윤혜원

다음 오늘의 웹툰 크로링 성공!

Showing 504 changed files with 4930 additions and 0 deletions
This diff is collapsed. Click to expand it.
1 +#!/bin/sh
2 +basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
3 +
4 +case `uname` in
5 + *CYGWIN*) basedir=`cygpath -w "$basedir"`;;
6 +esac
7 +
8 +if [ -x "$basedir/node" ]; then
9 + "$basedir/node" "$basedir/../he/bin/he" "$@"
10 + ret=$?
11 +else
12 + node "$basedir/../he/bin/he" "$@"
13 + ret=$?
14 +fi
15 +exit $ret
1 +@IF EXIST "%~dp0\node.exe" (
2 + "%~dp0\node.exe" "%~dp0\..\he\bin\he" %*
3 +) ELSE (
4 + @SETLOCAL
5 + @SET PATHEXT=%PATHEXT:;.JS;=;%
6 + node "%~dp0\..\he\bin\he" %*
7 +)
...\ No newline at end of file ...\ No newline at end of file
1 +#!/bin/sh
2 +basedir=$(dirname "$(echo "$0" | sed -e 's,\\,/,g')")
3 +
4 +case `uname` in
5 + *CYGWIN*) basedir=`cygpath -w "$basedir"`;;
6 +esac
7 +
8 +if [ -x "$basedir/node" ]; then
9 + "$basedir/node" "$basedir/../prettyjson/bin/prettyjson" "$@"
10 + ret=$?
11 +else
12 + node "$basedir/../prettyjson/bin/prettyjson" "$@"
13 + ret=$?
14 +fi
15 +exit $ret
1 +@IF EXIST "%~dp0\node.exe" (
2 + "%~dp0\node.exe" "%~dp0\..\prettyjson\bin\prettyjson" %*
3 +) ELSE (
4 + @SETLOCAL
5 + @SET PATHEXT=%PATHEXT:;.JS;=;%
6 + node "%~dp0\..\prettyjson\bin\prettyjson" %*
7 +)
...\ No newline at end of file ...\ No newline at end of file
1 + MIT License
2 +
3 + Copyright (c) Microsoft Corporation. All rights reserved.
4 +
5 + Permission is hereby granted, free of charge, to any person obtaining a copy
6 + of this software and associated documentation files (the "Software"), to deal
7 + in the Software without restriction, including without limitation the rights
8 + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 + copies of the Software, and to permit persons to whom the Software is
10 + furnished to do so, subject to the following conditions:
11 +
12 + The above copyright notice and this permission notice shall be included in all
13 + copies or substantial portions of the Software.
14 +
15 + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 + SOFTWARE
1 +# Installation
2 +> `npm install --save @types/cheerio`
3 +
4 +# Summary
5 +This package contains type definitions for Cheerio (https://github.com/cheeriojs/cheerio).
6 +
7 +# Details
8 +Files were exported from https://github.com/DefinitelyTyped/DefinitelyTyped/tree/master/types/cheerio
9 +
10 +Additional Details
11 + * Last updated: Mon, 26 Nov 2018 15:56:46 GMT
12 + * Dependencies: none
13 + * Global values: cheerio
14 +
15 +# Credits
16 +These definitions were written by Bret Little <https://github.com/blittle>, VILIC VANE <http://vilic.info>, Wayne Maurer <https://github.com/wmaurer>, Umar Nizamani <https://github.com/umarniz>, LiJinyao <https://github.com/LiJinyao>, Chennakrishna <https://github.com/chennakrishna8>, AzSiAz <https://github.com/AzSiAz>.
1 +// Type definitions for Cheerio v0.22.0
2 +// Project: https://github.com/cheeriojs/cheerio
3 +// Definitions by: Bret Little <https://github.com/blittle>, VILIC VANE <http://vilic.info>, Wayne Maurer <https://github.com/wmaurer>, Umar Nizamani <https://github.com/umarniz>, LiJinyao <https://github.com/LiJinyao>, Chennakrishna <https://github.com/chennakrishna8>, AzSiAz <https://github.com/AzSiAz>
4 +// Definitions: https://github.com/DefinitelyTyped/DefinitelyTyped
5 +
6 +interface Cheerio {
7 + // Document References
8 + // Cheerio https://github.com/cheeriojs/cheerio
9 + // JQuery http://api.jquery.com
10 +
11 + [index: number]: CheerioElement;
12 + length: number;
13 +
14 + // Attributes
15 +
16 + attr(): {[attr: string]: string};
17 + attr(name: string): string;
18 + attr(name: string, value: any): Cheerio;
19 +
20 + data(): any;
21 + data(name: string): any;
22 + data(name: string, value: any): any;
23 +
24 + val(): string;
25 + val(value: string): Cheerio;
26 +
27 + removeAttr(name: string): Cheerio;
28 +
29 + has(selector: string): Cheerio;
30 + has(element: CheerioElement): Cheerio;
31 +
32 + hasClass(className: string): boolean;
33 + addClass(classNames: string): Cheerio;
34 +
35 + removeClass(): Cheerio;
36 + removeClass(className: string): Cheerio;
37 + removeClass(func: (index: number, className: string) => string): Cheerio;
38 +
39 + toggleClass(className: string): Cheerio;
40 + toggleClass(className: string, toggleSwitch: boolean): Cheerio;
41 + toggleClass(toggleSwitch?: boolean): Cheerio;
42 + toggleClass(func: (index: number, className: string, toggleSwitch: boolean) => string, toggleSwitch?: boolean): Cheerio;
43 +
44 + is(selector: string): boolean;
45 + is(element: CheerioElement): boolean;
46 + is(element: CheerioElement[]): boolean;
47 + is(selection: Cheerio): boolean;
48 + is(func: (index: number, element: CheerioElement) => boolean): boolean;
49 +
50 + // Form
51 + serialize(): string;
52 + serializeArray(): {name: string, value: string}[];
53 +
54 + // Traversing
55 +
56 + find(selector: string): Cheerio;
57 + find(element: Cheerio): Cheerio;
58 +
59 + parent(selector?: string): Cheerio;
60 + parents(selector?: string): Cheerio;
61 + parentsUntil(selector?: string, filter?: string): Cheerio;
62 + parentsUntil(element: CheerioElement, filter?: string): Cheerio;
63 + parentsUntil(element: Cheerio, filter?: string): Cheerio;
64 +
65 + prop(name: string): any;
66 + prop(name: string, value: any): Cheerio;
67 +
68 + closest(): Cheerio;
69 + closest(selector: string): Cheerio;
70 +
71 + next(selector?: string): Cheerio;
72 + nextAll(): Cheerio;
73 + nextAll(selector: string): Cheerio;
74 +
75 + nextUntil(selector?: string, filter?: string): Cheerio;
76 + nextUntil(element: CheerioElement, filter?: string): Cheerio;
77 + nextUntil(element: Cheerio, filter?: string): Cheerio;
78 +
79 + prev(selector?: string): Cheerio;
80 + prevAll(): Cheerio;
81 + prevAll(selector: string): Cheerio;
82 +
83 + prevUntil(selector?: string, filter?: string): Cheerio;
84 + prevUntil(element: CheerioElement, filter?: string): Cheerio;
85 + prevUntil(element: Cheerio, filter?: string): Cheerio;
86 +
87 + slice(start: number, end?: number): Cheerio;
88 +
89 + siblings(selector?: string): Cheerio;
90 +
91 + children(selector?: string): Cheerio;
92 +
93 + contents(): Cheerio;
94 +
95 + each(func: (index: number, element: CheerioElement) => any): Cheerio;
96 + map(func: (index: number, element: CheerioElement) => any): Cheerio;
97 +
98 + filter(selector: string): Cheerio;
99 + filter(selection: Cheerio): Cheerio;
100 + filter(element: CheerioElement): Cheerio;
101 + filter(elements: CheerioElement[]): Cheerio;
102 + filter(func: (index: number, element: CheerioElement) => boolean): Cheerio;
103 +
104 + not(selector: string): Cheerio;
105 + not(selection: Cheerio): Cheerio;
106 + not(element: CheerioElement): Cheerio;
107 + not(func: (index: number, element: CheerioElement) => boolean): Cheerio;
108 +
109 + first(): Cheerio;
110 + last(): Cheerio;
111 +
112 + eq(index: number): Cheerio;
113 +
114 + get(): any[];
115 + get(index: number): any;
116 +
117 + index(): number;
118 + index(selector: string): number;
119 + index(selection: Cheerio): number;
120 +
121 + end(): Cheerio;
122 +
123 + add(selectorOrHtml: string): Cheerio;
124 + add(selector: string, context: Document): Cheerio;
125 + add(element: CheerioElement): Cheerio;
126 + add(elements: CheerioElement[]): Cheerio;
127 + add(selection: Cheerio): Cheerio;
128 +
129 + addBack():Cheerio;
130 + addBack(filter: string):Cheerio;
131 +
132 + // Manipulation
133 + appendTo(target: Cheerio) : Cheerio
134 + prependTo(target: Cheerio) : Cheerio
135 +
136 + append(content: string, ...contents: any[]): Cheerio;
137 + append(content: Document, ...contents: any[]): Cheerio;
138 + append(content: Document[], ...contents: any[]): Cheerio;
139 + append(content: Cheerio, ...contents: any[]): Cheerio;
140 +
141 + prepend(content: string, ...contents: any[]): Cheerio;
142 + prepend(content: Document, ...contents: any[]): Cheerio;
143 + prepend(content: Document[], ...contents: any[]): Cheerio;
144 + prepend(content: Cheerio, ...contents: any[]): Cheerio;
145 +
146 + after(content: string, ...contents: any[]): Cheerio;
147 + after(content: Document, ...contents: any[]): Cheerio;
148 + after(content: Document[], ...contents: any[]): Cheerio;
149 + after(content: Cheerio, ...contents: any[]): Cheerio;
150 +
151 + insertAfter(content: string): Cheerio;
152 + insertAfter(content: Document): Cheerio;
153 + insertAfter(content: Cheerio): Cheerio;
154 +
155 + before(content: string, ...contents: any[]): Cheerio;
156 + before(content: Document, ...contents: any[]): Cheerio;
157 + before(content: Document[], ...contents: any[]): Cheerio;
158 + before(content: Cheerio, ...contents: any[]): Cheerio;
159 +
160 + insertBefore(content: string): Cheerio;
161 + insertBefore(content: Document): Cheerio;
162 + insertBefore(content: Cheerio): Cheerio;
163 +
164 + remove(selector?: string): Cheerio;
165 +
166 + replaceWith(content: string): Cheerio;
167 + replaceWith(content: CheerioElement): Cheerio;
168 + replaceWith(content: CheerioElement[]): Cheerio;
169 + replaceWith(content: Cheerio): Cheerio;
170 + replaceWith(content: () => Cheerio): Cheerio;
171 +
172 + empty(): Cheerio;
173 +
174 + html(): string | null;
175 + html(html: string): Cheerio;
176 +
177 + text(): string;
178 + text(text: string): Cheerio;
179 +
180 + wrap(content: string): Cheerio;
181 + wrap(content: Document): Cheerio;
182 + wrap(content: Cheerio): Cheerio;
183 +
184 + css(propertyName: string): string;
185 + css(propertyNames: string[]): string[];
186 + css(propertyName: string, value: string): Cheerio;
187 + css(propertyName: string, value: number): Cheerio;
188 + css(propertyName: string, func: (index: number, value: string) => string): Cheerio;
189 + css(propertyName: string, func: (index: number, value: string) => number): Cheerio;
190 + css(properties: Object): Cheerio;
191 +
192 + // Rendering
193 +
194 + // Miscellaneous
195 +
196 + clone(): Cheerio;
197 +
198 + // Not Documented
199 +
200 + toArray(): CheerioElement[];
201 +}
202 +
203 +interface CheerioOptionsInterface {
204 + // Document References
205 + // Cheerio https://github.com/cheeriojs/cheerio
206 + // HTMLParser2 https://github.com/fb55/htmlparser2/wiki/Parser-options
207 + // DomHandler https://github.com/fb55/DomHandler
208 +
209 + xmlMode?: boolean;
210 + decodeEntities?: boolean;
211 + lowerCaseTags?: boolean;
212 + lowerCaseAttributeNames?: boolean;
213 + recognizeCDATA?: boolean;
214 + recognizeSelfClosing?: boolean;
215 + normalizeWhitespace?: boolean;
216 + ignoreWhitespace?: boolean;
217 +}
218 +
219 +interface CheerioSelector {
220 + (selector: string): Cheerio;
221 + (selector: string, context: string): Cheerio;
222 + (selector: string, context: CheerioElement): Cheerio;
223 + (selector: string, context: CheerioElement[]): Cheerio;
224 + (selector: string, context: Cheerio): Cheerio;
225 + (selector: string, context: string, root: string): Cheerio;
226 + (selector: string, context: CheerioElement, root: string): Cheerio;
227 + (selector: string, context: CheerioElement[], root: string): Cheerio;
228 + (selector: string, context: Cheerio, root: string): Cheerio;
229 + (selector: any): Cheerio;
230 +}
231 +
232 +interface CheerioStatic extends CheerioSelector {
233 + // Document References
234 + // Cheerio https://github.com/cheeriojs/cheerio
235 + // JQuery http://api.jquery.com
236 + xml(): string;
237 + root(): Cheerio;
238 + contains(container: CheerioElement, contained: CheerioElement): boolean;
239 + parseHTML(data: string, context?: Document, keepScripts?: boolean): Document[];
240 +
241 + html(options?: CheerioOptionsInterface): string;
242 + html(selector: string, options?: CheerioOptionsInterface): string;
243 + html(element: Cheerio, options?: CheerioOptionsInterface): string;
244 + html(element: CheerioElement, options?: CheerioOptionsInterface): string;
245 +}
246 +
247 +interface CheerioElement {
248 + // Document References
249 + // Node Console
250 + tagName: string;
251 + type: string;
252 + name: string;
253 + attribs: {[attr: string]: string};
254 + children: CheerioElement[];
255 + childNodes: CheerioElement[];
256 + lastChild: CheerioElement;
257 + firstChild: CheerioElement;
258 + next: CheerioElement;
259 + nextSibling: CheerioElement;
260 + prev: CheerioElement;
261 + previousSibling: CheerioElement;
262 + parent: CheerioElement;
263 + parentNode: CheerioElement;
264 + nodeValue: string;
265 + data?: string;
266 +}
267 +
268 +interface CheerioAPI extends CheerioSelector, CheerioStatic {
269 + load(html: string, options?: CheerioOptionsInterface): CheerioStatic;
270 + load(element: CheerioElement, options?: CheerioOptionsInterface): CheerioStatic;
271 +}
272 +
273 +interface Document { }
274 +
275 +declare var cheerio:CheerioAPI;
276 +
277 +declare module "cheerio" {
278 + export = cheerio;
279 +}
1 +{
2 + "_from": "@types/cheerio@^0.22.7",
3 + "_id": "@types/cheerio@0.22.10",
4 + "_inBundle": false,
5 + "_integrity": "sha512-fOM/Jhv51iyugY7KOBZz2ThfT1gwvsGCfWxpLpZDgkGjpEO4Le9cld07OdskikLjDUQJ43dzDaVRSFwQlpdqVg==",
6 + "_location": "/@types/cheerio",
7 + "_phantomChildren": {},
8 + "_requested": {
9 + "type": "range",
10 + "registry": true,
11 + "raw": "@types/cheerio@^0.22.7",
12 + "name": "@types/cheerio",
13 + "escapedName": "@types%2fcheerio",
14 + "scope": "@types",
15 + "rawSpec": "^0.22.7",
16 + "saveSpec": null,
17 + "fetchSpec": "^0.22.7"
18 + },
19 + "_requiredBy": [
20 + "/cheerio-httpcli"
21 + ],
22 + "_resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.10.tgz",
23 + "_shasum": "780d552467824be4a241b29510a7873a7432c4a6",
24 + "_spec": "@types/cheerio@^0.22.7",
25 + "_where": "D:\\OSS project\\YTMT\\node_modules\\cheerio-httpcli",
26 + "bugs": {
27 + "url": "https://github.com/DefinitelyTyped/DefinitelyTyped/issues"
28 + },
29 + "bundleDependencies": false,
30 + "contributors": [
31 + {
32 + "name": "Bret Little",
33 + "url": "https://github.com/blittle"
34 + },
35 + {
36 + "name": "VILIC VANE",
37 + "url": "http://vilic.info"
38 + },
39 + {
40 + "name": "Wayne Maurer",
41 + "url": "https://github.com/wmaurer"
42 + },
43 + {
44 + "name": "Umar Nizamani",
45 + "url": "https://github.com/umarniz"
46 + },
47 + {
48 + "name": "LiJinyao",
49 + "url": "https://github.com/LiJinyao"
50 + },
51 + {
52 + "name": "Chennakrishna",
53 + "url": "https://github.com/chennakrishna8"
54 + },
55 + {
56 + "name": "AzSiAz",
57 + "url": "https://github.com/AzSiAz"
58 + }
59 + ],
60 + "dependencies": {},
61 + "deprecated": false,
62 + "description": "TypeScript definitions for Cheerio",
63 + "homepage": "https://github.com/DefinitelyTyped/DefinitelyTyped#readme",
64 + "license": "MIT",
65 + "main": "",
66 + "name": "@types/cheerio",
67 + "repository": {
68 + "type": "git",
69 + "url": "git+https://github.com/DefinitelyTyped/DefinitelyTyped.git"
70 + },
71 + "scripts": {},
72 + "typeScriptVersion": "2.0",
73 + "types": "index",
74 + "typesPublisherContentHash": "2e27eac8c695bdef169dcf5482e7a17f84ae485381d5de916cdddaedbd3787be",
75 + "version": "0.22.10"
76 +}
1 +'use strict';
2 +var callsites = require('callsites');
3 +
4 +module.exports = function () {
5 + return callsites()[2].getFileName();
6 +};
1 +{
2 + "_from": "caller-path@^0.1.0",
3 + "_id": "caller-path@0.1.0",
4 + "_inBundle": false,
5 + "_integrity": "sha1-lAhe9jWB7NPaqSREqP6U6CV3dR8=",
6 + "_location": "/caller-path",
7 + "_phantomChildren": {},
8 + "_requested": {
9 + "type": "range",
10 + "registry": true,
11 + "raw": "caller-path@^0.1.0",
12 + "name": "caller-path",
13 + "escapedName": "caller-path",
14 + "rawSpec": "^0.1.0",
15 + "saveSpec": null,
16 + "fetchSpec": "^0.1.0"
17 + },
18 + "_requiredBy": [
19 + "/require-uncached"
20 + ],
21 + "_resolved": "https://registry.npmjs.org/caller-path/-/caller-path-0.1.0.tgz",
22 + "_shasum": "94085ef63581ecd3daa92444a8fe94e82577751f",
23 + "_spec": "caller-path@^0.1.0",
24 + "_where": "D:\\OSS project\\YTMT\\node_modules\\require-uncached",
25 + "author": {
26 + "name": "Sindre Sorhus",
27 + "email": "sindresorhus@gmail.com",
28 + "url": "http://sindresorhus.com"
29 + },
30 + "bugs": {
31 + "url": "https://github.com/sindresorhus/caller-path/issues"
32 + },
33 + "bundleDependencies": false,
34 + "dependencies": {
35 + "callsites": "^0.2.0"
36 + },
37 + "deprecated": false,
38 + "description": "Get the path of the caller module",
39 + "devDependencies": {
40 + "mocha": "*"
41 + },
42 + "engines": {
43 + "node": ">=0.10.0"
44 + },
45 + "files": [
46 + "index.js"
47 + ],
48 + "homepage": "https://github.com/sindresorhus/caller-path#readme",
49 + "keywords": [
50 + "caller",
51 + "calling",
52 + "module",
53 + "path",
54 + "parent",
55 + "callsites",
56 + "callsite",
57 + "stacktrace",
58 + "stack",
59 + "trace",
60 + "function",
61 + "file"
62 + ],
63 + "license": "MIT",
64 + "name": "caller-path",
65 + "repository": {
66 + "type": "git",
67 + "url": "git+https://github.com/sindresorhus/caller-path.git"
68 + },
69 + "scripts": {
70 + "test": "mocha"
71 + },
72 + "version": "0.1.0"
73 +}
1 +# caller-path [![Build Status](https://travis-ci.org/sindresorhus/caller-path.svg?branch=master)](https://travis-ci.org/sindresorhus/caller-path)
2 +
3 +> Get the path of the caller module
4 +
5 +You can't use [`module.parent`](http://nodejs.org/api/modules.html#modules_module_parent) as modules are cached and it will return the first caller module, not necessarily the current one.
6 +
7 +
8 +## Install
9 +
10 +```
11 +$ npm install --save caller-path
12 +```
13 +
14 +
15 +## Usage
16 +
17 +```js
18 +// foo.js
19 +var callerPath = require('caller-path');
20 +
21 +module.exports = function () {
22 + console.log(callerPath());
23 + //=> /Users/sindresorhus/dev/unicorn/bar.js
24 +}
25 +```
26 +
27 +```js
28 +// bar.js
29 +var foo = require('./foo');
30 +foo();
31 +```
32 +
33 +
34 +## License
35 +
36 +MIT © [Sindre Sorhus](http://sindresorhus.com)
1 +'use strict';
2 +module.exports = function () {
3 + var _ = Error.prepareStackTrace;
4 + Error.prepareStackTrace = function (_, stack) { return stack };
5 + var stack = new Error().stack.slice(1);
6 + Error.prepareStackTrace = _;
7 + return stack;
8 +};
1 +{
2 + "_from": "callsites@^0.2.0",
3 + "_id": "callsites@0.2.0",
4 + "_inBundle": false,
5 + "_integrity": "sha1-r6uWJikQp/M8GaV3WCXGnzTjUMo=",
6 + "_location": "/callsites",
7 + "_phantomChildren": {},
8 + "_requested": {
9 + "type": "range",
10 + "registry": true,
11 + "raw": "callsites@^0.2.0",
12 + "name": "callsites",
13 + "escapedName": "callsites",
14 + "rawSpec": "^0.2.0",
15 + "saveSpec": null,
16 + "fetchSpec": "^0.2.0"
17 + },
18 + "_requiredBy": [
19 + "/caller-path"
20 + ],
21 + "_resolved": "http://registry.npmjs.org/callsites/-/callsites-0.2.0.tgz",
22 + "_shasum": "afab96262910a7f33c19a5775825c69f34e350ca",
23 + "_spec": "callsites@^0.2.0",
24 + "_where": "D:\\OSS project\\YTMT\\node_modules\\caller-path",
25 + "author": {
26 + "name": "Sindre Sorhus",
27 + "email": "sindresorhus@gmail.com",
28 + "url": "http://sindresorhus.com"
29 + },
30 + "bugs": {
31 + "url": "https://github.com/sindresorhus/callsites/issues"
32 + },
33 + "bundleDependencies": false,
34 + "deprecated": false,
35 + "description": "Get callsites from the V8 stack trace API",
36 + "devDependencies": {
37 + "mocha": "*"
38 + },
39 + "engines": {
40 + "node": ">=0.10.0"
41 + },
42 + "files": [
43 + "index.js"
44 + ],
45 + "homepage": "https://github.com/sindresorhus/callsites#readme",
46 + "keywords": [
47 + "callsites",
48 + "callsite",
49 + "v8",
50 + "stacktrace",
51 + "stack",
52 + "trace",
53 + "function",
54 + "file",
55 + "line",
56 + "debug"
57 + ],
58 + "license": "MIT",
59 + "name": "callsites",
60 + "repository": {
61 + "type": "git",
62 + "url": "git+https://github.com/sindresorhus/callsites.git"
63 + },
64 + "scripts": {
65 + "test": "mocha"
66 + },
67 + "version": "0.2.0"
68 +}
1 +# callsites [![Build Status](https://travis-ci.org/sindresorhus/callsites.svg?branch=master)](https://travis-ci.org/sindresorhus/callsites)
2 +
3 +> Get callsites from the [V8 stack trace API](https://code.google.com/p/v8/wiki/JavaScriptStackTraceApi)
4 +
5 +
6 +## Install
7 +
8 +```sh
9 +$ npm install --save callsites
10 +```
11 +
12 +
13 +## Usage
14 +
15 +```js
16 +var callsites = require('callsites');
17 +
18 +function unicorn() {
19 + console.log(callsites()[0].getFileName());
20 + //=> /Users/sindresorhus/dev/callsites/test.js
21 +}
22 +
23 +unicorn();
24 +```
25 +
26 +## API
27 +
28 +Returns an array of callsite objects with the following methods:
29 +
30 +- `getThis`: returns the value of this
31 +- `getTypeName`: returns the type of this as a string. This is the name of the function stored in the constructor field of this, if available, otherwise the object's [[Class]] internal property.
32 +- `getFunction`: returns the current function
33 +- `getFunctionName`: returns the name of the current function, typically its name property. If a name property is not available an attempt will be made to try to infer a name from the function's context.
34 +- `getMethodName`: returns the name of the property of this or one of its prototypes that holds the current function
35 +- `getFileName`: if this function was defined in a script returns the name of the script
36 +- `getLineNumber`: if this function was defined in a script returns the current line number
37 +- `getColumnNumber`: if this function was defined in a script returns the current column number
38 +- `getEvalOrigin`: if this function was created using a call to eval returns a CallSite object representing the location where eval was called
39 +- `isToplevel`: is this a toplevel invocation, that is, is this the global object?
40 +- `isEval`: does this call take place in code defined by a call to eval?
41 +- `isNative`: is this call in native V8 code?
42 +- `isConstructor`: is this a constructor call?
43 +
44 +
45 +## License
46 +
47 +MIT © [Sindre Sorhus](http://sindresorhus.com)
1 +{
2 + "extends": "eslint:recommended",
3 + "env": {
4 + "node": true,
5 + "es6": false,
6 + "browser": false
7 + },
8 + "parserOptions": {
9 + "sourceType": "module",
10 + "ecmaFeatures": {
11 + "modules": true
12 + }
13 + },
14 + "plugins": [
15 + ],
16 + "rules": {
17 + "curly": [2, "multi-line"],
18 + "camelcase": [1, {
19 + "properties": "never"
20 + }],
21 + "require-jsdoc": 0,
22 + "valid-jsdoc": 0,
23 + "valid-typeof": 2,
24 + "no-array-constructor": 2,
25 + "no-bitwise": 2,
26 + "no-buffer-constructor": 0,
27 + "no-caller": 1,
28 + "no-catch-shadow": 2,
29 + "no-class-assign": 2,
30 + "no-compare-neg-zero": 2,
31 + "no-cond-assign": 2,
32 + "no-confusing-arrow": 2,
33 + "no-console": [1, {
34 + "allow": ["info", "warn", "error"]
35 + }],
36 + "no-const-assign": 2,
37 + "no-constant-condition": 2,
38 + "no-control-regex": 2,
39 + "no-debugger": 2,
40 + "no-delete-var": 2,
41 + "no-dupe-args": 2,
42 + "no-dupe-class-members": 2,
43 + "no-dupe-keys": 2,
44 + "no-duplicate-case": 2,
45 + "no-duplicate-imports": 2,
46 + "no-else-return": [2, {
47 + "allowElseIf": false
48 + }],
49 + "no-empty": [1, {
50 + "allowEmptyCatch": true
51 + }],
52 + "no-empty-character-class": 2,
53 + "no-eq-null": 2,
54 + "no-eval": 2,
55 + "no-ex-assign": 2,
56 + "no-extend-native": 2,
57 + "no-extra-bind": 1,
58 + "no-extra-boolean-cast": 2,
59 + "no-extra-parens": [0, "all", {
60 + "returnAssign": false,
61 + "nestedBinaryExpressions": false
62 + }],
63 + "no-extra-semi": 2,
64 + "no-fallthrough": 2,
65 + "no-floating-decimal": 2,
66 + "no-func-assign": 2,
67 + "no-global-assign": 2,
68 + "no-implicit-coercion": 2,
69 + "no-implied-eval": 2,
70 + "no-inner-declarations": 1,
71 + "no-invalid-regexp": 2,
72 + "no-invalid-this": 2,
73 + "no-irregular-whitespace": 2,
74 + "no-labels": 2,
75 + "no-lonely-if": 1,
76 + "no-loop-func": 0,
77 + "no-multi-assign": 1,
78 + "no-multi-str": 1,
79 + "no-multiple-empty-lines": [1, {"max": 2}],
80 + "no-nested-ternary": 2,
81 + "no-new": 1,
82 + "no-new-func": 2,
83 + "no-new-require": 2,
84 + "no-new-wrappers": 1,
85 + "no-obj-calls": 2,
86 + "no-octal": 2,
87 + "no-octal-escape": 2,
88 + "no-param-reassign": 0,
89 + "no-path-concat": 2,
90 + "no-process-env": 0,
91 + "no-proto": 2,
92 + "no-redeclare": 2,
93 + "no-regex-spaces": 1,
94 + "no-return-assign": 2,
95 + "no-self-compare": 1,
96 + "no-self-assign": [2, {
97 + "props": true
98 + }],
99 + "no-sequences": 2,
100 + "no-shadow-restricted-names": 2,
101 + "no-sparse-arrays": 2,
102 + "no-this-before-super": 2,
103 + "no-tabs": 2,
104 + "no-throw-literal": 2,
105 + "prefer-promise-reject-errors": 2,
106 + "no-template-curly-in-string": 1,
107 + "no-undef": 2,
108 + "no-undef-init": 2,
109 + "no-undefined": 2,
110 + "no-unexpected-multiline": 2,
111 + "no-unneeded-ternary": 2,
112 + "no-unreachable": 1,
113 + "no-unsafe-negation": 2,
114 + "no-unused-expressions": 1,
115 + "no-unused-vars": [2, {
116 + "vars": "all",
117 + "args": "none",
118 + "caughtErrors": "none",
119 + "ignoreRestSiblings": true
120 + }],
121 + "no-use-before-define": 2,
122 + "no-useless-call": 2,
123 + "no-useless-concat": 1,
124 + "no-useless-escape": 0,
125 + "no-void": 2,
126 + "no-with": 2,
127 + "no-return-await": 1,
128 + "no-await-in-loop": 2,
129 + "nonblock-statement-body-position": [2, "beside"],
130 + "use-isnan": 2,
131 + "accessor-pairs": 2,
132 + "block-scoped-var": 2,
133 + "func-name-matching": [2, {
134 + "includeCommonJSModuleExports": true
135 + }],
136 + "func-call-spacing": [2, "never"],
137 + "consistent-return": [2, {
138 + "treatUndefinedAsUnspecified": true
139 + }],
140 + "linebreak-style": [0, "unix"],
141 + "new-cap": 2,
142 + "new-parens": 2,
143 + "newline-after-var": [0, "always"],
144 + "generator-star-spacing": [2, {
145 + "before": true,
146 + "after": false
147 + }],
148 + "complexity": [2, 20],
149 + "lines-around-directive": 1,
150 + "dot-notation": 1,
151 + "guard-for-in": 1,
152 + "for-direction": 2,
153 + "require-await": 2,
154 + "require-yield": 2,
155 + "arrow-parens": [2, "always", {
156 + "requireForBlockBody": true
157 + }],
158 + "arrow-spacing": 2,
159 + "callback-return": [1, [
160 + "callback", "cb", "next", "done"
161 + ]],
162 + "getter-return": 2,
163 + "operator-linebreak": [1, "before", {
164 + "overrides": {
165 + "&&": "after",
166 + "||": "after"
167 + }
168 + }],
169 + "wrap-iife": [2, "inside", {
170 + "functionPrototypeMethods": true
171 + }],
172 + "function-paren-newline": 0,
173 + "yoda": 2,
174 + "prefer-arrow-callback": 0,
175 + "prefer-template": 0,
176 + "prefer-const": 0,
177 + "prefer-destructuring": 0,
178 + "prefer-spread": 0,
179 + "prefer-numeric-literals": 1,
180 + "rest-spread-spacing": 0,
181 + "constructor-super": 2,
182 + "template-tag-spacing": 2,
183 + "default-case": 1,
184 + "sort-keys": 0,
185 + "vars-on-top": 0,
186 + "eqeqeq": [2, "always", {
187 + "null": "always"
188 + }],
189 + "radix": 2,
190 + "consistent-this": [2, "_this"],
191 + "eol-last": 2,
192 + "func-style": 0,
193 + "jsx-quotes": [2, "prefer-double"],
194 + "semi": [2, "always"],
195 + "semi-spacing": [2, {
196 + "before": false,
197 + "after": true
198 + }],
199 + "one-var": [2, "never"],
200 + "quote-props": [2, "as-needed"],
201 + "init-declarations": [2, "always"],
202 + "computed-property-spacing": [2, "never"],
203 + "no-underscore-dangle": [2, {
204 + "allowAfterThis": true,
205 + "allowAfterSuper": true,
206 + "allow": [
207 + "_this",
208 + "_root", "_documentInfo", "_html", "_text",
209 + "_originalRoot", "_traceRoute", "_$"
210 + ]
211 + }],
212 + "comma-dangle": [2, "never"],
213 + "comma-style": [2, "last"],
214 + "brace-style": [1, "1tbs", {
215 + "allowSingleLine": true
216 + }],
217 + "no-mixed-spaces-and-tabs": [2, "smart-tabs"],
218 + "no-mixed-operators": [2, {
219 + "groups": [
220 + ["+", "-", "*", "/", "%", "**"],
221 + ["&", "|", "^", "~", "<<", ">>", ">>>"],
222 + ["==", "!=", "===", "!==", ">", ">=", "<", "<="],
223 + ["&&", "||"],
224 + ["in", "instanceof"]
225 + ],
226 + "allowSamePrecedence": true
227 + }],
228 + "indent": [2, 2, {
229 + "SwitchCase": 1,
230 + "ImportDeclaration": "first",
231 + "MemberExpression": 0,
232 + "CallExpression": {
233 + "arguments": "first"
234 + },
235 + "FunctionDeclaration": {
236 + "parameters": "first",
237 + "body": 1
238 + }
239 + }
240 + ],
241 + "keyword-spacing": [2, {
242 + "before": true,
243 + "after": true,
244 + "overrides": {}
245 + }],
246 + "lines-between-class-members": [2, "always"],
247 + "lines-around-comment": [2, {
248 + "beforeBlockComment": true,
249 + "afterBlockComment": false,
250 + "beforeLineComment": false,
251 + "afterLineComment": false,
252 + "allowBlockStart": true,
253 + "allowBlockEnd": true,
254 + "allowObjectStart": true,
255 + "allowObjectEnd": true,
256 + "allowArrayStart": true,
257 + "allowArrayEnd": true
258 + }],
259 + "space-before-blocks": [2, "always"],
260 + "space-before-function-paren": [2, {
261 + "named": "never",
262 + "asyncArrow": "always"
263 + }],
264 + "space-in-parens": [2, "never"],
265 + "space-infix-ops": 2,
266 + "space-unary-ops": [2, {
267 + "words": true,
268 + "nonwords": false,
269 + "overrides": {
270 + "!": true,
271 + "!!": true
272 + }
273 + }],
274 + "semi-style": [2, "last"],
275 + "switch-colon-spacing": 2,
276 + "multiline-ternary": [2, "always-multiline"],
277 + "object-curly-spacing": [2, "always", {
278 + "objectsInObjects": false,
279 + "arraysInObjects": false
280 + }],
281 + "object-curly-newline": [0, {
282 + "ObjectExpression": "always",
283 + "ObjectPattern": {"multiline": true}
284 + }],
285 + "object-property-newline": 0,
286 + "no-trailing-spaces": [2, {
287 + "skipBlankLines": false
288 + }],
289 + "object-shorthand": [1, "consistent"],
290 + "key-spacing": [2, {
291 + "beforeColon": false,
292 + "afterColon": true
293 + }],
294 + "comma-spacing": [2, {
295 + "before": false,
296 + "after": true
297 + }],
298 + "no-multi-spaces": [2, {
299 + "ignoreEOLComments": true,
300 + "exceptions": {
301 + "ImportDeclaration": true,
302 + "VariableDeclarator": true,
303 + "Property": true
304 + }
305 + }],
306 + "array-bracket-spacing": [2, "always", {
307 + "singleValue": true,
308 + "objectsInArrays": false,
309 + "arraysInArrays": true
310 + }],
311 + "array-bracket-newline": [2, "consistent"],
312 + "quotes": [1,
313 + "single", "avoid-escape"
314 + ],
315 + "max-statements-per-line": [1, { "max": 1 }],
316 + "max-len": [1, {
317 + "code": 100,
318 + "tabWidth": 2,
319 + "ignoreComments": true,
320 + "ignoreTrailingComments": true,
321 + "ignoreUrls": true,
322 + "ignoreStrings": true,
323 + "ignoreTemplateLiterals": true,
324 + "ignoreRegExpLiterals": true
325 + }],
326 + "max-lines": [1, 2000],
327 + "max-params": [1, 5],
328 + "max-depth": [1, 4],
329 + "max-nested-callbacks": [1, 5],
330 + "max-statements": [1, 30]
331 + }
332 +}
1 +{
2 + "excludeFiles": [],
3 + "fileExtensions": [
4 + ".js"
5 + ],
6 + "requireCurlyBraces": [
7 + "if",
8 + "else",
9 + "for",
10 + "while",
11 + "do",
12 + "try",
13 + "catch",
14 + "case",
15 + "default"
16 + ],
17 + "requireSpaceAfterKeywords": [
18 + "do",
19 + "for",
20 + "if",
21 + "else",
22 + "switch",
23 + "case",
24 + "try",
25 + "catch",
26 + "void",
27 + "while",
28 + "with",
29 + "return",
30 + "typeof",
31 + "function"
32 + ],
33 + "requireSpaceBeforeBlockStatements": true,
34 + "requireParenthesesAroundIIFE": true,
35 + "requireSpacesInConditionalExpression": {
36 + "afterTest": true,
37 + "beforeConsequent": true,
38 + "afterConsequent": true,
39 + "beforeAlternate": true
40 + },
41 + "requireSpacesInAnonymousFunctionExpression": {
42 + "beforeOpeningRoundBrace": true,
43 + "beforeOpeningCurlyBrace": true
44 + },
45 + "requireSpacesInNamedFunctionExpression": {
46 + "beforeOpeningCurlyBrace": true
47 + },
48 + "disallowMultipleVarDecl": true,
49 + "requireBlocksOnNewline": true,
50 + "disallowEmptyBlocks": true,
51 + "disallowSpacesInsideParentheses": true,
52 + "requireSpacesInsideObjectBrackets": "all",
53 + "requireSpacesInsideArrayBrackets": {
54 + "allExcept": [ "[", "]", "{", "}" ]
55 + },
56 + "disallowQuotedKeysInObjects": "allButReserved",
57 + "disallowSpaceAfterObjectKeys": false,
58 + "requireSpaceBeforeObjectValues": true,
59 + "requireCommaBeforeLineBreak": true,
60 + "disallowSpaceAfterPrefixUnaryOperators": [
61 + "++",
62 + "--"
63 + ],
64 + "requireSpaceAfterPrefixUnaryOperators": [
65 + "+",
66 + "~",
67 + "!"
68 + ],
69 + "disallowSpaceBeforePostfixUnaryOperators": [
70 + "++",
71 + "--"
72 + ],
73 + "requireSpaceBeforeBinaryOperators": [
74 + "=",
75 + "+",
76 + "-",
77 + "/",
78 + "*",
79 + "==",
80 + "===",
81 + "!=",
82 + "!=="
83 + ],
84 + "requireSpaceAfterBinaryOperators": [
85 + "=",
86 + ",",
87 + "+",
88 + "-",
89 + "/",
90 + "*",
91 + "==",
92 + "===",
93 + "!=",
94 + "!=="
95 + ],
96 + "disallowImplicitTypeConversion": [
97 + "string"
98 + ],
99 + "requireCamelCaseOrUpperCaseIdentifiers": "ignoreProperties",
100 + "disallowKeywords": [
101 + "with"
102 + ],
103 + "disallowMultipleLineStrings": true,
104 + "disallowMultipleLineBreaks": false,
105 + "disallowMixedSpacesAndTabs": true,
106 + "disallowTrailingWhitespace": true,
107 + "disallowTrailingComma": true,
108 + "requireLineFeedAtFileEnd": true,
109 + "requireCapitalizedConstructors": true,
110 + "requireDotNotation": true,
111 + "maximumLineLength": 120,
112 + "disallowYodaConditions": true,
113 + "requireSpaceAfterLineComment": true,
114 + "requireAnonymousFunctions": {
115 + "allExcept": [
116 + "declarations"
117 + ]
118 + },
119 + "disallowNewlineBeforeBlockStatements": true,
120 + "validateQuoteMarks": {
121 + "mark": "'",
122 + "escape": true
123 + },
124 + "validateIndentation": 2,
125 + "safeContextKeyword": [
126 + "_this"
127 + ]
128 +}
This diff is collapsed. Click to expand it.
1 +Copyright (c) 2013-2015 ktty1220
2 +
3 +The MIT License
4 +
5 +Permission is hereby granted, free of charge, to any person obtaining
6 +a copy of this software and associated documentation files (the
7 +"Software"), to deal in the Software without restriction, including
8 +without limitation the rights to use, copy, modify, merge, publish,
9 +distribute, sublicense, and/or sell copies of the Software, and to
10 +permit persons to whom the Software is furnished to do so, subject to
11 +the following conditions:
12 +
13 +The above copyright notice and this permission notice shall be
14 +included in all copies or substantial portions of the Software.
15 +
16 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17 +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18 +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19 +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20 +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21 +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22 +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
This diff is collapsed. Click to expand it.
1 +// このtsファイルをtscでコンパイルしてエラーにならなければOK的なゆるいチェック
2 +import * as cli from '../../cheerio-httpcli';
3 +import * as fs from 'fs';
4 +
5 +cli.set('debug', true);
6 +cli.set('browser', 'edge');
7 +cli.set('timeout', 3000);
8 +cli.set('agentOptions', {
9 + secureProtocol: 'TLSv1_2_method'
10 +}, true);
11 +console.info(cli.headers);
12 +
13 +cli.download
14 +.on('ready', (stream) => {
15 + stream.pipe(fs.createWriteStream('/path/to/image.png'));
16 + console.info(`${stream.url.href}をダウンロードしました`);
17 +})
18 +.on('error', (err) => {
19 + console.error(`${err.url}をダウンロードできませんでした: ${err.message}`);
20 +})
21 +.on('end', () => {
22 + console.info('ダウンロードが完了しました');
23 +})
24 +.parallel = 4;
25 +
26 +const url = 'http://foo.bar.baz.co.ne.jp/';
27 +cli.fetch(url, (_err, $, _res, _body) => {
28 + const docInfo = $.documentInfo();
29 + console.info(docInfo.url);
30 + console.info(docInfo.encoding);
31 + console.info($.xml());
32 + $('a').css('color', 'black').tick().url({
33 + absolute: false
34 + });
35 + $('img.thumbnail').download();
36 + $('button').eq(3).click()
37 + .then((result) => {
38 + const key = 'hoge';
39 + console.info(
40 + result.response.cookies[key],
41 + result.$('#content').entityHtml(),
42 + result.$('form').field()
43 + );
44 + })
45 + .catch(console.error)
46 + .finally(() => console.info('done'));
47 +});
48 +
49 +cli.fetch(url, 'euc-jp')
50 +.then((result2) => {
51 + console.info(result2.response.headers);
52 +});
53 +
54 +{
55 + const { error, $, response, body } = cli.fetchSync(url, { q: 'hoge' });
56 + console.info(error, $, response, body);
57 +}
58 +
59 +// #18
60 +{
61 + const wrapper = async (u: string): Promise<cli.CheerioStaticEx> => {
62 + const result = await cli.fetch(u);
63 + const type: string = result.response.headers['content-type'] as string;
64 +
65 + /* rejects */
66 + if (result.error !== undefined && result.error !== null) {
67 + throw new Error('happend-error');
68 + }
69 +
70 + if (! /text\/html/.test(type)) {
71 + throw new Error('not-html');
72 + }
73 +
74 + /* resolve result */
75 + return result.$;
76 + };
77 +
78 + wrapper(url).then((result) => {
79 + console.dir(result);
80 + });
81 +}
1 +{
2 + "compilerOptions": {
3 + "module": "commonjs",
4 + "target": "es5",
5 + "lib": [
6 + "dom",
7 + "es2015"
8 + ],
9 + "moduleResolution": "node",
10 + "allowSyntheticDefaultImports": true,
11 + "allowUnreachableCode": false,
12 + "allowUnusedLabels": false,
13 + "alwaysStrict": true,
14 + "suppressImplicitAnyIndexErrors": true,
15 + "noImplicitAny": true,
16 + "noImplicitReturns": true,
17 + "noImplicitThis": true,
18 + "noUnusedLocals": true,
19 + "noUnusedParameters": true,
20 + "noFallthroughCasesInSwitch": true,
21 + "suppressImplicitAnyIndexErrors": true,
22 + "forceConsistentCasingInFileNames": true,
23 + "strict": true,
24 + "strictFunctionTypes": true,
25 + "strictNullChecks": true,
26 + "pretty": true,
27 + "importHelpers": false,
28 + "noEmitOnError": true,
29 + "noEmit": true
30 + }
31 +}
1 +#!/usr/bin/env node
2 +/*eslint no-invalid-this:0*/
3 +/*jshint -W100*/
4 +
5 +'use strict';
6 +
7 +/**
8 + * 5ちゃんねる検索サンプル
9 + *
10 + * 以下のword変数の内容で検索します
11 + */
12 +var word = 'ぬるぽ';
13 +
14 +
15 +var client = require('../index');
16 +
17 +client.fetch('http://www.5ch.net/')
18 +.then(function (result) {
19 + return result.$('form').eq(0).submit({
20 + q: word
21 + });
22 +})
23 +.then(function (result) {
24 + var $ = result.$;
25 + var results = [];
26 + $('.list_line').each(function () {
27 + var $a = $(this).find('a').eq(0);
28 + results.push({
29 + title: $a.text().trim(),
30 + url: $a.attr('href'),
31 + info: $(this).find('.list_line_info_container').text().trim()
32 + });
33 + });
34 + console.info(results);
35 +})
36 +.catch(function (err) {
37 + console.error('エラーが発生しました', err);
38 +})
39 +.finally(function () {
40 + console.info('終了します');
41 +});
1 +#!/usr/bin/env node
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +/**
7 + * excite翻訳サンプル
8 + *
9 + * 以下のstr変数の内容を翻訳します(言語自動判定)
10 + */
11 +var str = 'お前はもう死んでいる';
12 +
13 +
14 +var client = require('../index');
15 +
16 +console.info('デバッグオプションを有効にします');
17 +client.set('debug', true);
18 +
19 +console.info('excite翻訳ページにアクセスします');
20 +client.fetch('http://www.excite.co.jp/world/')
21 +.then(function (result) {
22 + return result.$('#formTrans').submit({
23 + auto_detect: 'on',
24 + before: str
25 + });
26 +})
27 +.then(function (result) {
28 + console.info('「' + str + '」=>「' + result.$('#after').val() + '」');
29 +})
30 +.catch(function (err) {
31 + console.error('エラーが発生しました', err);
32 +})
33 +.finally(function () {
34 + console.info('終了します');
35 +});
1 +#!/usr/bin/env node
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +/**
7 + *****************************************************************
8 + * !!! このサンプルは2015-08-30現在のfacebookでは動作しません !!!
9 + *****************************************************************
10 + * facebookログインサンプル
11 + *
12 + * 以下のusernameとpasswordを書き換えてから実行してください
13 + */
14 +var username = 'hogehoge';
15 +var password = 'fugafuga';
16 +
17 +
18 +var client = require('../index');
19 +
20 +console.info('facebookTOPページにアクセスします');
21 +client.fetch('http://www.facebook.com/')
22 +.then(function (result) {
23 + console.info('ログインフォームを送信します');
24 + return result.$('#login_form').submit({
25 + email: username,
26 + pass: password
27 + });
28 +})
29 +.then(function (result) {
30 + console.info('クッキー', result.response.cookies);
31 + console.info('ユーザー名を取得します');
32 + console.info(result.$('._2dpb').text());
33 +})
34 +.catch(function (err) {
35 + console.error('エラーが発生しました', err);
36 +})
37 +.finally(function () {
38 + console.info('終了します');
39 +});
1 +#!/usr/bin/env node
2 +/*eslint no-invalid-this:0*/
3 +/*jshint -W100*/
4 +
5 +'use strict';
6 +
7 +/**
8 + * Google検索サンプル
9 + *
10 + * 以下のword変数の内容で検索します
11 + */
12 +var word = 'ドラえもん';
13 +
14 +
15 +var client = require('../index');
16 +
17 +// [重要] google検索の場合はfollowMetaRefreshをfalseにする(README.md参照)
18 +client.set('followMetaRefresh', false);
19 +
20 +client.fetch('http://www.google.co.jp/search', { q: word }, function (err, $, res, body) {
21 + if (err) {
22 + console.error(err);
23 + return;
24 + }
25 +
26 + var results = [];
27 + // 検索結果が個別に格納されている要素をループ
28 + $('#rso .g').each(function () {
29 + // 各検索結果のタイトル部分とURL、概要を取得
30 + var $h3 = $(this).find('h3');
31 + var url = $h3.find('a').attr('href');
32 + if (url) {
33 + results.push({
34 + title: $h3.text(),
35 + url: url,
36 + description: $(this).find('.st').text()
37 + });
38 + }
39 + });
40 +
41 + console.info(results);
42 +});
1 +#!/usr/bin/env node
2 +/*eslint no-invalid-this:0*/
3 +/*jshint -W100*/
4 +
5 +'use strict';
6 +
7 +/**
8 + * はてなキーワード検索サンプル
9 + *
10 + * 以下のword変数の内容で検索します
11 + */
12 +var word = 'ネッシー';
13 +
14 +
15 +var client = require('../index');
16 +
17 +console.info('はてなキーワードTOPページにアクセスします');
18 +client.fetch('http://d.hatena.ne.jp/keyword/')
19 +.then(function (result) {
20 + console.info('最近更新されたキーワード一覧');
21 + var $ = result.$;
22 + $('#updatekeywords ul li .name').each(function () {
23 + // NEW!は邪魔なので削除
24 + $('.new', $(this)).remove();
25 + console.info('* ' + $(this).text());
26 + });
27 + console.info('「' + word + '」を検索します');
28 + return $('#index-search').submit({
29 + word: word
30 + });
31 +})
32 +.then(function (result) {
33 + console.info(result.$('.keyword-body').text().trim().replace(/[\t\u3000]/g, ''));
34 +})
35 +.catch(function (err) {
36 + console.error('エラーが発生しました', err);
37 +})
38 +.finally(function () {
39 + console.info('終了します');
40 +});
1 +#!/usr/bin/env node
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +/**
7 + * ニコニコ動画ログインサンプル
8 + *
9 + * 以下のusernameとpasswordを書き換えてから実行してください
10 + */
11 +var username = 'hogehoge';
12 +var password = 'fugafuga';
13 +
14 +
15 +var client = require('../index');
16 +
17 +console.info('ニコニコTOPページにアクセスします');
18 +client.fetch('http://nicovideo.jp/')
19 +.then(function (result) {
20 + console.info('ログインリンクをクリックします');
21 + return result.$('#sideNav .loginBtn').click();
22 +})
23 +.then(function (result) {
24 + console.info('ログインフォームを送信します');
25 + return result.$('#login_form').submit({
26 + mail_tel: username,
27 + password: password
28 + });
29 +})
30 +.then(function (result) {
31 + console.info('ログイン可否をチェックします');
32 + if (! result.response.headers['x-niconico-id']) {
33 + throw new Error('login failed');
34 + }
35 + console.info('クッキー', result.response.cookies);
36 + console.info('マイページに移動します');
37 + return client.fetch('http://www.nicovideo.jp/my/top');
38 +})
39 +.then(function (result) {
40 + console.info('マイページに表示されるアカウント名を取得します');
41 + console.info(result.$('#siteHeaderUserNickNameContainer').text());
42 +})
43 +.catch(function (err) {
44 + console.error('エラーが発生しました', err.message);
45 +})
46 +.finally(function () {
47 + console.info('終了します');
48 +});
1 +#!/usr/bin/env node
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +/**
7 + * Promise形式での利用サンプル
8 + */
9 +
10 +
11 +var client = require('../index');
12 +
13 +// なんとなくgooglebotのUser-Agentをセット
14 +client.set('browser', 'googlebot');
15 +
16 +// Yahooのトップページを取得
17 +client.fetch('http://www.yahoo.co.jp/')
18 +.then(function (result) {
19 + console.info('<then>', result.$('title').text());
20 + // Googleのトップページを取得
21 + return client.fetch('http://www.google.co.jp/');
22 +}).then(function (result) {
23 + console.info('<then>', result.$('title').text());
24 + // 例外を発生させる
25 + throw new Error('!!! error !!!');
26 +}).catch(function (err) {
27 + // 例外発生時の処理
28 + console.error('<catch>', err.message);
29 +}).finally(function () {
30 + // 最終的に必ず実行される処理
31 + console.info('<finally>');
32 +});
1 +#!/usr/bin/env node
2 +/*eslint no-invalid-this:0*/
3 +/*jshint -W100*/
4 +
5 +'use strict';
6 +
7 +/**
8 + * 同期リクエストによるGoogle検索サンプル
9 + *
10 + * 以下のword変数の内容で検索します
11 + */
12 +var word = 'チュパカブラ';
13 +
14 +
15 +var client = require('../index');
16 +
17 +// [重要] google検索の場合はfollowMetaRefreshをfalseにする(README.md参照)
18 +client.set('followMetaRefresh', false);
19 +
20 +console.info('--- Bingで検索 ---');
21 +var result1 = client.fetchSync('http://www.bing.com/search', { q: word });
22 +if (result1.error) {
23 + console.error(result1.error);
24 +} else {
25 + var results1 = [];
26 + // 検索結果が個別に格納されている要素をループ
27 + var $ = result1.$;
28 + $('.b_algo').each(function () {
29 + // 各検索結果のタイトル部分とURL、概要を取得
30 + var $h2 = $(this).find('h2');
31 + var url = $h2.find('a').attr('href');
32 + if (url) {
33 + results1.push({
34 + title: $h2.text(),
35 + url: url,
36 + description: $(this).find('.b_caption p').text()
37 + });
38 + }
39 + });
40 + console.info(results1);
41 +}
42 +
43 +console.info('\n--- Googleで検索 ---');
44 +var result2 = client.fetchSync('http://www.google.co.jp/search', { q: word });
45 +if (result2.error) {
46 + console.error(result2.error);
47 +} else {
48 + var results2 = [];
49 + // 検索結果が個別に格納されている要素をループ
50 + var _$ = result2.$;
51 + _$('#rso .g').each(function () {
52 + // 各検索結果のタイトル部分とURL、概要を取得
53 + var $h3 = _$(this).find('h3');
54 + var url = $h3.find('a').attr('href');
55 + if (url) {
56 + results2.push({
57 + title: $h3.text(),
58 + url: url,
59 + description: _$(this).find('.st').text()
60 + });
61 + }
62 + });
63 + console.info(results2);
64 +}
1 +#!/usr/bin/env node
2 +/*eslint no-invalid-this:0*/
3 +/*jshint -W100*/
4 +
5 +'use strict';
6 +
7 +var C = require('colors/safe');
8 +
9 +/**
10 + * tiqav画像取得サンプル
11 + */
12 +
13 +var fs = require('fs');
14 +var path = require('path');
15 +var client = require('../index');
16 +
17 +// 画像保存フォルダ作成
18 +var imgdir = path.join(__dirname, 'img');
19 +if (! fs.existsSync(imgdir)) {
20 + fs.mkdirSync(imgdir);
21 +}
22 +
23 +// ダウンロードマネージャー設定
24 +client.download
25 +.on('ready', function (stream) {
26 + // ダウンロード完了時の処理(各ファイルごとに呼ばれる)
27 + var file = stream.url.pathname.match(/([^\/]+)$/)[1];
28 + var savepath = path.join(imgdir, file);
29 + stream.pipe(fs.createWriteStream(savepath));
30 + console.info(C.green('SUCCESS'), C.blue(stream.url.href) + ' => ' + C.yellow(savepath));
31 + console.info(C.magenta('STATE'), this.state);
32 +})
33 +.on('error', function (err) {
34 + // ダウンロード失敗時の処理(各ファイルごとに呼ばれる)
35 + console.error(C.red('ERROR'), err);
36 + console.info(C.magenta('STATE'), this.state);
37 +})
38 +.on('end', function (err) {
39 + // ダウンロードキューが空になった時の処理
40 + console.info(C.green.bold('COMPLETE'), this.state);
41 +});
42 +
43 +
44 +// fetch start
45 +console.info(C.cyan('INFO'), 'tiqavにアクセスします');
46 +client.fetch('http://tiqav.com/')
47 +.then(function (result) {
48 + // 画像を根こそぎダウンロード
49 + var $imgs = result.$('.box img');
50 + console.info(C.cyan('INFO'), $imgs.length + '個の画像があります');
51 + $imgs.download();
52 +})
53 +.catch(function (err) {
54 + console.error(C.red('ERROR'), err);
55 +})
56 +.finally(function () {
57 + console.info(C.cyan('INFO'), 'スクレイピングが終了しました');
58 +});
1 +#!/usr/bin/env node
2 +/*eslint no-invalid-this:0*/
3 +/*jshint -W100*/
4 +
5 +'use strict';
6 +
7 +/**
8 + *****************************************************************
9 + * !!! 2017-02-06現在サイトが消えているので動作しません !!!
10 + *****************************************************************
11 + * 2chスレタイ検索サンプル
12 + *
13 + * 以下のword変数の内容で検索します
14 + */
15 +var word = '話題';
16 +
17 +
18 +var client = require('../index');
19 +
20 +client.fetch('http://ttsearch.net/')
21 +.then(function (result) {
22 + return result.$('form').eq(0).submit({
23 + k: word
24 + });
25 +})
26 +.then(function (result) {
27 + var $ = result.$;
28 + var results = [];
29 + $('a.title').each(function () {
30 + results.push({
31 + title: $(this).text().trim(),
32 + url: $(this).attr('href')
33 + });
34 + });
35 + console.info(results);
36 +})
37 +.catch(function (err) {
38 + console.error('エラーが発生しました', err);
39 +})
40 +.finally(function () {
41 + console.info('終了します');
42 +});
1 +#!/usr/bin/env node
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +/**
7 + * ブラウザ設定確認
8 + */
9 +var client = require('../index');
10 +
11 +client.set('debug', true);
12 +client.set('browser', 'firefox');
13 +
14 +client.fetch('http://www.useragentstring.com/')
15 +.then(function (result) {
16 + console.info(result.$('#dieTabelle th').text());
17 +})
18 +.catch(function (err) {
19 + console.error(err);
20 +});
1 +import * as http from 'http';
2 +import * as url from 'url';
3 +import * as stream from 'stream';
4 +//tslint:disable:next-line: no-import-side-effect
5 +import 'cheerio';
6 +
7 +// cheerio-httpcli本体
8 +declare namespace CheerioHttpcli {
9 + interface FetchResponse extends http.IncomingMessage {
10 + cookies: {[ name: string ]: string};
11 + }
12 +
13 + // CheerioStatic拡張
14 + interface CheerioStaticEx extends CheerioStatic {
15 + documentInfo(): { url: string, encoding: string | null, isXml: boolean };
16 + entityHtml(options?: CheerioOptionsInterface): string;
17 + entityHtml(selector: string, options?: CheerioOptionsInterface): string;
18 + entityHtml(element: Cheerio, options?: CheerioOptionsInterface): string;
19 + entityHtml(element: CheerioElement, options?: CheerioOptionsInterface): string;
20 + }
21 +
22 + interface FetchResult {
23 + error: Error;
24 + $: CheerioStaticEx;
25 + response: FetchResponse;
26 + body: string;
27 + }
28 + type FetchCallback = (error: Error, $: CheerioStaticEx, response: FetchResponse, body: string) => void;
29 +
30 + // やっつけPromise
31 + interface Promise {
32 + then(callback: (result: FetchResult) => void): Promise;
33 + catch(callbck: (error: Error) => void): Promise;
34 + finally(callback: () => void): Promise;
35 + }
36 +
37 + namespace Download {
38 + interface Stream extends stream.Stream {
39 + url: url.Url;
40 + type: string;
41 + length: number;
42 + toBuffer(callback: (error: Error, buffer: Buffer) => void): void;
43 + end(): void;
44 + }
45 + interface ErrorEx extends Error {
46 + url: string;
47 + }
48 +
49 + export interface Manager {
50 + parallel: number;
51 + state: { queue: number, complete: number, error: number };
52 + clearCache(): void;
53 + on(event: 'ready', handler: ((stream: Stream) => void)): Manager;
54 + on(event: 'error', handler: ((error: ErrorEx) => void)): Manager;
55 + on(event: 'end', handler: (() => void)): Manager;
56 + }
57 + }
58 +
59 + type FreeObject = {[ name: string ]: string};
60 + const headers: FreeObject;
61 + const agentOptions: FreeObject;
62 + const timeout: number;
63 + const maxDataSize: number;
64 + const gzip: boolean;
65 + const referer: boolean;
66 + const followMetaRefresh: boolean;
67 + const forceHtml: boolean;
68 + const debug: boolean;
69 + const browser: string;
70 + const iconv: string;
71 + const version: string;
72 + const download: Download.Manager;
73 +
74 + function reset(): void;
75 +
76 + function set(name: 'browser' | 'iconv', value: string): void;
77 + function set(name: 'timeout' | 'maxDataSize', value: number): void;
78 + function set(name: 'gzip' | 'referer' | 'followMetaRefresh' | 'forceHtml' | 'debug', value: boolean): void;
79 + function set(name: 'headers' | 'agentOptions', value: FreeObject, nomerge?: boolean): void;
80 +
81 + function setIconvEngine(icmod: 'iconv' | 'iconv-jp' | 'iconv-lite'): void;
82 + function setBrowser(type:
83 + 'ie' | 'edge' | 'chrome' | 'firefox' |
84 + 'opera' | 'vivaldi' | 'safari' |
85 + 'ipad' | 'iphone' | 'ipod' | 'android' |
86 + 'googlebot'): boolean;
87 +
88 + function fetch(url: string, param: {[ name: string ]: any}, encode: string, callback: FetchCallback): void;
89 + function fetch(url: string, param: {[ name: string ]: any}, callback: FetchCallback): void;
90 + function fetch(url: string, encode: string, callback: FetchCallback): void;
91 + function fetch(url: string, callback: FetchCallback): void;
92 +
93 + function fetch(url: string, param: {[ name: string ]: any}, encode: string): Promise;
94 + function fetch(url: string, param: {[ name: string ]: any}): Promise;
95 + function fetch(url: string, encode: string): Promise;
96 + function fetch(url: string): Promise;
97 +
98 + function fetchSync(url: string, param: {[ name: string ]: any}, encode: string): FetchResult;
99 + function fetchSync(url: string, param: {[ name: string ]: any}): FetchResult;
100 + function fetchSync(url: string, encode: string): FetchResult;
101 + function fetchSync(url: string): FetchResult;
102 +}
103 +
104 +// cheerio拡張メソッド(オリジナルのinterfaceにマージ)
105 +declare global {
106 + interface Cheerio {
107 + click(callback: CheerioHttpcli.FetchCallback): void;
108 + click(): CheerioHttpcli.Promise;
109 + clickSync(): CheerioHttpcli.FetchResult;
110 + download(srcAttr?: string | string[]): void;
111 + field(): {[ name: string ]: string | number};
112 + field(name: string): string | number;
113 + field(name: string, value: string | (() => string), onNotFound?: 'append' | 'throw'): Cheerio;
114 + field(name: {[ name: string ]: string | number}, onNotFound?: 'append' | 'throw'): Cheerio;
115 + entityHtml(): string;
116 + entityHtml(html: string): Cheerio;
117 + submit(param: {[ name: string ]: string | number}, callback: CheerioHttpcli.FetchCallback): void;
118 + submit(callback: CheerioHttpcli.FetchCallback): void;
119 + submit(param?: {[ name: string ]: string | number}): CheerioHttpcli.Promise;
120 + submitSync(param?: {[ name: string ]: string | number}): CheerioHttpcli.FetchResult;
121 + tick(): Cheerio;
122 + untick(): Cheerio;
123 + url(optFilter: { absolute?: boolean, relative?: boolean, invalid?: boolean }, srcAttrs?: string | string[]): string | string[];
124 + url(srcAttrs?: string | string[]): string | string[];
125 + }
126 +}
127 +
128 +export = CheerioHttpcli;
1 +module.exports = require('./lib/core');
1 +{
2 + "ie": "Mozilla/5.0 (Windows NT 10.0; Trident/7.0; rv:11.0) like Gecko",
3 + "edge": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063",
4 + "chrome": "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
5 + "firefox": "Mozilla/5.0 (Windows NT 10.0; rv:54.0) Gecko/20100101 Firefox/59.0",
6 + "opera": "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; ja) Presto/2.10.289 Version/12.00",
7 + "vivaldi": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.143 Safari/537.36 Vivaldi/1.95.1077.45",
8 + "safari": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8",
9 + "ipad": "Mozilla/5.0 (iPad; CPU OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1",
10 + "iphone": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
11 + "ipod": "Mozilla/5.0 (iPod; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3",
12 + "android": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 7 Build/MOB30X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Safari/537.36",
13 + "ps4": "Mozilla/5.0 (PlayStation 4 5.00) AppleWebKit/601.2 (KHTML, like Gecko)",
14 + "3ds": "Mozilla/5.0 (Nintendo 3DS; U; ; ja) Version/1.7567.JP",
15 + "googlebot": "Googlebot/2.1 (+http://www.google.com/bot.html)"
16 +}
1 +/*jshint -W100*/
2 +
3 +'use strict';
4 +
5 +var cheerio = require('cheerio');
6 +var each = require('foreach');
7 +
8 +/**
9 + * cheerioオブジェクト拡張モジュール(プロトタイプにメソッド追加)
10 + */
11 +module.exports = function (encoding, client) {
12 + each([
13 + 'html',
14 + 'click',
15 + 'submit',
16 + 'tick-untick',
17 + 'field',
18 + 'url',
19 + 'download'
20 + ], function (m) {
21 + require('./cheerio/' + m)(encoding, client, cheerio);
22 + });
23 +
24 + return cheerio;
25 +};
1 +/*jshint -W100*/
2 +
3 +'use strict';
4 +
5 +var cutil = require('./util');
6 +
7 +module.exports = function (encoding, client, cheerio) {
8 + /**
9 + * a要素のリンク/submit系ボタンのクリックをエミュレート
10 + *
11 + * a要素: リンク先のページを取得
12 + * submit系ボタン: 所属フォームのsubmit
13 + *
14 + * @param callback リクエスト完了時のコールバック関数(err, response, body(buffer))
15 + */
16 + var emulateClick = function (elem, callback) {
17 + var doc = cutil.documentInfo(elem);
18 + var $link = null;
19 +
20 + // a要素でなければエラー
21 + try {
22 + if (elem.length === 0) {
23 + throw new Error('no elements');
24 + }
25 +
26 + // 複数ある場合は先頭の要素のみ
27 + $link = elem.eq(0);
28 + // submit系要素の場合はsubmit()に飛ばす
29 + var type = $link.attr('type');
30 + var is = {
31 + a: $link.is('a'),
32 + input: $link.is('input'),
33 + button: $link.is('button')
34 + };
35 + if ((is.input || is.button) && cutil.inArray([ 'submit', 'image' ], type)) {
36 + var $form = $link.closest('form');
37 + var param = {};
38 + var name = cutil.paramFilter($link.attr('name'));
39 + if (name.length > 0) {
40 + if (type === 'submit') {
41 + // submit: 押したボタンのnameとvalueを送信する
42 + param[name] = $link.val() || $link.attr('value');
43 + } else {
44 + // image: 押したボタンのname.xとname.y座標を送信する(ダミーなので0)
45 + param[name + '.x'] = 0;
46 + param[name + '.y'] = 0;
47 + }
48 + }
49 + return $form.submit(param, callback);
50 + }
51 + // submit系要素でもa要素でもなければエラー
52 + if (! is.a) {
53 + throw new Error('element is not clickable');
54 + }
55 + } catch (e) {
56 + return client.error(e.message, {
57 + param: { uri: doc.url },
58 + callback: callback
59 + });
60 + }
61 +
62 + return client.run('GET', $link.url(), {}, null, callback);
63 + };
64 +
65 + /**
66 + * 非同期クリック
67 + */
68 + cheerio.prototype.click = function (callback) {
69 + return emulateClick(this, callback);
70 + };
71 +
72 + /**
73 + * 同期クリック
74 + */
75 + cheerio.prototype.clickSync = function (callback) {
76 + return emulateClick(this, 'sync');
77 + };
78 +};
1 +/*eslint no-invalid-this:0*/
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +var util = require('util');
7 +var events = require('events');
8 +var stream = require('stream');
9 +var async = require('async');
10 +var assign = require('object-assign');
11 +var cutil = require('./util');
12 +
13 +module.exports = function (encoding, client, cheerio) {
14 + var urlCache = [];
15 +
16 + /**
17 + * ダウンロードイベント管理用クラス
18 + *
19 + * すべてのダウンロード結果はこのクラスにemit()される
20 + * - success: ダウンロード完了時(url, buffer)
21 + * - error: ダウンロード失敗時(error)
22 + */
23 + var DownloadEvent = (function () {
24 + function DownloadEvent() {
25 + events.EventEmitter.call(this);
26 + this.parallel = 3;
27 + this.state = Object.freeze({
28 + queue: 0, // ダウンロードキューに残っている順番待ち画像の数
29 + complete: 0, // ダウンロードが完了した画像の数
30 + error: 0 // ダウンロードが失敗した画像の数
31 + });
32 + }
33 + util.inherits(DownloadEvent, events.EventEmitter);
34 +
35 + /**
36 + * URLキャッシュクリア
37 + */
38 + DownloadEvent.prototype.clearCache = function () {
39 + urlCache.length = 0;
40 + };
41 +
42 + return DownloadEvent;
43 + })();
44 +
45 + var downloadEvent = new DownloadEvent();
46 + Object.defineProperty(client.core, 'download', {
47 + enumerable: true,
48 + get: function () {
49 + return downloadEvent;
50 + }
51 + });
52 +
53 +
54 + /**
55 + * ダウンロードストリームクラス
56 + */
57 + var DownloadStream = (function () {
58 + function DownloadStream(res) {
59 + stream.PassThrough.call(this);
60 + this.url = (res.request) ? res.request.uri : 'base64';
61 + this.type = res.headers['content-type'];
62 + this.length = Number(res.headers['content-length'] || -1);
63 +
64 + // タイムアウト時間を過ぎてもStreamの読み出しが行われていない場合は放置されているとみなす
65 + this.__timer = setTimeout((function () {
66 + if (! this.isUsed()) {
67 + clearTimeout(this.__timer);
68 + this.__timer = null;
69 + this.emit('error', new Error('stream timeout (maybe stream is not used)'));
70 + }
71 + }).bind(this), client.core.timeout);
72 + }
73 + util.inherits(DownloadStream, stream.PassThrough);
74 +
75 + /**
76 + * Stream => Buffer
77 + *
78 + * @param cb 変換後のBufferを受け取るコールバック関数(err, buffer)
79 + */
80 + DownloadStream.prototype.toBuffer = function (cb) {
81 + if (! (cb instanceof Function)) {
82 + throw new Error('callback is not function');
83 + }
84 + if (this.isUsed()) {
85 + throw new Error('stream has already been read');
86 + }
87 +
88 + var buffer = [];
89 + this.on('data', function (chunk) {
90 + buffer.push(chunk);
91 + });
92 + this.on('error', function (err) {
93 + cb.call(this, err);
94 + });
95 + this.on('end', function () {
96 + cb.call(this, null, Buffer.concat(buffer));
97 + });
98 + };
99 +
100 + /**
101 + * Streamの読み出しが開始されたかどうか(on('data')/pipe()が使用された形跡でチェック)
102 + *
103 + * @return true: 開始された
104 + */
105 + DownloadStream.prototype.isUsed = function () {
106 + return (this._readableState.pipesCount > 0 || this.listeners('data').length > 0);
107 + };
108 +
109 + /**
110 + * 手動でend()が呼ばれた場合はスキップ扱いにする
111 + */
112 + DownloadStream.prototype.end = function () {
113 + if (! this.__timer) {
114 + return;
115 + }
116 + clearTimeout(this.__timer);
117 + this.__timer = null;
118 + this.emit('end');
119 + };
120 +
121 + return DownloadStream;
122 + })();
123 +
124 +
125 + /**
126 + * イベント管理用クラスカウントを更新
127 + */
128 + function updateState(counts) {
129 + client.core.download.state = Object.freeze(
130 + assign({}, client.core.download.state, counts)
131 + );
132 + }
133 +
134 +
135 + /**
136 + * ダウンロード統括管理用クラス
137 + */
138 + var DownloadManager = (function () {
139 + var jobRunning = false;
140 +
141 + /**
142 + * ダウンロードループ実行
143 + */
144 + function downloadJob(manager) {
145 + // 実行フラグON
146 + jobRunning = true;
147 +
148 + // 現在キューに入っている分を全部切り出して一時キューに移動
149 + var qLen = manager.queue.length;
150 + var tmp = {
151 + queue: manager.queue.splice(0, qLen),
152 + complete: 0,
153 + error: 0,
154 + applyState: function (complete, error) {
155 + // 一時キューの処理状況をダウンロードマネージャーに反映させる
156 + this.complete += complete;
157 + this.error += error;
158 + updateState({
159 + complete: client.core.download.state.complete + complete,
160 + error: client.core.download.state.error + error,
161 + queue: manager.queue.length + this.queue.length - this.complete - this.error
162 + });
163 + }
164 + };
165 +
166 + // 一時キュー内のURLを順番にダウンロード(同時処理数: parallel)
167 + async.eachLimit(tmp.queue, manager.parallel, function (url, next) {
168 + var req = null; // リクエストオブジェクト
169 + var strm = null; // ダウンロードストリームオブジェクト
170 +
171 + // 失敗時の処理
172 + var onError = function (err) {
173 + tmp.applyState(0, 1);
174 + err.url = url;
175 + client.core.download.emit('error', err);
176 + req.abort();
177 + next();
178 + return;
179 + };
180 +
181 + // ストリームで取得する場合はgzipにしない
182 + var options = client.prepare('GET', url, {}, null);
183 + options.param.gzip = false;
184 + try {
185 + req = client.request(options.param);
186 + } catch (e) {
187 + e.type = 'Request Exception';
188 + onError(e);
189 + return;
190 + }
191 +
192 + req
193 + .on('response', function (res) {
194 + if (String(res.statusCode).substr(0, 2) !== '20') {
195 + var err = new Error('server status');
196 + err.statusCode = res.statusCode;
197 + err.type = 'Invalid Response';
198 + onError(err);
199 + return;
200 + }
201 +
202 + // ダウンロードストリームオブジェクトを作成してレスポンスを流し込む
203 + strm = new DownloadStream(res);
204 + // ダウンロード完了時
205 + strm
206 + .on('end', function () {
207 + tmp.applyState(1, 0);
208 + req.abort();
209 + next();
210 + })
211 + .on('error', function (err) {
212 + err.type = 'Stream Error';
213 + onError(err);
214 + });
215 +
216 + client.core.download.emit('ready', strm);
217 + req.pipe(strm);
218 + })
219 + // 複数回発生するようなのでonce
220 + .once('error', function (err) {
221 + err.type = 'Request Error';
222 + return onError(err);
223 + });
224 + }, function (err) {
225 + // 現在のダウンロード中にキューに追加されているURLがあるかもしれないので
226 + // 再度loopイベントを飛ばしておく
227 + jobRunning = false;
228 + manager.emit('loop');
229 + });
230 + }
231 +
232 + function DownloadManager() {
233 + this.queue = []; // ダウンロード待ちURL配列
234 +
235 + this.on('loop', (function () {
236 + if (jobRunning) {
237 + // 二重処理防止
238 + return;
239 + }
240 +
241 + if (this.queue.length > 0) {
242 + downloadJob(this);
243 + } else {
244 + client.core.download.emit('end');
245 + }
246 + }).bind(this));
247 + }
248 + util.inherits(DownloadManager, events.EventEmitter);
249 +
250 + /**
251 + * ダウンロードキューにURLを追加
252 + *
253 + * @param url ダウンロードするURL
254 + * @return true: キューに登録された
255 + */
256 + DownloadManager.prototype.addQueue = function (url) {
257 + if (! url) {
258 + return false;
259 + }
260 + if (cutil.inArray(urlCache, url)) {
261 + // 登録/ダウンロード済み
262 + return false;
263 + }
264 + urlCache.push(url);
265 + process.nextTick((function () {
266 + this.queue.push(url);
267 + this.emit('loop');
268 + }).bind(this));
269 + return true;
270 + };
271 +
272 + return DownloadManager;
273 + })();
274 + var manager = new DownloadManager();
275 +
276 +
277 + /**
278 + * img要素の画像をダウンロード
279 + *
280 + * @param srcAttrs (imgのみ)srcよりも優先して取得する属性名(文字列 or 配列)
281 + * @return キューに登録した数
282 + */
283 + cheerio.prototype.download = function (srcAttrs) {
284 + // ダウンロードマネージャーの設定がされていない
285 + if (client.core.download.listeners('ready').length === 0) {
286 + throw new Error('download manager configured no event');
287 + }
288 +
289 + var doc = cutil.documentInfo(this);
290 + var $ = cheerio;
291 +
292 + // 最初に全要素がimg要素かどうかチェック
293 + this.each(function () {
294 + if (! $(this).is('img')) {
295 + throw new Error('element is not img');
296 + }
297 + });
298 +
299 + // 同時実行数チェック
300 + var parallel = parseInt(client.core.download.parallel, 10);
301 + if (parallel < 1 || parallel > 5) {
302 + throw new Error('valid download parallel range is 1 and 5');
303 + }
304 + manager.parallel = parallel;
305 +
306 + var queued = 0;
307 + this.each(function () {
308 + var $elem = $(this);
309 +
310 + // ここの$はfetch()を経由していないので_documentInfoがない
311 + if (! $elem._root) {
312 + $elem._root = {
313 + 0: { _documentInfo: doc }
314 + };
315 + }
316 +
317 + // Base64埋め込み画像の場合はBuffer化して即返す
318 + var b64chk = ($elem.attr('src') || '').match(/^data:(image\/\w+);base64,([\s\S]+)$/i);
319 + if (b64chk) {
320 + updateState({
321 + complete: client.core.download.state.complete + 1
322 + });
323 + var b64buf = cutil.newBuffer(b64chk[2], 'base64');
324 + var strm = new DownloadStream({
325 + headers: {
326 + 'content-type': b64chk[1],
327 + 'content-length': b64buf.length
328 + }
329 + });
330 + client.core.download.emit('ready', strm);
331 + strm.write(b64buf);
332 + strm.end();
333 + queued++;
334 + return;
335 + }
336 +
337 + var url = $elem.url({ invalid: false }, srcAttrs);
338 + if (manager.addQueue(url)) {
339 + queued++;
340 + }
341 + });
342 +
343 + return queued;
344 + };
345 +};
1 +/*eslint no-invalid-this:0*/
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +var typeOf = require('type-of');
7 +var each = require('foreach');
8 +var cutil = require('./util');
9 +
10 +module.exports = function (encoding, client, cheerio) {
11 + /**
12 + * フォーム内部品の値取得/設定
13 + *
14 + * jQueryの$().attrや$().cssと同じ使用感
15 + * 1. nameのみ指定した場合はそのname部品の値を取得
16 + * 2. name, valueを指定した場合はそのname部品の値をvalueに設定
17 + * 3. valueに関数を指定した場合はその関数の戻り値をname部品の値に設定
18 + * 4. nameに文字列でなくname:valueの連想配列を指定した場合は複数要素をまとめて設定
19 + *
20 + * @param name 対象のフォーム部品のname or name: valueの連想配列
21 + * @param value 設定する値 or 値を返す関数 or undefined
22 + * @param onNotFound 指定したnameの要素が見つからない場合の挙動('throw', 'append')
23 + * @return 1: name部品の値 or 2-4: this(メソッドチェーン用)
24 + */
25 + cheerio.prototype.field = function (name, value, onNotFound) {
26 + var $ = cheerio;
27 + var $form = null;
28 +
29 + // form要素でなければエラー
30 + if (this.length === 0) {
31 + throw new Error('no elements');
32 + }
33 +
34 + // 複数ある場合は先頭のフォームのみ
35 + $form = this.eq(0);
36 + if (! $form.is('form')) {
37 + throw new Error('element is not form');
38 + }
39 +
40 + // *** 値取得モード ***
41 + var argLen = arguments.length;
42 + var isGet = ((argLen === 0) || (typeOf(name) === 'string' && argLen === 1));
43 + if (isGet) {
44 + // cheerio.serializeArray()だと値のない部品を拾ってくれないようなので自力でやる
45 + var fieldInfo = {};
46 + $form.find('input,textarea,select').each(function (idx) {
47 + var $e = $(this);
48 + var name = $e.attr('name');
49 + var type = ($e.attr('type') || '').toLowerCase();
50 + var val = $e.val() || $e.attr('value');
51 + if (! name) {
52 + return;
53 + }
54 + // submit系要素はjavascriptでform.submit()した時にはパラメータとして付加しない
55 + // (ブラウザと同じ挙動)
56 + if (cutil.inArray([ 'submit', 'image' ], type)) {
57 + return;
58 + }
59 + fieldInfo[name] = fieldInfo[name] || {};
60 + fieldInfo[name].count = (fieldInfo[name].count || 0) + 1;
61 + fieldInfo[name].params = fieldInfo[name].params || [];
62 +
63 + // radioは複数同nameがあるのが普通なので設定値を配列にしない
64 + if (type === 'radio' && ! fieldInfo[name].force) {
65 + fieldInfo[name].force = 'single';
66 + }
67 + // selectでmultipleの場合は強制的に設定値を配列にする
68 + if ($e.is('select') && $e.attr('multiple') && ! fieldInfo[name].force) {
69 + fieldInfo[name].force = 'multi';
70 + }
71 +
72 + if (cutil.inArray([ 'checkbox', 'radio' ], type) && ! $e.attr('checked')) {
73 + return;
74 + }
75 + if (typeOf(val) === 'array') {
76 + fieldInfo[name].params = fieldInfo[name].params.concat(val);
77 + } else {
78 + fieldInfo[name].params.push(val);
79 + }
80 + });
81 + // 複数同nameのcheckboxやmultipleのselect以外は値の配列化を解除
82 + var fieldParams = {};
83 + each(fieldInfo, function (info, name) {
84 + fieldParams[name] = info.params;
85 + if (info.force !== 'multi' && (info.force === 'single' || info.count === 1)) {
86 + fieldParams[name] = fieldParams[name].shift();
87 + }
88 + });
89 +
90 + // 引数未指定の場合はそのフォーム内の全要素のname:valueを連想配列で返す
91 + return (argLen === 0) ? fieldParams : fieldParams[name];
92 + }
93 +
94 + // *** 値設定モード ***
95 + var values = {};
96 + switch (typeOf(name)) {
97 + case 'string': {
98 + // name: valueの連想配列化してvaluesにセット
99 + values[name] = value;
100 + break;
101 + }
102 + case 'object': {
103 + values = name;
104 + // 連想配列で指定した場合はvalueの位置にonNotFoundが入っているのでずらす
105 + onNotFound = value;
106 + break;
107 + }
108 + default: {
109 + // それ以外の型は受け付けない
110 + throw new Error('name is not string or object');
111 + }
112 + }
113 +
114 + each(values, function (v, k, o) {
115 + // valueが関数で指定されている場合は実行して値ベースにそろえる
116 + var realValue = (typeOf(v) === 'function') ? v() : v;
117 +
118 + // 同じnameで別部品とかあってもさすがにそれはフォーム側の問題な気がするので無視
119 + var selector = [ 'input', 'textarea', 'select' ].map(function (s) {
120 + return s + '[name="' + k + '"]';
121 + }).join(',');
122 + var $parts = $form.find(selector);
123 + var pType = $parts.attr('type');
124 +
125 + if ($parts.length === 0) {
126 + // nameに該当する部品が見つからない場合はonNotFoundに従う
127 + if (onNotFound === 'append') {
128 + // append: 新規に要素を作成してフォームに付加
129 + var iType = 'hidden';
130 + if (typeOf(realValue) === 'array') {
131 + // 値が配列: checkbox
132 + iType = 'checkbox';
133 + } else {
134 + // 値が文字列: hidden
135 + realValue = [ realValue ];
136 + }
137 + each(realValue, function (val) {
138 + var $input = $('<input/>').attr({ type: iType, name: k, value: val });
139 + if (iType === 'checkbox') {
140 + $input.attr('checked', 'checked');
141 + }
142 + $form.append($input);
143 + });
144 + return;
145 + }
146 + if (onNotFound === 'throw') {
147 + // throw: エラー
148 + throw new Error('Element named "' + k + '" could not be found in this form');
149 + }
150 + }
151 +
152 + if (cutil.inArray([ 'checkbox', 'radio' ], pType)) {
153 + // radioの場合は指定したvalueが該当しなければ何もしない
154 + if (pType === 'radio') {
155 + realValue = String(realValue);
156 + var partsValues = $parts.map(function (idx) {
157 + return $(this).val();
158 + }).get();
159 + if (! cutil.inArray(partsValues, realValue)) {
160 + return;
161 + }
162 + }
163 +
164 + // tick/untickで値を操作
165 + if (typeOf(realValue) !== 'array') {
166 + realValue = [ realValue ];
167 + }
168 + $parts.untick().each(function (idx) {
169 + if (cutil.inArray(realValue, $(this).val())) {
170 + $(this).tick();
171 + }
172 + });
173 + return;
174 + }
175 +
176 + $parts.val(realValue);
177 + });
178 +
179 + return this;
180 + };
181 +};
1 +/*eslint no-invalid-this:0*/
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +var util = require('util');
7 +var cutil = require('./util');
8 +
9 +module.exports = function (encoding, client, cheerio) {
10 + /**
11 + * cheerio.load()の際にdecodeEntitiesをtrueにするとhtml()メソッドで文字列が
12 + * すべてエンティティ表記になってしまうのでエンティティをデコードするように拡張する
13 + *
14 + * @param str 指定した場合はその文字列を要素のHTMLとして設定
15 + * 指定しない場合は要素に設定されているHTMLを返す
16 + */
17 + // cheerioデフォルトのhtmlメソッドをentityHtmlとして退避
18 + cheerio.prototype.entityHtml = cheerio.prototype.html;
19 +
20 + cheerio.prototype.html = function (str) {
21 + // cheerioデフォルトのhtml()結果をデコード(エンティティ可読文字化)したものを返す
22 + return cutil.decodeEntities(this.entityHtml(str));
23 + };
24 +
25 + /***
26 + * [DEPRECATED] 将来削除予定
27 + */
28 + var deprecatedMessage = '%s() will be removed in the future)';
29 + cheerio.prototype._text = function (str) {
30 + cutil.colorMessage('DEPRECATED', util.format(deprecatedMessage, '_text'));
31 + return this.text(str);
32 + };
33 +
34 + cheerio.prototype._html = function (str) {
35 + cutil.colorMessage('DEPRECATED', util.format(deprecatedMessage, '_html'));
36 + return this.html(str);
37 + };
38 +};
1 +/*jshint -W100*/
2 +
3 +'use strict';
4 +
5 +var urlParser = require('url');
6 +var typeOf = require('type-of');
7 +var each = require('foreach');
8 +var cutil = require('./util');
9 +
10 +module.exports = function (encoding, client, cheerio) {
11 + /**
12 + * form要素からの送信をエミュレート
13 + *
14 + * @param param 疑似設定するフォーム送信パラメータ
15 + * @param callback リクエスト完了時のコールバック関数(err, response, body(buffer))
16 + */
17 + var emulateSubmit = function (elem, param, callback) {
18 + if (param === 'sync' || typeOf(param) === 'function') {
19 + callback = param;
20 + param = {};
21 + }
22 + param = param || {};
23 +
24 + var doc = cutil.documentInfo(elem);
25 + var $form = null;
26 +
27 + // form要素でなければエラー
28 + try {
29 + if (elem.length === 0) {
30 + throw new Error('no elements');
31 + }
32 +
33 + // 複数ある場合は先頭のフォームのみ
34 + $form = elem.eq(0);
35 + if (! $form.is('form')) {
36 + throw new Error('element is not form');
37 + }
38 + } catch (e) {
39 + return client.error(e.message, {
40 + param: { uri: doc.url },
41 + callback: callback
42 + });
43 + }
44 +
45 + // methodとURL確定
46 + var method = ($form.attr('method') || 'GET').toUpperCase();
47 + var url = urlParser.resolve(doc.url, $form.attr('action') || '');
48 +
49 + // フォーム送信パラメータ作成
50 + // 1. デフォルトパラメータ($form.field())を取得した後に
51 + // 2. 引数で指定したパラメータ(param)で上書き
52 + var formParam = {};
53 + each([ $form.field(), param ], function (fp) {
54 + each(fp, function (val, name) {
55 + var fparam = cutil.paramFilter(val);
56 + var fvalue = (typeOf(fparam) === 'array') ? fparam : [ fparam ];
57 + // 空パラメータでもname=のみで送信するための仕込み
58 + if (fvalue.length === 0) {
59 + fvalue.push('');
60 + }
61 + formParam[name] = fvalue;
62 + });
63 + });
64 +
65 + // 各種エンコーディングに対応したURLエンコードをする必要があるのでパラメータ文字列を自力で作成
66 + var formParamArray = [];
67 + // accept-charsetは先頭で指定されているものを使用
68 + var enc = ($form.attr('accept-charset') || doc.encoding).split(/[\s,]+/)[0];
69 + each(formParam, function (val, name) {
70 + formParamArray.push(val.map(function (vv) {
71 + var escName = encoding.escape(enc, name);
72 + var escVal = encoding.escape(enc, vv);
73 + if (method === 'POST') {
74 + // application/x-www-form-urlencodedでは半角スペースは%20ではなく+にする
75 + escName = escName.replace(/%20/g, '+');
76 + escVal = escVal.replace(/%20/g, '+');
77 + }
78 + return escName + '=' + escVal;
79 + }).join('&'));
80 + });
81 + var formParamStr = formParamArray.join('&');
82 +
83 + // GETの場合はURLに繋げてパラメータを空にする(そうしないと上手く動かないケースがたまにあった)
84 + if (method === 'GET') {
85 + var join = (url.indexOf('?') === -1) ? '?' : '&';
86 + if (formParamStr.length > 0) {
87 + url += join + formParamStr;
88 + }
89 + formParamStr = {};
90 + }
91 +
92 + return client.run(method, url, formParamStr, null, callback);
93 + };
94 +
95 + /**
96 + * 非同期フォーム送信
97 + */
98 + cheerio.prototype.submit = function (param, callback) {
99 + return emulateSubmit(this, param, callback);
100 + };
101 +
102 + /**
103 + * 同期フォーム送信
104 + */
105 + cheerio.prototype.submitSync = function (param) {
106 + return emulateSubmit(this, param, 'sync');
107 + };
108 +};
1 +/*eslint no-invalid-this:0*/
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +var each = require('foreach');
7 +var cutil = require('./util');
8 +
9 +module.exports = function (encoding, client, cheerio) {
10 + /**
11 + * チェックボックス/ラジオボタンの選択クリックをエミュレート
12 + *
13 + * @param elem checked状態を変更するcheerio要素
14 + * @param checked 設定する値('checked' or undefined)
15 + */
16 + var emulateTick = function (elem, checked) {
17 + var $ = cheerio;
18 +
19 + if ($(elem).length === 0) {
20 + throw new Error('no elements');
21 + }
22 +
23 + // checkboxとradioの振り分け
24 + var $targets = {
25 + checkbox: [],
26 + radio: []
27 + };
28 + var radioGroups = [];
29 + $(elem).each(function (i) {
30 + var $e = $(this);
31 + var type = $e.attr('type');
32 + if (! $e.is('input') || ! cutil.inArray([ 'checkbox', 'radio' ], type)) {
33 + // input[type=checkbox/radio]以外が混じっていたらエラー
34 + throw new Error('element is not checkbox or radio');
35 + }
36 + // radio: 同グループで複数要素がtick対象となっている場合は先頭以外の要素は無視
37 + if (type === 'radio' && checked) {
38 + var name = $e.attr('name').toLowerCase();
39 + if (cutil.inArray(radioGroups, name)) {
40 + return;
41 + }
42 + radioGroups.push(name);
43 + }
44 + $targets[type].push($e);
45 + });
46 +
47 + // 振り分けたcheckboxとradioに対してそれぞれ選択状態の変更を行う
48 + each($targets, function ($elem, type) {
49 + if (type === 'radio' && checked) {
50 + // radioかつtickの場合はまず同グループの選択済みradioを全部未選択にする
51 + each($elem, function ($e) {
52 + var name = $e.attr('name');
53 + $e
54 + .closest('form') // 所属するフォーム
55 + .find('input[type=radio][name="' + name + '"]') // 同グループのradio
56 + .removeAttr('checked'); // 選択状態
57 + });
58 + }
59 +
60 + each($elem, function ($e) {
61 + if (checked) {
62 + $e.attr('checked', checked);
63 + } else {
64 + $e.removeAttr('checked');
65 + }
66 + });
67 + });
68 +
69 + return elem;
70 + };
71 +
72 + /**
73 + * チェックボックス/ラジオボタンを選択状態にする
74 + */
75 + cheerio.prototype.tick = function () {
76 + return emulateTick(this, 'checked');
77 + };
78 +
79 + /**
80 + * チェックボックス/ラジオボタンの選択状態を解除する
81 + */
82 + cheerio.prototype.untick = function () {
83 + return emulateTick(this);
84 + };
85 +};
1 +/*eslint no-invalid-this:0*/
2 +/*jshint -W100*/
3 +
4 +'use strict';
5 +
6 +var urlParser = require('url');
7 +var valUrl = require('valid-url');
8 +var typeOf = require('type-of');
9 +var cutil = require('./util');
10 +var assign = require('object-assign');
11 +
12 +module.exports = function (encoding, client, cheerio) {
13 + /**
14 + * a要素/img要素/script要素/link要素の絶対URLを取得
15 + *
16 + * @param optFilter 取得するURLのフィルタリングオプション
17 + * @param srcAttrs (imgのみ)srcよりも優先して取得する属性名(文字列 or 配列)
18 + * @return 絶対URLもしくはその配列
19 + */
20 + cheerio.prototype.url = function (optFilter, srcAttrs) {
21 + var doc = cutil.documentInfo(this);
22 + var $ = cheerio;
23 + var result = [];
24 +
25 + if (cutil.inArray([ 'string', 'array' ], typeOf(optFilter))) {
26 + srcAttrs = optFilter;
27 + optFilter = {};
28 + } else {
29 + optFilter = optFilter || {};
30 + }
31 +
32 + var filter = assign({
33 + relative: true, // 相対リンク
34 + absolute: true, // 絶対リンク
35 + invalid: true // URL以外
36 + }, optFilter);
37 +
38 + srcAttrs = srcAttrs || [ 'data-original', 'data-lazy-src', 'data-src' ];
39 + if (typeOf(srcAttrs) !== 'array') {
40 + srcAttrs = [ srcAttrs ];
41 + }
42 + srcAttrs.push('src');
43 +
44 + // a要素/img要素/script要素でなければエラー
45 + this.each(function () {
46 + var $elem = $(this);
47 + var is = {
48 + a: $elem.is('a'),
49 + img: $elem.is('img'),
50 + script: $elem.is('script'),
51 + link: $elem.is('link')
52 + };
53 + if (! is.a && ! is.img && ! is.script && ! is.link) {
54 + throw new Error('element is not link, img, script or link');
55 + }
56 +
57 + // URLを取り出して絶対化
58 + var srcUrl = null;
59 + if (is.a || is.link) {
60 + srcUrl = $elem.attr('href');
61 + } else if (is.script) {
62 + srcUrl = $elem.attr('src');
63 + } else {
64 + // imgの場合はsrcAttrsの優先順に従って属性を見ていく
65 + for (var i = 0; i < srcAttrs.length; i++) {
66 + srcUrl = $elem.attr(srcAttrs[i]);
67 + if (srcUrl) {
68 + break;
69 + }
70 + }
71 + }
72 +
73 + var absUrl = (srcUrl) ? urlParser.resolve(doc.url, srcUrl) : srcUrl;
74 +
75 + // 除外判定
76 + if (valUrl.isWebUri(absUrl)) {
77 + var isAbsoluteLink = /^[a-z]+:\/\//i.test(srcUrl);
78 + if (isAbsoluteLink && ! filter.absolute) {
79 + return;
80 + }
81 + if (! isAbsoluteLink && ! filter.relative) {
82 + return;
83 + }
84 + } else if (! filter.invalid) {
85 + return;
86 + }
87 + result.push(absUrl);
88 + });
89 +
90 + // 要素数が1の場合は配列でなく文字列で返す
91 + return (this.length === 1) ? result[0] : result;
92 + };
93 +};
1 +/*jshint -W100*/
2 +
3 +'use strict';
4 +
5 +var he = require('he');
6 +var typeOf = require('type-of');
7 +var colors = require('colors/safe');
8 +
9 +/**
10 + * 汎用関数 - エンティティのデコード
11 + *
12 + * @param str エンティティ化された文字列
13 + */
14 +module.exports.decodeEntities = function (str) {
15 + // 文字列でない場合(cheerioオブジェクトなど)はそのまま返す
16 + if (typeOf(str) !== 'string') {
17 + return str;
18 + }
19 + return he.decode(str);
20 +};
21 +
22 +/**
23 + * 汎用関数 - パラメータの正規化
24 + *
25 + * @param val GET/POSTパラメータ
26 + */
27 +module.exports.paramFilter = function (val) {
28 + // 0はパラメータとして有効なので残す
29 + // null/undefinedは空文字にして返す
30 + if (typeOf(val) !== 'number' && ! val) {
31 + val = '';
32 + }
33 + return val;
34 +};
35 +
36 +/**
37 + * 汎用関数 - cheerio拡張情報_documentInfo取得
38 + *
39 + * @param $ 拡張cheerioオブジェクト
40 + * @return client.jsでWEBページ情報取得時にセットされた_documentInfo
41 + */
42 +module.exports.documentInfo = function ($) {
43 + if ($.cheerio !== '[cheerio object]') {
44 + throw new Error('argument is not cheerio object');
45 + }
46 + // 大元の_rootは_originalRootという名称で保持されているらしい by cheerio/lib/static.js
47 + return $._root[0]._documentInfo || $._originalRoot._documentInfo;
48 +};
49 +
50 +/**
51 + * 汎用関数 - PHPでいうin_array()
52 + * cheerioとは無関係
53 + *
54 + * @param array 調べる配列
55 + * @param val 調べる値
56 + * @return true or false
57 + */
58 +module.exports.inArray = function (array, val) {
59 + if (typeOf(array) !== 'array') {
60 + throw new Error(array + ' is not Array');
61 + }
62 + return (array.indexOf(val) !== -1);
63 +};
64 +
65 +/**
66 + * 汎用関数 - Buffer初期化
67 + * Buffer.fromが実装されているバージョンならそちらを優先して使用
68 + * cheerioとは無関係
69 + *
70 + * @param val 初期化内容
71 + * @param type 初期化型
72 + * @return Buffer
73 + */
74 +module.exports.newBuffer = function (val, type) {
75 + return (Buffer.from) ? Buffer.from(val, type) : new Buffer(val, type);
76 +};
77 +
78 +/**
79 + * 汎用関数 - 色付きメッセージ表示
80 + * cheerioとは無関係
81 + *
82 + * @param type メッセージの種類
83 + * @param msg 表示するメッセージ
84 + */
85 +module.exports.colorMessage = function (type, msg) {
86 + /*eslint-disable key-spacing*/
87 + var colorConf = {
88 + DEPRECATED : 'yellow',
89 + WARNING : 'magenta'
90 + };
91 + /*eslint-enable key-spacing*/
92 +
93 + // スタックトレースを取得
94 + var stackTrace = null;
95 + try {
96 + throw new Error('dummy');
97 + } catch (e) {
98 + stackTrace = e.stack.split(/[\r\n]+/).filter(function (v, v2, v3) {
99 + return /^\s*at\s+.*:[\d]+:[\d]+/.test(v);
100 + }).map(function (v) {
101 + return v.trim();
102 + });
103 + }
104 +
105 + // メッセージのトリガーとなった箇所を取得
106 + var at = '';
107 + for (var i = 0; i < stackTrace.length; i++) {
108 + var trace = stackTrace[i];
109 + if (! /^at/.test(trace)) continue;
110 + if (/[\\\/]cheerio-httpcli[\\\/]lib[\\\/]/.test(trace)) continue;
111 + at = trace;
112 + break;
113 + }
114 +
115 + console.warn(colors[colorConf[type] || 'white'](
116 + '[' + type + '] ' + msg + ' ' + at
117 + ));
118 +};
This diff is collapsed. Click to expand it.
1 +/*jshint -W100*/
2 +
3 +'use strict';
4 +
5 +var assign = require('object-assign');
6 +var typeOf = require('type-of');
7 +var each = require('foreach');
8 +var cheerioExtend = require('./cheerio-extend');
9 +var util = require('./cheerio/util.js');
10 +var encoding = require('./encoding');
11 +var client = require('./client');
12 +var pkg = require('../package.json');
13 +var browsers = require('./browsers.json');
14 +
15 +/**
16 + * cheerio-httpcli内部からプロパティを直接更新する際の黒魔術
17 + */
18 +var propertyUpdater = {
19 + // 内部的にプロパティを直接更新する際の照合キー
20 + Key: Math.random().toString(36).substr(2),
21 +
22 + // プロパティ更新時の値を黒魔術で包み込む
23 + wrap: function (value) {
24 + return [ this.key, value ];
25 + },
26 +
27 + // プロパティ更新時の値を黒魔術から取り出す
28 + unwrap: function (value) {
29 + if ((value instanceof Array) &&
30 + value.length === 2 && value[0] === this.key) {
31 + return value[1];
32 + }
33 +
34 + util.colorMessage('DEPRECATED', 'direct property update will be refused in the future. use set(key, value)');
35 + //throw new Error(direct property update is not permitted. use set(key, value)');
36 + return value;
37 + }
38 +};
39 +
40 +// リクエストヘッダを作り直す
41 +var rebuildHeaders = function (value) {
42 + var tmp = {};
43 + var nullKeys = [];
44 + // リクエストヘッダは小文字に統一する & 値にnullが入っているキーは削除
45 + each(value, function (val, key) {
46 + if (value[key] === null) {
47 + nullKeys.push(key.toLowerCase());
48 + }
49 + tmp[key.toLowerCase()] = val;
50 + });
51 + each(nullKeys, function (key) {
52 + delete tmp[key];
53 + });
54 + return tmp;
55 +};
56 +
57 +// 通常プロパティ作成(property変数内で管理)
58 +var defineNormalProperties = function (cli) {
59 + /*eslint-disable key-spacing*/
60 + var property = {
61 + // リクエストヘッダ
62 + headers : { types: [ 'object' ], value: null },
63 + // タイムアウトまでの時間
64 + timeout : { types: [ 'number' ], value: null },
65 + // gzip転送する/しない
66 + gzip : { types: [ 'boolean' ], value: null },
67 + // Refererを自動設定する/しない
68 + referer : { types: [ 'boolean' ], value: null },
69 + // <meta[http-equiv=refresh]>を検知してリダイレクトする/しない
70 + followMetaRefresh : { types: [ 'boolean' ], value: null },
71 + // 受信を許可する最大のサイズ
72 + maxDataSize : { types: [ 'number', 'null' ], value: null },
73 + // XML自動判別を使用しない
74 + forceHtml : { types: [ 'boolean' ], value: null },
75 + // requestモジュールに渡すagentOptions
76 + agentOptions : { types: [ 'object' ], value: null },
77 + // デバッグオプション
78 + debug : { types: [ 'boolean' ], value: null }
79 + };
80 + /*eslint-enable key-spacing*/
81 +
82 + // プロパティ登録(直接更新時にはDEPRECATEDメッセージを表示)
83 + Object.keys(property).forEach(function (prop) {
84 + Object.defineProperty(cli, prop, {
85 + enumerable: true,
86 + get: function () {
87 + // TODO 現在は直接更新も可としているのでコメントアウトしておく
88 + //if (typeOf(property[prop].value) === 'object') {
89 + // // オブジェクトの場合は複製を返す
90 + // var copy = {};
91 + // each(property[prop].value, function (val, key) {
92 + // copy[key] = val;
93 + // });
94 + // return copy;
95 + //}
96 + return property[prop].value;
97 + },
98 + set: function (value) {
99 + value = propertyUpdater.unwrap(value);
100 +
101 + // 型チェック
102 + var types = property[prop].types;
103 + var vtype = typeOf(value);
104 + if (types.indexOf(vtype) === -1 || (vtype === 'number' && value < 0)) {
105 + util.colorMessage(
106 + 'WARNING',
107 + 'invalid value: ' + String(value) + '. '
108 + + 'property "' + prop + '" can accept only ' + types.join(' or ')
109 + );
110 + return;
111 + }
112 +
113 + // headersのキーを全部小文字に変換 & 値がnullのキーは削除
114 + if (prop === 'headers') {
115 + value = rebuildHeaders(value);
116 + }
117 +
118 + property[prop].value = value;
119 + }
120 + });
121 + });
122 +};
123 +
124 +// 特殊プロパティ作成(動的に値を用意する)
125 +var defineSpecialProperties = function (cli) {
126 + // browserプロパティ
127 + Object.defineProperty(cli, 'browser', {
128 + enumerable: true,
129 + get: function () {
130 + // User-Agentとブラウザテンプレが一致するものを探す
131 + var ua = this.headers['user-agent'];
132 + if (! ua) {
133 + return null;
134 + }
135 + var browser = Object.keys(browsers).filter(function (name) {
136 + return (browsers[name] === ua);
137 + });
138 + if (browser.length > 0) {
139 + return browser[0];
140 + }
141 + return 'custom';
142 + },
143 + set: function (type) {
144 + type = propertyUpdater.unwrap(type);
145 +
146 + if (type in browsers) {
147 + this.set('headers', {
148 + 'User-Agent': browsers[type]
149 + });
150 + return;
151 + }
152 + util.colorMessage('WARNING', 'unknown browser: ' + type);
153 + }
154 + });
155 +
156 + // iconvプロパティ
157 + Object.defineProperty(cli, 'iconv', {
158 + enumerable: true,
159 + get: function () {
160 + return encoding.getIconvType();
161 + },
162 + set: function (icmod) {
163 + icmod = propertyUpdater.unwrap(icmod);
164 +
165 + if (! encoding.iconvLoad(icmod)) {
166 + throw new Error('Cannot find module "' + icmod + '"');
167 + }
168 + }
169 + });
170 +
171 + // バージョン情報プロパティ
172 + Object.defineProperty(cli, 'version', {
173 + enumerable: true,
174 + get: function () {
175 + return pkg.version;
176 + }
177 + });
178 +};
179 +
180 +// WIP マルチインスタンス化に向けて構造変更中
181 +var CheerioHttpCli = (function () {
182 + /**
183 + * コンストラクタ
184 + */
185 + function CheerioHttpCli() {
186 + defineNormalProperties(this);
187 + defineSpecialProperties(this);
188 + }
189 +
190 + /**
191 + * プロパティや内部情報の初期化
192 + */
193 + CheerioHttpCli.prototype.reset = function () {
194 + // リクエストヘッダ
195 + this.set('headers', {}, true);
196 + // タイムアウトまでの時間(効いているかどうか不明)
197 + this.set('timeout', 30000);
198 + // gzip転送する/しない
199 + this.set('gzip', true);
200 + // Refererを自動設定する/しない
201 + this.set('referer', true);
202 + // <meta[http-equiv=refresh]>を検知してリダイレクトする/しない
203 + this.set('followMetaRefresh', false);
204 + // 受信を許可する最大のサイズ
205 + this.set('maxDataSize', null);
206 + // XML自動判別を使用しない
207 + this.set('forceHtml', false);
208 + // requestモジュールに渡すagentOptions
209 + this.set('agentOptions', {}, true);
210 + // デバッグオプション
211 + this.set('debug', false);
212 +
213 + client.reset();
214 + return this;
215 + };
216 +
217 + /**
218 + * プロパティを操作
219 + *
220 + * @param name 操作するプロパティ名
221 + * @param value 挿入する値
222 + * @param nomerge trueのときマージを行わない
223 + */
224 + CheerioHttpCli.prototype.set = function (name, value, nomerge) {
225 + // 特殊プロパティ
226 + if ([ 'browser', 'iconv' ].indexOf(name) !== -1) {
227 + this[name] = propertyUpdater.wrap(value);
228 + return this;
229 + }
230 +
231 + // プロパティが存在するかチェック
232 + if (! Object.keys(this).some((function (prop) {
233 + return (prop === name && typeOf(this[prop]) !== 'function');
234 + }).bind(this))) {
235 + throw new Error('no such property "' + name + '"');
236 + }
237 +
238 + // オブジェクトへの代入ならマージする(黒魔術使用)
239 + if (! nomerge && typeOf(this[name]) === 'object' && typeOf(value) === 'object') {
240 + this[name] = propertyUpdater.wrap(assign(this[name], value));
241 + } else {
242 + this[name] = propertyUpdater.wrap(value);
243 + }
244 + return this;
245 + };
246 +
247 + /**
248 + * [DEPRECATED] 使用するiconvモジュールを指定
249 + *
250 + * @param icmod iconvモジュール名(iconv|iconv-jp|iconv-lite)
251 + */
252 + CheerioHttpCli.prototype.setIconvEngine = function (icmod) {
253 + util.colorMessage('DEPRECATED', 'setIconvEngine() will be removed in the future. use set("iconv", value)');
254 + this.set('iconv', icmod);
255 + };
256 +
257 + /**
258 + * [DEPRECATED] ブラウザごとのUser-Agentをワンタッチ設定
259 + *
260 + * @param browser ブラウザ種類(see browsers.json)
261 + * @return 設定できた/できなかった
262 + */
263 + CheerioHttpCli.prototype.setBrowser = function (type) {
264 + util.colorMessage('DEPRECATED', 'setBrowser() will be removed in the future. use set("browser", value)');
265 + this.set('browser', type);
266 + };
267 +
268 + /**
269 + * GETによる非同期httpリクエストを実行
270 + *
271 + * @param url リクエスト先のURL
272 + * @param param リクエストパラメータ
273 + * @param encode 取得先のHTMLのエンコーディング(default: 自動判定)
274 + * @param callback リクエスト完了時のコールバック関数(err, cheerio, response, body)
275 + */
276 + CheerioHttpCli.prototype.fetch = function (url, param, encode, callback) {
277 + return client.run('GET', url, param, encode, callback);
278 + };
279 +
280 + /**
281 + * GETによる同期httpリクエストを実行
282 + *
283 + * @param url リクエスト先のURL
284 + * @param param リクエストパラメータ
285 + * @param encode 取得先のHTMLのエンコーディング(default: 自動判定)
286 + * @param callback リクエスト完了時のコールバック関数(err, cheerio, response, body)
287 + */
288 + CheerioHttpCli.prototype.fetchSync = function (url, param, encode) {
289 + return client.run('GET', url, param, encode, 'sync');
290 + };
291 +
292 + return CheerioHttpCli;
293 +})();
294 +
295 +var mainClient = new CheerioHttpCli();
296 +// clientオブジェクト内で使用する外部オブジェクトを登録
297 +mainClient.reset();
298 +client.core = mainClient;
299 +client.encoding = encoding;
300 +client.cheerio = cheerioExtend(encoding, client);
301 +
302 +module.exports = mainClient;
1 +/*jshint -W100*/
2 +
3 +'use strict';
4 +
5 +var jschardet = require('jschardet');
6 +var cutil = require('./cheerio/util');
7 +
8 +/**
9 + * <head>タグ内からエンコーディングを判定する正規表現
10 + */
11 +/*eslint-disable key-spacing*/
12 +var reEnc = {
13 + head : /<head[\s>]([\s\S]*?)<\/head>/i,
14 + charset : /<meta[^>]*[\s;]+charset\s*=\s*["']?([\w\-_]+)["']?/i
15 +};
16 +
17 +/**
18 + * iconvモジュール情報
19 + */
20 +var iconvMod = {
21 + name : null,
22 + engine : null,
23 + func : null,
24 + cache : {}
25 +};
26 +/*eslint-enable key-spacing*/
27 +
28 +/**
29 + * encodingモジュール本体
30 + */
31 +var encoding = {
32 + /**
33 + * メソッド
34 + */
35 +
36 + /**
37 + * iconvモジュールをロード
38 + *
39 + * @param module iconvモジュール名(iconv|iconv-jp|iconv-lite)
40 + * @return ロードできた/できなかった
41 + */
42 + iconvLoad: function (module) {
43 + // モジュール名チェック
44 + if (! /^iconv(-(jp|lite))?$/.test(module)) {
45 + return false;
46 + }
47 +
48 + // モジュールをロード
49 + try {
50 + iconvMod.engine = require(module);
51 + } catch (/*eslint-disable no-unused-vars*/ e /*eslint-enable no-unused-vars*/) {
52 + return false;
53 + }
54 +
55 + if (iconvMod.engine.Iconv) {
56 + // iconv/iconv-jpはIconvというメソッドを持っている
57 + iconvMod.func = function (enc, buffer, revert) {
58 + enc = enc.toUpperCase();
59 + var from = enc;
60 + var to = 'UTF-8';
61 + if (revert) {
62 + from = 'UTF-8';
63 + to = enc;
64 + }
65 + var cacheKey = from + ':' + to;
66 + if (! (cacheKey in iconvMod.cache)) {
67 + // Iconvオブジェクトをキャッシュする
68 + iconvMod.cache[cacheKey] = new iconvMod.engine.Iconv(from, to + '//TRANSLIT//IGNORE');
69 + }
70 + return iconvMod.cache[cacheKey].convert(buffer);
71 + };
72 + } else {
73 + // iconv-lite用
74 + iconvMod.func = function (enc, buffer, revert) {
75 + if (! iconvMod.engine.encodingExists(enc)) {
76 + // iconv/iconv-jpとエラーオブジェクトの形を合わせる
77 + var err = new Error('EINVAL, Conversion not supported.');
78 + err.errno = 22;
79 + err.code = 'EINVAL';
80 + throw err;
81 + }
82 + return iconvMod.engine[(revert) ? 'encode' : 'decode'](buffer, enc);
83 + };
84 + }
85 + iconvMod.name = module;
86 + return true;
87 + },
88 +
89 + /**
90 + * 使用中のiconvモジュールの種類を取得
91 + *
92 + * @return iconvモジュール名(iconv|iconv-jp|iconv-lite)
93 + */
94 + getIconvType: function () {
95 + return iconvMod.name;
96 + },
97 +
98 + /**
99 + * エンコーディング名指定がUTF-8かどうか
100 + *
101 + * @param enc エンコーディング指定名('utf-8', 'shift_jis', ...)
102 + * @return true or false
103 + */
104 + isUTF8: function (enc) {
105 + return /^utf\-?8$/i.test(enc);
106 + },
107 +
108 + /**
109 + * HTML(Buffer)のエンコーディングをUTF-8に変換
110 + *
111 + * @param enc 変換元のエンコーディング
112 + * @param buffer HTML(Buffer)
113 + * @return UTF-8に変換後のHTML(Buffer)
114 + */
115 + convert: function (enc, buffer) {
116 + if (this.isUTF8(enc)) {
117 + return buffer;
118 + }
119 + if (/(shift_jis|sjis)/i.test(enc)) {
120 + // Shift_JISを指定してIconvで変換すると半角チルダが波ダッシュ(0x301C)に変換されてしまうのでCP932に変更
121 + enc = 'CP932';
122 + }
123 + return iconvMod.func(enc, buffer);
124 + },
125 +
126 + /**
127 + * パラメータのURL%エンコード(各種エンコーディング対応)
128 + *
129 + * @param enc 変換先のエンコーディング
130 + * @param str URLエンコードする文字列
131 + * @return encで指定したエンコーディングでURL%エンコードした文字列
132 + */
133 + escape: function (enc, str) {
134 + // var re = /^[\w\.\(\)\-!~*']+$/; // encodeURIComponent互換
135 + var re = /^[\w~.-]+$/; // RFC-3986準拠
136 + str = String(str);
137 + if (re.test(str)) {
138 + // エンコード不要
139 + return str;
140 + }
141 +
142 + // UTF-8から指定したエンコーディングに変換したバッファを回してエスケープ文字列作成
143 + var buffer = cutil.newBuffer(str);
144 + if (! this.isUTF8(enc)) {
145 + buffer = iconvMod.func(enc, buffer, true);
146 + }
147 + return Array.prototype.slice.call(buffer).map(function (b) {
148 + if (b < 128) {
149 + var c = String.fromCharCode(b);
150 + if (re.test(c)) {
151 + return c;
152 + }
153 + }
154 + return '%' + ('0' + b.toString(16).toUpperCase()).substr(-2);
155 + }).join('');
156 + },
157 +
158 + /**
159 + * jschardetモジュールによるHTMLのエンコーディング判定
160 + *
161 + * @param buffer HTML(Buffer)
162 + * @return 判定できた場合はエンコーディング名
163 + */
164 + detectByBuffer: function (buffer) {
165 + var enc = jschardet.detect(buffer);
166 + // 高精度で判定できた場合のみ
167 + if (enc && enc.encoding && (enc.confidence || 0) >= 0.99) {
168 + return enc.encoding;
169 + }
170 + return null;
171 + },
172 +
173 + /**
174 + * <head>タグ内から正規表現でエンコーディング判定
175 + *
176 + * @param buffer HTML(Buffer)
177 + * @return 判定できた場合はエンコーディング名
178 + */
179 + detectByHeader: function (buffer) {
180 + var head = buffer.toString('ascii').match(reEnc.head);
181 + if (head) {
182 + var charset = head[1].match(reEnc.charset);
183 + if (charset) {
184 + return charset[1].trim();
185 + }
186 + }
187 + return null;
188 + },
189 +
190 + /**
191 + * HTMLエンコーディング判定(自動判定 -> <head>タグ判定の順)
192 + *
193 + * @param buffer HTML(Buffer)
194 + * @return 判定できた場合はエンコーディング名
195 + */
196 + detect: function (buffer) {
197 + return this.detectByBuffer(buffer) || this.detectByHeader(buffer);
198 + }
199 +};
200 +
201 +// 初期状態では iconv > iconv-lite の優先順でロードしておく
202 +/*eslint-disable no-unused-expressions*/
203 +encoding.iconvLoad('iconv') || encoding.iconvLoad('iconv-lite');
204 +/*eslint-enable no-unused-expressions*/
205 +
206 +module.exports = encoding;
1 +/*jshint -W100*/
2 +
3 +'use strict';
4 +
5 +var client = require('./client');
6 +
7 +process.stdin.resume();
8 +process.stdin.setEncoding('utf8');
9 +
10 +var input = '';
11 +process.stdin.on('data', function (chunk) {
12 + input += chunk;
13 +});
14 +
15 +process.stdin.on('end', function () {
16 + var args = JSON.parse(input);
17 + client.core = args.core;
18 +
19 + // こちらでjarを作りなおす
20 + args.param.jar = client.engine.jar();
21 + Object.keys(args.cookies).forEach(function (key) {
22 + var val = args.cookies[key];
23 + var cookie = client.engine.cookie(key + '=' + val);
24 + args.param.jar.setCookie(cookie, args.param.uri);
25 + });
26 +
27 + client.request(args.param, args.core.gzip, function (err, res, body) {
28 + if (err) {
29 + process.stderr.write(err.message);
30 + }
31 + process.stdout.write(JSON.stringify({
32 + body: body,
33 + response: (res) ? res.toJSON() : null,
34 + cookies: args.param.jar.getCookies(args.param.uri) || []
35 + }));
36 + });
37 +});
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
1 +/**
2 + * Export cheerio (with )
3 + */
4 +
5 +exports = module.exports = require('./lib/cheerio');
6 +
7 +/*
8 + Export the version
9 +*/
10 +
11 +exports.version = require('./package.json').version;
1 +var domEach = require('../utils').domEach,
2 + _ = {
3 + pick: require('lodash.pick'),
4 + };
5 +
6 +var toString = Object.prototype.toString;
7 +
8 +/**
9 + * Set / Get css.
10 + *
11 + * @param {String|Object} prop
12 + * @param {String} val
13 + * @return {self}
14 + * @api public
15 + */
16 +
17 +exports.css = function(prop, val) {
18 + if (arguments.length === 2 ||
19 + // When `prop` is a "plain" object
20 + (toString.call(prop) === '[object Object]')) {
21 + return domEach(this, function(idx, el) {
22 + setCss(el, prop, val, idx);
23 + });
24 + } else {
25 + return getCss(this[0], prop);
26 + }
27 +};
28 +
29 +/**
30 + * Set styles of all elements.
31 + *
32 + * @param {String|Object} prop
33 + * @param {String} val
34 + * @param {Number} idx - optional index within the selection
35 + * @return {self}
36 + * @api private
37 + */
38 +
39 +function setCss(el, prop, val, idx) {
40 + if ('string' == typeof prop) {
41 + var styles = getCss(el);
42 + if (typeof val === 'function') {
43 + val = val.call(el, idx, styles[prop]);
44 + }
45 +
46 + if (val === '') {
47 + delete styles[prop];
48 + } else if (val != null) {
49 + styles[prop] = val;
50 + }
51 +
52 + el.attribs.style = stringify(styles);
53 + } else if ('object' == typeof prop) {
54 + Object.keys(prop).forEach(function(k){
55 + setCss(el, k, prop[k]);
56 + });
57 + }
58 +}
59 +
60 +/**
61 + * Get parsed styles of the first element.
62 + *
63 + * @param {String} prop
64 + * @return {Object}
65 + * @api private
66 + */
67 +
68 +function getCss(el, prop) {
69 + var styles = parse(el.attribs.style);
70 + if (typeof prop === 'string') {
71 + return styles[prop];
72 + } else if (Array.isArray(prop)) {
73 + return _.pick(styles, prop);
74 + } else {
75 + return styles;
76 + }
77 +}
78 +
79 +/**
80 + * Stringify `obj` to styles.
81 + *
82 + * @param {Object} obj
83 + * @return {Object}
84 + * @api private
85 + */
86 +
87 +function stringify(obj) {
88 + return Object.keys(obj || {})
89 + .reduce(function(str, prop){
90 + return str += ''
91 + + (str ? ' ' : '')
92 + + prop
93 + + ': '
94 + + obj[prop]
95 + + ';';
96 + }, '');
97 +}
98 +
99 +/**
100 + * Parse `styles`.
101 + *
102 + * @param {String} styles
103 + * @return {Object}
104 + * @api private
105 + */
106 +
107 +function parse(styles) {
108 + styles = (styles || '').trim();
109 +
110 + if (!styles) return {};
111 +
112 + return styles
113 + .split(';')
114 + .reduce(function(obj, str){
115 + var n = str.indexOf(':');
116 + // skip if there is no :, or if it is the first/last character
117 + if (n < 1 || n === str.length-1) return obj;
118 + obj[str.slice(0,n).trim()] = str.slice(n+1).trim();
119 + return obj;
120 + }, {});
121 +}
1 +// https://github.com/jquery/jquery/blob/2.1.3/src/manipulation/var/rcheckableType.js
2 +// https://github.com/jquery/jquery/blob/2.1.3/src/serialize.js
3 +var submittableSelector = 'input,select,textarea,keygen',
4 + r20 = /%20/g,
5 + rCRLF = /\r?\n/g,
6 + _ = {
7 + map: require('lodash.map')
8 + };
9 +
10 +exports.serialize = function() {
11 + // Convert form elements into name/value objects
12 + var arr = this.serializeArray();
13 +
14 + // Serialize each element into a key/value string
15 + var retArr = _.map(arr, function(data) {
16 + return encodeURIComponent(data.name) + '=' + encodeURIComponent(data.value);
17 + });
18 +
19 + // Return the resulting serialization
20 + return retArr.join('&').replace(r20, '+');
21 +};
22 +
23 +exports.serializeArray = function() {
24 + // Resolve all form elements from either forms or collections of form elements
25 + var Cheerio = this.constructor;
26 + return this.map(function() {
27 + var elem = this;
28 + var $elem = Cheerio(elem);
29 + if (elem.name === 'form') {
30 + return $elem.find(submittableSelector).toArray();
31 + } else {
32 + return $elem.filter(submittableSelector).toArray();
33 + }
34 + }).filter(
35 + // Verify elements have a name (`attr.name`) and are not disabled (`:disabled`)
36 + '[name!=""]:not(:disabled)'
37 + // and cannot be clicked (`[type=submit]`) or are used in `x-www-form-urlencoded` (`[type=file]`)
38 + + ':not(:submit, :button, :image, :reset, :file)'
39 + // and are either checked/don't have a checkable state
40 + + ':matches([checked], :not(:checkbox, :radio))'
41 + // Convert each of the elements to its value(s)
42 + ).map(function(i, elem) {
43 + var $elem = Cheerio(elem);
44 + var name = $elem.attr('name');
45 + var val = $elem.val();
46 +
47 + // If there is no value set (e.g. `undefined`, `null`), then return nothing
48 + if (val == null) {
49 + return null;
50 + } else {
51 + // If we have an array of values (e.g. `<select multiple>`), return an array of key/value pairs
52 + if (Array.isArray(val)) {
53 + return _.map(val, function(val) {
54 + // We trim replace any line endings (e.g. `\r` or `\r\n` with `\r\n`) to guarantee consistency across platforms
55 + // These can occur inside of `<textarea>'s`
56 + return {name: name, value: val.replace( rCRLF, '\r\n' )};
57 + });
58 + // Otherwise (e.g. `<input type="text">`, return only one key/value pair
59 + } else {
60 + return {name: name, value: val.replace( rCRLF, '\r\n' )};
61 + }
62 + }
63 + // Convert our result to an array
64 + }).get();
65 +};
1 +/*
2 + Module dependencies
3 +*/
4 +
5 +var parse = require('./parse'),
6 + isHtml = require('./utils').isHtml,
7 + _ = {
8 + extend: require('lodash.assignin'),
9 + bind: require('lodash.bind'),
10 + forEach: require('lodash.foreach'),
11 + defaults: require('lodash.defaults')
12 + };
13 +
14 +/*
15 + * The API
16 + */
17 +
18 +var api = [
19 + require('./api/attributes'),
20 + require('./api/traversing'),
21 + require('./api/manipulation'),
22 + require('./api/css'),
23 + require('./api/forms')
24 +];
25 +
26 +/*
27 + * Instance of cheerio
28 + */
29 +
30 +var Cheerio = module.exports = function(selector, context, root, options) {
31 + if (!(this instanceof Cheerio)) return new Cheerio(selector, context, root, options);
32 +
33 + this.options = _.defaults(options || {}, this.options);
34 +
35 + // $(), $(null), $(undefined), $(false)
36 + if (!selector) return this;
37 +
38 + if (root) {
39 + if (typeof root === 'string') root = parse(root, this.options);
40 + this._root = Cheerio.call(this, root);
41 + }
42 +
43 + // $($)
44 + if (selector.cheerio) return selector;
45 +
46 + // $(dom)
47 + if (isNode(selector))
48 + selector = [selector];
49 +
50 + // $([dom])
51 + if (Array.isArray(selector)) {
52 + _.forEach(selector, _.bind(function(elem, idx) {
53 + this[idx] = elem;
54 + }, this));
55 + this.length = selector.length;
56 + return this;
57 + }
58 +
59 + // $(<html>)
60 + if (typeof selector === 'string' && isHtml(selector)) {
61 + return Cheerio.call(this, parse(selector, this.options).children);
62 + }
63 +
64 + // If we don't have a context, maybe we have a root, from loading
65 + if (!context) {
66 + context = this._root;
67 + } else if (typeof context === 'string') {
68 + if (isHtml(context)) {
69 + // $('li', '<ul>...</ul>')
70 + context = parse(context, this.options);
71 + context = Cheerio.call(this, context);
72 + } else {
73 + // $('li', 'ul')
74 + selector = [context, selector].join(' ');
75 + context = this._root;
76 + }
77 + // $('li', node), $('li', [nodes])
78 + } else if (!context.cheerio) {
79 + context = Cheerio.call(this, context);
80 + }
81 +
82 + // If we still don't have a context, return
83 + if (!context) return this;
84 +
85 + // #id, .class, tag
86 + return context.find(selector);
87 +};
88 +
89 +/**
90 + * Mix in `static`
91 + */
92 +
93 +_.extend(Cheerio, require('./static'));
94 +
95 +/*
96 + * Set a signature of the object
97 + */
98 +
99 +Cheerio.prototype.cheerio = '[cheerio object]';
100 +
101 +/*
102 + * Cheerio default options
103 + */
104 +
105 +Cheerio.prototype.options = {
106 + withDomLvl1: true,
107 + normalizeWhitespace: false,
108 + xmlMode: false,
109 + decodeEntities: true
110 +};
111 +
112 +/*
113 + * Make cheerio an array-like object
114 + */
115 +
116 +Cheerio.prototype.length = 0;
117 +Cheerio.prototype.splice = Array.prototype.splice;
118 +
119 +/*
120 + * Make a cheerio object
121 + *
122 + * @api private
123 + */
124 +
125 +Cheerio.prototype._make = function(dom, context) {
126 + var cheerio = new this.constructor(dom, context, this._root, this.options);
127 + cheerio.prevObject = this;
128 + return cheerio;
129 +};
130 +
131 +/**
132 + * Turn a cheerio object into an array
133 + */
134 +
135 +Cheerio.prototype.toArray = function() {
136 + return this.get();
137 +};
138 +
139 +/**
140 + * Plug in the API
141 + */
142 +api.forEach(function(mod) {
143 + _.extend(Cheerio.prototype, mod);
144 +});
145 +
146 +var isNode = function(obj) {
147 + return obj.name || obj.type === 'text' || obj.type === 'comment';
148 +};
1 +/*
2 + Module Dependencies
3 +*/
4 +var htmlparser = require('htmlparser2');
5 +
6 +/*
7 + Parser
8 +*/
9 +exports = module.exports = function(content, options) {
10 + var dom = exports.evaluate(content, options),
11 + // Generic root element
12 + root = exports.evaluate('<root></root>', options)[0];
13 +
14 + root.type = 'root';
15 +
16 + // Update the dom using the root
17 + exports.update(dom, root);
18 +
19 + return root;
20 +};
21 +
22 +exports.evaluate = function(content, options) {
23 + // options = options || $.fn.options;
24 +
25 + var dom;
26 +
27 + if (typeof content === 'string' || Buffer.isBuffer(content)) {
28 + dom = htmlparser.parseDOM(content, options);
29 + } else {
30 + dom = content;
31 + }
32 +
33 + return dom;
34 +};
35 +
36 +/*
37 + Update the dom structure, for one changed layer
38 +*/
39 +exports.update = function(arr, parent) {
40 + // normalize
41 + if (!Array.isArray(arr)) arr = [arr];
42 +
43 + // Update parent
44 + if (parent) {
45 + parent.children = arr;
46 + } else {
47 + parent = null;
48 + }
49 +
50 + // Update neighbors
51 + for (var i = 0; i < arr.length; i++) {
52 + var node = arr[i];
53 +
54 + // Cleanly remove existing nodes from their previous structures.
55 + var oldParent = node.parent || node.root,
56 + oldSiblings = oldParent && oldParent.children;
57 + if (oldSiblings && oldSiblings !== arr) {
58 + oldSiblings.splice(oldSiblings.indexOf(node), 1);
59 + if (node.prev) {
60 + node.prev.next = node.next;
61 + }
62 + if (node.next) {
63 + node.next.prev = node.prev;
64 + }
65 + }
66 +
67 + if (parent) {
68 + node.prev = arr[i - 1] || null;
69 + node.next = arr[i + 1] || null;
70 + } else {
71 + node.prev = node.next = null;
72 + }
73 +
74 + if (parent && parent.type === 'root') {
75 + node.root = parent;
76 + node.parent = null;
77 + } else {
78 + node.root = null;
79 + node.parent = parent;
80 + }
81 + }
82 +
83 + return parent;
84 +};
85 +
86 +// module.exports = $.extend(exports);
1 +/**
2 + * Module dependencies
3 + */
4 +
5 +var serialize = require('dom-serializer'),
6 + select = require('css-select'),
7 + parse = require('./parse'),
8 + _ = {
9 + merge: require('lodash.merge'),
10 + defaults: require('lodash.defaults')
11 + };
12 +
13 +/**
14 + * $.load(str)
15 + */
16 +
17 +exports.load = function(content, options) {
18 + var Cheerio = require('./cheerio');
19 +
20 + options = _.defaults(options || {}, Cheerio.prototype.options);
21 +
22 + var root = parse(content, options);
23 +
24 + var initialize = function(selector, context, r, opts) {
25 + if (!(this instanceof initialize)) {
26 + return new initialize(selector, context, r, opts);
27 + }
28 + opts = _.defaults(opts || {}, options);
29 + return Cheerio.call(this, selector, context, r || root, opts);
30 + };
31 +
32 + // Ensure that selections created by the "loaded" `initialize` function are
33 + // true Cheerio instances.
34 + initialize.prototype = Object.create(Cheerio.prototype);
35 + initialize.prototype.constructor = initialize;
36 +
37 + // Mimic jQuery's prototype alias for plugin authors.
38 + initialize.fn = initialize.prototype;
39 +
40 + // Keep a reference to the top-level scope so we can chain methods that implicitly
41 + // resolve selectors; e.g. $("<span>").(".bar"), which otherwise loses ._root
42 + initialize.prototype._originalRoot = root;
43 +
44 + // Add in the static methods
45 + _.merge(initialize, exports);
46 +
47 + // Add in the root
48 + initialize._root = root;
49 + // store options
50 + initialize._options = options;
51 +
52 + return initialize;
53 +};
54 +
55 +/*
56 +* Helper function
57 +*/
58 +
59 +function render(that, dom, options) {
60 + if (!dom) {
61 + if (that._root && that._root.children) {
62 + dom = that._root.children;
63 + } else {
64 + return '';
65 + }
66 + } else if (typeof dom === 'string') {
67 + dom = select(dom, that._root, options);
68 + }
69 +
70 + return serialize(dom, options);
71 +}
72 +
73 +/**
74 + * $.html([selector | dom], [options])
75 + */
76 +
77 +exports.html = function(dom, options) {
78 + var Cheerio = require('./cheerio');
79 +
80 + // be flexible about parameters, sometimes we call html(),
81 + // with options as only parameter
82 + // check dom argument for dom element specific properties
83 + // assume there is no 'length' or 'type' properties in the options object
84 + if (Object.prototype.toString.call(dom) === '[object Object]' && !options && !('length' in dom) && !('type' in dom))
85 + {
86 + options = dom;
87 + dom = undefined;
88 + }
89 +
90 + // sometimes $.html() used without preloading html
91 + // so fallback non existing options to the default ones
92 + options = _.defaults(options || {}, this._options, Cheerio.prototype.options);
93 +
94 + return render(this, dom, options);
95 +};
96 +
97 +/**
98 + * $.xml([selector | dom])
99 + */
100 +
101 +exports.xml = function(dom) {
102 + var options = _.defaults({xmlMode: true}, this._options);
103 +
104 + return render(this, dom, options);
105 +};
106 +
107 +/**
108 + * $.text(dom)
109 + */
110 +
111 +exports.text = function(elems) {
112 + if (!elems) {
113 + elems = this.root();
114 + }
115 +
116 + var ret = '',
117 + len = elems.length,
118 + elem;
119 +
120 + for (var i = 0; i < len; i++) {
121 + elem = elems[i];
122 + if (elem.type === 'text') ret += elem.data;
123 + else if (elem.children && elem.type !== 'comment') {
124 + ret += exports.text(elem.children);
125 + }
126 + }
127 +
128 + return ret;
129 +};
130 +
131 +/**
132 + * $.parseHTML(data [, context ] [, keepScripts ])
133 + * Parses a string into an array of DOM nodes. The `context` argument has no
134 + * meaning for Cheerio, but it is maintained for API compatibility with jQuery.
135 + */
136 +exports.parseHTML = function(data, context, keepScripts) {
137 + var parsed;
138 +
139 + if (!data || typeof data !== 'string') {
140 + return null;
141 + }
142 +
143 + if (typeof context === 'boolean') {
144 + keepScripts = context;
145 + }
146 +
147 + parsed = this.load(data);
148 + if (!keepScripts) {
149 + parsed('script').remove();
150 + }
151 +
152 + // The `children` array is used by Cheerio internally to group elements that
153 + // share the same parents. When nodes created through `parseHTML` are
154 + // inserted into previously-existing DOM structures, they will be removed
155 + // from the `children` array. The results of `parseHTML` should remain
156 + // constant across these operations, so a shallow copy should be returned.
157 + return parsed.root()[0].children.slice();
158 +};
159 +
160 +/**
161 + * $.root()
162 + */
163 +exports.root = function() {
164 + return this(this._root);
165 +};
166 +
167 +/**
168 + * $.contains()
169 + */
170 +exports.contains = function(container, contained) {
171 +
172 + // According to the jQuery API, an element does not "contain" itself
173 + if (contained === container) {
174 + return false;
175 + }
176 +
177 + // Step up the descendants, stopping when the root element is reached
178 + // (signaled by `.parent` returning a reference to the same object)
179 + while (contained && contained !== contained.parent) {
180 + contained = contained.parent;
181 + if (contained === container) {
182 + return true;
183 + }
184 + }
185 +
186 + return false;
187 +};
1 +var parse = require('./parse'),
2 + render = require('dom-serializer');
3 +
4 +/**
5 + * HTML Tags
6 + */
7 +
8 +var tags = { tag: true, script: true, style: true };
9 +
10 +/**
11 + * Check if the DOM element is a tag
12 + *
13 + * isTag(type) includes <script> and <style> tags
14 + */
15 +
16 +exports.isTag = function(type) {
17 + if (type.type) type = type.type;
18 + return tags[type] || false;
19 +};
20 +
21 +/**
22 + * Convert a string to camel case notation.
23 + * @param {String} str String to be converted.
24 + * @return {String} String in camel case notation.
25 + */
26 +
27 +exports.camelCase = function(str) {
28 + return str.replace(/[_.-](\w|$)/g, function(_, x) {
29 + return x.toUpperCase();
30 + });
31 +};
32 +
33 +/**
34 + * Convert a string from camel case to "CSS case", where word boundaries are
35 + * described by hyphens ("-") and all characters are lower-case.
36 + * @param {String} str String to be converted.
37 + * @return {string} String in "CSS case".
38 + */
39 +exports.cssCase = function(str) {
40 + return str.replace(/[A-Z]/g, '-$&').toLowerCase();
41 +};
42 +
43 +/**
44 + * Iterate over each DOM element without creating intermediary Cheerio instances.
45 + *
46 + * This is indented for use internally to avoid otherwise unnecessary memory pressure introduced
47 + * by _make.
48 + */
49 +
50 +exports.domEach = function(cheerio, fn) {
51 + var i = 0, len = cheerio.length;
52 + while (i < len && fn.call(cheerio, i, cheerio[i]) !== false) ++i;
53 + return cheerio;
54 +};
55 +
56 +/**
57 + * Create a deep copy of the given DOM structure by first rendering it to a
58 + * string and then parsing the resultant markup.
59 + *
60 + * @argument {Object} dom - The htmlparser2-compliant DOM structure
61 + * @argument {Object} options - The parsing/rendering options
62 + */
63 +exports.cloneDom = function(dom, options) {
64 + return parse(render(dom, options), options).children;
65 +};
66 +
67 +/*
68 + * A simple way to check for HTML strings or ID strings
69 + */
70 +
71 +var quickExpr = /^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/;
72 +
73 +/*
74 + * Check if string is HTML
75 + */
76 +exports.isHtml = function(str) {
77 + // Faster than running regex, if str starts with `<` and ends with `>`, assume it's HTML
78 + if (str.charAt(0) === '<' && str.charAt(str.length - 1) === '>' && str.length >= 3) return true;
79 +
80 + // Run the regex
81 + var match = quickExpr.exec(str);
82 + return !!(match && match[1]);
83 +};
1 +{
2 + "_from": "cheerio@^0.22.0",
3 + "_id": "cheerio@0.22.0",
4 + "_inBundle": false,
5 + "_integrity": "sha1-qbqoYKP5tZWmuBsahocxIe06Jp4=",
6 + "_location": "/cheerio-httpcli/cheerio",
7 + "_phantomChildren": {},
8 + "_requested": {
9 + "type": "range",
10 + "registry": true,
11 + "raw": "cheerio@^0.22.0",
12 + "name": "cheerio",
13 + "escapedName": "cheerio",
14 + "rawSpec": "^0.22.0",
15 + "saveSpec": null,
16 + "fetchSpec": "^0.22.0"
17 + },
18 + "_requiredBy": [
19 + "/cheerio-httpcli"
20 + ],
21 + "_resolved": "http://registry.npmjs.org/cheerio/-/cheerio-0.22.0.tgz",
22 + "_shasum": "a9baa860a3f9b595a6b81b1a86873121ed3a269e",
23 + "_spec": "cheerio@^0.22.0",
24 + "_where": "D:\\OSS project\\YTMT\\node_modules\\cheerio-httpcli",
25 + "author": {
26 + "name": "Matt Mueller",
27 + "email": "mattmuelle@gmail.com",
28 + "url": "mat.io"
29 + },
30 + "bugs": {
31 + "url": "https://github.com/cheeriojs/cheerio/issues"
32 + },
33 + "bundleDependencies": false,
34 + "dependencies": {
35 + "css-select": "~1.2.0",
36 + "dom-serializer": "~0.1.0",
37 + "entities": "~1.1.1",
38 + "htmlparser2": "^3.9.1",
39 + "lodash.assignin": "^4.0.9",
40 + "lodash.bind": "^4.1.4",
41 + "lodash.defaults": "^4.0.1",
42 + "lodash.filter": "^4.4.0",
43 + "lodash.flatten": "^4.2.0",
44 + "lodash.foreach": "^4.3.0",
45 + "lodash.map": "^4.4.0",
46 + "lodash.merge": "^4.4.0",
47 + "lodash.pick": "^4.2.1",
48 + "lodash.reduce": "^4.4.0",
49 + "lodash.reject": "^4.4.0",
50 + "lodash.some": "^4.4.0"
51 + },
52 + "deprecated": false,
53 + "description": "Tiny, fast, and elegant implementation of core jQuery designed specifically for the server",
54 + "devDependencies": {
55 + "benchmark": "^2.1.0",
56 + "coveralls": "^2.11.9",
57 + "expect.js": "~0.3.1",
58 + "istanbul": "^0.4.3",
59 + "jquery": "^3.0.0",
60 + "jsdom": "^9.2.1",
61 + "jshint": "^2.9.2",
62 + "mocha": "^2.5.3",
63 + "xyz": "~0.5.0"
64 + },
65 + "engines": {
66 + "node": ">= 0.6"
67 + },
68 + "files": [
69 + "index.js",
70 + "lib"
71 + ],
72 + "homepage": "https://github.com/cheeriojs/cheerio#readme",
73 + "keywords": [
74 + "htmlparser",
75 + "jquery",
76 + "selector",
77 + "scraper",
78 + "parser",
79 + "html"
80 + ],
81 + "license": "MIT",
82 + "main": "./index.js",
83 + "name": "cheerio",
84 + "repository": {
85 + "type": "git",
86 + "url": "git://github.com/cheeriojs/cheerio.git"
87 + },
88 + "scripts": {
89 + "test": "make test"
90 + },
91 + "version": "0.22.0"
92 +}
1 +Copyright (c) 2015, Salesforce.com, Inc.
2 +All rights reserved.
3 +
4 +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
5 +
6 +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
7 +
8 +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
9 +
10 +3. Neither the name of Salesforce.com nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
11 +
12 +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
This diff is collapsed. Click to expand it.
1 +/*!
2 + * Copyright (c) 2015, Salesforce.com, Inc.
3 + * All rights reserved.
4 + *
5 + * Redistribution and use in source and binary forms, with or without
6 + * modification, are permitted provided that the following conditions are met:
7 + *
8 + * 1. Redistributions of source code must retain the above copyright notice,
9 + * this list of conditions and the following disclaimer.
10 + *
11 + * 2. Redistributions in binary form must reproduce the above copyright notice,
12 + * this list of conditions and the following disclaimer in the documentation
13 + * and/or other materials provided with the distribution.
14 + *
15 + * 3. Neither the name of Salesforce.com nor the names of its contributors may
16 + * be used to endorse or promote products derived from this software without
17 + * specific prior written permission.
18 + *
19 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 + * POSSIBILITY OF SUCH DAMAGE.
30 + */
31 +'use strict';
32 +var Store = require('./store').Store;
33 +var permuteDomain = require('./permuteDomain').permuteDomain;
34 +var pathMatch = require('./pathMatch').pathMatch;
35 +var util = require('util');
36 +
37 +function MemoryCookieStore() {
38 + Store.call(this);
39 + this.idx = {};
40 +}
41 +util.inherits(MemoryCookieStore, Store);
42 +exports.MemoryCookieStore = MemoryCookieStore;
43 +MemoryCookieStore.prototype.idx = null;
44 +
45 +// Since it's just a struct in RAM, this Store is synchronous
46 +MemoryCookieStore.prototype.synchronous = true;
47 +
48 +// force a default depth:
49 +MemoryCookieStore.prototype.inspect = function() {
50 + return "{ idx: "+util.inspect(this.idx, false, 2)+' }';
51 +};
52 +
53 +// Use the new custom inspection symbol to add the custom inspect function if
54 +// available.
55 +if (util.inspect.custom) {
56 + MemoryCookieStore.prototype[util.inspect.custom] = MemoryCookieStore.prototype.inspect;
57 +}
58 +
59 +MemoryCookieStore.prototype.findCookie = function(domain, path, key, cb) {
60 + if (!this.idx[domain]) {
61 + return cb(null,undefined);
62 + }
63 + if (!this.idx[domain][path]) {
64 + return cb(null,undefined);
65 + }
66 + return cb(null,this.idx[domain][path][key]||null);
67 +};
68 +
69 +MemoryCookieStore.prototype.findCookies = function(domain, path, cb) {
70 + var results = [];
71 + if (!domain) {
72 + return cb(null,[]);
73 + }
74 +
75 + var pathMatcher;
76 + if (!path) {
77 + // null means "all paths"
78 + pathMatcher = function matchAll(domainIndex) {
79 + for (var curPath in domainIndex) {
80 + var pathIndex = domainIndex[curPath];
81 + for (var key in pathIndex) {
82 + results.push(pathIndex[key]);
83 + }
84 + }
85 + };
86 +
87 + } else {
88 + pathMatcher = function matchRFC(domainIndex) {
89 + //NOTE: we should use path-match algorithm from S5.1.4 here
90 + //(see : https://github.com/ChromiumWebApps/chromium/blob/b3d3b4da8bb94c1b2e061600df106d590fda3620/net/cookies/canonical_cookie.cc#L299)
91 + Object.keys(domainIndex).forEach(function (cookiePath) {
92 + if (pathMatch(path, cookiePath)) {
93 + var pathIndex = domainIndex[cookiePath];
94 +
95 + for (var key in pathIndex) {
96 + results.push(pathIndex[key]);
97 + }
98 + }
99 + });
100 + };
101 + }
102 +
103 + var domains = permuteDomain(domain) || [domain];
104 + var idx = this.idx;
105 + domains.forEach(function(curDomain) {
106 + var domainIndex = idx[curDomain];
107 + if (!domainIndex) {
108 + return;
109 + }
110 + pathMatcher(domainIndex);
111 + });
112 +
113 + cb(null,results);
114 +};
115 +
116 +MemoryCookieStore.prototype.putCookie = function(cookie, cb) {
117 + if (!this.idx[cookie.domain]) {
118 + this.idx[cookie.domain] = {};
119 + }
120 + if (!this.idx[cookie.domain][cookie.path]) {
121 + this.idx[cookie.domain][cookie.path] = {};
122 + }
123 + this.idx[cookie.domain][cookie.path][cookie.key] = cookie;
124 + cb(null);
125 +};
126 +
127 +MemoryCookieStore.prototype.updateCookie = function(oldCookie, newCookie, cb) {
128 + // updateCookie() may avoid updating cookies that are identical. For example,
129 + // lastAccessed may not be important to some stores and an equality
130 + // comparison could exclude that field.
131 + this.putCookie(newCookie,cb);
132 +};
133 +
134 +MemoryCookieStore.prototype.removeCookie = function(domain, path, key, cb) {
135 + if (this.idx[domain] && this.idx[domain][path] && this.idx[domain][path][key]) {
136 + delete this.idx[domain][path][key];
137 + }
138 + cb(null);
139 +};
140 +
141 +MemoryCookieStore.prototype.removeCookies = function(domain, path, cb) {
142 + if (this.idx[domain]) {
143 + if (path) {
144 + delete this.idx[domain][path];
145 + } else {
146 + delete this.idx[domain];
147 + }
148 + }
149 + return cb(null);
150 +};
151 +
152 +MemoryCookieStore.prototype.removeAllCookies = function(cb) {
153 + this.idx = {};
154 + return cb(null);
155 +}
156 +
157 +MemoryCookieStore.prototype.getAllCookies = function(cb) {
158 + var cookies = [];
159 + var idx = this.idx;
160 +
161 + var domains = Object.keys(idx);
162 + domains.forEach(function(domain) {
163 + var paths = Object.keys(idx[domain]);
164 + paths.forEach(function(path) {
165 + var keys = Object.keys(idx[domain][path]);
166 + keys.forEach(function(key) {
167 + if (key !== null) {
168 + cookies.push(idx[domain][path][key]);
169 + }
170 + });
171 + });
172 + });
173 +
174 + // Sort by creationIndex so deserializing retains the creation order.
175 + // When implementing your own store, this SHOULD retain the order too
176 + cookies.sort(function(a,b) {
177 + return (a.creationIndex||0) - (b.creationIndex||0);
178 + });
179 +
180 + cb(null, cookies);
181 +};
1 +/*!
2 + * Copyright (c) 2015, Salesforce.com, Inc.
3 + * All rights reserved.
4 + *
5 + * Redistribution and use in source and binary forms, with or without
6 + * modification, are permitted provided that the following conditions are met:
7 + *
8 + * 1. Redistributions of source code must retain the above copyright notice,
9 + * this list of conditions and the following disclaimer.
10 + *
11 + * 2. Redistributions in binary form must reproduce the above copyright notice,
12 + * this list of conditions and the following disclaimer in the documentation
13 + * and/or other materials provided with the distribution.
14 + *
15 + * 3. Neither the name of Salesforce.com nor the names of its contributors may
16 + * be used to endorse or promote products derived from this software without
17 + * specific prior written permission.
18 + *
19 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 + * POSSIBILITY OF SUCH DAMAGE.
30 + */
31 +"use strict";
32 +/*
33 + * "A request-path path-matches a given cookie-path if at least one of the
34 + * following conditions holds:"
35 + */
36 +function pathMatch (reqPath, cookiePath) {
37 + // "o The cookie-path and the request-path are identical."
38 + if (cookiePath === reqPath) {
39 + return true;
40 + }
41 +
42 + var idx = reqPath.indexOf(cookiePath);
43 + if (idx === 0) {
44 + // "o The cookie-path is a prefix of the request-path, and the last
45 + // character of the cookie-path is %x2F ("/")."
46 + if (cookiePath.substr(-1) === "/") {
47 + return true;
48 + }
49 +
50 + // " o The cookie-path is a prefix of the request-path, and the first
51 + // character of the request-path that is not included in the cookie- path
52 + // is a %x2F ("/") character."
53 + if (reqPath.substr(cookiePath.length, 1) === "/") {
54 + return true;
55 + }
56 + }
57 +
58 + return false;
59 +}
60 +
61 +exports.pathMatch = pathMatch;
1 +/*!
2 + * Copyright (c) 2015, Salesforce.com, Inc.
3 + * All rights reserved.
4 + *
5 + * Redistribution and use in source and binary forms, with or without
6 + * modification, are permitted provided that the following conditions are met:
7 + *
8 + * 1. Redistributions of source code must retain the above copyright notice,
9 + * this list of conditions and the following disclaimer.
10 + *
11 + * 2. Redistributions in binary form must reproduce the above copyright notice,
12 + * this list of conditions and the following disclaimer in the documentation
13 + * and/or other materials provided with the distribution.
14 + *
15 + * 3. Neither the name of Salesforce.com nor the names of its contributors may
16 + * be used to endorse or promote products derived from this software without
17 + * specific prior written permission.
18 + *
19 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 + * POSSIBILITY OF SUCH DAMAGE.
30 + */
31 +"use strict";
32 +var pubsuffix = require('./pubsuffix-psl');
33 +
34 +// Gives the permutation of all possible domainMatch()es of a given domain. The
35 +// array is in shortest-to-longest order. Handy for indexing.
36 +function permuteDomain (domain) {
37 + var pubSuf = pubsuffix.getPublicSuffix(domain);
38 + if (!pubSuf) {
39 + return null;
40 + }
41 + if (pubSuf == domain) {
42 + return [domain];
43 + }
44 +
45 + var prefix = domain.slice(0, -(pubSuf.length + 1)); // ".example.com"
46 + var parts = prefix.split('.').reverse();
47 + var cur = pubSuf;
48 + var permutations = [cur];
49 + while (parts.length) {
50 + cur = parts.shift() + '.' + cur;
51 + permutations.push(cur);
52 + }
53 + return permutations;
54 +}
55 +
56 +exports.permuteDomain = permuteDomain;
1 +/*!
2 + * Copyright (c) 2018, Salesforce.com, Inc.
3 + * All rights reserved.
4 + *
5 + * Redistribution and use in source and binary forms, with or without
6 + * modification, are permitted provided that the following conditions are met:
7 + *
8 + * 1. Redistributions of source code must retain the above copyright notice,
9 + * this list of conditions and the following disclaimer.
10 + *
11 + * 2. Redistributions in binary form must reproduce the above copyright notice,
12 + * this list of conditions and the following disclaimer in the documentation
13 + * and/or other materials provided with the distribution.
14 + *
15 + * 3. Neither the name of Salesforce.com nor the names of its contributors may
16 + * be used to endorse or promote products derived from this software without
17 + * specific prior written permission.
18 + *
19 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 + * POSSIBILITY OF SUCH DAMAGE.
30 + */
31 +'use strict';
32 +var psl = require('psl');
33 +
34 +function getPublicSuffix(domain) {
35 + return psl.get(domain);
36 +}
37 +
38 +exports.getPublicSuffix = getPublicSuffix;
1 +/*!
2 + * Copyright (c) 2015, Salesforce.com, Inc.
3 + * All rights reserved.
4 + *
5 + * Redistribution and use in source and binary forms, with or without
6 + * modification, are permitted provided that the following conditions are met:
7 + *
8 + * 1. Redistributions of source code must retain the above copyright notice,
9 + * this list of conditions and the following disclaimer.
10 + *
11 + * 2. Redistributions in binary form must reproduce the above copyright notice,
12 + * this list of conditions and the following disclaimer in the documentation
13 + * and/or other materials provided with the distribution.
14 + *
15 + * 3. Neither the name of Salesforce.com nor the names of its contributors may
16 + * be used to endorse or promote products derived from this software without
17 + * specific prior written permission.
18 + *
19 + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
23 + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 + * POSSIBILITY OF SUCH DAMAGE.
30 + */
31 +'use strict';
32 +/*jshint unused:false */
33 +
34 +function Store() {
35 +}
36 +exports.Store = Store;
37 +
38 +// Stores may be synchronous, but are still required to use a
39 +// Continuation-Passing Style API. The CookieJar itself will expose a "*Sync"
40 +// API that converts from synchronous-callbacks to imperative style.
41 +Store.prototype.synchronous = false;
42 +
43 +Store.prototype.findCookie = function(domain, path, key, cb) {
44 + throw new Error('findCookie is not implemented');
45 +};
46 +
47 +Store.prototype.findCookies = function(domain, path, cb) {
48 + throw new Error('findCookies is not implemented');
49 +};
50 +
51 +Store.prototype.putCookie = function(cookie, cb) {
52 + throw new Error('putCookie is not implemented');
53 +};
54 +
55 +Store.prototype.updateCookie = function(oldCookie, newCookie, cb) {
56 + // recommended default implementation:
57 + // return this.putCookie(newCookie, cb);
58 + throw new Error('updateCookie is not implemented');
59 +};
60 +
61 +Store.prototype.removeCookie = function(domain, path, key, cb) {
62 + throw new Error('removeCookie is not implemented');
63 +};
64 +
65 +Store.prototype.removeCookies = function(domain, path, cb) {
66 + throw new Error('removeCookies is not implemented');
67 +};
68 +
69 +Store.prototype.removeAllCookies = function(cb) {
70 + throw new Error('removeAllCookies is not implemented');
71 +}
72 +
73 +Store.prototype.getAllCookies = function(cb) {
74 + throw new Error('getAllCookies is not implemented (therefore jar cannot be serialized)');
75 +};
1 +// generated by genversion
2 +module.exports = '2.5.0'
1 +{
2 + "_from": "tough-cookie@^2.3.4",
3 + "_id": "tough-cookie@2.5.0",
4 + "_inBundle": false,
5 + "_integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==",
6 + "_location": "/cheerio-httpcli/tough-cookie",
7 + "_phantomChildren": {},
8 + "_requested": {
9 + "type": "range",
10 + "registry": true,
11 + "raw": "tough-cookie@^2.3.4",
12 + "name": "tough-cookie",
13 + "escapedName": "tough-cookie",
14 + "rawSpec": "^2.3.4",
15 + "saveSpec": null,
16 + "fetchSpec": "^2.3.4"
17 + },
18 + "_requiredBy": [
19 + "/cheerio-httpcli"
20 + ],
21 + "_resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz",
22 + "_shasum": "cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2",
23 + "_spec": "tough-cookie@^2.3.4",
24 + "_where": "D:\\OSS project\\YTMT\\node_modules\\cheerio-httpcli",
25 + "author": {
26 + "name": "Jeremy Stashewsky",
27 + "email": "jstash@gmail.com"
28 + },
29 + "bugs": {
30 + "url": "https://github.com/salesforce/tough-cookie/issues"
31 + },
32 + "bundleDependencies": false,
33 + "contributors": [
34 + {
35 + "name": "Alexander Savin"
36 + },
37 + {
38 + "name": "Ian Livingstone"
39 + },
40 + {
41 + "name": "Ivan Nikulin"
42 + },
43 + {
44 + "name": "Lalit Kapoor"
45 + },
46 + {
47 + "name": "Sam Thompson"
48 + },
49 + {
50 + "name": "Sebastian Mayr"
51 + }
52 + ],
53 + "dependencies": {
54 + "psl": "^1.1.28",
55 + "punycode": "^2.1.1"
56 + },
57 + "deprecated": false,
58 + "description": "RFC6265 Cookies and Cookie Jar for node.js",
59 + "devDependencies": {
60 + "async": "^1.4.2",
61 + "genversion": "^2.1.0",
62 + "nyc": "^11.6.0",
63 + "string.prototype.repeat": "^0.2.0",
64 + "vows": "^0.8.2"
65 + },
66 + "engines": {
67 + "node": ">=0.8"
68 + },
69 + "files": [
70 + "lib"
71 + ],
72 + "homepage": "https://github.com/salesforce/tough-cookie",
73 + "keywords": [
74 + "HTTP",
75 + "cookie",
76 + "cookies",
77 + "set-cookie",
78 + "cookiejar",
79 + "jar",
80 + "RFC6265",
81 + "RFC2965"
82 + ],
83 + "license": "BSD-3-Clause",
84 + "main": "./lib/cookie",
85 + "name": "tough-cookie",
86 + "repository": {
87 + "type": "git",
88 + "url": "git://github.com/salesforce/tough-cookie.git"
89 + },
90 + "scripts": {
91 + "cover": "nyc --reporter=lcov --reporter=html vows test/*_test.js",
92 + "test": "vows test/*_test.js",
93 + "version": "genversion lib/version.js && git add lib/version.js"
94 + },
95 + "version": "2.5.0"
96 +}
1 +{
2 + "_from": "cheerio-httpcli",
3 + "_id": "cheerio-httpcli@0.7.3",
4 + "_inBundle": false,
5 + "_integrity": "sha512-6grjmuzSwjv/8gsDwZcNbrWL5Ce9trbrO4rkqSJC7ci3DmDkePFAmXfMZgokO8xiy9ZvRMT2jxvwTlt6e5cHTA==",
6 + "_location": "/cheerio-httpcli",
7 + "_phantomChildren": {
8 + "css-select": "1.2.0",
9 + "dom-serializer": "0.1.0",
10 + "entities": "1.1.2",
11 + "htmlparser2": "3.10.0",
12 + "lodash.assignin": "4.2.0",
13 + "lodash.bind": "4.2.1",
14 + "lodash.defaults": "4.2.0",
15 + "lodash.filter": "4.6.0",
16 + "lodash.flatten": "4.4.0",
17 + "lodash.foreach": "4.5.0",
18 + "lodash.map": "4.6.0",
19 + "lodash.merge": "4.6.1",
20 + "lodash.pick": "4.4.0",
21 + "lodash.reduce": "4.6.0",
22 + "lodash.reject": "4.6.0",
23 + "lodash.some": "4.6.0",
24 + "psl": "1.1.29",
25 + "punycode": "2.1.1"
26 + },
27 + "_requested": {
28 + "type": "tag",
29 + "registry": true,
30 + "raw": "cheerio-httpcli",
31 + "name": "cheerio-httpcli",
32 + "escapedName": "cheerio-httpcli",
33 + "rawSpec": "",
34 + "saveSpec": null,
35 + "fetchSpec": "latest"
36 + },
37 + "_requiredBy": [
38 + "#USER",
39 + "/"
40 + ],
41 + "_resolved": "https://registry.npmjs.org/cheerio-httpcli/-/cheerio-httpcli-0.7.3.tgz",
42 + "_shasum": "a8a00032abeadda909849e5d28bdb500242f11a7",
43 + "_spec": "cheerio-httpcli",
44 + "_where": "D:\\OSS project\\YTMT\\routes",
45 + "author": {
46 + "name": "ktty1220",
47 + "email": "ktty1220@gmail.com"
48 + },
49 + "bugs": {
50 + "url": "https://github.com/ktty1220/cheerio-httpcli/issues"
51 + },
52 + "bundleDependencies": false,
53 + "dependencies": {
54 + "@types/cheerio": "^0.22.7",
55 + "async": "^2.6.0",
56 + "cheerio": "^0.22.0",
57 + "colors": "^1.2.1",
58 + "foreach": "^2.0.5",
59 + "he": "^1.1.1",
60 + "iconv-lite": "^0.4.21",
61 + "jschardet": "^1.6.0",
62 + "object-assign": "^4.1.1",
63 + "os-locale": "^2.1.0",
64 + "prettyjson": "^1.2.1",
65 + "request": "^2.85.0",
66 + "require-uncached": "^1.0.3",
67 + "rsvp": "^4.8.2",
68 + "spawn-sync": "^1.0.15",
69 + "tough-cookie": "^2.3.4",
70 + "type-of": "^2.0.1",
71 + "valid-url": "^1.0.9"
72 + },
73 + "deprecated": false,
74 + "description": "http client module with cheerio & iconv(-lite) & promise",
75 + "devDependencies": {
76 + "@types/node": "^8.0.47",
77 + "constants": "^0.0.2",
78 + "dev-null": "^0.1.1",
79 + "eslint": "^4.19.1",
80 + "espower-loader": "^1.2.2",
81 + "intelli-espower-loader": "^1.0.1",
82 + "isstream": "^0.1.2",
83 + "istanbul": "^0.4.5",
84 + "mocha": "^5.1.1",
85 + "mocha-clean": "^1.0.0",
86 + "node-static": "^0.7.10",
87 + "pkg-ok": "^2.2.0",
88 + "power-assert": "^1.5.0",
89 + "pre-commit": "^1.2.2",
90 + "random-string": "^0.2.0",
91 + "strip-ansi": "^4.0.0",
92 + "yargs": "^11.0.0"
93 + },
94 + "engines": {
95 + "node": ">=4"
96 + },
97 + "homepage": "https://github.com/ktty1220/cheerio-httpcli#readme",
98 + "keywords": [
99 + "cheerio",
100 + "http",
101 + "dom",
102 + "scrape"
103 + ],
104 + "license": "MIT",
105 + "main": "index.js",
106 + "name": "cheerio-httpcli",
107 + "pre-commit": [
108 + "lint"
109 + ],
110 + "repository": {
111 + "type": "git",
112 + "url": "git://github.com/ktty1220/cheerio-httpcli.git"
113 + },
114 + "scripts": {
115 + "cov": "istanbul cover -x index.js -x testrunner.js testrunner.js -- -R nyan || true",
116 + "cs": "jscs lib test example || true",
117 + "dep": "depcheck --parsers=*.js:es6,*.json:json --ignores=iconv-lite,os-locale,constants,pre-commit,@types/*,espower-loader || true",
118 + "dts": "tsc -p dts",
119 + "lint": "eslint lib test example --ext .js",
120 + "prepublish": "pkg-ok",
121 + "test": "node testrunner.js || true",
122 + "todo": "leasot -S lib/**/**/* test/** example/** || true"
123 + },
124 + "typings": "./index.d.ts",
125 + "version": "0.7.3"
126 +}
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
No preview for this file type
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff could not be displayed because it is too large.
This diff could not be displayed because it is too large.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff could not be displayed because it is too large.
This diff is collapsed. Click to expand it.
This diff could not be displayed because it is too large.
This diff is collapsed. Click to expand it.
This diff could not be displayed because it is too large.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.
This diff is collapsed. Click to expand it.