crawler - npm Package Compare versions

Comparing version 2.0.0-beta.1 to 2.0.0-beta.3

dist/crawler.d.ts

 /// <reference types="node" />
 import { EventEmitter } from "events";
-import type { crawlerOptions, requestOptions } from "./types/crawler.js";
+import type { CrawlerOptions, RequestConfig, CrawlerResponse } from "./types/crawler.js";
 declare class Crawler extends EventEmitter {
@@ -8,7 +8,7 @@ private _limiters;
     private _proxyIndex;
-    options: crawlerOptions;
+    options: CrawlerOptions;
     globalOnlyOptions: string[];
     seen: any;
-    constructor(options?: crawlerOptions);
-    private _checkHtml;
+    constructor(options?: CrawlerOptions);
+    private _detectHtmlOnHeaders;
     private _schedule;
@@ -31,3 +31,3 @@ private _execute;
      */
-    setLimiter(rateLimiterId: number, property: string, value: any): void;
+    setLimiter(rateLimiterId: number, property: string, value: unknown): void;
     /**
@@ -50,3 +50,3 @@ *
      */
-    send: (options: string | requestOptions) => Promise<any>;
+    send: (options: RequestConfig) => Promise<CrawlerResponse>;
     /**
@@ -57,3 +57,3 @@ * @deprecated
      */
-    direct: (options: string | requestOptions) => Promise<any>;
+    direct: (options: RequestConfig) => Promise<CrawlerResponse>;
     /**
@@ -72,3 +72,3 @@ *
      */
-    add: (options: string | requestOptions | requestOptions[]) => void;
+    add: (options: RequestConfig) => void;
     /**
@@ -79,5 +79,5 @@ * @deprecated
      */
-    queue: (options: string | requestOptions | requestOptions[]) => void;
+    queue: (options: RequestConfig) => void;
 }
 export default Crawler;
 //# sourceMappingURL=crawler.d.ts.map

dist/crawler.js

@@ -20,3 +20,3 @@ import { EventEmitter } from "events";
         this._proxyIndex = 0;
-        this._checkHtml = (headers) => {
+        this._detectHtmlOnHeaders = (headers) => {
             const contentType = headers["content-type"];
@@ -158,3 +158,3 @@ if (/xml|html/i.test(contentType))
             if (options.jQuery === true) {
-                if (response.body === "" || !this._checkHtml(response.headers)) {
+                if (response.body === "" || !this._detectHtmlOnHeaders(response.headers)) {
                     log.warn("response body is not HTML, skip injecting. Set jQuery to false to mute this warning.");
@@ -258,3 +258,3 @@ }
                 })
-                    .catch((err) => log.error(err));
+                    .catch((error) => log.error(error));
             });
@@ -307,4 +307,4 @@ };
         })
-            .catch((err) => {
-            log.error(err);
+            .catch((error) => {
+            log.error(error);
         });
@@ -311,0 +311,0 @@ this.on("_release", () => {

dist/logger.d.ts

 export declare const logOptions: {
-    type: any;
+    type: "hidden" | "json" | "pretty";
     name: string;
@@ -4,0 +4,0 @@ hideLogPositionForProduction: boolean;

dist/options.d.ts

@@ -1,4 +0,5 @@
-export declare const getCharset: (headers: Record<string, string>) => null | string;
-export declare const getValidOptions: (options: unknown) => Object;
-export declare const alignOptions: (options: any) => any;
+import { RequestConfig, RequestOptions } from "./types/crawler.js";
+export declare const getCharset: (headers: Record<string, unknown>) => null | string;
+export declare const getValidOptions: (options: RequestConfig) => RequestOptions;
+export declare const alignOptions: (options: RequestOptions) => any;
 //# sourceMappingURL=options.d.ts.map

dist/options.js

@@ -62,2 +62,3 @@ import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
         stringifyJson: options.stringifyJson ?? options.jsonReplacer,
+        cookieJar: options.cookieJar ?? options.jar,
         timeout: { request: options.timeout },
@@ -74,6 +75,6 @@ };
     }
-    const defaultagent = {
+    const defaultagent = options["proxy"] ? {
         https: new HttpsProxyAgent({ proxy: options["proxy"] }),
         http: new HttpProxyAgent({ proxy: options["proxy"] }),
-    };
+    } : undefined;
     // http2 proxy
@@ -80,0 +81,0 @@ if (options.http2 === true && options.proxy) {

dist/types/crawler.d.ts

@@ -1,8 +0,12 @@
-declare global {
-    var mainModule: string;
-}
-type globalOnlyOptions = {
+export type GlobalOnlyOptions = {
+    /**
+     * Global Only option.
+     * @default 10
+     * @description The maximum number of requests that can be sent simultaneously.
+     * @example If the value is 10, the crawler will send at most 10 requests at the same time.
+     * Note: The maxConnections(> 1) will be valid only if the global ratelimit is set to be 0.
+     */
     maxConnections: number;
     /**
-     * Global option.
+     * Global Only option.
      * @default 10
@@ -13,3 +17,3 @@ * @description The number of levels of priority. Can be only assigned at the beginning.
     /**
-     * Global option.
+     * Global Only option.
      * @default 1000
@@ -21,3 +25,3 @@ * @description The default priority of the tasks. Can be only assigned at the beginning.
     /**
-     * Global option.
+     * Global Only option.
      * @default false
@@ -29,3 +33,3 @@ * @description If true, the crawler will skip duplicate tasks.
     /**
-     * Global option.
+     * Global Only option.
      * @default false
@@ -36,3 +40,3 @@ * @description If true, the crawler will dynamically reallocate the tasks within the queue blocked due to header blocking to other queues.
     /**
-     * Global option.
+     * Global Only option.
      * @default undefined
@@ -43,3 +47,3 @@ * @description If passed, the crawler will rotate the user agent for each request. The "userAgents" option must be an array if activated.
 };
-type requestOptions = {
+export type RequestOptions = {
     forceUTF8?: boolean;
@@ -91,6 +95,3 @@ /**
      */
-    qs?: Record<string, unknown>;
-    /**
-     * @description The query string of the URL.
-     */
+    qs?: string | Record<string, unknown>;
     searchParams?: Record<string, unknown>;
@@ -134,8 +135,9 @@ /**
     stringifyJson?: Function;
-    preRequest?: (options: requestOptions, done?: (error?: Error | null) => void) => void;
+    preRequest?: (options: RequestOptions, done?: (error?: Error | null) => void) => void;
     release?: () => void;
     callback?: (error: any, response: unknown, done: unknown) => void;
 };
-type crawlerOptions = Partial<globalOnlyOptions> & requestOptions;
-export { crawlerOptions, requestOptions };
+export type RequestConfig = string | RequestOptions | RequestOptions[];
+export type CrawlerOptions = Partial<GlobalOnlyOptions> & RequestOptions;
+export type CrawlerResponse = any;
 //# sourceMappingURL=crawler.d.ts.map

package.json

 {
     "name": "crawler",
-    "version": "2.0.0-beta.1",
+    "version": "2.0.0-beta.3",
     "description": "New TypeScript Crawler Test version",
@@ -33,4 +33,5 @@ "exports": "./dist/index.js",
         "sinon": "^17.0.1",
+        "tough-cookie": "^4.1.4",
         "tsx": "^4.7.3"
     }
 }

README.md

@@ -1,2 +0,554 @@
-# crawler
-New Test Version of Node-Crawler
+
+<p align="center">
+  <a href="https://github.com/bda-research/node-crawler">
+    <img alt="Node.js" src="https://raw.githubusercontent.com/bda-research/node-crawler/master/crawler_primary.png" width="400"/>
+  </a>
+</p>
+
+
+#
+[![npm package](https://nodei.co/npm/crawler.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/crawler/)
+
+[![CircleCI](https://circleci.com/gh/bda-research/node-crawler/tree/master.svg?style=svg)](https://circleci.com/gh/bda-research/node-crawler/tree/master)
+[![Coverage Status](https://coveralls.io/repos/github/bda-research/node-crawler/badge.svg?branch=master)](https://coveralls.io/github/bda-research/node-crawler?branch=master)
+[![NPM download][download-image]][download-url]
+[![Package Quality][quality-image]][quality-url]
+[![Gitter](https://img.shields.io/badge/gitter-join_chat-blue.svg?style=flat-square)](https://gitter.im/node-crawler/discuss?utm_source=badge)
+
+[quality-image]: https://packagequality.com/shield/crawler.svg
+[quality-url]: https://packagequality.com/#?package=crawler
+[download-image]: https://img.shields.io/npm/dm/crawler.svg?style=flat-square
+[download-url]: https://npmjs.org/package/crawler
+
+
+Most powerful, popular and production crawling/scraping package for Node, happy hacking :)
+
+Features:
+
+ * Server-side DOM & automatic jQuery insertion with Cheerio (default) or JSDOM,
+ * Configurable pool size and retries,
+ * Control rate limit,
+ * Priority queue of requests,
+ * `forceUTF8` mode to let crawler deal for you with charset detection and conversion,
+ * Compatible with 4.x or newer version.
+
+Here is the [CHANGELOG](https://github.com/bda-research/node-crawler/blob/master/CHANGELOG.md)
+
+Thanks to [Authuir](https://github.com/authuir), we have a [Chinese](http://node-crawler.readthedocs.io/zh_CN/latest/) docs. Other languages are welcomed!
+
+# Table of Contents
+- [Get started](#get-started)
+  * [Install](#install)
+  * [Basic usage](#basic-usage)
+  * [Slow down](#slow-down)
+  * [Custom parameters](#custom-parameters)
+  * [Raw body](#raw-body)
+  * [preRequest](#prerequest)
+- [Advanced](#advanced)
+  * [Send request directly](#send-request-directly)
+  * [Work with bottleneck](#work-with-bottleneck)
+  * [Class:Crawler](#classcrawler)
+    + [Event: 'schedule'](#event-schedule)
+    + [Event: 'limiterChange'](#event-limiterchange)
+    + [Event: 'request'](#event-request)
+    + [Event: 'drain'](#event-drain)
+    + [crawler.queue(uri|options)](#crawlerqueueurioptions)
+    + [crawler.queueSize](#crawlerqueuesize)
+  * [Options reference](#options-reference)
+    + [Basic request options](#basic-request-options)
+    + [Callbacks](#callbacks)
+    + [Schedule options](#schedule-options)
+    + [Retry options](#retry-options)
+    + [Server-side DOM options](#server-side-dom-options)
+    + [Charset encoding](#charset-encoding)
+    + [Cache](#cache)
+    + [Http headers](#http-headers)
+  * [Work with Cheerio or JSDOM](#work-with-cheerio-or-jsdom)
+    + [Working with Cheerio](#working-with-cheerio)
+    + [Work with JSDOM](#work-with-jsdom)
+- [How to test](#how-to-test)
+  * [Alternative: Docker](#alternative-docker)
+- [Rough todolist](#rough-todolist)
+
+# Get started
+
+## Install
+
+```sh
+$ npm install crawler
+```
+
+## Basic usage
+
+```js
+const Crawler = require('crawler');
+
+const c = new Crawler({
+    maxConnections: 10,
+    // This will be called for each crawled page
+    callback: (error, res, done) => {
+        if (error) {
+            console.log(error);
+        } else {
+            const $ = res.$;
+            // $ is Cheerio by default
+            //a lean implementation of core jQuery designed specifically for the server
+            console.log($('title').text());
+        }
+        done();
+    }
+});
+
+// Queue just one URL, with default callback
+c.queue('http://www.amazon.com');
+
+// Queue a list of URLs
+c.queue(['http://www.google.com/','http://www.yahoo.com']);
+
+// Queue URLs with custom callbacks & parameters
+c.queue([{
+    uri: 'http://parishackers.org/',
+    jQuery: false,
+
+    // The global callback won't be called
+    callback: (error, res, done) => {
+        if (error) {
+            console.log(error);
+        } else {
+            console.log('Grabbed', res.body.length, 'bytes');
+        }
+        done();
+    }
+}]);
+
+// Queue some HTML code directly without grabbing (mostly for tests)
+c.queue([{
+    html: '<p>This is a <strong>test</strong></p>'
+}]);
+```
+
+## Slow down
+Use `rateLimit` to slow down when you are visiting web sites.
+
+```js
+const Crawler = require('crawler');
+
+const c = new Crawler({
+    rateLimit: 1000, // `maxConnections` will be forced to 1
+    callback: (err, res, done) => {
+        console.log(res.$('title').text());
+        done();
+    }
+});
+
+c.queue(tasks);//between two tasks, minimum time gap is 1000 (ms)
+```
+
+## Custom parameters
+
+Sometimes you have to access variables from previous request/response session, what should you do is passing parameters as same as options:
+
+```js
+c.queue({
+    uri: 'http://www.google.com',
+    parameter1: 'value1',
+    parameter2: 'value2',
+    parameter3: 'value3'
+})
+```
+
+then access them in callback via `res.options`
+
+```js
+console.log(res.options.parameter1);
+```
+
+Crawler picks options only needed by request, so don't worry about the redundancy.
+
+## Raw body
+
+If you are downloading files like image, pdf, word etc, you have to save the raw response body which means Crawler shouldn't convert it to string. To make it happen, you need to set encoding to null
+
+```js
+const Crawler = require('crawler');
+const fs = require('fs');
+
+const c = new Crawler({
+    encoding: null,
+    jQuery: false,// set false to suppress warning message.
+    callback: (err, res, done) => {
+        if (err) {
+            console.error(err.stack);
+        } else {
+            fs.createWriteStream(res.options.filename).write(res.body);
+        }
+
+        done();
+    }
+});
+
+c.queue({
+    uri: 'https://nodejs.org/static/images/logos/nodejs-1920x1200.png',
+    filename: 'nodejs-1920x1200.png'
+});
+
+```
+
+## preRequest
+
+If you want to do something either synchronously or asynchronously before each request, you can try the code below. Note that direct requests won't trigger preRequest.
+
+```js
+const c = new Crawler({
+    preRequest: (options, done) => {
+        // 'options' here is not the 'options' you pass to 'c.queue', instead, it's the options that is going to be passed to 'request' module
+        console.log(options);
+    	// when done is called, the request will start
+    	done();
+    },
+    callback: (err, res, done) => {
+        if (err) {
+    	    console.log(err);
+    	} else {
+    	    console.log(res.statusCode);
+    	}
+    }
+});
+
+c.queue({
+    uri: 'http://www.google.com',
+    // this will override the 'preRequest' defined in crawler
+    preRequest: (options, done) => {
+        setTimeout(() => {
+    	    console.log(options);
+    	    done();
+    	}, 1000);
+    }
+});
+```
+# Advanced
+## Send request directly
+
+In case you want to send a request directly without going through the scheduler in Crawler, try the code below. `direct` takes the same options as `queue`, please refer to [options](#options-reference) for detail. The difference is when calling `direct`, `callback` must be defined explicitly, with two arguments `error` and `response`, which are the same as that of `callback` of method `queue`.
+
+```js
+crawler.direct({
+    uri: 'http://www.google.com',
+    skipEventRequest: false, // default to true, direct requests won't trigger Event:'request'
+    callback: (error, response) => {
+        if (error) {
+            console.log(error)
+        } else {
+            console.log(response.statusCode);
+        }
+    }
+});
+```
+
+## Work with Http2
+
+Node-crawler now supports http request. Proxy functionality for http2 request does not be included now. It will be added in the future.
+
+```js
+crawler.queue({
+    //unit test work with httpbin http2 server. It could be used for test
+    uri: 'https://nghttp2.org/httpbin/status/200',
+    method: 'GET',
+    http2: true, //set http2 to be true will make a http2 request
+    callback: (error, response, done) => {
+        if (error) {
+            console.error(error);
+            return done();
+        }
+
+        console.log(`inside callback`);
+        console.log(response.body);
+        return done();
+    }
+})
+```
+
+## Work with bottleneck
+
+Control rate limit for with limiter. All tasks submit to a limiter will abide the `rateLimit` and `maxConnections` restrictions of the limiter. `rateLimit` is the minimum time gap between two tasks. `maxConnections` is the maximum number of tasks that can be running at the same time. Limiters are independent of each other. One common use case is setting different limiters for different proxies. One thing is worth noticing, when `rateLimit` is set to a non-zero value, `maxConnections` will be forced to 1.
+
+```js
+const Crawler = require('crawler');
+
+const c = new Crawler({
+    rateLimit: 2000,
+    maxConnections: 1,
+    callback: (error, res, done) => {
+        if (error) {
+            console.log(error);
+        } else {
+            const $ = res.$;
+            console.log($('title').text());
+        }
+        done();
+    }
+});
+
+// if you want to crawl some website with 2000ms gap between requests
+c.queue('http://www.somewebsite.com/page/1');
+c.queue('http://www.somewebsite.com/page/2');
+c.queue('http://www.somewebsite.com/page/3');
+
+// if you want to crawl some website using proxy with 2000ms gap between requests for each proxy
+c.queue({
+    uri:'http://www.somewebsite.com/page/1',
+    limiter:'proxy_1',
+    proxy:'proxy_1'
+});
+c.queue({
+    uri:'http://www.somewebsite.com/page/2',
+    limiter:'proxy_2',
+    proxy:'proxy_2'
+});
+c.queue({
+    uri:'http://www.somewebsite.com/page/3',
+    limiter:'proxy_3',
+    proxy:'proxy_3'
+});
+c.queue({
+    uri:'http://www.somewebsite.com/page/4',
+    limiter:'proxy_1',
+    proxy:'proxy_1'
+});
+```
+
+Normally, all limiter instances in limiter cluster in crawler are instantiated with options specified in crawler constructor. You can change property of any limiter by calling the code below. Currently, we only support changing property 'rateLimit' of limiter. Note that the default limiter can be accessed by `c.setLimiterProperty('default', 'rateLimit', 3000)`. We strongly recommend that you leave limiters unchanged after their instantiation unless you know clearly what you are doing.
+```js
+const c = new Crawler({});
+c.setLimiterProperty('limiterName', 'propertyName', value);
+```
+
+
+## Class:Crawler
+
+### Event: 'schedule'
+ * `options` [Options](#options-reference)
+
+Emitted when a task is being added to scheduler.
+
+```js
+crawler.on('schedule', (options) => {
+    options.proxy = 'http://proxy:port';
+});
+```
+
+### Event: 'limiterChange'
+ * `options` [Options](#options-reference)
+ * `limiter` [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type)
+
+Emitted when limiter has been changed.
+
+### Event: 'request'
+ * `options` [Options](#options-reference)
+
+Emitted when crawler is ready to send a request.
+
+If you are going to modify options at last stage before requesting, just listen on it.
+
+```js
+crawler.on('request', (options) => {
+    options.qs.timestamp = new Date().getTime();
+});
+```
+
+### Event: 'drain'
+
+Emitted when queue is empty.
+
+```js
+crawler.on('drain', () => {
+    // For example, release a connection to database.
+    db.end();// close connection to MySQL
+});
+```
+
+### crawler.queue(uri|options)
+ * `uri` [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type)
+ * `options` [Options](#options-reference)
+
+Enqueue a task and wait for it to be executed.
+
+### crawler.queueSize
+ * [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type)
+
+Size of queue, read-only
+
+
+## Options reference
+
+
+You can pass these options to the Crawler() constructor if you want them to be global or as
+items in the queue() calls if you want them to be specific to that item (overwriting global options)
+
+This options list is a strict superset of [mikeal's request options](https://github.com/mikeal/request#requestoptions-callback) and will be directly passed to
+the request() method.
+
+### Basic request options
+
+ * `options.uri`: [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type) The url you want to crawl.
+ * `options.timeout`: [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type) In milliseconds (Default 15000).
+ * [All mikeal's request options are accepted](https://github.com/mikeal/request#requestoptions-callback).
+
+### Callbacks
+
+ * `callback(error, res, done)`: Function that will be called after a request was completed
+     * `error`: [Error](https://nodejs.org/api/errors.html)
+     * `res`: [http.IncomingMessage](https://nodejs.org/api/http.html#http_class_http_incomingmessage) A response of standard IncomingMessage includes `$` and `options`
+         * `res.statusCode`: [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type) HTTP status code. E.G.`200`
+         * `res.body`: [Buffer](https://nodejs.org/api/buffer.html) | [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type) HTTP response content which could be a html page, plain text or xml document e.g.
+         * `res.headers`: [Object](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object) HTTP response headers
+         * `res.request`: [Request](https://github.com/request/request)  An instance of Mikeal's `Request` instead of [http.ClientRequest](https://nodejs.org/api/http.html#http_class_http_clientrequest)
+             * `res.request.uri`: [urlObject](https://nodejs.org/api/url.html#url_url_strings_and_url_objects) HTTP request entity of parsed url
+             * `res.request.method`: [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type) HTTP request method. E.G. `GET`
+             * `res.request.headers`: [Object](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object) HTTP request headers
+         * `res.options`: [Options](#options-reference) of this task
+         * `$`: [jQuery Selector](https://api.jquery.com/category/selectors/) A selector for  html or xml document.
+     * `done`: [Function](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Function) It must be called when you've done your work in callback.
+
+### Schedule options
+
+ * `options.maxConnections`: [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type) Size of the worker pool (Default 10).
+ * `options.rateLimit`: [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type) Number of milliseconds to delay between each requests (Default 0).
+ * `options.priorityRange`: [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type) Range of acceptable priorities starting from 0 (Default 10).
+ * `options.priority`: [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type) Priority of this request (Default 5). Low values have higher priority.
+
+### Retry options
+
+ * `options.retries`: [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type) Number of retries if the request fails (Default 3),
+ * `options.retryTimeout`: [Number](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Number_type) Number of milliseconds to wait before retrying (Default 10000),
+
+### Server-side DOM options
+
+ * `options.jQuery`: [Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Boolean_type)|[String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type)|[Object](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object) Use `cheerio` with default configurations to inject document if true or 'cheerio'. Or use customized `cheerio` if an object with [Parser options](https://github.com/fb55/htmlparser2/wiki/Parser-options). Disable injecting jQuery selector if false. If you have memory leak issue in your project, use 'whacko', an alternative parser,to avoid that. (Default true)
+
+### Charset encoding
+
+ * `options.forceUTF8`: [Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Boolean_type) If true crawler will get charset from HTTP headers or meta tag in html and convert it to UTF8 if necessary. Never worry about encoding anymore! (Default true),
+ * `options.incomingEncoding`: [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type) With forceUTF8: true to set encoding manually (Default null) so that crawler will not have to detect charset by itself. For example, `incomingEncoding: 'windows-1255'`. See [all supported encodings](https://github.com/ashtuchkin/iconv-lite/wiki/Supported-Encodings)
+
+### Cache
+
+ * `options.skipDuplicates`: [Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Boolean_type) If true skips URIs that were already crawled, without even calling callback() (Default false). __This is not recommended__, it's better to handle outside `Crawler` use [seenreq](https://github.com/mike442144/seenreq)
+
+### Http headers
+
+ * `options.rotateUA`: [Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Boolean_type) If true, `userAgent` should be an array and rotate it (Default false)
+ * `options.userAgent`: [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type)|[Array](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array), If `rotateUA` is false, but `userAgent` is an array, crawler will use the first one.
+ * `options.referer`: [String](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#String_type) If truthy sets the HTTP referer header
+ * `options.removeRefererHeader`: [Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Boolean_type) If true preserves the set referer during redirects
+ * `options.headers`: [Object](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object) Raw key-value of http headers
+
+### Http2
+
+ * `options.http2`: [Boolean](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Data_structures#Boolean_type) If true, request will be sent in http2 protocol (Default false)
+
+### Https socks5
+```js
+const Agent = require('socks5-https-client/lib/Agent');
+//...
+const c = new Crawler({
+    // rateLimit: 2000,
+    maxConnections: 20,
+    agentClass: Agent, //adding socks5 https agent
+    method: 'GET',
+    strictSSL: true,
+    agentOptions: {
+        socksHost: 'localhost',
+        socksPort: 9050
+    },
+    // debug: true,
+    callback: (error, res, done) => {
+        if (error) {
+            console.log(error);
+        }
+        done();
+    }
+});
+```
+
+
+## Work with Cheerio or JSDOM
+
+
+Crawler by default use [Cheerio](https://github.com/cheeriojs/cheerio) instead of [JSDOM](https://github.com/tmpvar/jsdom). JSDOM is more robust, if you want to use JSDOM you will have to require it `require('jsdom')` in your own script before passing it to crawler.
+
+### Working with Cheerio
+```js
+jQuery: true //(default)
+//OR
+jQuery: 'cheerio'
+//OR
+jQuery: {
+    name: 'cheerio',
+    options: {
+        normalizeWhitespace: true,
+        xmlMode: true
+    }
+}
+```
+These parsing options are taken directly from [htmlparser2](https://github.com/fb55/htmlparser2/wiki/Parser-options), therefore any options that can be used in `htmlparser2` are valid in cheerio as well. The default options are:
+
+```js
+{
+    normalizeWhitespace: false,
+    xmlMode: false,
+    decodeEntities: true
+}
+```
+
+For a full list of options and their effects, see [this](https://github.com/fb55/DomHandler) and
+[htmlparser2's options](https://github.com/fb55/htmlparser2/wiki/Parser-options).
+[source](https://github.com/cheeriojs/cheerio#loading)
+
+### Work with JSDOM
+
+In order to work with JSDOM you will have to install it in your project folder `npm install jsdom`, and pass it to crawler.
+
+```js
+const jsdom = require('jsdom');
+const Crawler = require('crawler');
+
+const c = new Crawler({
+    jQuery: jsdom
+});
+```
+
+# How to test
+
+Crawler uses `nock` to mock http request, thus testing no longer relying on http server.
+
+```bash
+$ npm install
+$ npm test
+$ npm run cover # code coverage
+```
+
+## Alternative: Docker
+
+After [installing Docker](http://docs.docker.com/), you can run:
+
+```bash
+# Builds the local test environment
+$ docker build -t node-crawler .
+
+# Runs tests
+$ docker run node-crawler sh -c "npm install && npm test"
+
+# You can also ssh into the container for easier debugging
+$ docker run -i -t node-crawler bash
+```
+
+
+# Rough todolist
+
+ * Introducing zombie to deal with page with complex ajax
+ * Refactoring the code to be more maintainable
+ * Make Sizzle tests pass (JSDOM bug? https://github.com/tmpvar/jsdom/issues#issue/81)
+ * Promise support
+ * Commander support
+ * Middleware support

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

105742

increased by25.98%

Lines of code

1321

increased by0.3%

Number of lines in readme file

554

increased by18366.67%

Number of medium supply chain risk alerts

1

decreased by-50%

Worsened metrics

Dev dependency count

9

increased by12.5%

No dependency changes