crawler-ninja - npm Package Compare versions

Comparing version 0.2.0 to 0.2.1

default-lists/domain-black-list.js

		module.exports.list = function () {

		return [
		"mozilla.org",
		"mozilla.fr",
		"microsoft.com",
		"mozilla.net",
		"addthis",
		"mozilla",
		"mozilla",
		"microsoft",
		"mozilla",
		"google",
		@@ -9,0 +10,0 @@ "picasaweb",

index.js

		@@ -41,3 +41,2 @@ var timers = require('timers');
		var DEFAULT_CACHE = false;
		var DEFAULT_METHOD = 'GET';
		var DEFAULT_REFERER = false;
		@@ -87,3 +86,3 @@
		* - queueModuleName : the npm module name used for the job queue. By default : async-queue
		* - queueParams : the params to pass to the job queue when create it.
		* - queueParams : the params to pass to the job queue when create it.
		*
		@@ -286,3 +285,2 @@ * + all options provided by nodejs request : https://github.com/request/request
		var options = {
		method : DEFAULT_METHOD,
		referer : DEFAULT_REFERER,
		@@ -289,0 +287,0 @@ skipDuplicates : DEFAULT_SKIP_DUPLICATES,

lib/http/http-request.js

		@@ -72,3 +72,3 @@
		var start = new Date();
		request(_.pick.apply(this,[options].concat(requestArgs)), function(error,response) {
		request.get(_.pick.apply(this,[options].concat(requestArgs)), function(error,response) {

		@@ -75,0 +75,0 @@ var end = new Date() - start;

lib/queue/request-queue.js

		@@ -35,3 +35,3 @@ /**
		function init (options, crawlCallback, recrawlCallback, endCallback, proxies) {
		console.log("Job Queue Module : " + options.queueModuleName);

		requestQueue = require(options.queueModuleName);
		@@ -38,0 +38,0 @@ requestQueue.init(options, onUrlToCrawl, endCallback);

package.json

		{
		"name": "crawler-ninja",
		"version": "0.2.0",
		"version": "0.2.1",
		"description": "A web crawler made for the SEO based on plugins. Please wait or contribute ... still in beta",
		@@ -5,0 +5,0 @@ "main": "index.js",

New alerts