simplecrawler - npm Package Compare versions

Comparing version 0.2.8 to 0.2.9

lib/crawler.js

@@ -352,3 +352,3 @@ // Simplecrawler
 		// Worst case scenario is we make some junky requests.
-		/^javascript\:[a-z0-9]+\(['"][^'"\s]+/ig
+		/^javascript\:[a-z0-9\$\_\.]+\(['"][^'"\s]+/ig
 	];
@@ -364,2 +364,3 @@
 				.replace(/^\/\//, queueItem.protocol + "://")
+				.replace(/\&/i,"&")
 				.split(/\s+/g)
@@ -569,3 +570,3 @@ .shift()
 	fetchDenied = crawler._fetchConditions.reduce(function(prev,callback) {
-		return fetchDenied || !callback(parsedURL);
+		return prev || !callback(parsedURL);
 	},false);
@@ -572,0 +573,0 @@

package.json

 {
   "name": "simplecrawler",
   "description": "Very straigntforward web crawler. Uses EventEmitter. Generates queue statistics and has a basic cache mechanism with extensible backend.",
-  "version": "0.2.8",
+  "version": "0.2.9",
   "homepage": "http://github.com/cgiffard/node-simplecrawler",
@@ -6,0 +6,0 @@ "author": "Christopher Giffard <[e-mail]>",

test/testcrawl.js

@@ -16,2 +16,7 @@ // Runs a very simple crawl on an HTTP server
 		asyncCrawler = new Crawler("127.0.0.1","/",3000);
+	
+	// Speed up tests. No point waiting for every request when we're running
+	// our own server.
+	localCrawler.interval = asyncCrawler.interval = 1;
+	
 	var linksDiscovered = 0;
@@ -62,3 +67,3 @@
 				evtDone();
-			},250);
+			},100);
 		});
@@ -65,0 +70,0 @@

No alert changes

Improved metrics

Total package byte prevSize

104918

increased by0.17%

Lines of code

2212

increased by0.18%

Number of lines in readme file

565

increased by0.18%

No dependency changes