supercrawler - npm Package Compare versions

Comparing version 0.10.0 to 0.10.1

lib/Crawler.js

@@ -313,3 +313,4 @@ var Crawler,
     encoding: null,
-    followRedirect: Boolean(followRedirect)
+    followRedirect: Boolean(followRedirect),
+    gzip: true
   }).catch(function (err) {
@@ -316,0 +317,0 @@ err = new error.RequestError("A request error occured. " + err.message);

package.json

 {
   "name": "supercrawler",
   "description": "A web crawler. Supercrawler automatically crawls websites. Define custom handlers to parse content. Obeys robots.txt, rate limits and concurrency limits.",
-  "version": "0.10.0",
+  "version": "0.10.1",
   "homepage": "https://github.com/brendonboshell/supercrawler",
@@ -6,0 +6,0 @@ "author": "Brendon Boshell <[e-mail]>",

README.md

@@ -312,2 +312,7 @@ # Node.js Web Crawler
 
+### 0.10.1
+
+* [Fixed] Request sends `Accept-Encoding: gzip, deflate` header, so the
+responses arrive compressed (saving data transfer).
+
 ### 0.10.0
@@ -314,0 +319,0 @@

test/Crawler.spec.js

@@ -499,2 +499,18 @@ var proxyquire = require('proxyquire'),
     });
+
+    it("asks for a gzipped response", function (done) {
+      var crawler = new Crawler({
+        interval: 10
+      });
+
+      crawler.start();
+
+      setTimeout(function () {
+        crawler.stop();
+        sinon.assert.calledWith(requestSpy, sinon.match({
+          gzip: true
+        }));
+        done();
+      }, 100);
+    });
   });
@@ -501,0 +517,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

99053

increased by0.49%

Lines of code

2343

increased by0.6%

Number of lines in readme file

403

increased by1.26%

No dependency changes