supercrawler
Advanced tools
Comparing version 0.3.1 to 0.3.2
{ | ||
"name": "supercrawler", | ||
"description": "A web crawler. Supercrawler automatically crawls websites. Define custom handlers to parse content. Obeys robots.txt, rate limits and concurrency limits.", | ||
"version": "0.3.1", | ||
"version": "0.3.2", | ||
"homepage": "https://github.com/brendonboshell/supercrawler", | ||
@@ -6,0 +6,0 @@ "author": "Brendon Boshell <brendonboshell@gmail.com>", |
@@ -263,3 +263,3 @@ var proxyquire = require('proxyquire'), | ||
var crawler = new Crawler({ | ||
interval: 50, // 50 ms | ||
interval: 20, // 20 ms | ||
concurrentRequestsLimit: 1, | ||
@@ -557,3 +557,3 @@ robotsCacheTime: 100 // 100ms | ||
done(); | ||
}, 15); | ||
}, 100); | ||
}); | ||
@@ -560,0 +560,0 @@ |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
82660