sitemap-stream-parser
Advanced tools
Comparing version 1.0.0 to 1.1.0
18
index.js
// Generated by CoffeeScript 1.10.0 | ||
(function() { | ||
var agentOptions, async, domain, headers, parseSitemaps, request, sax, sitemapParser, zlib, | ||
var agentOptions, async, domain, headers, request, sax, sitemapParser, zlib, | ||
bind = function(fn, me){ return function(){ return fn.apply(me, arguments); }; }; | ||
@@ -103,4 +103,7 @@ | ||
parseSitemaps = function(urls, url_cb, done) { | ||
exports.parseSitemaps = function(urls, url_cb, done) { | ||
var queue; | ||
if (!(urls instanceof Array)) { | ||
urls = [urls]; | ||
} | ||
sitemapParser = new sitemapParser(url_cb, function(sitemap) { | ||
@@ -116,9 +119,8 @@ return queue.push(sitemap); | ||
exports.parseSitemap = function(url, url_cb, done) { | ||
return parseSitemaps([url], url_cb, done); | ||
}; | ||
exports.parseRobot = function(url, url_cb, done) { | ||
exports.sitemapsInRobots = function(url, cb) { | ||
return request.get(url, function(err, res, body) { | ||
var matches; | ||
if (err) { | ||
return cb(err); | ||
} | ||
matches = []; | ||
@@ -128,3 +130,3 @@ body.replace(/^Sitemap:\s?([^\s]+)$/igm, function(m, p1) { | ||
}); | ||
return parseSitemaps(matches, url_cb, done); | ||
return cb(null, matches); | ||
}); | ||
@@ -131,0 +133,0 @@ }; |
{ | ||
"name": "sitemap-stream-parser", | ||
"version": "1.0.0", | ||
"version": "1.1.0", | ||
"description": "Get a list of URLs from one or more sitemaps", | ||
"main": "index.js", | ||
"scripts": { | ||
"test": "echo \"Error: no test specified\" && exit 1" | ||
"prepublish": "coffee -c index.coffee" | ||
}, | ||
"author": "Erwin van der Koogh <erwin@404check.io>", | ||
"license": "ISC", | ||
"license": "Apache-2.0", | ||
"dependencies": { | ||
@@ -16,13 +16,9 @@ "async": "^1.5.0", | ||
}, | ||
"license" : "Apache-2.0", | ||
"repository" : { | ||
"type" : "git", | ||
"url" : "https://github.com/evanderkoogh/node-sitemap-stream-parser.git" | ||
"repository": { | ||
"type": "git", | ||
"url": "https://github.com/evanderkoogh/node-sitemap-stream-parser.git" | ||
}, | ||
"scripts": { | ||
"prepublish": "coffee -c index.coffee" | ||
}, | ||
"devDependencies": { | ||
"coffee-script": "^1.10.0" | ||
} | ||
} | ||
} |
# node-sitemap-stream-parser | ||
A streaming parser for sitemap files. Is able to deal with deeply nested sitemaps with 100+ million urls in them. | ||
A streaming parser for sitemap files. It is able to deal with deeply nested sitemaps with 100+ million urls in them. | ||
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Mixed license
License(Experimental) Package contains multiple licenses.
Found 1 instance in 1 package
1
116
4
17727