robots-parser
Advanced tools
Comparing version 1.0.1 to 1.0.2
{ | ||
"name": "robots-parser", | ||
"version": "1.0.1", | ||
"version": "1.0.2", | ||
"description": "Robots.txt parser.", | ||
@@ -5,0 +5,0 @@ "main": "index.js", |
@@ -269,3 +269,3 @@ var libUrl = require('url'); | ||
parsedUrl.port = parsedUrl.port || 80; | ||
parsedUrl.hostname = punycode.toUnicode(parsedUrl.hostname); | ||
parsedUrl.hostname = parsedUrl.hostname && punycode.toUnicode(parsedUrl.hostname); | ||
@@ -272,0 +272,0 @@ // The base URL must match otherwise this robots.txt is not valid for it. |
@@ -218,3 +218,5 @@ var robotsParser = require('../index'); | ||
'http://www.example.net/test/index.html', | ||
'http://www.examsple.com/test/' | ||
'http://www.examsple.com/test/', | ||
'example.com/test/', | ||
':::::;;`\\|/.example.com/test/' | ||
]; | ||
@@ -221,0 +223,0 @@ |
21830
640
5