robotstxt - npm Package Compare versions

Comparing version 0.0.2-1 to 0.0.3-0

index.js

@@ -198,16 +198,23 @@ (function() {
       req = handler.request(options, __bind(function(res) {
-        res.setEncoding(encoding);
-        res.on("data", __bind(function(chunk) {
-          return txtA.push(chunk);
-        }, this));
-        res.on("end", __bind(function() {
-          txt = txtA.join('');
-          this.emit("crawled", txt);
-          return this.parse(txt);
-        }, this));
-        return null;
+        if (res.statusCode !== 200) {
+          return this.emit("error", new Error('invalid status code - is: HTTP ' + res.statusCode + ' - should: HTTP 200'));
+        } else {
+          res.setEncoding(encoding);
+          res.on("data", __bind(function(chunk) {
+            return txtA.push(chunk);
+          }, this));
+          res.on("end", __bind(function() {
+            txt = txtA.join('');
+            this.emit("crawled", txt);
+            return this.parse(txt);
+          }, this));
+          return null;
+        }
       }, this));
       req.setHeader("User-Agent", user_agent);
       req.end();
-      return null;
+      null;
+      return req.on('error', __bind(function(e) {
+        return this.emit("error", e);
+      }, this));
     };
@@ -297,6 +304,6 @@ RobotsTxt.prototype.parse = function(txt) {
       }
-      if (myGateKeeper) {
+      if (myGateKeeper != null) {
         return this.emit("ready", myGateKeeper);
       } else {
-        return this.emit("error", myGateKeeper);
+        return this.emit("error", 'gatekeeper is ' + typeof myGateKeeper);
       }
@@ -303,0 +310,0 @@ };

package.json

@@ -19,3 +19,3 @@ {
   "main"          : "./index.js",
-  "version"       : "0.0.2-1"
+  "version"       : "0.0.3-0"
 }

README.md

@@ -29,2 +29,3 @@ robots.txt parser for node.js
       #returns false
+      #note: only the path and the query of the url gets parsed, everything else (i.e. http://, the domain-name) gets ditched
       console.log gate_keeper.isAllowed 'http://www.google.com/setnewsprefs?sfsdfg'
@@ -34,2 +35,3 @@ #returns false
       #returns true
+      #note: only the path and the query of the url gets parsed, everything else (i.e. http://, the domain-name) gets ditched
       console.log gate_keeper.isDisallowed 'http://www.google.com/setnewsprefs?sfsdfg' 
@@ -36,0 +38,0 @@ #returns true

No alert changes

Improved metrics

Total package byte prevSize

27084

increased by8.75%

Number of package files

11

increased by22.22%

Lines of code

381

increased by7.93%

Number of lines in readme file

108

increased by1.89%

No dependency changes