robotstxt - npm Package Compare versions

Comparing version 0.0.4-1 to 0.0.4-3

index.js

@@ -115,3 +115,7 @@ (function() {
         } else {
-          return xu.path;
+          if (_(url).endsWith('?')) {
+            return xu.path + '?';
+          } else {
+            return xu.path;
+          }
         }
@@ -148,2 +152,13 @@ }
     };
+    GateKeeper.prototype.getCrawlDelay = function(user_agent) {
+      var delay, _ref;
+      if (user_agent == null) {
+        user_agent = this.user_agent;
+      }
+      user_agent = user_agent.toLowerCase();
+      delay = ((_ref = this.groups[user_agent]) != null ? _ref.crawl_delay : void 0) || this.groups['*'].crawl_delay;
+      if (delay != null) {
+        return Number(delay);
+      }
+    };
     GateKeeper.prototype.groups = {};
@@ -202,5 +217,4 @@ GateKeeper.prototype.user_agent = null;
       req = handler.request(options, __bind(function(res) {
-        if (res.statusCode !== 200) {
-          return this.emit("error", new Error('invalid status code - is: HTTP ' + res.statusCode + ' - should: HTTP 200'));
-        } else {
+        var _ref;
+        if ((200 <= (_ref = res.statusCode) && _ref < 300)) {
           res.setEncoding(encoding);
@@ -216,2 +230,4 @@ res.on("data", __bind(function(chunk) {
           return null;
+        } else {
+          return this.emit("error", new Error('invalid status code - is: HTTP ' + res.statusCode + ' - should: HTTP 200'));
         }
@@ -241,3 +257,7 @@ }, this));
             doublepoint = line.indexOf(':');
-            kvA = [line.substr(0, doublepoint), line.substr(doublepoint + 1)];
+            if (_(line).includes('#')) {
+              kvA = [line.substr(0, doublepoint), line.substr(doublepoint + 1, line.indexOf('#') - (doublepoint + 1))];
+            } else {
+              kvA = [line.substr(0, doublepoint), line.substr(doublepoint + 1)];
+            }
             if (kvA.length !== 2) {
@@ -273,4 +293,6 @@ return false;
               }
-            } else if (kvA[0] === 'sitemap') {
-              ;
+            } else if (kvA[0] === 'sitemap') {} else if (kvA[0] === 'crawl-delay') {
+              if (currUserAgentGroup) {
+                return currUserAgentGroup.crawl_delay = kvA[1];
+              }
             } else {
@@ -322,3 +344,3 @@ regExStr = kvA[1] + '';
         } else {
-          ;
+
         }
@@ -325,0 +347,0 @@ }, this);

package.json

@@ -19,3 +19,3 @@ {
   "main"          : "./index.js",
-  "version"       : "0.0.4-1"
+  "version"       : "0.0.4-3"
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

32198

increased by6.38%

Lines of code

423

increased by5.22%

No dependency changes