Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More
Socket
Sign inDemoInstall
Socket

robotstxt

Package Overview
Dependencies
Maintainers
1
Versions
17
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

robotstxt - npm Package Compare versions

Comparing version 0.0.4-3 to 0.0.4-4

37

index.js

@@ -22,3 +22,15 @@ (function() {

function GateKeeper(user_agent) {
this.getCrawlDelay = __bind(this.getCrawlDelay, this);
this.getGroup = __bind(this.getGroup, this);
this.setUserAgent = __bind(this.setUserAgent, this);
this.cleanUrl = __bind(this.cleanUrl, this);
this.why = __bind(this.why, this);
this.whatsUp = __bind(this.whatsUp, this);
this.isDisallowed = __bind(this.isDisallowed, this);
this.isAllowed = __bind(this.isAllowed, this); this.user_agent = null;
this.setUserAgent(user_agent);
this.groups = {};
this.user_agent_group = {
'*': '*'
};
}

@@ -33,3 +45,3 @@ GateKeeper.prototype.isAllowed = function(url, _allowed) {

prio = 0;
check = function(matchO) {
check = __bind(function(matchO) {
if (matchO) {

@@ -48,3 +60,3 @@ if (matchO.type === 'disallow') {

}
};
}, this);
for (_i = 0, _len = a.length; _i < _len; _i++) {

@@ -67,5 +79,5 @@ matchO = a[_i];

group = this.getGroup();
return r = this.groups[group].rules.map(function(e) {
return r = this.groups[group].rules.map(__bind(function(e) {
return e(url);
});
}, this));
};

@@ -78,3 +90,3 @@ GateKeeper.prototype.why = function(url) {

conflict = false;
test = function(matchO) {
test = __bind(function(matchO) {
if (matchO) {

@@ -98,3 +110,3 @@ if (!ra[0]) {

}
};
}, this);
for (_i = 0, _len = a.length; _i < _len; _i++) {

@@ -168,7 +180,2 @@ matchO = a[_i];

};
GateKeeper.prototype.groups = {};
GateKeeper.prototype.user_agent = null;
GateKeeper.prototype.user_agent_group = {
'*': '*'
};
return GateKeeper;

@@ -186,2 +193,3 @@ })();

this.parse = __bind(this.parse, this);
this.crawl = __bind(this.crawl, this);
if (this.url) {

@@ -275,2 +283,3 @@ this.uri = parseUri(this.url);

if (!myGateKeeper) {
delete myGateKeeper;
myGateKeeper = new GateKeeper(this.user_agent);

@@ -297,3 +306,5 @@ }

}
} else if (kvA[0] === 'sitemap') {} else if (kvA[0] === 'crawl-delay') {
} else if (kvA[0] === 'sitemap') {
;
} else if (kvA[0] === 'crawl-delay') {
if (currUserAgentGroup) {

@@ -348,3 +359,3 @@ return currUserAgentGroup.crawl_delay = kvA[1];

} else {
;
}

@@ -351,0 +362,0 @@ }, this);

@@ -19,3 +19,3 @@ {

"main" : "./index.js",
"version" : "0.0.4-3"
"version" : "0.0.4-4"
}

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc