crawler-ninja-expired - npm Package Compare versions

Comparing version 1.0.0 to 1.0.1

index.js

@@ -10,5 +10,7 @@ var fs              = require('fs');
 var EXPIRED_TXT_FILE = process.cwd() + "/expireds.txt";
+var NOT_AVAILABLE_TXT_FILE = process.cwd() + "/not-available-errors.txt";
 var crawlLog = logs.Logger;
 
 var expiredTxtFile = EXPIRED_TXT_FILE;
+var notAvailableTxtFile = NOT_AVAILABLE_TXT_FILE;
 /**
@@ -30,2 +32,9 @@ *  Find expired domains
     }
+
+    if (params && params.notAvailableTxtFile) {
+      notAvailableTxtFile = params.notAvailableTxtFile;
+    }
+
+    this.domains = [];
+
 }
@@ -39,6 +48,9 @@
         this.expiredLog.info({"50*" : true, status : result.statusCode, url : result.url});
-        check(URI.domain(result.uri), callback);
+        this.check(URI.domain(result.uri), callback);
 
       }
-      callback();
+      else {
+        callback();
+      }
+
 }
@@ -52,3 +64,3 @@
           this.expiredLog.info({expired : true, domain : domain, url : result.url});
-          check(URI.domain(result.uri), callback);
+          this.check(URI.domain(result.uri), callback);
         }
@@ -64,4 +76,13 @@ else {
 
-check = function(domain, callback) {
+Plugin.prototype.check = function(domain, callback) {
     crawlLog.info({"url" : domain, "step" : "expired-domains-plugin", "message" : "check"});
+
+    if (this.domains.indexOf(domain) > -1) {
+      return callback();
+    }
+    else {
+      this.domains.push(domain);
+    }
+
+    var self = this;
     domainChecker({domain : domain}, function(error, result) {
@@ -74,3 +95,11 @@
         var line = result.domain + "," + (result.pr ? result.pr : "no-pr") + "," + result.available + "," + (result.whois.toString() ?  result.whois.toString(): "no-whois") + "\n";
-        fs.appendFile(expiredTxtFile, line);
+
+        if (result.available == "NOT-AVAILABLE") {
+
+            fs.appendFile(notAvailableTxtFile, line);
+        }
+        else {
+            fs.appendFile(expiredTxtFile, line);
+        }
+
         callback();
@@ -77,0 +106,0 @@ });

package.json

 {
   "name": "crawler-ninja-expired",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Expired domains finder for crawler.ninja",
@@ -5,0 +5,0 @@ "main": "index.js",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

8770

increased by7.44%

Lines of code

140

increased by16.67%

No dependency changes