linkinator - npm Package Compare versions

Comparing version 2.14.5 to 2.15.0

build/src/cli.js

@@ -54,2 +54,11 @@ #!/usr/bin/env node
 
+      --retry-errors,
+          Automatically retry requests that return 5xx or unknown response.
+
+      --retry-errors-count,
+          How many times should an error be retried?
+
+      --retry-errors-jitter,
+          Random jitter applied to error retry.
+
       --server-root
@@ -95,2 +104,5 @@ When scanning a locally directory, customize the location on disk
         retry: { type: 'boolean' },
+        retryErrors: { type: 'boolean' },
+        retryErrorsCount: { type: 'number', default: 3 },
+        retryErrorsJitter: { type: 'number', default: 3000 },
         urlRewriteSearch: { type: 'string' },
@@ -147,2 +159,5 @@ urlReWriteReplace: { type: 'string' },
         retry: flags.retry,
+        retryErrors: flags.retryErrors,
+        retryErrorsCount: Number(flags.retryErrorsCount),
+        retryErrorsJitter: Number(flags.retryErrorsJitter),
     };
@@ -149,0 +164,0 @@ if (flags.skip) {

build/src/config.d.ts

@@ -14,2 +14,5 @@ export interface Flags {
     retry?: boolean;
+    retryErrors?: boolean;
+    retryErrorsCount?: number;
+    retryErrorsJitter?: number;
     urlRewriteSearch?: string;
@@ -16,0 +19,0 @@ urlRewriteReplace?: string;

build/src/index.d.ts

@@ -36,2 +36,3 @@ /// <reference types="node" />
     delayCache: Map<string, number>;
+    retryErrorsCache: Map<string, number>;
     checkOptions: CheckOptions;
@@ -41,2 +42,5 @@ queue: Queue;
     retry: boolean;
+    retryErrors: boolean;
+    retryErrorsCount: number;
+    retryErrorsJitter: number;
 }
@@ -80,2 +84,8 @@ export declare const headers: {
     shouldRetryAfter(res: GaxiosResponse, opts: CrawlOptions): boolean;
+    /**
+     * If the response is a 5xx or synthetic 0 response retry N times.
+     * @param status Status returned by request or 0 if request threw.
+     * @param opts CrawlOptions used during this request
+     */
+    shouldRetryOnError(status: number, opts: CrawlOptions): boolean;
 }
@@ -82,0 +92,0 @@ /**

build/src/index.js

@@ -68,2 +68,3 @@ "use strict";
         const delayCache = new Map();
+        const retryErrorsCache = new Map();
         for (const path of options.path) {
@@ -73,2 +74,3 @@ const url = new url_1.URL(path);
             queue.add(async () => {
+                var _a, _b;
                 await this.crawl({
@@ -81,5 +83,9 @@ url,
                     delayCache,
+                    retryErrorsCache,
                     queue,
                     rootPath: path,
                     retry: !!opts.retry,
+                    retryErrors: !!opts.retryErrors,
+                    retryErrorsCount: (_a = opts.retryErrorsCount) !== null && _a !== void 0 ? _a : 3,
+                    retryErrorsJitter: (_b = opts.retryErrorsJitter) !== null && _b !== void 0 ? _b : 3000,
                 });
@@ -234,2 +240,7 @@ });
         }
+        // If retryErrors is enabled, retry 5xx and 0 status (which indicates
+        // a network error likely occurred):
+        if (this.shouldRetryOnError(status, opts)) {
+            return;
+        }
         // Assume any 2xx status is 👌
@@ -293,2 +304,3 @@ if (status >= 200 && status < 300) {
                             delayCache: opts.delayCache,
+                            retryErrorsCache: opts.retryErrorsCache,
                             results: opts.results,
@@ -300,2 +312,5 @@ checkOptions: opts.checkOptions,
                             retry: opts.retry,
+                            retryErrors: opts.retryErrors,
+                            retryErrorsCount: opts.retryErrorsCount,
+                            retryErrorsJitter: opts.retryErrorsJitter,
                         });
@@ -356,2 +371,41 @@ });
     }
+    /**
+     * If the response is a 5xx or synthetic 0 response retry N times.
+     * @param status Status returned by request or 0 if request threw.
+     * @param opts CrawlOptions used during this request
+     */
+    shouldRetryOnError(status, opts) {
+        const maxRetries = opts.retryErrorsCount;
+        const retryAfter = opts.retryErrorsJitter;
+        if (!opts.retryErrors) {
+            return false;
+        }
+        // Only retry 0 and >5xx status codes:
+        if (status > 0 && status < 500) {
+            return false;
+        }
+        // check to see if there is already a request to wait for this host
+        if (opts.retryErrorsCache.has(opts.url.host)) {
+            // use whichever time is higher in the cache
+            const currentRetries = opts.retryErrorsCache.get(opts.url.host);
+            if (currentRetries > maxRetries)
+                return false;
+            opts.retryErrorsCache.set(opts.url.host, currentRetries + 1);
+        }
+        else {
+            opts.retryErrorsCache.set(opts.url.host, 1);
+        }
+        opts.queue.add(async () => {
+            await this.crawl(opts);
+        }, {
+            delay: retryAfter,
+        });
+        const retryDetails = {
+            url: opts.url.href,
+            status: status,
+            secondsUntilRetry: Math.round(retryAfter / 1000),
+        };
+        this.emit('retry', retryDetails);
+        return true;
+    }
 }
@@ -358,0 +412,0 @@ exports.LinkChecker = LinkChecker;

build/src/options.d.ts

@@ -16,2 +16,5 @@ export interface UrlRewriteExpression {
     retry?: boolean;
+    retryErrors?: boolean;
+    retryErrorsCount?: number;
+    retryErrorsJitter?: number;
     urlRewriteExpressions?: UrlRewriteExpression[];
@@ -18,0 +21,0 @@ }

package.json

 {
   "name": "linkinator",
   "description": "Find broken links, missing images, etc in your HTML. Scurry around your site and find all those broken links.",
-  "version": "2.14.5",
+  "version": "2.15.0",
   "license": "MIT",
@@ -41,7 +41,7 @@ "repository": "JustinBeckwith/linkinator",
     "@types/glob": "^7.1.3",
-    "@types/marked": "^2.0.0",
+    "@types/marked": "^3.0.0",
     "@types/meow": "^5.0.0",
     "@types/mime": "^2.0.3",
     "@types/mocha": "^9.0.0",
-    "@types/node": "^14.0.0",
+    "@types/node": "^16.0.0",
     "@types/server-destroy": "^1.0.0",
@@ -59,3 +59,3 @@ "@types/sinon": "^10.0.0",
     "semantic-release": "^18.0.0",
-    "sinon": "^11.0.0",
+    "sinon": "^12.0.0",
     "strip-ansi": "^6.0.0",
@@ -62,0 +62,0 @@ "typescript": "^4.0.0"

README.md

@@ -73,2 +73,11 @@ # 🐿 linkinator
 
+    --retry-errors,
+        Automatically retry requests that return 5xx or unknown response.
+
+    --retry-errors-count,
+        How many times should an error be retried?
+
+    --retry-errors-jitter,
+        Random jitter applied to error retry.
+
     --server-root
@@ -75,0 +84,0 @@ When scanning a locally directory, customize the location on disk

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

103831

increased by5.62%

Lines of code

1429

increased by6.09%

Number of lines in readme file

360

increased by2.56%

No dependency changes