linkinator - npm Package Compare versions

Comparing version 2.13.1 to 2.13.2

build/src/queue.d.ts

@@ -0,3 +1,5 @@
+/// <reference types="node" />
+import { EventEmitter } from 'events';
 export interface QueueOptions {
-    concurrency?: number;
+    concurrency: number;
 }
@@ -7,9 +9,14 @@ export interface QueueItemOptions {
 }
+export declare interface Queue {
+    on(event: 'done', listener: () => void): this;
+}
 export declare type AsyncFunction = () => Promise<void>;
-export declare class Queue {
+export declare class Queue extends EventEmitter {
     private q;
-    private activeTimers;
+    private activeFunctions;
+    private concurrency;
     constructor(options: QueueOptions);
     add(fn: AsyncFunction, options?: QueueItemOptions): void;
+    private tick;
     onIdle(): Promise<void>;
 }

build/src/queue.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Queue = void 0;
-const p_queue_1 = require("p-queue");
-class Queue {
+const events_1 = require("events");
+class Queue extends events_1.EventEmitter {
     constructor(options) {
-        this.activeTimers = 0;
-        this.q = new p_queue_1.default({
-            concurrency: options.concurrency,
+        super();
+        this.q = [];
+        this.activeFunctions = 0;
+        this.concurrency = options.concurrency;
+    }
+    add(fn, options) {
+        const delay = (options === null || options === void 0 ? void 0 : options.delay) || 0;
+        const timeToRun = Date.now() + delay;
+        this.q.push({
+            fn,
+            timeToRun,
         });
+        setTimeout(() => this.tick(), delay);
     }
-    add(fn, options) {
-        if (options === null || options === void 0 ? void 0 : options.delay) {
-            setTimeout(() => {
-                this.q.add(fn);
-                this.activeTimers--;
-            }, options.delay);
-            this.activeTimers++;
+    tick() {
+        // Check if we're complete
+        if (this.activeFunctions === 0 && this.q.length === 0) {
+            this.emit('done');
+            return;
         }
-        else {
-            this.q.add(fn);
+        for (let i = 0; i < this.q.length; i++) {
+            // Check if we have too many concurrent functions executing
+            if (this.activeFunctions >= this.concurrency) {
+                return;
+            }
+            // grab the element at the front of the array
+            const item = this.q.shift();
+            // make sure this element is ready to execute - if not, to the back of the stack
+            if (item.timeToRun > Date.now()) {
+                this.q.push(item);
+            }
+            else {
+                // this function is ready to go!
+                this.activeFunctions++;
+                item.fn().finally(() => {
+                    this.activeFunctions--;
+                    this.tick();
+                });
+            }
         }
     }
     async onIdle() {
-        await this.q.onIdle();
-        await new Promise(resolve => {
-            if (this.activeTimers === 0) {
-                resolve();
-                return;
-            }
-            const timer = setInterval(async () => {
-                if (this.activeTimers === 0) {
-                    await this.q.onIdle();
-                    clearInterval(timer);
-                    resolve();
-                    return;
-                }
-            }, 500);
+        return new Promise(resolve => {
+            this.on('done', () => resolve());
         });
@@ -40,0 +52,0 @@ }

build/src/server.js

@@ -6,8 +6,11 @@ "use strict";
 const path = require("path");
-const util = require("util");
 const fs = require("fs");
+const util_1 = require("util");
 const marked = require("marked");
-const serve = require("serve-handler");
+const mime = require("mime");
+const escape = require("escape-html");
 const enableDestroy = require("server-destroy");
-const readFile = util.promisify(fs.readFile);
+const readFile = util_1.promisify(fs.readFile);
+const stat = util_1.promisify(fs.stat);
+const readdir = util_1.promisify(fs.readdir);
 /**
@@ -19,24 +22,6 @@ * Spin up a local HTTP server to serve static requests from disk
 async function startWebServer(options) {
+    const root = path.resolve(options.root);
     return new Promise((resolve, reject) => {
         const server = http
-            .createServer(async (req, res) => {
-            const pathParts = req.url.split('/').filter(x => !!x);
-            if (pathParts.length > 0) {
-                const ext = path.extname(pathParts[pathParts.length - 1]);
-                if (options.markdown && ext.toLowerCase() === '.md') {
-                    const filePath = path.join(path.resolve(options.root), req.url);
-                    const data = await readFile(filePath, { encoding: 'utf-8' });
-                    const result = marked(data, { gfm: true });
-                    res.writeHead(200, {
-                        'content-type': 'text/html',
-                    });
-                    res.end(result);
-                    return;
-                }
-            }
-            return serve(req, res, {
-                public: options.root,
-                directoryListing: options.directoryListing,
-            });
-        })
+            .createServer((req, res) => handleRequest(req, res, root, options))
             .listen(options.port, () => resolve(server))
@@ -48,2 +33,74 @@ .on('error', reject);
 exports.startWebServer = startWebServer;
+async function handleRequest(req, res, root, options) {
+    var _a, _b, _c;
+    const pathParts = ((_a = req.url) === null || _a === void 0 ? void 0 : _a.split('/')) || [];
+    const originalPath = path.join(root, ...pathParts);
+    if ((_b = req.url) === null || _b === void 0 ? void 0 : _b.endsWith('/')) {
+        pathParts.push('index.html');
+    }
+    const localPath = path.join(root, ...pathParts);
+    if (!localPath.startsWith(root)) {
+        res.writeHead(500);
+        res.end();
+        return;
+    }
+    const maybeListing = options.directoryListing && localPath.endsWith(`${path.sep}index.html`);
+    try {
+        const stats = await stat(localPath);
+        const isDirectory = stats.isDirectory();
+        if (isDirectory) {
+            // this means we got a path with no / at the end!
+            const doc = "<html><body>Redirectin'</body></html>";
+            res.statusCode = 301;
+            res.setHeader('Content-Type', 'text/html; charset=UTF-8');
+            res.setHeader('Content-Length', Buffer.byteLength(doc));
+            res.setHeader('Location', req.url + '/');
+            res.end(doc);
+            return;
+        }
+    }
+    catch (err) {
+        if (!maybeListing) {
+            return return404(res, err);
+        }
+    }
+    try {
+        let data = await readFile(localPath, { encoding: 'utf8' });
+        let mimeType = mime.getType(localPath);
+        const isMarkdown = (_c = req.url) === null || _c === void 0 ? void 0 : _c.toLocaleLowerCase().endsWith('.md');
+        if (isMarkdown && options.markdown) {
+            data = marked(data, { gfm: true });
+            mimeType = 'text/html; charset=UTF-8';
+        }
+        res.setHeader('Content-Type', mimeType);
+        res.setHeader('Content-Length', Buffer.byteLength(data));
+        res.writeHead(200);
+        res.end(data);
+    }
+    catch (err) {
+        if (maybeListing) {
+            try {
+                const files = await readdir(originalPath);
+                const fileList = files
+                    .filter(f => escape(f))
+                    .map(f => `<li><a href="${f}">${f}</a></li>`)
+                    .join('\r\n');
+                const data = `<html><body><ul>${fileList}</ul></body></html>`;
+                res.writeHead(200);
+                res.end(data);
+                return;
+            }
+            catch (err) {
+                return return404(res, err);
+            }
+        }
+        else {
+            return return404(res, err);
+        }
+    }
+}
+function return404(res, err) {
+    res.writeHead(404);
+    res.end(JSON.stringify(err));
+}
 //# sourceMappingURL=server.js.map

package.json

 {
   "name": "linkinator",
   "description": "Find broken links, missing images, etc in your HTML. Scurry around your site and find all those broken links.",
-  "version": "2.13.1",
+  "version": "2.13.2",
   "license": "MIT",
   "repository": "JustinBeckwith/linkinator",
+  "author": "Justin Beckwith",
   "main": "build/src/index.js",
@@ -18,6 +19,5 @@ "types": "build/src/index.d.ts",
     "fix": "gts fix",
-    "codecov": "c8 report --reporter=json && codecov -f coverage/*.json",
     "lint": "gts lint",
     "build-binaries": "pkg . --out-path build/binaries",
-    "docs-test": "npm link && linkinator ./README.md"
+    "docs-test": "node build/src/cli.js ./README.md"
   },
@@ -27,2 +27,3 @@ "dependencies": {
     "cheerio": "^1.0.0-rc.5",
+    "escape-html": "^1.0.3",
     "gaxios": "^4.0.0",
@@ -33,4 +34,3 @@ "glob": "^7.1.6",
     "meow": "^9.0.0",
-    "p-queue": "^6.2.1",
-    "serve-handler": "^6.1.3",
+    "mime": "^2.5.0",
     "server-destroy": "^1.0.1",
@@ -42,8 +42,9 @@ "update-notifier": "^5.0.0"
     "@types/cheerio": "0.22.23",
+    "@types/escape-html": "^1.0.0",
     "@types/glob": "^7.1.3",
     "@types/marked": "^1.2.0",
     "@types/meow": "^5.0.0",
+    "@types/mime": "^2.0.3",
     "@types/mocha": "^8.0.0",
     "@types/node": "^12.7.12",
-    "@types/serve-handler": "^6.1.0",
     "@types/server-destroy": "^1.0.0",
@@ -61,2 +62,3 @@ "@types/sinon": "^9.0.0",
     "sinon": "^9.0.0",
+    "strip-ansi": "^6.0.0",
     "typescript": "^4.0.0"
@@ -85,4 +87,8 @@ },
       "build/test"
+    ],
+    "reporter": [
+      "html",
+      "text"
     ]
   }
 }

README.md

 # 🐿 linkinator
+
 > A super simple site crawler and broken link checker.
 
-[![npm version](https://img.shields.io/npm/v/linkinator.svg)](https://www.npmjs.org/package/linkinator)
-[![Build Status](https://img.shields.io/github/workflow/status/JustinBeckwith/linkinator/ci/master)](https://github.com/JustinBeckwith/linkinator/actions?query=branch%3Amaster+workflow%3Aci)
-[![codecov](https://img.shields.io/codecov/c/github/JustinBeckwith/linkinator/master)](https://codecov.io/gh/JustinBeckwith/linkinator)
+[![npm version](https://img.shields.io/npm/v/linkinator)](https://www.npmjs.org/package/linkinator)
+[![Build Status](https://img.shields.io/github/workflow/status/JustinBeckwith/linkinator/ci/main)](https://github.com/JustinBeckwith/linkinator/actions?query=branch%3Amain+workflow%3Aci)
+[![codecov](https://img.shields.io/codecov/c/github/JustinBeckwith/linkinator/main)](https://codecov.io/gh/JustinBeckwith/linkinator)
 [![Known Vulnerabilities](https://img.shields.io/snyk/vulnerabilities/github/JustinBeckwith/linkinator)](https://snyk.io/test/github/JustinBeckwith/linkinator)
-[![Code Style: Google](https://img.shields.io/badge/code%20style-google-blueviolet.svg)](https://github.com/google/gts)
-[![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release)
+[![Code Style: Google](https://img.shields.io/badge/code%20style-google-blueviolet)](https://github.com/google/gts)
+[![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079)](https://github.com/semantic-release/semantic-release)
 
 
 Behold my latest inator! The `linkinator` provides an API and CLI for crawling websites and validating links.  It's got a ton of sweet features:
+
 - 🔥 Easily perform scans on remote sites or local files
@@ -22,3 +24,3 @@ - 🔥 Scan any element that includes links, not just `<a href>`
 ```sh
-$ npm install linkinator
+npm install linkinator
 ```
@@ -32,3 +34,3 @@
 
-```
+```text
 $ linkinator LOCATIONS [ --arguments ]
@@ -93,3 +95,3 @@
 ```sh
-$ npx linkinator http://jbeckwith.com
+npx linkinator http://jbeckwith.com
 ```
@@ -100,3 +102,3 @@
 ```sh
-$ npx linkinator ./docs
+npx linkinator ./docs
 ```
@@ -107,3 +109,3 @@
 ```sh
-$ npx linkinator ./docs --recurse
+npx linkinator ./docs --recurse
 ```
@@ -114,3 +116,3 @@
 ```sh
-$ npx linkinator ./docs --skip www.googleapis.com
+npx linkinator ./docs --skip www.googleapis.com
 ```
@@ -121,3 +123,3 @@
 ```sh
-$ linkinator http://jbeckwith.com --skip '^(?!http://jbeckwith.com)'
+linkinator http://jbeckwith.com --skip '^(?!http://jbeckwith.com)'
 ```
@@ -128,3 +130,3 @@
 ```sh
-$ linkinator ./docs --format CSV
+linkinator ./docs --format CSV
 ```
@@ -135,3 +137,3 @@
 ```sh
-$ linkinator ./README.md --markdown
+linkinator ./README.md --markdown
 ```
@@ -142,6 +144,7 @@
 ```sh
-$ linkinator "**/*.md" --markdown
+linkinator "**/*.md" --markdown
 ```
 
 ### Configuration file
+
 You can pass options directly to the `linkinator` CLI, or you can define a config file.  By default, `linkinator` will look for a `linkinator.config.json` file in the current working directory.
@@ -167,6 +170,7 @@
 ```sh
-$ linkinator --config /some/path/your-config.json
+linkinator --config /some/path/your-config.json
 ```
 
 ## GitHub Actions
+
 You can use `linkinator` as a GitHub Action as well, using [JustinBeckwith/linkinator-action](https://github.com/JustinBeckwith/linkinator-action):
@@ -195,4 +199,6 @@
 
-#### linkinator.check(options)
+### linkinator.check(options)
+
 Asynchronous method that runs a site wide scan. Options come in the form of an object that includes:
+
 - `path` (string|string[]) - A fully qualified path to the url to be scanned, or the path(s) to the directory on disk that contains files to be scanned. *required*.
@@ -210,4 +216,6 @@ - `concurrency` (number) -  The number of connections to make simultaneously. Defaults to 100.
 
-#### linkinator.LinkChecker()
+### linkinator.LinkChecker()
+
 Constructor method that can be used to create a new `LinkChecker` instance.  This is particularly useful if you want to receive events as the crawler crawls.  Exposes the following events:
+
 - `pagestart` (string) - Provides the url that the crawler has just started to scan.
@@ -219,4 +227,6 @@ - `link` (object) - Provides an object with
 
-### Simple example
+### Examples
 
+#### Simple example
+
 ```js
@@ -256,3 +266,3 @@ const link = require('linkinator');
 
-### Complete example
+#### Complete example
 
@@ -317,8 +327,11 @@ In most cases you're going to want to respond to events, as running the check command can kinda take a long time.
 ### Using a proxy
+
 This library supports proxies via the `HTTP_PROXY` and `HTTPS_PROXY` environment variables.  This [guide](https://www.golinuxcloud.com/set-up-proxy-http-proxy-environment-variable/) provides a nice overview of how to format and set these variables.
 
 ### Globbing
+
 You may have noticed in the example, when using a glob the pattern is encapsulated in quotes:
+
 ```sh
-$ linkinator "**/*.md" --markdown
+linkinator "**/*.md" --markdown
 ```
@@ -329,11 +342,15 @@
 ### Debugging
+
 Oftentimes when a link fails, it's an easy to spot typo, or a clear 404.  Other times ... you may need more details on exactly what went wrong.  To see a full call stack for the HTTP request failure, use `--verbosity DEBUG`:
+
 ```sh
-$ linkinator https://jbeckwith.com --verbosity DEBUG
+linkinator https://jbeckwith.com --verbosity DEBUG
 ```
 
 ### Controlling Output
+
 The `--verbosity` flag offers preset options for controlling the output, but you may want more control.  Using [`jq`](https://stedolan.github.io/jq/) and `--format JSON` - you can do just that!
+
 ```sh
-$ linkinator https://jbeckwith.com --verbosity DEBUG --format JSON | jq '.links | .[] | select(.state | contains("BROKEN"))'
+linkinator https://jbeckwith.com --verbosity DEBUG --format JSON | jq '.links | .[] | select(.state | contains("BROKEN"))'
 ```
@@ -340,0 +357,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

No contributors or author data

Maintenance

Package does not specify a list of contributors or an author in package.json.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

94470

increased by5.67%

Lines of code

1291

increased by6.26%

Number of low maintenance alerts

1

decreased by-50%

Number of lines in readme file

344

increased by5.2%

Worsened metrics

Dev dependency count

23

increased by9.52%

Dependency changes

• + Addedescape-html@^1.0.3

• + Addedmime@^2.5.0

• + Addedescape-html@1.0.3(transitive)

• + Addedmime@2.6.0(transitive)

• - Removedp-queue@^6.2.1

• - Removedserve-handler@^6.1.3

• - Removedbytes@3.0.0(transitive)

• - Removedcontent-disposition@0.5.2(transitive)

• - Removedeventemitter3@4.0.7(transitive)

• - Removedmime-db@1.33.0(transitive)

• - Removedmime-types@2.1.18(transitive)

• - Removedp-finally@1.0.0(transitive)

• - Removedp-queue@6.6.2(transitive)

• - Removedp-timeout@3.2.0(transitive)

• - Removedpath-is-inside@1.0.2(transitive)

• - Removedpath-to-regexp@3.3.0(transitive)

• - Removedrange-parser@1.2.0(transitive)

• - Removedserve-handler@6.1.6(transitive)