linkinator
Advanced tools
Comparing version 2.13.1 to 2.13.2
@@ -0,3 +1,5 @@ | ||
/// <reference types="node" /> | ||
import { EventEmitter } from 'events'; | ||
export interface QueueOptions { | ||
concurrency?: number; | ||
concurrency: number; | ||
} | ||
@@ -7,9 +9,14 @@ export interface QueueItemOptions { | ||
} | ||
export declare interface Queue { | ||
on(event: 'done', listener: () => void): this; | ||
} | ||
export declare type AsyncFunction = () => Promise<void>; | ||
export declare class Queue { | ||
export declare class Queue extends EventEmitter { | ||
private q; | ||
private activeTimers; | ||
private activeFunctions; | ||
private concurrency; | ||
constructor(options: QueueOptions); | ||
add(fn: AsyncFunction, options?: QueueItemOptions): void; | ||
private tick; | ||
onIdle(): Promise<void>; | ||
} |
"use strict"; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
exports.Queue = void 0; | ||
const p_queue_1 = require("p-queue"); | ||
class Queue { | ||
const events_1 = require("events"); | ||
class Queue extends events_1.EventEmitter { | ||
constructor(options) { | ||
this.activeTimers = 0; | ||
this.q = new p_queue_1.default({ | ||
concurrency: options.concurrency, | ||
super(); | ||
this.q = []; | ||
this.activeFunctions = 0; | ||
this.concurrency = options.concurrency; | ||
} | ||
add(fn, options) { | ||
const delay = (options === null || options === void 0 ? void 0 : options.delay) || 0; | ||
const timeToRun = Date.now() + delay; | ||
this.q.push({ | ||
fn, | ||
timeToRun, | ||
}); | ||
setTimeout(() => this.tick(), delay); | ||
} | ||
add(fn, options) { | ||
if (options === null || options === void 0 ? void 0 : options.delay) { | ||
setTimeout(() => { | ||
this.q.add(fn); | ||
this.activeTimers--; | ||
}, options.delay); | ||
this.activeTimers++; | ||
tick() { | ||
// Check if we're complete | ||
if (this.activeFunctions === 0 && this.q.length === 0) { | ||
this.emit('done'); | ||
return; | ||
} | ||
else { | ||
this.q.add(fn); | ||
for (let i = 0; i < this.q.length; i++) { | ||
// Check if we have too many concurrent functions executing | ||
if (this.activeFunctions >= this.concurrency) { | ||
return; | ||
} | ||
// grab the element at the front of the array | ||
const item = this.q.shift(); | ||
// make sure this element is ready to execute - if not, to the back of the stack | ||
if (item.timeToRun > Date.now()) { | ||
this.q.push(item); | ||
} | ||
else { | ||
// this function is ready to go! | ||
this.activeFunctions++; | ||
item.fn().finally(() => { | ||
this.activeFunctions--; | ||
this.tick(); | ||
}); | ||
} | ||
} | ||
} | ||
async onIdle() { | ||
await this.q.onIdle(); | ||
await new Promise(resolve => { | ||
if (this.activeTimers === 0) { | ||
resolve(); | ||
return; | ||
} | ||
const timer = setInterval(async () => { | ||
if (this.activeTimers === 0) { | ||
await this.q.onIdle(); | ||
clearInterval(timer); | ||
resolve(); | ||
return; | ||
} | ||
}, 500); | ||
return new Promise(resolve => { | ||
this.on('done', () => resolve()); | ||
}); | ||
@@ -40,0 +52,0 @@ } |
@@ -6,8 +6,11 @@ "use strict"; | ||
const path = require("path"); | ||
const util = require("util"); | ||
const fs = require("fs"); | ||
const util_1 = require("util"); | ||
const marked = require("marked"); | ||
const serve = require("serve-handler"); | ||
const mime = require("mime"); | ||
const escape = require("escape-html"); | ||
const enableDestroy = require("server-destroy"); | ||
const readFile = util.promisify(fs.readFile); | ||
const readFile = util_1.promisify(fs.readFile); | ||
const stat = util_1.promisify(fs.stat); | ||
const readdir = util_1.promisify(fs.readdir); | ||
/** | ||
@@ -19,24 +22,6 @@ * Spin up a local HTTP server to serve static requests from disk | ||
async function startWebServer(options) { | ||
const root = path.resolve(options.root); | ||
return new Promise((resolve, reject) => { | ||
const server = http | ||
.createServer(async (req, res) => { | ||
const pathParts = req.url.split('/').filter(x => !!x); | ||
if (pathParts.length > 0) { | ||
const ext = path.extname(pathParts[pathParts.length - 1]); | ||
if (options.markdown && ext.toLowerCase() === '.md') { | ||
const filePath = path.join(path.resolve(options.root), req.url); | ||
const data = await readFile(filePath, { encoding: 'utf-8' }); | ||
const result = marked(data, { gfm: true }); | ||
res.writeHead(200, { | ||
'content-type': 'text/html', | ||
}); | ||
res.end(result); | ||
return; | ||
} | ||
} | ||
return serve(req, res, { | ||
public: options.root, | ||
directoryListing: options.directoryListing, | ||
}); | ||
}) | ||
.createServer((req, res) => handleRequest(req, res, root, options)) | ||
.listen(options.port, () => resolve(server)) | ||
@@ -48,2 +33,74 @@ .on('error', reject); | ||
exports.startWebServer = startWebServer; | ||
async function handleRequest(req, res, root, options) { | ||
var _a, _b, _c; | ||
const pathParts = ((_a = req.url) === null || _a === void 0 ? void 0 : _a.split('/')) || []; | ||
const originalPath = path.join(root, ...pathParts); | ||
if ((_b = req.url) === null || _b === void 0 ? void 0 : _b.endsWith('/')) { | ||
pathParts.push('index.html'); | ||
} | ||
const localPath = path.join(root, ...pathParts); | ||
if (!localPath.startsWith(root)) { | ||
res.writeHead(500); | ||
res.end(); | ||
return; | ||
} | ||
const maybeListing = options.directoryListing && localPath.endsWith(`${path.sep}index.html`); | ||
try { | ||
const stats = await stat(localPath); | ||
const isDirectory = stats.isDirectory(); | ||
if (isDirectory) { | ||
// this means we got a path with no / at the end! | ||
const doc = "<html><body>Redirectin'</body></html>"; | ||
res.statusCode = 301; | ||
res.setHeader('Content-Type', 'text/html; charset=UTF-8'); | ||
res.setHeader('Content-Length', Buffer.byteLength(doc)); | ||
res.setHeader('Location', req.url + '/'); | ||
res.end(doc); | ||
return; | ||
} | ||
} | ||
catch (err) { | ||
if (!maybeListing) { | ||
return return404(res, err); | ||
} | ||
} | ||
try { | ||
let data = await readFile(localPath, { encoding: 'utf8' }); | ||
let mimeType = mime.getType(localPath); | ||
const isMarkdown = (_c = req.url) === null || _c === void 0 ? void 0 : _c.toLocaleLowerCase().endsWith('.md'); | ||
if (isMarkdown && options.markdown) { | ||
data = marked(data, { gfm: true }); | ||
mimeType = 'text/html; charset=UTF-8'; | ||
} | ||
res.setHeader('Content-Type', mimeType); | ||
res.setHeader('Content-Length', Buffer.byteLength(data)); | ||
res.writeHead(200); | ||
res.end(data); | ||
} | ||
catch (err) { | ||
if (maybeListing) { | ||
try { | ||
const files = await readdir(originalPath); | ||
const fileList = files | ||
.filter(f => escape(f)) | ||
.map(f => `<li><a href="${f}">${f}</a></li>`) | ||
.join('\r\n'); | ||
const data = `<html><body><ul>${fileList}</ul></body></html>`; | ||
res.writeHead(200); | ||
res.end(data); | ||
return; | ||
} | ||
catch (err) { | ||
return return404(res, err); | ||
} | ||
} | ||
else { | ||
return return404(res, err); | ||
} | ||
} | ||
} | ||
function return404(res, err) { | ||
res.writeHead(404); | ||
res.end(JSON.stringify(err)); | ||
} | ||
//# sourceMappingURL=server.js.map |
{ | ||
"name": "linkinator", | ||
"description": "Find broken links, missing images, etc in your HTML. Scurry around your site and find all those broken links.", | ||
"version": "2.13.1", | ||
"version": "2.13.2", | ||
"license": "MIT", | ||
"repository": "JustinBeckwith/linkinator", | ||
"author": "Justin Beckwith", | ||
"main": "build/src/index.js", | ||
@@ -18,6 +19,5 @@ "types": "build/src/index.d.ts", | ||
"fix": "gts fix", | ||
"codecov": "c8 report --reporter=json && codecov -f coverage/*.json", | ||
"lint": "gts lint", | ||
"build-binaries": "pkg . --out-path build/binaries", | ||
"docs-test": "npm link && linkinator ./README.md" | ||
"docs-test": "node build/src/cli.js ./README.md" | ||
}, | ||
@@ -27,2 +27,3 @@ "dependencies": { | ||
"cheerio": "^1.0.0-rc.5", | ||
"escape-html": "^1.0.3", | ||
"gaxios": "^4.0.0", | ||
@@ -33,4 +34,3 @@ "glob": "^7.1.6", | ||
"meow": "^9.0.0", | ||
"p-queue": "^6.2.1", | ||
"serve-handler": "^6.1.3", | ||
"mime": "^2.5.0", | ||
"server-destroy": "^1.0.1", | ||
@@ -42,8 +42,9 @@ "update-notifier": "^5.0.0" | ||
"@types/cheerio": "0.22.23", | ||
"@types/escape-html": "^1.0.0", | ||
"@types/glob": "^7.1.3", | ||
"@types/marked": "^1.2.0", | ||
"@types/meow": "^5.0.0", | ||
"@types/mime": "^2.0.3", | ||
"@types/mocha": "^8.0.0", | ||
"@types/node": "^12.7.12", | ||
"@types/serve-handler": "^6.1.0", | ||
"@types/server-destroy": "^1.0.0", | ||
@@ -61,2 +62,3 @@ "@types/sinon": "^9.0.0", | ||
"sinon": "^9.0.0", | ||
"strip-ansi": "^6.0.0", | ||
"typescript": "^4.0.0" | ||
@@ -85,4 +87,8 @@ }, | ||
"build/test" | ||
], | ||
"reporter": [ | ||
"html", | ||
"text" | ||
] | ||
} | ||
} |
# 🐿 linkinator | ||
> A super simple site crawler and broken link checker. | ||
[![npm version](https://img.shields.io/npm/v/linkinator.svg)](https://www.npmjs.org/package/linkinator) | ||
[![Build Status](https://img.shields.io/github/workflow/status/JustinBeckwith/linkinator/ci/master)](https://github.com/JustinBeckwith/linkinator/actions?query=branch%3Amaster+workflow%3Aci) | ||
[![codecov](https://img.shields.io/codecov/c/github/JustinBeckwith/linkinator/master)](https://codecov.io/gh/JustinBeckwith/linkinator) | ||
[![npm version](https://img.shields.io/npm/v/linkinator)](https://www.npmjs.org/package/linkinator) | ||
[![Build Status](https://img.shields.io/github/workflow/status/JustinBeckwith/linkinator/ci/main)](https://github.com/JustinBeckwith/linkinator/actions?query=branch%3Amain+workflow%3Aci) | ||
[![codecov](https://img.shields.io/codecov/c/github/JustinBeckwith/linkinator/main)](https://codecov.io/gh/JustinBeckwith/linkinator) | ||
[![Known Vulnerabilities](https://img.shields.io/snyk/vulnerabilities/github/JustinBeckwith/linkinator)](https://snyk.io/test/github/JustinBeckwith/linkinator) | ||
[![Code Style: Google](https://img.shields.io/badge/code%20style-google-blueviolet.svg)](https://github.com/google/gts) | ||
[![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release) | ||
[![Code Style: Google](https://img.shields.io/badge/code%20style-google-blueviolet)](https://github.com/google/gts) | ||
[![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079)](https://github.com/semantic-release/semantic-release) | ||
Behold my latest inator! The `linkinator` provides an API and CLI for crawling websites and validating links. It's got a ton of sweet features: | ||
- 🔥 Easily perform scans on remote sites or local files | ||
@@ -22,3 +24,3 @@ - 🔥 Scan any element that includes links, not just `<a href>` | ||
```sh | ||
$ npm install linkinator | ||
npm install linkinator | ||
``` | ||
@@ -32,3 +34,3 @@ | ||
``` | ||
```text | ||
$ linkinator LOCATIONS [ --arguments ] | ||
@@ -93,3 +95,3 @@ | ||
```sh | ||
$ npx linkinator http://jbeckwith.com | ||
npx linkinator http://jbeckwith.com | ||
``` | ||
@@ -100,3 +102,3 @@ | ||
```sh | ||
$ npx linkinator ./docs | ||
npx linkinator ./docs | ||
``` | ||
@@ -107,3 +109,3 @@ | ||
```sh | ||
$ npx linkinator ./docs --recurse | ||
npx linkinator ./docs --recurse | ||
``` | ||
@@ -114,3 +116,3 @@ | ||
```sh | ||
$ npx linkinator ./docs --skip www.googleapis.com | ||
npx linkinator ./docs --skip www.googleapis.com | ||
``` | ||
@@ -121,3 +123,3 @@ | ||
```sh | ||
$ linkinator http://jbeckwith.com --skip '^(?!http://jbeckwith.com)' | ||
linkinator http://jbeckwith.com --skip '^(?!http://jbeckwith.com)' | ||
``` | ||
@@ -128,3 +130,3 @@ | ||
```sh | ||
$ linkinator ./docs --format CSV | ||
linkinator ./docs --format CSV | ||
``` | ||
@@ -135,3 +137,3 @@ | ||
```sh | ||
$ linkinator ./README.md --markdown | ||
linkinator ./README.md --markdown | ||
``` | ||
@@ -142,6 +144,7 @@ | ||
```sh | ||
$ linkinator "**/*.md" --markdown | ||
linkinator "**/*.md" --markdown | ||
``` | ||
### Configuration file | ||
You can pass options directly to the `linkinator` CLI, or you can define a config file. By default, `linkinator` will look for a `linkinator.config.json` file in the current working directory. | ||
@@ -167,6 +170,7 @@ | ||
```sh | ||
$ linkinator --config /some/path/your-config.json | ||
linkinator --config /some/path/your-config.json | ||
``` | ||
## GitHub Actions | ||
You can use `linkinator` as a GitHub Action as well, using [JustinBeckwith/linkinator-action](https://github.com/JustinBeckwith/linkinator-action): | ||
@@ -195,4 +199,6 @@ | ||
#### linkinator.check(options) | ||
### linkinator.check(options) | ||
Asynchronous method that runs a site wide scan. Options come in the form of an object that includes: | ||
- `path` (string|string[]) - A fully qualified path to the url to be scanned, or the path(s) to the directory on disk that contains files to be scanned. *required*. | ||
@@ -210,4 +216,6 @@ - `concurrency` (number) - The number of connections to make simultaneously. Defaults to 100. | ||
#### linkinator.LinkChecker() | ||
### linkinator.LinkChecker() | ||
Constructor method that can be used to create a new `LinkChecker` instance. This is particularly useful if you want to receive events as the crawler crawls. Exposes the following events: | ||
- `pagestart` (string) - Provides the url that the crawler has just started to scan. | ||
@@ -219,4 +227,6 @@ - `link` (object) - Provides an object with | ||
### Simple example | ||
### Examples | ||
#### Simple example | ||
```js | ||
@@ -256,3 +266,3 @@ const link = require('linkinator'); | ||
### Complete example | ||
#### Complete example | ||
@@ -317,8 +327,11 @@ In most cases you're going to want to respond to events, as running the check command can kinda take a long time. | ||
### Using a proxy | ||
This library supports proxies via the `HTTP_PROXY` and `HTTPS_PROXY` environment variables. This [guide](https://www.golinuxcloud.com/set-up-proxy-http-proxy-environment-variable/) provides a nice overview of how to format and set these variables. | ||
### Globbing | ||
You may have noticed in the example, when using a glob the pattern is encapsulated in quotes: | ||
```sh | ||
$ linkinator "**/*.md" --markdown | ||
linkinator "**/*.md" --markdown | ||
``` | ||
@@ -329,11 +342,15 @@ | ||
### Debugging | ||
Oftentimes when a link fails, it's an easy to spot typo, or a clear 404. Other times ... you may need more details on exactly what went wrong. To see a full call stack for the HTTP request failure, use `--verbosity DEBUG`: | ||
```sh | ||
$ linkinator https://jbeckwith.com --verbosity DEBUG | ||
linkinator https://jbeckwith.com --verbosity DEBUG | ||
``` | ||
### Controlling Output | ||
The `--verbosity` flag offers preset options for controlling the output, but you may want more control. Using [`jq`](https://stedolan.github.io/jq/) and `--format JSON` - you can do just that! | ||
```sh | ||
$ linkinator https://jbeckwith.com --verbosity DEBUG --format JSON | jq '.links | .[] | select(.state | contains("BROKEN"))' | ||
linkinator https://jbeckwith.com --verbosity DEBUG --format JSON | jq '.links | .[] | select(.state | contains("BROKEN"))' | ||
``` | ||
@@ -340,0 +357,0 @@ |
Sorry, the diff of this file is not supported yet
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
No contributors or author data
MaintenancePackage does not specify a list of contributors or an author in package.json.
Found 1 instance in 1 package
94470
1291
1
344
23
+ Addedescape-html@^1.0.3
+ Addedmime@^2.5.0
+ Addedescape-html@1.0.3(transitive)
+ Addedmime@2.6.0(transitive)
- Removedp-queue@^6.2.1
- Removedserve-handler@^6.1.3
- Removedbytes@3.0.0(transitive)
- Removedcontent-disposition@0.5.2(transitive)
- Removedeventemitter3@4.0.7(transitive)
- Removedmime-db@1.33.0(transitive)
- Removedmime-types@2.1.18(transitive)
- Removedp-finally@1.0.0(transitive)
- Removedp-queue@6.6.2(transitive)
- Removedp-timeout@3.2.0(transitive)
- Removedpath-is-inside@1.0.2(transitive)
- Removedpath-to-regexp@3.3.0(transitive)
- Removedrange-parser@1.2.0(transitive)
- Removedserve-handler@6.1.6(transitive)