url-reader - npm Package Compare versions

Comparing version 1.0.0 to 1.0.1

dist/core.d.ts

@@ -1,19 +0,12 @@
-interface IOptions {
-    urls: string[];
-    timeout: number;
-}
-interface IReaderResult {
-    title: string;
-    content: string;
-    length: number;
-    textContent: string;
-    excerpt: string;
-}
+import { IOptions, IReaderResult } from './interface';
 export declare class URLReader {
     private browser;
     private timeout;
+    private turndown;
     constructor();
     init(): Promise<void>;
     read(options: IOptions): Promise<IReaderResult[]>;
+    private readDoc;
+    private html2md;
 }
-export {};
+export default URLReader;

dist/core.js

@@ -0,10 +1,16 @@
+import { JSDOM } from 'jsdom';
+import Turndown from 'turndown';
 import puppeteer from 'puppeteer';
 import { Readability } from '@mozilla/readability';
-import { JSDOM } from 'jsdom';
+class URLReaderError extends Error {
+}
+;
 export class URLReader {
     browser;
     timeout;
+    turndown;
     constructor() {
         this.timeout = 60000;
         this.browser = null;
+        this.turndown = new Turndown();
     }
@@ -14,9 +20,11 @@ async init() {
             return;
-        this.browser = await puppeteer.launch();
+        this.browser = await puppeteer.launch({
+            headless: true,
+        });
     }
     async read(options) {
-        const { urls, timeout } = options;
+        const { urls, timeout, enableMarkdown = true, runScripts } = options;
         const { browser, timeout: defaultTimeout } = this;
         if (!browser)
-            throw new Error('browser is null');
+            throw new URLReaderError('browser is null.');
         const results = [];
@@ -26,3 +34,3 @@ for (const url of urls) {
             const res = await page.goto(url, {
-                timeout: timeout || defaultTimeout
+                timeout: timeout ?? defaultTimeout,
             });
@@ -33,13 +41,17 @@ const txt = await res?.text();
             const doc = new JSDOM(txt, {
-                url
+                url,
+                runScripts,
             });
-            const reader = new Readability(doc.window.document);
-            const article = reader.parse();
+            const article = await this.readDoc(doc.window.document);
+            let markdown = '';
+            if (enableMarkdown)
+                markdown = await this.html2md(article?.content ?? '');
             if (article) {
                 results.push({
+                    length: article.length,
                     title: article.title,
-                    content: article.content,
-                    length: article.length,
-                    textContent: article.textContent,
-                    excerpt: article.excerpt
+                    html: article.content,
+                    text: article.textContent,
+                    markdown,
+                    excerpt: article.excerpt,
                 });
@@ -51,2 +63,12 @@ }
     }
+    async readDoc(doc) {
+        const reader = new Readability(doc);
+        return reader.parse();
+    }
+    async html2md(html) {
+        if (!html)
+            return '';
+        return this.turndown?.turndown(html);
+    }
 }
+export default URLReader;

package.json

@@ -8,2 +8,3 @@ {
     "url-reader",
+    "urlReader",
     "json",
@@ -14,3 +15,3 @@ "html",
   ],
-  "version": "1.0.0",
+  "version": "1.0.1",
   "main": "./dist/index.js",
@@ -21,29 +22,31 @@ "author": "zac ma",
   "scripts": {
+    "start": "node ./dist/app.js",
+    "dev": "nodemon --exec tsx ./src/app.ts",
     "build": "tsc --project tsconfig.build.json --outDir dist/",
+    "lint": "yarn eslint .",
     "test": "jest"
   },
   "engines": {
-    "node": ">=20.0.0"
+    "node": ">=20.11.0"
   },
   "dependencies": {
     "@mozilla/readability": "^0.5.0",
+    "fastify": "^4.26.2",
     "jsdom": "^24.0.0",
-    "puppeteer": "^22.6.5"
+    "puppeteer": "^22.6.5",
+    "turndown": "^7.1.3"
   },
   "devDependencies": {
-    "@eslint/eslintrc": "^3.0.2",
-    "@eslint/js": "^9.0.0",
-    "@tsconfig/recommended": "^1.0.6",
+    "@eslint/js": "^9.1.1",
+    "@stylistic/eslint-plugin": "^1.7.2",
     "@types/jsdom": "^21.1.6",
     "@types/node": "^20.12.7",
-    "@typescript-eslint/eslint-plugin": "^6.4.0",
-    "eslint": "^9.0.0",
-    "eslint-config-standard-with-typescript": "^43.0.1",
-    "eslint-plugin-import": "^2.25.2",
-    "eslint-plugin-n": "^15.0.0 || ^16.0.0 ",
-    "eslint-plugin-promise": "^6.0.0",
+    "@types/turndown": "^5.0.4",
+    "eslint": "^9.1.1",
     "globals": "^15.0.0",
+    "tsx": "^4.7.3",
     "typescript": "^5.4.5",
-    "typescript-eslint": "^7.7.0"
+    "typescript-eslint": "^7.7.1",
+    "nodemon": "^3.1.0"
   }
 }

README.md

 # URL READER
 
-This project allows you to read the content of a URL.
+This project helps you to read the content of URLs, and return the title, length, html, text, markdown, excerpt.
 
+> "node": ">=20.11.0"
+
 ## Installation
@@ -9,13 +11,76 @@
 yarn install urlreader
+# or npm install urlreader
 ```
 
-### Troubleshooting
+## Usage
 
-- puppeteer
+```ts
+import URLReader from 'urlreader';
 
-the puppeteer will download the chromium browser
+const reader = new URLReader();
+await reader.init();
 
+const results = await reader.read({
+  urls: ['https://www.google.com'],
+  timeout: 10000, // ms, default: 60000
+  enableMarkdown: false, // default: true
+  runScripts: 'dangerously', // run the scripts included in the HTML and fetch remote resources, default is closed.
+});
+```
+
+Parsed Result:
+
+```ts
+interface IReaderResult {
+  title: string;
+  length: number;
+  html: string;
+  text: string;
+  markdown?: string;
+  excerpt: string;
+}
+```
+
+## Server
+
+* start server
+
 ```bash
+git clone https://github.com/yokingma/url-reader.git
+cd urlReader
 
+# default listen on port 3030
+yarn run start
 ```
 
+* api
+
+```txt
+GET /reader?url=https://www.google.com
+
+POST /reader
+{
+  urls: ['https://www.google.com', 'https://www.bing.com']
+}
+```
+
+## Docker
+
+```bash
+docker build -t urlreader . # urlreader is your image's tag name
+```
+
+## Tips
+
+- puppeteer
+When you install Puppeteer, it will automatically downloads a recent version of Chrome for Testing (~170MB macOS, ~282MB Linux, ~280MB Windows) and a chrome-headless-shell binary.
+
+## Troubleshooting
+
+- install error with puppeteer
+
+```txt
+Error [ERR_TLS_CERT_ALTNAME_INVALID]: Hostname/IP does not match certificate's altnames...
+```
+
+remove .npmrc file and re-install.

src/core.ts

@@ -0,21 +1,13 @@
+import { JSDOM } from 'jsdom';
+import Turndown from 'turndown';
 import puppeteer, { type Browser } from 'puppeteer';
 import { Readability } from '@mozilla/readability';
-import { JSDOM } from 'jsdom';
+import { IOptions, IReaderResult } from './interface';
 
-interface IOptions {
-  urls: string[]
-  timeout: number
-}
+class URLReaderError extends Error {};
 
-interface IReaderResult {
-  title: string
-  content: string
-  length: number
-  textContent: string
-  excerpt: string
-}
-
 export class URLReader {
-  private browser: null | Browser
-  private timeout: number
+  private browser: null | Browser;
+  private timeout: number;
+  private turndown: Turndown;
 
@@ -25,2 +17,3 @@ constructor() {
     this.browser = null;
+    this.turndown = new Turndown();
   }
@@ -30,9 +23,11 @@
     if (this.browser) return;
-    this.browser = await puppeteer.launch();
+    this.browser = await puppeteer.launch({
+      headless: true,
+    });
   }
 
   public async read(options: IOptions) {
-    const { urls, timeout } = options;
+    const { urls, timeout, enableMarkdown = true, runScripts } = options;
     const { browser, timeout: defaultTimeout } = this;
-    if (!browser) throw new Error('browser is null');
+    if (!browser) throw new URLReaderError('browser is null.');
     const results: IReaderResult[] = [];
@@ -42,24 +37,26 @@ for (const url of urls) {
       const res = await page.goto(url, {
-        timeout: timeout || defaultTimeout
-      })
-
+        timeout: timeout ?? defaultTimeout,
+      });
       const txt = await res?.text();
       if (!txt) continue;
-  
+
       const doc = new JSDOM(txt, {
-        url
+        url,
+        runScripts,
       });
-      const reader = new Readability(doc.window.document);
-      const article = reader.parse();
+      const article = await this.readDoc(doc.window.document);
+      let markdown = '';
+      if (enableMarkdown) markdown = await this.html2md(article?.content ?? '');
       if (article) {
         results.push({
+          length: article.length,
           title: article.title,
-          content: article.content,
-          length: article.length,
-          textContent: article.textContent,
-          excerpt: article.excerpt
-        })
+          html: article.content,
+          text: article.textContent,
+          markdown,
+          excerpt: article.excerpt,
+        });
       }
 
-      await page.close()
+      await page.close();
     }
@@ -69,2 +66,14 @@
   }
+
+  private async readDoc(doc: Document) {
+    const reader = new Readability(doc);
+    return reader.parse();
+  }
+
+  private async html2md(html: string) {
+    if (!html) return '';
+    return this.turndown?.turndown(html);
+  }
 }
+
+export default URLReader;

src/index.ts

@@ -1,1 +0,1 @@
-export * from './core';
+export * from './core';

tsconfig.json

 {
-  "extends": "@tsconfig/recommended/tsconfig.json",
   "compilerOptions": {
+    "rootDir": "./src",
     "declaration": true,
-    "module": "ES2022",
+    "module": "ESNext",
     "moduleResolution": "node",
-    "target": "ES2022",
+    "target": "ESNext",
     "outDir": "dist",
     "noImplicitAny": true,
+    "allowSyntheticDefaultImports": true,
     "strict": true,
-    // "noEmit": true,
+    "noEmit": false,
     "typeRoots": [
@@ -18,3 +19,4 @@ "node_modules/@types"
     "src/**/*.ts",
-    "test/**/*.ts", "tsconfig.build.json",
+    "test/**/*.ts",
+    "tsconfig.build.json",
   ],
@@ -25,4 +27,5 @@ "exclude": [
     "node_modules",
-    "jest.config.js"
+    "jest.config.js",
+    "eslint.config.js"
   ]
 }

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

13632

121.08%

Dev dependency count

11

-21.43%

Number of package files

20

81.82%

Lines of code

343

83.42%

Number of lines in readme file

86

309.52%

Worsened metrics

Dependency count

5

66.67%

Number of low supply chain risk alerts

2

100%

Number of medium supply chain risk alerts

1

Infinity%

Dependency changes

• + Added

  fastify@^4.26.2

• + Added

  turndown@^7.1.3

• + Added

  @fastify/ajv-compiler@3.6.0
  (transitive)

• + Added

  @fastify/error@3.4.1
  (transitive)

• + Added

  @fastify/fast-json-stringify-compiler@4.3.0
  (transitive)

• + Added

  @fastify/merge-json-schemas@0.1.1
  (transitive)

• + Added

  @mixmark-io/domino@2.2.0
  (transitive)

• + Added

  abstract-logging@2.0.1
  (transitive)

• + Added

  ajv@8.17.1
  (transitive)

• + Added

  ajv-formats@2.1.13.0.1
  (transitive)

• + Added

  atomic-sleep@1.0.0
  (transitive)

• + Added

  avvio@8.4.0
  (transitive)

• + Added

  cookie@0.7.2
  (transitive)

• + Added

  fast-content-type-parse@1.1.0
  (transitive)

• + Added

  fast-decode-uri-component@1.0.1
  (transitive)

• + Added

  fast-deep-equal@3.1.3
  (transitive)

• + Added

  fast-json-stringify@5.16.1
  (transitive)

• + Added

  fast-querystring@1.1.2
  (transitive)

• + Added

  fast-redact@3.5.0
  (transitive)

• + Added

  fast-uri@2.4.03.0.6
  (transitive)

• + Added

  fastify@4.29.1
  (transitive)

• + Added

  fastq@1.19.1
  (transitive)

• + Added

  find-my-way@8.2.2
  (transitive)

• + Added

  forwarded@0.2.0
  (transitive)

• + Added

  ipaddr.js@1.9.1
  (transitive)

• + Added

  json-schema-ref-resolver@1.0.1
  (transitive)

• + Added

  json-schema-traverse@1.0.0
  (transitive)

• + Added

  light-my-request@5.14.0
  (transitive)

• + Added

  on-exit-leak-free@2.1.2
  (transitive)

• + Added

  pino@9.7.0
  (transitive)

• + Added

  pino-abstract-transport@2.0.0
  (transitive)

• + Added

  pino-std-serializers@7.0.0
  (transitive)

• + Added

  process-warning@3.0.05.0.0
  (transitive)

• + Added

  proxy-addr@2.0.7
  (transitive)

• + Added

  quick-format-unescaped@4.0.4
  (transitive)

• + Added

  real-require@0.2.0
  (transitive)

• + Added

  require-from-string@2.0.2
  (transitive)

• + Added

  ret@0.4.3
  (transitive)

• + Added

  reusify@1.1.0
  (transitive)

• + Added

  rfdc@1.4.1
  (transitive)

• + Added

  safe-regex2@3.1.0
  (transitive)

• + Added

  safe-stable-stringify@2.5.0
  (transitive)

• + Added

  secure-json-parse@2.7.0
  (transitive)

• + Added

  set-cookie-parser@2.7.1
  (transitive)

• + Added

  sonic-boom@4.2.0
  (transitive)

• + Added

  split2@4.2.0
  (transitive)

• + Added

  thread-stream@3.1.0
  (transitive)

• + Added

  toad-cache@3.7.0
  (transitive)

• + Added

  turndown@7.2.0
  (transitive)