fetch-filecache-for-crawling - npm Package Compare versions

Comparing version 4.1.0 to 5.0.0

examples/example-etag.js

@@ -13,7 +13,6 @@ /**
 async function run() {
-  const etag = '"5421-55a8e11cc2280"';
+  const etag = '"5421-55a8e11cc2280-gzip"';
   console.log(`Fetch https://www.w3.org/TR/2012/REC-hr-time-20121217/ with etag ${etag}`);
   let resp = await fetch('https://www.w3.org/TR/2012/REC-hr-time-20121217/', {
-    refresh: 'once', // Needed because of invalid cache semantics in W3C servers
-    compress: false,
+    refresh: 'once',
     headers: {
@@ -28,4 +27,3 @@ 'If-None-Match': etag
   resp = await fetch('https://www.w3.org/TR/2012/REC-hr-time-20121217/', {
-    refresh: 'force',
-    compress: false,
+    refresh: 'force'
   });
@@ -32,0 +30,0 @@ console.log(`Received HTTP status ${resp.status} with etag ${resp.headers.get('etag')}`)

fetch-filecache.js

@@ -11,5 +11,3 @@ /**
 const filenamifyUrl = require('filenamify-url');
-const baseFetch = require('node-fetch');
-const Response = require('node-fetch').Response;
-const rimraf = require('rimraf');
+const { rimraf } = require('rimraf');
 const path = require('path');
@@ -162,3 +160,3 @@ const fs = require('fs');
  */
-async function fetch(url, options) {
+async function cacheFetch(url, options) {
   // We may modify request options in place, let's make a shallow copy
@@ -208,3 +206,3 @@ options = Object.assign({}, options);
     cacheFolderReset[config.cacheFolder] = true;
-    await rimraf(config.cacheFolder + '/*');
+    await rimraf(config.cacheFolder + '/*', { glob: true });
   }
@@ -447,3 +445,3 @@
     log('fetch and save response to cache');
-    return new Promise((resolve, reject) => {
+    return new Promise(async (resolve, reject) => {
       let writable = fs.createWriteStream(cacheFilename);
@@ -457,3 +455,13 @@ writable.on('close', _ => {
       writable.on('error', reject);
-      response.body.pipe(writable);
+      const reader = response.body.getReader();
+      while (true) {
+        const chunk = await reader.read();
+        if (chunk.done) {
+          writable.close();
+          break;
+        }
+        else {
+          writable.write(chunk.value);
+        }
+      }
     });
@@ -483,3 +491,3 @@ }
       try {
-        return await baseFetch(url, options);
+        return await fetch(url, options);
       }
@@ -540,5 +548,5 @@ catch (err) {
 
-module.exports = fetch;
+module.exports = cacheFetch;
 module.exports.setParameter = function (name, value) {
   globalConfig[name] = value;
 }

package.json

 {
   "name": "fetch-filecache-for-crawling",
-  "version": "4.1.0",
+  "version": "5.0.0",
   "description": "Implementation of a `fetch` that extends the implementation from `node-fetch` to add an HTTP cache using a local cache folder for crawling purpose.",
@@ -19,9 +19,8 @@ "main": "fetch-filecache.js",
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=18.0.0"
   },
   "dependencies": {
     "filenamify-url": "^2.1.2",
-    "node-fetch": "^2.6.0",
-    "rimraf": "^3.0.2"
+    "rimraf": "^5.0.1"
   }
 }

README.md

 # Implementation of fetch with a file-based HTTP cache for crawling purpose
 
 Node.js module that exports a `fetch` function that extends the implementation
-from `node-fetch` to add an HTTP cache using a local cache folder.
+of Node.js native `fetch` to add an HTTP cache using a local cache folder.
 
@@ -30,3 +30,3 @@ The code was developed for a particular scenario with specific requirements in
 
-Run `npm install fetch-filecache-for-crawling`
+Run `npm install fetch-filecache-for-crawling`.
 
@@ -33,0 +33,0 @@ ## Usage

Fixed alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

30697

increased by3.67%

Dependency count

2

decreased by-33.33%

Number of package files

13

increased by18.18%

Lines of code

669

increased by5.69%

Worsened metrics

Number of medium supply chain risk alerts

27

increased by17.39%

Dependency changes

• + Added@isaacs/cliui@8.0.2(transitive)

• + Added@pkgjs/parseargs@0.11.0(transitive)

• + Addedansi-regex@5.0.16.1.0(transitive)

• + Addedansi-styles@4.3.06.2.1(transitive)

• + Addedbrace-expansion@2.0.1(transitive)

• + Addedcolor-convert@2.0.1(transitive)

• + Addedcolor-name@1.1.4(transitive)

• + Addedcross-spawn@7.0.6(transitive)

• + Addedeastasianwidth@0.2.0(transitive)

• + Addedemoji-regex@8.0.09.2.2(transitive)

• + Addedforeground-child@3.3.0(transitive)

• + Addedglob@10.4.5(transitive)

• + Addedis-fullwidth-code-point@3.0.0(transitive)

• + Addedisexe@2.0.0(transitive)

• + Addedjackspeak@3.4.3(transitive)

• + Addedlru-cache@10.4.3(transitive)

• + Addedminimatch@9.0.5(transitive)

• + Addedminipass@7.1.2(transitive)

• + Addedpackage-json-from-dist@1.0.1(transitive)

• + Addedpath-key@3.1.1(transitive)

• + Addedpath-scurry@1.11.1(transitive)

• + Addedrimraf@5.0.10(transitive)

• + Addedshebang-command@2.0.0(transitive)

• + Addedshebang-regex@3.0.0(transitive)

• + Addedsignal-exit@4.1.0(transitive)

• + Addedstring-width@4.2.35.1.2(transitive)

• + Addedstrip-ansi@6.0.17.1.0(transitive)

• + Addedwhich@2.0.2(transitive)

• + Addedwrap-ansi@7.0.08.1.0(transitive)

• - Removednode-fetch@^2.6.0

• - Removedbrace-expansion@1.1.11(transitive)

• - Removedconcat-map@0.0.1(transitive)

• - Removedfs.realpath@1.0.0(transitive)

• - Removedglob@7.2.3(transitive)

• - Removedinflight@1.0.6(transitive)

• - Removedinherits@2.0.4(transitive)

• - Removedminimatch@3.1.2(transitive)

• - Removednode-fetch@2.7.0(transitive)

• - Removedonce@1.4.0(transitive)

• - Removedpath-is-absolute@1.0.1(transitive)

• - Removedrimraf@3.0.2(transitive)

• - Removedtr46@0.0.3(transitive)

• - Removedwebidl-conversions@3.0.1(transitive)

• - Removedwhatwg-url@5.0.0(transitive)

• - Removedwrappy@1.0.2(transitive)

• Updatedrimraf@^5.0.1