nodejs-file-downloader - npm Package Compare versions

Comparing version 4.5.3 to 4.6.0

CHANGELOG.md

@@ -0,1 +1,7 @@
+
+## 4.6.0 10/06/2021
+
+### Feature
+- Added the ability to skip downloads if the same file name exists.
+
 ## 4.5.3 09/05/2021
@@ -2,0 +8,0 @@

Download.js

@@ -13,3 +13,3 @@ const fs = require('fs');
 const writeFile = util.promisify(fs.writeFile);
-const { deduceFileName } = require('./utils/fileName');
+const { deduceFileName,exists } = require('./utils/fileName');
 const unlink = util.promisify(fs.unlink)
@@ -19,2 +19,3 @@ const rename = util.promisify(fs.rename)
 
+
 module.exports = class Download {
@@ -28,3 +29,3 @@
    * @param {string} [config.fileName = undefined] 
-   * @param {boolean} [config.cloneFiles=true] 
+   * @param {boolean | string} [config.cloneFiles=true] //Can also be the string "skip"
    * @param {number} [config.timeout=6000]   
@@ -83,2 +84,9 @@ * @param {object} [config.headers = undefined]
 
+        // bailout skip file
+        if (this.config.fileName && this.config.cloneFiles === 'skip') {
+            if (await exists(this.config.directory + '/' + this.config.fileName)) {
+                return;
+            }
+        }
+
         try {
@@ -142,2 +150,9 @@ const { dataStream, originalResponse } = await this._request();
 
+            if (finalName === this.config.fileName &&
+                this.config.cloneFiles === 'skip' &&
+                await exists(this.config.directory + '/' + this.config.fileName)) {
+                // will skip this request
+                return;
+            }
+
             if (this.config.onBeforeSave) {
@@ -331,4 +346,3 @@ const clientOverideName = await this.config.onBeforeSave(finalName)
 
-        if (this.config.cloneFiles) {
-
+        if (this.config.cloneFiles === true) {
             var fileProcessor = new FileProcessor({ useSynchronousMode: this.config.useSynchronousMode, fileName, path: this.config.directory })
@@ -335,0 +349,0 @@

Downloader.js

@@ -1,3 +0,1 @@
-
-
 const rpur = require('./utils/rpur')
@@ -22,3 +20,3 @@ const { capitalize } = require('./utils/string')
   cloneFiles: {
-    type: 'boolean',
+    type: ['boolean', 'string'],
     mandatory: false
@@ -39,7 +37,8 @@ },
 
-      if (typeof config[prop] !== configTypes[prop].type)
-        generateTypeError(prop)
+        if (-1 === [].concat(configTypes[prop].type).indexOf(typeof config[prop]))
+            generateTypeError(prop)
     }
-    if (config.hasOwnProperty(prop) && typeof config[prop] !== configTypes[prop].type)
-      generateTypeError(prop)
+
+      if (config.hasOwnProperty(prop) &&  -1 === [].concat(configTypes[prop].type).indexOf(typeof config[prop]))
+          generateTypeError(prop)
   }
@@ -58,3 +57,4 @@ }
    * @param {string} [config.fileName = undefined] 
-   * @param {boolean} [config.cloneFiles=true] 
+   * @param {boolean | string} [config.cloneFiles=true] true will create a duplicate. false will overwrite the existing file. The string "skip"
+   * will cause the downloader to skip the download process in case a file with the same name already exists.
    * @param {number} [config.timeout=6000] 
@@ -61,0 +61,0 @@ * @param {number} [config.maxAttempts=1]

main.test.js

@@ -1,2 +0,1 @@
-
 // const axios = require('axios');
@@ -155,2 +154,40 @@ const expect = require('expect')
 
+    it ('Should skip same name request', async () => {
+        let deducedName;
+        const host = randomHost()
+        let downloadTimes = 0;
+        nock(`http://www.${host}.com`)
+            .get('/contentType')
+            .reply(200, (uri, requestBody) => {
+                downloadTimes += 1
+                return fs.createReadStream(Path.join(__dirname, 'fixtures/Desert.jpg'))
+                //   fs.readFile(Path.join(__dirname, 'fixtures/Desert.jpg'), cb) // Error-first callback
+            }, {
+                'Content-Type': 'image/jpeg',
+                'Content-Length': '23642'
+            })
+        const downloader = new Downloader({
+            url: `http://www.${host}.com/contentType`,
+            directory: "./downloads",
+            fileName: "testfile.jpg",
+            cloneFiles: 'skip'
+        })
+
+        const downloader2 = new Downloader({
+            url: `http://www.${host}.com/contentType`,
+            directory: "./downloads",
+            fileName: "testfile.jpg",
+            cloneFiles: 'skip'
+        })
+
+        // debugger;
+        await downloader.download();
+        await downloader.download();
+        await downloader2.download();
+        // debugger
+        await verifyFile('./downloads/testfile.jpg', 23642);
+        expect(downloadTimes).toBe(1);
+        //  console.log(verify)
+    })
+
     it('Should get NaN in onProgress', async () => {
@@ -829,3 +866,3 @@
         // return;
-        // }  
+        // }
 
@@ -914,3 +951,3 @@ // throw new Error();
         // return;
-        // }  
+        // }
 
@@ -1087,3 +1124,2 @@ // throw new Error();
 
-   
 
@@ -1093,2 +1129,3 @@
 
+
     // it('Should timeout during stream, twice', async function () {
@@ -1270,5 +1307,5 @@ //     let error;
  /**
-     * 
-     * @param {string} path 
-     * @param {number} [size] 
+     *
+     * @param {string} path
+     * @param {number} [size]
      */
@@ -1275,0 +1312,0 @@ function verifyFile(path, size) {

package.json

 {
   "name": "nodejs-file-downloader",
-  "version": "4.5.3",
+  "version": "4.6.0",
   "description": "A file downloader for NodeJs",
@@ -5,0 +5,0 @@ "main": "Downloader.js",

readme.md

@@ -137,2 +137,3 @@ nodejs-file-downloader is a simple utility for downloading files. It hides the complexity of dealing with streams, redirects, paths and duplicate file names. Can automatically repeat failed downloads.
 ```
+You can also set cloneFiles to the string "skip". This will cause the downloader to simply skip a download, If a file with the same name exists.
 
@@ -139,0 +140,0 @@  

utils/fileName.js

 const sanitize = require('sanitize-filename');
 const path = require('path');
 var mime = require('mime-types')
+const { promises: Fs } = require('fs')
 
@@ -10,2 +11,3 @@
 
+
 /**
@@ -17,6 +19,6 @@ *
 function deduceFileNameFromUrl(url) {
-    // debugger;
-    const cleanUrl = removeQueryString(url);
-    const baseName = sanitize(path.basename(cleanUrl));
-    return baseName;
+  // debugger;
+  const cleanUrl = removeQueryString(url);
+  const baseName = sanitize(path.basename(cleanUrl));
+  return baseName;
 
@@ -35,21 +37,21 @@ }
   // debugger
-    //First option
-    const fileNameFromContentDisposition = getFileNameFromContentDisposition(headers['content-disposition'] || headers['Content-Disposition']);
-    // console.log('filenamecontentdisposition', fileNameFromContentDisposition)
-    if (fileNameFromContentDisposition) return fileNameFromContentDisposition;
+  //First option
+  const fileNameFromContentDisposition = getFileNameFromContentDisposition(headers['content-disposition'] || headers['Content-Disposition']);
+  // console.log('filenamecontentdisposition', fileNameFromContentDisposition)
+  if (fileNameFromContentDisposition) return fileNameFromContentDisposition;
 
-    // debugger;
-    //Second option
-    if (path.extname(url)) {//First check if the url even has an extension
-        const fileNameFromUrl = deduceFileNameFromUrl(url);
-        if (fileNameFromUrl) return fileNameFromUrl;
-    }
+  // debugger;
+  //Second option
+  if (path.extname(url)) {//First check if the url even has an extension
+    const fileNameFromUrl = deduceFileNameFromUrl(url);
+    if (fileNameFromUrl) return fileNameFromUrl;
+  }
 
-    //Third option
-    const fileNameFromContentType = getFileNameFromContentType(headers['content-type'] || headers['Content-Type'],url)
-    if (fileNameFromContentType) return fileNameFromContentType
+  //Third option
+  const fileNameFromContentType = getFileNameFromContentType(headers['content-type'] || headers['Content-Type'], url)
+  if (fileNameFromContentType) return fileNameFromContentType
 
 
-    //Fallback option
-    return sanitize(url)
+  //Fallback option
+  return sanitize(url)
 
@@ -61,45 +63,54 @@
 function removeQueryString(url) {
-    return url.split(/[?#]/)[0];
+  return url.split(/[?#]/)[0];
 }
 
-function getFileNameFromContentType(contentType,url) {
+function getFileNameFromContentType(contentType, url) {
 
-    // var contentType = this.response.headers['content-type'] || this.response.headers['Content-Type'];
-    // console.log(contentType)
-    let extension = mime.extension(contentType)
+  // var contentType = this.response.headers['content-type'] || this.response.headers['Content-Type'];
+  // console.log(contentType)
+  let extension = mime.extension(contentType)
 
-    url = removeQueryString(url);
-    const fileNameWithoutExtension = removeExtension(path.basename(url));
-    return `${sanitize(fileNameWithoutExtension)}.${extension}`;
+  url = removeQueryString(url);
+  const fileNameWithoutExtension = removeExtension(path.basename(url));
+  return `${sanitize(fileNameWithoutExtension)}.${extension}`;
+}
+
+
+function getFileNameFromContentDisposition(contentDisposition) {
+  // debugger;
+  // const contentDisposition = this.response.headers['content-disposition'] || this.response.headers['Content-Disposition'];
+  if (!contentDisposition || !contentDisposition.includes('filename=')) {
+    return "";
   }
+  let filename = "";
+  var filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
+  var matches = filenameRegex.exec(contentDisposition);
+  if (matches != null && matches[1]) {
+    filename = matches[1].replace(/['"]/g, '');
+  }
 
+  return filename ? sanitize(filename) : "";
+}
 
-  function getFileNameFromContentDisposition(contentDisposition) {
-    // debugger;
-    // const contentDisposition = this.response.headers['content-disposition'] || this.response.headers['Content-Disposition'];
-    if (!contentDisposition || !contentDisposition.includes('filename=')) {
-      return "";
-    }
-    let filename = "";
-    var filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
-    var matches = filenameRegex.exec(contentDisposition);
-    if (matches != null && matches[1]) {
-      filename = matches[1].replace(/['"]/g, '');
-    }
-
-    return filename ? sanitize(filename) : "";
+function removeExtension(str) {
+  // debugger;
+  const arr = str.split('.');
+  if (arr.length == 1) {
+    return str;
   }
+  return arr.slice(0, -1).join('.')
 
-  function removeExtension(str) {
-    // debugger;
-    const arr = str.split('.');
-    if (arr.length == 1) {
-      return str;
-    }
-    return arr.slice(0, -1).join('.')
 
 
+}
 
+async function exists(path) {
+  try {
+    await Fs.access(path)
+    return true
+  } catch {
+    return false
   }
+}
 
-module.exports = { deduceFileName }
+module.exports = { deduceFileName, exists }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

195960

increased by1.32%

Lines of code

2260

increased by2.59%

Number of lines in readme file

299

increased by0.34%

Worsened metrics

Number of low supply chain risk alerts

6

increased by20%

No dependency changes