nodejs-file-downloader - npm Package Compare versions

Comparing version 1.0.5 to 1.1.0

Downloader.js

@@ -6,2 +6,3 @@ const fs = require('fs');
 const util = require('util');
+var HttpsProxyAgent = require('https-proxy-agent');
 const { EventEmitter } = require('events')
@@ -14,2 +15,3 @@ const path = require('path');
 const mkdir = util.promisify(fs.mkdir);
+const writeFile = util.promisify(fs.writeFile);
 
@@ -61,7 +63,11 @@
    * @param {string} [config.fileName] 
-   * @param {boolean} [config.cloneFiles] 
+   * @param {boolean} [config.cloneFiles=true] 
+   * @param {number} [config.timeout=6000] 
+   * @param {object} [config.headers] 
+   * @param {string} [config.proxy] 
+   * @param {string} [config.auth] 
    */
   constructor(config) {
     super();
-    if(!config || typeof config !== 'object'){
+    if (!config || typeof config !== 'object') {
       throw new Error('Must provide a valid config object')
@@ -74,3 +80,7 @@ }
       fileName: null,
+      timeout: 6000,
+      // proxy: null,
+      // auth:null,
       cloneFiles: true,
+      shouldBufferResponse: false
     }
@@ -88,12 +98,31 @@
 
+    // this.config.httpsAgent = new HttpsProxyAgent(this.config.proxy)
+
   }
 
-  async createReadStream(url) {
+  /**
+   * 
+   * @param {boolean} shouldBuffer 
+   * @return {Promise<axios.AxiosResponse>}
+   */
+  async makeRequest(shouldBuffer) {
+
+    const httpsAgent = this.config.proxy ? new HttpsProxyAgent(this.config.proxy) : null; 
+    // debugger;
     const response = await axios({
       method: 'get',
       url: this.config.url,
-      responseType: 'stream'
+      timeout: this.config.timeout,
+      headers: this.config.headers,
+      httpsAgent,
+      responseType: shouldBuffer ? 'arraybuffer' : 'stream'
     })
-    // console.log(response.constructor)
-    // debugger;
+
+    return response;
+  }
+
+  async createReadStream() {
+
+    const response = await this.makeRequest(false)
+
     if (this._events.response) {
@@ -104,5 +133,3 @@ this.emit('response', response)
     this.fileSize = parseInt(contentLength);
-    // console.log('content-length',response.headers['content-length'])
-    // console.log('Content-Length',response.headers['Content-Length'])
-    // debugger;
+
     this.response = response;
@@ -118,4 +145,14 @@ return response.data;
 
+  async downloadAndBuffer() {
+    // debugger;
+    const response = await this.makeRequest(true);
+    this.response = response;
+    const fileName = await this.getFinalFileName();
+    // const write = this.createWriteStream(`${this.config.directory}/${fileName}`)
+    await writeFile(`${this.config.directory}/${fileName}`, response.data)
 
+  }
+
   download() {
+    if (this.config.shouldBufferResponse) return this.downloadAndBuffer()
     // debugger;
@@ -127,26 +164,4 @@ const that = this;
         const read = await this.createReadStream(this.config.url);
-        let fileName;
-        if (this.config.fileName) {
-          fileName = this.config.fileName
-        } else {
-          fileName = this.deduceFileName()
-        }
-        // debugger;
-        var fileProcessor = new FileProcessor({ fileName, path: this.config.directory })
-        // debugger;
-        if (! await fileProcessor.pathExists(this.config.directory)) {
-          // debugger;
-          try {
-            await mkdir(this.config.directory, { recursive: true });
-          } catch (error) {
-            // debugger;
-          }
+        const fileName = await this.getFinalFileName();
 
-        }
-        if (this.config.cloneFiles) {
-
-
-          // debugger;
-          fileName = await fileProcessor.getAvailableFileName()
-        }
         const progress = new Transform({
@@ -156,7 +171,7 @@ // writableObjectMode: true,
           transform(chunk, encoding, callback) {
-           
+
             that.currentDataSize += chunk.byteLength;
-            
+
             that.percentage = ((that.currentDataSize / that.fileSize) * 100).toFixed(2)
-           
+
             if (that._events.progress) {
@@ -171,12 +186,5 @@ that.emit('progress', that.percentage, chunk);
         const write = this.createWriteStream(`${this.config.directory}/${fileName}`)
-        // write.on('error',(error)=>{console.log('error from write',error)})
-        // console.log(write)
-        // setTimeout(()=>{
-        //   read.unpipe();
-        //    setTimeout(()=>{
-           
-        //    },5000)
-        // },5000)
-        await pipeline(read, progress,write )
 
+        await pipeline(read, progress, write)
+
         resolve();
@@ -189,9 +197,40 @@ } catch (error) {
 
-  /**
-   * Returns the file name from content-disposition if exists. Empty string otherwise.
-   * @return {string} filename
-   */
-  getFileNameFromContentDisposition(contentDisposition) {
+  async getFinalFileName() {
+    let fileName;
+    if (this.config.fileName) {
+      fileName = this.config.fileName
+    } else {
+      fileName = this.deduceFileName(this.config.url, this.response.headers)
+    }
+    // debugger;
+    var fileProcessor = new FileProcessor({ fileName, path: this.config.directory })
+    // debugger;
+    // if (! await fileProcessor.pathExists(this.config.directory)) {
+    if (!fileProcessor.pathExists(this.config.directory)) {
+      // debugger;
+      try {
+        await mkdir(this.config.directory, { recursive: true });
+      } catch (error) {
+        // debugger;
+      }
 
+    }
+    if (this.config.cloneFiles) {
 
+
+      // debugger;
+      // fileName = await fileProcessor.getAvailableFileName()
+      fileName = fileProcessor.getAvailableFileName()
+    }
+
+    return fileName;
+  }
+
+
+  getFileNameFromContentDisposition(contentDisposition) {
+    // debugger;
+    // const contentDisposition = this.response.headers['content-disposition'] || this.response.headers['Content-Disposition'];
+    if (!contentDisposition || !contentDisposition.includes('filename=')) {
+      return "";
+    }
     let filename = "";
@@ -204,34 +243,78 @@ var filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
 
-    return filename;
+    return filename ? sanitize(filename) : "";
   }
 
+  getFileNameFromContentType(contentType) {
 
+    // var contentType = this.response.headers['content-type'] || this.response.headers['Content-Type'];
+    // console.log(contentType)
+    let extension = mime.extension(contentType)
+
+    const url = this.removeQueryString(this.config.url);
+    const fileNameWithoutExtension = this.removeExtension(path.basename(url));
+    return `${sanitize(fileNameWithoutExtension)}.${extension}`;
+  }
+
+
+  removeQueryString(url) {
+    return url.split(/[?#]/)[0];
+  }
+
+  removeExtension(str) {
+    // debugger;
+    const arr = str.split('.');
+    if (arr.length == 1) {
+      return str;
+    }
+    return arr.slice(0, -1).join('.')
+
+
+
+  }
+
+
   /**
+   * 
+   * @param {string} url 
+   * @return {string} fileName
+   */
+  deduceFileNameFromUrl(url) {
+    // debugger;
+    const cleanUrl = this.removeQueryString(url);
+    const baseName = sanitize(path.basename(cleanUrl));
+    return baseName;
+
+  }
+
+
+  /**
    * Deduce the fileName, covering various scenarios.
-   * @return {string} sanitizedBaseName
+   * @param {string} url
+   * @param {Object} headers
+   * @return {string} fileName
    */
-  deduceFileName() {
+  deduceFileName(url, headers) {
 
-    const headers = this.response.headers;
-    const contentDisposition = headers['content-disposition'] || headers['Content-Disposition'];
-    const contentType = this.response.headers['content-type'] || headers['Content-Type'];
-    // console.log('content-type',contentType)
-    if (contentDisposition && contentDisposition.includes('filename=')) {
-      return this.getFileNameFromContentDisposition(contentDisposition);
-    }
-    // console.log('content-disposition',contentDisposition)
-    const baseName = path.basename(this.config.url);
-    const extension = path.extname(baseName);
-    const sanitizedBaseName = sanitize(baseName)
-    if (extension) {
-      return sanitizedBaseName;
-    }
 
-    const extensionFromContentType = mime.extension(contentType)
-    if (extensionFromContentType) {
-      return `${sanitizedBaseName}.${extensionFromContentType}`
+    //First option
+    const fileNameFromContentDisposition = this.getFileNameFromContentDisposition(headers['content-disposition'] || headers['Content-Disposition']);
+    // console.log('filenamecontentdisposition', fileNameFromContentDisposition)
+    if (fileNameFromContentDisposition) return fileNameFromContentDisposition;
+
+    // debugger;
+    //Second option
+    if (path.extname(url)) {//First check if the url even has an extension
+      const fileNameFromUrl = this.deduceFileNameFromUrl(url);
+      if (fileNameFromUrl) return fileNameFromUrl;
     }
 
-    return sanitizedBaseName;
+    //Third option
+    const fileNameFromContentType = this.getFileNameFromContentType(headers['content-type'] || headers['Content-Type'])
+    if (fileNameFromContentType) return fileNameFromContentType
 
+
+    //Fallback option
+    return sanitize(url)
+
+
   }
@@ -238,0 +321,0 @@ }

Downloader.test.js

@@ -15,2 +15,9 @@
 
+    // beforeEach((done) => {
+    //     rimraf.sync("./downloads");
+    //     // console.log('done')
+    //     // deleteFolderRecursive('./downloads')
+    //     done();
+    // })
+
     before((done) => {
@@ -50,4 +57,4 @@ rimraf.sync("./downloads");
 
-    it('Should download a picture', async () => {
-        mock.onGet("/Desert.jpg").reply(
+    it('Should download a picture and use content-type', async () => {
+        mock.onGet("/contentType").reply(
             200,
@@ -62,3 +69,3 @@ fs.createReadStream(Path.join(__dirname, 'fixtures/Desert.jpg')),
         const downloader = new Downloader({
-            url: '/Desert.jpg',
+            url: '/contentType',
             directory: "./downloads",
@@ -83,3 +90,3 @@ cloneFiles: false,
 
-        await verifyFile('./downloads/Desert.jpg', 845941);
+        await verifyFile('./downloads/contentType.jpeg', 845941);
         //  console.log(verify)
@@ -154,3 +161,3 @@
             url: '/Desert.jpg',
-            directory: "./downloads",
+            directory: "./downloads/May/2020",
             // cloneFiles: true
@@ -162,3 +169,3 @@ })
 
-        await verifyFile('./downloads/Desert2.jpg', 845941);
+        await verifyFile('./downloads/May/2020/Desert2.jpg', 845941);
         //  console.log(verify)
@@ -219,45 +226,98 @@
 
+    it('Should download a picture and get the name from content-disposition ', async () => {
+        // rimraf.sync('./')
+        mock.onGet("/contentDisposition").reply(
+            200,
+            fs.createReadStream(Path.join(__dirname, 'fixtures/Hydrangeas.jpg')),
+            {
+                'Content-Disposition':'Content-Disposition: attachment; filename="contentDispositionFile.jpg"'
+                // 'Content-Type': 'image/jpeg',
+                // 'Content-Length': '845941'
+            }
+
+        )
+        const downloader = new Downloader({
+            url: '/contentDisposition',
+            directory: "./downloads"            
+        })
+        // .on('progress',(p)=>{})
+        //   console.log(downloader)
+        // debugger;
+        await downloader.download();
+
+        await verifyFile('./downloads/contentDispositionFile.jpg');
+        //  console.log(verify)
+
+        console.log('Download complete')
+    })
+
+
+    it('Should download a picture with a querystring after the extension ', async () => {
+        
+        mock.onGet("/Hydrangeas.jpg?width=400&height=300").reply(
+            200,
+            fs.createReadStream(Path.join(__dirname, 'fixtures/Hydrangeas.jpg')),
+            {
+                'Content-Type': 'image/jpeg',
+                // 'Content-Length': '845941'
+            }
+
+        )
+        const downloader = new Downloader({
+            url: '/Hydrangeas.jpg?width=400&height=300',
+            directory: "./downloads"            
+        }).on('progress',(p)=>{})
+        //   console.log(downloader)
+        // debugger;
+        await downloader.download();
+
+        await verifyFile('./downloads/Hydrangeas.jpg');
+        //  console.log(verify)
+
+        console.log('Download complete')
+    })
+
     
 
-    // it('Should download two pictures, with name appending', async () => {
-    //     try {
-    //         mock.onGet("/Koala.jpg").reply(
-    //             200,
-    //             fs.createReadStream(Path.join(__dirname, 'fixtures/Koala.jpg')),
-    //             {
-    //                 'Content-Type': 'image/jpeg',
-    //                 'Content-Length': '780831'
-    //             }
+    it('Should download two pictures, with name appending', async () => {
+        try {
+            mock.onGet("/Koala.jpg").reply(
+                200,
+                fs.createReadStream(Path.join(__dirname, 'fixtures/Koala.jpg')),
+                {
+                    'Content-Type': 'image/jpeg',
+                    'Content-Length': '780831'
+                }
 
-    //         )
-    //         const downloader = new Downloader({
-    //             url: '/Koala.jpg',
-    //             directory: "./downloads",
-    //             cloneFiles: false
-    //         })
-    //         //   console.log(downloader)
-    //         // debugger;
-    //         await downloader.download();
+            )
+            const downloader = new Downloader({
+                url: '/Koala.jpg',
+                directory: "./downloads",
+                cloneFiles: false
+            })
+            //   console.log(downloader)
+            // debugger;
+            await downloader.download();
 
-    //         await verifyFile('./downloads/Koala.jpg', 780831);
+            await verifyFile('./downloads/Koala.jpg', 780831);
 
-    //         const downloader2 = new Downloader({
-    //             url: '/Koala.jpg',
-    //             directory: "./downloads",
-    //         })
-    //         //   console.log(downloader)
-    //         // debugger;
-    //         await downloader2.download();
+            const downloader2 = new Downloader({
+                url: '/Koala.jpg',
+                directory: "./downloads",
+            })
+            //   console.log(downloader)
+            // debugger;
+            await downloader2.download();
 
 
-    //         // await verifyFile('./downloads/Koala2.jpg', 780831);
-    //         //  console.log(verify)
+            // await verifyFile('./downloads/Koala2.jpg', 780831);
+            //  console.log(verify)
 
-    //         console.log('Download complete')
-    //     } catch (error) {
-    //         console.log(error)
-    //         throw error
-    //     }
+            console.log('Download complete')
+        } catch (error) {
+            console.log(error)
+            throw error
+        }
 
-    // })
+    })
 
@@ -264,0 +324,0 @@

FileProcessor.js

@@ -18,32 +18,42 @@ const path = require('path');
 
-    async getAvailableFileName() {
+    // async getAvailableFileName() {
+    getAvailableFileName() {
 
-        return await this.createNewFileName(this.originalFileName);
+        // return await this.createNewFileName(this.originalFileName);
+        return this.createNewFileName(this.originalFileName);
     }
 
-    pathExists(path) {
-        return new Promise((resolve, reject) => {
-            fs.open(path, 'r', (err, fd) => {
-                if (err) {
-                    // debugger;
-                    if (err.code === 'ENOENT') {
-                        return resolve(false)
-                    }
+    // pathExists(path) {
+    //     return new Promise((resolve, reject) => {
+    //         fs.open(path, 'r', (err, fd) => {
+    //             if (err) {
+    //                 // debugger;
+    //                 if (err.code === 'ENOENT') {
+    //                     return resolve(false)
+    //                 }
 
-                    reject(err)
-                } else {
-                    // debugger;
-                    resolve(true)
-                }
+    //                 reject(err)
+    //             } else {
+    //                 // debugger;
+    //                 resolve(true)
+    //             }
 
 
-            });
-        })
+    //         });
+    //     })
 
+    // }
+    pathExists(path) {
+        if(fs.existsSync(path)) return true;
+
+        return false;
+
     }
 
-    async createNewFileName(fileName, counter = 1) {
+    // async createNewFileName(fileName, counter = 1) {
+    createNewFileName(fileName, counter = 1) {
 
 
-        if (! await this.fileNameExists(fileName)) {
+        // if (! await this.fileNameExists(fileName)) {
+        if (!this.fileNameExists(fileName)) {
             // console.log('new file name', newFileName)
@@ -56,3 +66,4 @@ return fileName;
 
-        return await this.createNewFileName(newFileName, counter);
+        // return await this.createNewFileName(newFileName, counter);
+        return this.createNewFileName(newFileName, counter);
 
@@ -59,0 +70,0 @@ }

package.json

 {
   "name": "nodejs-file-downloader",
-  "version": "1.0.5",
+  "version": "1.1.0",
   "description": "A file fownloader for NodeJs",
@@ -24,2 +24,3 @@ "main": "Downloader.js",
     "axios": "^0.19.2",
+    "https-proxy-agent": "^5.0.0",
     "mime-types": "^2.1.27",
@@ -31,4 +32,4 @@ "sanitize-filename": "^1.6.3"
     "axios-mock-adapter": "^1.18.1",
-    "expect": "^26.0.1",   
-    "mocha": "^7.1.2",    
+    "expect": "^26.0.1",
+    "mocha": "^7.1.2",
     "rimraf": "^3.0.2"
@@ -35,0 +36,0 @@ },

New alerts

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

2808021

increased by27.16%

Number of package files

11

increased by22.22%

Lines of code

574

increased by25.05%

Worsened metrics

Dependency count

4

increased by33.33%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedhttps-proxy-agent@^5.0.0

• + Addedagent-base@6.0.2(transitive)

• + Addeddebug@4.3.5(transitive)

• + Addedhttps-proxy-agent@5.0.1(transitive)

• + Addedms@2.1.2(transitive)