nodejs-file-downloader - npm Package Compare versions

Comparing version 1.2.3 to 2.0.0

Downloader.js

@@ -6,11 +6,9 @@ const fs = require('fs');
 const util = require('util');
-var HttpsProxyAgent = require('https-proxy-agent');
+// var HttpsProxyAgent = require('https-proxy-agent');
 const { EventEmitter } = require('events')
-const path = require('path');
-const sanitize = require('sanitize-filename');
 const FileProcessor = require('./FileProcessor');
-var mime = require('mime-types')
 const pipeline = util.promisify(stream.pipeline);
 const mkdir = util.promisify(fs.mkdir);
 const writeFile = util.promisify(fs.writeFile);
+const {deduceFileName} = require('./fileName')
 
@@ -65,4 +63,4 @@
    * @param {object} [config.headers] 
-   * @param {string} [config.proxy] 
-   * @param {string} [config.auth] 
+   * @param {object} [config.httpsAgent] 
+   * @param {boolean} [config.shouldBufferResponse = false] 
    * @param {boolean} [config.useSynchronousMode = false] 
@@ -82,4 +80,3 @@ */
       useSynchronousMode:false,
-      // proxy: null,
-      // auth:null,
+      proxy: null,
       cloneFiles: true,
@@ -104,10 +101,46 @@ shouldBufferResponse: false
   /**
-   * 
-   * @param {boolean} shouldBuffer 
    * @return {Promise<axios.AxiosResponse>}
    */
-  async makeRequest(shouldBuffer) {
+  async request(){
+    const response = await this._makeRequest();
+    this.response = response;
+    if (this._events.response) {
+      this.emit('response', response)
+    }
+    const contentLength = response.headers['content-length'] || response.headers['Content-Length'];
+    this.fileSize = parseInt(contentLength);
+    return response; 
 
-    const httpsAgent = this.config.proxy ? new HttpsProxyAgent(this.config.proxy) : null; 
+  }
+
+  /**
+   * @return {Promise<void>}
+   */
+  async save(){
+    if(this.config.shouldBufferResponse){
+      // debugger;
+      return this._saveFromBuffer(this.response.data);
+    }
     // debugger;
+    await this._saveFromReadableStream(this.response.data);
+  }
+
+
+  /**
+   * @return {Promise<void>}
+   */
+  async download() {
+
+    await this.request();
+    await this.save()
+  }
+
+  /**
+   * 
+   * @return {Promise<axios.AxiosResponse>}
+   */
+  async _makeRequest() {
+   
+    const shouldBuffer = this.config.shouldBufferResponse
+    const httpsAgent = this.config.httpsAgent; 
     const response = await axios({
@@ -121,2 +154,4 @@ method: 'get',
     })
+    // debugger;
+    // this.response = response;
 
@@ -126,5 +161,5 @@ return response;
 
-  async createReadStream() {
+  async _createReadStream() {
 
-    const response = await this.makeRequest(false)
+    const response = await this._makeRequest()
 
@@ -137,3 +172,3 @@ if (this._events.response) {
 
-    this.response = response;
+    // this.response = response;
     return response.data;
@@ -143,3 +178,3 @@ }
 
-  createWriteStream(fullPath) {
+  _createWriteStream(fullPath) {
     // console.log(fullPath)
@@ -149,41 +184,37 @@ return fs.createWriteStream(fullPath)
 
-  async downloadAndBuffer() {
-    // debugger;
-    const response = await this.makeRequest(true);
-    this.response = response;
-    const fileName = await this.getFinalFileName();
-    // const write = this.createWriteStream(`${this.config.directory}/${fileName}`)
-    await writeFile(`${this.config.directory}/${fileName}`, response.data)
 
-  }
-
-  download() {
-    if (this.config.shouldBufferResponse) return this.downloadAndBuffer()
-    // debugger;
+  _getProgressStream(){
     const that = this;
+    const progress = new Transform({
+      // writableObjectMode: true,
 
-    return new Promise(async (resolve, reject) => {
-      try {
-        const read = await this.createReadStream(this.config.url);
-        const fileName = await this.getFinalFileName();
+      transform(chunk, encoding, callback) {
 
-        const progress = new Transform({
-          // writableObjectMode: true,
+        that.currentDataSize += chunk.byteLength;
 
-          transform(chunk, encoding, callback) {
+        that.percentage = ((that.currentDataSize / that.fileSize) * 100).toFixed(2)
 
-            that.currentDataSize += chunk.byteLength;
+        if (that._events.progress) {
+          that.emit('progress', that.percentage, chunk);
+        }
 
-            that.percentage = ((that.currentDataSize / that.fileSize) * 100).toFixed(2)
+        // Push the data onto the readable queue.
+        callback(null, chunk);
+      }
+    });
 
-            if (that._events.progress) {
-              that.emit('progress', that.percentage, chunk);
-            }
+    return progress;
+    
+  }
 
-            // Push the data onto the readable queue.
-            callback(null, chunk);
-          }
-        });
-        const write = this.createWriteStream(`${this.config.directory}/${fileName}`)
+  _saveFromReadableStream(read){
+  
 
+    return new Promise(async (resolve, reject) => {
+      try {       
+        const fileName = await this._getFinalFileName();
+
+        const progress = this._getProgressStream();
+        const write = this._createWriteStream(`${this.config.directory}/${fileName}`)
+
         await pipeline(read, progress, write)
@@ -198,4 +229,17 @@
 
-  async getFinalFileName() {
+  async _saveFromBuffer(buffer) {
     // debugger;
+    // const response = await this._makeRequest(true);
+    // this.response = response;
+    const fileName = await this._getFinalFileName();
+    // debugger;
+    // const write = this.createWriteStream(`${this.config.directory}/${fileName}`)
+    await writeFile(`${this.config.directory}/${fileName}`, buffer)
+
+  }
+
+  
+
+  async _getFinalFileName() {
+    // debugger;
     let fileName;
@@ -205,3 +249,3 @@ if (this.config.fileName) {
     } else {
-      fileName = this.deduceFileName(this.config.url, this.response.headers)
+      fileName = deduceFileName(this.config.url, this.response.headers)
     }
@@ -233,94 +277,4 @@ // debugger;
 
-  getFileNameFromContentDisposition(contentDisposition) {
-    // debugger;
-    // const contentDisposition = this.response.headers['content-disposition'] || this.response.headers['Content-Disposition'];
-    if (!contentDisposition || !contentDisposition.includes('filename=')) {
-      return "";
-    }
-    let filename = "";
-    var filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
-    var matches = filenameRegex.exec(contentDisposition);
-    if (matches != null && matches[1]) {
-      filename = matches[1].replace(/['"]/g, '');
-    }
-
-    return filename ? sanitize(filename) : "";
-  }
-
-  getFileNameFromContentType(contentType) {
-
-    // var contentType = this.response.headers['content-type'] || this.response.headers['Content-Type'];
-    // console.log(contentType)
-    let extension = mime.extension(contentType)
-
-    const url = this.removeQueryString(this.config.url);
-    const fileNameWithoutExtension = this.removeExtension(path.basename(url));
-    return `${sanitize(fileNameWithoutExtension)}.${extension}`;
-  }
-
-
-  removeQueryString(url) {
-    return url.split(/[?#]/)[0];
-  }
-
-  removeExtension(str) {
-    // debugger;
-    const arr = str.split('.');
-    if (arr.length == 1) {
-      return str;
-    }
-    return arr.slice(0, -1).join('.')
-
-
-
-  }
-
-
-  /**
-   * 
-   * @param {string} url 
-   * @return {string} fileName
-   */
-  deduceFileNameFromUrl(url) {
-    // debugger;
-    const cleanUrl = this.removeQueryString(url);
-    const baseName = sanitize(path.basename(cleanUrl));
-    return baseName;
-
-  }
-
-
-  /**
-   * Deduce the fileName, covering various scenarios.
-   * @param {string} url
-   * @param {Object} headers
-   * @return {string} fileName
-   */
-  deduceFileName(url, headers) {
-
-
-    //First option
-    const fileNameFromContentDisposition = this.getFileNameFromContentDisposition(headers['content-disposition'] || headers['Content-Disposition']);
-    // console.log('filenamecontentdisposition', fileNameFromContentDisposition)
-    if (fileNameFromContentDisposition) return fileNameFromContentDisposition;
-
-    // debugger;
-    //Second option
-    if (path.extname(url)) {//First check if the url even has an extension
-      const fileNameFromUrl = this.deduceFileNameFromUrl(url);
-      if (fileNameFromUrl) return fileNameFromUrl;
-    }
-
-    //Third option
-    const fileNameFromContentType = this.getFileNameFromContentType(headers['content-type'] || headers['Content-Type'])
-    if (fileNameFromContentType) return fileNameFromContentType
-
-
-    //Fallback option
-    return sanitize(url)
-
-
-  }
 }
 
 

Downloader.test.js

@@ -29,3 +29,3 @@
 
-    
+
     /**
@@ -50,9 +50,9 @@ *
             });
-           
+
         })
-       
+
     }
 
-  
 
+
     it('Should download a picture and use content-type', async () => {
@@ -173,3 +173,3 @@ mock.onGet("/contentType").reply(
     it('Should handle a file without content-length', async () => {
-        
+
         mock.onGet("/Koala.jpg").reply(
@@ -186,4 +186,4 @@ 200,
             url: '/Koala.jpg',
-            directory: "./downloads"            
-        }).on('progress',(p)=>{})
+            directory: "./downloads"
+        }).on('progress', (p) => { })
         //   console.log(downloader)
@@ -200,3 +200,3 @@ // debugger;
     it('Should handle a file without content-type', async () => {
-        
+
         mock.onGet("/Lighthouse.jpg").reply(
@@ -213,4 +213,4 @@ 200,
             url: '/Lighthouse.jpg',
-            directory: "./downloads"            
-        }).on('progress',(p)=>{})
+            directory: "./downloads"
+        }).on('progress', (p) => { })
         //   console.log(downloader)
@@ -232,3 +232,3 @@ // debugger;
             {
-                'Content-Disposition':'Content-Disposition: attachment; filename="contentDispositionFile.jpg"'
+                'Content-Disposition': 'Content-Disposition: attachment; filename="contentDispositionFile.jpg"'
                 // 'Content-Type': 'image/jpeg',
@@ -241,3 +241,3 @@ // 'Content-Length': '845941'
             url: '/contentDisposition',
-            directory: "./downloads"            
+            directory: "./downloads"
         })
@@ -257,3 +257,3 @@ // .on('progress',(p)=>{})
     it('Should download a picture with a querystring after the extension ', async () => {
-        
+
         mock.onGet("/Hydrangeas.jpg?width=400&height=300").reply(
@@ -270,4 +270,4 @@ 200,
             url: '/Hydrangeas.jpg?width=400&height=300',
-            directory: "./downloads"            
-        }).on('progress',(p)=>{})
+            directory: "./downloads"
+        }).on('progress', (p) => { })
         //   console.log(downloader)
@@ -283,4 +283,4 @@ // debugger;
 
-    
 
+
     it('Should download two pictures, with name appending', async () => {
@@ -328,5 +328,46 @@ try {
 
+    it('Should download an image, with shouldBufferResponse', async () => {
+        const stream = fs.createReadStream(Path.join(__dirname, 'fixtures/Koala.jpg'));
+        // const buffer=[]
+        const chunks = []
+        for await (let chunk of stream) {
+            chunks.push(chunk)
+        }
 
+        const buffer = Buffer.concat(chunks)
 
+        mock.onGet("/Koala.jpg").reply(
+            200,
+            buffer,
+            {
+                'Content-Type': 'image/jpeg',
+                'Content-Length': '29051'
+            }
+
+        )
+
+        try {
+            const downloader = new Downloader({
+                url: '/Koala.jpg',
+                directory: "./downloads",
+                cloneFiles: false,
+                shouldBufferResponse: true
+            })
+            //   console.log(downloader)
+            // debugger;
+            await downloader.download();
+
+            // await verifyFile('./downloads/Koala.jpg', 29051);
+        } catch (error) {
+            debugger;
+        }
+
+
+
+
+    })
+
+
+
 })
 

package.json

 {
   "name": "nodejs-file-downloader",
-  "version": "1.2.3",
+  "version": "2.0.0",
   "description": "A file downloader for NodeJs",
@@ -24,3 +24,2 @@ "main": "Downloader.js",
     "axios": "^0.19.2",
-    "https-proxy-agent": "^5.0.0",
     "mime-types": "^2.1.27",
@@ -27,0 +26,0 @@ "sanitize-filename": "^1.6.3"

readme.md

 nodejs-file-downloader is a simple utility for downloading files. It hides the complexity of dealing with streams, paths and duplicate file names.
 
+If you encounter any bugs or have a question, please don't hesitate to open an issue.
+
 ## Installation
@@ -14,2 +16,3 @@
   * [Overwrite existing files](#overwrite-existing-files)  
+  * [Get response and then download](#get-response-and-then-download)  
 
@@ -96,1 +99,28 @@ ## Examples
  
+
+#### Get response and then download
+
+There is an alternative way to using Downloader.download():
+
+```javascript
+
+  const downloader = new Downloader({     
+      url: 'http://212.183.159.230/200MB.zip',     
+      directory: "./",        
+  }) 
+
+  const response = await downloader.request()//This function just performs the request. The file isn't actually being downloaded yet. It returns an Axios response object. You can refer to their docs for more details.
+
+  //Now you can do something with the response, like check the headers
+  if(response.headers['content-length'] > 1000000){
+    await downloader.save()
+  }else{
+    console.log('File is too big!')
+  }  
+
+  //Note that Downloader.download() simply combines these two function calls.
+
+
+```
+
+ 

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

129216

increased by2.2%

Dependency count

3

decreased by-25%

Number of package files

10

increased by11.11%

Lines of code

665

increased by10.83%

Number of lines in readme file

125

increased by31.58%

Number of medium supply chain risk alerts

0

decreased by-100%

Dependency changes

• - Removedhttps-proxy-agent@^5.0.0

• - Removedagent-base@6.0.2(transitive)

• - Removeddebug@4.3.7(transitive)

• - Removedhttps-proxy-agent@5.0.1(transitive)

• - Removedms@2.1.3(transitive)