nodejs-file-downloader - npm Package Compare versions

Comparing version 2.0.1 to 2.1.0

Downloader.js

@@ -6,9 +6,8 @@ const fs = require('fs');
 const util = require('util');
-// var HttpsProxyAgent = require('https-proxy-agent');
-const { EventEmitter } = require('events')
-const FileProcessor = require('./FileProcessor');
+const FileProcessor = require('./utils/FileProcessor');
 const pipeline = util.promisify(stream.pipeline);
 const mkdir = util.promisify(fs.mkdir);
 const writeFile = util.promisify(fs.writeFile);
-const {deduceFileName} = require('./fileName')
+const { deduceFileName } = require('./utils/fileName');
+const rpur = require('./utils/rpur');
 
@@ -51,5 +50,6 @@
 
-module.exports = class Downloader extends EventEmitter {
 
+module.exports = class Downloader {
 
+
   /**
@@ -60,7 +60,11 @@ *
    * @param {string} [config.directory]    
-   * @param {string} [config.fileName] 
+   * @param {string} [config.fileName = undefined] 
    * @param {boolean} [config.cloneFiles=true] 
    * @param {number} [config.timeout=6000] 
-   * @param {object} [config.headers] 
-   * @param {object} [config.httpsAgent] 
+   * @param {number} [config.maxAttempts=1] 
+   * @param {object} [config.headers = undefined] 
+   * @param {object} [config.httpsAgent = undefined] 
+   * @param {function} [config.onError = undefined] 
+   * @param {function} [config.onResponse = undefined] 
+   * @param {function} [config.onProgress = undefined] 
    * @param {boolean} [config.shouldBufferResponse = false] 
@@ -70,3 +74,3 @@ * @param {boolean} [config.useSynchronousMode = false]
   constructor(config) {
-    super();
+    // super();
     if (!config || typeof config !== 'object') {
@@ -79,8 +83,13 @@ throw new Error('Must provide a valid config object')
       directory: './',
-      fileName: null,
+      fileName: undefined,
       timeout: 6000,
-      useSynchronousMode:false,
-      proxy: null,
-      cloneFiles: true,
-      shouldBufferResponse: false
+      maxAttempts:1,
+      useSynchronousMode: false,
+      httpsAgent:undefined,
+      headers:undefined,
+      cloneFiles: true,      
+      shouldBufferResponse: false,
+      onResponse:undefined,
+      onError:undefined,
+      onProgress:undefined
     }
@@ -93,2 +102,6 @@
 
+    if(this.config.filename){
+      this.config.fileName = this.config.filename 
+    }
+
     this.response = null;
@@ -103,14 +116,21 @@ this.readStream = null;
 
+  //For EventEmitter backwards compatibility
+  on(event,callback){
+    this.config[`on${capitalize(event)}`] = callback
+  }
+
   /**
    * @return {Promise<axios.AxiosResponse>}
    */
-  async request(){
-    const response = await this._makeRequest();
+  async request() {
+    // const response = await this._makeRequest();
+    // const response = await this._makeRequestUntilSuccessful();
+    const response = await this._makeUntilSuccessful(this._makeRequest);
     this.response = response;
-    if (this._events.response) {
-      this.emit('response', response)
+    if (this.config.onResponse) {
+      await this.config.onResponse(response);
     }
     const contentLength = response.headers['content-length'] || response.headers['Content-Length'];
     this.fileSize = parseInt(contentLength);
-    return response; 
+    return response;
 
@@ -122,9 +142,11 @@ }
    */
-  async save(){
-    if(this.config.shouldBufferResponse){
+  async save() {
+    if (this.config.shouldBufferResponse) {
       // debugger;
-      return this._saveFromBuffer(this.response.data);
+      // return this._saveFromBuffer(this.response.data);
+      return this._makeUntilSuccessful(async()=>{await this._saveFromBuffer(this.response.data)});
     }
     // debugger;
-    await this._saveFromReadableStream(this.response.data);
+    // await this._saveFromReadableStream(this.response.data);
+    await this._makeUntilSuccessful(async()=>{await this._saveFromReadableStream(this.response.data)});
   }
@@ -139,6 +161,38 @@
     await this.request();
+    // debugger;
     await this.save()
   }
 
+  
+
   /**
+   * @param {Function} asyncFunc
+   * @return {Promise<any>} 
+   */
+  async _makeUntilSuccessful(asyncFunc) {
+
+    let data;
+    // debugger;
+    const func = asyncFunc.bind(this)
+    await rpur(async () => {
+      // debugger;
+
+      data = await func();
+      // debugger;
+    }, {
+      onError: async(e) => {
+        // debugger;
+        if (this.config.onError) {
+          await this.config.onError(e);
+        }
+      },
+      maxAttempts:this.config.maxAttempts
+      // maxAttempts:1
+    })
+    // debugger;
+    return data;
+
+
+  }
+  /**
    * 
@@ -148,5 +202,5 @@ * @return {Promise<axios.AxiosResponse>}
   async _makeRequest() {
-   
+    // debugger;
     const shouldBuffer = this.config.shouldBufferResponse
-    const httpsAgent = this.config.httpsAgent; 
+    const httpsAgent = this.config.httpsAgent;
     const response = await axios({
@@ -166,17 +220,4 @@ method: 'get',
 
-  async _createReadStream() {
 
-    const response = await this._makeRequest()
 
-    if (this._events.response) {
-      this.emit('response', response)
-    }
-    const contentLength = response.headers['content-length'] || response.headers['Content-Length'];
-    this.fileSize = parseInt(contentLength);
-
-    // this.response = response;
-    return response.data;
-  }
-
-
   _createWriteStream(fullPath) {
@@ -188,3 +229,3 @@ // console.log(fullPath)
 
-  _getProgressStream(){
+  _getProgressStream() {
     const that = this;
@@ -200,4 +241,4 @@ const progress = new Transform({
 
-        if (that._events.progress) {
-          that.emit('progress', that.percentage, chunk);
+        if (that.config.onProgress) {
+          that.config.onProgress(that.percentage, chunk);
         }
@@ -211,22 +252,14 @@
     return progress;
-    
+
   }
 
-  _saveFromReadableStream(read){
-  
+  async _saveFromReadableStream(read) {
+    // yoyo
+    const fileName = await this._getFinalFileName();
 
-    return new Promise(async (resolve, reject) => {
-      try {       
-        const fileName = await this._getFinalFileName();
+    const progress = this._getProgressStream();
+    const write = this._createWriteStream(`${this.config.directory}/${fileName}`)
 
-        const progress = this._getProgressStream();
-        const write = this._createWriteStream(`${this.config.directory}/${fileName}`)
+    await pipeline(read, progress, write)
 
-        await pipeline(read, progress, write)
-
-        resolve();
-      } catch (error) {
-        reject(error)
-      }
-    })
   }
@@ -245,4 +278,4 @@
 
-  
 
+
   async _getFinalFileName() {
@@ -257,3 +290,3 @@ // debugger;
     // debugger;
-    var fileProcessor = new FileProcessor({useSynchronousMode:this.config.useSynchronousMode, fileName, path: this.config.directory })
+    var fileProcessor = new FileProcessor({ useSynchronousMode: this.config.useSynchronousMode, fileName, path: this.config.directory })
     // debugger;
@@ -285,1 +318,5 @@ // if (! await fileProcessor.pathExists(this.config.directory)) {
 
+const capitalize = (s) => {
+  if (typeof s !== 'string') return ''
+  return s.charAt(0).toUpperCase() + s.slice(1)
+}

Downloader.test.js

@@ -70,5 +70,3 @@
             cloneFiles: false,
-            // fileName:'yoyo2.jpg'
-        })
-            .on('progress', (p, chunk) => {
+            onProgress:(p, chunk) => {
                 // console.log(p, chunk)
@@ -78,8 +76,21 @@ expect(!isNaN(parseFloat(p)) && isFinite(p)).toBe(true)
 
-            })
-            .on('response', (r) => {
+            },
+            onResponse:(r) => {
                 // console.log(Object.getPrototypeOf(r).constructor.name)
                 expect(r).toHaveProperty('data');
                 expect(r).toHaveProperty('headers');
-            })
+            }
+            // fileName:'yoyo2.jpg'
+        })
+            // .on('progress', (p, chunk) => {
+            //     // console.log(p, chunk)
+            //     expect(!isNaN(parseFloat(p)) && isFinite(p)).toBe(true)
+            //     expect(Object.getPrototypeOf(chunk).constructor.name).toBe('Buffer')
+
+            // })
+            // .on('response', (r) => {
+            //     // console.log(Object.getPrototypeOf(r).constructor.name)
+            //     expect(r).toHaveProperty('data');
+            //     expect(r).toHaveProperty('headers');
+            // })
         //   console.log(downloader)
@@ -92,3 +103,3 @@ // debugger;
 
-        console.log('Download complete')
+        
     })
@@ -119,3 +130,3 @@
 
-        console.log('Download complete')
+        
     })
@@ -145,3 +156,3 @@
 
-        console.log('Download complete')
+        
     })
@@ -172,3 +183,3 @@
 
-        console.log('Download complete')
+        
     })
@@ -189,4 +200,6 @@
             url: '/Koala.jpg',
-            directory: "./downloads"
-        }).on('progress', (p) => { })
+            directory: "./downloads",
+            // onProgress: (p) => { }
+        })
+        // .on('progress', (p) => { })
         //   console.log(downloader)
@@ -199,3 +212,3 @@ // debugger;
 
-        console.log('Download complete')
+        
     })
@@ -216,4 +229,5 @@
             url: '/Lighthouse.jpg',
-            directory: "./downloads"
-        }).on('progress', (p) => { })
+            directory: "./downloads",
+        })
+        // .on('progress', (p) => { })
         //   console.log(downloader)
@@ -226,3 +240,3 @@ // debugger;
 
-        console.log('Download complete')
+        
     })
@@ -254,3 +268,3 @@
 
-        console.log('Download complete')
+        
     })
@@ -273,3 +287,4 @@
             directory: "./downloads"
-        }).on('progress', (p) => { })
+        })
+        // .on('progress', (p) => { })
         //   console.log(downloader)
@@ -282,3 +297,3 @@ // debugger;
 
-        console.log('Download complete')
+        
     })
@@ -322,3 +337,3 @@
 
-            console.log('Download complete')
+            
         } catch (error) {
@@ -372,5 +387,191 @@ console.log(error)
 
+    it('Should repeat a request few times and fail', async () => {
 
 
+
+        mock.onGet("/400").reply(400)
+
+        try {
+            const downloader = new Downloader({
+                url: '/400',
+                directory: "./downloads",
+                maxAttempts:3
+            })
+            //   console.log(downloader)
+            // debugger;
+            await downloader.download();
+            // await downloader.request();
+            debugger;
+            // await downloader.save();
+
+            // await verifyFile('./downloads/Koala.jpg', 29051);
+        } catch (error) {
+            // expect(1+2).toBe(1)
+            expect(error.message).toBe('Request failed with status code 400')
+            // debugger;
+        }
+
+
+
+
+    })
+
+
+    it('Should fail twice and finally succeed', async () => {
+        const stream = fs.createReadStream(Path.join(__dirname, 'fixtures/Koala.jpg'));
+
+        let counter = 0;
+        mock.onGet("/400").reply(function (config) {
+            // debugger;
+            let status;
+            counter++
+            if (counter < 3) {
+                status = 400
+            } else {
+                status = 200
+            }
+            return [
+                status,
+                stream,
+                {'Content-Type': 'image/jpeg'}
+            ];
+        });
+
+        try {
+            var onErrorCount = 0
+
+            const downloader = new Downloader({
+                timeout: 1000,
+                url: '/400',
+                directory: "./downloads",
+                maxAttempts:3,
+                onError:(e) => {
+                    debugger;
+                    onErrorCount++;
+                    // console.log(e.message)
+                }
+
+            })
+            //   console.log(downloader)
+            // debugger;
+            // var onErrorCount = 0
+            // downloader.on('error', (e) => {
+            //     debugger;
+            //     onErrorCount++;
+            //     // console.log(e.message)
+            // })
+            // await downloader.download();
+           const request =  await downloader.request()
+            await downloader.save()
+            // debugger;
+            var s = request.data;
+            // debugger;
+            
+            
+            
+        } catch (error) {
+            // debugger;
+        }finally{
+            // debugger;
+            expect(s.constructor.name).toBe('ReadStream')
+            expect(onErrorCount).toBe(2)
+            await verifyFile('./downloads/400.jpeg', 29051);
+        }
+
+
+
+
+    })
+
+    it('Should fail once and finally fail', async () => {
+   
+
+       
+        mock.onGet("/500").reply(500);
+        var onErrorCount = 0;
+        try {
+            const downloader = new Downloader({
+                timeout: 1000,
+                url: '/500',
+                directory: "./downloads",
+                maxAttempts:1,
+                onError: (e) => {
+                    // debugger;
+                    onErrorCount++;
+                    // console.log(e.message)
+                }
+
+            })
+
+            // var onErrorCount = 0
+            // downloader.on('error', (e) => {
+            //     // debugger;
+            //     onErrorCount++;
+            //     // console.log(e.message)
+            // })
+            await downloader.download();
+
+        } catch (error) {
+            // debugger;
+        }finally{
+            // debugger;
+            // expect(s.constructor.name).toBe('ReadStream')
+            expect(onErrorCount).toBe(1)
+        }
+
+
+
+
+    })
+
+    // it('Should fail three times during stream', async function() {
+    //     // this.timeout(10000)
+        
+    //     mock.onGet("/fileThatDoesntExist").reply(function (config) {
+    //         // debugger;
+    //         const stream = fs.createReadStream(Path.join(__dirname, 'fixtures/Desert.jpg'));
+    //         stream.destroy();
+           
+    //         return [
+    //             200,
+    //             stream,
+    //             {}
+    //         ];
+    //     });
+
+    //     try {
+    //         const downloader = new Downloader({
+    //             timeout: 1000,
+    //             url: '/fileThatDoesntExist',
+    //             directory: "./downloads",
+
+    //         })
+
+    //         var onErrorCount = 0
+    //         downloader.on('error', (e) => {
+    //             onErrorCount++;
+    //         })
+    //        const request =  await downloader.request()
+    //        debugger;
+    //         await downloader.save()
+    //         debugger;
+    //         // await downloader.download();
+            
+            
+    //     } catch (error) {
+    //         debugger;
+    //         // console.log(error)
+    //     }finally{
+    //         debugger;
+    //         expect(onErrorCount).toBe(3)
+    //     }
+
+
+
+
+    // })
+
+
+
 })
 

package.json

 {
   "name": "nodejs-file-downloader",
-  "version": "2.0.1",
+  "version": "2.1.0",
   "description": "A file downloader for NodeJs",
@@ -5,0 +5,0 @@ "main": "Downloader.js",

readme.md

@@ -1,2 +0,2 @@
-nodejs-file-downloader is a simple utility for downloading files. It hides the complexity of dealing with streams, paths and duplicate file names.
+nodejs-file-downloader is a simple utility for downloading files. It hides the complexity of dealing with streams, paths and duplicate file names. Can automatically repeat failed downloads.
 
@@ -17,2 +17,3 @@ If you encounter any bugs or have a question, please don't hesitate to open an issue.
   * [Get response and then download](#get-response-and-then-download)  
+  * [Repeat failed downloads automatically](#repeat-failed-downloads-automatically)  
 
@@ -53,8 +54,7 @@ ## Examples
       url: 'http://212.183.159.230/200MB.zip',     
-      directory: "./downloads/2020/May",//Sub directories will also be automatically created if they do not exist.           
-    })
-
-    downloader.on('progress',(percentage)=>{//Downloader is an event emitter. You can register a "progress" event.
-        console.log('% ',percentage)
-    })
+      directory: "./downloads/2020/May",//Sub directories will also be automatically created if they do not exist.  
+      onProgress:function(percentage){//Gets called with each chunk.
+           console.log('% ',percentage)   
+      }         
+    })    
     
@@ -127,1 +127,26 @@ await downloader.download();
  
+
+
+
+#### Repeat failed downloads automatically
+
+The program can repeat any failed http request or a stream automatically. Just set the maxAttempts property.
+Note that this applies separately both to the http request and the stream(each will have 3 maxAttemps, in this example).
+
+```javascript
+
+  const downloader = new Downloader({     
+      url: 'http://212.183.159.230/200MB.zip',     
+      directory: "./",
+      maxAttempts:3,//Default is 1.
+      onError:function(error){//You can also hook into each failed attempt.
+        console.log('Error from attempt ',error)
+      }        
+  })   
+
+  await downloader.dowload();
+
+
+```
+
+ 

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

139431

increased by7.91%

Number of package files

12

increased by20%

Lines of code

930

increased by39.85%

Number of lines in readme file

150

increased by20%

No dependency changes