Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

nodejs-file-downloader

Package Overview
Dependencies
Maintainers
1
Versions
52
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

nodejs-file-downloader - npm Package Compare versions

Comparing version 4.4.2 to 4.5.0

Download.js
RequestWrapper.js
testServer.js
utils/abort.js
utils/string.js

315

Downloader.js

@@ -1,22 +0,8 @@

const fs = require('fs');
// const axios = require('axios');
const { request } = require('./request');
const stream = require('stream');
var HttpsProxyAgent = require('https-proxy-agent');
// const {WritableStream}= fs;
const { Transform } = require('stream')
const util = require('util');
const FileProcessor = require('./utils/FileProcessor');
const pipelinePromisified = util.promisify(stream.pipeline);
const mkdir = util.promisify(fs.mkdir);
const writeFile = util.promisify(fs.writeFile);
const { deduceFileName } = require('./utils/fileName');
const rpur = require('./utils/rpur');
// const { resolve } = require('path');
// const {IncomingMessage} = require('http')
const unlink = util.promisify(fs.unlink)
const rename = util.promisify(fs.rename)
const rpur = require('./utils/rpur')
const { capitalize } = require('./utils/string')
const Download = require('./Download');
const configTypes = {

@@ -42,3 +28,28 @@ url: {

mandatory: false
}
},
onProgress: {
type: 'function',
mandatory: false
},
onResponse: {
type: 'function',
mandatory: false
},
shouldStop: {
type: 'function',
mandatory: false
},
onBeforeSave: {
type: 'function',
mandatory: false
},
onError: {
type: 'function',
mandatory: false
},
maxAttempts: {
type: 'number',
mandatory: false
},
};

@@ -121,11 +132,4 @@

// this.readStream = null;
// this.buffer = null;
// this.responseHeaders = null;
// this.response = null;
this.percentage = 0;
this.fileSize = null;
this.currentDataSize = 0;
this._currentDownload = null;
}

@@ -139,97 +143,25 @@

/**
* @return {Promise<void>}
*/
async download() {
await this._verifyDirectoryExists(this.config.directory)
await this._makeUntilSuccessful(async () => {
this._resetData()
const response = await this._request();
// const readStream = response.readStream
if (this.config.onResponse) {
// debugger
const shouldContinue = await this.config.onResponse(response);
if (shouldContinue === false) {
return;
}
}
// const finalName = await this._getFinalFileName(response.headers);
// const finalPath = `${this.config.directory}/${finalName}`;
await this._save(response)
})
// debugger
}
async _verifyDirectoryExists(directory) {
await mkdir(directory, { recursive: true });
}
_resetData() {
this.percentage = 0;
this.fileSize = null;
this.currentDataSize = 0;
}
/**
* @return {Promise<IncomingMessage} response
*/
async _request() {
// this.resetData()
const response = await this._makeRequest();
// debugger;
const contentLength = response.headers['content-length'] || response.headers['Content-Length'];
this.fileSize = parseInt(contentLength);
return response;
}
/**
* @param {IncomingMessage} response
* @return {Promise<void>}
*/
async _save(response) {
try {
// debugger
let finalName = await this._getFinalFileName(response.headers);
async download() {
const that = this;
const { url, directory, fileName, cloneFiles, timeout, headers, httpsAgent, proxy, onResponse, onBeforeSave, onProgress, shouldBufferResponse, useSynchronousMode } = that.config;
if(this.config.onBeforeSave){
// debugger
const clientOverideName = await this.config.onBeforeSave(finalName)
if(clientOverideName && typeof clientOverideName === 'string'){
finalName = clientOverideName;
}
}
const finalPath = `${this.config.directory}/${finalName}`;
var tempPath = this._getTempFilePath(finalPath);
if (this.config.shouldBufferResponse) {
const buffer = await this._createBufferFromResponseStream(response);
await this._saveFromBuffer(buffer, tempPath);
// await this._saveFromBuffer(buffer, finalPath);
} else {
await this._saveFromReadableStream(response, tempPath);
// await this._saveFromReadableStream(response, finalPath);
}
// debugger;
await this._renameTempFileToFinalName(tempPath, finalPath)
//Repeat downloading process until success
await that._makeUntilSuccessful(async () => {
const download = new Download({ url, directory, fileName, cloneFiles, timeout, headers, httpsAgent, proxy, onResponse, onBeforeSave, onProgress, shouldBufferResponse, useSynchronousMode });
this._currentDownload = download
} catch (error) {
// debugger
await this._removeFailedFile(tempPath)
throw error;
}
await download.start();
})
}
/**

@@ -242,3 +174,2 @@ * @param {Function} asyncFunc

let data;
// debugger;
const func = asyncFunc.bind(this)

@@ -250,3 +181,2 @@ await rpur(async () => {

onError: async (e) => {
// debugger;
if (this.config.onError) {

@@ -257,3 +187,7 @@ await this.config.onError(e);

shouldStop: async (e) => {
// debugger
if (e.code === 'ERR_REQUEST_CANCELLED')//Means the request was cancelled, therefore no repetition is required.
return true;
if (this.config.shouldStop) {

@@ -272,23 +206,6 @@ if (await this.config.shouldStop(e) === true) {

}
/**
*
* @return {Promise<IncomingMessage>}
*/
async _makeRequest() {
const { timeout, headers, proxy, url, httpsAgent } = this.config;
const options = {
timeout,
headers
}
if (httpsAgent) {
options.httpsAgent = httpsAgent;
}
else if (proxy) {
// debugger
options.httpsAgent = new HttpsProxyAgent(proxy)
}
var response = await request(url, options);
cancel() {
return response;
this._currentDownload.cancel();
}

@@ -298,143 +215,5 @@

/**
*
* @param {string} fullPath
* @return {Promie<WritableStream>}
*/
_createWriteStream(fullPath) {
// debugger
return fs.createWriteStream(fullPath)
}
async _createBufferFromResponseStream(stream) {
const chunks = []
for await (let chunk of stream) {
chunks.push(chunk)
}
const buffer = Buffer.concat(chunks)
return buffer;
}
_getProgressStream() {
const that = this;
const progress = new Transform({
transform(chunk, encoding, callback) {
that.currentDataSize += chunk.byteLength;
if (that.fileSize) {
that.percentage = ((that.currentDataSize / that.fileSize) * 100).toFixed(2)
} else {
that.percentage = NaN
}
const remainingFracture = (100 - that.percentage) / 100;
const remainingSize = Math.round(remainingFracture * that.fileSize);
if (that.config.onProgress) {
that.config.onProgress(that.percentage, chunk, remainingSize);
}
// Push the data onto the readable queue.
callback(null, chunk);
}
});
return progress;
}
async _pipeStreams(arrayOfStreams) {
// try {
await pipelinePromisified(...arrayOfStreams);
// } catch (error) {
// debugger;
// }
}
async _saveFromReadableStream(read, path) {
// debugger;
const streams = [read];
const write = this._createWriteStream(path)
if (this.config.onProgress) {
const progressStream = this._getProgressStream()
streams.push(progressStream);
}
streams.push(write)
// debugger
await this._pipeStreams(streams)
// debugger
}
async _saveFromBuffer(buffer, path) {
// debugger;
// const tempPath = this._getTempFilePath(path);
await writeFile(path, buffer)
}
async _removeFailedFile(path) {
await unlink(path);
}
async _renameTempFileToFinalName(temp, final) {
// debugger;
await rename(temp, final)
}
/**
*
* @param {string} finalpath
*/
_getTempFilePath(finalpath) {
return `${finalpath}.download`;
}
/**
* @param {object} responseHeaders
*/
async _getFinalFileName(responseHeaders) {
let fileName;
if (this.config.fileName) {
fileName = this.config.fileName
} else {
fileName = deduceFileName(this.config.url, responseHeaders)
}
if (this.config.cloneFiles) {
var fileProcessor = new FileProcessor({ useSynchronousMode: this.config.useSynchronousMode, fileName, path: this.config.directory })
fileName = await fileProcessor.getAvailableFileName()
}
return fileName;
}
}
const capitalize = (s) => {
if (typeof s !== 'string') return ''
return s.charAt(0).toUpperCase() + s.slice(1)
}

286

Downloader.test.js
// const axios = require('axios');
const expect = require('expect')
// const request = require('supertest');
const { app,server } = require('./testServer');
// var MockAdapter = require("axios-mock-adapter");

@@ -36,3 +39,8 @@ // var mock = new MockAdapter(axios);

after((done)=>{
server.close();
done();
})
/**

@@ -128,3 +136,3 @@ *

cloneFiles: false,
onBeforeSave:(name)=>{
onBeforeSave: (name) => {
deducedName = name;

@@ -162,3 +170,3 @@ // return 'yoyoyoy'

cloneFiles: false,
onBeforeSave:(name)=>{
onBeforeSave: (name) => {
deducedName = name;

@@ -195,3 +203,3 @@ return 'override.jpg'

cloneFiles: false,
onProgress: (p, chunk,remaining) => {
onProgress: (p, chunk, remaining) => {
// debugger;

@@ -609,5 +617,6 @@ expect(isNaN(p)).toBe(true)

onError: function () {
debugger
counter++;
},
})

@@ -654,7 +663,7 @@ // console.log(downloader)

},
shouldStop:function(e){
debugger
if(e.statusCode && e.statusCode === 404){
shouldStop: function (e) {
// debugger
if (e.statusCode && e.statusCode === 404) {
return true;
}
}
}

@@ -764,3 +773,3 @@ })

let fileExists= false
let fileExists = false
await downloader.download();

@@ -773,10 +782,10 @@

} catch (error) {
debugger
// debugger
//The "error" should be caught here, and the test should pass
debugger
// debugger
}
debugger
if(fileExists)throw new Error("Download hasn't stopped")
// debugger
if (fileExists) throw new Error("Download hasn't stopped")

@@ -830,3 +839,3 @@ // throw new Error();

},
fileName:'yoyo.jpg',
fileName: 'yoyo.jpg',
url: `http://www.${host}.com/Koala.jpg`,

@@ -844,3 +853,3 @@ directory: "./downloads",

} catch (error) {
debugger
// debugger
} finally {

@@ -864,170 +873,143 @@ // debugger

// it('Should fail three times, then succeed', async function () {
it('Should get ERR_REQUEST_CANCELLED error after cancellation, while streaming', async function () {
// try {
// let counter = 0
let errorCounter = 0
const downloader = new Downloader({
// var onErrorCount = 0
// const host = randomHost()
// nock(`http://www.${host}.com`)
// .get('/Koala.jpg')
// .reply(function (uri, requestBody) {
// debugger
// counter++
// const stream = fs.createReadStream(Path.join(__dirname, 'fixtures/Koala.jpg'));
// const code = counter <= 3 ? 500 : 200;
// return [
// code,
// stream,
// {}
// ];
// })
// .persist()
// // .reply(500)
fileName: 'cancelled.jpg',
maxAttempts: 4,
url: `http://localhost:3002/cancelWhileStream`,
directory: "./downloads",
onResponse() {
debugger
downloader.cancel()
},
onError(){
errorCounter++
}
// const downloader = new Downloader({
// // timeout: 1000,
// maxAttempts: 4,
// url: `http://www.${host}.com/Koala.jpg`,
// directory: "./downloads",
// onError: function (e) {
// debugger;
// onErrorCount++;
// },
// onProgress:(p)=>{
// debugger
// console.log(p)
// }
// })
})
let error;
try {
// setTimeout(() => {
// downloader.cancel()
// }, 1000)
// debugger
await downloader.download();
// console.log('success')
// debugger
// await downloader.download();
} catch (e) {
error = e
// expect(error.code).toBe('ERR_REQUEST_CANCELLED')
} finally {
// console.log(error)
debugger
expect(errorCounter).toBe(1)
if (!error || error.code !== 'ERR_REQUEST_CANCELLED') {
throw new Error('Cancellation did not work')
}
}
// } catch (error) {
// debugger;
// } finally {
// debugger;
// expect(onErrorCount).toBe(4)
// }
// })
})
it('Should get ERR_REQUEST_CANCELLED error after cancellation, before stream', async function () {
let errorCounter = 0
const downloader = new Downloader({
fileName: 'cancelled.jpg',
maxAttempts: 4,
url: `http://localhost:3002/cancelBeforeStream`,
directory: "./downloads",
onResponse() {
debugger
// downloader.cancel()
},
onError(){
errorCounter++
}
// it('Should fail three times during stream, and then succeed', async function () {
// // this.timeout(10000)
// // let counter = 0
// // const stream = fs.createReadStream(Path.join(__dirname, 'fixtures/Koala.jpg'));
// // mock.onGet("/koala.jpg").reply(function (config) {
// // // const { Readable } = require('stream');
})
let error;
try {
setTimeout(() => {
downloader.cancel()
}, 1000)
debugger
await downloader.download();
// console.log('success')
// debugger
} catch (e) {
error = e
// expect(error.code).toBe('ERR_REQUEST_CANCELLED')
} finally {
// console.log(error)
debugger
expect(errorCounter).toBe(1)
if (!error || error.code !== 'ERR_REQUEST_CANCELLED') {
throw new Error('Cancellation did not work')
}
}
// // const modifiedStream = Readable.from((async function* () {
// // counter++
// // if (counter === 4) {
// // for await (const chunk of stream) {
// // yield chunk;
// // }
// // } else {
// // throw new Error('LOL');
// // }
})
// // })());
// // return [
// // 200,
// // modifiedStream,
// // {}
// // ];
// // });
it('Should timeout during stream, twice', async function () {
this.timeout(0)
try {
// let counter = 0
// try {
// let counter = 0
var onErrorCount = 0
const downloader = new Downloader({
timeout: 2000,
// debugMode:true,
maxAttempts: 2,
// onResponse:function(r){
// if(r.headers=== 'yoyo'){
// error='yoyo'
// return false
// }
// },
url: `http://localhost:3002/timeout`,
directory: "./downloads",
onError: function (e) {
debugger;
// console.log('error')
onErrorCount++;
}
})
// var onErrorCount = 0
// const host = randomHost()
// nock(`http://www.${host}.com`)
// .get('/Koala.jpg')
// .reply(function(uri, requestBody) {
// const stream = fs.createReadStream(Path.join(__dirname, 'fixtures/Koala.jpg'));
// // stream.emit('error')
// // stream.emit('close')
// const modifiedStream = Readable.from((async function* () {
// counter++
// debugger;
// // await timeout(1000);
// debugger
// // if (counter === 4) {
// for await (const chunk of stream) {
// await timeout(1000);
// yield chunk;
// }
// // } else {
// // debugger;
// // throw new Error('LOL');
// // modifiedStream.emit('error','yoyo')
// // }
await downloader.download();
debugger
} catch (error) {
debugger;
// console.log('final error',error)
} finally {
debugger;
expect(onErrorCount).toBe(2)
// await verifyFile('./downloads/koala.jpg', 29051);
}
// })());
// return [
// 200,
// modifiedStream,
// // stream,
// {}
// ];
// })
// .persist()
// let error;
// const downloader = new Downloader({
// timeout: 500,
// // debugMode:true,
// maxAttempts: 1,
// // onResponse:function(r){
// // if(r.headers=== 'yoyo'){
// // error='yoyo'
// // return false
// // }
// // },
// url: `http://www.${host}.com/Koala.jpg`,
// directory: "./downloads",
// onError: function (e) {
// debugger;
// onErrorCount++;
// }
// })
})
// // const response = await downloader.request()
// // debugger;
// // await downloader.save()
// // debugger;
// await downloader.download();
// } catch (error) {
// debugger;
// // console.log(error)
// } finally {
// debugger;
// expect(onErrorCount).toBe(1)
// // await verifyFile('./downloads/koala.jpg', 29051);
// }
// })
})

@@ -1034,0 +1016,0 @@

3

package.json
{
"name": "nodejs-file-downloader",
"version": "4.4.2",
"version": "4.5.0",
"description": "A file downloader for NodeJs",

@@ -31,2 +31,3 @@ "main": "Downloader.js",

"expect": "^26.0.1",
"express": "^4.17.1",
"mocha": "^7.1.2",

@@ -33,0 +34,0 @@ "nock": "^13.0.4",

39

readme.md

@@ -20,2 +20,3 @@ nodejs-file-downloader is a simple utility for downloading files. It hides the complexity of dealing with streams, redirects, paths and duplicate file names. Can automatically repeat failed downloads.

* [Prevent unnecessary repetition](#prevent-unnecessary-repetition)
* [Cancel a download](#cancel-a-download)
* [Use a Proxy](#use-a-proxy)

@@ -237,2 +238,40 @@ - [Error handling](#error-handling)

#### Cancel a download
This feature is new. Kindly report any bugs you encounter.
Useful for Electron apps.
```javascript
const downloader = new Downloader({
url: 'http://212.183.159.230/200MB.zip',
directory: "./",
})
try {
//Mocking cancellation
setTimeout(()=>{
downloader.cancel()
},2000)
await downloader.download();
//If the download is cancelled, the promise will not be resolved, so this part is never reached
console.log('done');
} catch (error) {//When the download is cancelled, 'ERR_REQUEST_CANCELLED' error is thrown. This is how you can handle cancellations in your code.
if(error.code === 'ERR_REQUEST_CANCELLED'){
//do something after cancellation..
}else{
//handle general error..
}
}
```
&nbsp;
#### Use a proxy

@@ -239,0 +278,0 @@

2

utils/fileName.js

@@ -32,3 +32,3 @@ const sanitize = require('sanitize-filename');

// debugger
//First option

@@ -35,0 +35,0 @@ const fileNameFromContentDisposition = getFileNameFromContentDisposition(headers['content-disposition'] || headers['Content-Disposition']);

request.js
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc