Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

nodejs-file-downloader

Package Overview
Dependencies
Maintainers
1
Versions
52
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

nodejs-file-downloader - npm Package Compare versions

Comparing version 1.2.3 to 2.0.0

fileName.js

228

Downloader.js

@@ -6,11 +6,9 @@ const fs = require('fs');

const util = require('util');
var HttpsProxyAgent = require('https-proxy-agent');
// var HttpsProxyAgent = require('https-proxy-agent');
const { EventEmitter } = require('events')
const path = require('path');
const sanitize = require('sanitize-filename');
const FileProcessor = require('./FileProcessor');
var mime = require('mime-types')
const pipeline = util.promisify(stream.pipeline);
const mkdir = util.promisify(fs.mkdir);
const writeFile = util.promisify(fs.writeFile);
const {deduceFileName} = require('./fileName')

@@ -65,4 +63,4 @@

* @param {object} [config.headers]
* @param {string} [config.proxy]
* @param {string} [config.auth]
* @param {object} [config.httpsAgent]
* @param {boolean} [config.shouldBufferResponse = false]
* @param {boolean} [config.useSynchronousMode = false]

@@ -82,4 +80,3 @@ */

useSynchronousMode:false,
// proxy: null,
// auth:null,
proxy: null,
cloneFiles: true,

@@ -104,10 +101,46 @@ shouldBufferResponse: false

/**
*
* @param {boolean} shouldBuffer
* @return {Promise<axios.AxiosResponse>}
*/
async makeRequest(shouldBuffer) {
async request(){
const response = await this._makeRequest();
this.response = response;
if (this._events.response) {
this.emit('response', response)
}
const contentLength = response.headers['content-length'] || response.headers['Content-Length'];
this.fileSize = parseInt(contentLength);
return response;
const httpsAgent = this.config.proxy ? new HttpsProxyAgent(this.config.proxy) : null;
}
/**
* @return {Promise<void>}
*/
async save(){
if(this.config.shouldBufferResponse){
// debugger;
return this._saveFromBuffer(this.response.data);
}
// debugger;
await this._saveFromReadableStream(this.response.data);
}
/**
* @return {Promise<void>}
*/
async download() {
await this.request();
await this.save()
}
/**
*
* @return {Promise<axios.AxiosResponse>}
*/
async _makeRequest() {
const shouldBuffer = this.config.shouldBufferResponse
const httpsAgent = this.config.httpsAgent;
const response = await axios({

@@ -121,2 +154,4 @@ method: 'get',

})
// debugger;
// this.response = response;

@@ -126,5 +161,5 @@ return response;

async createReadStream() {
async _createReadStream() {
const response = await this.makeRequest(false)
const response = await this._makeRequest()

@@ -137,3 +172,3 @@ if (this._events.response) {

this.response = response;
// this.response = response;
return response.data;

@@ -143,3 +178,3 @@ }

createWriteStream(fullPath) {
_createWriteStream(fullPath) {
// console.log(fullPath)

@@ -149,41 +184,37 @@ return fs.createWriteStream(fullPath)

async downloadAndBuffer() {
// debugger;
const response = await this.makeRequest(true);
this.response = response;
const fileName = await this.getFinalFileName();
// const write = this.createWriteStream(`${this.config.directory}/${fileName}`)
await writeFile(`${this.config.directory}/${fileName}`, response.data)
}
download() {
if (this.config.shouldBufferResponse) return this.downloadAndBuffer()
// debugger;
_getProgressStream(){
const that = this;
const progress = new Transform({
// writableObjectMode: true,
return new Promise(async (resolve, reject) => {
try {
const read = await this.createReadStream(this.config.url);
const fileName = await this.getFinalFileName();
transform(chunk, encoding, callback) {
const progress = new Transform({
// writableObjectMode: true,
that.currentDataSize += chunk.byteLength;
transform(chunk, encoding, callback) {
that.percentage = ((that.currentDataSize / that.fileSize) * 100).toFixed(2)
that.currentDataSize += chunk.byteLength;
if (that._events.progress) {
that.emit('progress', that.percentage, chunk);
}
that.percentage = ((that.currentDataSize / that.fileSize) * 100).toFixed(2)
// Push the data onto the readable queue.
callback(null, chunk);
}
});
if (that._events.progress) {
that.emit('progress', that.percentage, chunk);
}
return progress;
}
// Push the data onto the readable queue.
callback(null, chunk);
}
});
const write = this.createWriteStream(`${this.config.directory}/${fileName}`)
_saveFromReadableStream(read){
return new Promise(async (resolve, reject) => {
try {
const fileName = await this._getFinalFileName();
const progress = this._getProgressStream();
const write = this._createWriteStream(`${this.config.directory}/${fileName}`)
await pipeline(read, progress, write)

@@ -198,4 +229,17 @@

async getFinalFileName() {
async _saveFromBuffer(buffer) {
// debugger;
// const response = await this._makeRequest(true);
// this.response = response;
const fileName = await this._getFinalFileName();
// debugger;
// const write = this.createWriteStream(`${this.config.directory}/${fileName}`)
await writeFile(`${this.config.directory}/${fileName}`, buffer)
}
async _getFinalFileName() {
// debugger;
let fileName;

@@ -205,3 +249,3 @@ if (this.config.fileName) {

} else {
fileName = this.deduceFileName(this.config.url, this.response.headers)
fileName = deduceFileName(this.config.url, this.response.headers)
}

@@ -233,94 +277,4 @@ // debugger;

getFileNameFromContentDisposition(contentDisposition) {
// debugger;
// const contentDisposition = this.response.headers['content-disposition'] || this.response.headers['Content-Disposition'];
if (!contentDisposition || !contentDisposition.includes('filename=')) {
return "";
}
let filename = "";
var filenameRegex = /filename[^;=\n]*=((['"]).*?\2|[^;\n]*)/;
var matches = filenameRegex.exec(contentDisposition);
if (matches != null && matches[1]) {
filename = matches[1].replace(/['"]/g, '');
}
return filename ? sanitize(filename) : "";
}
getFileNameFromContentType(contentType) {
// var contentType = this.response.headers['content-type'] || this.response.headers['Content-Type'];
// console.log(contentType)
let extension = mime.extension(contentType)
const url = this.removeQueryString(this.config.url);
const fileNameWithoutExtension = this.removeExtension(path.basename(url));
return `${sanitize(fileNameWithoutExtension)}.${extension}`;
}
removeQueryString(url) {
return url.split(/[?#]/)[0];
}
removeExtension(str) {
// debugger;
const arr = str.split('.');
if (arr.length == 1) {
return str;
}
return arr.slice(0, -1).join('.')
}
/**
*
* @param {string} url
* @return {string} fileName
*/
deduceFileNameFromUrl(url) {
// debugger;
const cleanUrl = this.removeQueryString(url);
const baseName = sanitize(path.basename(cleanUrl));
return baseName;
}
/**
* Deduce the fileName, covering various scenarios.
* @param {string} url
* @param {Object} headers
* @return {string} fileName
*/
deduceFileName(url, headers) {
//First option
const fileNameFromContentDisposition = this.getFileNameFromContentDisposition(headers['content-disposition'] || headers['Content-Disposition']);
// console.log('filenamecontentdisposition', fileNameFromContentDisposition)
if (fileNameFromContentDisposition) return fileNameFromContentDisposition;
// debugger;
//Second option
if (path.extname(url)) {//First check if the url even has an extension
const fileNameFromUrl = this.deduceFileNameFromUrl(url);
if (fileNameFromUrl) return fileNameFromUrl;
}
//Third option
const fileNameFromContentType = this.getFileNameFromContentType(headers['content-type'] || headers['Content-Type'])
if (fileNameFromContentType) return fileNameFromContentType
//Fallback option
return sanitize(url)
}
}

73

Downloader.test.js

@@ -29,3 +29,3 @@

/**

@@ -50,9 +50,9 @@ *

});
})
}
it('Should download a picture and use content-type', async () => {

@@ -173,3 +173,3 @@ mock.onGet("/contentType").reply(

it('Should handle a file without content-length', async () => {
mock.onGet("/Koala.jpg").reply(

@@ -186,4 +186,4 @@ 200,

url: '/Koala.jpg',
directory: "./downloads"
}).on('progress',(p)=>{})
directory: "./downloads"
}).on('progress', (p) => { })
// console.log(downloader)

@@ -200,3 +200,3 @@ // debugger;

it('Should handle a file without content-type', async () => {
mock.onGet("/Lighthouse.jpg").reply(

@@ -213,4 +213,4 @@ 200,

url: '/Lighthouse.jpg',
directory: "./downloads"
}).on('progress',(p)=>{})
directory: "./downloads"
}).on('progress', (p) => { })
// console.log(downloader)

@@ -232,3 +232,3 @@ // debugger;

{
'Content-Disposition':'Content-Disposition: attachment; filename="contentDispositionFile.jpg"'
'Content-Disposition': 'Content-Disposition: attachment; filename="contentDispositionFile.jpg"'
// 'Content-Type': 'image/jpeg',

@@ -241,3 +241,3 @@ // 'Content-Length': '845941'

url: '/contentDisposition',
directory: "./downloads"
directory: "./downloads"
})

@@ -257,3 +257,3 @@ // .on('progress',(p)=>{})

it('Should download a picture with a querystring after the extension ', async () => {
mock.onGet("/Hydrangeas.jpg?width=400&height=300").reply(

@@ -270,4 +270,4 @@ 200,

url: '/Hydrangeas.jpg?width=400&height=300',
directory: "./downloads"
}).on('progress',(p)=>{})
directory: "./downloads"
}).on('progress', (p) => { })
// console.log(downloader)

@@ -283,4 +283,4 @@ // debugger;

it('Should download two pictures, with name appending', async () => {

@@ -328,5 +328,46 @@ try {

it('Should download an image, with shouldBufferResponse', async () => {
const stream = fs.createReadStream(Path.join(__dirname, 'fixtures/Koala.jpg'));
// const buffer=[]
const chunks = []
for await (let chunk of stream) {
chunks.push(chunk)
}
const buffer = Buffer.concat(chunks)
mock.onGet("/Koala.jpg").reply(
200,
buffer,
{
'Content-Type': 'image/jpeg',
'Content-Length': '29051'
}
)
try {
const downloader = new Downloader({
url: '/Koala.jpg',
directory: "./downloads",
cloneFiles: false,
shouldBufferResponse: true
})
// console.log(downloader)
// debugger;
await downloader.download();
// await verifyFile('./downloads/Koala.jpg', 29051);
} catch (error) {
debugger;
}
})
})

3

package.json
{
"name": "nodejs-file-downloader",
"version": "1.2.3",
"version": "2.0.0",
"description": "A file downloader for NodeJs",

@@ -24,3 +24,2 @@ "main": "Downloader.js",

"axios": "^0.19.2",
"https-proxy-agent": "^5.0.0",
"mime-types": "^2.1.27",

@@ -27,0 +26,0 @@ "sanitize-filename": "^1.6.3"

30

readme.md
nodejs-file-downloader is a simple utility for downloading files. It hides the complexity of dealing with streams, paths and duplicate file names.
If you encounter any bugs or have a question, please don't hesitate to open an issue.
## Installation

@@ -14,2 +16,3 @@

* [Overwrite existing files](#overwrite-existing-files)
* [Get response and then download](#get-response-and-then-download)

@@ -96,1 +99,28 @@ ## Examples

&nbsp;
#### Get response and then download
There is an alternative way to using Downloader.download():
```javascript
const downloader = new Downloader({
url: 'http://212.183.159.230/200MB.zip',
directory: "./",
})
const response = await downloader.request()//This function just performs the request. The file isn't actually being downloaded yet. It returns an Axios response object. You can refer to their docs for more details.
//Now you can do something with the response, like check the headers
if(response.headers['content-length'] > 1000000){
await downloader.save()
}else{
console.log('File is too big!')
}
//Note that Downloader.download() simply combines these two function calls.
```
&nbsp;
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc