wget-improved - npm Package Compare versions

Comparing version 1.5.0 to 3.0.0

lib/wget.js

@@ -1,10 +0,10 @@
-'use strict'
+'use strict';
 
-var http = require('http');
-var https = require('https');
-var tunnel = require('tunnel');
-var url = require('url');
-var zlib = require('zlib');
-var fs = require('fs');
-var EventEmitter = require('events').EventEmitter;
+const http = require('http');
+const https = require('https');
+const tunnel = require('tunnel');
+const url = require('url');
+const zlib = require('zlib');
+const fs = require('fs');
+const EventEmitter = require('events').EventEmitter;
 
@@ -21,8 +21,7 @@ /**
 function download(src, output, options, _parentEvent, redirects) {
-    if(typeof redirects === "undefined") {
+    if (typeof redirects === 'undefined') {
         redirects = 0;
     }
-    var downloader = _parentEvent || new EventEmitter(),
+    let downloader = _parentEvent || new EventEmitter(),
         srcUrl,
-        tunnelAgent,
         req;
@@ -38,73 +37,117 @@
     srcUrl = url.parse(src);
+    if (!srcUrl.protocol || srcUrl.prototype === null) {
+        downloader.emit(
+            'error',
+            'Cannot parse url protocol for ' +
+                src +
+                ', please specify either HTTP or HTTPS'
+        );
+        return false;
+    }
     srcUrl.protocol = cleanProtocol(srcUrl.protocol);
 
-    req = request({
-        protocol: srcUrl.protocol,
-        host: srcUrl.hostname,
-        port: srcUrl.port,
-        path: srcUrl.pathname + (srcUrl.search || ""),
-        proxy: options?options.proxy:undefined,
-        auth: options.auth?options.auth:undefined,
-        method: 'GET'
-    }, function(res) {
-        var fileSize, writeStream, downloadedSize;
-        var gunzip = zlib.createGunzip();
+    req = request(
+        {
+            protocol: srcUrl.protocol,
+            host: srcUrl.hostname,
+            port: srcUrl.port,
+            path: srcUrl.pathname + (srcUrl.search || ''),
+            proxy: options ? options.proxy : undefined,
+            auth: options.auth ? options.auth : undefined,
+            method: 'GET'
+        },
+        function(res) {
+            let fileSize, writeStream, downloadedSize;
+            let gunzip = zlib.createGunzip();
 
-        // Handle 302 redirects
-        if(res.statusCode === 301 || res.statusCode === 302 || res.statusCode === 307) {
-            redirects++;
-            if(redirects >= 10) {
-                downloader.emit('error', 'Infinite redirect loop detected');
+            // Handle 302 redirects
+            if (
+                res.statusCode === 301 ||
+                res.statusCode === 302 ||
+                res.statusCode === 307
+            ) {
+                var newLocation = res.headers.location;
+                if (res.headers.location.indexOf('/') === 0) {
+                    newLocation =
+                        srcUrl.protocol +
+                        '://' +
+                        srcUrl.hostname +
+                        (srcUrl.port ? ':' + srcUrl.port : '') +
+                        res.headers.location;
+                }
+
+                redirects++;
+                if (redirects >= 10) {
+                    downloader.emit('error', 'Infinite redirect loop detected');
+                    return false;
+                }
+                download(newLocation, output, options, downloader, redirects);
             }
-            download(res.headers.location, output, options, downloader, redirects);
-        }
 
-        if (res.statusCode === 200) {
-            downloadedSize = 0;
-            fileSize = res.headers['content-length'];
-            writeStream = fs.createWriteStream(output, {
-                flags: 'w+',
-                encoding: 'binary'
-            });
+            if (res.statusCode === 200) {
+                downloadedSize = 0;
+                fileSize = Number(res.headers['content-length']);
 
-            res.on('error', function(err) {
-                writeStream.end();
-                downloader.emit('error', err);
-            });
+                // If content length header is not sent, there is no way to determine file size.
+                if (isNaN(fileSize) === true) {
+                    fileSize = null;
+                }
+                writeStream = fs.createWriteStream(output, {
+                    flags: 'w+',
+                    encoding: 'binary'
+                });
 
-            var encoding = "";
-            if(typeof res.headers['content-encoding'] === "string") {
-                encoding = res.headers['content-encoding'];
-            }
+                res.on('error', function(err) {
+                    writeStream.end();
+                    downloader.emit('error', err);
+                });
 
-            // If the user has specified to unzip, and the file is gzip encoded, pipe to gunzip
-            if(options.gunzip === true && encoding === "gzip") {
-                res.pipe(gunzip);
-            } else {
-                res.pipe(writeStream);
-            }
+                var encoding = '';
+                if (typeof res.headers['content-encoding'] === 'string') {
+                    encoding = res.headers['content-encoding'];
+                }
 
-            //emit a start event so the user knows the file-size he's gonna receive
-            downloader.emit('start', fileSize);
+                // If the user has specified to unzip, and the file is gzip encoded, pipe to gunzip
+                if (options.gunzip === true && encoding === 'gzip') {
+                    res.pipe(gunzip);
+                } else {
+                    res.pipe(writeStream);
+                }
 
-            // Data handlers
-            res.on('data', function(chunk) {
-                downloadedSize += chunk.length;
-                downloader.emit('progress', downloadedSize/fileSize);
-            });
-            gunzip.on('data', function(chunk) {
-                writeStream.write(chunk);
-            });
+                //emit a start event so the user knows the file-size he's gonna receive
+                downloader.emit('start', fileSize);
 
-            writeStream.on('finish', function() {
-                writeStream.end();
-                downloader.emit('end', "Finished writing to disk");
-                req.end('finished');
-            });
-        } else if(res.statusCode !== 200 && res.statusCode !== 301 && res.statusCode !== 302) {
-            downloader.emit('error', 'Server responded with unhandled status: ' + res.statusCode);
+                // Data handlers
+                res.on('data', function(chunk) {
+                    downloadedSize += chunk.length;
+                    downloader.emit(
+                        'progress',
+                        calculateProgress(fileSize, downloadedSize)
+                    );
+                    downloader.emit('bytes', downloadedSize);
+                });
+                gunzip.on('data', function(chunk) {
+                    writeStream.write(chunk);
+                });
+
+                writeStream.on('finish', function() {
+                    writeStream.end();
+                    downloader.emit('progress', 1);
+                    downloader.emit('end', 'Finished writing to disk');
+                    req.end('finished');
+                });
+            } else if (
+                res.statusCode !== 200 &&
+                res.statusCode !== 301 &&
+                res.statusCode !== 302
+            ) {
+                downloader.emit(
+                    'error',
+                    'Server responded with unhandled status: ' + res.statusCode
+                );
+            }
         }
-    });
+    );
 
-    req.end('done');
+    req.flushHeaders();
     req.on('error', function(err) {
@@ -121,3 +164,5 @@ downloader.emit('error', err);
 function request(options, callback) {
-    var newOptions = {}, newProxy = {}, key;
+    var newOptions = {},
+        newProxy = {},
+        key;
     options = parseOptions('request', options);
@@ -132,5 +177,5 @@ if (options.protocol === 'http') {
             if (options.proxy.protocol === 'http') {
-                options.agent = tunnel.httpOverHttp({proxy: newProxy});
+                options.agent = tunnel.httpOverHttp({ proxy: newProxy });
             } else if (options.proxy.protocol === 'https') {
-                options.agent = tunnel.httpOverHttps({proxy: newProxy});
+                options.agent = tunnel.httpOverHttps({ proxy: newProxy });
             } else {
@@ -155,5 +200,5 @@ throw options.proxy.protocol + ' proxy is not supported!';
             if (options.proxy.protocol === 'http') {
-                options.agent = tunnel.httpsOverHttp({proxy: newProxy});
+                options.agent = tunnel.httpsOverHttp({ proxy: newProxy });
             } else if (options.proxy.protocol === 'https') {
-                options.agent = tunnel.httpsOverHttps({proxy: newProxy});
+                options.agent = tunnel.httpsOverHttps({ proxy: newProxy });
             } else {
@@ -170,3 +215,3 @@ throw options.proxy.protocol + ' proxy is not supported!';
     }
-    throw 'only allow http or https request!';
+    throw 'Your URL must use either HTTP or HTTPS.';
 }
@@ -185,3 +230,3 @@
                 options.proxy.proxyAuth = proxy.auth;
-                options.proxy.headers = {'User-Agent': 'Node'};
+                options.proxy.headers = { 'User-Agent': 'Node-Wget' };
             }
@@ -205,3 +250,3 @@ }
                 options.proxy.proxyAuth = proxy.auth;
-                options.proxy.headers = {'User-Agent': 'Node'};
+                options.proxy.headers = { 'User-Agent': 'Node-Wget' };
             }
@@ -216,6 +261,20 @@ }
 function cleanProtocol(str) {
-    return str.trim().toLowerCase().replace(/:$/, '');
+    return str
+        .trim()
+        .toLowerCase()
+        .replace(/:$/, '');
 }
 
+function calculateProgress(fileSize, totalDownloaded) {
+    // If we don't know the size of the download, we Microsoft(tm) the progress bar into a moving target...
+    if (fileSize === null) {
+        var length = String(totalDownloaded).length;
+        // This guarantees that totalDownloaded is never greater than or equal to fileSize.
+        fileSize = Math.pow(10, length) + 1;
+    }
+
+    return totalDownloaded / fileSize;
+}
+
 exports.download = download;
 exports.request = request;

package.json

 {
     "name": "wget-improved",
-    "version": "1.5.0",
+    "version": "3.0.0",
     "description": "wget in nodejs, forked from wuchengwei/node-wget to add improvements and help maintain the project",
@@ -13,3 +13,3 @@ "keywords": [
     ],
-    "author": "Michael Barajas <[e-mail]>",
+    "author": "Bearjaws <[e-mail]>",
     "repository": {
@@ -38,4 +38,9 @@ "type": "git",
         "chai": "^4.0.2",
-        "mocha": "^3.4.2"
+        "express": "^4.15.3",
+        "mocha": "^3.4.2",
+        "request": "^2.81.0"
+    },
+    "scripts": {
+        "test": "mocha test/test.js"
     }
 }

README.md

@@ -6,3 +6,3 @@ # wget-improved
 Improvements over [wuchengwei/node-wget](https://github.com/wuchengwei/node-wget)
-- Handles 302 redirects (including infinite redirect loops)
+- Handles 3xx redirects (including infinite redirect loops)
 - Passes URL parameters
@@ -22,9 +22,9 @@ - Better error reporting
 ```js
-var wget = require('wget-improved');
-var src = 'http://nodejs.org/images/logo.svg';
-var output = '/tmp/logo.svg';
-var options = {
+const wget = require('wget-improved');
+const src = 'http://nodejs.org/images/logo.svg';
+const output = '/tmp/logo.svg';
+const options = {
     // see options below
 };
-var download = wget.download(src, output, options);
+let download = wget.download(src, output, options);
 download.on('error', function(err) {
@@ -40,2 +40,3 @@ console.log(err);
 download.on('progress', function(progress) {
+    typeof progress === 'number'
     // code to show progress bar
@@ -48,4 +49,4 @@ });
 ```js
-var wget = require('wget');
-var options = {
+const wget = require('wget');
+const options = {
     protocol: 'https',
@@ -57,4 +58,4 @@ host: 'raw.github.com',
 };
-var req = wget.request(options, function(res) {
-    var content = '';
+let req = wget.request(options, function(res) {
+    let content = '';
     if (res.statusCode === 200) {
@@ -106,4 +107,16 @@ res.on('error', function(err) {
 
-## Todo
+## Changes from 2.0.0 to 3.0.0
+**Progress is now returned as a Number instead of a String**
 
-- Enable gzip when using request method
+**On start filesize can return null when the remote server does not provided content-lenth**
+
+Exception for not specifying protocol is now: `Your URL must use either HTTP or HTTPS.`
+
+Supports handling redirects that return a relative URL.
+
+You can now get events for the **total** number of bytes downloaded `download.on('bytes', function(bytes) {}...)`
+
+Request headers can be specified by passing an object to options.headers.
+
+Unit tests have been added for most download functionality and error cases and are a requirement for all PRs going forward!
+

test/test.js

@@ -1,11 +0,81 @@
-let wget = require('../lib/wget');
+let crypto = require('crypto');
+let fs = require('fs');
 let expect = require('chai').expect;
+let request = require('request');
 
-describe("Download Tests", function() {
-    // with a proxy:
-    it("Should be able to download the NPM logo", function(done) {
-        let download = wget.download('https://www.npmjs.com/static/images/npm-logo.svg', '/tmp/npm-logo.svg');
-        // @todo upgrade these tests to use a more consistent environment, with its own http server and files.
+let wget = require('../lib/wget');
 
+let baseHTTP = 'http://localhost:8884';
+let metadata = {};
+before(function() {
+    let server = require('./server');
+    return server().then(function() {
+        request(baseHTTP + '/file/metadata', function(err, res, body) {
+            metadata = JSON.parse(body);
+        });
+    });
+});
+
+describe('Download Tests', function() {
+    it('Should be able to download a file', function(done) {
+        let download = wget.download(
+            'http://localhost:8884/file',
+            '/tmp/wget-test-file.bin'
+        );
+        let bytes = 0;
         download.on('error', function(err) {
+            done(err);
+        });
+        download.on('start', function(fileSize) {
+            expect(fileSize).to.be.a('number');
+            expect(fileSize).to.equal(metadata.size);
+        });
+        download.on('end', function(output) {
+            let file = fs.readFileSync('/tmp/wget-test-file.bin');
+            let hash = crypto
+                .createHash('sha256')
+                .update(file)
+                .digest('hex');
+            expect(output).to.equal('Finished writing to disk');
+            expect(hash).to.equal(metadata.hash);
+            expect(bytes).to.equal(1024 * 1024);
+            done();
+        });
+        download.on('bytes', function(input) {
+            expect(input).to.be.above(0);
+            bytes = input;
+        });
+        download.on('progress', function(progress) {
+            expect(progress).to.be.above(0);
+            expect(progress).to.be.below(1.00000000000001);
+        });
+    });
+
+    it('Should handle a server that does not have content-length header', function(done) {
+        let download = wget.download(
+            'http://localhost:9933/',
+            '/tmp/wget-bs-test.bin'
+        );
+        download.on('error', function(err) {
+            done(err);
+        });
+        download.on('start', function(fileSize) {
+            expect(fileSize).to.be.null;
+        });
+        download.on('progress', function(progress) {
+            expect(progress).to.be.above(0);
+            expect(progress).to.be.below(1.00000000000001);
+        });
+
+        download.on('end', function() {
+            done();
+        });
+    });
+
+    it('Should not append to the previous file.', function(done) {
+        let download = wget.download(
+            'http://localhost:8884/file',
+            '/tmp/wget-test-file.bin'
+        );
+        download.on('error', function(err) {
             console.log(err);
@@ -15,17 +85,97 @@ expect(err).to.be.null;
         });
-        download.on('start', function(fileSize) {
-            expect(fileSize).to.be.a('string');
-            fileSize = Number(fileSize);
-            expect(fileSize).to.be.above(200);
-            expect(fileSize).to.be.below(500);
-        });
         download.on('end', function(output) {
+            let file = fs.readFileSync('/tmp/wget-test-file.bin');
+            let hash = crypto
+                .createHash('sha256')
+                .update(file)
+                .digest('hex');
             expect(output).to.equal('Finished writing to disk');
+            expect(hash).to.equal(metadata.hash);
             done();
-            process.exit();
         });
+    });
+
+    it('Should handle 302 redirects that end with file download.', function(done) {
+        let download = wget.download(
+            'http://localhost:8884/file/redirect',
+            '/tmp/wget-test-file2.bin'
+        );
+        download.on('end', function(output) {
+            request(baseHTTP + '/file/redirect/metadata', function(
+                err,
+                res,
+                body
+            ) {
+                let meta = JSON.parse(body);
+                let file = fs.readFileSync('/tmp/wget-test-file2.bin');
+                let hash = crypto
+                    .createHash('sha256')
+                    .update(file)
+                    .digest('hex');
+                expect(output).to.equal('Finished writing to disk');
+                expect(hash).to.equal(meta.hash);
+                done();
+            });
+        });
+    });
+
+    it('Should handle infinite redirects', function(done) {
+        let download = wget.download(
+            'http://localhost:8884/file/redirect/infinite',
+            '/tmp/wget-test-file2.bin'
+        );
+
+        download.on('error', function(err) {
+            expect(err).to.equal('Infinite redirect loop detected');
+            done();
+        });
+    });
+
+    it('Should handle relative path redirect', function(done) {
+        let download = wget.download(
+            'http://localhost:8884/file/redirect/relative',
+            '/tmp/wget-test-file3.bin'
+        );
+        download.on('error', function(err) {
+            done(err);
+        });
+        download.on('start', function(fileSize) {
+            expect(fileSize).to.be.a('number');
+            expect(fileSize).to.equal(metadata.size);
+        });
+        download.on('end', function(output) {
+            request(baseHTTP + '/file/redirect/metadata', function(
+                err,
+                res,
+                body
+            ) {
+                let meta = JSON.parse(body);
+                let file = fs.readFileSync('/tmp/wget-test-file2.bin');
+                let hash = crypto
+                    .createHash('sha256')
+                    .update(file)
+                    .digest('hex');
+                expect(output).to.equal('Finished writing to disk');
+                expect(hash).to.equal(meta.hash);
+                done();
+            });
+        });
+
         download.on('progress', function(progress) {
             expect(progress).to.be.above(0);
+            expect(progress).to.be.below(1.00000000000001);
         });
     });
-});
+
+    it('Should handle invalid protocol (no http/https)', function(done) {
+        try {
+            let download = wget.download(
+                'localhost:8884/file/redirect/infinite',
+                '/tmp/wget-test-file2.bin'
+            );
+        } catch (err) {
+            expect(err).to.equal('Your URL must use either HTTP or HTTPS.');
+            done();
+        }
+    });
+});

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

23904

increased by70.78%

Number of package files

10

increased by25%

Lines of code

494

increased by124.55%

Number of lines in readme file

117

increased by12.5%

Worsened metrics

Dev dependency count

4

increased by100%

Number of low supply chain risk alerts

2

increased by100%

Number of medium supply chain risk alerts

4

increased by100%

No dependency changes