proxy-chain - npm Package Compare versions

Comparing version 0.4.10-beta.1 to 1.0.0-beta.0

build/server.js

@@ -311,10 +311,10 @@ 'use strict';
                     //  URI in absolute-form as the request-target"
-                    var parsed = (0, _tools.parseUrl)(request.url);
 
-                    // If srcRequest.url does not match the regexp tools.HOST_HEADER_REGEX
-                    // or the url is too long it will not be parsed so we throw error here.
-                    if (!parsed) {
+                    var parsed = void 0;
+                    try {
+                        parsed = (0, _tools.parseUrl)(request.url);
+                    } catch (e) {
+                        // If URL is invalid, throw HTTP 400 error
                         throw new RequestError('Target "' + request.url + '" could not be parsed', 400);
                     }
-
                     // If srcRequest.url is something like '/some-path', this is most likely a normal HTTP request
@@ -375,12 +375,17 @@ if (!parsed.protocol) {
                 if (funcResult && funcResult.upstreamProxyUrl) {
-                    handlerOpts.upstreamProxyUrlParsed = (0, _tools.parseUrl)(funcResult.upstreamProxyUrl);
+                    try {
+                        handlerOpts.upstreamProxyUrlParsed = (0, _tools.parseUrl)(funcResult.upstreamProxyUrl);
+                    } catch (e) {
+                        throw new Error('Invalid "upstreamProxyUrl" provided: ' + e + ' (was "' + funcResult.upstreamProxyUrl + '"');
+                    }
 
-                    if (handlerOpts.upstreamProxyUrlParsed) {
-                        if (!handlerOpts.upstreamProxyUrlParsed.hostname || !handlerOpts.upstreamProxyUrlParsed.port) {
-                            throw new Error('Invalid "upstreamProxyUrl" provided: URL must have hostname and port (was "' + funcResult.upstreamProxyUrl + '")');
-                        }
-                        if (handlerOpts.upstreamProxyUrlParsed.protocol !== 'http:') {
-                            throw new Error('Invalid "upstreamProxyUrl" provided: URL must have the "http" protocol (was "' + funcResult.upstreamProxyUrl + '")');
-                        }
+                    if (!handlerOpts.upstreamProxyUrlParsed.hostname || !handlerOpts.upstreamProxyUrlParsed.port) {
+                        throw new Error('Invalid "upstreamProxyUrl" provided: URL must have hostname and port (was "' + funcResult.upstreamProxyUrl + '")'); // eslint-disable-line max-len
                     }
+                    if (handlerOpts.upstreamProxyUrlParsed.protocol !== 'http:') {
+                        throw new Error('Invalid "upstreamProxyUrl" provided: URL must have the "http" protocol (was "' + funcResult.upstreamProxyUrl + '")'); // eslint-disable-line max-len
+                    }
+                    if (/:/.test(handlerOpts.upstreamProxyUrlParsed.username)) {
+                        throw new Error('Invalid "upstreamProxyUrl" provided: The username cannot contain the colon (:) character according to RFC 7617.'); // eslint-disable-line max-len
+                    }
                 }
@@ -387,0 +392,0 @@

build/tcp_tunnel_tools.js

@@ -32,13 +32,21 @@ 'use strict';
 
+    var parsedProxyUrl = (0, _tools.parseUrl)(proxyUrl);
+    if (!parsedProxyUrl.hostname || !parsedProxyUrl.port) {
+        throw new Error('The proxy URL must contain hostname and port (was "' + proxyUrl + '")');
+    }
+    if (parsedProxyUrl.protocol !== 'http:') {
+        throw new Error('The proxy URL must have the "http" protocol (was "' + proxyUrl + '")');
+    }
+    if (/:/.test(parsedProxyUrl.username)) {
+        throw new Error('The proxy URL username cannot contain the colon (:) character according to RFC 7617.');
+    }
+
     // TODO: More and better validations - yeah, make sure targetHost is really a hostname
-    var _targetHost$split = targetHost.split(':'),
-        _targetHost$split2 = _slicedToArray(_targetHost$split, 2),
-        trgHostname = _targetHost$split2[0],
-        trgPort = _targetHost$split2[1];
 
-    if (!trgHostname || !trgPort) throw new Error('target needs to include both hostname and port.');
+    var _split = (targetHost || '').split(':'),
+        _split2 = _slicedToArray(_split, 2),
+        trgHostname = _split2[0],
+        trgPort = _split2[1];
 
-    var parsedProxyUrl = (0, _tools.parseUrl)(proxyUrl);
-    if (!parsedProxyUrl.hostname) throw new Error('proxyUrl needs to include atleast hostname');
-    if (parsedProxyUrl.protocol !== 'http:') throw new Error('Currently only "http" protocol is supported');
+    if (!trgHostname || !trgPort) throw new Error('The target host needs to include both hostname and port.');
 
@@ -51,58 +59,56 @@ var options = _extends({
 
-    var promise = new Promise(function (resolve, reject) {
-        if (options.port) return resolve(options.port);
-        // TODO: Use port: 0 instead!
-        (0, _tools.findFreePort)().then(resolve).catch(reject);
-    }).then(function (port) {
-        var server = _net2.default.createServer();
+    var server = _net2.default.createServer();
 
-        var log = function log() {
-            var _console;
+    var log = function log() {
+        var _console;
 
-            if (options.verbose) (_console = console).log.apply(_console, arguments);
-        };
+        if (options.verbose) (_console = console).log.apply(_console, arguments);
+    };
 
-        server.on('connection', function (srcSocket) {
-            runningServers[port].connections = srcSocket;
-            var remoteAddress = srcSocket.remoteAddress + ':' + srcSocket.remotePort;
-            log('new client connection from %s', remoteAddress);
+    server.on('connection', function (srcSocket) {
+        var port = server.address().port;
 
-            srcSocket.pause();
+        runningServers[port].connections = srcSocket;
+        var remoteAddress = srcSocket.remoteAddress + ':' + srcSocket.remotePort;
+        log('new client connection from %s', remoteAddress);
 
-            var tunnel = new _tcp_tunnel2.default({
-                srcSocket: srcSocket,
-                upstreamProxyUrlParsed: parsedProxyUrl,
-                trgParsed: {
-                    hostname: trgHostname,
-                    port: trgPort
-                },
-                log: log
-            });
+        srcSocket.pause();
 
-            tunnel.run();
+        var tunnel = new _tcp_tunnel2.default({
+            srcSocket: srcSocket,
+            upstreamProxyUrlParsed: parsedProxyUrl,
+            trgParsed: {
+                hostname: trgHostname,
+                port: trgPort
+            },
+            log: log
+        });
 
-            srcSocket.on('data', onConnData);
-            srcSocket.on('close', onConnClose);
-            srcSocket.on('error', onConnError);
+        tunnel.run();
 
-            function onConnData(d) {
-                log('connection data from %s: %j', remoteAddress, d);
-            }
+        srcSocket.on('data', onConnData);
+        srcSocket.on('close', onConnClose);
+        srcSocket.on('error', onConnError);
 
-            function onConnClose() {
-                log('connection from %s closed', remoteAddress);
-            }
+        function onConnData(d) {
+            log('connection data from %s: %j', remoteAddress, d);
+        }
 
-            function onConnError(err) {
-                log('Connection %s error: %s', remoteAddress, err.message);
-            }
-        });
+        function onConnClose() {
+            log('connection from %s closed', remoteAddress);
+        }
 
-        return new Promise(function (resolve) {
-            server.listen(port, function (err) {
-                if (err) return reject(err);
-                log('server listening to ', server.address());
-                runningServers[port] = { server: server, connections: [] };
-                resolve(options.hostname + ':' + port);
-            });
+        function onConnError(err) {
+            log('Connection %s error: %s', remoteAddress, err.message);
+        }
+    });
+
+    var promise = new Promise(function (resolve) {
+        // Let the system pick a random listening port
+        server.listen(0, function (err) {
+            if (err) return reject(err);
+            var address = server.address();
+            log('server listening to ', address);
+            runningServers[address.port] = { server: server, connections: [] };
+            resolve(options.hostname + ':' + address.port);
         });
@@ -109,0 +115,0 @@ });

build/tools.js

@@ -6,14 +6,8 @@ 'use strict';
 });
-exports.nodeify = exports.maybeAddProxyAuthorizationHeader = exports.findFreePort = exports.PORT_SELECTION_CONFIG = exports.addHeader = exports.parseProxyAuthorizationHeader = exports.redactParsedUrl = exports.redactUrl = exports.parseUrl = exports.isInvalidHeader = exports.isHopByHopHeader = exports.parseHostHeader = undefined;
+exports.nodeify = exports.maybeAddProxyAuthorizationHeader = exports.PORT_SELECTION_CONFIG = exports.addHeader = exports.parseProxyAuthorizationHeader = exports.redactParsedUrl = exports.redactUrl = exports.parseUrl = exports.isInvalidHeader = exports.isHopByHopHeader = exports.parseHostHeader = undefined;
 
 var _http_common = require('_http_common');
 
-var _portastic = require('portastic');
+// eslint-disable-line
 
-var _portastic2 = _interopRequireDefault(_portastic);
-
-function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
-
-// import through from 'through';
-
 var HOST_HEADER_REGEX = /^((([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9-]*[A-Za-z0-9]))(:([0-9]+))?$/;
@@ -29,3 +23,2 @@
  */
-// eslint-disable-line
 var parseHostHeader = exports.parseHostHeader = function parseHostHeader(hostHeader) {
@@ -63,14 +56,21 @@ var matches = HOST_HEADER_REGEX.exec(hostHeader || '');
 
+var bulletproofDecodeURIComponent = function bulletproofDecodeURIComponent(encodedURIComponent) {
+    try {
+        return decodeURIComponent(encodedURIComponent);
+    } catch (e) {
+        return encodedURIComponent;
+    }
+};
+
 /**
- * Wraps `new URL(url)` and adds following:
+ * Parses a URL using Node.js' `new URL(url)` and adds the following features:
  *  - `port` is casted to number / null from string
  *  - `path` field is added (pathname + search)
- *  - malformed or relative urls are parsed as well (always, the given string is returned as is in
- *    few fields, other are left undefined)
+ *  - both username and password is URI-decoded
+ *  - `auth` field is added (username + ":" + password, or empty string)
  *
- * Using `new URL` causes following:
- *  - we are unable to distiguish empty password and missing password
+ * Note that compared to the old implementation using `url.parse()`, the new function:
+ *  - is unable to distinguish empty password and missing password
  *  - password and username are empty string if not present (or empty)
  *  - we are able to parse IPv6
- *  - there might be issues with password being urlencoded
  *
@@ -81,42 +81,36 @@ * @param url
 var parseUrl = exports.parseUrl = function parseUrl(url) {
-    // NOTE: Not using url.parse() because it can't handle IPv6 and other special URLs
-    try {
-        var urlObj = new URL(url);
+    // NOTE: In the past we used url.parse() here, but it can't handle IPv6 and other special URLs,
+    // so we moved to new URL()
+    var urlObj = new URL(url);
 
-        var parsed = {
-            hash: urlObj.hash,
-            host: urlObj.host,
-            hostname: urlObj.hostname,
-            href: urlObj.href,
-            origin: urlObj.origin,
-            password: urlObj.password,
-            username: urlObj.username,
-            pathname: urlObj.pathname,
-            // Path was present on the original UrlObject, it's kept for backwards compatibility
-            path: '' + urlObj.pathname + urlObj.search,
-            // Port is turned into a number if available
-            port: urlObj.port ? parseInt(urlObj.port, 10) : null,
-            protocol: urlObj.protocol,
-            scheme: null,
-            search: urlObj.search,
-            searchParams: urlObj.searchParams
-        };
+    var parsed = {
+        auth: urlObj.username || urlObj.password ? urlObj.username + ':' + urlObj.password : '',
+        hash: urlObj.hash,
+        host: urlObj.host,
+        hostname: urlObj.hostname,
+        href: urlObj.href,
+        origin: urlObj.origin,
+        // The username and password might not be correctly URI-encoded, try to make it work anyway
+        username: bulletproofDecodeURIComponent(urlObj.username),
+        password: bulletproofDecodeURIComponent(urlObj.password),
+        pathname: urlObj.pathname,
+        // Path was present on the original UrlObject, it's kept for backwards compatibility
+        path: '' + urlObj.pathname + urlObj.search,
+        // Port is turned into a number if available
+        port: urlObj.port ? parseInt(urlObj.port, 10) : null,
+        protocol: urlObj.protocol,
+        scheme: null,
+        search: urlObj.search,
+        searchParams: urlObj.searchParams
+    };
 
-        // Add scheme field (as some other external tools rely on that)
-        if (parsed.protocol) {
-            var matches = /^([a-z0-9]+):$/i.exec(parsed.protocol);
-            if (matches && matches.length === 2) {
-                parsed.scheme = matches[1];
-            }
+    // Add scheme field (as some other external tools rely on that)
+    if (parsed.protocol) {
+        var matches = /^([a-z0-9]+):$/i.exec(parsed.protocol);
+        if (matches && matches.length === 2) {
+            parsed.scheme = matches[1];
         }
+    }
 
-        return parsed;
-    } catch (e) {
-        // Malformed (or relative) urls need to be treated as well.
-        return {
-            pathname: url,
-            path: url,
-            href: url
-        };
-    }
+    return parsed;
 };
@@ -220,23 +214,12 @@
 
-var findFreePort = exports.findFreePort = function findFreePort() {
-    // Let 'min' be a random value in the first half of the PORT_FROM-PORT_TO range,
-    // to reduce a chance of collision if other ProxyChain is started at the same time.
-    var half = Math.floor((PORT_SELECTION_CONFIG.TO - PORT_SELECTION_CONFIG.FROM) / 2);
-
-    var opts = {
-        min: PORT_SELECTION_CONFIG.FROM + Math.floor(Math.random() * half),
-        max: PORT_SELECTION_CONFIG.TO,
-        retrieve: 1
-    };
-
-    return _portastic2.default.find(opts).then(function (ports) {
-        if (ports.length < 1) throw new Error('There are no more free ports in range from ' + PORT_SELECTION_CONFIG.FROM + ' to ' + PORT_SELECTION_CONFIG.TO); // eslint-disable-line max-len
-        return ports[0];
-    });
-};
-
 var maybeAddProxyAuthorizationHeader = exports.maybeAddProxyAuthorizationHeader = function maybeAddProxyAuthorizationHeader(parsedUrl, headers) {
-    if (parsedUrl && parsedUrl.username) {
-        var auth = parsedUrl.username;
-        if (parsedUrl.password || parsedUrl.password === '') auth += ':' + parsedUrl.password;
+    if (parsedUrl && (parsedUrl.username || parsedUrl.password)) {
+        // According to RFC 7617 (see https://tools.ietf.org/html/rfc7617#page-5):
+        //  "Furthermore, a user-id containing a colon character is invalid, as
+        //   the first colon in a user-pass string separates user-id and password
+        //   from one another; text after the first colon is part of the password.
+        //   User-ids containing colons cannot be encoded in user-pass strings."
+        // So to be correct and avoid strange errors later, we just throw an error
+        if (/:/.test(parsedUrl.username)) throw new Error('The proxy username cannot contain the colon (:) character according to RFC 7617.');
+        var auth = (parsedUrl.username || '') + ':' + (parsedUrl.password || '');
         headers['Proxy-Authorization'] = 'Basic ' + Buffer.from(auth).toString('base64');
@@ -243,0 +226,0 @@ }

CHANGELOG.md

@@ -0,1 +1,28 @@
+1.0.0 / 2021-02-17
+===================
+- **BREAKING:** The `parseUrl()` function slightly changed its behavior (see README for details):
+  - it no longer returns an object on invalid URLs and throws an exception instead
+  - it URI-decodes username and password if possible
+    (if not, the function keeps the username and password as is)
+  - it adds back `auth` property for better backwards compatibility
+- The above change should make it possible to pass upstream proxy URLs containing
+  special characters, such as `http://user:pass:[e-mail]`
+  or `http://us%35er:[e-mail]`. The parsing is done on a best-effort basis.
+  The safest way is to always URI-encode username and password before constructing
+  the URL, according to RFC 3986.
+  This change should finally fix issues:
+  [#89](https://github.com/apify/proxy-chain/issues/89),
+  [#67](https://github.com/apify/proxy-chain/issues/67),
+  and [#108](https://github.com/apify/proxy-chain/issues/108)
+- **BREAKING:** Improved error handling in `createTunnel()` and `prepareRequestFunction()` functions
+  and provided better error messages. Both functions now fail if the upstream proxy
+  URL contains colon (`:`) character in the username, in order to comply with RFC 7617.
+  The functions now fail fast with a reasonable error, rather later and with cryptic errors.
+- **BREAKING:** The `createTunnel()` function now lets the system assign potentially
+  random listening TCP port, instead of the previous selection from range from 20000 to 60000.
+- **BREAKING:** The undocumented `findFreePort()` function was moved from tools.js to test/tools.js
+- Got rid of the "portastic" NPM package and thus reduced bundle size by ~50%
+- Various code improvements and better tests.
+- Updated packages.
+
 0.4.9 / 2021-01-26
@@ -2,0 +29,0 @@ ===================

package.json

 {
   "name": "proxy-chain",
-  "version": "0.4.10-beta.1",
+  "version": "1.0.0-beta.0",
   "description": "Node.js implementation of a proxy server (think Squid) with support for SSL, authentication, upstream proxy chaining, and protocol tunneling.",
@@ -46,3 +46,2 @@ "main": "build/index.js",
   "dependencies": {
-    "portastic": "^1.0.1",
     "underscore": "^1.9.1"
@@ -72,2 +71,3 @@ },
     "phantomjs-prebuilt": "^2.1.16",
+    "portastic": "^1.0.1",
     "proxy": "^1.0.1",
@@ -74,0 +74,0 @@ "request": "^2.83.0",

README.md

@@ -45,4 +45,5 @@ # Programmable HTTP proxy server for Node.js
 
-    // Custom function to authenticate proxy requests and provide the URL to chained upstream proxy.
-    // It must return an object (or promise resolving to the object) with the following form:
+    // Custom user-defined function to authenticate incoming proxy requests,
+    // and optionally provide the URL to chained upstream proxy.
+    // The function must return an object (or promise resolving to the object) with the following signature:
     // { requestAuthentication: Boolean, upstreamProxyUrl: String }
@@ -62,3 +63,4 @@ // If the function is not defined or is null, the server runs in simple mode.
         return {
-            // Require clients to authenticate with username 'bob' and password 'TopSecret'
+            // If set to true, the client is sent HTTP 407 resposne with the Proxy-Authenticate header set,
+            // requiring Basic authentication. Here you can verify user credentials.
             requestAuthentication: username !== 'bob' || password !== 'TopSecret',
@@ -69,6 +71,8 @@
             // to the target server. This field is ignored if "requestAuthentication" is true.
+            // The username and password should be URI-encoded, in case it contains some special characters.
+            // See `parseUrl()` function for details.
             upstreamProxyUrl: `http://username:[e-mail]:3128`,
 
             // If "requestAuthentication" is true, you can use the following property
-            // to define a custom error message instead of the default "Proxy credentials required"
+            // to define a custom error message to return to the client instead of the default "Proxy credentials required"
             failMsg: 'Bad username or password, please try again.',
@@ -311,7 +315,26 @@ };
 
-Parses url string with `new URL(url)` and normalizes the result (eg. port is converted to number), path (ie. pathname + search) is added
-to the result.
+An utility function for parsing URLs.
+It parses the URL using Node.js' `new URL(url)` and adds the following features:
 
-For non-urls the given string is treated as if it was relative url.
+- The result is a vanilla JavaScript object
+- `port` field is casted to number / null from string
+- `path` field is added (pathname + search)
+- both username and password is URI-decoded if possible
+  (if not, the function keeps the username and password as is)
+- `auth` field is added, and it contains username + ":" + password, or an empty string.
 
+If the URL is invalid, the function throws an error.
+
+The username and password parsing should make it possible to parse proxy URLs containing
+special characters, such as `http://user:pass:[e-mail]`
+or `http://us%35er:[e-mail]`. The parsing is done on a best-effort basis.
+The safest way is to always URI-encode username and password before constructing
+the URL, according to RFC 3986.
+
+Note that compared to the old implementation using `url.parse()`, the new function:
+
+ - is unable to distinguish empty password and missing password
+ - password and username are empty string if not present (or empty)
+ - we are able to parse IPv6
+
 ### `redactUrl(url, passwordReplacement)`
@@ -318,0 +341,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

134760

increased by2.44%

Dependency count

1

decreased by-50%

Number of lines in readme file

344

increased by7.17%

Worsened metrics

Dev dependency count

28

increased by3.7%

Lines of code

1932

decreased by-0.1%

Dependency changes

• - Removedportastic@^1.0.1

• - Removedbluebird@2.11.0(transitive)

• - Removedcommander@2.20.3(transitive)

• - Removeddebug@2.6.9(transitive)

• - Removedms@2.0.0(transitive)

• - Removedportastic@1.0.1(transitive)