co-urllib - npm Package Compare versions

Comparing version 0.0.1 to 0.1.0

index.js

@@ -8,2 +8,3 @@ /**!
  *   dead_horse <[e-mail]> (http://deadhorse.me)
+ *   fengmk2 <[e-mail]> (http://fengmk2.cnpmjs.org)
  */
@@ -17,22 +18,339 @@
 
-var urllib = require('urllib');
+var debug = require('debug')('co-urllib');
 var thunkify = require('thunkify');
-var _urllib = {};
+var http = require('http');
+var https = require('https');
+var urlutil = require('url');
+var qs = require('querystring');
+var path = require('path');
+var fs = require('fs');
+var zlib = require('zlib');
+var readall = require('co-readall');
+var assertTimeout = require('co-assert-timeout');
+var ua = require('default-user-agent');
+var gunzip = thunkify(zlib.gunzip);
 
-Object.keys(urllib).forEach(function (key) {
-  if (key === 'request') {
-    return;
+var pkg = JSON.parse(fs.readFileSync(path.join(__dirname, 'package.json')));
+
+var REQUEST_ID = 0;
+
+function createRequest(httplib) {
+  return function (options, args) {
+    var reqId = ++REQUEST_ID;
+    return function (done) {
+      var called = false;
+      function _done(err, result) {
+        if (called) {
+          return;
+        }
+        called = true;
+        done(err, result);
+      }
+
+      var req = httplib.request(options, function (res) {
+        _done(null, {req: req, res: res});
+      });
+
+      req.requestId = reqId;
+
+      req.on('error', function onerror(err) {
+        if (err.name === 'Error') {
+          err.name = 'RequestError';
+        }
+        debug('Request#%d %s `req error` event emit, %s: %s', reqId, options.path, err.name, err.message);
+        _done(err);
+      });
+
+      if (args.stream) {
+        args.stream.pipe(req);
+      } else {
+        req.end(args.body);
+      }
+    };
+  };
+}
+
+var httpRequest = createRequest(http);
+var httpsRequest = createRequest(https);
+
+var USER_AGENT = ua('node-co-urllib', pkg.version);
+
+// change Agent.maxSockets to 1000
+exports.agent = new http.Agent();
+exports.agent.maxSockets = 1000;
+
+exports.httpsAgent = new https.Agent();
+exports.httpsAgent.maxSockets = 1000;
+
+/**
+ * The default request timeout(in milliseconds) 5000ms.
+ * @type {Number}
+ * @const
+ */
+exports.TIMEOUT = 5000;
+
+/**
+ * Handle all http request, both http and https support well.
+ *
+ * @example
+ *
+ * // GET http://httptest.cnodejs.net
+ * var result = yield *urllib.request('http://httptest.cnodejs.net/test/get');
+ * // POST http://httptest.cnodejs.net
+ * var args = { type: 'post', data: { foo: 'bar' } };
+ * var result = yield *urllib.request('http://httptest.cnodejs.net/test/post', args);
+ *
+ * @param {String|Object} url
+ * @param {Object} [args], optional
+ *   - {Object} [data]: request data, will auto be query stringify.
+ *   - {String|Buffer} [content]: optional, if set content, `data` will ignore.
+ *   - {ReadStream} [stream]: read stream to sent.
+ *   - {WriteStream} [writeStream]: writable stream to save response data.
+ *       If you use this, callback's data should be null.
+ *       We will just `pipe(ws, {end: true})`.
+ *   - {String} [method]: optional, could be GET | POST | DELETE | PUT, default is GET
+ *   - {String} [dataType]: optional, `text` or `json`, default is text
+ *   - {Object} [headers]: optional, request headers
+ *   - {Number} [timeout]: request timeout(in milliseconds), default is `exports.TIMEOUT`
+ *   - {Agent} [agent]: optional, http agent. Set `false` if you does not use agent.
+ *   - {Agent} [httpsAgent]: optional, https agent. Set `false` if you does not use agent.
+ *   - {String} [auth]: Basic authentication i.e. 'user:password' to compute an Authorization header.
+ *   - {String|Buffer|Array} [ca]: An array of strings or Buffers of trusted certificates.
+ *       If this is omitted several well known "root" CAs will be used, like VeriSign.
+ *       These are used to authorize connections.
+ *       Notes: This is necessary only if the server uses the self-signed certificate
+ *   - {Boolean} [rejectUnauthorized]: If true, the server certificate is verified against the list of supplied CAs.
+ *       An 'error' event is emitted if verification fails. Default: true.
+ *   - {String|Buffer} [pfx]: A string or Buffer containing the private key,
+ *       certificate and CA certs of the server in PFX or PKCS12 format.
+ *   - {String|Buffer} [key]: A string or Buffer containing the private key of the client in PEM format.
+ *       Notes: This is necessary only if using the client certificate authentication
+ *   - {String|Buffer} [cert]: A string or Buffer containing the certificate key of the client in PEM format.
+ *       Notes: This is necessary only if using the client certificate authentication
+ *   - {String} [passphrase]: A string of passphrase for the private key or pfx.
+ *   - {Boolean} [followRedirect]: Follow HTTP 3xx responses as redirects. defaults to false.
+ *   - {Number} [maxRedirects]: The maximum number of redirects to follow, defaults to 10.
+ *   - {Function(options)} [beforeRequest]: Before request hook, you can change every thing here.
+ *   - {Boolean} [gzip]: Accept gzip response content and auto decode it, default is `false`.
+ * @return {Object} result, contains `data` and `res`
+ *   - {Buffer|Object} data
+ *   - {Response} res
+ * @api public
+ */
+function *request(url, args) {
+  args = args || {};
+
+  args.timeout = args.timeout || exports.TIMEOUT;
+  args.maxRedirects = args.maxRedirects || 10;
+  var parsedUrl = typeof url === 'string' ? urlutil.parse(url) : url;
+
+  var method = (args.type || args.method || parsedUrl.method || 'GET').toUpperCase();
+  var port = parsedUrl.port || 80;
+  var _request = httpRequest;
+  var agent = args.agent || exports.agent;
+
+  if (parsedUrl.protocol === 'https:') {
+    _request = httpsRequest;
+    agent = args.httpsAgent || exports.httpsAgent;
+    if (args.httpsAgent === false) {
+      agent = false;
+    }
+    if (!parsedUrl.port) {
+      port = 443;
+    }
   }
 
-  _urllib.__defineGetter__(key, function () {
-    return urllib[key];
+  if (args.agent === false) {
+    agent = false;
+  }
+
+  var options = {
+    host: parsedUrl.hostname || parsedUrl.host || 'localhost',
+    path: parsedUrl.path || '/',
+    method: method,
+    port: port,
+    agent: agent,
+    headers: args.headers || {}
+  };
+
+  var sslNames = ['ca', 'pfx', 'key', 'cert', 'passphrase'];
+  for (var i = 0; i < sslNames.length; i++) {
+    var name = sslNames[i];
+    if (args[name]) {
+      options[name] = args[name];
+    }
+  }
+
+  if (args.rejectUnauthorized !== undefined) {
+    options.rejectUnauthorized = args.rejectUnauthorized;
+  }
+
+  var auth = args.auth || parsedUrl.auth;
+  if (auth) {
+    options.auth = auth;
+  }
+
+  var body = args.content || args.data;
+  var isReadAction = method === 'GET' || method === 'HEAD';
+  if (!args.content) {
+    if (body && !(typeof body === 'string' || Buffer.isBuffer(body))) {
+      if (isReadAction) {
+        // read: GET, HEAD, use query string
+        body = qs.stringify(body);
+      } else {
+        // auto add application/x-www-form-urlencoded when using urlencode form request
+        if (!options.headers['Content-Type'] && !options.headers['content-type']) {
+          options.headers['Content-Type'] = 'application/x-www-form-urlencoded';
+        }
+
+        var contentType = options.headers['Content-Type'] || options.headers['content-type'];
+        if (contentType === 'application/json') {
+          body = JSON.stringify(body);
+        } else {
+          // 'application/x-www-form-urlencoded'
+          body = qs.stringify(body);
+        }
+      }
+    }
+  }
+
+  // if it's a GET or HEAD request, data should be sent as query string
+  if (isReadAction && body) {
+    options.path += (parsedUrl.query ? '&' : '?') + body;
+    body = null;
+  }
+
+  if (body) {
+    var length = body.length;
+    if (!Buffer.isBuffer(body)) {
+      length = Buffer.byteLength(body);
+    }
+    options.headers['Content-Length'] = length;
+  }
+
+  args.dataType = args.dataType || args.datatype;
+
+  if (args.dataType === 'json') {
+    options.headers.Accept = 'application/json';
+  }
+
+  if (typeof args.beforeRequest === 'function') {
+    // you can use this hook to change every thing.
+    args.beforeRequest(options);
+  }
+
+  // set user-agent
+  if (!options.headers['User-Agent'] && !options.headers['user-agent']) {
+    options.headers['User-Agent'] = USER_AGENT;
+  }
+
+  var enableGzip = args.gzip === true;
+  if (enableGzip) {
+    var acceptEncoding = options.headers['Accept-Encoding'] || options.headers['accept-encoding'];
+    if (acceptEncoding) {
+      enableGzip = false; // user want to handle response content decode themself
+    } else {
+      options.headers['Accept-Encoding'] = 'gzip';
+    }
+  }
+
+  debug('%s %s, headers: %j', method, url, options.headers);
+
+  var r;
+  try {
+    r = yield assertTimeout(_request(options, {body: body, stream: args.stream}), args.timeout);
+  } catch (err) {
+    if (err.status === 408) {
+      err.name = 'ConnectionTimeoutError';
+    }
+    throw err;
+  }
+
+  var req = r.req;
+  var res = r.res;
+
+  var reqId = req.requestId;
+
+  debug('Request#%d %s `req response` event emit: status %d, headers: %j',
+    reqId, options.path, res.statusCode, res.headers);
+
+  if ((res.statusCode === 302 || res.statusCode === 301) && args.followRedirect) {  // handle redirect
+    args._followRedirectCount = (args._followRedirectCount || 0) + 1;
+    if (!res.headers.location) {
+      var err = new Error('Got statusCode ' + res.statusCode + ' but cannot resolve next location from headers');
+      err.name = 'FollowRedirectError';
+      throw err;
+    } else if (args._followRedirectCount > args.maxRedirects) {
+      var err = new Error('Exceeded ' + args.maxRedirects + ' maxRedirects. Probably stuck in a redirect loop ' + url);
+      err.name = 'MaxRedirectError';
+      throw err;
+    }
+
+    var _url = urlutil.resolve(url, res.headers.location);
+    debug('Request#%d %s: `redirected` from %s to %s', reqId, options.path, url, _url);
+    return yield *request(_url, args);
+  }
+
+  var aborted = false;
+  res.on('aborted', function () {
+    aborted = true;
   });
 
-  _urllib.__defineSetter__(key, function (val) {
-    urllib[key] = val;
+  req.on('close', function () {
+    debug('Request#%d %s: `req close` event emit', reqId, options.path);
   });
-});
 
-_urllib.request = thunkify(urllib.request);
+  var data;
+  try {
+    data = yield assertTimeout(readall(res, args.writeStream), args.timeout);
+  } catch (err) {
+    err.requestId = reqId;
+    if (err.status === 408) {
+      req.abort(); // try to abort response handle
+      err.name = 'ResponseTimeoutError';
+    }
+    throw err;
+  }
 
-module.exports = _urllib;
+  if (aborted) {
+    var err = new Error('Remote socket was terminated before `response.end()` was called');
+    err.name = 'RemoteSocketClosedError';
+    throw err;
+  }
+
+  var size = data && data.length || 0;
+
+  debug('Request#%d %s %s %s got %d bytes body',
+    reqId, method, url, res.statusCode, size);
+
+  var encoding = res.headers['content-encoding'];
+
+  if (size > 0) {
+    if (enableGzip) {
+      if (encoding && encoding.toLowerCase() === 'gzip') {
+        var gzipLength = data.length;
+        data = yield gunzip(data);
+        encoding = null;
+        debug('gunzip %d bytes body to %d bytes', gzipLength, data.length);
+      }
+    }
+
+    // if response body not decode, dont touch it
+    if (!encoding && args.dataType === 'json') {
+      try {
+        data = JSON.parse(data);
+      } catch (err) {
+        err.name = 'JSONResponseFormatError';
+        throw err;
+      }
+    }
+  }
+
+  return {
+    data: data,
+    status: res.statusCode,
+    headers: res.headers
+  };
+}
+
+exports.request = request;

package.json

 {
   "name": "co-urllib",
-  "version": "0.0.1",
-  "description": "co wrapper for urllib",
+  "version": "0.1.0",
+  "description": "co version of urllib",
   "main": "index.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "make test-all"
   },
@@ -19,11 +19,24 @@ "repository": {
   },
-  "files": ["index.js"],
   "homepage": "https://github.com/dead-horse/co-urllib",
   "dependencies": {
-    "thunkify": "0.0.1",
-    "urllib": "0.5.5"
+    "co-assert-timeout": "0.0.3",
+    "co-readall": "0.0.1",
+    "debug": "0.7.4",
+    "default-user-agent": "0.0.1",
+    "thunkify": "0.0.1"
   },
   "devDependencies": {
-
+    "autod": "*",
+    "co": "3.0.4",
+    "co-sleep": "0.0.1",
+    "contributors": "*",
+    "cov": "*",
+    "istanbul": "git://github.com/gotwarlost/istanbul.git#harmony",
+    "jshint": "*",
+    "koa": "0.5.1",
+    "koa-middlewares": "0.0.9",
+    "mocha": "*",
+    "should": "3.1.3",
+    "speeds": "0.0.1"
   }
 }

README.md

 co-urllib
 =========
 
-co wraper for urllib
+co version of [urllib](https://github.com/fengmk2/urllib).
 
+Not just thunkify, `co-urllib.request` is a [Generator](http://wiki.ecmascript.org/doku.php?id=harmony:generators).
+
 [![NPM](https://nodei.co/npm/co-urllib.png?downloads=true)](https://nodei.co/npm/co-urllib/)
 
+Code coverage: [100%](http://qtestbucket.qiniudn.com/cov/html/co-urllib/0.0.1/index.html)
 
@@ -15,22 +18,36 @@ ## Usage
 
-urllib.TIME_OUT = 300;
-urllib.agent.maxSockets = 10;
-
 co(function *() {
   var result = yield urllib.request('http://baidu.com');
-  var data = result[0]; // response data
-  var res = result[1]; // response object
-  console.log(res.statusCode);
+  var data = result.data; // response data
+  var status = result.status; // response status code
+  var headers = result.headers; // response headers
+  console.log(status);
 })();
 ```
 
+More documents, please @see [API of urllib](https://github.com/fengmk2/urllib#api-doc)
+
 ## Licences
+
 (The MIT License)
 
-Copyright (c) 2013 dead-horse and other contributors
+Copyright (c) 2013 - 2014 dead-horse and other contributors
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the 'Software'), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
 
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Network access

Supply chain risk

This module accesses the network.

Found 2 instances in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Fixed alerts

No tests

Quality

Package does not have any tests. This is a strong signal of a poorly maintained or low quality package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

16854

increased by494.08%

Number of package files

9

increased by200%

Lines of code

312

increased by1014.29%

Number of low quality alerts

1

decreased by-50%

Number of lines in readme file

53

increased by47.22%

Worsened metrics

Dependency count

5

increased by150%

Dev dependency count

12

increased byInfinity%

Number of low supply chain risk alerts

1

increased byInfinity%

Number of medium supply chain risk alerts

3

increased byInfinity%

Dependency changes

• + Addedco-assert-timeout@0.0.3

• + Addedco-readall@0.0.1

• + Addeddebug@0.7.4

• + Addeddefault-user-agent@0.0.1

• + Addedco-assert-timeout@0.0.3(transitive)

• + Addedco-readall@0.0.1(transitive)

• + Addeddebug@0.7.43.2.7(transitive)

• + Addeddefault-user-agent@0.0.1(transitive)

• + Addedms@0.6.22.1.3(transitive)

• + Addedreadall@1.1.0(transitive)

• - Removedurllib@0.5.5

• - Removedbuffer-concat@0.0.1(transitive)

• - Removeddebug@0.7.2(transitive)

• - Removedurllib@0.5.5(transitive)