req-fast - npm Package Compare versions

Comparing version 0.2.0 to 0.2.1

examples/basic.js

 var req = require('../');
-req('http://www.google.com', function(err, resp){
+req('http://www.bing.com', function(err, resp){
   if(err){
@@ -4,0 +4,0 @@ return console.log('[ERROR]', err.message);

lib/req_stream.js

@@ -19,9 +19,6 @@ // Copyright 2014 Tjatse
 var Stream = require('stream').Stream,
-  http = require('http'),
-  https = require('https'),
-  tunnel = require('tunnel'),
   URI = require('URIjs'),
   util = require('util'),
-  ua = require('random-ua'),
-  qs = require('querystring'),
+  http = require('http'),
+  myUtil = require('./util'),
   zlib = require('zlib');
@@ -58,5 +55,10 @@
   }
-  this.genOptions(options);
 
-  this.next(this.request);
+  var finalOptions = myUtil.genOptions(options);
+  for(var k in finalOptions){
+    this[k] = finalOptions[k];
+  }
+  finalOptions = null;
+
+  myUtil.next(this.request, this);
 };
@@ -66,5 +68,12 @@
 RequestStream.prototype.request = function () {
+  // memorize redirects
   if(typeof this.redirects == 'undefined'){
     this.redirects = [];
   }
+
+  // memorize cookies
+  if(typeof this.cookies == 'undefined' && this.options.trackCookie){
+    this.cookies = {};
+  }
+
   var req = this.client.request(this.options, function (res) {
@@ -83,5 +92,11 @@ var status = res.statusCode;
 
+        if(this.options.trackCookie){
+          var cookies = myUtil.cookieJar(res.headers['set-cookie']);
+          for(var k in cookies){
+            this.cookies[k] = cookies[k];
+          }
+        }
         // CAUTION: abort previous request at first.
         try{ req && req.abort(); req = null; }catch(err){ }
-        return this.next(this.request);
+        return myUtil.next(this.request, this);
       }else{
@@ -115,4 +130,7 @@ // just fake it, no more redirects.
       redirects: this.redirects,
+      cookies: this.cookies,
       statusCode: res.statusCode
     });
+    delete this.redirects;
+    delete this.cookies;
 
@@ -157,137 +175,2 @@ // gzip,deflate encoding
   req.end();
-};
-
-// next i/o event
-RequestStream.prototype.next = function(fn){
-  var io;
-  if (typeof setImmediate != 'undefined') {
-    io = setImmediate;
-  } else {
-    io = process.nextTick;
-  }
-  io(fn.bind(this));
-}
-
-// generate options.
-RequestStream.prototype.genOptions = function (options) {
-  // reset max redirects to three.
-  if (!options.disableRedirect && typeof options.maxRedirects != 'number') {
-    options.maxRedirects = 3;
-  }
-
-  // cookies
-  var cookies = options.cookies && Object.keys(options.cookies).map(function(key){
-    return key + '=' + options.cookies[key];
-  }).join(';');
-
-  // headers' keys with lowercase.
-  var headers = {
-    'connection':'keep-alive',
-    'accept':'text/html,text/javascript,application/json,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-    'cookie':cookies || ''
-  };
-  if (options.headers && typeof options.headers == 'object') {
-    for (var k in options.headers) {
-      headers[k.toLowerCase()] = options.headers[k];
-    }
-  }
-  delete options.cookies;
-
-  // gzip
-  if (!options.disableGzip) {
-    headers['accept-encoding'] = 'gzip,deflate,sdch';
-  } else {
-    delete headers['accept-encoding'];
-  }
-
-  // http 1.0+ no-cache
-  headers.pragma = headers.pragma || 'no-cache';
-  // http 1.1 no-cache
-  headers['cache-control'] = headers['cache-control'] || 'no-cache';
-
-  // make sure data exists;
-  if(typeof options.data != 'object' || Object.keys(options.data).length <= 0){
-    delete options.data;
-  }
-  if (!options.method && options.data) {
-    options.method = 'POST';
-  }
-  options.method && (options.method = options.method.toUpperCase());
-
-  // http 1.1
-  if(!~['OPTIONS', 'GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'TRACE', 'CONNECT'].indexOf(options.method)){
-    options.method = 'GET';
-  }
-
-  // generate user-agent.
-  if(typeof options.agent != 'boolean'){
-    options.agent = true;
-  }
-  if (options.agent) {
-    headers['user-agent'] = ua.generate();
-  }
-  delete options.agent;
-
-  // data type.
-  if(options.dataType){
-    options.dataType = options.dataType.toUpperCase();
-  }
-  if(!options.dataType || !~['JSON', 'FORM'].indexOf(options.dataType)){
-    options.dataType = 'JSON';
-  }
-  var dataUtil = {
-    'JSON': { wrapper: JSON, contentType: 'application/json' },
-    'FORM':{ wrapper: qs, contentType: 'application/x-www-form-urlencoded' }
-  };
-
-  // raw uri.
-  var uriEntity = URI(options.uri);
-  headers.host = uriEntity.host();
-
-  if (options.data) {
-    if(!!~['POST', 'PUT', 'PATCH'].indexOf(options.method)){
-      options._data = dataUtil[options.dataType].wrapper.stringify(options.data);
-      // TODO: file, multipart/form-data
-      var ctKey = 'content-type', ctExist = (ctKey in headers);
-      headers[ctKey] = (ctExist ? (headers[ctKey] + ';') : '') + dataUtil[options.dataType].contentType;
-      headers['content-length'] = Buffer.byteLength(options._data, 'utf-8');
-    }else{
-      var query;
-      uriEntity.search( qs.stringify(options.data) + ((query = uriEntity.query()) ? ('&' + query) : ''));
-    }
-  }
-
-
-  options.path = uriEntity.path() + uriEntity.search();
-  options.host = uriEntity.hostname();
-  options.port = uriEntity.port();
-
-  // http && https
-  if (uriEntity.protocol() == 'https') {
-    this.client = https;
-    !options.port && (options.port = 443);
-    // can not use protocol here, complicated with [http.request]'s options.
-    options.__protocol = 'https';
-  } else {
-    this.client = http;
-    !options.port && (options.port = 80);
-    options.__protocol = 'http';
-  }
-
-  // proxy.
-  if (typeof options.proxy != 'object' || !options.proxy.host) {
-    // CAUTION: proxy must be an JSON object, and including "host" field at least.
-    //    https://www.npmjs.org/package/tunnel
-    delete options.proxy;
-  }
-
-  if (options.proxy) {
-    options.agent = tunnel[options.__protocol + 'OverHttp']({
-      proxy:options.proxy
-    });
-    delete options.proxy;
-  }
-  options.headers = headers;
-
-  this.options = options;
 };

lib/req.js

@@ -20,2 +20,3 @@ // Copyright 2014 Tjatse
   qs = require('querystring'),
+  util = require('./util'),
   iconv = require('iconv-lite');
@@ -58,23 +59,14 @@
 
-      // get cookies
-      var resCookies = {},
-        reqCookies = resp.headers['set-cookie'];
+      resp.body = bodyBuffer;
 
-      if (reqCookies) {
-        reqCookies.forEach(function (cookies) {
-          if (!cookies) {
-            return;
-          }
-          cookies.split(';').forEach(function (cookie) {
-            var kv = cookie.split('=');
-            if (kv.length == 2 && kv[1]) {
-              resCookies[kv[0].trim()] = kv[1].trim();
-            }
-          });
-        });
+      var lastCookies = util.cookieJar(resp.headers['set-cookie']);
+      // merge cookies.
+      if(resp.cookies){
+        for(var k in lastCookies){
+          resp.cookie[k] = lastCookies[k];
+        }
+      }else{
+        resp.cookies = lastCookies;
       }
 
-      resp.body = bodyBuffer;
-      resp.cookies = resCookies;
-
       var contentType = getContentType(resp),
@@ -81,0 +73,0 @@ charset = options.charset || contentType.charset || 'utf-8';

package.json

 {
   "name": "req-fast",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "This module is designed to be the fast, lightweight way to fetch the web content(HTML stream).",
   "main": "lib/req.js",
   "scripts": {
-    "test": "./node_modules/.bin/mocha -R spec -t 20000"
+    "test": "./node_modules/.bin/mocha -R spec -t 30000"
   },
@@ -9,0 +9,0 @@ "repository": {

README.md

@@ -47,2 +47,3 @@ # req-fast
   - **disableGzip** Request compressed content from server and automatic decompress response content, if this option sets to `true`, this feature will be disabled.
+  - **trackCookie** A value indicating whether gathering all the cookies when following redirect or not, `false` by default, `false` means gathering the cookie of last request only.
   - **cookies** It should be key/value pairs.
@@ -49,0 +50,0 @@ - **headers** Http headers, it should be key/value pairs, and some default values were:

test/basic.js

@@ -7,5 +7,5 @@ var req = require('../'),
 describe('basic request', function(){
-  describe('from google', function(){
+  describe('from bing', function(){
     it('everything goes fine', function(done){
-      req('http://www.google.com', function(err, resp){
+      req('http://www.bing.com', function(err, resp){
         should.not.exist(err);
@@ -12,0 +12,0 @@ expect(resp).to.be.an('object');

test/cookie.js

@@ -42,3 +42,21 @@ var req = require('../'),
     });
+  });
+
+  describe('were sent from server', function(){
+    it('should be detected even redirecting', function(done){
+      req({
+        url: 'http://httpbin.org/cookies/set',
+        method: 'get',
+        trackCookie: true,
+        data: {
+          'username':'tjatse'
+        }
+      }, function(err, resp){
+        should.not.exist(err);
+        should.exist(resp.cookies);
+        expect(resp.cookies).to.have.property('username', 'tjatse');
+        done();
+      });
+    });
   })
 });

test/pipe.js

@@ -9,16 +9,17 @@ var req = require('../'),
 
-  describe('from google',function(){
+  describe('from bing',function(){
     it('should create file and have HTML content',function(done){
-      var pipeGoogleHome = function(){
-        req('http://www.google.com').pipe(fs.createWriteStream('google.html'));
-        // check file 5 seconds later.
-        setTimeout(function(){
-          fs.exists('google.html', function(exists){
+      var savedPath = __dirname + '/bing.html';
+      var pipePingHome = function(){
+        var rs = req('http://www.bing.com');
+        rs.pipe(fs.createWriteStream(savedPath));
+        rs.on('end', function(){
+          fs.exists(savedPath, function(exists){
             exists.should.be.ok;
             if(exists){
-              return fs.readFile('google.html', function(err, body){
+              return fs.readFile(savedPath, {'encoding': 'utf-8'}, function(err, body){
                 should.not.exist(err);
                 should.exist(body);
                 expect(body).to.match(/^\s*</);
-                try{fs.unlink('google.html')}catch(err){}
+                try{fs.unlink(savedPath)}catch(err){}
                 done();
@@ -29,9 +30,9 @@ });
           });
-        }, 5000);
+        });
       };
-      fs.exists('google.html', function(exists){
+      fs.exists(savedPath, function(exists){
         if(exists){
-          return fs.unlink('google.html', pipeGoogleHome);
+          return fs.unlink(savedPath, pipePingHome);
         }
-        pipeGoogleHome();
+        pipePingHome();
       });
@@ -38,0 +39,0 @@ });

test/redirect.js

@@ -32,3 +32,3 @@ var req = require('../'),
       req({
-        url: 'http://httpbin.org/redirect/4',
+        url: 'http://httpbin.org/redirect/2',
         maxRedirects: 10
@@ -35,0 +35,0 @@ }, function(err, resp){

No alert changes

Improved metrics

Total package byte prevSize

53883

increased by3.66%

Number of package files

27

increased by3.85%

Lines of code

1054

increased by7.11%

Number of lines in readme file

214

increased by0.47%

Worsened metrics

Number of medium supply chain risk alerts

4

increased by33.33%

No dependency changes