browser - npm Package Compare versions

Comparing version 0.1.1 to 0.1.2

browser.js

@@ -6,2 +6,3 @@ var u2r   = require('u2r');
 var CookieManager = require('./CookieManager');
+require('./buffer-concat');
 
@@ -102,3 +103,3 @@ var debug = true;
       /*
-      this.out.text = '';
+      this.out.result = '';
       this.terminate();
@@ -110,7 +111,10 @@ */
 
-  $j('charset', function(contentType) {
+  $j('contentType', function(contentType) {
+    var vals = contentType.split("; charset=");
+    contentType = vals[0];
+    var charset = vals[1] || "binary";
     if (typeof contentType != 'string' || !contentType) {
       return 'utf-8';
     }
-    return contentType.split('charset=')[1].toLowerCase();
+    return Junjo.multi(contentType, charset);
   })
@@ -124,22 +128,39 @@ .failSafe('utf-8')
 
-  $j('iconv', function(res, charset) {
+  $j('resultStream', function(res, contentType, charset) {
     var stream;
+    if (charset == "binary") {
+      stream = res;
 
-    if (charset.split(/[-_]/).join('') == 'utf8') {
-      stream = res;
+      this.absorb(stream, 'data', function(data, result) {
+        return (result) ? Buffer.concat(result, data) : data;
+      });
     }
     else {
-      if (debug) console.eyellow("converting charset", charset, "to utf-8");
-      var iconv = spawn('iconv', ['-f', charset, '-t', 'utf-8']);
-      res.pipe(iconv.stdin);
-      stream = iconv.stdout;
+      if (charset.toLowerCase().split(/[-_]/).join('') == 'utf8') {
+        stream = res;
+      }
+      else {
+        if (debug) console.eyellow("converting charset", charset, "to utf-8");
+        var iconv = spawn('iconv', ['-f', charset, '-t', 'utf-8']);
+        res.pipe(iconv.stdin);
+        stream = iconv.stdout;
+      }
+      stream.setEncoding("utf8");
+    
+      this.absorb(res, 'data', function(data, result) {
+        return (result) ? result + data : data;
+      });
+      this.length = res.headers['content-length'];
     }
-
-    this.absorbData(stream);
   })
   .post(function(err, out) {
-    this.out['text'] = out;
+    var len = out.length;
+    if (this.length && this.length != len) {
+      console.ered("content-length in header (", this.length, ") and the actual length (", len, ") don't match");
+    }
+    this.out['result'] = out;
+
   })
   .err()
-  .after('request', 'charset');
+  .after('request', 'contentType');
 
@@ -259,3 +280,3 @@
       if (debug) console.eyellow("redirected to ", out.location);
-      console.ered("original text", out.text);
+      console.ered("original result", out.result);
       var $jb = new jbrowser();
@@ -262,0 +283,0 @@ $jb.maxRedirect   = $b.maxRedirect;

package.json

 {
   "name": "browser"
-, "version": "0.1.1"
+, "version": "0.1.2"
 , "description": "browsing urls with cookies, that is, we can scrape with authenticated pages!"
@@ -5,0 +5,0 @@ , "tags": ["scraping", "browse", "url", "cookie", "crawling", "login", "sign-in"]

README.md

@@ -35,3 +35,3 @@ browse
     $b.browse(function(err, out) {
-      var window = jsdom(out.text).createWindow();
+      var window = jsdom(out.result).createWindow();
       var $ = jquery.create(window);
@@ -61,9 +61,9 @@ var postdata = {
      *   err: error object or null
-     *   out: { text: result body, ...}
+     *   out: { result : result body, ...}
      */
     $b.on("end", function(err, out) {
-      if (!out || !out.text) return res.end("no result");
+      if (!out || !out.result) return res.end("no result");
       res.writeHead(200, {'Content-Type' : 'text/html'});
-      console.log("result", out.text);
-      res.end(out.text);
+      console.log("result", out.result);
+      res.end(out.result);
     });
@@ -90,3 +90,3 @@ $b.run();
 #### keys of out object ####
- - text: response text
+ - result : response data(Buffer or String)
  - statusCode
@@ -93,0 +93,0 @@ - location

No alert changes

Improved metrics

Total package byte prevSize

16114

increased by9.81%

Number of package files

6

increased by20%

Lines of code

338

increased by15.36%

No dependency changes