urllib - npm Package Compare versions

Comparing version 0.0.1 to 0.1.0

lib/urllib.js

@@ -8,3 +8,3 @@ /**
   , qs = require('querystring')
-  , Iconv  = require('iconv').Iconv;
+  , Iconv  = require('./iconv').Iconv;
 
@@ -39,2 +39,8 @@ /**
 		charset = matchs[1].toLowerCase();
+	} else if(req.headers['content-type']) {
+		content_type = data.slice(0, end).toString();
+		matchs = CHARTSET_RE.exec(content_type);
+		if(matchs) {
+			charset = matchs[1].toLowerCase();
+		}
 	}
@@ -47,5 +53,10 @@ return charset;
 	options.method = (options.method || 'get').toLowerCase();
-	if(options.handler_redirect === undefined) {
-		options.handler_redirect = true;
+	var handle_redirect = options.handle_redirect;
+	var handle_data = options.handle_data;
+	if(handle_redirect === undefined) {
+		handle_redirect = true;
 	}
+	if(handle_data === undefined) {
+		handle_data = true;
+	}
 	var urlinfo = urllib.parse(url);
@@ -71,3 +82,9 @@ var op = {
 	}
-	return {body: body, options: op, protocol: urlinfo.protocol || 'http:'};
+	return {
+		body: body, 
+		options: op, 
+		protocol: urlinfo.protocol || 'http:', 
+		handle_redirect: handle_redirect,
+		handle_data: handle_data
+	};
 };
@@ -91,36 +108,50 @@
 		data = null;
+		options = null;
 	} else if(typeof(options) === 'function') {
 		callback = options;
+		options = null;
 	}
+	options = options || {};
 	var params = get_request_options(url, data, options);
+//	console.log(params);
 	var req =  http.request(params.options, function(res) {
-//		console.log("Got response: " + res.statusCode);
+//		console.log("Got response: " + res.statusCode, params.handle_redirect);
 //		console.log('headers:', res.headers);
-		if((res.statusCode == 301 || res.statusCode == 302) && options.handler_redirect) {
-			//console.log(res.statusCode, 'redirect', res.headers.location);
-			return urlopen(res.headers.location, callback);
+		if((res.statusCode == 301 || res.statusCode == 302) && params.handle_redirect) {
+//			console.log(res.statusCode, 'redirect', res.headers.location);
+			if(params.options.method == 'post') {
+				// redirect only support get
+				options.method = 'get';
+			}
+			return urlopen(res.headers.location, null, options, callback);
 		}
-		var buffers = [], size = 0;
-		res.on('data', function(buffer) {
-			buffers.push(buffer);
-			size += buffer.length;
-		});
-		res.on('end', function() {
-			var buffer = new Buffer(size), pos = 0;
-			for(var i = 0, len = buffers.length; i < len; i++) {
-				buffers[i].copy(buffer, pos);
-				pos += buffers[i].length;
-			}
-			// 'content-type': 'text/html;charset=gbk',
-			var charset = get_charset({headers:{}}, buffer);
-			// console.log('guess charset:', charset);
-			if(charset && charset != 'utf-8' && charset != 'utf8') {
-				var charset_iconv = new Iconv(charset, 'utf8');
-				buffer = charset_iconv.convert(buffer);
-			}
-			callback(null, buffer, res);
-		});
+		res.realurl = url;
+		// handle data or by your self
+		if(params.handle_data) {
+			var buffers = [], size = 0;
+			res.on('data', function(buffer) {
+				buffers.push(buffer);
+				size += buffer.length;
+			});
+			res.on('end', function() {
+				var buffer = new Buffer(size), pos = 0;
+				for(var i = 0, len = buffers.length; i < len; i++) {
+					buffers[i].copy(buffer, pos);
+					pos += buffers[i].length;
+				}
+				// 'content-type': 'text/html;charset=gbk',
+				var charset = get_charset({headers:{}}, buffer);
+				// console.log('guess charset:', charset);
+				if(charset && charset != 'utf-8' && charset != 'utf8') {
+					var charset_iconv = new Iconv(charset, 'utf8');
+					buffer = charset_iconv.convert(buffer);
+				}
+				callback(null, buffer, res);
+			});
+		} else {
+			callback(null, null, res);
+		}
 	});
 	req.on('error', function(e) {
-		console.log("Got error: " + e.message);
+		//console.log("Got error: " + e.message);
 		callback(e);
@@ -127,0 +158,0 @@ });

package.json

 {
-  "author": "fengmk2 <[e-mail]> (http://fengmk2.cnblogs.com)",
+  "author": "fengmk2 <[e-mail]> (http://github.com/fengmk2)",
   "name": "urllib",
   "description": "Help in opening URLs (mostly HTTP) in a complex world — basic and digest authentication, redirections, cookies and more. Like python's _urllib_ module.",
-  "version": "0.0.1",
+  "version": "0.1.0",
   "homepage": "http://github.com/fengmk2/urllib",
@@ -7,0 +7,0 @@ "repository": {

README.md

@@ -11,3 +11,5 @@ # urllib
 
-    urlget('http://www.baidu.com/', {wd: 'cnodejs'}, function(err, data, res) {
+### HTTP GET
+
+    urllib.urlget('http://www.baidu.com/', {wd: 'cnodejs'}, function(err, data, res) {
         console.log(res.statusCode);
@@ -17,1 +19,18 @@ console.log(res.headers);
     });
+
+http post juet use 'urlpost' replace 'urlget'
+
+### Fetch HTTP headers only
+
+    urllib.urlget('http://www.google.com/', null, {handle_data: false}, function(err, _, res) {
+        console.log(res.statusCode);
+        console.log(res.headers);
+    };
+    
+### Don\'t handle 301 or 302 redirect
+    
+    urllib.urlget('http://www.google.com/', null, {handle_redirect: false}, function(err, data, res) {
+        console.log(res.statusCode);
+        console.log(res.headers);
+    };
+    

test/urllib.test.js

@@ -0,2 +1,8 @@
+/**
+ * Module dependencies.
+ */
 
+var urllib = require('../')
+  , assert = require('assert');
+
 module.exports = {
@@ -6,10 +12,48 @@
 		var cases = [
-		    ['baidu.com', 'gb2312'],
+		    ['http://baidu.com', null], // <meta http-equiv="refresh" content="0;url=http://www.baidu.com/">
+		    ['http://www.baidu.com', 'gb2312'],
 		    ['http://www.baidu.com/s?kw=cnodejs', 'gbk'],
 		    ['http://cnodejs.org/', 'utf-8'],
-		    ['http://cnodejs.org/', 'utf-8'],
+		    ['http://cnodejs.org/log', null],
+		    ['http://taobao.com', 'gb2312'],
 		    ['http://weibo.com', 'utf-8']
 		];
+		cases.forEach(function(c){
+			urllib.urlget(c[0], function(err, data, res) {
+				var end = data.length;
+				if(end > 512) {
+					end = 512;
+				}
+				var log = '\nurl: ' + c[0];
+				log += '\nerror: ' + err;
+				log += '\nstatus: ' + res.statusCode;
+				log += '\nheaders: ' + JSON.stringify(res.headers);
+				log += '\nhtml head: ' + data.slice(0, end).toString();
+				var charset = urllib.get_charset(res, data);
+				assert.equal(charset, c[1], log);
+				//console.log(res.realurl);
+			});
+		});
 	},
 	
+	'redirect': function(){
+		var cases = [
+ 		    // ['http://baidu.com', 'http://www.baidu.com/'], // <meta http-equiv="refresh" content="0;url=http://www.baidu.com/">
+ 		    ['http://www.baidu.com/s?kw=cnodejs', 'http://www.baidu.com/s?kw=cnodejs'],
+ 		    ['http://cnodejs.org/', 'http://cnodejs.org/blog/'],
+ 		    ['http://taobao.com', 'http://www.taobao.com/'],
+ 		    ['http://t.cn/heVleY', 'http://user.qzone.qq.com/309389312/blog/1305512153']
+ 		];
+		cases.forEach(function(c){
+			urllib.urlget(c[0], null, {handle_data: false}, function(err, data, res) {
+				var log = '\nurl: ' + c[0];
+				log += '\nrealurl: ' + res.realurl;
+				log += '\nerror: ' + err;
+				log += '\nstatus: ' + res.statusCode;
+				log += '\nheaders: ' + JSON.stringify(res.headers);
+				assert.equal(res.realurl, c[1], log);
+			});
+		});
+	},
+	
 	'get_request_options': function() {
@@ -19,5 +63,8 @@
 	
-	'urlget': function() {
-		
-	}
+//	'urlpost': function() {
+//		urllib.urlpost('http://www.postyourtest.com/tests', {kw: 'cnodejs'}, function(err, data, res) {
+//			console.log(res.headers);
+//			console.log(data.slice(0, 512).toString())
+//		});
+//	}
 };

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

29224

increased by357.63%

Number of package files

10

increased by42.86%

Lines of code

719

increased by252.45%

Number of lines in readme file

34

increased by112.5%

Worsened metrics

Number of medium supply chain risk alerts

3

increased by200%

No dependency changes