req-fast - npm Package Compare versions

Comparing version 0.2.18 to 0.2.19

lib/req.js

@@ -11,2 +11,3 @@ 'use strict'
 var regMeta = /<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)/i
+var regIEComments = /<!--\[if [lg]te? IE \d+\]>.+<!\[endif\]-->/gi
 
@@ -143,3 +144,3 @@ module.exports = req
   if (!charset && mime === 'text/html') {
-    var html = resp.body.toString('ascii')
+    var html = resp.body.toString('ascii').replace(regIEComments, '')
     var meta
@@ -146,0 +147,0 @@ if ((meta = html.match(regMeta)) && meta.length > 1) {

lib/util.js

@@ -6,3 +6,3 @@ 'use strict'
 var tunnel = require('tunnel2')
-var ua = require('random-ua')
+var UserAgent = require('user-agents')
 var http = require('http')
@@ -17,2 +17,3 @@ var https = require('https')
 var allowedDataType = ['JSON', 'FORM']
+var userAgent = new UserAgent()
 
@@ -158,4 +159,5 @@ util.postMethods = postMethods
   if (options.agent) {
-    debug.enabled && debug('-> generating user agent...')
-    headers['user-agent'] = ua.generate()
+    var ua = headers['user-agent'] || userAgent.random().toString()
+    debug.enabled && debug('-> generating user agent: %s', ua)
+    headers['user-agent'] = ua
   }
@@ -219,2 +221,3 @@ delete options.agent
   }
+  debug(`-> request headers: ${JSON.stringify(headers, null, 2)}`)
   options.headers = headers
@@ -221,0 +224,0 @@

package.json

 {
   "name": "req-fast",
-  "version": "0.2.18",
+  "version": "0.2.19",
   "description": "This module is designed to be the fast, lightweight way to fetch the web content(HTML stream).",
@@ -32,5 +32,5 @@ "main": "lib/req.js",
     "iconv-lite": "^0.4.15",
-    "random-ua": "0.0.6",
     "tunnel2": "0.0.4",
-    "urijs": "^1.18.4"
+    "urijs": "^1.18.4",
+    "user-agents": "^1.0.387"
   },
@@ -37,0 +37,0 @@ "devDependencies": {

No alert changes

Improved metrics

Total package byte prevSize

41276

increased by0.66%

Lines of code

672

increased by0.6%

Dependency changes

• + Addeduser-agents@^1.0.387

• + Addedlodash.clonedeep@4.5.0(transitive)

• + Addeduser-agents@1.1.470(transitive)

• - Removedrandom-ua@0.0.6

• - Removedrandom-ua@0.0.6(transitive)