node-readability - npm Package Compare versions

Comparing version 0.2.1 to 0.3.0

package.json

 {
   "name": "node-readability",
-  "version": "0.2.1",
+  "version": "0.3.0",
   "author": "Zihua Li",
@@ -22,4 +22,5 @@ "description": "Turning any web page into a clean view.",
   "dependencies": {
-    "fetch": "0.3.x",
-    "jsdom": "0.8.x"
+    "jsdom": "0.8.x",
+    "request": "~2.31.0",
+    "encoding": "~0.1.7"
   },
@@ -34,4 +35,5 @@ "engines": [
     "mocha": "~1.8.2",
-    "should": "~1.2.2"
+    "should": "~2.1.1",
+    "nock": "~0.27.1"
   }
 }

README.md

@@ -46,22 +46,21 @@ # Readability
 
-node-readability will pass the options to [fetch](https://github.com/andris9/fetch) directly.
+node-readability will pass the options to [request](https://github.com/mikeal/request) directly.
+See request lib to view all available options.
 
-Possible option values
-
- * **maxRedirects** how many redirects allowed, defaults to 10
- * **disableRedirects** set to true if redirects are not allowed, defaults to false
- * **headers** optional header fields, in the form of `{'Header-Field':'value'}`
- * **maxResponseLength** maximum allowed length for the file, the remainder is cut off. Defaults to `Infinity`
- * **method** defaults to GET
- * **payload** request body
- * **disableGzip** set to false, to disable content gzipping, needed for Node v0.5.9 which has buggy zlib
- * **cookies** an array of cookie definitions in the form of `['name=val']`
- * **cookieJar** for sharing cookies between requests, see below
- * **outputEncoding**
- * **disableDecoding** set to true to disable automatic charset decoding to utf-8
- * **overrideCharset** set input encoding
- * **asyncDnsLoookup** use high performance asynchronous DNS resolution based on c-ares instead of a thread pool calling getaddrinfo(3)
- * **timeout** set a timeout in ms
- * **agent** pass-through http.request agent parameter
-
+node-readability has additional option cleanRules which allow set your own validation rule for tags.
+If true rule is valid, otherwise no.
+options.cleanRules = [callback(obj, tagName)]
+```
+read(url, {
+        cleanRulers : [
+          function(obj, tag) {
+            if(tag === 'object') {
+              if(obj.getAttribute('class') === 'BrightcoveExperience') {
+                return true;
+              }
+            }
+          }
+        ]
+      }, function(err, article, response) {});
+```
 ## article object
@@ -83,2 +82,6 @@
 
+## meta object
+
+response object from request lib. If you need to get current url after all redirect or get some headers it can be useful.
+
 The document of the web page generated by jsdom. You can use it to access the DOM directly(for example, `article.document.getElementById('main')`).
@@ -93,1 +96,5 @@
 This code is under the Apache License 2.0.  http://www.apache.org/licenses/LICENSE-2.0
+
+
+[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/luin/node-readability/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
+

src/helpers.js

@@ -23,2 +23,8 @@ var url = require("url");
 
+var cleanRules = [];
+
+module.exports.setCleanRules = function(rules) {
+  cleanRules = rules;
+};
+
 /**
@@ -389,8 +395,26 @@ * Prepare the HTML document for readability to scrape it.
 
+
+
   for (var y = targetList.length - 1; y >= 0; y--) {
-    /* Allow youtube and vimeo videos through as people usually want to see those. */
-    if (isEmbed && targetList[y].innerHTML.search(regexps.videoRe) !== -1) {
+    //------- user clean handler -----------------
+    var validRule = false;
+    for(var i = 0; i < cleanRules.length; i++) {
+      if(cleanRules[i](targetList[y], tag) === true) {
+        validRule = true;
+        break;
+      }
+    }
+
+    if(validRule) {
       continue;
     }
+    //------- end user clean handler -----------------
 
+    /* Allow youtube and vimeo videos through as people usually want to see those. */
+    if (isEmbed) {
+      if(targetList[y].innerHTML.search(regexps.videoRe) !== -1) {
+        continue;
+      }
+    }
+
     targetList[y].parentNode.removeChild(targetList[y]);
@@ -397,0 +421,0 @@ }

src/readability.js

 var jsdom = require('jsdom');
-var fetchUrl = require('fetch').fetchUrl;
+var request = require('request');
 var helpers = require('./helpers');
+var encodinglib = require("encoding");
 
+
 exports.debug = function (debug) {
@@ -11,3 +13,3 @@ helpers.debug(debug);
 
-function Readability(document) {
+function Readability(document, options) {
   this._document = document;
@@ -18,2 +20,3 @@ this.iframeLoads = 0;
   this._articleContent = '';
+  helpers.setCleanRules(options.cleanRulers || []);
 
@@ -103,2 +106,48 @@ this.cache = {};
 
+function _findHTMLCharset(htmlbuffer){
+
+  var body = htmlbuffer.toString("ascii"),
+    input, meta, charset;
+
+  if(meta = body.match(/<meta\s+http-equiv=["']content-type["'][^>]*?>/i)){
+    input = meta[0];
+  }
+
+  if(input){
+    charset = input.match(/charset\s?=\s?([a-zA-Z\-0-9]*);?/);
+    if(charset){
+      charset = (charset[1] || "").trim().toLowerCase();
+    }
+  }
+
+  if(!charset && (meta = body.match(/<meta\s+charset=["'](.*?)["']/i))){
+    charset = (meta[1] || "").trim().toLowerCase();
+  }
+
+  return charset;
+}
+
+function _parseContentType(str){
+  if(!str){
+    return {};
+  }
+  var parts = str.split(";"),
+    mimeType = parts.shift(),
+    charset, chparts;
+
+  for(var i=0, len = parts.length; i<len; i++){
+    chparts = parts[i].split("=");
+    if(chparts.length>1){
+      if(chparts[0].trim().toLowerCase() == "charset"){
+        charset = chparts[1];
+      }
+    }
+  }
+
+  return {
+    mimeType: (mimeType || "").trim().toLowerCase(),
+    charset: (charset || "UTF-8").trim().toLowerCase() // defaults to UTF-8
+  }
+}
+
 function read(html, options, callback) {
@@ -110,4 +159,26 @@ if (typeof options === 'function') {
 
+  var overrideEncoding = options.encoding;
+
+  options.encoding = null;
+
   if (html.indexOf('<') === -1) {
-    fetchUrl(html, options, jsdomParse);
+    request(html, options, function(err, res, buffer) {
+      if(err) {
+        return callback(err);
+      }
+
+      var content_type = _parseContentType(res.headers['content-type']);
+
+      if(content_type.mimeType == "text/html"){
+        content_type.charset = _findHTMLCharset(buffer) || content_type.charset;
+      }
+
+      content_type.charset = (overrideEncoding || content_type.charset || "utf-8").trim().toLowerCase();
+
+      if(!content_type.charset.match(/^utf-?8$/i)){
+        buffer = encodinglib.convert(buffer, "UTF-8", content_type.charset);
+      }
+
+      jsdomParse(null, res, buffer.toString());
+    });
   } else {
@@ -127,6 +198,12 @@ jsdomParse(null, null, html);
       done: function (errors, window) {
-        window.document.originalURL = html;
+        if(meta) {
+          window.document.originalURL = meta.request.uri.href;
+        } else {
+          window.document.originalURL = null;
+        }
+
         if (errors) return callback(errors);
         if (!window.document.body) return callback(new Error('No body tag was found.'));
-        callback(null, new Readability(window.document, options));
+        // add meta information to callback
+        callback(null, new Readability(window.document, options), meta);
       }
@@ -133,0 +210,0 @@ });

New alerts

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Found 1 instance in 1 package

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Found 1 instance in 1 package

Fixed alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

161987

377.85%

Number of package files

12

20%

Lines of code

1236

78.87%

Number of lines in readme file

98

7.69%

Worsened metrics

Dependency count

3

50%

Dev dependency count

3

50%

Number of low supply chain risk alerts

3

200%

Dependency changes

• + Added

  encoding@~0.1.7

• + Added

  request@~2.31.0

• + Added

  asn1@0.1.11
  (transitive)

• + Added

  assert-plus@0.1.5
  (transitive)

• + Added

  async@0.9.2
  (transitive)

• + Added

  aws-sign2@0.5.0
  (transitive)

• + Added

  boom@0.4.2
  (transitive)

• + Added

  combined-stream@0.0.7
  (transitive)

• + Added

  cryptiles@0.2.2
  (transitive)

• + Added

  ctype@0.5.3
  (transitive)

• + Added

  delayed-stream@0.0.5
  (transitive)

• + Added

  forever-agent@0.5.2
  (transitive)

• + Added

  form-data@0.1.4
  (transitive)

• + Added

  hawk@1.0.0
  (transitive)

• + Added

  hoek@0.9.1
  (transitive)

• + Added

  http-signature@0.10.1
  (transitive)

• + Added

  mime@1.2.11
  (transitive)

• + Added

  node-uuid@1.4.8
  (transitive)

• + Added

  oauth-sign@0.3.0
  (transitive)

• + Added

  qs@0.6.6
  (transitive)

• + Added

  request@2.31.0
  (transitive)

• + Added

  sntp@0.2.4
  (transitive)

• + Added

  tough-cookie@0.9.15
  (transitive)

• + Added

  tunnel-agent@0.3.0
  (transitive)

• - Removed

  fetch@0.3.x

• - Removed

  ajv@6.12.6
  (transitive)

• - Removed

  asn1@0.2.6
  (transitive)

• - Removed

  assert-plus@1.0.0
  (transitive)

• - Removed

  asynckit@0.4.0
  (transitive)

• - Removed

  aws-sign2@0.7.0
  (transitive)

• - Removed

  aws4@1.13.2
  (transitive)

• - Removed

  bcrypt-pbkdf@1.0.2
  (transitive)

• - Removed

  caseless@0.12.0
  (transitive)

• - Removed

  combined-stream@1.0.8
  (transitive)

• - Removed

  core-util-is@1.0.2
  (transitive)

• - Removed

  dashdash@1.14.1
  (transitive)

• - Removed

  delayed-stream@1.0.0
  (transitive)

• - Removed

  ecc-jsbn@0.1.2
  (transitive)

• - Removed

  extend@3.0.2
  (transitive)

• - Removed

  extsprintf@1.3.0
  (transitive)

• - Removed

  fast-deep-equal@3.1.3
  (transitive)

• - Removed

  fast-json-stable-stringify@2.1.0
  (transitive)

• - Removed

  fetch@0.3.6
  (transitive)

• - Removed

  forever-agent@0.6.1
  (transitive)

• - Removed

  form-data@2.3.3
  (transitive)

• - Removed

  getpass@0.1.7
  (transitive)

• - Removed

  har-schema@2.0.0
  (transitive)

• - Removed

  har-validator@5.1.5
  (transitive)

• - Removed

  http-signature@1.2.0
  (transitive)

• - Removed

  is-typedarray@1.0.0
  (transitive)

• - Removed

  isstream@0.1.2
  (transitive)

• - Removed

  jsbn@0.1.1
  (transitive)

• - Removed

  json-schema@0.4.0
  (transitive)

• - Removed

  json-schema-traverse@0.4.1
  (transitive)

• - Removed

  jsprim@1.4.2
  (transitive)

• - Removed

  mime-db@1.52.0
  (transitive)

• - Removed

  mime-types@2.1.35
  (transitive)

• - Removed

  oauth-sign@0.9.0
  (transitive)

• - Removed

  performance-now@2.1.0
  (transitive)

• - Removed

  psl@1.15.0
  (transitive)

• - Removed

  qs@6.5.3
  (transitive)

• - Removed

  request@2.88.2
  (transitive)

• - Removed

  sshpk@1.18.0
  (transitive)

• - Removed

  tough-cookie@2.5.0
  (transitive)

• - Removed

  tunnel-agent@0.6.0
  (transitive)

• - Removed

  tweetnacl@0.14.5
  (transitive)

• - Removed

  uri-js@4.4.1
  (transitive)

• - Removed

  uuid@3.4.0
  (transitive)

• - Removed

  verror@1.10.0
  (transitive)