cheerio-httpcli - npm Package Compare versions

Comparing version 0.7.2 to 0.7.3

CHANGELOG.md

@@ -0,1 +1,12 @@
+# 0.7.3 (2018-04-28)
+
+* `form`の`accept-charset`属性の複数のエンコーディング指定に対応(先頭で指定されているものを使用)
+* Node.jsバージョン10でテストが失敗したりDEPRECATEDメッセージが出るのを修正
+* サポート対象Node.jsバージョンを4に変更
+* `setBrowser()`メソッドで指定する一部のブラウザのバージョンを一部更新
+* `setBrowser()`メソッドで指定するブラウザ種類に`ps4`と`3ds`を追加
+* exampleの2chを5chに変更
+* 依存ライブラリを(ほぼ)最新バージョンに更新
+* READMEにhttps接続に関するエラーについてのTipsを追加
+
 # 0.7.2 (2017-11-10)
@@ -2,0 +13,0 @@

example/hatena-keyword.js

@@ -33,3 +33,3 @@ #!/usr/bin/env node
 .then(function (result) {
-  console.info(result.$('.keyword-body').text().trim().replace(/[\u0009\u3000]/g, ''));
+  console.info(result.$('.keyword-body').text().trim().replace(/[\t\u3000]/g, ''));
 })
@@ -36,0 +36,0 @@ .catch(function (err) {

lib/browsers.json

 {
   "ie": "Mozilla/5.0 (Windows NT 10.0; Trident/7.0; rv:11.0) like Gecko",
-  "edge": "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.79 Safari/537.36 Edge/14.14393",
-  "chrome": "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36",
-  "firefox": "Mozilla/5.0 (Windows NT 10.0; rv:54.0) Gecko/20100101 Firefox/54.0",
+  "edge": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36 Edge/15.15063",
+  "chrome": "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36",
+  "firefox": "Mozilla/5.0 (Windows NT 10.0; rv:54.0) Gecko/20100101 Firefox/59.0",
   "opera": "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; ja) Presto/2.10.289 Version/12.00",
-  "vivaldi": "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.117 Safari/537.36 Vivaldi/1.3.551.30",
-  "safari": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8) AppleWebKit/536.25 (KHTML, like Gecko) Version/6.0 Safari/536.25",
-  "ipad": "Mozilla/5.0 (iPad; CPU OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53",
-  "iphone": "Mozilla/5.0 (iPhone; CPU iPhone OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A403 Safari/8536.25",
+  "vivaldi": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.143 Safari/537.36 Vivaldi/1.95.1077.45",
+  "safari": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/603.3.8 (KHTML, like Gecko) Version/10.1.2 Safari/603.3.8",
+  "ipad": "Mozilla/5.0 (iPad; CPU OS 9_3_2 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13F69 Safari/601.1",
+  "iphone": "Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1",
   "ipod": "Mozilla/5.0 (iPod; CPU iPhone OS 5_0_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A405 Safari/7534.48.3",
-  "android": "Mozilla/5.0 (Linux; Android 4.1.1; Nexus 7 Build/JRO03S) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Safari/535.19",
+  "android": "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 7 Build/MOB30X) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.83 Safari/537.36",
+  "ps4": "Mozilla/5.0 (PlayStation 4 5.00) AppleWebKit/601.2 (KHTML, like Gecko)",
+  "3ds": "Mozilla/5.0 (Nintendo 3DS; U; ; ja) Version/1.7567.JP",
   "googlebot": "Googlebot/2.1 (+http://www.google.com/bot.html)"
 }

lib/cheerio/download.js

@@ -323,3 +323,3 @@ /*eslint no-invalid-this:0*/
         });
-        var b64buf = new Buffer(b64chk[2], 'base64');
+        var b64buf = cutil.newBuffer(b64chk[2], 'base64');
         var strm = new DownloadStream({
@@ -326,0 +326,0 @@ headers: {

lib/cheerio/submit.js

@@ -67,3 +67,4 @@ /*jshint -W100*/
     var formParamArray = [];
-    var enc = $form.attr('accept-charset') || doc.encoding;
+    // accept-charsetは先頭で指定されているものを使用
+    var enc = ($form.attr('accept-charset') || doc.encoding).split(/[\s,]+/)[0];
     each(formParam, function (val, name) {
@@ -70,0 +71,0 @@ formParamArray.push(val.map(function (vv) {

lib/cheerio/util.js

@@ -66,2 +66,15 @@ /*jshint -W100*/
 /**
+ * 汎用関数 - Buffer初期化
+ * Buffer.fromが実装されているバージョンならそちらを優先して使用
+ * cheerioとは無関係
+ *
+ * @param val 初期化内容
+ * @param type 初期化型
+ * @return Buffer
+ */
+module.exports.newBuffer = function (val, type) {
+  return (Buffer.from) ? Buffer.from(val, type) : new Buffer(val, type);
+};
+
+/**
  * 汎用関数 - 色付きメッセージ表示
@@ -68,0 +81,0 @@ * cheerioとは無関係

lib/client.js

@@ -339,3 +339,3 @@ /*jshint -W100*/
     var err = (message) ? new Error(message) : null;
-    var body = (json.body) ? new Buffer(json.body) : null;
+    var body = (json.body) ? cutil.newBuffer(json.body) : null;
     var obj = this.responseToResult(err, json.response, body, options);
@@ -342,0 +342,0 @@ if (obj.err) {

lib/encoding.js

@@ -6,2 +6,3 @@ /*jshint -W100*/
 var jschardet = require('jschardet');
+var cutil     = require('./cheerio/util');
 
@@ -143,3 +144,3 @@ /**
     // UTF-8から指定したエンコーディングに変換したバッファを回してエスケープ文字列作成
-    var buffer = new Buffer(str);
+    var buffer = cutil.newBuffer(str);
     if (! this.isUTF8(enc)) {
@@ -146,0 +147,0 @@ buffer = iconvMod.func(enc, buffer, true);

package.json

 {
   "name": "cheerio-httpcli",
-  "version": "0.7.2",
+  "version": "0.7.3",
   "description": "http client module with cheerio & iconv(-lite) & promise",
   "main": "index.js",
   "engines": {
-    "node": ">=0.12.0"
+    "node": ">=4"
   },
@@ -24,10 +24,9 @@ "scripts": {
   "dependencies": {
-    "@types/cheerio": "^0.22.5",
-    "@types/node": "^8.0.47",
-    "async": "^2.5.0",
+    "@types/cheerio": "^0.22.7",
+    "async": "^2.6.0",
     "cheerio": "^0.22.0",
-    "colors": "^1.1.2",
+    "colors": "^1.2.1",
     "foreach": "^2.0.5",
     "he": "^1.1.1",
-    "iconv-lite": "^0.4.19",
+    "iconv-lite": "^0.4.21",
     "jschardet": "^1.6.0",
@@ -37,15 +36,15 @@ "object-assign": "^4.1.1",
     "prettyjson": "^1.2.1",
-    "request": "^2.83.0",
+    "request": "^2.85.0",
     "require-uncached": "^1.0.3",
-    "rsvp": "^4.7.0",
+    "rsvp": "^4.8.2",
     "spawn-sync": "^1.0.15",
-    "tough-cookie": "^2.3.3",
+    "tough-cookie": "^2.3.4",
     "type-of": "^2.0.1",
-    "valid-url": "^1.0.9",
-    "yargs": "^10.0.3"
+    "valid-url": "^1.0.9"
   },
   "devDependencies": {
+    "@types/node": "^8.0.47",
+    "constants": "^0.0.2",
     "dev-null": "^0.1.1",
-    "constants": "^0.0.2",
-    "eslint": "^4.10.0",
+    "eslint": "^4.19.1",
     "espower-loader": "^1.2.2",
@@ -55,10 +54,11 @@ "intelli-espower-loader": "^1.0.1",
     "istanbul": "^0.4.5",
-    "mocha": "^4.0.1",
+    "mocha": "^5.1.1",
     "mocha-clean": "^1.0.0",
     "node-static": "^0.7.10",
-    "pkg-ok": "^1.1.0",
-    "power-assert": "^1.4.4",
+    "pkg-ok": "^2.2.0",
+    "power-assert": "^1.5.0",
     "pre-commit": "^1.2.2",
     "random-string": "^0.2.0",
-    "strip-ansi": "^4.0.0"
+    "strip-ansi": "^4.0.0",
+    "yargs": "^11.0.0"
   },
@@ -65,0 +65,0 @@ "repository": {

README.md

@@ -323,2 +323,4 @@ # cheerio-httpcli - Node.js用WEBスクレイピングモジュール
 * android
+* ps4
+* 3ds
 * googlebot
@@ -1178,2 +1180,16 @@
 
+### httpsページ接続時の「unable to verify the first certificate」エラー
+
+cheerio-httpcliでアクセスしたサーバー側のSSL証明書設定に不備があると発生することがあります。
+
+基本的にはサーバー側で対応してもらわないと接続はできないのですが、スクリプトの最初の方に以下の指定をするとSSL証明書検証でエラーが発生しても無理やり処理を続行してアクセスすることが可能です。
+
+```js
+process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;  // SSL証明書検証エラーを無視する設定
+
+var client = require('cheerio-httpcli');
+```
+
+ただし、この方法は安全でないサイトにアクセスできるという危険な指定なので、あくまでも緊急時においてのみ自己責任で設定するようにしてください。
+
 ### XMLの名前空間付きタグの指定方法
@@ -1228,2 +1244,2 @@
 
-© 2013-2017 [ktty1220](mailto:[e-mail])
+© 2013-2018 [ktty1220](mailto:[e-mail])

test/_helper.js

@@ -282,3 +282,4 @@ /*eslint max-statements:[1, 100]*/
   readBuffer: function (file) {
-    return new Buffer(fs.readFileSync(path.join(__dirname, file)));
+    var contents = fs.readFileSync(path.join(__dirname, file));
+    return (Buffer.from) ? Buffer.from(contents) : new Buffer(contents);
   },
@@ -285,0 +286,0 @@

test/cheerio-submit.js

@@ -336,4 +336,17 @@ /*eslint-env mocha*/
       });
+
+      it('accept-chaset属性あり(複数) => accept-charsetで指定された先頭のURLエンコードで送信される(' + expectedEncodings[enc] + ')', function () {
+        var param = { q: 'さしすせそ' };
+        return cli.fetch(helper.url('form', enc))
+        .then(function (result1) {
+          return result1.$('form[name="multi-charset"]').submit(param);
+        })
+        .then(function (result2) {
+          var actual = result2.response.headers['request-url'];
+          var expected = '/~info?q=' + escapes[param.q][expectedEncodings[enc]];
+          assert(actual === expected);
+        });
+      });
     });
   });
 });

test/https.js

@@ -60,3 +60,3 @@ /*eslint-env mocha*/
       assert(err.code === 'EPROTO');
-      assert(err.message.indexOf('handshake failure:') !== -1);
+      assert(err.message.indexOf('SSL routines:') !== -1);
       assert(err.url === url);
@@ -78,3 +78,3 @@ assert(! res);
       assert(err.code === 'EPROTO');
-      assert(err.message.indexOf('handshake failure:') !== -1);
+      assert(err.message.indexOf('SSL routines:') !== -1);
       assert(err.url === url);
@@ -81,0 +81,0 @@ assert(! res);

test/sync.js

@@ -285,2 +285,11 @@ /*eslint-env mocha*/
       });
+
+      it('accept-chaset属性あり(複数) => accept-charsetで指定された先頭のURLエンコードで送信される(' + expectedEncodings[enc] + ')', function () {
+        var param = { q: 'さしすせそ' };
+        var result1 = cli.fetchSync(helper.url('form', enc));
+        var result2 = result1.$('form[name="multi-charset"]').submitSync(param);
+        var actual = result2.response.headers['request-url'];
+        var expected = '/~info?q=' + escapes[param.q][expectedEncodings[enc]];
+        assert(actual === expected);
+      });
     });
@@ -287,0 +296,0 @@ });

No alert changes

Improved metrics

Total package byte prevSize

413707

increased by0.99%

Dependency count

18

decreased by-10%

Lines of code

7689

increased by0.48%

Number of lines in readme file

1243

increased by1.3%

Worsened metrics

Dev dependency count

17

increased by13.33%

Dependency changes

• + Added@types/node@22.13.0(transitive)

• + Addedundici-types@6.20.0(transitive)

• - Removed@types/node@^8.0.47

• - Removedyargs@^10.0.3

• - Removed@types/node@8.10.66(transitive)

• - Removedansi-regex@2.1.13.0.1(transitive)

• - Removedcamelcase@4.1.0(transitive)

• - Removedcliui@4.1.0(transitive)

• - Removedcode-point-at@1.1.0(transitive)

• - Removeddecamelize@1.2.0(transitive)

• - Removedfind-up@2.1.0(transitive)

• - Removedget-caller-file@1.0.3(transitive)

• - Removedis-fullwidth-code-point@1.0.02.0.0(transitive)

• - Removedlocate-path@2.0.0(transitive)

• - Removednumber-is-nan@1.0.1(transitive)

• - Removedp-limit@1.3.0(transitive)

• - Removedp-locate@2.0.0(transitive)

• - Removedp-try@1.0.0(transitive)

• - Removedpath-exists@3.0.0(transitive)

• - Removedrequire-directory@2.1.1(transitive)

• - Removedrequire-main-filename@1.0.1(transitive)

• - Removedset-blocking@2.0.0(transitive)

• - Removedstring-width@1.0.22.1.1(transitive)

• - Removedstrip-ansi@3.0.14.0.0(transitive)

• - Removedwhich-module@2.0.1(transitive)

• - Removedwrap-ansi@2.1.0(transitive)

• - Removedy18n@3.2.2(transitive)

• - Removedyargs@10.1.2(transitive)

• - Removedyargs-parser@8.1.0(transitive)

• Updated@types/cheerio@^0.22.7

• Updatedasync@^2.6.0

• Updatedcolors@^1.2.1

• Updatediconv-lite@^0.4.21

• Updatedrequest@^2.85.0

• Updatedrsvp@^4.8.2

• Updatedtough-cookie@^2.3.4