crawler.plugins.downloader - npm Package Compare versions

Comparing version 0.1.20 to 0.1.21

config.js

 module.exports = {
     "options": {
-        "seneca": {
+        "senecaOptions": {
             "timeout": 60000
@@ -9,16 +9,2 @@ }
         "pre": {
-            "redis-store-expires": {
-                "host": "47.92.126.120",
-                "port": 6379,
-                "auth": "crawler",
-                "expire": 20,
-                "entityspec": {
-                    "-/-/downloads": {
-                        "expire": 60
-                    }
-                }
-            },
-            // "consul-registry": {
-            //     "host": "47.92.126.120"
-            // },
             "mesh": {
@@ -28,9 +14,6 @@ "isbase": false,
                 "host": process.env.HOST,
-                // "discover": {
-                //     "registry": {
-                //         "active": true
-                //     }
-                // },
                 "listen": [{
-                    "pin": "role:crawler.plugin.downloader,cmd:*"
+                    "type": "tcp",
+                    "pin": "role:crawler.plugin.downloader,cmd:*",
+                    "timeout": 60000
                 }]
@@ -37,0 +20,0 @@ }

out/container.js

@@ -10,2 +10,3 @@ "use strict";
 var superagent_1 = require("./engines/superagent");
+var phantom_1 = require("./engines/phantom");
 var constants_1 = require("./constants");
@@ -21,3 +22,4 @@ var proxy_1 = require("./proxy");
 exports.container.bind(constants_1.Types.engine).to(superagent_1.SuperAgentEngine).inSingletonScope();
+exports.container.bind(constants_1.Types.engine).to(phantom_1.PhantomEngine).inSingletonScope();
 exports.container.bind(crawler_plugins_common_1.Types._plugin).to(download_1.DownloadPlugin).inSingletonScope().whenNoAncestorNamed("DownloadPlugin");
 //# sourceMappingURL=container.js.map

out/engines/superagent.js

@@ -89,2 +89,3 @@ "use strict";
                         _l.trys.push([1, 3, , 4]);
+                        console.log(path, params, data);
                         curReq = request(method.toString(), path);
@@ -91,0 +92,0 @@ params && curReq.query(params);

out/index.js

@@ -43,3 +43,4 @@ "use strict";
 var seneca = new crawler_plugins_common_1.Seneca(container_1.container, {
-    tag: "crawler.plugin.downloader"
+    tag: "crawler.plugin.downloader",
+    timeout: 60000
 });
@@ -46,0 +47,0 @@ seneca.seneca

out/plugins/download.js

@@ -120,5 +120,3 @@ "use strict";
         var url = _a.url, _b = _a.path, path = _b === void 0 ? "" : _b, params = _a.params, data = _a.data, header = _a.header, _c = _a.method, method = _c === void 0 ? "get" : _c, _d = _a.engine, engine = _d === void 0 ? "superagent" : _d, _e = _a._id, _id = _e === void 0 ? "" : _e;
-        /**
-         * 添加接口信息
-         */
+        var start = Date.now();
         this.proxy.proxy.loadConfig({
@@ -139,2 +137,3 @@ "engine": engine,
         });
+        console.log(url, "-----downloader 成功；耗时：", Date.now() - start, "ms");
         /**
@@ -141,0 +140,0 @@ * 调用接口

package.json

 {
   "name": "crawler.plugins.downloader",
-  "version": "0.1.20",
+  "version": "0.1.21",
   "description": "爬虫下载插件",
@@ -21,5 +21,6 @@ "main": "out/index.js",
   "dependencies": {
-    "crawler.plugins.common": "^0.1.20",
+    "crawler.plugins.common": "^0.1.21",
     "inversify": "^4.2.0",
     "modelproxy": "^0.5.19",
+    "node-horseman": "^3.3.0",
     "reflect-metadata": "^0.1.10",
@@ -44,2 +45,4 @@ "request": "^2.81.0",
     "@types/inversify": "^2.0.33",
+    "@types/joi": "^10.4.3",
+    "@types/nightmare": "^1.6.35",
     "@types/node": "^8.0.14",
@@ -46,0 +49,0 @@ "@types/reflect-metadata": "0.0.5",

readme.md

@@ -7,4 +7,5 @@ # 爬虫下载模块
 
-1. （request） nodejs中http请求的模块。
-2. （phantomjs）未实现。
+1.request） nodejs中http请求的模块。
+2.phantomjs）载入js，不加载image和css。
+3.(superagent) nodejs中http请求的模块。
 
@@ -11,0 +12,0 @@ > 当前支持的模式

No alert changes

Improved metrics

Total package byte prevSize

69199

increased by22.29%

Number of package files

35

increased by9.38%

Lines of code

1018

increased by22.95%

Number of lines in readme file

137

increased by0.74%

Worsened metrics

Dependency count

19

increased by5.56%

Dev dependency count

10

increased by25%

Dependency changes

• + Addednode-horseman@^3.3.0

• + Addedcookies.txt@0.1.2(transitive)

• + Addeddata-uri-to-buffer@0.0.4(transitive)

• + Addeddefaults@1.0.4(transitive)

• + Addednode-horseman@3.3.0(transitive)

• + Addednode-phantom-simple@2.2.4(transitive)

• Updatedcrawler.plugins.common@^0.1.21