declarative-crawler - npm Package Compare versions

Comparing version 0.0.11 to 0.0.12

dist/source/crawler/CrawlerScheduler.js

@@ -112,3 +112,3 @@ "use strict";
 
-        var crawlerNames, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, crawlerName, crawler;
+        var crawlerNames, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _loop, _iterator, _step;
 
@@ -132,59 +132,41 @@ return regeneratorRuntime.wrap(function _callee2$(_context2) {
                 _context2.prev = 4;
-                _iterator = crawlerNames[Symbol.iterator]();
 
-              case 6:
-                if (_iteratorNormalCompletion = (_step = _iterator.next()).done) {
-                  _context2.next = 24;
-                  break;
-                }
+                _loop = function _loop() {
+                  var crawlerName = _step.value;
 
-                crawlerName = _step.value;
-                crawler = this.crawlers[crawlerName];
-                // 当爬虫尚未运行时运行该爬虫
+                  var crawler = _this2.crawlers[crawlerName];
+                  // 当爬虫尚未运行时运行该爬虫
+                  if (!crawler.isRunning) {
+                    // 发出消息提示爬虫已启动
+                    _supervisor.dcEmitter.emit("Crawler", new _CrawlerMessage2.default(_CrawlerMessage2.default.START, crawler));
 
-                if (crawler.isRunning) {
-                  _context2.next = 21;
-                  break;
-                }
+                    // 异步运行该爬虫
+                    crawler.run().then(function (result) {
+                      _supervisor.dcEmitter.emit("Crawler", new _CrawlerMessage2.default(_CrawlerMessage2.default.FINISH, crawler));
+                    }, function (error) {
+                      // 出现异常之后，重置当前爬虫
+                      crawler.reset();
 
-                // 发出消息提示爬虫已启动
-                _supervisor.dcEmitter.emit("Crawler", new _CrawlerMessage2.default(_CrawlerMessage2.default.START, crawler));
+                      _supervisor.dcEmitter.emit("Crawler", new _CrawlerMessage2.default(_CrawlerMessage2.default.ERROR, crawler, error));
+                    });
+                  }
+                };
 
-                _context2.prev = 11;
-                _context2.next = 14;
-                return crawler.run();
+                for (_iterator = crawlerNames[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {
+                  _loop();
+                }
 
-              case 14:
-
-                _supervisor.dcEmitter.emit("Crawler", new _CrawlerMessage2.default(_CrawlerMessage2.default.FINISH, crawler));
-                _context2.next = 21;
+                // 设置定时器，更新爬虫执行安排
+                _context2.next = 13;
                 break;
 
-              case 17:
-                _context2.prev = 17;
-                _context2.t0 = _context2["catch"](11);
-
-                // 出现异常之后，重置当前爬虫
-                crawler.reset();
-
-                _supervisor.dcEmitter.emit("Crawler", new _CrawlerMessage2.default(_CrawlerMessage2.default.ERROR, crawler, _context2.t0));
-
-              case 21:
-                _iteratorNormalCompletion = true;
-                _context2.next = 6;
-                break;
-
-              case 24:
-                _context2.next = 30;
-                break;
-
-              case 26:
-                _context2.prev = 26;
-                _context2.t1 = _context2["catch"](4);
+              case 9:
+                _context2.prev = 9;
+                _context2.t0 = _context2["catch"](4);
                 _didIteratorError = true;
-                _iteratorError = _context2.t1;
+                _iteratorError = _context2.t0;
 
-              case 30:
-                _context2.prev = 30;
-                _context2.prev = 31;
+              case 13:
+                _context2.prev = 13;
+                _context2.prev = 14;
 
@@ -195,7 +177,7 @@ if (!_iteratorNormalCompletion && _iterator.return) {
 
-              case 33:
-                _context2.prev = 33;
+              case 16:
+                _context2.prev = 16;
 
                 if (!_didIteratorError) {
-                  _context2.next = 36;
+                  _context2.next = 19;
                   break;
@@ -206,11 +188,9 @@ }
 
-              case 36:
-                return _context2.finish(33);
+              case 19:
+                return _context2.finish(16);
 
-              case 37:
-                return _context2.finish(30);
+              case 20:
+                return _context2.finish(13);
 
-              case 38:
-
-                // 设置定时器，更新爬虫执行安排
+              case 21:
                 setInterval(function () {
@@ -220,3 +200,3 @@ _this2.scheduleCrawler();
 
-              case 39:
+              case 22:
               case "end":
@@ -226,3 +206,3 @@ return _context2.stop();
           }
-        }, _callee2, this, [[4, 26, 30, 38], [11, 17], [31,, 33, 37]]);
+        }, _callee2, this, [[4, 9, 13, 21], [14,, 16, 20]]);
       }));
@@ -229,0 +209,0 @@

dist/source/crawler/store/Store.js

@@ -106,2 +106,3 @@ "use strict";
 
+        // 将所有的错误信息编码存放
         crawlerStatusList.push({
@@ -113,3 +114,3 @@ name: crawler.name,
           lastFinishTime: crawlerStatistics.lastFinishTime,
-          lastError: crawlerStatistics.lastError ? JSON.parse(crawlerStatistics.lastError.message) : undefined
+          lastError: crawlerStatistics.lastError ? crawlerStatistics.lastError.message : undefined
         });
@@ -116,0 +117,0 @@ }

package.json

 {
   "name": "declarative-crawler",
-  "version": "0.0.11",
+  "version": "0.0.12",
   "description": "Declarative and Observable Distributed Crawler For Web, RDB, OS, also can act as a Monitor or ETL for your system",
@@ -5,0 +5,0 @@ "scripts": {

src/source/crawler/CrawlerScheduler.js

@@ -79,19 +79,20 @@ // @flow
 
-        try {
-          // 运行单个爬虫
-          await crawler.run();
+        // 异步运行该爬虫
+        crawler.run().then(
+          result => {
+            dcEmitter.emit(
+              "Crawler",
+              new CrawlerMessage(CrawlerMessage.FINISH, crawler)
+            );
+          },
+          error => {
+            // 出现异常之后，重置当前爬虫
+            crawler.reset();
 
-          dcEmitter.emit(
-            "Crawler",
-            new CrawlerMessage(CrawlerMessage.FINISH, crawler)
-          );
-        } catch (error) {
-          // 出现异常之后，重置当前爬虫
-          crawler.reset();
-
-          dcEmitter.emit(
-            "Crawler",
-            new CrawlerMessage(CrawlerMessage.ERROR, crawler, error)
-          );
-        }
+            dcEmitter.emit(
+              "Crawler",
+              new CrawlerMessage(CrawlerMessage.ERROR, crawler, error)
+            );
+          }
+        );
       }
@@ -98,0 +99,0 @@ }

src/source/crawler/store/Store.js

@@ -85,2 +85,3 @@ // @flow
 
+      // 将所有的错误信息编码存放
       crawlerStatusList.push({
@@ -93,3 +94,3 @@ name: crawler.name,
         lastError: crawlerStatistics.lastError
-          ? JSON.parse(crawlerStatistics.lastError.message)
+          ? crawlerStatistics.lastError.message
           : undefined
@@ -96,0 +97,0 @@ });

No alert changes

Worsened metrics

Total package byte prevSize

254681

decreased by-0.08%

Lines of code

4162

decreased by-0.22%

No dependency changes