@democracy-deutschland/dip21-scraper - npm Package Compare versions

Comparing version 0.1.0 to 0.1.1

distribution/scraper.js

@@ -245,20 +245,23 @@ 'use strict';
 
-    this.getFreeBrowser = () => this.stack.find(({ used }) => !used);
+    this.takeSearchableValues = (() => {
+      var _ref7 = _asyncToGenerator(function* ({ browserObj }) {
+        browserObj.used = true; // eslint-disable-line no-param-reassign
+        const searchBody = yield browserObj.browser.getBeratungsablaeufeSearchPage();
+        const searchOptions = yield browserObj.browser.getBeratungsablaeufeSearchOptions({
+          body: searchBody
+        });
+        if (searchOptions.vorgangstyp.length === 0) {
+          throw new Error();
+        }
+        browserObj.used = false; // eslint-disable-line no-param-reassign
+        return {
+          periods: searchOptions.wahlperioden,
+          operationTypes: searchOptions.vorgangstyp
+        };
+      });
 
-    this.takeSearchableValues = _asyncToGenerator(function* () {
-      const browserObj = _this.getFreeBrowser();
-      browserObj.used = true;
-      const searchBody = yield browserObj.browser.getBeratungsablaeufeSearchPage();
-      const searchOptions = yield browserObj.browser.getBeratungsablaeufeSearchOptions({
-        body: searchBody
-      });
-      if (searchOptions.vorgangstyp.length === 0) {
-        throw new Error();
-      }
-      browserObj.used = false;
-      return {
-        periods: searchOptions.wahlperioden,
-        operationTypes: searchOptions.vorgangstyp
+      return function (_x4) {
+        return _ref7.apply(this, arguments);
       };
-    });
+    })();
 
@@ -327,3 +330,3 @@ this.startSearch = (() => {
 
-      return function (_x4) {
+      return function (_x5) {
         return _ref8.apply(this, arguments);
@@ -340,3 +343,3 @@ };
 
-      return function (_x5) {
+      return function (_x6) {
         return _ref9.apply(this, arguments);
@@ -346,9 +349,15 @@ };
 
-    this.timeout = _asyncToGenerator(function* () {
-      return new Promise(function (resolve) {
-        setTimeout(function () {
-          resolve();
-        }, _.random(1000, 5000));
+    this.timeout = (() => {
+      var _ref10 = _asyncToGenerator(function* ({ min, max } = { min: 1000, max: 5000 }) {
+        return new Promise(function (resolve) {
+          setTimeout(function () {
+            resolve();
+          }, _.random(min, max));
+        });
       });
-    });
+
+      return function () {
+        return _ref10.apply(this, arguments);
+      };
+    })();
   }
@@ -367,3 +376,3 @@
           _this2.stack = yield Promise.all(_this2.createBrowserStack({
-            size: browserStackSize
+            size: Math.max(browserStackSize, 1)
           }));
@@ -379,7 +388,24 @@ stackCreated = true;
         try {
-          _this2.availableFilters = yield _this2.takeSearchableValues();
+          _this2.availableFilters = yield _this2.takeSearchableValues({ browserObj: _this2.stack[0] });
           hasData = true;
         } catch (error) {
           console.log('bundestag down (search)');
-          yield _this2.timeout();
+          yield _this2.timeout({ min: 10000, max: 10000 });
+          yield _this2.createNewBrowser({ browserObject: _this2.stack[0] }).then((() => {
+            var _ref11 = _asyncToGenerator(function* (newBrowser) {
+              _this2.stack[0] = newBrowser;
+            });
+
+            return function (_x7) {
+              return _ref11.apply(this, arguments);
+            };
+          })()).catch((() => {
+            var _ref12 = _asyncToGenerator(function* (error2) {
+              _this2.options.logError({ error2 });
+            });
+
+            return function (_x8) {
+              return _ref12.apply(this, arguments);
+            };
+          })());
         }
@@ -401,8 +427,8 @@ }
       yield Promise.all(_this2.stack.map((() => {
-        var _ref11 = _asyncToGenerator(function* (browser, browserIndex) {
+        var _ref13 = _asyncToGenerator(function* (browser, browserIndex) {
           yield _this2.startAnalyse(browserIndex);
         });
 
-        return function (_x6, _x7) {
-          return _ref11.apply(this, arguments);
+        return function (_x9, _x10) {
+          return _ref13.apply(this, arguments);
         };
@@ -441,16 +467,16 @@ })())).then(_asyncToGenerator(function* () {
           yield _this3.createNewBrowser({ browserObject: _this3.stack[browserIndex] }).then((() => {
-            var _ref13 = _asyncToGenerator(function* (newBrowser) {
+            var _ref15 = _asyncToGenerator(function* (newBrowser) {
               _this3.stack[browserIndex] = newBrowser;
             });
 
-            return function (_x8) {
-              return _ref13.apply(this, arguments);
+            return function (_x11) {
+              return _ref15.apply(this, arguments);
             };
           })()).catch((() => {
-            var _ref14 = _asyncToGenerator(function* (error) {
+            var _ref16 = _asyncToGenerator(function* (error) {
               _this3.options.logError({ error });
             });
 
-            return function (_x9) {
-              return _ref14.apply(this, arguments);
+            return function (_x12) {
+              return _ref16.apply(this, arguments);
             };
@@ -480,3 +506,3 @@ })());
           })).catch((() => {
-            var _ref16 = _asyncToGenerator(function* (error) {
+            var _ref18 = _asyncToGenerator(function* (error) {
               _this3.options.logError({ error });
@@ -498,16 +524,16 @@ _this3.procedures[linkIndex].scraped = false;
                 yield _this3.createNewBrowser({ browserObject: _this3.stack[browserIndex] }).then((() => {
-                  var _ref17 = _asyncToGenerator(function* (newBrowser) {
+                  var _ref19 = _asyncToGenerator(function* (newBrowser) {
                     _this3.stack[browserIndex] = newBrowser;
                   });
 
-                  return function (_x11) {
-                    return _ref17.apply(this, arguments);
+                  return function (_x14) {
+                    return _ref19.apply(this, arguments);
                   };
                 })()).catch((() => {
-                  var _ref18 = _asyncToGenerator(function* (error2) {
+                  var _ref20 = _asyncToGenerator(function* (error2) {
                     _this3.options.logError({ error2 });
                   });
 
-                  return function (_x12) {
-                    return _ref18.apply(this, arguments);
+                  return function (_x15) {
+                    return _ref20.apply(this, arguments);
                   };
@@ -518,4 +544,4 @@ })());
 
-            return function (_x10) {
-              return _ref16.apply(this, arguments);
+            return function (_x13) {
+              return _ref18.apply(this, arguments);
             };
@@ -522,0 +548,0 @@ })());

package.json

 {
   "name": "@democracy-deutschland/dip21-scraper",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "scraper for the german \"Bundestag\"",
@@ -5,0 +5,0 @@ "main": "distribution/scraper.js",

README.md

@@ -5,3 +5,3 @@ ![Screenshot](https://github.com/demokratie-live/democracy-assets/blob/master/images/forfb2.png)
 
-[![Build Status](https://travis-ci.org/bundestag/dip21-scraper.svg?branch=master)](https://travis-ci.org/bundestag/dip21-scraper)    [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/XXXX/badge)](https://bestpractices.coreinfrastructure.org/projects/XXXX)
+[![Build Status](https://travis-ci.org/bundestag/dip21-scraper.svg?branch=master)](https://travis-ci.org/bundestag/dip21-scraper)
 
@@ -8,0 +8,0 @@ The dip21 Scraper for <a href="https://www.bundestag.de">Bundestag.de</a>.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

272581

increased by0.3%

Lines of code

886

increased by2.67%

No dependency changes