data-sourcer - npm Package Compare versions

Comparing version 1.9.1 to 1.10.0

abstracts/list-crawler.js

@@ -250,10 +250,48 @@ 'use strict';
 
-	navigateToList: function(page, list, options, done) {
-		if (list.link.url) {
-			this.navigateByHardCodedUrl(page, list.link.url, options, done);
-		} else if (list.link.selector) {
-			this.navigateByClicking(page, list.link.selector, done);
+	evaluateInPageContext: function(page, fn, args, done) {
+		if (!_.isFunction(fn)) {
+			return done(new Error('Invalid argument ("fn"): Function expected.'));
 		}
+		if (_.isFunction(args)) {
+			done = args;
+			args = null;
+		}
+		args = args || [];
+		try {
+			_.each(args, function(arg) {
+				if (JSON.parse(JSON.stringify(arg)) !== arg) {
+					throw new Error('Arguments passed to page context must be JSON serializable.');
+				}
+			});
+		} catch (error) {
+			return done(error);
+		}
+		page.waitForNavigation().then(function() {
+			return page.evaluate.apply(page, [fn].concat(args)).then(function() {
+				done();
+			});
+		}).catch(done);
 	},
 
+	navigateToList: function(page, list, options, done) {
+		var evaluateInPageContext = this.evaluateInPageContext.bind(this);
+		var navigateByHardCodedUrl = this.navigateByHardCodedUrl.bind(this);
+		var navigateByClicking = this.navigateByClicking.bind(this);
+		async.seq(
+			function(next) {
+				if (!list.link.evaluate) return next();
+				var fn = list.link.evaluate.fn;
+				var args = list.link.evaluate.args;
+				evaluateInPageContext(page, fn, args, next);
+			},
+			function(next) {
+				if (list.link.url) {
+					navigateByHardCodedUrl(page, list.link.url, options, next);
+				} else if (list.link.selector) {
+					navigateByClicking(page, list.link.selector, next);
+				}
+			}
+		)(done);
+	},
+
 	navigateByClicking: function(page, selector, done) {
@@ -260,0 +298,0 @@ var doNavigateByClicking = async.seq(

changelog.md

 # Changelog
 
+* v1.10.0:
+  * Added link.evaluate to list-crawler abstract
+  * Updated dependencies
 * v1.9.1:
@@ -4,0 +7,0 @@ * Updated dependencies

package.json

 {
   "name": "data-sourcer",
-  "version": "1.9.1",
+  "version": "1.10.0",
   "description": "Get (and filter) data from multiple different data sources quickly and efficiently.",
@@ -8,3 +8,2 @@ "main": "index.js",
     "lint": "./node_modules/.bin/eslint --config .eslintrc.js *.js abstracts/*.js test/**/*.js",
-    "prepare": "npm run test",
     "pretest": "npm run lint",
@@ -24,4 +23,4 @@ "test": "npm run test:unit",
   "funding": {
-    "type" : "individual",
-    "url" : "https://degreesofzero.com/donate.html?project=data-sourcer"
+    "type": "individual",
+    "url": "https://degreesofzero.com/donate.html?project=data-sourcer"
   },
@@ -35,5 +34,5 @@ "readmeFilename": "readme.md",
     "debug": "4.1.1",
-    "puppeteer": "2.1.1",
+    "puppeteer": "3.0.2",
     "request": "2.88.2",
-    "underscore": "1.9.2",
+    "underscore": "1.10.2",
     "user-agents": "1.0.559",
@@ -46,4 +45,4 @@ "xml2js": "0.4.23"
     "express": "4.17.1",
-    "mocha": "7.1.1"
+    "mocha": "7.1.2"
   }
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

52190

increased by2.24%

Lines of code

1271

increased by3%

Dependency changes

• + Added@types/node@22.10.1(transitive)

• + Added@types/yauzl@2.10.3(transitive)

• + Addedbase64-js@1.5.1(transitive)

• + Addedbl@4.1.0(transitive)

• + Addedbuffer@5.7.1(transitive)

• + Addedchownr@1.1.4(transitive)

• + Addedend-of-stream@1.4.4(transitive)

• + Addedextract-zip@2.0.1(transitive)

• + Addedfs-constants@1.0.0(transitive)

• + Addedget-stream@5.2.0(transitive)

• + Addedieee754@1.2.1(transitive)

• + Addedmkdirp-classic@0.5.3(transitive)

• + Addedpump@3.0.2(transitive)

• + Addedpuppeteer@3.0.2(transitive)

• + Addedreadable-stream@3.6.2(transitive)

• + Addedrimraf@3.0.2(transitive)

• + Addedsafe-buffer@5.2.1(transitive)

• + Addedstring_decoder@1.3.0(transitive)

• + Addedtar-fs@2.1.1(transitive)

• + Addedtar-stream@2.2.0(transitive)

• + Addedthrough@2.3.8(transitive)

• + Addedunbzip2-stream@1.4.3(transitive)

• + Addedunderscore@1.10.2(transitive)

• + Addedundici-types@6.20.0(transitive)

• + Addedws@7.5.10(transitive)

• - Removedasync-limiter@1.0.1(transitive)

• - Removedbuffer-from@1.1.2(transitive)

• - Removedconcat-stream@1.6.2(transitive)

• - Removedcore-util-is@1.0.3(transitive)

• - Removeddebug@2.6.9(transitive)

• - Removedextract-zip@1.7.0(transitive)

• - Removedisarray@1.0.0(transitive)

• - Removedminimist@1.2.8(transitive)

• - Removedmkdirp@0.5.6(transitive)

• - Removedms@2.0.0(transitive)

• - Removedprocess-nextick-args@2.0.1(transitive)

• - Removedpuppeteer@2.1.1(transitive)

• - Removedreadable-stream@2.3.8(transitive)

• - Removedrimraf@2.7.1(transitive)

• - Removedsafe-buffer@5.1.2(transitive)

• - Removedstring_decoder@1.1.1(transitive)

• - Removedtypedarray@0.0.6(transitive)

• - Removedunderscore@1.9.2(transitive)

• - Removedws@6.2.3(transitive)

• Updatedpuppeteer@3.0.2

• Updatedunderscore@1.10.2