@crawlee/core - npm Package Compare versions

crawlers/statistics.d.ts

		@@ -0,1 +1,2 @@
		import { KeyValueStore } from '../storages/key_value_store';
		import { Configuration } from '../configuration';
		@@ -37,3 +38,3 @@ /**
		readonly requestRetryHistogram: number[];
		private keyValueStore;
		private keyValueStore?;
		private persistStateKey;
		@@ -111,2 +112,3 @@ private logIntervalMillis;
		logMessage?: string;
		keyValueStore?: KeyValueStore;
		config?: Configuration;
		@@ -113,0 +115,0 @@ }

crawlers/statistics.js

		@@ -5,3 +5,2 @@ "use strict";
		const tslib_1 = require("tslib");
		/* eslint-disable max-classes-per-file */
		const ow_1 = tslib_1.__importDefault(require("ow"));
		@@ -93,3 +92,3 @@ const log_1 = require("../log");
		writable: true,
		value: null
		value: undefined
		});
		@@ -153,7 +152,9 @@ Object.defineProperty(this, "persistStateKey", {
		logMessage: ow_1.default.optional.string,
		keyValueStore: ow_1.default.optional.object,
		config: ow_1.default.optional.object,
		}));
		const { logIntervalSecs = 60, logMessage = 'Statistics', config = configuration_1.Configuration.getGlobalConfig(), } = options;
		const { logIntervalSecs = 60, logMessage = 'Statistics', keyValueStore, config = configuration_1.Configuration.getGlobalConfig(), } = options;
		this.logIntervalMillis = logIntervalSecs * 1000;
		this.logMessage = logMessage;
		this.keyValueStore = keyValueStore;
		this.listener = this.persistState.bind(this);
		@@ -252,3 +253,3 @@ this.events = config.getEventManager();
		async startCapturing() {
		this.keyValueStore = await key_value_store_1.KeyValueStore.open();
		this.keyValueStore ?? (this.keyValueStore = await key_value_store_1.KeyValueStore.open());
		await this._maybeLoadStatistics();
		@@ -255,0 +256,0 @@ if (this.state.crawlerStartedAt === null) {

events/local_event_manager.js

		@@ -5,3 +5,2 @@ "use strict";
		const tslib_1 = require("tslib");
		// eslint-disable-next-line max-classes-per-file
		const os_1 = tslib_1.__importDefault(require("os"));
		@@ -8,0 +7,0 @@ const log_1 = tslib_1.__importDefault(require("@apify/log"));

package.json

		{
		"name": "@crawlee/core",
		"version": "3.0.0-alpha.42",
		"version": "3.0.0-alpha.43",
		"description": "The scalable web crawling and scraping library for JavaScript/Node.js. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
		@@ -52,3 +52,3 @@ "engines": {
		"dependencies": {
		"@apify/consts": "^1.4.2",
		"@apify/consts": "^1.7.0",
		"@apify/datastructures": "^1.0.1",
		@@ -58,15 +58,12 @@ "@apify/log": "^1.2.0",
		"@apify/timeout": "^0.2.1",
		"@apify/utilities": "^1.2.5",
		"@crawlee/browser-pool": "^3.0.0-alpha.42",
		"@crawlee/utils": "^3.0.0-alpha.42",
		"@crawlee/memory-storage": "^3.0.0-alpha.42",
		"got-scraping": "^3.2.7",
		"@crawlee/browser-pool": "^3.0.0-alpha.43",
		"@crawlee/memory-storage": "^3.0.0-alpha.43",
		"iconv-lite": "^0.6.3",
		"minimatch": "^5.0.1",
		"ow": "^0.28.0",
		"stream-json": "^1.7.3",
		"ow": "^0.28.1",
		"stream-json": "^1.7.4",
		"tldts": "^5.7.69",
		"tough-cookie": "^4.0.0",
		"ws": "^7.5.3"
		"ws": "^7.5.7"
		}
		}

README.md

		@@ -21,2 +21,4 @@ # Crawlee: The scalable web crawling and scraping library for JavaScript

		> Would you like to work with us on Apify SDK or similar projects? [We are hiring!](https://apify.com/jobs#senior-node.js-engineer)

		## Motivation
		@@ -23,0 +25,0 @@

storages/request_list.js

		@@ -600,3 +600,3 @@ "use strict";
		// Skip requests with duplicate uniqueKey
		if (!this.uniqueKeyToIndex.hasOwnProperty(uniqueKey)) { // eslint-disable-line no-prototype-builtins
		if (!this.uniqueKeyToIndex.hasOwnProperty(uniqueKey)) {
		this.uniqueKeyToIndex[uniqueKey] = this.requests.length;
		@@ -606,3 +606,3 @@ this.requests.push(request);
		else if (this.keepDuplicateUrls) {
		this.log.warning(`Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`); // eslint-disable-line max-len
		this.log.warning(`Duplicate uniqueKey: ${uniqueKey} found while the keepDuplicateUrls option was set. Check your sources' unique keys.`);
		}
		@@ -609,0 +609,0 @@ }

crawlers/statistics.d.ts.map