@harvestapi/scraper - npm Package Compare versions

Comparing version 1.4.1 to 1.4.2

dist/base/types.d.ts

@@ -11,3 +11,3 @@ import { ApiItemResponse, ApiListResponse } from '../types';
         logger: Required<ScraperOptions>['logger'];
-    }) => any;
+    } & Partial<ApiItemResponse<TItemShot | TItemDetails>>) => any;
     onFirstPageFetched?: (args: {
@@ -14,0 +14,0 @@ data: ApiListResponse<TItemShot> | null;

dist/index.cjs.js

@@ -826,3 +826,3 @@ 'use strict';
         this.undefinedPagination = false;
-        this.onItemScraped = async ({ item }) => {
+        this.onItemScraped = async ({ item, ...apiArgs }) => {
             var _a, _b, _c, _d, _e, _f;
@@ -835,3 +835,3 @@ const logger = {
                 this.inMemoryItems.push(item);
-                void ((_b = (_a = this.options).onItemScraped) === null || _b === void 0 ? void 0 : _b.call(_a, { item, logger }));
+                void ((_b = (_a = this.options).onItemScraped) === null || _b === void 0 ? void 0 : _b.call(_a, { item, logger, ...apiArgs }));
             }
@@ -842,6 +842,6 @@ if (this.options.outputType === 'sqlite') {
                 });
-                void ((_d = (_c = this.options).onItemScraped) === null || _d === void 0 ? void 0 : _d.call(_c, { item, logger }));
+                void ((_d = (_c = this.options).onItemScraped) === null || _d === void 0 ? void 0 : _d.call(_c, { item, logger, ...apiArgs }));
             }
             if (this.options.outputType === 'callback') {
-                await ((_f = (_e = this.options).onItemScraped) === null || _f === void 0 ? void 0 : _f.call(_e, { item, logger }));
+                await ((_f = (_e = this.options).onItemScraped) === null || _f === void 0 ? void 0 : _f.call(_e, { item, logger, ...apiArgs }));
             }
@@ -942,3 +942,3 @@ };
         });
-        this.onItemScrapedQueue = createConcurrentQueues(this.options.outputType === 'sqlite' ? 1 : concurrency, ({ item }) => this.onItemScraped({ item }));
+        this.onItemScrapedQueue = createConcurrentQueues(this.options.outputType === 'sqlite' ? 1 : concurrency, ({ item, ...rest }) => this.onItemScraped({ item, ...rest }));
         this.stats.requestsStartTime = new Date();
@@ -1058,3 +1058,3 @@ this.stats.pages = 1;
                     this.stats.itemsSuccess++;
-                    await this.onItemScrapedQueue({ item: itemDetails.element });
+                    await this.onItemScrapedQueue({ item: itemDetails.element, ...itemDetails });
                     details.push(itemDetails.element);
@@ -1145,3 +1145,2 @@ }
         const results = await this.scraper.fetchApi({ path: 'linkedin/job-search', params });
-        console.log(`searchJobs results`, results.pagination);
         return results;
@@ -1148,0 +1147,0 @@ }

dist/index.esm.js

@@ -824,3 +824,3 @@ import { randomUUID } from 'crypto';
         this.undefinedPagination = false;
-        this.onItemScraped = async ({ item }) => {
+        this.onItemScraped = async ({ item, ...apiArgs }) => {
             var _a, _b, _c, _d, _e, _f;
@@ -833,3 +833,3 @@ const logger = {
                 this.inMemoryItems.push(item);
-                void ((_b = (_a = this.options).onItemScraped) === null || _b === void 0 ? void 0 : _b.call(_a, { item, logger }));
+                void ((_b = (_a = this.options).onItemScraped) === null || _b === void 0 ? void 0 : _b.call(_a, { item, logger, ...apiArgs }));
             }
@@ -840,6 +840,6 @@ if (this.options.outputType === 'sqlite') {
                 });
-                void ((_d = (_c = this.options).onItemScraped) === null || _d === void 0 ? void 0 : _d.call(_c, { item, logger }));
+                void ((_d = (_c = this.options).onItemScraped) === null || _d === void 0 ? void 0 : _d.call(_c, { item, logger, ...apiArgs }));
             }
             if (this.options.outputType === 'callback') {
-                await ((_f = (_e = this.options).onItemScraped) === null || _f === void 0 ? void 0 : _f.call(_e, { item, logger }));
+                await ((_f = (_e = this.options).onItemScraped) === null || _f === void 0 ? void 0 : _f.call(_e, { item, logger, ...apiArgs }));
             }
@@ -940,3 +940,3 @@ };
         });
-        this.onItemScrapedQueue = createConcurrentQueues(this.options.outputType === 'sqlite' ? 1 : concurrency, ({ item }) => this.onItemScraped({ item }));
+        this.onItemScrapedQueue = createConcurrentQueues(this.options.outputType === 'sqlite' ? 1 : concurrency, ({ item, ...rest }) => this.onItemScraped({ item, ...rest }));
         this.stats.requestsStartTime = new Date();
@@ -1056,3 +1056,3 @@ this.stats.pages = 1;
                     this.stats.itemsSuccess++;
-                    await this.onItemScrapedQueue({ item: itemDetails.element });
+                    await this.onItemScrapedQueue({ item: itemDetails.element, ...itemDetails });
                     details.push(itemDetails.element);
@@ -1143,3 +1143,2 @@ }
         const results = await this.scraper.fetchApi({ path: 'linkedin/job-search', params });
-        console.log(`searchJobs results`, results.pagination);
         return results;
@@ -1146,0 +1145,0 @@ }

package.json

 {
   "name": "@harvestapi/scraper",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "main": "dist/index.cjs.js",
@@ -5,0 +5,0 @@ "module": "dist/index.esm.js",

README.md

@@ -818,10 +818,4 @@ # HarvestAPI scraping tools
 
-###### item
+`object` & `Partial`\<[`ApiItemResponse`](#apiitemresponsetitem)\<`TItemShot` \| `TItemDetails`\>\>
 
-`TItemShot` \| `TItemDetails`
-
-###### logger
-
-`Required`\<[`ScraperOptions`](#scraperoptions)\>\[`"logger"`\]
-
 ###### Returns
@@ -828,0 +822,0 @@

No alert changes

Improved metrics

Total package byte prevSize

333816

0.12%

Worsened metrics

Lines of code

3004

-0.07%

Number of lines in readme file

2169

-0.28%

No dependency changes