@crawlee/memory-storage - npm Package Compare versions

Comparing version 3.11.5-beta.9 to 3.11.5-beta.10

cache-helpers.js

@@ -240,2 +240,3 @@ "use strict";
     const entries = new Set();
+    let forefrontRequestIds = [];
     for await (const entry of directoryEntries) {
@@ -257,2 +258,3 @@ if (entry.isFile()) {
                     handledRequestCount = metadata.handledRequestCount;
+                    forefrontRequestIds = metadata?.forefrontRequestIds ?? [];
                     break;
@@ -300,2 +302,4 @@ }
     newClient.handledRequestCount = handledRequestCount;
+    // @ts-expect-error - Assigning to private property
+    newClient.forefrontRequestIds = forefrontRequestIds;
     for (const requestId of entries) {
@@ -302,0 +306,0 @@ const entry = new fs_3.RequestQueueFileSystemEntry({

package.json

 {
     "name": "@crawlee/memory-storage",
-    "version": "3.11.5-beta.9",
+    "version": "3.11.5-beta.10",
     "description": "A simple in-memory storage implementation of the Apify API",
@@ -52,3 +52,3 @@ "engines": {
         "@apify/log": "^2.4.0",
-        "@crawlee/types": "3.11.5-beta.9",
+        "@crawlee/types": "3.11.5-beta.10",
         "@sapphire/async-queue": "^1.5.0",
@@ -70,3 +70,3 @@ "@sapphire/shapeshift": "^3.0.0",
     },
-    "gitHead": "416c1c9edfdf8d4f0165a9b9f87bd4a5cb1dd0ca"
+    "gitHead": "c2cf67c52d0c62a5c235af14c327fd993d4b2def"
 }

resource-clients/request-queue.d.ts

@@ -28,2 +28,3 @@ import type * as storage from '@crawlee/types';
     private readonly mutex;
+    private forefrontRequestIds;
     private readonly requests;
@@ -38,2 +39,3 @@ private readonly client;
     delete(): Promise<void>;
+    private requestKeyIterator;
     listHead(options?: storage.ListOptions): Promise<storage.QueueHead>;
@@ -40,0 +42,0 @@ listAndLockHead(options: storage.ListAndLockOptions): Promise<storage.ListAndLockHeadResult>;

resource-clients/request-queue.js

@@ -80,2 +80,8 @@ "use strict";
         });
+        Object.defineProperty(this, "forefrontRequestIds", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
         Object.defineProperty(this, "requests", {
@@ -151,2 +157,10 @@ enumerable: true,
     }
+    *requestKeyIterator(rqClient) {
+        for (let i = this.forefrontRequestIds.length - 1; i >= 0; i--) {
+            yield this.forefrontRequestIds[i];
+        }
+        for (const key of rqClient.requests.keys()) {
+            yield key;
+        }
+    }
     async listHead(options = {}) {
@@ -164,6 +178,15 @@ const { limit } = shapeshift_1.s
         const items = [];
-        for (const storageEntry of existingQueueById.requests.values()) {
+        // Tracks processed request IDs to avoid duplicates when a request is in both `forefrontRequestIds` and `requests`.
+        const seenRequestIds = new Set();
+        // Tracks handled request IDs from `forefrontRequestIds` to be removed.
+        const handledForefrontIds = new Set();
+        for (const requestId of this.requestKeyIterator(existingQueueById)) {
             if (items.length === limit) {
                 break;
             }
+            if (seenRequestIds.has(requestId)) {
+                continue;
+            }
+            seenRequestIds.add(requestId);
+            const storageEntry = existingQueueById.requests.get(requestId);
             let { orderNo } = storageEntry;
@@ -180,3 +203,7 @@ let loaded;
             }
+            else if (this.forefrontRequestIds.includes(requestId)) {
+                handledForefrontIds.add(requestId);
+            }
         }
+        this.forefrontRequestIds = this.forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
         return {
@@ -202,8 +229,20 @@ limit,
         try {
-            for (const storageEntry of queue.requests.values()) {
+            // Tracks processed request IDs to avoid duplicates (when a request is in both `forefrontRequestIds` and `requests`).
+            const seenRequestIds = new Set();
+            // Tracks handled request IDs from `forefrontRequestIds` (to be all removed at once).
+            const handledForefrontIds = new Set();
+            for (const requestId of this.requestKeyIterator(queue)) {
                 if (items.length === limit) {
                     break;
                 }
+                if (seenRequestIds.has(requestId)) {
+                    continue;
+                }
+                seenRequestIds.add(requestId);
+                const storageEntry = queue.requests.get(requestId);
                 // This is set to null when the request has been handled, so we don't need to re-fetch from fs
                 if (storageEntry.orderNo === null) {
+                    if (this.forefrontRequestIds.includes(requestId)) {
+                        handledForefrontIds.add(requestId);
+                    }
                     continue;
@@ -220,2 +259,3 @@ }
             }
+            this.forefrontRequestIds = this.forefrontRequestIds.filter((id) => !handledForefrontIds.has(id));
             return {
@@ -313,2 +353,5 @@ limit,
         }
+        if (options.forefront) {
+            this.forefrontRequestIds.push(requestModel.id);
+        }
         return {
@@ -359,2 +402,5 @@ requestId: requestModel.id,
             }
+            if (options.forefront) {
+                this.forefrontRequestIds.push(requestModel.id);
+            }
             result.processedRequests.push({
@@ -414,2 +460,5 @@ requestId: requestModel.id,
         existingQueueById.updateTimestamps(true);
+        if (options.forefront && !requestIsHandledAfterUpdate) {
+            this.forefrontRequestIds.push(requestModel.id);
+        }
         return {
@@ -460,3 +509,6 @@ requestId: requestModel.id,
         }
-        const data = this.toRequestQueueInfo();
+        const data = {
+            ...this.toRequestQueueInfo(),
+            forefrontRequestIds: this.forefrontRequestIds,
+        };
         (0, background_handler_1.scheduleBackgroundTask)({
@@ -463,0 +515,0 @@ action: 'update-metadata',

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

337099

increased by1.42%

Lines of code

2926

increased by2.02%

Dependency changes

• + Added@crawlee/types@3.11.5-beta.10(transitive)

• - Removed@crawlee/types@3.11.5-beta.9(transitive)

• Updated@crawlee/types@3.11.5-beta.10