scarlet-task - npm Package Compare versions

Comparing version 2.1.0 to 3.0.0

lib/index.d.ts

 /**
- * Nyaa ヾ(●゜▽゜●)♡ ～ This module name is to remember its
- * initial use for me - to find a touhou project
- * song.
+ * Nyaa ヾ(●゜▽゜●)♡ ～ This module name is to remember its initial use for me -
+ * to find a Touhou Project song.
  *
- * If no task queue, the website you crawled will
- * be DDOSed by your node.js code.
+ * If no task queue, the website you crawled will be DDoSed by your Node.js
+ * code.
  *
  * So you need to have a task queue.
  *
- * Otherwise, you can use this queue not only in
- * crawler.
+ * Otherwise, you can use this queue not only in crawler.
  *
  * (　ﾟ∀ﾟ)つ≡≡≡♡♡♡)`ν゜)ｸﾞｼｬ
  *
- *                                   -- XadillaX
- *                                    05/06/2014
+ *                                                                   -- XadillaX
+ *                                                                    05/06/2014
  */
 export * from './scarlet';
-export * from './task_object';
+export * from './task-object';

lib/index.js

@@ -19,21 +19,19 @@ "use strict";
 /**
- * Nyaa ヾ(●゜▽゜●)♡ ～ This module name is to remember its
- * initial use for me - to find a touhou project
- * song.
+ * Nyaa ヾ(●゜▽゜●)♡ ～ This module name is to remember its initial use for me -
+ * to find a Touhou Project song.
  *
- * If no task queue, the website you crawled will
- * be DDOSed by your node.js code.
+ * If no task queue, the website you crawled will be DDoSed by your Node.js
+ * code.
  *
  * So you need to have a task queue.
  *
- * Otherwise, you can use this queue not only in
- * crawler.
+ * Otherwise, you can use this queue not only in crawler.
  *
  * (　ﾟ∀ﾟ)つ≡≡≡♡♡♡)`ν゜)ｸﾞｼｬ
  *
- *                                   -- XadillaX
- *                                    05/06/2014
+ *                                                                   -- XadillaX
+ *                                                                    05/06/2014
  */
 /* eslint-enable */
 __exportStar(require("./scarlet"), exports);
-__exportStar(require("./task_object"), exports);
+__exportStar(require("./task-object"), exports);

lib/scarlet.d.ts

 /// <reference types="node" />
 import { EventEmitter } from 'events';
-import { TaskObject } from './task_object';
+import { TaskObject } from './task-object';
 export type TaskProcessor<T> = (task: TaskObject<T>) => void | Promise<void>;
 export type AfterFinishProcessor = () => void | Promise<void>;
 interface IQueuingItem<T> {
-    queueId: number;
+    queueId: number | null;
     task: T;
     processor: TaskProcessor<T>;
 }
-type IQueuingItemArray<T> = IQueuingItem<T>[];
+/**
+ * Scarlet is a task queue that allows you to process tasks in parallel.
+ */
 export declare class Scarlet {
@@ -17,4 +19,5 @@ #private;
     processedCount: number;
-    queue: IQueuingItemArray<any>[];
+    spouts: (IQueuingItem<any> | null)[];
     running: boolean[];
+    queue: IQueuingItem<any>[];
     emitter: EventEmitter;
@@ -24,10 +27,39 @@ afterFinishCount: number;
     afterFinishProcessor: AfterFinishProcessor | undefined;
+    /**
+     * Create a new Scarlet instance.
+     * @param {number} queueCount The number of queues to create.
+     */
     constructor(queueCount?: number);
+    /**
+     * Get the number of processed tasks.
+     * @return {number} The number of processed tasks.
+     */
     numberOfProcessed(): number;
+    /**
+     * Reset the number of processed tasks.
+     */
     resetNumberOfProcessed(): void;
-    push<T>(task: T, processor: TaskProcessor<T>, debugStr?: boolean): void;
-    taskDone<T>(taskObject: TaskObject<T>, debugStr?: boolean): void;
+    /**
+     * Push a task into the queue.
+     * @param {T} task The task to push into the queue.
+     * @param {TaskProcessor<T>} processor The processor to run the task.
+     */
+    push<T>(task: T, processor: TaskProcessor<T>): void;
+    /**
+     * Mark a task as done.
+     * @param {TaskObject<T>} taskObject The task object to mark as done.
+     */
+    taskDone<T>(taskObject: TaskObject<T>): void;
+    /**
+     * Set the after finish processor.
+     * @param {number} count The number of tasks to process before calling the processor.
+     * @param {AfterFinishProcessor} processor The processor to call after the tasks are processed.
+     * @param {boolean} loop Whether to loop the after finish processor.
+     */
     afterFinish(count: number, processor: AfterFinishProcessor, loop?: boolean): void;
+    /**
+     * Clear the after finish processor.
+     */
     clearAfterFinish(): void;
 }
 export {};

lib/scarlet.js

@@ -12,3 +12,3 @@ "use strict";
 const events_1 = require("events");
-const task_object_1 = require("./task_object");
+const task_object_1 = require("./task-object");
 let asyncRun;
@@ -21,8 +21,16 @@ if (!((_a = globalThis.process) === null || _a === void 0 ? void 0 : _a.nextTick)) {
 }
+/**
+ * Scarlet is a task queue that allows you to process tasks in parallel.
+ */
 class Scarlet {
+    /**
+     * Create a new Scarlet instance.
+     * @param {number} queueCount The number of queues to create.
+     */
     constructor(queueCount = 1) {
         _Scarlet_instances.add(this);
         this.processedCount = 0;
+        this.spouts = [];
+        this.running = [];
         this.queue = [];
-        this.running = [];
         this.emitter = new events_1.EventEmitter();
@@ -33,70 +41,74 @@ this.afterFinishCount = -1;
         for (let i = 0; i < queueCount; i++) {
-            this.queue.push([]);
+            this.spouts.push(null);
             this.running.push(false);
         }
-        this.emitter.on('done', (queueId, debugStr) => {
+        this.emitter.on('done', (queueId) => {
             if (queueId >= this.queueCount || queueId < 0) {
                 throw new Error(`Invalid queueId, ${queueId} !== ${this.queueCount}`);
             }
-            if (debugStr) {
-                console.log(`Scarlet done (${this.queue.map(q => q.length).join(', ')})`);
+            this.spouts[queueId] = null;
+            this.processedCount++;
+            // Call the `afterFinishProcessor` if it's set.
+            let needRunFinishProcessor;
+            if (this.afterFinishLoop) {
+                needRunFinishProcessor = !(this.processedCount % this.afterFinishCount);
             }
-            asyncRun(() => {
-                __classPrivateFieldGet(this, _Scarlet_instances, "m", _Scarlet_runTask).call(this, queueId);
-            });
+            else {
+                needRunFinishProcessor = this.processedCount === this.afterFinishCount;
+            }
+            if (needRunFinishProcessor && this.processedCount !== -1 && typeof this.afterFinishProcessor === 'function') {
+                this.afterFinishProcessor();
+                if (!this.afterFinishLoop) {
+                    this.clearAfterFinish();
+                }
+            }
+            asyncRun(() => __classPrivateFieldGet(this, _Scarlet_instances, "m", _Scarlet_runTask).call(this, queueId));
         });
     }
+    /**
+     * Get the number of processed tasks.
+     * @return {number} The number of processed tasks.
+     */
     numberOfProcessed() {
         return this.processedCount;
     }
+    /**
+     * Reset the number of processed tasks.
+     */
     resetNumberOfProcessed() {
         this.processedCount = 0;
     }
-    push(task, processor, debugStr) {
-        // Choose a minimun queue
-        let min = 0;
+    /**
+     * Push a task into the queue.
+     * @param {T} task The task to push into the queue.
+     * @param {TaskProcessor<T>} processor The processor to run the task.
+     */
+    push(task, processor) {
+        // Push the task into the queue.
+        this.queue.push({ queueId: null, task, processor });
+        // Start this task if any queue is stopped.
         for (let i = 0; i < this.queueCount; i++) {
-            if (this.queue[i].length < this.queue[min].length) {
-                min = i;
+            if (!this.running[i]) {
+                this.running[i] = true;
+                asyncRun(() => __classPrivateFieldGet(this, _Scarlet_instances, "m", _Scarlet_runTask).call(this, i));
+                break;
             }
         }
-        // Push the task into the queue
-        this.queue[min].push({
-            queueId: min,
-            task,
-            processor,
-        });
-        if (debugStr) {
-            console.log(`Scarlet pushed (${this.queue.map(q => q.length).join(', ')})`);
-        }
-        // Start the queue if it's not running
-        if (!this.running[min]) {
-            this.running[min] = true;
-            asyncRun(() => {
-                __classPrivateFieldGet(this, _Scarlet_instances, "m", _Scarlet_runTask).call(this, min);
-            });
-        }
     }
-    taskDone(taskObject, debugStr) {
+    /**
+     * Mark a task as done.
+     * @param {TaskObject<T>} taskObject The task object to mark as done.
+     */
+    taskDone(taskObject) {
         if (taskObject.hasDone)
             return;
-        asyncRun(() => {
-            this.emitter.emit('done', taskObject.queueId, !!debugStr);
-        });
-        this.processedCount++;
-        // Call the afterFinishProcessor if it's set
-        let needRunFinishProcessor;
-        if (this.afterFinishLoop) {
-            needRunFinishProcessor = !(this.processedCount % this.afterFinishCount);
-        }
-        else {
-            needRunFinishProcessor = this.processedCount === this.afterFinishCount;
-        }
-        if (needRunFinishProcessor && this.processedCount !== -1 && typeof this.afterFinishProcessor === 'function') {
-            this.afterFinishProcessor();
-            if (!this.afterFinishLoop) {
-                this.clearAfterFinish();
-            }
-        }
+        const queueId = taskObject.queueId;
+        asyncRun(() => this.emitter.emit('done', queueId));
     }
+    /**
+     * Set the after finish processor.
+     * @param {number} count The number of tasks to process before calling the processor.
+     * @param {AfterFinishProcessor} processor The processor to call after the tasks are processed.
+     * @param {boolean} loop Whether to loop the after finish processor.
+     */
     afterFinish(count, processor, loop) {
@@ -107,2 +119,5 @@ this.afterFinishCount = count;
     }
+    /**
+     * Clear the after finish processor.
+     */
     clearAfterFinish() {
@@ -116,16 +131,13 @@ this.afterFinishCount = -1;
 _Scarlet_instances = new WeakSet(), _Scarlet_runTask = function _Scarlet_runTask(queueId) {
-    const queue = this.queue[queueId];
-    // If empty
-    if (queue.length === 0) {
+    const task = this.queue.shift();
+    // If task item is not existing, stop the queue.
+    if (task === undefined) {
         this.running[queueId] = false;
         return;
     }
+    task.queueId = queueId;
     this.running[queueId] = true;
-    // Get the task
-    const task = queue.shift();
-    if (task.queueId !== queueId) {
-        throw new Error(`Invalid queueId, ${task.queueId} !== ${queueId}`);
-    }
+    this.spouts[queueId] = task;
     const taskObject = new task_object_1.TaskObject(task.task, queueId, this);
     task.processor(taskObject);
 };

package.json

 {
   "name": "scarlet-task",
-  "version": "2.1.0",
-  "description": "A task queue module for node.js. You can set several children-queue for one task queue.",
+  "version": "3.0.0",
+  "description": "An advanced task queue module for Node.js with support for multiple child queues, enabling controlled concurrent processing of asynchronous tasks.",
   "main": "lib/index.js",
@@ -21,3 +21,3 @@ "types": "lib/index.d.ts",
   "scripts": {
-    "example": "node example/hackernews.js",
+    "example": "node examples/hackernews.js",
     "build": "tsc -p ./tsconfig.build.json",
@@ -24,0 +24,0 @@ "prepublish": "npm run build"

README.md

@@ -5,20 +5,24 @@ # Scarlet Task
 
-A task queue module for node.js. You can set several children-queue for one task queue.
+Scarlet Task is an advanced task queue module for Node.js, featuring the ability to configure multiple child queues for
+a single task queue.
 
 ## Why named Scarlet? ๛ก(ｰ̀ωｰ́ก)
 
-At first, I wrote this module is for searching one song in [萌否收音機](https://moe.fm). And last I found that song named <[the Embodiment of Scarlet Devil](https://moe.fm/listen?song=79922)>.
+The story behind this module's name is quite serendipitous. One day, while browsing [萌否收音機](https://moe.fm), I
+stumbled upon a captivating song. Though its title slipped my mind, I distinctly remembered its considerable length.
 
-For rembembering this and for my favorite [Flandre Scarlet](http://touhou.wikia.com/wiki/Flandre_Scarlet), I named this module `Scarlet Task`.
+Driven by a desire to rediscover this elusive track, I developed this queue module. It allowed me to perform an
+exhaustive search across the entire Moe FM platform. After a thorough exploration, I finally unearthed the song that had
+been haunting my memory: <[the Embodiment of Scarlet Devil](https://moe.fm/listen?song=79922)>.
 
-## Usage
+To commemorate this musical treasure hunt and pay homage to my favorite character,
+[Flandre Scarlet](http://touhou.wikia.com/wiki/Flandre_Scarlet), I christened this module 'Scarlet Task'.
+It's a nod to both the journey of rediscovery and the Scarlet-themed song that sparked it all.
 
-For one situation, once you want to crawl one website. If you use primitive `node.js`, it will like you're DDOSing that website.
+## Use Case
 
-So you need a task queue to help you. It will process tasks in queue one by one.
+Scarlet Task is particularly useful for scenarios requiring controlled asynchronous operations, such as web crawling. By
+allowing you to set up multiple child queues, it enables concurrent processing while maintaining control over
+parallelism, preventing issues like unintentional DDoS-like behavior on target websites.
 
-What's more, you can set that one queue has several children-queue to work concurrently.
-
-And you can use it at any other situation that suitable.
-
 ## Installation
@@ -63,3 +67,5 @@
 
-> ***Notice:*** In the `processor` function, you should call `taskObject.done()` or `taskQueue.taskDone(taskObject)` when you think this task is done. And then the `taskQueue` will process next task. The parameter `taskObject` is a parameter that `taskQueue` passed to you.
+> ***Notice:*** In the `processor` function, you should call `taskObject.done()` or `taskQueue.taskDone(taskObject)`
+> when you think this task is done. And then the `taskQueue` will process next task. The parameter `taskObject` is a
+> parameter that `taskQueue` passed to you.
 
@@ -66,0 +72,0 @@ You can push task(s) at anytime.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

20094

increased by32.69%

Number of package files

11

increased by22.22%

Lines of code

361

increased by41.02%

Number of lines in readme file

129

increased by4.88%

No dependency changes