@aacc/runner - npm Package Compare versions

Comparing version 0.0.0 to 0.0.1

dist/cjs/index.js

 'use strict';
 
-Object.defineProperty(exports, '__esModule', { value: true });
-
 var path = require('node:path');
@@ -11,6 +9,3 @@ var os = require('node:os');
 
-function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
-
-function _interopNamespace(e) {
-  if (e && e.__esModule) return e;
+function _interopNamespaceDefault(e) {
   var n = Object.create(null);
@@ -28,45 +23,43 @@ if (e) {
   }
-  n["default"] = e;
+  n.default = e;
   return Object.freeze(n);
 }
 
-var path__namespace = /*#__PURE__*/_interopNamespace(path);
-var os__namespace = /*#__PURE__*/_interopNamespace(os);
-var wt__namespace = /*#__PURE__*/_interopNamespace(wt);
-var Comlink__namespace = /*#__PURE__*/_interopNamespace(Comlink);
-var nodeEndpoint__default = /*#__PURE__*/_interopDefaultLegacy(nodeEndpoint);
+var path__namespace = /*#__PURE__*/_interopNamespaceDefault(path);
+var os__namespace = /*#__PURE__*/_interopNamespaceDefault(os);
+var wt__namespace = /*#__PURE__*/_interopNamespaceDefault(wt);
+var Comlink__namespace = /*#__PURE__*/_interopNamespaceDefault(Comlink);
 
+/* eslint-disable no-await-in-loop */
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
 async function runner(options) {
   const numOfWorkers = Math.min(options.files.length, options.workers ? Math.min(options.workers, os__namespace.cpus().length) : os__namespace.cpus().length);
+  const chunkedFiles = [];
+  const chunkSize = 50;
+  for (let i = 0; i < options.files.length; i += chunkSize) {
+    chunkedFiles.push(options.files.slice(i, i + chunkSize));
+  }
   const workers = [];
-
   for (let i = 0; i < numOfWorkers; i += 1) {
     const worker = new wt__namespace.Worker(path__namespace.join(__dirname, './worker.js'));
-    workers.push(Comlink__namespace.wrap(nodeEndpoint__default["default"](worker)));
+    workers.push(Comlink__namespace.wrap(nodeEndpoint(worker)));
   }
-
-  let currentFile = 0;
+  let currentChunk = 0;
   await Promise.all(workers.map(async worker => {
-    while (currentFile < options.files.length) {
-      // eslint-disable-next-line no-plusplus
-      const fileIndex = currentFile++; // eslint-disable-next-line no-await-in-loop
-
-      await worker.processFile(Comlink__namespace.proxy(options.processor), {
-        filePath: options.files[fileIndex]
+    await worker.loadProcessor(options.processor);
+    while (currentChunk < chunkedFiles.length) {
+      const chunkIndex = currentChunk;
+      currentChunk += 1;
+      const results = await worker.processFiles({
+        filePaths: chunkedFiles[chunkIndex]
       });
+      if (options.onProcessed) {
+        await (options === null || options === void 0 ? void 0 : options.onProcessed(results));
+      }
     }
-
     worker[Comlink__namespace.releaseProxy]();
-    return worker;
   }));
 }
-runner({
-  workers: 2,
-  files: [path__namespace.join(__dirname, 'index.js'), path__namespace.join(__dirname, 'worker.js'), path__namespace.join(__dirname, 'index.js'), path__namespace.join(__dirname, 'worker.js')],
-  processor: options => {
-    console.log('filePath: ', options.filePath);
-    console.log('fileContent: ', options.fileContent.slice(0, 10));
-  }
-}).catch(console.error);
 
 exports.runner = runner;

dist/cjs/worker.js

@@ -8,6 +8,3 @@ 'use strict';
 
-function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
-
-function _interopNamespace(e) {
-  if (e && e.__esModule) return e;
+function _interopNamespaceDefault(e) {
   var n = Object.create(null);
@@ -25,21 +22,48 @@ if (e) {
   }
-  n["default"] = e;
+  n.default = e;
   return Object.freeze(n);
 }
 
-var wt__namespace = /*#__PURE__*/_interopNamespace(wt);
-var fs__namespace = /*#__PURE__*/_interopNamespace(fs);
-var Comlink__namespace = /*#__PURE__*/_interopNamespace(Comlink);
-var nodeEndpoint__default = /*#__PURE__*/_interopDefaultLegacy(nodeEndpoint);
+var wt__namespace = /*#__PURE__*/_interopNamespaceDefault(wt);
+var fs__namespace = /*#__PURE__*/_interopNamespaceDefault(fs);
+var Comlink__namespace = /*#__PURE__*/_interopNamespaceDefault(Comlink);
 
+/* eslint-disable @typescript-eslint/no-unsafe-assignment */
+// eslint-disable-next-line @typescript-eslint/ban-ts-comment
+// @ts-nocheck
 const workerApi = {
-  async processFile(processor, options) {
-    const content = await fs__namespace.promises.readFile(options.filePath, 'utf-8');
-    await processor({
-      filePath: options.filePath,
-      fileContent: content
+  processor: null,
+  async loadProcessor(processorPath) {
+    const {
+      default: processor
+    } = await import(processorPath);
+    this.processor = processor;
+  },
+  /**
+   * @deprecated Use `processFiles` instead.
+   */
+  async processFile(options) {
+    if (!this.processor) throw new Error('No processor loaded');
+    const fileContent = await fs__namespace.promises.readFile(options.filePath, 'utf-8');
+    const result = await this.processor({
+      fileContent,
+      filePath: options.filePath
     });
+    return result ?? null;
+  },
+  async processFiles(options) {
+    if (!this.processor) throw new Error('No processor loaded');
+    await Promise.all(options.filePaths.map(async filePath => {
+      const fileContent = await fs__namespace.promises.readFile(filePath, 'utf-8');
+      const result = await this.processor({
+        fileContent,
+        filePath: options.filePath
+      });
+      return result ?? null;
+    }));
+
+    // return results ?? []
   }
+};
 
-};
-Comlink__namespace.expose(workerApi, nodeEndpoint__default["default"](wt__namespace.parentPort));
+Comlink__namespace.expose(workerApi, nodeEndpoint(wt__namespace.parentPort));

dist/types/index.d.ts

@@ -1,9 +0,10 @@
-import type { Processor } from './worker';
+import type { WorkerResult } from './worker';
 interface RunnerOptions {
     workers?: number;
     files: string[];
-    processor: Processor;
+    processor: string;
+    onProcessed?: <T = null>(result: WorkerResult) => Promise<T> | T;
 }
-export declare function runner(options: RunnerOptions): Promise<void>;
+export declare function runner(options: RunnerOptions): Promise<WorkerResult>;
 export {};
 //# sourceMappingURL=index.d.ts.map

dist/types/worker.d.ts

@@ -1,12 +0,20 @@
-declare type ProcessorOptions = {
+type ProcessorOptions = {
     filePath: string;
     fileContent: string;
 };
-export declare type Processor = (options: ProcessorOptions) => Promise<void> | void;
+export type Processor<T = null> = (options: ProcessorOptions) => Promise<T> | T;
+export interface ProcessorImport {
+    default: Processor;
+}
 export interface WorkerApi {
-    processFile: (processor: Processor, options: {
+    processor: null | Processor;
+    loadProcessor(processorPath: string): Promise<void>;
+    processFile: <T = null>(options: {
         filePath: string;
-    }) => Promise<void>;
+    }) => Promise<T>;
+    processFiles: <T = null>(options: {
+        filePaths: string[];
+    }) => Promise<T>;
 }
 export {};
 //# sourceMappingURL=worker.d.ts.map

package.json

 {
   "name": "@aacc/runner",
-  "version": "0.0.0",
+  "version": "0.0.1",
   "description": "Simple API for processing files in a worker pool",
@@ -25,3 +25,3 @@ "author": "Aaron Casanova <[e-mail]>",
     "type-check:watch": "npm run type-check -- --watch",
-    "lint": "TIMING=1 eslint . --ext .js,.ts --cache",
+    "lint": "TIMING=1 eslint . --ext .js,.jsx,.ts,.tsx,.cjs,.mjs --cache",
     "prepublishOnly": "npm run build"
@@ -33,3 +33,3 @@ },
   "dependencies": {
-    "comlink": "^4.3.1"
+    "comlink": "^4.4.1"
   },
@@ -41,9 +41,13 @@ "devDependencies": {
     "@aacc/tsconfigs": "*",
-    "@rollup/plugin-babel": "^5.3.1",
-    "@rollup/plugin-commonjs": "^21.1.0",
-    "@rollup/plugin-node-resolve": "^13.2.1",
-    "@types/figlet": "^1.5.4",
-    "rollup": "^2.70.2",
+    "@babel/parser": "^7.18.11",
+    "@babel/traverse": "^7.18.11",
+    "@rollup/plugin-babel": "^6.0.4",
+    "@rollup/plugin-commonjs": "^25.0.7",
+    "@rollup/plugin-node-resolve": "^15.2.3",
+    "@types/figlet": "^1.5.6",
+    "globby": "^11.1.0",
+    "jscodeshift": "^0.15.0",
+    "rollup": "^4.9.4",
     "rollup-plugin-preserve-shebang": "^1.0.1",
-    "typescript": "^4.7.3"
+    "typescript": "~5.3.3"
   },
@@ -50,0 +54,0 @@ "browserslist": [

No alert changes

Improved metrics

Total package byte prevSize

10925

24.05%

Lines of code

225

38.04%

Worsened metrics

Dev dependency count

15

36.36%

Dependency changes

• Updated

  comlink@^4.4.1