gatsby-worker
Advanced tools
Comparing version 2.4.0-next.0 to 2.4.0-next.1
@@ -6,2 +6,14 @@ # Changelog: `gatsby-worker` | ||
## [2.3.0](https://github.com/gatsbyjs/gatsby/commits/gatsby-worker@2.3.0/packages/gatsby-worker) (2022-12-13) | ||
[🧾 Release notes](https://www.gatsbyjs.com/docs/reference/release-notes/v5.3) | ||
#### Features | ||
- Show original stack trace [#37206](https://github.com/gatsbyjs/gatsby/issues/37206) ([3b75594](https://github.com/gatsbyjs/gatsby/commit/3b75594d9d45371c483df942ee680eef54471b82)) | ||
#### Chores | ||
- update dependency typescript to ^4.9.3 [#37192](https://github.com/gatsbyjs/gatsby/issues/37192) ([204ed0e](https://github.com/gatsbyjs/gatsby/commit/204ed0e220eb9c1c5cdec692e82bce8e3e82e915)) | ||
## [2.2.0](https://github.com/gatsbyjs/gatsby/commits/gatsby-worker@2.2.0/packages/gatsby-worker) (2022-11-25) | ||
@@ -8,0 +20,0 @@ |
"use strict"; | ||
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault"); | ||
exports.__esModule = true; | ||
exports.isWorker = exports.getMessenger = void 0; | ||
var _signalExit = _interopRequireDefault(require("signal-exit")); | ||
var _fsExtra = _interopRequireDefault(require("fs-extra")); | ||
var _types = require("./types"); | ||
var _utils = require("./utils"); | ||
let counter = 0; | ||
/** | ||
@@ -23,9 +16,6 @@ * Used to check wether current context is executed in worker process | ||
exports.isWorker = isWorker; | ||
let getMessenger = function () { | ||
return undefined; | ||
}; | ||
exports.getMessenger = getMessenger; | ||
if (process.send && process.env.GATSBY_WORKER_MODULE_PATH && process.env.GATSBY_WORKER_IN_FLIGHT_DUMP_LOCATION) { | ||
@@ -42,3 +32,2 @@ const workerInFlightsDumpLocation = process.env.GATSBY_WORKER_IN_FLIGHT_DUMP_LOCATION; | ||
}); | ||
function ensuredSendToMain(msg) { | ||
@@ -52,3 +41,2 @@ inFlightMessages.add(msg); | ||
} | ||
function onError(error) { | ||
@@ -58,7 +46,5 @@ if (error == null) { | ||
} | ||
const msg = [_types.ERROR, ++counter, error.constructor && error.constructor.name, error.message, error.stack, error]; | ||
ensuredSendToMain(msg); | ||
} | ||
function onResult(result) { | ||
@@ -68,5 +54,3 @@ const msg = [_types.RESULT, ++counter, result]; | ||
} | ||
const MESSAGING_VERSION = 1; | ||
exports.getMessenger = getMessenger = function () { | ||
@@ -77,3 +61,2 @@ return { | ||
}, | ||
sendMessage(msg) { | ||
@@ -83,13 +66,9 @@ const poolMsg = [_types.CUSTOM_MESSAGE, ++counter, msg]; | ||
}, | ||
messagingVersion: MESSAGING_VERSION | ||
}; | ||
}; | ||
const child = require(process.env.GATSBY_WORKER_MODULE_PATH); | ||
function messageHandler(msg) { | ||
if (msg[0] === _types.EXECUTE) { | ||
let result; | ||
try { | ||
@@ -101,3 +80,2 @@ result = child[msg[2]].call(child, ...msg[3]); | ||
} | ||
if ((0, _utils.isPromise)(result)) { | ||
@@ -116,5 +94,4 @@ result.then(onResult, onError); | ||
} | ||
process.on(`message`, messageHandler); | ||
ensuredSendToMain([_types.WORKER_READY, ++counter]); | ||
} |
"use strict"; | ||
var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefault"); | ||
exports.__esModule = true; | ||
@@ -10,17 +9,9 @@ var _exportNames = { | ||
exports.WorkerPool = void 0; | ||
var _child_process = require("child_process"); | ||
var _fsExtra = _interopRequireDefault(require("fs-extra")); | ||
var _os = _interopRequireDefault(require("os")); | ||
var _path = _interopRequireDefault(require("path")); | ||
var _taskQueue = require("./task-queue"); | ||
var _types = require("./types"); | ||
var _child = require("./child"); | ||
Object.keys(_child).forEach(function (key) { | ||
@@ -32,5 +23,3 @@ if (key === "default" || key === "__esModule") return; | ||
}); | ||
const childWrapperPath = require.resolve(`./child`); | ||
class TaskInfo { | ||
@@ -46,5 +35,3 @@ constructor(opts) { | ||
} | ||
} | ||
/** | ||
@@ -66,2 +53,3 @@ * Worker pool is a class that allow you to queue function execution across multiple | ||
*/ | ||
workers = []; | ||
@@ -72,3 +60,2 @@ taskQueue = new _taskQueue.TaskQueue(); | ||
counter = 0; | ||
constructor(workerPath, options) { | ||
@@ -82,5 +69,3 @@ this.workerPath = workerPath; | ||
const module = require(workerPath); | ||
const exportNames = Object.keys(module); | ||
for (const exportName of exportNames) { | ||
@@ -94,3 +79,2 @@ if (typeof module[exportName] !== `function`) { | ||
} | ||
single[exportName] = this.scheduleWorkSingle.bind(this, exportName); | ||
@@ -104,18 +88,13 @@ all[exportName] = this.scheduleWorkAll.bind(this, exportName); | ||
} | ||
startAll() { | ||
this.counter = 0; | ||
const tmpDir = _fsExtra.default.mkdtempSync(_path.default.join(_os.default.tmpdir(), `gatsby-worker`)); | ||
const options = this.options; | ||
for (let workerId = 1; workerId <= ((_options$numWorkers = options === null || options === void 0 ? void 0 : options.numWorkers) !== null && _options$numWorkers !== void 0 ? _options$numWorkers : 1); workerId++) { | ||
var _options$numWorkers, _options$env; | ||
const workerInFlightsDumpLocation = _path.default.join(tmpDir, `worker-${workerId}.json`); | ||
const worker = (0, _child_process.fork)(childWrapperPath, { | ||
cwd: process.cwd(), | ||
env: { ...process.env, | ||
env: { | ||
...process.env, | ||
...((_options$env = options === null || options === void 0 ? void 0 : options.env) !== null && _options$env !== void 0 ? _options$env : {}), | ||
@@ -138,3 +117,2 @@ GATSBY_WORKER_ID: workerId.toString(), | ||
} | ||
worker.send(msg, undefined, undefined, error => { | ||
@@ -155,3 +133,2 @@ if (error && worker.connected) { | ||
}; | ||
const workerProcessMessageHandler = msg => { | ||
@@ -180,5 +157,3 @@ if (!Array.isArray(msg)) { | ||
} | ||
workerInfo.lastMessage = msg[1]; | ||
if (msg[0] === _types.RESULT) { | ||
@@ -188,3 +163,2 @@ if (!workerInfo.currentTask) { | ||
} | ||
const task = workerInfo.currentTask; | ||
@@ -198,5 +172,3 @@ workerInfo.currentTask = undefined; | ||
} | ||
let error = msg[5]; | ||
if (error !== null && typeof error === `object`) { | ||
@@ -206,7 +178,6 @@ const extra = error; | ||
const Ctor = typeof NativeCtor === `function` ? NativeCtor : Error; | ||
error = new Ctor(msg[3]); // @ts-ignore type doesn't exist on Error, but that's what jest-worker does for errors :shrug: | ||
error = new Ctor(msg[3]); | ||
// @ts-ignore type doesn't exist on Error, but that's what jest-worker does for errors :shrug: | ||
error.type = msg[2]; | ||
error.stack = msg[4]; | ||
for (const key in extra) { | ||
@@ -218,3 +189,2 @@ if (Object.prototype.hasOwnProperty.call(extra, key)) { | ||
} | ||
const task = workerInfo.currentTask; | ||
@@ -232,3 +202,2 @@ workerInfo.currentTask = undefined; | ||
}; | ||
worker.on(`message`, workerProcessMessageHandler); | ||
@@ -238,3 +207,2 @@ worker.on(`exit`, async (code, signal) => { | ||
const pendingMessages = await _fsExtra.default.readJSON(workerInFlightsDumpLocation); | ||
if (Array.isArray(pendingMessages)) { | ||
@@ -245,16 +213,14 @@ for (const msg of pendingMessages) { | ||
} | ||
try { | ||
await _fsExtra.default.remove(workerInFlightsDumpLocation); | ||
} catch {// this is just cleanup, failing to delete this file | ||
} catch { | ||
// this is just cleanup, failing to delete this file | ||
// won't cause | ||
} | ||
} | ||
if (workerInfo.currentTask) { | ||
// worker exited without finishing a task | ||
workerInfo.currentTask.reject(new Error(`Worker exited before finishing task`)); | ||
} // remove worker from list of workers | ||
} | ||
// remove worker from list of workers | ||
this.workers.splice(this.workers.indexOf(workerInfo), 1); | ||
@@ -270,2 +236,3 @@ workerExitResolve({ | ||
} | ||
/** | ||
@@ -275,4 +242,2 @@ * Kills worker processes and rejects and ongoing or pending tasks. | ||
*/ | ||
end() { | ||
@@ -282,4 +247,5 @@ const results = this.workers.map(async workerInfo => { | ||
const endMessage = [_types.END, ++this.counter]; | ||
workerInfo.send(endMessage); // force exit if worker doesn't exit gracefully quickly | ||
workerInfo.send(endMessage); | ||
// force exit if worker doesn't exit gracefully quickly | ||
const forceExitTimeout = setTimeout(() => { | ||
@@ -297,3 +263,2 @@ workerInfo.kill(`SIGKILL`); | ||
} | ||
this.workers = []; | ||
@@ -304,7 +269,6 @@ this.idleWorkers = new Set(); | ||
} | ||
/** | ||
* Kills all running worker processes and spawns a new pool of processes | ||
*/ | ||
async restart() { | ||
@@ -314,3 +278,2 @@ await Promise.all(this.end()); | ||
} | ||
getWorkerInfo() { | ||
@@ -323,3 +286,2 @@ return this.workers.map(worker => { | ||
} | ||
checkForWork(workerInfo) { | ||
@@ -329,3 +291,2 @@ // check if there is task in queue | ||
const task = taskNode.value; | ||
if (!task.assignedToWorker || task.assignedToWorker === workerInfo) { | ||
@@ -336,8 +297,7 @@ this.doWork(task, workerInfo); | ||
} | ||
} // no task found, so just marking worker as idle | ||
} | ||
// no task found, so just marking worker as idle | ||
this.idleWorkers.add(workerInfo); | ||
} | ||
async doWork(taskInfo, workerInfo) { | ||
@@ -351,6 +311,4 @@ // block worker | ||
} | ||
scheduleWork(taskInfo) { | ||
let workerToExecuteTaskNow; | ||
if (taskInfo.assignedToWorker) { | ||
@@ -363,3 +321,2 @@ if (this.idleWorkers.has(taskInfo.assignedToWorker)) { | ||
} | ||
if (workerToExecuteTaskNow) { | ||
@@ -370,6 +327,4 @@ this.doWork(taskInfo, workerToExecuteTaskNow); | ||
} | ||
return taskInfo.promise; | ||
} | ||
scheduleWorkSingle(functionName, ...args) { | ||
@@ -381,3 +336,2 @@ return this.scheduleWork(new TaskInfo({ | ||
} | ||
scheduleWorkAll(functionName, ...args) { | ||
@@ -390,20 +344,14 @@ return this.workers.map(workerInfo => this.scheduleWork(new TaskInfo({ | ||
} | ||
onMessage(listener) { | ||
this.listeners.push(listener); | ||
} | ||
sendMessage(msg, workerId) { | ||
const worker = this.workers[workerId - 1]; | ||
if (!worker) { | ||
throw new Error(`There is no worker with "${workerId}" id.`); | ||
} | ||
const poolMsg = [_types.CUSTOM_MESSAGE, ++this.counter, msg]; | ||
worker.send(poolMsg); | ||
} | ||
} | ||
exports.WorkerPool = WorkerPool; |
@@ -5,3 +5,2 @@ "use strict"; | ||
exports.TaskQueue = void 0; | ||
/** | ||
@@ -13,3 +12,2 @@ * Task queue implemented with doubly linked list | ||
let currentHead = this.head; | ||
while (currentHead) { | ||
@@ -20,2 +18,3 @@ yield currentHead; | ||
} | ||
/** | ||
@@ -25,4 +24,2 @@ * Puts new task at the end of the list | ||
*/ | ||
enqueue(task) { | ||
@@ -32,3 +29,2 @@ const newNode = { | ||
}; | ||
if (this.tail) { | ||
@@ -40,5 +36,5 @@ this.tail.next = newNode; | ||
} | ||
this.tail = newNode; | ||
} | ||
/** | ||
@@ -48,8 +44,5 @@ * Remove a task node from the queue | ||
*/ | ||
remove(taskNode) { | ||
if (taskNode === this.head) { | ||
this.head = taskNode.next; | ||
if (this.head) { | ||
@@ -67,11 +60,8 @@ this.head.prev = undefined; | ||
taskNode.next.prev = taskNode.prev; | ||
} // if node is not the head then it will have .prev | ||
} | ||
// if node is not the head then it will have .prev | ||
taskNode.prev.next = taskNode.next; | ||
} | ||
} | ||
} | ||
exports.TaskQueue = TaskQueue; |
@@ -5,7 +5,4 @@ "use strict"; | ||
exports.isRunning = exports.isPromise = void 0; | ||
const isPromise = obj => !!obj && (typeof obj === `object` || typeof obj === `function`) && typeof obj.then === `function`; | ||
exports.isPromise = isPromise; | ||
const isRunning = pid => { | ||
@@ -21,3 +18,2 @@ try { | ||
}; | ||
exports.isRunning = isRunning; |
{ | ||
"name": "gatsby-worker", | ||
"description": "Utility to create worker pools", | ||
"version": "2.4.0-next.0", | ||
"version": "2.4.0-next.1", | ||
"author": "Michal Piechowiak<misiek.piechowiak@gmail.com>", | ||
@@ -13,3 +13,3 @@ "bugs": { | ||
"fs-extra": "^10.0.0", | ||
"signal-exit": "^3.0.5" | ||
"signal-exit": "^3.0.7" | ||
}, | ||
@@ -19,3 +19,3 @@ "devDependencies": { | ||
"@babel/register": "^7.15.3", | ||
"babel-preset-gatsby-package": "^3.4.0-next.0", | ||
"babel-preset-gatsby-package": "^3.4.0-next.1", | ||
"cross-env": "^7.0.3", | ||
@@ -46,3 +46,3 @@ "rimraf": "^3.0.2", | ||
}, | ||
"gitHead": "7938eb07143c06953ae0197d26533ba34eecfc29" | ||
"gitHead": "5df8a99b7fd6de53a8f3088138cc574f55f4cfd5" | ||
} |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
46269
620
Updatedsignal-exit@^3.0.7