mobiletto-base - npm Package Compare versions

Comparing version 2.0.24 to 2.0.25

lib/cjs/functions.js

@@ -23,2 +23,3 @@ "use strict";
 const util_js_1 = require("./util.js");
+const mobiletto_1 = require("./mobiletto");
 function mirrorDir(source, sourcePath, visitor) {
@@ -276,10 +277,10 @@ return __awaiter(this, void 0, void 0, function* () {
     destroy: (client) => () => __awaiter(void 0, void 0, void 0, function* () {
-        if (client.queueWorkers) {
+        if (client.mq) {
             const workerClosePromises = [];
-            client.queueWorkers.forEach((w) => workerClosePromises.push(w.close(true)));
+            client.mq.workers.forEach((w) => workerClosePromises.push(w.close(true)));
             yield Promise.all(workerClosePromises);
+            yield client.mq.events.close();
+            yield client.mq.queue.close();
+            delete mobiletto_1.ALL_MQ[client.id];
         }
-        if (client.queueEvents) {
-            yield client.queueEvents.close();
-        }
         const cache = client.getCache();
@@ -286,0 +287,0 @@ if (cache) {

lib/cjs/mobiletto.d.ts

@@ -1,8 +0,6 @@
-import { QueueEvents, Worker } from "bullmq";
 import { MobilettoOptions } from "mobiletto-common";
-import { MobilettoConnection } from "./types.js";
+import { MobilettoConnection, MobilettoQueue } from "./types.js";
 import { MobilettoEncryptionSettings } from "./crypt.js";
-export declare const ALL_WORKERS: Worker[];
-export declare const ALL_QUEUE_EVENTS: QueueEvents[];
+export declare const ALL_MQ: Record<string, MobilettoQueue>;
 export declare function mobiletto(driverPath: string, key: string, secret?: string | null, opts?: MobilettoOptions | null, encryption?: MobilettoEncryptionSettings): Promise<MobilettoConnection>;
 export declare function connect(driverPath: string, key: string, secret?: string | null, opts?: MobilettoOptions | null, encryption?: MobilettoEncryptionSettings): Promise<MobilettoConnection>;

lib/cjs/mobiletto.js

@@ -38,3 +38,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.connect = exports.mobiletto = exports.ALL_QUEUE_EVENTS = exports.ALL_WORKERS = void 0;
+exports.connect = exports.mobiletto = exports.ALL_MQ = void 0;
 /* eslint-disable @typescript-eslint/ban-ts-comment */
@@ -55,4 +55,3 @@ const path_1 = require("path");
 const ENC_PAD_SEP = " ~ ";
-exports.ALL_WORKERS = [];
-exports.ALL_QUEUE_EVENTS = [];
+exports.ALL_MQ = {};
 function mobiletto(driverPath, key, secret, opts, encryption) {
@@ -243,9 +242,8 @@ return __awaiter(this, void 0, void 0, function* () {
         });
-        const META_LOAD_QUEUE_NAME = `/tmp/_/loadMetaQueue_${client.id}_`;
-        const META_LOAD_JOB_NAME = `/tmp/_/loadMetaJob_${client.id}_`;
-        let META_LOAD_QUEUE = null;
+        const queueName = `metaQ_${client.id}`;
+        const jobName = `metaJ_${client.id}`;
         const META_HANDLERS = {};
         const META_ERR_HANDLERS = {};
         const metaLoadQueue = () => {
-            if (META_LOAD_QUEUE === null) {
+            if (!client.mq) {
                 if (!client.redisConfig) {
@@ -262,14 +260,14 @@ const message = "metaLoadQueue: redis is required but not enabled";
                     },
-                    prefix: client.redisConfig.prefix + "_" + META_LOAD_QUEUE_NAME,
+                    prefix: client.redisConfig.prefix + "_" + queueName,
                 };
-                META_LOAD_QUEUE = new bullmq_1.Queue(META_LOAD_QUEUE_NAME, queueOptions);
+                const queue = new bullmq_1.Queue(queueName, queueOptions);
+                const workers = [];
                 const numWorkers = enc.metaWorkers || crypt_js_1.DEFAULT_META_WORKERS;
                 for (let i = 0; i < numWorkers; i++) {
-                    const worker = new bullmq_1.Worker(META_LOAD_QUEUE_NAME, (job) => __awaiter(this, void 0, void 0, function* () { return yield _singleMeta(job); }), queueOptions);
-                    exports.ALL_WORKERS.push(worker);
-                    client.queueWorkers.push(worker);
+                    const worker = new bullmq_1.Worker(queueName, (job) => __awaiter(this, void 0, void 0, function* () { return yield _singleMeta(job); }), queueOptions);
+                    workers.push(worker);
                 }
-                const queueEvents = new bullmq_1.QueueEvents(META_LOAD_QUEUE_NAME, queueOptions);
-                queueEvents.on("completed", ({ jobId, returnvalue }) => {
-                    mobiletto_common_1.logger.info(`${META_LOAD_JOB_NAME} completed job ${jobId} with result: ${returnvalue}`);
+                const events = new bullmq_1.QueueEvents(queueName, queueOptions);
+                events.on("completed", ({ jobId, returnvalue }) => {
+                    mobiletto_common_1.logger.info(`${jobName} completed job ${jobId} with result: ${returnvalue}`);
                     if (META_HANDLERS[jobId]) {
@@ -279,4 +277,4 @@ META_HANDLERS[jobId](JSON.parse(returnvalue));
                 });
-                queueEvents.on("failed", ({ jobId, failedReason }) => {
-                    mobiletto_common_1.logger.info(`${META_LOAD_JOB_NAME} failed job ${jobId} with result: ${failedReason}`);
+                events.on("failed", ({ jobId, failedReason }) => {
+                    mobiletto_common_1.logger.info(`${jobName} failed job ${jobId} with result: ${failedReason}`);
                     if (META_ERR_HANDLERS[jobId]) {
@@ -286,6 +284,6 @@ META_ERR_HANDLERS[jobId](failedReason);
                 });
-                client.queueEvents = queueEvents;
-                exports.ALL_QUEUE_EVENTS.push(queueEvents);
+                client.mq = { queue, workers, events };
+                exports.ALL_MQ[client.id] = client.mq;
             }
-            return META_LOAD_QUEUE;
+            return client.mq.queue;
         };
@@ -310,3 +308,3 @@ const _loadMeta = (dirent, entries) => __awaiter(this, void 0, void 0, function* () {
                 const job = { mobilettoJobID, dirent, entry };
-                yield mq.add(META_LOAD_JOB_NAME, job);
+                yield mq.add(jobName, job);
             }
@@ -313,0 +311,0 @@ yield new Promise((resolve) => waitForFiles(resolve));

lib/cjs/register.js

@@ -17,6 +17,16 @@ "use strict";
 const shutdownMobiletto = () => __awaiter(void 0, void 0, void 0, function* () {
-    const closePromises = [];
-    mobiletto_1.ALL_WORKERS.forEach((w) => closePromises.push(w.close(true)));
-    mobiletto_1.ALL_QUEUE_EVENTS.forEach((qe) => closePromises.push(qe.close()));
-    yield Promise.all(closePromises);
+    const workerPromises = [];
+    const eventsPromises = [];
+    const queuePromises = [];
+    const clientIds = Object.keys(mobiletto_1.ALL_MQ);
+    clientIds.forEach((id) => {
+        const mq = mobiletto_1.ALL_MQ[id];
+        mq.workers.forEach((w) => workerPromises.push(w.close(true)));
+        eventsPromises.push(mq.events.close());
+        queuePromises.push(mq.queue.close());
+        delete mobiletto_1.ALL_MQ[id];
+    });
+    yield Promise.all(workerPromises);
+    yield Promise.all(eventsPromises);
+    yield Promise.all(queuePromises);
     yield (0, redis_js_1.teardown)();
@@ -23,0 +33,0 @@ });

lib/cjs/types.d.ts

 /// <reference types="node" />
 import { CacheLike } from "./cache";
 import { MobilettoListOptions, MobilettoMetadata, MobilettoMinimalClient, MobilettoMirrorResults, MobilettoOptions, MobilettoRedisConfig, MobilettoWriteSource, MobilettoVisitor } from "mobiletto-common";
-import { QueueEvents, Worker } from "bullmq";
+import { Queue, QueueEvents, Worker } from "bullmq";
 export type MobilettoConnection = MobilettoMinimalClient & {
@@ -18,9 +18,13 @@ safeList: (path?: string, opts?: MobilettoListOptions) => Promise<MobilettoMetadata[]>;
 export type MobilettoDriverParameter = MobilettoConnectionFunction | MobilettoDriver;
+export type MobilettoQueue = {
+    queue: Queue;
+    workers: Worker[];
+    events: QueueEvents;
+};
 export type MobilettoClient = MobilettoConnection & {
-    id?: string;
+    id: string;
     redisConfig: MobilettoRedisConfig;
     cache: CacheLike;
     redis: () => CacheLike;
-    queueWorkers: Worker[];
-    queueEvents: QueueEvents;
+    mq?: MobilettoQueue;
     scopedCache: (cacheName: string, size?: number) => CacheLike;
@@ -27,0 +31,0 @@ flush: () => Promise<void>;

lib/esm/functions.js

@@ -17,2 +17,3 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
 import { MOBILETTO_TMP, reader } from "./util.js";
+import { ALL_MQ } from "./mobiletto";
 function mirrorDir(source, sourcePath, visitor) {
@@ -269,10 +270,10 @@ return __awaiter(this, void 0, void 0, function* () {
     destroy: (client) => () => __awaiter(void 0, void 0, void 0, function* () {
-        if (client.queueWorkers) {
+        if (client.mq) {
             const workerClosePromises = [];
-            client.queueWorkers.forEach((w) => workerClosePromises.push(w.close(true)));
+            client.mq.workers.forEach((w) => workerClosePromises.push(w.close(true)));
             yield Promise.all(workerClosePromises);
+            yield client.mq.events.close();
+            yield client.mq.queue.close();
+            delete ALL_MQ[client.id];
         }
-        if (client.queueEvents) {
-            yield client.queueEvents.close();
-        }
         const cache = client.getCache();
@@ -279,0 +280,0 @@ if (cache) {

lib/esm/mobiletto.d.ts

@@ -1,8 +0,6 @@
-import { QueueEvents, Worker } from "bullmq";
 import { MobilettoOptions } from "mobiletto-common";
-import { MobilettoConnection } from "./types.js";
+import { MobilettoConnection, MobilettoQueue } from "./types.js";
 import { MobilettoEncryptionSettings } from "./crypt.js";
-export declare const ALL_WORKERS: Worker[];
-export declare const ALL_QUEUE_EVENTS: QueueEvents[];
+export declare const ALL_MQ: Record<string, MobilettoQueue>;
 export declare function mobiletto(driverPath: string, key: string, secret?: string | null, opts?: MobilettoOptions | null, encryption?: MobilettoEncryptionSettings): Promise<MobilettoConnection>;
 export declare function connect(driverPath: string, key: string, secret?: string | null, opts?: MobilettoOptions | null, encryption?: MobilettoEncryptionSettings): Promise<MobilettoConnection>;

lib/esm/mobiletto.js

@@ -25,4 +25,3 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
 const ENC_PAD_SEP = " ~ ";
-export const ALL_WORKERS = [];
-export const ALL_QUEUE_EVENTS = [];
+export const ALL_MQ = {};
 export function mobiletto(driverPath, key, secret, opts, encryption) {
@@ -213,9 +212,8 @@ return __awaiter(this, void 0, void 0, function* () {
         });
-        const META_LOAD_QUEUE_NAME = `/tmp/_/loadMetaQueue_${client.id}_`;
-        const META_LOAD_JOB_NAME = `/tmp/_/loadMetaJob_${client.id}_`;
-        let META_LOAD_QUEUE = null;
+        const queueName = `metaQ_${client.id}`;
+        const jobName = `metaJ_${client.id}`;
         const META_HANDLERS = {};
         const META_ERR_HANDLERS = {};
         const metaLoadQueue = () => {
-            if (META_LOAD_QUEUE === null) {
+            if (!client.mq) {
                 if (!client.redisConfig) {
@@ -232,14 +230,14 @@ const message = "metaLoadQueue: redis is required but not enabled";
                     },
-                    prefix: client.redisConfig.prefix + "_" + META_LOAD_QUEUE_NAME,
+                    prefix: client.redisConfig.prefix + "_" + queueName,
                 };
-                META_LOAD_QUEUE = new Queue(META_LOAD_QUEUE_NAME, queueOptions);
+                const queue = new Queue(queueName, queueOptions);
+                const workers = [];
                 const numWorkers = enc.metaWorkers || DEFAULT_META_WORKERS;
                 for (let i = 0; i < numWorkers; i++) {
-                    const worker = new Worker(META_LOAD_QUEUE_NAME, (job) => __awaiter(this, void 0, void 0, function* () { return yield _singleMeta(job); }), queueOptions);
-                    ALL_WORKERS.push(worker);
-                    client.queueWorkers.push(worker);
+                    const worker = new Worker(queueName, (job) => __awaiter(this, void 0, void 0, function* () { return yield _singleMeta(job); }), queueOptions);
+                    workers.push(worker);
                 }
-                const queueEvents = new QueueEvents(META_LOAD_QUEUE_NAME, queueOptions);
-                queueEvents.on("completed", ({ jobId, returnvalue }) => {
-                    logger.info(`${META_LOAD_JOB_NAME} completed job ${jobId} with result: ${returnvalue}`);
+                const events = new QueueEvents(queueName, queueOptions);
+                events.on("completed", ({ jobId, returnvalue }) => {
+                    logger.info(`${jobName} completed job ${jobId} with result: ${returnvalue}`);
                     if (META_HANDLERS[jobId]) {
@@ -249,4 +247,4 @@ META_HANDLERS[jobId](JSON.parse(returnvalue));
                 });
-                queueEvents.on("failed", ({ jobId, failedReason }) => {
-                    logger.info(`${META_LOAD_JOB_NAME} failed job ${jobId} with result: ${failedReason}`);
+                events.on("failed", ({ jobId, failedReason }) => {
+                    logger.info(`${jobName} failed job ${jobId} with result: ${failedReason}`);
                     if (META_ERR_HANDLERS[jobId]) {
@@ -256,6 +254,6 @@ META_ERR_HANDLERS[jobId](failedReason);
                 });
-                client.queueEvents = queueEvents;
-                ALL_QUEUE_EVENTS.push(queueEvents);
+                client.mq = { queue, workers, events };
+                ALL_MQ[client.id] = client.mq;
             }
-            return META_LOAD_QUEUE;
+            return client.mq.queue;
         };
@@ -280,3 +278,3 @@ const _loadMeta = (dirent, entries) => __awaiter(this, void 0, void 0, function* () {
                 const job = { mobilettoJobID, dirent, entry };
-                yield mq.add(META_LOAD_JOB_NAME, job);
+                yield mq.add(jobName, job);
             }
@@ -283,0 +281,0 @@ yield new Promise((resolve) => waitForFiles(resolve));

lib/esm/register.js

@@ -12,8 +12,18 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
 import { logger, MobilettoError } from "mobiletto-common";
-import { ALL_QUEUE_EVENTS, ALL_WORKERS } from "./mobiletto";
+import { ALL_MQ } from "./mobiletto";
 export const shutdownMobiletto = () => __awaiter(void 0, void 0, void 0, function* () {
-    const closePromises = [];
-    ALL_WORKERS.forEach((w) => closePromises.push(w.close(true)));
-    ALL_QUEUE_EVENTS.forEach((qe) => closePromises.push(qe.close()));
-    yield Promise.all(closePromises);
+    const workerPromises = [];
+    const eventsPromises = [];
+    const queuePromises = [];
+    const clientIds = Object.keys(ALL_MQ);
+    clientIds.forEach((id) => {
+        const mq = ALL_MQ[id];
+        mq.workers.forEach((w) => workerPromises.push(w.close(true)));
+        eventsPromises.push(mq.events.close());
+        queuePromises.push(mq.queue.close());
+        delete ALL_MQ[id];
+    });
+    yield Promise.all(workerPromises);
+    yield Promise.all(eventsPromises);
+    yield Promise.all(queuePromises);
     yield teardown();
@@ -20,0 +30,0 @@ });

lib/esm/types.d.ts

 /// <reference types="node" />
 import { CacheLike } from "./cache";
 import { MobilettoListOptions, MobilettoMetadata, MobilettoMinimalClient, MobilettoMirrorResults, MobilettoOptions, MobilettoRedisConfig, MobilettoWriteSource, MobilettoVisitor } from "mobiletto-common";
-import { QueueEvents, Worker } from "bullmq";
+import { Queue, QueueEvents, Worker } from "bullmq";
 export type MobilettoConnection = MobilettoMinimalClient & {
@@ -18,9 +18,13 @@ safeList: (path?: string, opts?: MobilettoListOptions) => Promise<MobilettoMetadata[]>;
 export type MobilettoDriverParameter = MobilettoConnectionFunction | MobilettoDriver;
+export type MobilettoQueue = {
+    queue: Queue;
+    workers: Worker[];
+    events: QueueEvents;
+};
 export type MobilettoClient = MobilettoConnection & {
-    id?: string;
+    id: string;
     redisConfig: MobilettoRedisConfig;
     cache: CacheLike;
     redis: () => CacheLike;
-    queueWorkers: Worker[];
-    queueEvents: QueueEvents;
+    mq?: MobilettoQueue;
     scopedCache: (cacheName: string, size?: number) => CacheLike;
@@ -27,0 +31,0 @@ flush: () => Promise<void>;

package.json

 {
   "name": "mobiletto-base",
-  "version": "2.0.24",
+  "version": "2.0.25",
   "type": "module",
@@ -5,0 +5,0 @@ "description": "A storage layer that presents a uniform interface to Amazon S3, Backblaze B2, local and other storage systems.",

No alert changes

Improved metrics

Total package byte prevSize

158283

increased by0.13%

Lines of code

3013

increased by0.74%

No dependency changes