@metrichor/epi2me-web - npm Package Compare versions

Comparing version 5.1.5926490 to 5.1.6001567

cjs/epi2me-fs.js

@@ -8,2 +8,3 @@ "use strict";
 const ts_runtime_typecheck_1 = require("ts-runtime-typecheck");
+const ts_runtime_typecheck_2 = require("ts-runtime-typecheck");
 const aws_sdk_1 = __importDefault(require("aws-sdk"));
@@ -49,3 +50,3 @@ const fs_extra_1 = __importDefault(require("fs-extra")); /* MC-565 handle EMFILE & EXDIR gracefully; use Promises */
             let token;
-            ts_runtime_typecheck_1.assertDefined(this.config.instance.id_workflow_instance);
+            ts_runtime_typecheck_2.assertDefined(this.config.instance.id_workflow_instance);
             if (this.config.options.useGraphQL) {
@@ -56,3 +57,3 @@ const instanceTokenOptions = {
                 const result = await this.graphQL.instanceToken(instanceTokenOptions);
-                token = ts_runtime_typecheck_1.asDefined((_a = result.data) === null || _a === void 0 ? void 0 : _a.token);
+                token = ts_runtime_typecheck_2.asDefined((_a = result.data) === null || _a === void 0 ? void 0 : _a.token);
             }
@@ -93,3 +94,3 @@ else {
         try {
-            const queueURL = await this.discoverQueue(ts_runtime_typecheck_1.asOptString(this.config.instance.outputQueueName));
+            const queueURL = await this.discoverQueue(ts_runtime_typecheck_2.asOptString(this.config.instance.outputQueueName));
             const sqs = this.getSQSSessionedService();
@@ -99,3 +100,3 @@ return sqs
                 QueueUrl: queueURL,
-                ReceiptHandle: ts_runtime_typecheck_1.asString(message.ReceiptHandle),
+                ReceiptHandle: ts_runtime_typecheck_2.asString(message.ReceiptHandle),
             })
@@ -110,3 +111,3 @@ .promise();
             this.states.download.failure[error] = this.states.download.failure[error]
-                ? ts_runtime_typecheck_1.asNumber(this.states.download.failure[error]) + 1
+                ? ts_runtime_typecheck_2.asNumber(this.states.download.failure[error]) + 1
                 : 1;
@@ -118,3 +119,3 @@ throw error;
         if (this.config.instance.discoverQueueCache[queueName]) {
-            return ts_runtime_typecheck_1.asString(this.config.instance.discoverQueueCache[queueName]);
+            return ts_runtime_typecheck_2.asString(this.config.instance.discoverQueueCache[queueName]);
         }
@@ -135,3 +136,3 @@ this.log.debug(`discovering queue for ${queueName}`);
         }
-        const queueURL = ts_runtime_typecheck_1.asString(getQueue.QueueUrl);
+        const queueURL = ts_runtime_typecheck_2.asString(getQueue.QueueUrl);
         this.log.debug(`found queue ${queueURL}`);
@@ -239,21 +240,21 @@ this.config.instance.discoverQueueCache[queueName] = queueURL;
         const startData = (_a = result.data) === null || _a === void 0 ? void 0 : _a.startData;
-        ts_runtime_typecheck_1.assertDefined(startData, 'Workflow Start Data');
+        ts_runtime_typecheck_2.assertDefined(startData, 'Workflow Start Data');
         const { instance, bucket, idUser, remoteAddr } = startData;
-        ts_runtime_typecheck_1.assertDefined(instance, 'Workflow Instance');
+        ts_runtime_typecheck_2.assertDefined(instance, 'Workflow Instance');
         const { workflowImage, outputqueue, keyId, startDate, idWorkflowInstance, mappedTelemetry, telemetryNames } = instance;
-        const chain = ts_runtime_typecheck_1.isString(instance.chain) ? ts_runtime_typecheck_1.asString(instance.chain) : ts_runtime_typecheck_1.asDictionary(instance.chain);
+        const chain = ts_runtime_typecheck_2.isString(instance.chain) ? ts_runtime_typecheck_2.asString(instance.chain) : ts_runtime_typecheck_2.asDictionary(instance.chain);
         const regionName = workflowImage.region.name;
-        const idWorkflow = ts_runtime_typecheck_1.asOptIndex(workflowImage.workflow.idWorkflow);
+        const idWorkflow = ts_runtime_typecheck_2.asOptIndex(workflowImage.workflow.idWorkflow);
         const inputqueue = workflowImage.inputqueue;
         const map = {
-            bucket: ts_runtime_typecheck_1.asOptString(bucket),
-            id_user: ts_runtime_typecheck_1.asOptIndex(idUser),
-            remote_addr: ts_runtime_typecheck_1.asOptString(remoteAddr),
-            id_workflow_instance: ts_runtime_typecheck_1.asOptIndex(idWorkflowInstance),
-            key_id: ts_runtime_typecheck_1.asOptString(keyId),
-            start_date: ts_runtime_typecheck_1.asOptString(startDate),
-            outputQueueName: ts_runtime_typecheck_1.asOptString(outputqueue),
-            summaryTelemetry: ts_runtime_typecheck_1.asOptDictionary(mappedTelemetry),
-            telemetryNames: ts_runtime_typecheck_1.asOptDictionaryOf(ts_runtime_typecheck_1.isDictionaryOf(ts_runtime_typecheck_1.isString))(telemetryNames),
-            inputQueueName: ts_runtime_typecheck_1.asOptString(inputqueue),
+            bucket: ts_runtime_typecheck_2.asOptString(bucket),
+            id_user: ts_runtime_typecheck_2.asOptIndex(idUser),
+            remote_addr: ts_runtime_typecheck_2.asOptString(remoteAddr),
+            id_workflow_instance: ts_runtime_typecheck_2.asOptIndex(idWorkflowInstance),
+            key_id: ts_runtime_typecheck_2.asOptString(keyId),
+            start_date: ts_runtime_typecheck_2.asOptString(startDate),
+            outputQueueName: ts_runtime_typecheck_2.asOptString(outputqueue),
+            summaryTelemetry: ts_runtime_typecheck_2.asOptDictionary(mappedTelemetry),
+            telemetryNames: ts_runtime_typecheck_2.asOptDictionaryOf(ts_runtime_typecheck_2.isDictionaryOf(ts_runtime_typecheck_2.isString))(telemetryNames),
+            inputQueueName: ts_runtime_typecheck_2.asOptString(inputqueue),
             id_workflow: idWorkflow,
@@ -268,20 +269,26 @@ region: regionName,
         const conf = this.config.instance;
-        conf.id_workflow_instance = ts_runtime_typecheck_1.asOptIndex(instance.id_workflow_instance);
-        conf.id_workflow = ts_runtime_typecheck_1.asOptIndex(instance.id_workflow);
-        conf.remote_addr = ts_runtime_typecheck_1.asOptString(instance.remote_addr);
-        conf.key_id = ts_runtime_typecheck_1.asOptString(instance.key_id);
-        conf.bucket = ts_runtime_typecheck_1.asOptString(instance.bucket);
-        conf.start_date = ts_runtime_typecheck_1.asOptString(instance.start_date);
-        conf.id_user = ts_runtime_typecheck_1.asOptIndex(instance.id_user);
+        conf.id_workflow_instance = ts_runtime_typecheck_2.asOptIndex(instance.id_workflow_instance);
+        conf.id_workflow = ts_runtime_typecheck_2.asOptIndex(instance.id_workflow);
+        conf.remote_addr = ts_runtime_typecheck_2.asOptString(instance.remote_addr);
+        conf.key_id = ts_runtime_typecheck_2.asOptString(instance.key_id);
+        conf.bucket = ts_runtime_typecheck_2.asOptString(instance.bucket);
+        conf.start_date = ts_runtime_typecheck_2.asOptString(instance.start_date);
+        conf.id_user = ts_runtime_typecheck_2.asOptIndex(instance.id_user);
         // copy tuples with different names / structures
-        conf.inputQueueName = ts_runtime_typecheck_1.asOptString(instance.inputqueue);
-        conf.outputQueueName = ts_runtime_typecheck_1.asOptString(instance.outputqueue);
-        conf.region = ts_runtime_typecheck_1.asString(instance.region, this.config.options.region);
+        conf.inputQueueName = ts_runtime_typecheck_2.asOptString(instance.inputqueue);
+        conf.outputQueueName = ts_runtime_typecheck_2.asOptString(instance.outputqueue);
+        conf.region = ts_runtime_typecheck_2.asString(instance.region, this.config.options.region);
         conf.bucketFolder = `${instance.outputqueue}/${instance.id_user}/${instance.id_workflow_instance}`;
-        conf.summaryTelemetry = ts_runtime_typecheck_1.asOptDictionary(instance.telemetry); // MC-7056 for fetchTelemetry (summary) telemetry periodically
+        conf.summaryTelemetry = ts_runtime_typecheck_2.asOptDictionary(instance.telemetry); // MC-7056 for fetchTelemetry (summary) telemetry periodically
         // WARN this assumes chain is an object, but could it be a string?
         if (instance.chain) {
-            conf.chain = utils_fs_1.utilsFS.convertResponseToObject(ts_runtime_typecheck_1.asDictionary(instance.chain));
+            conf.chain = utils_fs_1.utilsFS.convertResponseToObject(ts_runtime_typecheck_2.asDictionary(instance.chain));
         }
     }
+    // WARN
+    // the purpose of this function is somewhat confused
+    // it is used after the instance is created, but validates
+    // some of the input parameters!
+    // we should replace it with a linear "validate, instantiate, run" flow
+    // when we depreciate and remove REST
     async autoConfigure(instance, autoStartCb) {
@@ -301,8 +308,20 @@ /*
          */
-        if (!this.config.options.inputFolders.length) {
-            throw new Error('must set inputFolder');
+        // NOTE these errors check the options
+        const { inputFolders, idDataset } = this.config.options;
+        const usingDataset = ts_runtime_typecheck_1.isDefined(idDataset);
+        if (ts_runtime_typecheck_1.isDefined(inputFolders)) {
+            if (usingDataset) {
+                throw new Error('cannot use a dataset and folders as an input');
+            }
+            if (inputFolders.length === 0) {
+                throw new Error('no input folders specified');
+            }
         }
+        else if (!usingDataset) {
+            throw new Error('no input folders specified');
+        }
         if (!this.config.options.outputFolder) {
             throw new Error('must set outputFolder');
         }
+        // NOTE these errors check the instance
         if (!this.config.instance.bucketFolder) {
@@ -317,2 +336,3 @@ throw new Error('bucketFolder must be set');
         }
+        // NOTE now for actual setup
         try {
@@ -329,3 +349,3 @@ await fs_extra_1.default.mkdirp(this.config.options.outputFolder);
         const instancesDir = path_1.default.join(rootDir(), 'instances');
-        const thisInstanceDir = path_1.default.join(instancesDir, ts_runtime_typecheck_1.makeString(this.config.instance.id_workflow_instance));
+        const thisInstanceDir = path_1.default.join(instancesDir, ts_runtime_typecheck_2.makeString(this.config.instance.id_workflow_instance));
         try {
@@ -340,7 +360,10 @@ await fs_extra_1.default.mkdirp(thisInstanceDir);
         }
-        // set up new tracking database
-        this.db = new db_1.DB(thisInstanceDir, {
-            idWorkflowInstance: ts_runtime_typecheck_1.makeString(this.config.instance.id_workflow_instance),
-            inputFolders: this.config.options.inputFolders,
-        }, this.log);
+        // NOTE don't need the database if we're running from a dataset
+        if (inputFolders) {
+            // set up new tracking database
+            this.db = new db_1.DB(thisInstanceDir, {
+                idWorkflowInstance: ts_runtime_typecheck_2.makeString(this.config.instance.id_workflow_instance),
+                inputFolders,
+            }, this.log);
+        }
         // MC-1828 - include instance id in telemetry file name
@@ -423,6 +446,6 @@ const fileName = this.config.instance.id_workflow_instance
                     });
-                    instanceObj = ts_runtime_typecheck_1.asDefined(response.data).workflowInstance;
+                    instanceObj = ts_runtime_typecheck_2.asDefined(response.data).workflowInstance;
                 }
                 else {
-                    instanceObj = await this.REST.workflowInstance(ts_runtime_typecheck_1.asDefined(this.config.instance.id_workflow_instance));
+                    instanceObj = await this.REST.workflowInstance(ts_runtime_typecheck_2.asDefined(this.config.instance.id_workflow_instance));
                 }
@@ -433,3 +456,3 @@ if (instanceObj.state === 'stopped') {
                         await this.stopEverything();
-                        const remoteShutdownCb = ts_runtime_typecheck_1.asOptFunction(this.config.options.remoteShutdownCb);
+                        const remoteShutdownCb = ts_runtime_typecheck_2.asOptFunction(this.config.options.remoteShutdownCb);
                         if (remoteShutdownCb) {
@@ -449,6 +472,9 @@ remoteShutdownCb(`instance was stopped remotely at ${instanceObj.stop_date}`);
         this.reportProgress();
-        this.uploadState$.next(true);
-        const startUpload = fileUploader_1.instantiateFileUpload(this);
-        // WARN this is async, but doesn't exit until the upload has been stopped
-        startUpload();
+        // NOTE don't need the uploader if we are running from a dataset
+        if (inputFolders) {
+            this.uploadState$.next(true);
+            const startUpload = fileUploader_1.instantiateFileUpload(this);
+            // WARN this is async, but doesn't exit until the upload has been stopped
+            startUpload();
+        }
         return instance;
@@ -479,3 +505,3 @@ }
         try {
-            const queueURL = await this.discoverQueue(ts_runtime_typecheck_1.asOptString(this.config.instance.outputQueueName));
+            const queueURL = await this.discoverQueue(ts_runtime_typecheck_2.asOptString(this.config.instance.outputQueueName));
             const len = await this.queueLength(queueURL);
@@ -494,3 +520,3 @@ if (len) {
             this.states.download.failure[err] = this.states.download.failure[err]
-                ? ts_runtime_typecheck_1.asNumber(this.states.download.failure[err]) + 1
+                ? ts_runtime_typecheck_2.asNumber(this.states.download.failure[err]) + 1
                 : 1;
@@ -509,3 +535,3 @@ }
         try {
-            const queueURL = await this.discoverQueue(ts_runtime_typecheck_1.asOptString(this.config.instance.outputQueueName));
+            const queueURL = await this.discoverQueue(ts_runtime_typecheck_2.asOptString(this.config.instance.outputQueueName));
             this.log.debug('fetching messages');
@@ -529,3 +555,3 @@ const sqs = this.getSQSSessionedService();
                 const existing = failures[msg];
-                failures[msg] = ts_runtime_typecheck_1.asNumber(existing, 0) + 1;
+                failures[msg] = ts_runtime_typecheck_2.asNumber(existing, 0) + 1;
             }
@@ -547,3 +573,3 @@ throw err;
         for (const message of receiveMessages.Messages) {
-            const id = ts_runtime_typecheck_1.makeString(message.MessageId);
+            const id = ts_runtime_typecheck_2.makeString(message.MessageId);
             workerPool[id] = 1;
@@ -583,3 +609,3 @@ // WARN this behaviour does not appear to be correct, where does the error go???
         try {
-            messageBody = JSON.parse(ts_runtime_typecheck_1.asString(message.Body));
+            messageBody = JSON.parse(ts_runtime_typecheck_2.asString(message.Body));
         }
@@ -597,3 +623,3 @@ catch (jsonError) {
         const { downloadMode } = this.config.options;
-        const telemetry = ts_runtime_typecheck_1.asOptDictionary(messageBody.telemetry);
+        const telemetry = ts_runtime_typecheck_2.asOptDictionary(messageBody.telemetry);
         /* MC-405 telemetry log to file */
@@ -607,4 +633,4 @@ if (telemetry && downloadMode.includes('telemetry')) {
                         .getObject({
-                        Bucket: ts_runtime_typecheck_1.asString(messageBody.bucket),
-                        Key: ts_runtime_typecheck_1.asString(telemetry.tm_path),
+                        Bucket: ts_runtime_typecheck_2.asString(messageBody.bucket),
+                        Key: ts_runtime_typecheck_2.asString(telemetry.tm_path),
                     })
@@ -614,3 +640,3 @@ .promise();
                     const body = data.Body;
-                    if (ts_runtime_typecheck_1.isUndefined(body)) {
+                    if (ts_runtime_typecheck_2.isUndefined(body)) {
                         throw new Error('Telemetry body is undefined');
@@ -655,4 +681,4 @@ }
         const idWorkflowInstance = this.config.instance.id_workflow_instance;
-        let folder = path_1.default.join(ts_runtime_typecheck_1.asString(this.config.options.outputFolder), ts_runtime_typecheck_1.isUndefined(idWorkflowInstance) ? '' : ts_runtime_typecheck_1.makeString(idWorkflowInstance));
-        const telemetryHintsFolder = ts_runtime_typecheck_1.asOptString((_a = ts_runtime_typecheck_1.asOptDictionary(telemetry === null || telemetry === void 0 ? void 0 : telemetry.hints)) === null || _a === void 0 ? void 0 : _a.folder);
+        let folder = path_1.default.join(ts_runtime_typecheck_2.asString(this.config.options.outputFolder), ts_runtime_typecheck_2.isUndefined(idWorkflowInstance) ? '' : ts_runtime_typecheck_2.makeString(idWorkflowInstance));
+        const telemetryHintsFolder = ts_runtime_typecheck_2.asOptString((_a = ts_runtime_typecheck_2.asOptDictionary(telemetry === null || telemetry === void 0 ? void 0 : telemetry.hints)) === null || _a === void 0 ? void 0 : _a.folder);
         /* MC-940: use folder hinting if present */
@@ -677,5 +703,5 @@ if (telemetryHintsFolder) {
             /* download file[s] from S3 */
-            const downloads = ts_runtime_typecheck_1.isArrayOf(ts_runtime_typecheck_1.isString)(messageBody.downloads)
+            const downloads = ts_runtime_typecheck_2.isArrayOf(ts_runtime_typecheck_2.isString)(messageBody.downloads)
                 ? messageBody.downloads
-                : [ts_runtime_typecheck_1.asString(messageBody.path)];
+                : [ts_runtime_typecheck_2.asString(messageBody.path)];
             await Promise.all(downloads.map(async (filepath) => {
@@ -686,3 +712,3 @@ const destination = path_1.default.join(folder, path_1.default.basename(filepath));
                     await this.initiateDownloadStream({
-                        bucket: ts_runtime_typecheck_1.asString(messageBody.bucket),
+                        bucket: ts_runtime_typecheck_2.asString(messageBody.bucket),
                         path: filepath,
@@ -697,5 +723,5 @@ }, message, destination);
         else {
-            const batchSummary = ts_runtime_typecheck_1.asOptDictionary(telemetry === null || telemetry === void 0 ? void 0 : telemetry.batch_summary);
+            const batchSummary = ts_runtime_typecheck_2.asOptDictionary(telemetry === null || telemetry === void 0 ? void 0 : telemetry.batch_summary);
             // telemetry-only mode uses readcount from message
-            const readCount = (_b = ts_runtime_typecheck_1.asOptNumber(batchSummary === null || batchSummary === void 0 ? void 0 : batchSummary.reads_num)) !== null && _b !== void 0 ? _b : 1;
+            const readCount = (_b = ts_runtime_typecheck_2.asOptNumber(batchSummary === null || batchSummary === void 0 ? void 0 : batchSummary.reads_num)) !== null && _b !== void 0 ? _b : 1;
             this.downloadState('success', 'incr', {
@@ -867,4 +893,4 @@ files: 1,
                         .changeMessageVisibility({
-                        QueueUrl: ts_runtime_typecheck_1.asString(queueUrl),
-                        ReceiptHandle: ts_runtime_typecheck_1.asString(receiptHandle),
+                        QueueUrl: ts_runtime_typecheck_2.asString(queueUrl),
+                        ReceiptHandle: ts_runtime_typecheck_2.asString(receiptHandle),
                         VisibilityTimeout: this.config.options.inFlightDelay.seconds,
@@ -907,6 +933,6 @@ })
         }
-        const instanceDir = path_1.default.join(rootDir(), 'instances', ts_runtime_typecheck_1.makeString(this.config.instance.id_workflow_instance));
+        const instanceDir = path_1.default.join(rootDir(), 'instances', ts_runtime_typecheck_2.makeString(this.config.instance.id_workflow_instance));
         const telemetryNames = (_b = (_a = this.config) === null || _a === void 0 ? void 0 : _a.instance) === null || _b === void 0 ? void 0 : _b.telemetryNames;
-        const idWorkflowInstance = ts_runtime_typecheck_1.makeString(this.config.instance.id_workflow_instance);
-        this.telemetry = telemetry_1.Telemetry.connect(idWorkflowInstance, this.graphQL, ts_runtime_typecheck_1.asDefined(telemetryNames));
+        const idWorkflowInstance = ts_runtime_typecheck_2.makeString(this.config.instance.id_workflow_instance);
+        this.telemetry = telemetry_1.Telemetry.connect(idWorkflowInstance, this.graphQL, ts_runtime_typecheck_2.asDefined(telemetryNames));
         // const reports$ = this.telemetry.telemetryReports$().pipe(filter(isDefined));
@@ -962,3 +988,3 @@ const destroySignal$ = new rxjs_1.Subject();
         var _a, _b;
-        const instanceDir = path_1.default.join(rootDir(), 'instances', ts_runtime_typecheck_1.makeString(this.config.instance.id_workflow_instance));
+        const instanceDir = path_1.default.join(rootDir(), 'instances', ts_runtime_typecheck_2.makeString(this.config.instance.id_workflow_instance));
         if (this.config.options.useGraphQL) {
@@ -972,8 +998,8 @@ // uses observeTelemetry instead
         const toFetch = [];
-        const summaryTelemetry = ts_runtime_typecheck_1.asDictionary(this.config.instance.summaryTelemetry);
+        const summaryTelemetry = ts_runtime_typecheck_2.asDictionary(this.config.instance.summaryTelemetry);
         Object.keys(summaryTelemetry).forEach((componentId) => {
             var _a;
-            const component = (_a = ts_runtime_typecheck_1.asDictionary(summaryTelemetry[componentId])) !== null && _a !== void 0 ? _a : {};
+            const component = (_a = ts_runtime_typecheck_2.asDictionary(summaryTelemetry[componentId])) !== null && _a !== void 0 ? _a : {};
             const firstReport = Object.keys(component)[0]; // poor show
-            let url = ts_runtime_typecheck_1.asOptString(component[firstReport]);
+            let url = ts_runtime_typecheck_2.asOptString(component[firstReport]);
             if (!url) {
@@ -980,0 +1006,0 @@ return;

cjs/fileUploader.js

@@ -43,2 +43,3 @@ "use strict";
     };
+    ts_runtime_typecheck_1.assertDefined(inputFolders, 'inputFolders');
     const scanner = createFileScanner({
@@ -45,0 +46,0 @@ database,

cjs/parseOptions.js

@@ -68,4 +68,7 @@ "use strict";
         // EPI2ME-FS options
-        inputFolders: ts_runtime_typecheck_1.asArrayOf(ts_runtime_typecheck_1.isString)(opt.inputFolders, []), outputFolder: ts_runtime_typecheck_1.asOptString(opt.outputFolder), awsAcceleration: ts_runtime_typecheck_1.asOptString(opt.awsAcceleration), agent_address: ts_runtime_typecheck_1.asOptString(opt.agent_address), telemetryCb: ts_runtime_typecheck_1.asOptFunction(opt.telemetryCb), dataCb: ts_runtime_typecheck_1.asOptFunction(opt.dataCb), remoteShutdownCb: ts_runtime_typecheck_1.asOptFunction(opt.remoteShutdownCb) });
+        inputFolders: ts_runtime_typecheck_1.asOptArrayOf(ts_runtime_typecheck_1.isString)(opt.inputFolders), outputFolder: ts_runtime_typecheck_1.asOptString(opt.outputFolder), awsAcceleration: ts_runtime_typecheck_1.asOptString(opt.awsAcceleration), agent_address: ts_runtime_typecheck_1.asOptString(opt.agent_address), telemetryCb: ts_runtime_typecheck_1.asOptFunction(opt.telemetryCb), dataCb: ts_runtime_typecheck_1.asOptFunction(opt.dataCb), remoteShutdownCb: ts_runtime_typecheck_1.asOptFunction(opt.remoteShutdownCb) });
     if (opt.inputFolder) {
+        if (!options.inputFolders) {
+            options.inputFolders = [];
+        }
         options.inputFolders.push(ts_runtime_typecheck_1.asString(opt.inputFolder));
@@ -72,0 +75,0 @@ }

esm/epi2me-fs.js

@@ -0,1 +1,2 @@
+import { isDefined } from 'ts-runtime-typecheck';
 import { asOptString, asString, asNumber, asDictionary, asOptIndex, asOptDictionary, asDefined, asOptFunction, asOptNumber, makeString, isString, isUndefined, asOptDictionaryOf, isDictionaryOf, isArrayOf, assertDefined, } from 'ts-runtime-typecheck';
@@ -271,2 +272,8 @@ import AWS from 'aws-sdk';
     }
+    // WARN
+    // the purpose of this function is somewhat confused
+    // it is used after the instance is created, but validates
+    // some of the input parameters!
+    // we should replace it with a linear "validate, instantiate, run" flow
+    // when we depreciate and remove REST
     async autoConfigure(instance, autoStartCb) {
@@ -286,8 +293,20 @@ /*
          */
-        if (!this.config.options.inputFolders.length) {
-            throw new Error('must set inputFolder');
+        // NOTE these errors check the options
+        const { inputFolders, idDataset } = this.config.options;
+        const usingDataset = isDefined(idDataset);
+        if (isDefined(inputFolders)) {
+            if (usingDataset) {
+                throw new Error('cannot use a dataset and folders as an input');
+            }
+            if (inputFolders.length === 0) {
+                throw new Error('no input folders specified');
+            }
         }
+        else if (!usingDataset) {
+            throw new Error('no input folders specified');
+        }
         if (!this.config.options.outputFolder) {
             throw new Error('must set outputFolder');
         }
+        // NOTE these errors check the instance
         if (!this.config.instance.bucketFolder) {
@@ -302,2 +321,3 @@ throw new Error('bucketFolder must be set');
         }
+        // NOTE now for actual setup
         try {
@@ -324,7 +344,10 @@ await fs.mkdirp(this.config.options.outputFolder);
         }
-        // set up new tracking database
-        this.db = new DB(thisInstanceDir, {
-            idWorkflowInstance: makeString(this.config.instance.id_workflow_instance),
-            inputFolders: this.config.options.inputFolders,
-        }, this.log);
+        // NOTE don't need the database if we're running from a dataset
+        if (inputFolders) {
+            // set up new tracking database
+            this.db = new DB(thisInstanceDir, {
+                idWorkflowInstance: makeString(this.config.instance.id_workflow_instance),
+                inputFolders,
+            }, this.log);
+        }
         // MC-1828 - include instance id in telemetry file name
@@ -431,6 +454,9 @@ const fileName = this.config.instance.id_workflow_instance
         this.reportProgress();
-        this.uploadState$.next(true);
-        const startUpload = instantiateFileUpload(this);
-        // WARN this is async, but doesn't exit until the upload has been stopped
-        startUpload();
+        // NOTE don't need the uploader if we are running from a dataset
+        if (inputFolders) {
+            this.uploadState$.next(true);
+            const startUpload = instantiateFileUpload(this);
+            // WARN this is async, but doesn't exit until the upload has been stopped
+            startUpload();
+        }
         return instance;
@@ -437,0 +463,0 @@ }

esm/fileUploader.js

@@ -37,2 +37,3 @@ import { loadInputFiles } from './inputScanner';
     };
+    assertDefined(inputFolders, 'inputFolders');
     const scanner = createFileScanner({
@@ -39,0 +40,0 @@ database,

esm/parseOptions.js

 import { FallbackLogger } from './Logger';
 import { version as VERSION } from '../package.json';
 import DEFAULTS from './default_options.json';
-import { isDictionary, asFunction, asString, asBoolean, asOptString, asNumber, asArrayOf, isString, asOptIndex, asOptFunction, } from 'ts-runtime-typecheck';
+import { isDictionary, asFunction, asString, asBoolean, asOptString, asNumber, asArrayOf, isString, asOptIndex, asOptFunction, asOptArrayOf, } from 'ts-runtime-typecheck';
 import { Duration } from './Duration';
@@ -60,4 +60,7 @@ function resolveLogger(log) {
         // EPI2ME-FS options
-        inputFolders: asArrayOf(isString)(opt.inputFolders, []), outputFolder: asOptString(opt.outputFolder), awsAcceleration: asOptString(opt.awsAcceleration), agent_address: asOptString(opt.agent_address), telemetryCb: asOptFunction(opt.telemetryCb), dataCb: asOptFunction(opt.dataCb), remoteShutdownCb: asOptFunction(opt.remoteShutdownCb) });
+        inputFolders: asOptArrayOf(isString)(opt.inputFolders), outputFolder: asOptString(opt.outputFolder), awsAcceleration: asOptString(opt.awsAcceleration), agent_address: asOptString(opt.agent_address), telemetryCb: asOptFunction(opt.telemetryCb), dataCb: asOptFunction(opt.dataCb), remoteShutdownCb: asOptFunction(opt.remoteShutdownCb) });
     if (opt.inputFolder) {
+        if (!options.inputFolders) {
+            options.inputFolders = [];
+        }
         options.inputFolders.push(asString(opt.inputFolder));
@@ -64,0 +67,0 @@ }

package.json

@@ -12,3 +12,3 @@ {
   "private": false,
-  "version": "5.1.5926490",
+  "version": "5.1.6001567",
   "main": "cjs/index-web.js",
@@ -15,0 +15,0 @@ "module": "esm/index-web.js",

types/Configuration.type.d.ts

@@ -26,3 +26,3 @@ import type { Index, Dictionary, UnknownFunction } from 'ts-runtime-typecheck';
         proxy?: string;
-        inputFolders: string[];
+        inputFolders?: string[];
         outputFolder?: string;
@@ -29,0 +29,0 @@ awsAcceleration?: string;

types/epi2me-options.type.d.ts

@@ -35,3 +35,3 @@ import type { Logger } from './Logger.type';
     inputFolder?: string;
-    inputFolders: string[];
+    inputFolders?: string[];
     outputFolder?: string;
@@ -38,0 +38,0 @@ awsAcceleration?: string;

No alert changes

Improved metrics

Total package byte prevSize

3049376

increased by0.08%

Lines of code

28226

increased by0.21%

No dependency changes