@golemio/core - npm Package Compare versions

Comparing version 1.17.2-dev.1574590891 to 1.17.2-dev.1576969835

dist/helpers/data-access/storage/AzureBlobStorageService.d.ts

 /// <reference types="node" />
 /// <reference types="node" />
 /// <reference types="node" />
+import { ILogger } from "../../logger";
 import { Readable } from "stream";
-import { ILogger } from "../../logger";
 import { AbstractStorageService } from "./AbstractStorageService";
@@ -13,2 +13,3 @@ export interface IAzureBlobStorageConfig {
     containerName: string;
+    uploadTimeoutInSeconds: number;
 }
@@ -18,2 +19,3 @@ export declare class AzureBlobStorageService extends AbstractStorageService {
     private readonly containerClient;
+    private readonly uploadTimeoutInMs;
     constructor(config: IAzureBlobStorageConfig, log: ILogger);
@@ -20,0 +22,0 @@ uploadStream(stream: Readable, pathPrefix: string, ext?: string, fileName?: string): Promise<void>;

dist/helpers/data-access/storage/AzureBlobStorageService.js

@@ -13,6 +13,6 @@ "use strict";
 exports.AzureBlobStorageService = void 0;
+const _helpers_1 = require("../..");
+const identity_1 = require("@azure/identity");
 const storage_blob_1 = require("@azure/storage-blob");
 const AbstractStorageService_1 = require("./AbstractStorageService");
-const _helpers_1 = require("../..");
-const identity_1 = require("@azure/identity");
 class AzureBlobStorageService extends AbstractStorageService_1.AbstractStorageService {
@@ -29,2 +29,3 @@ constructor(config, log) {
         });
+        this.uploadTimeoutInMs = config.uploadTimeoutInSeconds * 1000;
     }
@@ -37,8 +38,19 @@ uploadStream(stream, pathPrefix, ext = "", fileName) {
             const fileNameExt = ext ? `.${ext}` : "";
+            const controller = new AbortController();
             try {
-                const blobClient = this.containerClient.getBlockBlobClient(`${fileNameKey}${fileNameExt}`);
                 const options = {
                     blobHTTPHeaders: { blobContentType: "application/octet-stream" },
                     onProgress: this.handleUploadProgress(fileNameKey + fileNameExt),
+                    abortSignal: controller.signal,
                 };
+                const blobClient = this.containerClient.getBlockBlobClient(`${fileNameKey}${fileNameExt}`);
+                stream.on("error", (err) => {
+                    this.log.error(`[${this.constructor.name}] Data stream error (${fileNameKey}${fileNameExt})`, err);
+                    controller.abort();
+                });
+                stream.on("end", () => {
+                    this.log.verbose(`[${this.constructor.name}] Data stream ended (${fileNameKey}${fileNameExt})`);
+                    controller.abort();
+                });
+                setTimeout(() => controller.abort(), this.uploadTimeoutInMs);
                 yield blobClient.uploadStream(stream, undefined, undefined, options);
@@ -58,10 +70,11 @@ }
             try {
-                const blobClient = this.containerClient.getBlockBlobClient(`${fileNameKey}${fileNameExt}`);
-                if (!Buffer.isBuffer(data)) {
-                    data = Buffer.from(data);
-                }
                 const options = {
                     blobHTTPHeaders: { blobContentType: "application/octet-stream" },
                     onProgress: this.handleUploadProgress(fileNameKey + fileNameExt, Buffer.byteLength(data)),
+                    abortSignal: AbortSignal.timeout(this.uploadTimeoutInMs),
                 };
+                if (!Buffer.isBuffer(data)) {
+                    data = Buffer.from(data);
+                }
+                const blobClient = this.containerClient.getBlockBlobClient(`${fileNameKey}${fileNameExt}`);
                 yield blobClient.uploadData(data, options);
@@ -68,0 +81,0 @@ }

dist/input-gateway/data-access/StorageServiceProvider.js

@@ -31,2 +31,3 @@ "use strict";
                     containerName: config.getValue("env.AZURE_BLOB_STORAGE_CONTAINER_NAME", ""),
+                    uploadTimeoutInSeconds: Number.parseInt(config.getValue("env.AZURE_BLOB_UPLOAD_TIMEOUT_IN_SECONDS", "120")) || 120,
                 },
@@ -33,0 +34,0 @@ },

dist/input-gateway/helpers/SaveRawDataMiddleware.js

@@ -44,7 +44,13 @@ "use strict";
                     const fileName = `${now.format("yyyy-MM-dd")}/${now.format("HH_mm_ss.SSS")}`;
-                    this.storage.uploadStream(req, `${requestUrlPath.slice(1)}`, ext, fileName);
+                    this.storage.uploadStream(req, `${requestUrlPath.slice(1)}`, ext, fileName).catch((err) => {
+                        this.log.error(`Saving of data stream failed (${requestUrlPath})`, err);
+                    });
                     // save file with headers of the matched route
                     if (whitelistRegexp.headersRegexp.exec(requestUrlPath)) {
                         const headerFileName = `${fileName}_headers`;
-                        this.storage.uploadFile(JSON.stringify(req.headers), `${requestUrlPath.slice(1)}`, `json`, headerFileName);
+                        this.storage
+                            .uploadFile(JSON.stringify(req.headers), `${requestUrlPath.slice(1)}`, `json`, headerFileName)
+                            .catch((err) => {
+                            this.log.error(`Saving of headers failed (${requestUrlPath})`, err);
+                        });
                     }
@@ -51,0 +57,0 @@ }

dist/integration-engine/data-access/StorageServiceProvider.js

@@ -32,2 +32,3 @@ "use strict";
                     containerName: config.getValue("env.AZURE_BLOB_STORAGE_CONTAINER_NAME", ""),
+                    uploadTimeoutInSeconds: Number.parseInt(config.getValue("env.AZURE_BLOB_UPLOAD_TIMEOUT_IN_SECONDS", "120")) || 120,
                 },
@@ -34,0 +35,0 @@ },

package.json

 {
     "name": "@golemio/core",
-    "version": "1.17.2-dev.1574590891",
+    "version": "1.17.2-dev.1576969835",
     "description": "Golemio Core Module",
@@ -74,5 +74,5 @@ "main": "dist/index.js",
         "@abraham/reflection": "^0.10.0",
-        "@azure/data-tables": "^13.2.2",
-        "@azure/identity": "^4.0.1",
-        "@azure/storage-blob": "^12.11.0",
+        "@azure/data-tables": "^13.3.0",
+        "@azure/identity": "^4.5.0",
+        "@azure/storage-blob": "^12.26.0",
         "@golemio/errors": "2.0.5-dev.1288217017",
@@ -79,0 +79,0 @@ "@golemio/validator": "0.3.6-dev.1288217736",

No alert changes

Improved metrics

Total package byte prevSize

808994

0.32%

Lines of code

11844

0.19%

Dependency changes

• Updated

  @azure/data-tables@^13.3.0

• Updated

  @azure/identity@^4.5.0

• Updated

  @azure/storage-blob@^12.26.0