gcs-resumable-upload - npm Package Compare versions

Comparing version 3.2.1 to 3.3.0

build/src/index.d.ts

@@ -135,2 +135,6 @@ /*!
     userProject?: string;
+    /**
+     * Configuration options for retrying retriable errors.
+     */
+    retryOptions?: RetryOptions;
 }
@@ -148,2 +152,13 @@ export interface ConfigMetadata {
 }
+export interface RetryOptions {
+    retryDelayMultiplier?: number;
+    totalTimeout?: number;
+    maxRetryDelay?: number;
+    autoRetry?: boolean;
+    maxRetries?: number;
+    retryableErrorFn?: (err: ApiError) => boolean;
+}
+export interface ApiError extends Error {
+    code?: number;
+}
 export declare class Upload extends Pumpify {
@@ -180,2 +195,8 @@ bucket: string;
     contentLength: number | '*';
+    retryLimit: number;
+    maxRetryDelay: number;
+    retryDelayMultiplier: number;
+    maxRetryTotalTimeout: number;
+    timeOfFirstRequest: number;
+    retryableErrorFn?: (err: ApiError) => boolean;
     private bufferStream?;
@@ -201,2 +222,10 @@ private offsetStream?;
     private onResponse;
+    /**
+     * @param resp GaxiosResponse object from previous attempt
+     */
+    private attemptDelayedRetry;
+    /**
+     * @returns {number} the amount of time to wait before retrying the request
+     */
+    private getRetryDelay;
     private sanitizeEndpoint;
@@ -203,0 +232,0 @@ }

build/src/index.js

@@ -24,8 +24,17 @@ "use strict";
 const DEFAULT_API_ENDPOINT_REGEX = /.*\.googleapis\.com/;
+const MAX_RETRY_DELAY = 64;
+const RETRY_DELAY_MULTIPLIER = 2;
+const MAX_TOTAL_RETRY_TIMEOUT = 600;
+const AUTO_RETRY_VALUE = true;
 exports.PROTOCOL_REGEX = /^(\w*):\/\//;
 class Upload extends Pumpify {
     constructor(cfg) {
+        var _a, _b, _c, _d, _e, _f;
         super();
         this.numBytesWritten = 0;
         this.numRetries = 0;
+        this.retryLimit = RETRY_LIMIT;
+        this.maxRetryDelay = MAX_RETRY_DELAY;
+        this.retryDelayMultiplier = RETRY_DELAY_MULTIPLIER;
+        this.maxRetryTotalTimeout = MAX_TOTAL_RETRY_TIMEOUT;
         streamEvents(this);
@@ -85,2 +94,3 @@ cfg = cfg || {};
         });
+        const autoRetry = ((_a = cfg === null || cfg === void 0 ? void 0 : cfg.retryOptions) === null || _a === void 0 ? void 0 : _a.autoRetry) || AUTO_RETRY_VALUE;
         this.uriProvidedManually = !!cfg.uri;
@@ -90,2 +100,19 @@ this.uri = cfg.uri || this.get('uri');
         this.numRetries = 0;
+        if (autoRetry && ((_b = cfg === null || cfg === void 0 ? void 0 : cfg.retryOptions) === null || _b === void 0 ? void 0 : _b.maxRetries) !== undefined) {
+            this.retryLimit = cfg.retryOptions.maxRetries;
+        }
+        else if (!autoRetry) {
+            this.retryLimit = 0;
+        }
+        if (((_c = cfg === null || cfg === void 0 ? void 0 : cfg.retryOptions) === null || _c === void 0 ? void 0 : _c.maxRetryDelay) !== undefined) {
+            this.maxRetryDelay = cfg.retryOptions.maxRetryDelay;
+        }
+        if (((_d = cfg === null || cfg === void 0 ? void 0 : cfg.retryOptions) === null || _d === void 0 ? void 0 : _d.retryDelayMultiplier) !== undefined) {
+            this.retryDelayMultiplier = cfg.retryOptions.retryDelayMultiplier;
+        }
+        if (((_e = cfg === null || cfg === void 0 ? void 0 : cfg.retryOptions) === null || _e === void 0 ? void 0 : _e.totalTimeout) !== undefined) {
+            this.maxRetryTotalTimeout = cfg.retryOptions.totalTimeout;
+        }
+        this.timeOfFirstRequest = Date.now();
+        this.retryableErrorFn = (_f = cfg === null || cfg === void 0 ? void 0 : cfg.retryOptions) === null || _f === void 0 ? void 0 : _f.retryableErrorFn;
         const contentLength = cfg.metadata
@@ -257,6 +284,8 @@ ? Number(cfg.metadata.contentLength)
         let length = chunk.length;
-        if (typeof chunk === 'string')
+        if (typeof chunk === 'string') {
             length = Buffer.byteLength(chunk, enc);
-        if (numBytesWritten < offset)
+        }
+        if (numBytesWritten < offset) {
             chunk = chunk.slice(offset - numBytesWritten);
+        }
         this.numBytesWritten += length;
@@ -370,27 +399,48 @@ // only push data from the byte after the one we left off on
     onResponse(resp) {
-        if (resp.status === 404) {
-            if (this.numRetries < RETRY_LIMIT) {
-                this.numRetries++;
-                this.startUploading();
-            }
-            else {
-                this.destroy(new Error('Retry limit exceeded - ' + resp.data));
-            }
+        if ((this.retryableErrorFn &&
+            this.retryableErrorFn({
+                code: resp.status,
+                message: resp.statusText,
+                name: resp.statusText,
+            })) ||
+            resp.status === 404 ||
+            (resp.status > 499 && resp.status < 600)) {
+            this.attemptDelayedRetry(resp);
             return false;
         }
-        if (resp.status > 499 && resp.status < 600) {
-            if (this.numRetries < RETRY_LIMIT) {
-                const randomMs = Math.round(Math.random() * 1000);
-                const waitTime = Math.pow(2, this.numRetries) * 1000 + randomMs;
-                this.numRetries++;
-                setTimeout(this.continueUploading.bind(this), waitTime);
+        this.emit('response', resp);
+        return true;
+    }
+    /**
+     * @param resp GaxiosResponse object from previous attempt
+     */
+    attemptDelayedRetry(resp) {
+        if (this.numRetries < this.retryLimit) {
+            if (resp.status === 404) {
+                this.startUploading();
             }
             else {
-                this.destroy(new Error('Retry limit exceeded - ' + resp.data));
+                const retryDelay = this.getRetryDelay();
+                if (retryDelay <= 0) {
+                    this.destroy(new Error(`Retry total time limit exceeded - ${resp.data}`));
+                    return;
+                }
+                setTimeout(this.continueUploading.bind(this), retryDelay);
             }
-            return false;
+            this.numRetries++;
         }
-        this.emit('response', resp);
-        return true;
+        else {
+            this.destroy(new Error('Retry limit exceeded - ' + resp.data));
+        }
     }
+    /**
+     * @returns {number} the amount of time to wait before retrying the request
+     */
+    getRetryDelay() {
+        const randomMs = Math.round(Math.random() * 1000);
+        const waitTime = Math.pow(this.retryDelayMultiplier, this.numRetries) * 1000 + randomMs;
+        const maxAllowableDelayMs = this.maxRetryTotalTimeout * 1000 - (Date.now() - this.timeOfFirstRequest);
+        const maxRetryDelayMs = this.maxRetryDelay * 1000;
+        return Math.min(waitTime, maxRetryDelayMs, maxAllowableDelayMs);
+    }
     /*
@@ -397,0 +447,0 @@ * Prepare user-defined API endpoint for compatibility with our API.

CHANGELOG.md

@@ -7,2 +7,9 @@ # Changelog
 
+## [3.3.0](https://www.github.com/googleapis/gcs-resumable-upload/compare/v3.2.1...v3.3.0) (2021-07-19)
+
+
+### Features
+
+* customization options for retries ([#441](https://www.github.com/googleapis/gcs-resumable-upload/issues/441)) ([2007234](https://www.github.com/googleapis/gcs-resumable-upload/commit/200723407881c24f77e5b4dc3ed4527799945133))
+
 ### [3.2.1](https://www.github.com/googleapis/gcs-resumable-upload/compare/v3.2.0...v3.2.1) (2021-06-28)
@@ -9,0 +16,0 @@

package.json

 {
   "name": "gcs-resumable-upload",
-  "version": "3.2.1",
+  "version": "3.3.0",
   "description": "Upload a file to Google Cloud Storage with built-in resumable behavior",
@@ -60,9 +60,6 @@ "repository": "googleapis/gcs-resumable-upload",
     "@types/mockery": "^1.4.29",
-    "@types/nock": "^10.0.0",
-    "@types/node": "^10.3.0",
+    "@types/node": "^14.0.0",
     "@types/pumpify": "^1.4.1",
     "@types/sinon": "^10.0.0",
-    "assert-rejects": "^1.0.0",
     "c8": "^7.0.0",
-    "codecov": "^3.0.4",
     "gts": "^2.0.0",
@@ -69,0 +66,0 @@ "is-stream": "^2.0.0",

README.md

@@ -237,2 +237,62 @@ # gcs-resumable-upload
 
+##### config.retryOptions
+
+- Type: `object`
+- *Optional*
+
+Parameters used to control retrying operations.
+
+```js
+interface RetryOptions {
+  retryDelayMultiplier?: number;
+  totalTimeout?: number;
+  maxRetryDelay?: number;
+  autoRetry?: boolean;
+  maxRetries?: number;
+  retryableErrorFn?: (err: ApiError) => boolean;
+}
+```
+
+##### config.retryOptions.retryDelayMultiplier
+
+- Type: `number`
+- *Optional*
+
+Base number used for exponential backoff. Default 2.
+
+##### config.retryOptions.totalTimeout
+
+- Type: `number`
+- *Optional*
+
+Upper bound on the total amount of time to attempt retrying, in seconds. Default: 600.
+
+##### config.retryOptions.maxRetryDelay
+
+- Type: `number`
+- *Optional*
+
+The maximum time to delay between retries, in seconds. Default: 64.
+
+##### config.retryOptions.autoRetry
+
+- Type: `boolean`
+- *Optional*
+
+Whether or not errors should be retried. Default: true.
+
+##### config.retryOptions.maxRetries
+
+- Type: `number`
+- *Optional*
+
+The maximum number of retries to attempt. Default: 5.
+
+##### config.retryOptions.retryableErrorFn
+
+- Type: `function`
+- *Optional*
+
+Custom function returning a boolean inidicating whether or not to retry an error.
+
 ---
@@ -239,0 +299,0 @@ <a name="events"></a>

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Unidentified License

License

(Experimental) Something that seems like a license was found, but its contents could not be matched with a known license.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Unidentified License

License

(Experimental) Something that seems like a license was found, but its contents could not be matched with a known license.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

73301

increased by9.81%

Dev dependency count

18

decreased by-14.29%

Lines of code

728

increased by12.17%

Number of lines in readme file

366

increased by19.61%

No dependency changes