@google-cloud/storage - npm Package Compare versions

Comparing version 7.1.0 to 7.2.0

build/src/file.d.ts

@@ -13,3 +13,3 @@ /// <reference types="node" />
 import { GetSignedUrlResponse, GetSignedUrlCallback, URLSigner, Query } from './signer';
-import { Duplexify } from './nodejs-common/util';
+import { Duplexify, GCCL_GCS_CMD_KEY } from './nodejs-common/util';
 import { CRC32CValidatorGenerator } from './crc32c';
@@ -111,2 +111,3 @@ import { URL } from 'url';
     preconditionOpts?: PreconditionOptions;
+    [GCCL_GCS_CMD_KEY]?: resumableUpload.UploadConfig[typeof GCCL_GCS_CMD_KEY];
 }
@@ -204,2 +205,3 @@ export type CreateResumableUploadResponse = [string];
     decompress?: boolean;
+    [GCCL_GCS_CMD_KEY]?: string;
 }
@@ -206,0 +208,0 @@ export interface SaveOptions extends CreateWriteStreamOptions {

build/src/nodejs-common/service.js

@@ -135,2 +135,5 @@ "use strict";
         };
+        if (reqOpts[util_1.GCCL_GCS_CMD_KEY]) {
+            reqOpts.headers['x-goog-api-client'] += ` gccl-gcs-cmd/${reqOpts[util_1.GCCL_GCS_CMD_KEY]}`;
+        }
         if (reqOpts.shouldReturnStream) {
@@ -137,0 +140,0 @@ return this.makeAuthenticatedRequest(reqOpts);

build/src/nodejs-common/util.d.ts

@@ -22,2 +22,9 @@ /*!
 import { Interceptor } from './service-object';
+/**
+ * A unique symbol for providing a `gccl-gcs-cmd` value
+ * for the `X-Goog-API-Client` header.
+ *
+ * E.g. the `V` in `X-Goog-API-Client: gccl-gcs-cmd/V`
+ **/
+export declare const GCCL_GCS_CMD_KEY: unique symbol;
 export type ResponseBody = any;
@@ -125,3 +132,5 @@ export interface DuplexifyOptions extends DuplexOptions {
     request?: r.Options;
-    makeAuthenticatedRequest(reqOpts: r.OptionsWithUri, fnobj: {
+    makeAuthenticatedRequest(reqOpts: r.OptionsWithUri & {
+        [GCCL_GCS_CMD_KEY]?: string;
+    }, fnobj: {
         onAuthenticated(err: Error | null, authenticatedReqOpts?: r.Options): void;
@@ -139,2 +148,3 @@ }): void;
     projectId?: string;
+    [GCCL_GCS_CMD_KEY]?: string;
 }
@@ -328,3 +338,3 @@ export interface ParsedHttpResponseBody {
     maybeOptionsOrCallback<T = {}, C = (err?: Error) => void>(optionsOrCallback?: T | C, cb?: C): [T, C];
-    _getDefaultHeaders(): {
+    _getDefaultHeaders(gcclGcsCmd?: string): {
         'User-Agent': string;
@@ -331,0 +341,0 @@ 'x-goog-api-client': string;

build/src/nodejs-common/util.js

@@ -18,3 +18,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.util = exports.Util = exports.PartialFailureError = exports.ApiError = void 0;
+exports.util = exports.Util = exports.PartialFailureError = exports.ApiError = exports.GCCL_GCS_CMD_KEY = void 0;
 /*!
@@ -33,2 +33,9 @@ * @module common/util
 const packageJson = require('../../../package.json');
+/**
+ * A unique symbol for providing a `gccl-gcs-cmd` value
+ * for the `X-Goog-API-Client` header.
+ *
+ * E.g. the `V` in `X-Goog-API-Client: gccl-gcs-cmd/V`
+ **/
+exports.GCCL_GCS_CMD_KEY = Symbol.for('GCCL_GCS_CMD');
 // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -277,3 +284,3 @@ const duplexify = require('duplexify');
                 }
-                requestDefaults.headers = util._getDefaultHeaders();
+                requestDefaults.headers = util._getDefaultHeaders(reqOpts[exports.GCCL_GCS_CMD_KEY]);
                 const request = teeny_request_1.teenyRequest.defaults(requestDefaults);
@@ -526,3 +533,3 @@ request(authenticatedReqOpts, (err, resp, body) => {
         }
-        requestDefaults.headers = this._getDefaultHeaders();
+        requestDefaults.headers = this._getDefaultHeaders(reqOpts[exports.GCCL_GCS_CMD_KEY]);
         const options = {
@@ -652,7 +659,11 @@ request: teeny_request_1.teenyRequest.defaults(requestDefaults),
     }
-    _getDefaultHeaders() {
-        return {
+    _getDefaultHeaders(gcclGcsCmd) {
+        const headers = {
             'User-Agent': util.getUserAgentFromPackageJson(packageJson),
             'x-goog-api-client': `${(0, util_1.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-invocation-id/${uuid.v4()}`,
         };
+        if (gcclGcsCmd) {
+            headers['x-goog-api-client'] += ` gccl-gcs-cmd/${gcclGcsCmd}`;
+        }
+        return headers;
     }
@@ -659,0 +670,0 @@ }

build/src/resumable-upload.d.ts

@@ -7,2 +7,3 @@ /// <reference types="node" />
 import { RetryOptions, PreconditionOptions } from './storage';
+import { GCCL_GCS_CMD_KEY } from './nodejs-common/util';
 export declare const PROTOCOL_REGEX: RegExp;
@@ -135,2 +136,3 @@ export interface ErrorWithCode extends Error {
     retryOptions: RetryOptions;
+    [GCCL_GCS_CMD_KEY]?: string;
 }
@@ -137,0 +139,0 @@ export interface ConfigMetadata {

build/src/resumable-upload.js

@@ -15,2 +15,8 @@ "use strict";
 // limitations under the License.
+var __classPrivateFieldSet = (this && this.__classPrivateFieldSet) || function (receiver, state, value, kind, f) {
+    if (kind === "m") throw new TypeError("Private method is not writable");
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a setter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot write private member to an object whose class did not declare it");
+    return (kind === "a" ? f.call(receiver, value) : f ? f.value = value : state.set(receiver, value)), value;
+};
 var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
@@ -21,3 +27,3 @@ if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
 };
-var _Upload_instances, _Upload_resetLocalBuffersCache, _Upload_addLocalBufferCache;
+var _Upload_instances, _Upload_gcclGcsCmd, _Upload_resetLocalBuffersCache, _Upload_addLocalBufferCache;
 Object.defineProperty(exports, "__esModule", { value: true });
@@ -33,2 +39,3 @@ exports.createURI = exports.upload = exports.Upload = exports.PROTOCOL_REGEX = void 0;
 const util_1 = require("./util");
+const util_2 = require("./nodejs-common/util");
 const NOT_FOUND_STATUS_CODE = 404;
@@ -63,2 +70,3 @@ const RESUMABLE_INCOMPLETE_STATUS_CODE = 308;
         this.upstreamEnded = false;
+        _Upload_gcclGcsCmd.set(this, void 0);
         cfg = cfg || {};
@@ -123,2 +131,3 @@ if (!cfg.bucket || !cfg.file) {
         this.contentLength = isNaN(contentLength) ? '*' : contentLength;
+        __classPrivateFieldSet(this, _Upload_gcclGcsCmd, cfg[util_2.GCCL_GCS_CMD_KEY], "f");
         this.once('writing', () => {
@@ -312,2 +321,6 @@ if (this.uri) {
         }
+        let googAPIClient = `${(0, util_1.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-invocation-id/${this.currentInvocationId.uri}`;
+        if (__classPrivateFieldGet(this, _Upload_gcclGcsCmd, "f")) {
+            googAPIClient += ` gccl-gcs-cmd/${__classPrivateFieldGet(this, _Upload_gcclGcsCmd, "f")}`;
+        }
         // Check if headers already exist before creating new ones
@@ -323,3 +336,3 @@ const reqOpts = {
             headers: {
-                'x-goog-api-client': `${(0, util_1.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-invocation-id/${this.currentInvocationId.uri}`,
+                'x-goog-api-client': googAPIClient,
                 ...headers,
@@ -462,4 +475,8 @@ },
         });
+        let googAPIClient = `${(0, util_1.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-invocation-id/${this.currentInvocationId.chunk}`;
+        if (__classPrivateFieldGet(this, _Upload_gcclGcsCmd, "f")) {
+            googAPIClient += ` gccl-gcs-cmd/${__classPrivateFieldGet(this, _Upload_gcclGcsCmd, "f")}`;
+        }
         const headers = {
-            'x-goog-api-client': `${(0, util_1.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-invocation-id/${this.currentInvocationId.chunk}`,
+            'x-goog-api-client': googAPIClient,
         };
@@ -579,2 +596,6 @@ // If using multiple chunk upload, set appropriate header
     async getAndSetOffset() {
+        let googAPIClient = `${(0, util_1.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-invocation-id/${this.currentInvocationId.offset}`;
+        if (__classPrivateFieldGet(this, _Upload_gcclGcsCmd, "f")) {
+            googAPIClient += ` gccl-gcs-cmd/${__classPrivateFieldGet(this, _Upload_gcclGcsCmd, "f")}`;
+        }
         const opts = {
@@ -586,3 +607,3 @@ method: 'PUT',
                 'Content-Range': 'bytes */*',
-                'x-goog-api-client': `${(0, util_1.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-invocation-id/${this.currentInvocationId.offset}`,
+                'x-goog-api-client': googAPIClient,
             },
@@ -751,3 +772,3 @@ };
 exports.Upload = Upload;
-_Upload_instances = new WeakSet(), _Upload_resetLocalBuffersCache = function _Upload_resetLocalBuffersCache() {
+_Upload_gcclGcsCmd = new WeakMap(), _Upload_instances = new WeakSet(), _Upload_resetLocalBuffersCache = function _Upload_resetLocalBuffersCache() {
     this.localWriteCache = [];
@@ -754,0 +775,0 @@ this.localWriteCacheByteLength = 0;

build/src/transfer-manager.d.ts

@@ -44,4 +44,8 @@ /*!
     uploadId?: string;
+    autoAbortFailure?: boolean;
     partsMap?: Map<number, string>;
     validation?: 'md5' | false;
+    headers?: {
+        [key: string]: string;
+    };
 }
@@ -53,5 +57,8 @@ export interface MultiPartUploadHelper {
     partsMap?: Map<number, string>;
-    initiateUpload(): Promise<void>;
+    initiateUpload(headers?: {
+        [key: string]: string;
+    }): Promise<void>;
     uploadPart(partNumber: number, chunk: Buffer, validation?: 'md5' | false): Promise<void>;
     completeUpload(): Promise<GaxiosResponse | undefined>;
+    abortUpload(): Promise<void>;
 }
@@ -204,2 +211,6 @@ export type MultiPartHelperGenerator = (bucket: Bucket, fileName: string, uploadId?: string, partsMap?: Map<number, string>) => MultiPartUploadHelper;
      * specified in partsMap
+     * @property {object} [headers] headers to be sent when initiating the multipart upload.
+     * See {@link https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers| Request Headers: Initiate a Multipart Upload}
+     * @property {boolean} [autoAbortFailure] boolean to indicate if an in progress upload session will be automatically aborted upon failure. If not set,
+     * failures will be automatically aborted.
      * @experimental
@@ -206,0 +217,0 @@ */

build/src/transfer-manager.js

@@ -22,3 +22,3 @@ "use strict";
 };
-var _XMLMultiPartUploadHelper_instances, _XMLMultiPartUploadHelper_handleErrorResponse;
+var _XMLMultiPartUploadHelper_instances, _XMLMultiPartUploadHelper_setGoogApiClientHeaders, _XMLMultiPartUploadHelper_handleErrorResponse;
 Object.defineProperty(exports, "__esModule", { value: true });
@@ -34,2 +34,5 @@ exports.TransferManager = exports.MultiPartUploadError = void 0;
 const crypto_1 = require("crypto");
+const util_1 = require("./nodejs-common/util");
+const util_2 = require("./util");
+const packageJson = require('../../package.json');
 /**
@@ -71,2 +74,15 @@ * Default number of concurrently executing promises to use when calling uploadManyFiles.
 const EMPTY_REGEX = '(?:)';
+/**
+ * The `gccl-gcs-cmd` value for the `X-Goog-API-Client` header.
+ * Example: `gccl-gcs-cmd/tm.upload_many`
+ *
+ * @see {@link GCCL_GCS_CMD}.
+ * @see {@link GCCL_GCS_CMD_KEY}.
+ */
+const GCCL_GCS_CMD_FEATURE = {
+    UPLOAD_MANY: 'tm.upload_many',
+    DOWNLOAD_MANY: 'tm.download_many',
+    UPLOAD_SHARDED: 'tm.upload_sharded',
+    DOWNLOAD_SHARDED: 'tm.download_sharded',
+};
 const defaultMultiPartGenerator = (bucket, fileName, uploadId, partsMap) => {
@@ -96,3 +112,2 @@ return new XMLMultiPartUploadHelper(bucket, fileName, uploadId, partsMap);
         this.fileName = fileName;
-        // eslint-disable-next-line prettier/prettier
         this.baseUrl = `https://${bucket.name}.${new URL(this.bucket.storage.apiEndpoint).hostname}/${fileName}`;
@@ -114,3 +129,3 @@ this.xmlBuilder = new fast_xml_parser_1.XMLBuilder({ arrayNodeName: 'Part' });
      */
-    async initiateUpload() {
+    async initiateUpload(headers = {}) {
         const url = `${this.baseUrl}?uploads`;
@@ -120,2 +135,3 @@ return retry(async (bail) => {
                 const res = await this.authClient.request({
+                    headers: __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this, headers),
                     method: 'POST',
@@ -146,3 +162,3 @@ url,
         const url = `${this.baseUrl}?partNumber=${partNumber}&uploadId=${this.uploadId}`;
-        let headers = {};
+        let headers = __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this);
         if (validation === 'md5') {
@@ -188,2 +204,3 @@ const hash = (0, crypto_1.createHash)('md5').update(chunk).digest('base64');
                 const res = await this.authClient.request({
+                    headers: __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_setGoogApiClientHeaders).call(this),
                     url,
@@ -204,4 +221,45 @@ method: 'POST',
     }
+    /**
+     * Aborts an multipart upload that is in progress. Once aborted, any parts in the process of being uploaded fail,
+     * and future requests using the upload ID fail.
+     *
+     * @returns {Promise<void>}
+     */
+    async abortUpload() {
+        const url = `${this.baseUrl}?uploadId=${this.uploadId}`;
+        return retry(async (bail) => {
+            try {
+                const res = await this.authClient.request({
+                    url,
+                    method: 'DELETE',
+                });
+                if (res.data && res.data.error) {
+                    throw res.data.error;
+                }
+            }
+            catch (e) {
+                __classPrivateFieldGet(this, _XMLMultiPartUploadHelper_instances, "m", _XMLMultiPartUploadHelper_handleErrorResponse).call(this, e, bail);
+                return;
+            }
+        }, this.retryOptions);
+    }
 }
-_XMLMultiPartUploadHelper_instances = new WeakSet(), _XMLMultiPartUploadHelper_handleErrorResponse = function _XMLMultiPartUploadHelper_handleErrorResponse(err, bail) {
+_XMLMultiPartUploadHelper_instances = new WeakSet(), _XMLMultiPartUploadHelper_setGoogApiClientHeaders = function _XMLMultiPartUploadHelper_setGoogApiClientHeaders(headers = {}) {
+    let headerFound = false;
+    for (const [key, value] of Object.entries(headers)) {
+        if (key.toLocaleLowerCase().trim() === 'x-goog-api-client') {
+            headerFound = true;
+            // Prepend command feature to value, if not already there
+            if (!value.includes(GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED)) {
+                headers[key] = `${value} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`;
+            }
+            break;
+        }
+    }
+    // If the header isn't present, add it
+    if (!headerFound) {
+        headers['x-goog-api-client'] = `${(0, util_2.getRuntimeTrackingString)()} gccl/${packageJson.version} gccl-gcs-cmd/${GCCL_GCS_CMD_FEATURE.UPLOAD_SHARDED}`;
+    }
+    return headers;
+}, _XMLMultiPartUploadHelper_handleErrorResponse = function _XMLMultiPartUploadHelper_handleErrorResponse(err, bail) {
     if (this.bucket.storage.retryOptions.autoRetry &&
@@ -299,6 +357,9 @@ this.bucket.storage.retryOptions.retryableErrorFn(err)) {
                 ...options.passthroughOptions,
+                [util_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.UPLOAD_MANY,
             };
-            passThroughOptionsCopy.destination = filePath;
+            passThroughOptionsCopy.destination = filePath
+                .split(path.sep)
+                .join(path.posix.sep);
             if (options.prefix) {
-                passThroughOptionsCopy.destination = path.join(options.prefix, passThroughOptionsCopy.destination);
+                passThroughOptionsCopy.destination = path.posix.join(...options.prefix.split(path.sep), passThroughOptionsCopy.destination);
             }
@@ -376,2 +437,3 @@ promises.push(limit(() => this.bucket.upload(filePath, passThroughOptionsCopy)));
                 ...options.passthroughOptions,
+                [util_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_MANY,
             };
@@ -441,3 +503,9 @@ if (options.prefix) {
             chunkEnd = chunkEnd > size ? size : chunkEnd;
-            promises.push(limit(() => file.download({ start: chunkStart, end: chunkEnd }).then(resp => {
+            promises.push(limit(() => file
+                .download({
+                start: chunkStart,
+                end: chunkEnd,
+                [util_1.GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_SHARDED,
+            })
+                .then(resp => {
                 return fileToWrite.write(resp[0], 0, resp[0].length, chunkStart);
@@ -479,2 +547,6 @@ })));
      * specified in partsMap
+     * @property {object} [headers] headers to be sent when initiating the multipart upload.
+     * See {@link https://cloud.google.com/storage/docs/xml-api/post-object-multipart#request_headers| Request Headers: Initiate a Multipart Upload}
+     * @property {boolean} [autoAbortFailure] boolean to indicate if an in progress upload session will be automatically aborted upon failure. If not set,
+     * failures will be automatically aborted.
      * @experimental
@@ -521,3 +593,3 @@ */
             if (options.uploadId === undefined) {
-                await mpuHelper.initiateUpload();
+                await mpuHelper.initiateUpload(options.headers);
             }
@@ -542,2 +614,12 @@ const startOrResumptionByte = mpuHelper.partsMap.size * chunkSize;
         catch (e) {
+            if ((options.autoAbortFailure === undefined || options.autoAbortFailure) &&
+                mpuHelper.uploadId) {
+                try {
+                    await mpuHelper.abortUpload();
+                    return;
+                }
+                catch (e) {
+                    throw new MultiPartUploadError(e.message, mpuHelper.uploadId, mpuHelper.partsMap);
+                }
+            }
             throw new MultiPartUploadError(e.message, mpuHelper.uploadId, mpuHelper.partsMap);
@@ -544,0 +626,0 @@ }

package.json

 {
   "name": "@google-cloud/storage",
   "description": "Cloud Storage Client Library for Node.js",
-  "version": "7.1.0",
+  "version": "7.2.0",
   "license": "Apache-2.0",
@@ -30,10 +30,10 @@ "author": "Google Inc.",
   "scripts": {
-    "predocs": "npm run compile",
+    "predocs": "npm run compile -- --sourceMap",
     "docs": "jsdoc -c .jsdoc.js",
     "system-test": "mocha build/system-test --timeout 600000 --exit",
     "conformance-test": "mocha --parallel build/conformance-test/ --require build/conformance-test/globalHooks.js",
-    "preconformance-test": "npm run compile",
-    "presystem-test": "npm run compile",
+    "preconformance-test": "npm run compile -- --sourceMap",
+    "presystem-test": "npm run compile -- --sourceMap",
     "test": "c8 mocha build/test",
-    "pretest": "npm run compile",
+    "pretest": "npm run compile -- --sourceMap",
     "lint": "gts check",
@@ -62,3 +62,3 @@ "samples-test": "npm link && cd samples/ && npm link ../ && npm test && cd ../",
     "ent": "^2.2.0",
-    "fast-xml-parser": "^4.2.2",
+    "fast-xml-parser": "^4.3.0",
     "gaxios": "^6.0.2",
@@ -89,3 +89,3 @@ "google-auth-library": "^9.0.0",
     "@types/sinon": "^10.0.15",
-    "@types/tmp": "0.2.3",
+    "@types/tmp": "0.2.4",
     "@types/uuid": "^8.0.0",
@@ -105,3 +105,3 @@ "@types/yargs": "^17.0.10",
     "proxyquire": "^2.1.3",
-    "sinon": "^15.2.0",
+    "sinon": "^16.0.0",
     "tmp": "^0.2.0",
@@ -108,0 +108,0 @@ "typescript": "^5.1.6",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

874876

increased by0.99%

Lines of code

17625

increased by0.85%

Dependency changes

• Updatedfast-xml-parser@^4.3.0