@cumulus/aws-client - npm Package Compare versions

Comparing version 8.1.1 to 8.1.2

package.json

 {
   "name": "@cumulus/aws-client",
-  "version": "8.1.1",
+  "version": "8.1.2",
   "description": "Utilities for working with AWS",
@@ -46,5 +46,5 @@ "keywords": [
   "dependencies": {
-    "@cumulus/checksum": "8.1.1",
-    "@cumulus/errors": "8.1.1",
-    "@cumulus/logger": "8.1.1",
+    "@cumulus/checksum": "8.1.2",
+    "@cumulus/errors": "8.1.2",
+    "@cumulus/logger": "8.1.2",
     "aws-sdk": "^2.814.0",
@@ -58,3 +58,3 @@ "jsonpath-plus": "^1.1.0",
   },
-  "gitHead": "5ca7830a80c6c21eecece1f886418580aebc495d"
+  "gitHead": "9b5982281cefb50582fd7aad4cea67d29d02b6dc"
 }

README.md

@@ -67,2 +67,5 @@ # @cumulus/aws-client
 </dd>
+<dt><a href="#S3ObjectStore">S3ObjectStore</a></dt>
+<dd><p>Class to use when interacting with S3</p>
+</dd>
 </dl>
@@ -309,2 +312,3 @@
     * [.putFile(bucket, key, filename)](#module_S3.putFile) ⇒ <code>Promise</code>
+    * [.streamS3Upload(uploadStream, uploadParams)](#module_S3.streamS3Upload) ⇒ <code>Promise</code>
     * [.downloadS3File(s3Obj, filepath)](#module_S3.downloadS3File) ⇒ <code>Promise.&lt;string&gt;</code>
@@ -539,2 +543,18 @@ * [.getObjectSize(params)](#module_S3.getObjectSize) ⇒ <code>Promise.&lt;(number\|undefined)&gt;</code>
 
+<a name="module_S3.streamS3Upload"></a>
+
+### S3.streamS3Upload(uploadStream, uploadParams) ⇒ <code>Promise</code>
+Upload data to S3 using a stream
+
+We are not using `s3.upload().promise()` due to errors observed in testing
+with uncaught exceptions. By creating our own promise, we can ensure any
+errors from the streams or upload cause this promise to reject.
+
+**Kind**: static method of [<code>S3</code>](#module_S3)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| uploadStream | <code>Readable</code> | Stream of data to upload |
+| uploadParams | <code>Object</code> | see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property) |
+
 <a name="module_S3.downloadS3File"></a>
@@ -1076,3 +1096,52 @@
 **Returns**: <code>Promise.&lt;Object&gt;</code> - an S3 object description  
+<a name="S3ObjectStore"></a>
 
+## S3ObjectStore
+Class to use when interacting with S3
+
+**Kind**: global class  
+
+* [S3ObjectStore](#S3ObjectStore)
+    * [.signGetObject(objectUrl, [options], [queryParams])](#S3ObjectStore+signGetObject) ⇒ <code>Promise.&lt;string&gt;</code>
+    * [.signHeadObject(objectUrl, [options], [queryParams])](#S3ObjectStore+signHeadObject) ⇒ <code>Promise.&lt;string&gt;</code>
+
+<a name="S3ObjectStore+signGetObject"></a>
+
+### s3ObjectStore.signGetObject(objectUrl, [options], [queryParams]) ⇒ <code>Promise.&lt;string&gt;</code>
+Returns an HTTPS URL that can be used to perform a GET on the given object
+store URL
+
+**Kind**: instance method of [<code>S3ObjectStore</code>](#S3ObjectStore)  
+**Returns**: <code>Promise.&lt;string&gt;</code> - a signed URL  
+**Throws**:
+
+- TypeError - if the URL is not a recognized protocol or cannot be parsed
+
+
+| Param | Type | Description |
+| --- | --- | --- |
+| objectUrl | <code>string</code> | the URL of the object to sign |
+| [options] | <code>string</code> | options to pass to S3.getObject |
+| [queryParams] | <code>string</code> | a mapping of parameter key/values to put in the URL |
+
+<a name="S3ObjectStore+signHeadObject"></a>
+
+### s3ObjectStore.signHeadObject(objectUrl, [options], [queryParams]) ⇒ <code>Promise.&lt;string&gt;</code>
+Returns an HTTPS URL that can be used to perform a HEAD on the given object
+store URL
+
+**Kind**: instance method of [<code>S3ObjectStore</code>](#S3ObjectStore)  
+**Returns**: <code>Promise.&lt;string&gt;</code> - a signed URL  
+**Throws**:
+
+- TypeError - if the URL is not a recognized protocol or cannot be parsed
+
+
+| Param | Type | Description |
+| --- | --- | --- |
+| objectUrl | <code>string</code> | the URL of the object to sign |
+| [options] | <code>string</code> | options to pass to S3.getObject |
+| [queryParams] | <code>string</code> | a mapping of parameter key/values to put in the URL |
+
+
 ## About Cumulus
@@ -1079,0 +1148,0 @@

S3.d.ts

@@ -130,2 +130,14 @@ /**
 /**
+ * Upload data to S3 using a stream
+ *
+ * We are not using `s3.upload().promise()` due to errors observed in testing
+ * with uncaught exceptions. By creating our own promise, we can ensure any
+ * errors from the streams or upload cause this promise to reject.
+ *
+ * @param {Readable} uploadStream - Stream of data to upload
+ * @param {Object} uploadParams - see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
+ * @returns {Promise}
+ */
+export declare const streamS3Upload: (uploadStream: Readable, uploadParams: AWS.S3.PutObjectRequest) => Promise<unknown>;
+/**
  * Downloads the given s3Obj to the given filename in a streaming manner
@@ -132,0 +144,0 @@ *

S3.js

@@ -28,3 +28,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.moveObject = exports.multipartCopyObject = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.recursivelyDeleteS3Bucket = exports.deleteS3Files = exports.downloadS3Files = exports.fileExists = exports.getObjectReadStream = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
+exports.moveObject = exports.multipartCopyObject = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.recursivelyDeleteS3Bucket = exports.deleteS3Files = exports.downloadS3Files = exports.fileExists = exports.getObjectReadStream = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.streamS3Upload = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
 const fs_1 = __importDefault(require("fs"));
@@ -38,2 +38,3 @@ const isBoolean_1 = __importDefault(require("lodash/isBoolean"));
 const querystring_1 = __importDefault(require("querystring"));
+const stream_1 = require("stream");
 const util_1 = require("util");
@@ -210,2 +211,28 @@ const checksum_1 = require("@cumulus/checksum");
 /**
+ * Upload data to S3 using a stream
+ *
+ * We are not using `s3.upload().promise()` due to errors observed in testing
+ * with uncaught exceptions. By creating our own promise, we can ensure any
+ * errors from the streams or upload cause this promise to reject.
+ *
+ * @param {Readable} uploadStream - Stream of data to upload
+ * @param {Object} uploadParams - see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
+ * @returns {Promise}
+ */
+exports.streamS3Upload = (uploadStream, uploadParams) => new Promise((resolve, reject) => {
+    const pass = new stream_1.PassThrough();
+    uploadStream.pipe(pass);
+    uploadStream.on('error', reject);
+    pass.on('error', reject);
+    return services_1.s3().upload({
+        ...uploadParams,
+        Body: pass,
+    }, (err, uploadResponse) => {
+        if (err) {
+            return reject(err);
+        }
+        return resolve(uploadResponse);
+    });
+});
+/**
  * Downloads the given s3Obj to the given filename in a streaming manner
@@ -212,0 +239,0 @@ *

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

180990

increased by2.51%

Lines of code

3023

increased by1.31%

Number of lines in readme file

1159

increased by6.33%

Dependency changes

• + Added@cumulus/checksum@8.1.2(transitive)

• + Added@cumulus/errors@8.1.2(transitive)

• + Added@cumulus/logger@8.1.2(transitive)

• - Removed@cumulus/checksum@8.1.1(transitive)

• - Removed@cumulus/errors@8.1.1(transitive)

• - Removed@cumulus/logger@8.1.1(transitive)

• Updated@cumulus/checksum@8.1.2

• Updated@cumulus/errors@8.1.2

• Updated@cumulus/logger@8.1.2