@cumulus/aws-client - npm Package Compare versions

Comparing version 9.3.0 to 9.4.0

package.json

 {
   "name": "@cumulus/aws-client",
-  "version": "9.3.0",
+  "version": "9.4.0",
   "description": "Utilities for working with AWS",
@@ -46,5 +46,5 @@ "keywords": [
   "dependencies": {
-    "@cumulus/checksum": "9.3.0",
-    "@cumulus/errors": "9.3.0",
-    "@cumulus/logger": "9.3.0",
+    "@cumulus/checksum": "9.4.0",
+    "@cumulus/errors": "9.4.0",
+    "@cumulus/logger": "9.4.0",
     "aws-sdk": "^2.814.0",
@@ -58,3 +58,3 @@ "jsonpath-plus": "^1.1.0",
   },
-  "gitHead": "1e0ecaff0e7107e3c5a83f0479972bca0f223842"
+  "gitHead": "a1c0e94e07adf3a5d2c8df93f6f5f216f5ceafe7"
 }

README.md

@@ -331,2 +331,3 @@ # @cumulus/aws-client
     * [.putFile(bucket, key, filename)](#module_S3.putFile) ⇒ <code>Promise</code>
+    * [.streamS3Upload(uploadStream, uploadParams)](#module_S3.streamS3Upload) ⇒ <code>Promise</code>
     * [.downloadS3File(s3Obj, filepath)](#module_S3.downloadS3File) ⇒ <code>Promise.&lt;string&gt;</code>
@@ -563,2 +564,18 @@ * [.getObjectSize(params)](#module_S3.getObjectSize) ⇒ <code>Promise.&lt;(number\|undefined)&gt;</code>
 
+<a name="module_S3.streamS3Upload"></a>
+
+### S3.streamS3Upload(uploadStream, uploadParams) ⇒ <code>Promise</code>
+Upload data to S3 using a stream
+
+We are not using `s3.upload().promise()` due to errors observed in testing
+with uncaught exceptions. By creating our own promise, we can ensure any
+errors from the streams or upload cause this promise to reject.
+
+**Kind**: static method of [<code>S3</code>](#module_S3)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| uploadStream | <code>Readable</code> | Stream of data to upload |
+| uploadParams | <code>Object</code> | see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property) |
+
 <a name="module_S3.downloadS3File"></a>
@@ -886,3 +903,3 @@
     * [.receiveSQSMessages(queueUrl, options)](#module_SQS.receiveSQSMessages) ⇒ <code>Promise.&lt;Array&gt;</code>
-    * [.sqsQueueExists(queue)](#module_SQS.sqsQueueExists) ⇒ <code>Promise.&lt;boolean&gt;</code>
+    * [.sqsQueueExists(queueUrl)](#module_SQS.sqsQueueExists) ⇒ <code>Promise.&lt;boolean&gt;</code>
 
@@ -946,7 +963,7 @@ <a name="module_SQS.deleteSQSMessage"></a>
 
-### SQS.sqsQueueExists(queue) ⇒ <code>Promise.&lt;boolean&gt;</code>
+### SQS.sqsQueueExists(queueUrl) ⇒ <code>Promise.&lt;boolean&gt;</code>
 Test if an SQS queue exists
 
 **Kind**: static method of [<code>SQS</code>](#module_SQS)  
-**Returns**: <code>Promise.&lt;boolean&gt;</code> - a Promise that will resolve to a boolean indicating
+**Returns**: <code>Promise.&lt;boolean&gt;</code> - - a Promise that will resolve to a boolean indicating
                               if the queue exists  
@@ -956,3 +973,3 @@
 | --- | --- | --- |
-| queue | <code>Object</code> | queue name or url |
+| queueUrl | <code>Object</code> | queue url |
 
@@ -959,0 +976,0 @@ <a name="module_SecretsManager"></a>

S3.d.ts

@@ -130,2 +130,14 @@ /**
 /**
+ * Upload data to S3 using a stream
+ *
+ * We are not using `s3.upload().promise()` due to errors observed in testing
+ * with uncaught exceptions. By creating our own promise, we can ensure any
+ * errors from the streams or upload cause this promise to reject.
+ *
+ * @param {Readable} uploadStream - Stream of data to upload
+ * @param {Object} uploadParams - see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
+ * @returns {Promise}
+ */
+export declare const streamS3Upload: (uploadStream: Readable, uploadParams: AWS.S3.PutObjectRequest) => Promise<unknown>;
+/**
  * Downloads the given s3Obj to the given filename in a streaming manner
@@ -132,0 +144,0 @@ *

S3.js

@@ -28,3 +28,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.moveObject = exports.multipartCopyObject = exports.createS3Buckets = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.deleteS3Buckets = exports.recursivelyDeleteS3Bucket = exports.deleteS3Files = exports.downloadS3Files = exports.fileExists = exports.getObjectReadStream = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
+exports.moveObject = exports.multipartCopyObject = exports.createS3Buckets = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.deleteS3Buckets = exports.recursivelyDeleteS3Bucket = exports.deleteS3Files = exports.downloadS3Files = exports.fileExists = exports.getObjectReadStream = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.streamS3Upload = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
 const fs_1 = __importDefault(require("fs"));
@@ -38,2 +38,3 @@ const isBoolean_1 = __importDefault(require("lodash/isBoolean"));
 const querystring_1 = __importDefault(require("querystring"));
+const stream_1 = require("stream");
 const util_1 = require("util");
@@ -210,2 +211,28 @@ const checksum_1 = require("@cumulus/checksum");
 /**
+ * Upload data to S3 using a stream
+ *
+ * We are not using `s3.upload().promise()` due to errors observed in testing
+ * with uncaught exceptions. By creating our own promise, we can ensure any
+ * errors from the streams or upload cause this promise to reject.
+ *
+ * @param {Readable} uploadStream - Stream of data to upload
+ * @param {Object} uploadParams - see [S3.upload()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#upload-property)
+ * @returns {Promise}
+ */
+exports.streamS3Upload = (uploadStream, uploadParams) => new Promise((resolve, reject) => {
+    const pass = new stream_1.PassThrough();
+    uploadStream.pipe(pass);
+    uploadStream.on('error', reject);
+    pass.on('error', reject);
+    return services_1.s3().upload({
+        ...uploadParams,
+        Body: pass,
+    }, (err, uploadResponse) => {
+        if (err) {
+            return reject(err);
+        }
+        return resolve(uploadResponse);
+    });
+});
+/**
  * Downloads the given s3Obj to the given filename in a streaming manner
@@ -212,0 +239,0 @@ *

SQS.d.ts

@@ -7,2 +7,3 @@ /**
 }
+export declare const getQueueNameFromUrl: (queueUrl: string) => string | undefined;
 export declare const getQueueUrl: (sourceArn: string, queueName: string) => string;
@@ -66,8 +67,8 @@ export declare const getQueueUrlByName: (queueName: string) => Promise<string | undefined>;
  *
- * @param {Object} queue - queue name or url
- * @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
+ * @param {Object} queueUrl     - queue url
+ * @returns {Promise<boolean>}  - a Promise that will resolve to a boolean indicating
  *                               if the queue exists
  */
-export declare const sqsQueueExists: (queue: string) => Promise<boolean>;
+export declare const sqsQueueExists: (queueUrl: string) => Promise<boolean>;
 export {};
 //# sourceMappingURL=SQS.d.ts.map

SQS.js

@@ -28,3 +28,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sqsQueueExists = exports.deleteSQSMessage = exports.parseSQSMessageBody = exports.receiveSQSMessages = exports.sendSQSMessage = exports.getQueueAttributes = exports.deleteQueue = exports.createQueue = exports.getQueueUrlByName = exports.getQueueUrl = void 0;
+exports.sqsQueueExists = exports.deleteSQSMessage = exports.parseSQSMessageBody = exports.receiveSQSMessages = exports.sendSQSMessage = exports.getQueueAttributes = exports.deleteQueue = exports.createQueue = exports.getQueueUrlByName = exports.getQueueUrl = exports.getQueueNameFromUrl = void 0;
 const get_1 = __importDefault(require("lodash/get"));
@@ -38,2 +38,3 @@ const isObject_1 = __importDefault(require("lodash/isObject"));
 const utils_1 = require("./utils");
+exports.getQueueNameFromUrl = (queueUrl) => queueUrl.split('/').pop();
 exports.getQueueUrl = (sourceArn, queueName) => {
@@ -153,10 +154,10 @@ const arnParts = sourceArn.split(':');
  *
- * @param {Object} queue - queue name or url
- * @returns {Promise<boolean>} a Promise that will resolve to a boolean indicating
+ * @param {Object} queueUrl     - queue url
+ * @returns {Promise<boolean>}  - a Promise that will resolve to a boolean indicating
  *                               if the queue exists
  */
-exports.sqsQueueExists = async (queue) => {
-    const QueueName = queue.split('/').pop();
+exports.sqsQueueExists = async (queueUrl) => {
+    const QueueName = exports.getQueueNameFromUrl(queueUrl);
     if (!QueueName) {
-        throw new Error(`Unable to determine QueueName from ${queue}`);
+        throw new Error(`Unable to determine QueueName from ${queueUrl}`);
     }
@@ -163,0 +164,0 @@ try {

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

194579

increased by1.37%

Lines of code

3268

increased by1.3%

Number of lines in readme file

1204

increased by1.43%

Dependency changes

• + Added@cumulus/checksum@9.4.0(transitive)

• + Added@cumulus/errors@9.4.0(transitive)

• + Added@cumulus/logger@9.4.0(transitive)

• + Addedis-typed-array@1.1.15(transitive)

• + Addedmath-intrinsics@1.1.0(transitive)

• + Addedwhich-typed-array@1.1.18(transitive)

• - Removed@cumulus/checksum@9.3.0(transitive)

• - Removed@cumulus/errors@9.3.0(transitive)

• - Removed@cumulus/logger@9.3.0(transitive)

• - Removedis-typed-array@1.1.14(transitive)

• - Removedmath-intrinsics@1.0.0(transitive)

• - Removedwhich-typed-array@1.1.17(transitive)

• Updated@cumulus/checksum@9.4.0

• Updated@cumulus/errors@9.4.0

• Updated@cumulus/logger@9.4.0