@cumulus/aws-client - npm Package Compare versions

Comparing version 8.1.0 to 9.0.0

DynamoDb.d.ts

 /**
  * @module DynamoDb
  */
+import pRetry from 'p-retry';
+import { DocumentClient } from 'aws-sdk/lib/dynamodb/document_client';
 /**
@@ -23,3 +25,3 @@ * Call DynamoDb client get
     getParams?: object | undefined;
-}) => Promise<import("aws-sdk/clients/dynamodb").DocumentClient.AttributeMap>;
+}) => Promise<DocumentClient.AttributeMap>;
 /**
@@ -39,4 +41,4 @@ * Call DynamoDb client scan
         filter?: string | undefined;
-        names?: import("aws-sdk/clients/dynamodb").DocumentClient.ExpressionAttributeNameMap | undefined;
-        values?: import("aws-sdk/clients/dynamodb").DocumentClient.ExpressionAttributeValueMap | undefined;
+        names?: DocumentClient.ExpressionAttributeNameMap | undefined;
+        values?: DocumentClient.ExpressionAttributeValueMap | undefined;
     } | undefined;
@@ -46,5 +48,29 @@ fields?: string | undefined;
     select: string;
-    startKey?: import("aws-sdk/clients/dynamodb").DocumentClient.Key | undefined;
-}) => Promise<import("aws-sdk/lib/request").PromiseResult<import("aws-sdk/clients/dynamodb").DocumentClient.ScanOutput, import("aws-sdk").AWSError>>;
+    startKey?: DocumentClient.Key | undefined;
+}) => Promise<import("aws-sdk/lib/request").PromiseResult<DocumentClient.ScanOutput, import("aws-sdk").AWSError>>;
 /**
+ * Do a parallel scan of DynamoDB table using a document client.
+ *
+ * See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.ParallelScan.
+ * See [DocumentClient.scan()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB/DocumentClient.html#scan-property).
+ *
+ * @param {Object} params
+ * @param {number} params.totalSegments
+ *   Total number of segments to divide table into for parallel scanning
+ * @param {DocumentClient.ScanInput} params.scanParams
+ *   Params for the DynamoDB client scan operation
+ *   See https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Scan.html
+ * @param {function} params.processItemsFunc - Function used to process returned items by scan
+ * @param {DocumentClient} [params.dynamoDbClient] - Instance of Dynamo DB document client
+ * @param {pRetry.Options} [params.retryOptions] - Retry options for scan operations
+ * @returns {Promise}
+ */
+export declare const parallelScan: (params: {
+    totalSegments: number;
+    scanParams: DocumentClient.ScanInput;
+    processItemsFunc: (items: DocumentClient.ItemList) => Promise<void>;
+    dynamoDbClient?: DocumentClient | undefined;
+    retryOptions?: pRetry.Options | undefined;
+}) => Promise<void[]>;
+/**
  * Create a DynamoDB table and then wait for the table to exist
@@ -51,0 +77,0 @@ *

DynamoDb.js

@@ -5,4 +5,10 @@ "use strict";
  */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.deleteAndWaitForDynamoDbTableNotExists = exports.createAndWaitForDynamoDbTable = exports.scan = exports.get = void 0;
+exports.deleteAndWaitForDynamoDbTableNotExists = exports.createAndWaitForDynamoDbTable = exports.parallelScan = exports.scan = exports.get = void 0;
+const p_map_1 = __importDefault(require("p-map"));
+const p_retry_1 = __importDefault(require("p-retry"));
+const range_1 = __importDefault(require("lodash/range"));
 const errors_1 = require("@cumulus/errors");
@@ -93,2 +99,41 @@ const services_1 = require("./services");
 /**
+ * Do a parallel scan of DynamoDB table using a document client.
+ *
+ * See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.ParallelScan.
+ * See [DocumentClient.scan()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB/DocumentClient.html#scan-property).
+ *
+ * @param {Object} params
+ * @param {number} params.totalSegments
+ *   Total number of segments to divide table into for parallel scanning
+ * @param {DocumentClient.ScanInput} params.scanParams
+ *   Params for the DynamoDB client scan operation
+ *   See https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Scan.html
+ * @param {function} params.processItemsFunc - Function used to process returned items by scan
+ * @param {DocumentClient} [params.dynamoDbClient] - Instance of Dynamo DB document client
+ * @param {pRetry.Options} [params.retryOptions] - Retry options for scan operations
+ * @returns {Promise}
+ */
+exports.parallelScan = async (params) => {
+    const { totalSegments, scanParams, processItemsFunc, dynamoDbClient = services_1.dynamodbDocClient(), retryOptions, } = params;
+    return p_map_1.default(range_1.default(totalSegments), async (_, segmentIndex) => {
+        let exclusiveStartKey;
+        const segmentScanParams = {
+            ...scanParams,
+            TotalSegments: totalSegments,
+            Segment: segmentIndex,
+        };
+        /* eslint-disable no-await-in-loop */
+        do {
+            const { Items = [], LastEvaluatedKey, } = await p_retry_1.default(() => dynamoDbClient.scan(segmentScanParams).promise(), retryOptions);
+            exclusiveStartKey = LastEvaluatedKey;
+            segmentScanParams.ExclusiveStartKey = exclusiveStartKey;
+            await processItemsFunc(Items);
+        } while (exclusiveStartKey);
+        /* eslint-enable no-await-in-loop */
+        return Promise.resolve();
+    }, {
+        stopOnError: false,
+    });
+};
+/**
  * Create a DynamoDB table and then wait for the table to exist
@@ -95,0 +140,0 @@ *

DynamoDbSearchQueue.d.ts

 import * as AWS from 'aws-sdk';
+declare type searchType = 'scan' | 'query';
 /**
@@ -11,3 +12,3 @@ * Class to efficiently search all of the items in a DynamoDB table, without loading them all into
     private items;
-    constructor(params: AWS.DynamoDB.DocumentClient.ScanInput, searchType?: 'scan');
+    constructor(params: AWS.DynamoDB.DocumentClient.ScanInput, searchType?: searchType);
     /**
@@ -14,0 +15,0 @@ * Drain all values from the searchQueue, and return to the user.

index.d.ts

 export * as CloudFormation from './CloudFormation';
 export * as DynamoDb from './DynamoDb';
+export * as KMS from './KMS';
 export * as S3 from './S3';
@@ -4,0 +5,0 @@ export * as services from './services';

index.js

@@ -24,2 +24,3 @@ "use strict";
 exports.DynamoDb = __importStar(require("./DynamoDb"));
+exports.KMS = __importStar(require("./KMS"));
 exports.S3 = __importStar(require("./S3"));
@@ -26,0 +27,0 @@ exports.services = __importStar(require("./services"));

package.json

 {
   "name": "@cumulus/aws-client",
-  "version": "8.1.0",
+  "version": "9.0.0",
   "description": "Utilities for working with AWS",
@@ -46,5 +46,5 @@ "keywords": [
   "dependencies": {
-    "@cumulus/checksum": "8.1.0",
-    "@cumulus/errors": "8.1.0",
-    "@cumulus/logger": "8.1.0",
+    "@cumulus/checksum": "9.0.0",
+    "@cumulus/errors": "9.0.0",
+    "@cumulus/logger": "9.0.0",
     "aws-sdk": "^2.814.0",
@@ -58,3 +58,3 @@ "jsonpath-plus": "^1.1.0",
   },
-  "gitHead": "ced7e73ddeb4d22892ca3bc6be27fcec58654cb8"
+  "gitHead": "04ae1372d7c28a4d13c4e386307997ff45ddea21"
 }

README.md

@@ -144,2 +144,3 @@ # @cumulus/aws-client
     * [.scan](#module_DynamoDb.scan) ⇒ <code>Promise.&lt;Object&gt;</code>
+    * [.parallelScan(params)](#module_DynamoDb.parallelScan) ⇒ <code>Promise</code>
     * [.createAndWaitForDynamoDbTable(params)](#module_DynamoDb.createAndWaitForDynamoDbTable) ⇒ <code>Promise.&lt;Object&gt;</code>
@@ -184,2 +185,21 @@ * [.deleteAndWaitForDynamoDbTableNotExists(params)](#module_DynamoDb.deleteAndWaitForDynamoDbTableNotExists) ⇒ <code>Promise</code>
 
+<a name="module_DynamoDb.parallelScan"></a>
+
+### DynamoDb.parallelScan(params) ⇒ <code>Promise</code>
+Do a parallel scan of DynamoDB table using a document client.
+
+See https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Scan.html#Scan.ParallelScan.
+See [DocumentClient.scan()](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/DynamoDB/DocumentClient.html#scan-property).
+
+**Kind**: static method of [<code>DynamoDb</code>](#module_DynamoDb)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| params | <code>Object</code> |  |
+| params.totalSegments | <code>number</code> | Total number of segments to divide table into for parallel scanning |
+| params.scanParams | <code>DocumentClient.ScanInput</code> | Params for the DynamoDB client scan operation   See https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Scan.html |
+| params.processItemsFunc | <code>function</code> | Function used to process returned items by scan |
+| [params.dynamoDbClient] | <code>DocumentClient</code> | Instance of Dynamo DB document client |
+| [params.retryOptions] | <code>pRetry.Options</code> | Retry options for scan operations |
+
 <a name="module_DynamoDb.createAndWaitForDynamoDbTable"></a>
@@ -319,2 +339,3 @@
     * [.deleteS3Files(s3Objs)](#module_S3.deleteS3Files) ⇒ <code>Promise</code>
+    * [.deleteS3Buckets(buckets)](#module_S3.deleteS3Buckets) ⇒ <code>Promise</code>
     * [.uploadS3FileStream(fileStream, bucket, key, s3opts)](#module_S3.uploadS3FileStream) ⇒ <code>Promise</code>
@@ -327,2 +348,3 @@ * [.listS3Objects(bucket, prefix, skipFolders)](#module_S3.listS3Objects) ⇒ <code>Promise</code>
     * [.createBucket(Bucket)](#module_S3.createBucket) ⇒ <code>Promise</code>
+    * [.createS3Buckets(buckets)](#module_S3.createS3Buckets) ⇒ <code>Promise</code>
     * [.multipartCopyObject(params)](#module_S3.multipartCopyObject) ⇒ <code>Promise.&lt;{etag: string}&gt;</code>
@@ -667,2 +689,14 @@ * [.moveObject(params)](#module_S3.moveObject) ⇒ <code>Promise.&lt;undefined&gt;</code>
 
+<a name="module_S3.deleteS3Buckets"></a>
+
+### S3.deleteS3Buckets(buckets) ⇒ <code>Promise</code>
+Delete a list of buckets and all of their objects from S3
+
+**Kind**: static method of [<code>S3</code>](#module_S3)  
+**Returns**: <code>Promise</code> - the promised result of `S3.deleteBucket`  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| buckets | <code>Array</code> | list of bucket names |
+
 <a name="module_S3.uploadS3FileStream"></a>
@@ -778,2 +812,13 @@
 
+<a name="module_S3.createS3Buckets"></a>
+
+### S3.createS3Buckets(buckets) ⇒ <code>Promise</code>
+Create multiple S3 buckets
+
+**Kind**: static method of [<code>S3</code>](#module_S3)  
+
+| Param | Type | Description |
+| --- | --- | --- |
+| buckets | <code>Array.&lt;string&gt;</code> | the names of the S3 buckets to create |
+
 <a name="module_S3.multipartCopyObject"></a>
@@ -780,0 +825,0 @@

S3.d.ts

@@ -274,2 +274,9 @@ /**
 export declare const recursivelyDeleteS3Bucket: (bucket: string) => Promise<void>;
+/**
+* Delete a list of buckets and all of their objects from S3
+*
+* @param {Array} buckets - list of bucket names
+* @returns {Promise} the promised result of `S3.deleteBucket`
+**/
+export declare const deleteS3Buckets: (buckets: Array<string>) => Promise<any>;
 declare type FileInfo = {
@@ -379,2 +386,9 @@ filename: string;
 /**
+ * Create multiple S3 buckets
+ *
+ * @param {Array<string>} buckets - the names of the S3 buckets to create
+ * @returns {Promise}
+ */
+export declare const createS3Buckets: (buckets: Array<string>) => Promise<any>;
+/**
  * Copy an S3 object to another location in S3 using a multipart copy
@@ -381,0 +395,0 @@ *

S3.js

@@ -28,3 +28,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.moveObject = exports.multipartCopyObject = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.recursivelyDeleteS3Bucket = exports.deleteS3Files = exports.downloadS3Files = exports.fileExists = exports.getObjectReadStream = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
+exports.moveObject = exports.multipartCopyObject = exports.createS3Buckets = exports.createBucket = exports.getFileBucketAndKey = exports.validateS3ObjectChecksum = exports.calculateObjectHash = exports.listS3ObjectsV2 = exports.listS3Objects = exports.uploadS3FileStream = exports.uploadS3Files = exports.deleteS3Buckets = exports.recursivelyDeleteS3Bucket = exports.deleteS3Files = exports.downloadS3Files = exports.fileExists = exports.getObjectReadStream = exports.putJsonS3Object = exports.getJsonS3Object = exports.getTextObject = exports.getS3Object = exports.waitForObject = exports.getObject = exports.s3PutObjectTagging = exports.s3GetObjectTagging = exports.getObjectSize = exports.downloadS3File = exports.promiseS3Upload = exports.s3CopyObject = exports.putFile = exports.s3PutObject = exports.waitForObjectToExist = exports.s3ObjectExists = exports.headObject = exports.deleteS3Object = exports.s3TagSetToQueryString = exports.buildS3Uri = exports.parseS3Uri = exports.s3Join = void 0;
 const fs_1 = __importDefault(require("fs"));
@@ -449,2 +449,9 @@ const isBoolean_1 = __importDefault(require("lodash/isBoolean"));
 });
+/**
+* Delete a list of buckets and all of their objects from S3
+*
+* @param {Array} buckets - list of bucket names
+* @returns {Promise} the promised result of `S3.deleteBucket`
+**/
+exports.deleteS3Buckets = async (buckets) => Promise.all(buckets.map(exports.recursivelyDeleteS3Bucket));
 exports.uploadS3Files = (files, defaultBucket, keyPath, s3opts = {}) => {
@@ -622,2 +629,9 @@ let i = 0;
 exports.createBucket = (Bucket) => services_1.s3().createBucket({ Bucket }).promise();
+/**
+ * Create multiple S3 buckets
+ *
+ * @param {Array<string>} buckets - the names of the S3 buckets to create
+ * @returns {Promise}
+ */
+exports.createS3Buckets = async (buckets) => Promise.all(buckets.map(exports.createBucket));
 const createMultipartUpload = async (params) => {
@@ -624,0 +638,0 @@ const uploadParams = {

services.d.ts

@@ -19,2 +19,3 @@ import * as AWS from 'aws-sdk';
 export declare const secretsManager: (options?: object | undefined) => AWS.SecretsManager;
+export declare const systemsManager: (options?: object | undefined) => AWS.SSM;
 export declare const kms: (options?: object | undefined) => AWS.KMS;
@@ -21,0 +22,0 @@ export declare const es: (options?: object | undefined) => AWS.ES;

services.js

@@ -22,3 +22,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.sts = exports.es = exports.kms = exports.secretsManager = exports.sns = exports.cf = exports.sfn = exports.dynamodbDocClient = exports.dynamodbstreams = exports.dynamodb = exports.cloudwatch = exports.cloudwatchlogs = exports.cloudwatchevents = exports.sqs = exports.lambda = exports.kinesis = exports.s3 = exports.ec2 = exports.ecs = exports.apigateway = void 0;
+exports.sts = exports.es = exports.kms = exports.systemsManager = exports.secretsManager = exports.sns = exports.cf = exports.sfn = exports.dynamodbDocClient = exports.dynamodbstreams = exports.dynamodb = exports.cloudwatch = exports.cloudwatchlogs = exports.cloudwatchevents = exports.sqs = exports.lambda = exports.kinesis = exports.s3 = exports.ec2 = exports.ecs = exports.apigateway = void 0;
 const AWS = __importStar(require("aws-sdk"));
@@ -43,2 +43,3 @@ const awsClient = require("./client");
 exports.secretsManager = awsClient(AWS.SecretsManager, '2017-10-17');
+exports.systemsManager = awsClient(AWS.SSM, '2017-10-17');
 exports.kms = awsClient(AWS.KMS, '2014-11-01');
@@ -45,0 +46,0 @@ exports.es = awsClient(AWS.ES, '2015-01-01');

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

183449

increased by3.9%

Lines of code

3088

increased by3.49%

Number of lines in readme file

1135

increased by4.13%

Dependency changes

• + Added@cumulus/checksum@9.0.0(transitive)

• + Added@cumulus/errors@9.0.0(transitive)

• + Added@cumulus/logger@9.0.0(transitive)

• + Addedis-typed-array@1.1.15(transitive)

• + Addedmath-intrinsics@1.1.0(transitive)

• + Addedwhich-typed-array@1.1.18(transitive)

• - Removed@cumulus/checksum@8.1.0(transitive)

• - Removed@cumulus/errors@8.1.0(transitive)

• - Removed@cumulus/logger@8.1.0(transitive)

• - Removedis-typed-array@1.1.14(transitive)

• - Removedmath-intrinsics@1.0.0(transitive)

• - Removedwhich-typed-array@1.1.17(transitive)

• Updated@cumulus/checksum@9.0.0

• Updated@cumulus/errors@9.0.0

• Updated@cumulus/logger@9.0.0