@cumulus/common - npm Package Compare versions

Comparing version 1.5.4 to 1.5.5

aws.js

@@ -1,2 +0,1 @@
-/* eslint-disable no-param-reassign */
 'use strict';
@@ -15,2 +14,24 @@
 
+/**
+ * Join strings into an S3 key without a leading slash or double slashes
+ *
+ * @param {Array<string>} tokens - the strings to join
+ * @returns {string} the full S3 key
+ */
+function s3Join(tokens) {
+  const removeLeadingSlash = (token) => token.replace(/^\//, '');
+  const removeTrailingSlash = (token) => token.replace(/\/$/, '');
+  const isNotEmptyString = (token) => token.length > 0;
+
+  const key = tokens
+    .map(removeLeadingSlash)
+    .map(removeTrailingSlash)
+    .filter(isNotEmptyString)
+    .join('/');
+
+  if (tokens[tokens.length - 1].endsWith('/')) return `${key}/`;
+  return key;
+}
+exports.s3Join = s3Join;
+
 const region = exports.region = process.env.AWS_DEFAULT_REGION || 'us-east-1';
@@ -357,15 +378,24 @@ if (region) {
  * @param {Object} params - params for the s3.listObjectsV2 call
- * @returns {Promise.<Array>} - resolves to an array of objects corresponding to
+ * @returns {Promise<Array>} - resolves to an array of objects corresponding to
  *   the Contents property of the listObjectsV2 response
  */
 async function listS3ObjectsV2(params) {
-  const data = await exports.s3().listObjectsV2(params).promise();
+  // Fetch the first list of objects from S3
+  let listObjectsResponse = await exports.s3().listObjectsV2(params).promise();
+  let discoveredObjects = listObjectsResponse.Contents;
 
-  if (data.IsTruncated) {
-    const newParams = Object.assign({}, params);
-    newParams.ContinuationToken = data.NextContinuationToken;
-    return data.Contents.concat(await exports.listS3ObjectsV2(newParams));
+  // Keep listing more objects from S3 until we have all of them
+  while (listObjectsResponse.IsTruncated) {
+    listObjectsResponse = await exports.s3().listObjectsV2( // eslint-disable-line no-await-in-loop, function-paren-newline, max-len
+      // Update the params with a Continuation Token
+      Object.assign(
+        {},
+        params,
+        { ContinuationToken: listObjectsResponse.NextContinuationToken }
+      )
+    ).promise(); //eslint-disable-line function-paren-newline
+    discoveredObjects = discoveredObjects.concat(listObjectsResponse.Contents);
   }
 
-  return data.Contents;
+  return discoveredObjects;
 }
@@ -372,0 +402,0 @@ exports.listS3ObjectsV2 = listS3ObjectsV2;

package.json

 {
   "name": "@cumulus/common",
-  "version": "1.5.4",
+  "version": "1.5.5",
   "description": "Common utilities used across tasks",
@@ -43,3 +43,3 @@ "keywords": [
   "dependencies": {
-    "@cumulus/test-data": "^1.5.4",
+    "@cumulus/test-data": "^1.5.5",
     "ajv": "^5.2.2",
@@ -46,0 +46,0 @@ "async": "^2.0.0",

tests/aws.js

@@ -10,2 +10,16 @@ 'use strict';
 
+test('s3Join behaves as expected', (t) => {
+  t.is(aws.s3Join(['a', 'b', 'c']), 'a/b/c');
+
+  t.is(aws.s3Join(['a', 'b']), 'a/b');
+  t.is(aws.s3Join(['a', 'b/']), 'a/b/');
+  t.is(aws.s3Join(['a/', 'b']), 'a/b');
+  t.is(aws.s3Join(['/a', 'b']), 'a/b');
+  t.is(aws.s3Join(['a/', 'b']), 'a/b');
+
+  t.is(aws.s3Join(['a']), 'a');
+  t.is(aws.s3Join(['/a']), 'a');
+  t.is(aws.s3Join(['a/']), 'a/');
+});
+
 test('listS3ObjectsV2 handles non-truncated case', async (t) => {
@@ -12,0 +26,0 @@ const Bucket = randomString();

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Lines of code

2666

increased by1.45%

Worsened metrics

Total package byte prevSize

108510

decreased by-54.12%

Number of package files

30

decreased by-3.23%

Dependency changes

• + Addedis-typed-array@1.1.13(transitive)

• - Removedis-typed-array@1.1.14(transitive)

• Updated@cumulus/test-data@^1.5.5