@cumulus/common - npm Package Compare versions

Comparing version 1.11.1 to 1.11.2

aws.js

 'use strict';
 
 const AWS = require('aws-sdk');
-const cksum = require('cksum');
 const crypto = require('crypto');
@@ -11,2 +10,3 @@ const fs = require('fs');
 const pMap = require('p-map');
+const pRetry = require('p-retry');
 const pump = require('pump');
@@ -20,2 +20,3 @@ const url = require('url');
 const { noop } = require('./util');
+const { getFileChecksumFromStream } = require('./file');
 
@@ -284,3 +285,16 @@ /**
 
+
 /**
+* Puts object Tagging in S3
+* https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html#putObjectTagging-property
+*
+* @param {string} bucket - name of bucket
+* @param {string} key - key for object (filepath + filename)
+* @param {Object} tagging - tagging object
+* @returns {Promise} - returns response from `S3.getObjectTagging` as a promise
+**/
+exports.s3PutObjectTagging = (bucket, key, tagging) =>
+  exports.s3().putObjectTagging({ Bucket: bucket, Key: key, Tagging: tagging }).promise();
+
+/**
 * Get an object from S3
@@ -357,11 +371,9 @@ *
  */
-exports.deleteS3Files = (s3Objs) => {
-  log.info(`Starting deletion of ${s3Objs.length} object(s)`);
-  return pMap(
-    s3Objs,
-    (s3Obj) => exports.s3().deleteObject(s3Obj).promise(),
-    { concurrency: S3_RATE_LIMIT }
-  );
-};
+exports.deleteS3Files = (s3Objs) => pMap(
+  s3Objs,
+  (s3Obj) => exports.s3().deleteObject(s3Obj).promise(),
+  { concurrency: S3_RATE_LIMIT }
+);
 
+
 /**
@@ -555,6 +567,5 @@ * Delete a bucket and all of its objects from S3
   if (algorithm.toLowerCase() === 'cksum') {
-    return new Promise((resolve, reject) =>
+    return getFileChecksumFromStream(
       exports.s3().getObject(param).createReadStream()
-        .pipe(cksum.stream((value) => resolve(value.readUInt32BE(0))))
-        .on('error', reject));
+    );
   }
@@ -813,2 +824,9 @@
 
+exports.getStateMachineArn = (executionArn) => {
+  if (executionArn) {
+    return executionArn.replace('execution', 'stateMachine').split(':').slice(0, -1).join(':');
+  }
+  return null;
+};
+
 /**
@@ -873,1 +891,23 @@ * Parse event metadata to get location of granule on S3
 };
+
+const retryIfThrottlingException = (err) => {
+  if (exports.isThrottlingException(err)) throw err;
+  throw new pRetry.AbortError(err);
+};
+
+/**
+ * Wrap a function so that it will retry when a ThrottlingException is encountered.
+ *
+ * @param {Function} fn - the function to retry.  This function must return a Promise.
+ * @param {Object} options - retry options, documented here:
+ *   - https://github.com/sindresorhus/p-retry#options
+ *   - https://github.com/tim-kos/node-retry#retryoperationoptions
+ *   - https://github.com/tim-kos/node-retry#retrytimeoutsoptions
+ * @returns {Function} a function that will retry on a ThrottlingException
+ */
+exports.retryOnThrottlingException = (fn, options) =>
+  (...args) =>
+    pRetry(
+      () => fn(...args).catch(retryIfThrottlingException),
+      options
+    );

errors.js

@@ -0,1 +1,3 @@
+const isFunction = require('lodash.isfunction');
+
 /**
@@ -11,3 +13,8 @@ * Creates a new error type with the given name and parent class. Sets up
   function E(message) {
-    Error.captureStackTrace(this, this.constructor);
+    if (isFunction(Error.captureStackTrace)) {
+      Error.captureStackTrace(this, this.constructor);
+    }
+    else {
+      this.stack = (new Error(message)).stack;
+    }
     this.message = message;
@@ -62,2 +69,5 @@ }
 
+  // No CMR metadata file was present.
+  CMRMetaFileNotFound: createErrorType('CMRMetaFileNotFound'),
+
   // The provider info is missing error
@@ -85,4 +95,7 @@ ProviderNotFound: createErrorType('ProviderNotFound'),
 
+  // Error thrown when system encounters a conflicting request.
+  InvalidArgument: createErrorType('InvalidArgument'),
+
   // is raised if the PDR file doesn't match the collection
   MismatchPdrCollection: createErrorType('MismatchPdrCollection')
 };

index.js

 'use strict';
 
 exports.aws = require('./aws');
+exports.BucketsConfig = require('./BucketsConfig');
 exports.cliUtils = require('./cli-utils');
@@ -8,2 +9,3 @@ exports.CloudFormationGateway = require('./CloudFormationGateway');
 exports.constructCollectionId = require('./collection-config-store').constructCollectionId;
+exports.file = require('./file');
 exports.http = require('./http');
@@ -16,1 +18,3 @@ exports.log = require('./log');
 exports.keyPairProvider = require('./key-pair-provider');
+exports.concurrency = require('./concurrency');
+exports.errors = require('./errors');

key-pair-provider.js

@@ -16,3 +16,3 @@ /**
   /**
-   * Encrypt the given string using the given public key stored in the internal bucket
+   * Encrypt the given string using the given public key stored in the system_bucket.
    *
@@ -22,3 +22,3 @@ * @param {string} str - The string to encrypt
    * @param {string} bucket - the optional bucket name. if not provided will
-   *                          use env variable "internal"
+   *                          use env variable "system_bucket"
    * @param {stack} stack - the optional stack name. if not provided will
@@ -31,3 +31,3 @@ *                        use env variable "stackName"
     const pki = forge.pki;
-    const b = bucket || process.env.internal;
+    const b = bucket || process.env.system_bucket;
     const s = stack || process.env.stackName;
@@ -48,3 +48,3 @@ const pub = await s3().getObject({
    * @param {string} bucket - the optional bucket name. Defaults to the value of
-   *   the "internal" environment variable
+   *   the "system_bucket" environment variable
    * @param {string} stack - the optional stack name. Defaults to the value of
@@ -56,3 +56,3 @@ *   the "stackName" environment variable
     const pki = forge.pki;
-    const b = bucket || process.env.internal;
+    const b = bucket || process.env.system_bucket;
     const s = stack || process.env.stackName;
@@ -59,0 +59,0 @@ const priv = await s3().getObject({

package.json

 {
   "name": "@cumulus/common",
-  "version": "1.11.1",
+  "version": "1.11.2",
   "description": "Common utilities used across tasks",
@@ -43,4 +43,4 @@ "keywords": [
   "dependencies": {
-    "@cumulus/logger": "^1.11.1",
-    "@cumulus/test-data": "^1.11.1",
+    "@cumulus/logger": "^1.11.2",
+    "@cumulus/test-data": "^1.11.2",
     "ajv": "^5.2.2",
@@ -59,5 +59,7 @@ "async": "^2.0.0",
     "lodash.compact": "^3.0.1",
+    "lodash.isfunction": "^3.0.9",
     "lodash.isnumber": "^3.0.3",
     "lodash.isobject": "^3.0.2",
     "lodash.isstring": "^4.0.1",
+    "lodash.isundefined": "^3.0.1",
     "lodash.kebabcase": "^4.1.1",
@@ -78,5 +80,6 @@ "lodash.range": "^3.2.0",
     "nock": "^10.0.0",
-    "nyc": "^11.6.0",
+    "nyc": "^13.3.0",
     "sinon": "^7.1.1"
-  }
+  },
+  "gitHead": "383ca0f32d381a25b69a1ddef5c757c2f0c15ab6"
 }

step-functions.js

 'use strict';
 
-const pRetry = require('p-retry');
+// This entire module is deprecated.  Nothing in it is being used, and it should be removed in a
+// future release.  Any StepFunction-related functions should be found in StepFunctions.js.
+
 const uuidv4 = require('uuid/v4');
 
-const { sleep } = require('./util');
+const { deprecate, sleep } = require('./util');
 
-const {
-  isThrottlingException,
-  sfn,
-  toSfnExecutionName
-} = require('./aws');
+const { toSfnExecutionName } = require('./aws');
+const StepFunctions = require('./StepFunctions');
 
-const log = require('./log');
+deprecate('@cumulus/common/step-functions', '1.11.1', '@cumulus/common/StepFunctions');
 
@@ -48,11 +47,2 @@ /**
 
-const logSfnThrottlingException = (fn) => {
-  log.debug(`ThrottlingException in stepfunctions.${fn}(), will retry`);
-};
-
-const retryOnThrottlingException = (err) => {
-  if (isThrottlingException(err)) throw err;
-  throw new pRetry.AbortError(err);
-};
-
 /**
@@ -65,20 +55,6 @@ * Describe a Step Function Execution
  * @param {string} executionArn - ARN of the execution
- * @param {Object} [retryOptions] - see the options described [here](https://github.com/tim-kos/node-retry#retrytimeoutsoptions)
  * @returns {Promise<Object>} https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/StepFunctions.html#describeExecution-property
  */
-exports.describeExecution = (executionArn, retryOptions) => {
-  const fullRetryOptions = Object.assign(
-    {
-      onFailedAttempt: () => logSfnThrottlingException('describeExecution')
-    },
-    retryOptions
-  );
+exports.describeExecution = (executionArn) => StepFunctions.describeExecution({ executionArn });
 
-  return pRetry(
-    () => sfn().describeExecution({ executionArn }).promise()
-      .catch(retryOnThrottlingException),
-    fullRetryOptions
-  );
-};
-
 /**
@@ -90,12 +66,3 @@ * Test if a Step Function Execution exists
  */
-exports.executionExists = async (executionArn) => {
-  try {
-    await exports.describeExecution({ executionArn }).promise();
-    return true;
-  }
-  catch (err) {
-    if (err.code === 'ExecutionDoesNotExist') return false;
-    throw err;
-  }
-};
+exports.executionExists = StepFunctions.executionExists;
 
@@ -123,3 +90,3 @@ /**
   do {
-    if (await exports.executionExists(executionArn)) return;
+    if (await StepFunctions.executionExists(executionArn)) return;
     await sleep(intervalInMs);
@@ -126,0 +93,0 @@ } while (Date.now() < failAfter);

test-utils.js

@@ -23,9 +23,22 @@ /* eslint no-console: "off" */
 /**
- * Generate a 40-character random string
+ * Generate a [40 character] random string
  *
+ * @param {number} numBytes - number of bytes to use in creating a random string
+ *                 defaults to 20 to produce a 40 character string
  * @returns {string} - a random string
  */
-exports.randomString = () => crypto.randomBytes(20).toString('hex');
+exports.randomString = (numBytes = 20) => crypto.randomBytes(numBytes).toString('hex');
 
+
 /**
+ * Postpend a [10-character] random string to input identifier.
+ *
+ * @param {string} id - identifer to return
+ * @param {number} numBytes - number of bytes to use to compute random
+ *                 extension. Default 5 to produce 10 characters..
+ * @returns {string} - a random string
+ */
+exports.randomId = (id, numBytes = 5) => `${id}${exports.randomString(numBytes)}`;
+
+/**
  * Create a random granule id from the regular expression
@@ -40,2 +53,3 @@ *
 const localStackPorts = {
+  stepfunctions: 10000, // add a fake port to support test overrides
   apigateway: 4567,
@@ -241,1 +255,28 @@ cloudformation: 4581,
 exports.jlog = jlog;
+
+const throwThrottlingException = () => {
+  const throttlingException = new Error('ThrottlingException');
+  throttlingException.code = 'ThrottlingException';
+
+  throw throttlingException;
+};
+
+/**
+ * Return a function that throws a ThrottlingException the first time it is called, then returns as
+ * normal any other times.
+ *
+ * @param {Function} fn
+ * @returns {Function}
+ */
+exports.throttleOnce = (fn) => {
+  let throttleNextCall = true;
+
+  return (...args) => {
+    if (throttleNextCall) {
+      throttleNextCall = false;
+      throwThrottlingException();
+    }
+
+    return fn(...args);
+  };
+};

util.js

@@ -60,1 +60,18 @@ 'use strict';
 exports.noop = () => {}; // eslint-disable-line lodash/prefer-noop
+
+/**
+ * Replacement for lodash.omit returns a shallow copy of input object
+ * with keys removed.
+ * (lodash.omit will be removed in v5.0.0)
+ * https://github.com/lodash/lodash/wiki/Roadmap#v500-2019
+ *
+ * @param {Object} objectIn - input object
+ * @param {(string|string[])} keys - key or list of keys to remove from object
+ * @returns {Object} copy of objectIn without keys attached.
+ */
+exports.omit = (objectIn, keys) => {
+  const keysToRemove = [].concat(keys);
+  const objectOut = { ...objectIn };
+  keysToRemove.forEach((key) => delete objectOut[key]);
+  return objectOut;
+};

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

124438

increased by15.65%

Number of package files

32

increased by14.29%

Lines of code

2867

increased by7.02%

Worsened metrics

Dependency count

31

increased by6.9%

Dependency changes

• + Addedlodash.isfunction@^3.0.9

• + Addedlodash.isundefined@^3.0.1

• + Addedlodash.isfunction@3.0.9(transitive)

• + Addedlodash.isundefined@3.0.1(transitive)

• Updated@cumulus/logger@^1.11.2

• Updated@cumulus/test-data@^1.11.2