google-cloud-bucket - npm Package Compare versions

Comparing version 0.4.6 to 0.5.1

CHANGELOG.md

@@ -5,2 +5,17 @@ # Change Log
 
+<a name="0.5.1"></a>
+## [0.5.1](https://github.com/nicolasdao/google-cloud-bucket/compare/v0.5.0...v0.5.1) (2018-12-10)
+
+
+
+<a name="0.5.0"></a>
+# [0.5.0](https://github.com/nicolasdao/google-cloud-bucket/compare/v0.4.6...v0.5.0) (2018-12-10)
+
+
+### Features
+
+* Add support for configuring the retry timout ([6fcf1c4](https://github.com/nicolasdao/google-cloud-bucket/commit/6fcf1c4))
+
+
+
 <a name="0.4.6"></a>
@@ -7,0 +22,0 @@ ## [0.4.6](https://github.com/nicolasdao/google-cloud-bucket/compare/v0.4.5...v0.4.6) (2018-12-09)

index.js

@@ -24,3 +24,3 @@ /**
 	() => true, 
-	{ ignoreFailure: true, retryInterval: [200, 2000], retryAttempts: 10 })
+	{ ignoreFailure: true, retryInterval: [500, 2000], timeOut: options.timeout || 10000 })
 	.catch(e => {
@@ -33,2 +33,10 @@ if (options.retryCatch)
 
+const _throwHttpErrorIfBadStatus = res => Promise.resolve(null).then(() => {
+	if (res && res.status && res.status >= 400) {
+		const errorMsg = `Failed with error ${res.status}.${res.data ? ` Details:\n${JSON.stringify(res.data, null, ' ')}` : ''}`
+		throw new Error(errorMsg)
+	}
+	return res 
+}) 
+
 const _getBucketAndPathname = (filePath, options={}) => {
@@ -58,6 +66,10 @@ if (!filePath)
 
-	const putObject = (object, filePath, options) => getToken(auth).then(token => gcp.insert(object, filePath, token, options)).then(({ data }) => data)
-	const getObject = (bucket, filePath, options) => getToken(auth).then(token => gcp.get(bucket, filePath, token, options)).then(({ data }) => data)
+	const putObject = (object, filePath, options) => getToken(auth).then(token => _retryFn(() => gcp.insert(object, filePath, token, options), options))
+		.then(_throwHttpErrorIfBadStatus)
+		.then(({ data }) => data)
+	const getObject = (bucket, filePath, options) => getToken(auth).then(token => _retryFn(() => gcp.get(bucket, filePath, token, options), options))
+		.then(_throwHttpErrorIfBadStatus)
+		.then(({ data }) => data)
 	
-	const objectExists = (bucket, filePath) => getToken(auth).then(token => _retryFn(() => gcp.doesFileExist(bucket, filePath, token)).then(({ data }) => data))
+	const objectExists = (bucket, filePath, options={}) => getToken(auth).then(token => _retryFn(() => gcp.doesFileExist(bucket, filePath, token), options).then(({ data }) => data))
 	const getBucket = (bucket) => getToken(auth).then(token => _retryFn(() => gcp.config.get(bucket, token)).then(({ data }) => data))
@@ -81,3 +93,3 @@
 
-	const retryPutObject = (object, filePath, options={}) => _retryFn(() => putObject(object, filePath, options), options)
+	const retryPutObject = (object, filePath, options={}) => putObject(object, filePath, options)
 		.then(data => {
@@ -97,3 +109,3 @@ if (data)
 		const { bucket, file } = _getBucketAndPathname(filePath)
-		return _retryFn(() => getObject(bucket, file, options), options) 
+		return getObject(bucket, file, options)
 	})
@@ -106,3 +118,3 @@
 		removePublicAccess,
-		exists: (filepath) => Promise.resolve(null).then(() => {
+		exists: (filepath, options={}) => Promise.resolve(null).then(() => {
 			if(!filepath)
@@ -112,3 +124,3 @@ throw new Error('Missing required \'filepath\' argument')
 			const { bucket, file } = _getBucketAndPathname(filepath, { ignoreMissingFile: true })
-			return objectExists(bucket, file)
+			return objectExists(bucket, file, options)
 		}),
@@ -131,3 +143,3 @@ config: (bucket) => {
 				'get': () => getBucket(bucketName),
-				exists: () => objectExists(bucketName),
+				exists: (options={}) => objectExists(bucketName, null, options),
 				create: (options={}) => createBucket(bucketName, options),
@@ -150,3 +162,3 @@ delete: () => deleteBucket(bucketName),
 						file: filePath,
-						exists: () => objectExists(bucketName, filePath),
+						exists: (options={}) => objectExists(bucketName, filePath, options),
 						'get': (options={}) => retryGetObject(posix.join(bucketName, filePath), options),
@@ -153,0 +165,0 @@ insert: (object, options={}) => retryPutObject(object, posix.join(bucketName, filePath), options),

package.json

 {
   "name": "google-cloud-bucket",
-  "version": "0.4.6",
+  "version": "0.5.1",
   "description": "Nodejs package to add objects to a Google Cloud Bucket.",
@@ -5,0 +5,0 @@ "main": "index.js",

README.md

@@ -8,2 +8,3 @@ # Google Cloud Bucket ·  [![NPM](https://img.shields.io/npm/v/google-cloud-bucket.svg?style=flat)](https://www.npmjs.com/package/google-cloud-bucket) [![Tests](https://travis-ci.org/nicolasdao/google-cloud-bucket.svg?branch=master)](https://travis-ci.org/nicolasdao/google-cloud-bucket) [![License](https://img.shields.io/badge/License-BSD%203--Clause-blue.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Neap](https://neap.co/img/made_by_neap.svg)](#this-is-what-we-re-up-to)
 > * [How To Use It](#how-to-use-it) 
+> * [Extra Precautions To Making Robust Queries](#extra-precautions-to-making-robust-queries)
 > * [Annex](#annex) 
@@ -40,3 +41,2 @@ >    * [List Of All Google Cloud Platform Locations](#list-of-all-google-cloud-platform-locations)
 ## Show Me The Code
-
 ### Basics
@@ -229,2 +229,18 @@
 
+## Extra Precautions To Making Robust Queries
+### Avoiding Network Errors
+
+Networks errors (e.g. socket hang up, connect ECONNREFUSED) are a fact of life. To deal with those undeterministic errors, this library uses a simple exponential back off retry strategy, which will reprocess your read or write request for 10 seconds by default. You can increase that retry period as follow:
+
+```js
+// Retry timeout for CHECKING FILE EXISTS
+storage.exists('your-bucket/a-path/image.jpg', { timeout: 30000 }) // 30 seconds retry period timeout
+
+// Retry timeout for INSERTS
+storage.insert(someObject, 'your-bucket/a-path/filename.json', { timeout: 30000 }) // 30 seconds retry period timeout
+
+// Retry timeout for QUERIES
+storage.get('your-bucket/a-path/filename.json', { timeout: 30000 }) // 30 seconds retry period timeout
+```
+
 # Annex
@@ -231,0 +247,0 @@ ## List Of All Google Cloud Platform Locations

src/utils/core.js

@@ -205,2 +205,8 @@ /**
 
+const getRandomNumber = (start, end) => {
+	const size = end == undefined ? start : (end - start)
+	const offset = end == undefined ? 0 : start
+	return offset + Math.floor(Math.random() * size)
+}
+
 module.exports = {
@@ -226,3 +232,6 @@ identity: {
 		arrayAreDiff: arrayObjAreDiff
+	},
+	math: {
+		randomNumber: getRandomNumber
 	}
 }

src/utils/promise.js

@@ -9,3 +9,3 @@ /**
 
-const { obj: { merge } } = require('./core')
+const { obj: { merge }, math } = require('./core')
 const { arities } = require('./functional')
@@ -45,9 +45,14 @@
  * @param  {Function} fn        				[description]
- * @param  {Function} successFn 				Returns a boolean that determines whether a response is valid or not. 
- *                                  			Can be a normal function or a promise
- * @param  {Function} failureFn 				(Optional) Returns a boolean that determines whether an exception can be ignored or not. 
- *                                  			Can be a normal function or a promise                           			
+ * @param  {Function} successFn 				(res, options) => Returns a promise or a value. The value is a boolean or an object that determines 
+ *                                  			whether a response is valid or not. If the value is an object, that object might contain
+ *                                  			a 'retryInterval' which overrides the optional value. 
+ * @param  {Function} failureFn 				(Optional) (error, options) => Returns a promise or a value. The value is a boolean or an object that determines 
+ *                                  			whether a response is valid or not. If the value is an object, that object might contain
+ *                                  			a 'retryInterval' which overrides the optional value.                          			
  * @param  {Number}   options.retryAttempts   	default: 5. Number of retry
- * @param  {Number}   options.retryInterval   	default: 5000. Time interval in milliseconds between each retry
  * @param  {Number}   options.attemptsCount   	Current retry count. When that counter reaches the 'retryAttempts', the function stops.
+ * @param  {Number}   options.timeOut   		If specified, 'retryAttempts' and 'attemptsCount' are ignored
+ * @param  {Number}   options.retryInterval   	default: 5000. Time interval in milliseconds between each retry. It can also be a 2 items array.
+ *                                             	In that case, the retryInterval is a random number between the 2 ranges (e.g., [10, 100] => 54).
+ *                                             	The retry strategy increases the 'retryInterval' by a factor 1.5 after each failed attempt.
  * @param  {Boolean}  options.ignoreError   	In case of constant failure to pass the 'successFn' test, this function will either throw an error
@@ -62,33 +67,63 @@ *                                           	or return the current result without throwing an error if this flag is set to true.
 	'function fn, function successFn, function failureFn, object options={}',
-	({ fn, successFn, failureFn, options={} }) => Promise.resolve(null)
-		.then(() => fn()).then(data => ({ error: null, data }))
-		.catch(error => { 
-			if (options.ignoreFailure && !failureFn)
-				failureFn = () => true
-			return { error, data: null }
-		})
-		.then(({ error, data }) => Promise.resolve(null).then(() => {
-			if (error && failureFn)
-				return failureFn(error)
-			else if (error)
-				throw error 
-			else
-				return successFn(data)
-		})
-			.then(passed => {
-				if (!error && passed)
-					return data
-				else if ((!error && !passed) || (error && passed)) {
-					const { retryAttempts=5, retryInterval=5000, attemptsCount=0 } = options
-					if (attemptsCount < retryAttempts)
-						return delay(retryInterval).then(() => retry(fn, successFn, merge(options, { attemptsCount:attemptsCount+1 })))
-					else if (options.ignoreError)
+	({ fn, successFn, failureFn, options={} }) => { 
+		const start = Date.now()
+		return Promise.resolve(null)
+			.then(() => fn()).then(data => ({ error: null, data }))
+			.catch(error => { 
+				if (options.ignoreFailure && !failureFn)
+					failureFn = () => true
+				return { error, data: null }
+			})
+			.then(({ error, data }) => Promise.resolve(null)
+				.then(() => {
+					if (error && failureFn)
+						return failureFn(error, options)
+					else if (error)
+						throw error 
+					else
+						return successFn(data, options)
+				})
+				.then(passed => {
+					if (!error && passed)
 						return data
-					else 
-						throw new Error(options.errorMsg ? options.errorMsg : `${retryAttempts} attempts to retry the procedure failed to pass the test`)
-				} else 
-					throw error
-			})))
+					else if ((!error && !passed) || (error && passed)) {
+						let { retryAttempts=5, retryInterval=5000, attemptsCount=0, timeOut=null, startTime=null } = options
+						const delayFactor = (attemptsCount+1) <= 1 ? 1 : Math.pow(1.5, attemptsCount)
+						if (timeOut > 0) {
+							startTime = startTime || start
+							if (Date.now() - startTime < timeOut) {
+								const explicitRetryInterval = passed && passed.retryInterval > 0 ? passed.retryInterval : null
+								const i = (!explicitRetryInterval && Array.isArray(retryInterval) && retryInterval.length > 1)
+									? (() => {
+										if (typeof(retryInterval[0]) != 'number' || typeof(retryInterval[1]) != 'number')
+											throw new Error(`Wrong argument exception. When 'options.retryInterval' is an array, all elements must be numbers. Current: [${retryInterval.join(', ')}].`)
+										if (retryInterval[0] > retryInterval[1])
+											throw new Error(`Wrong argument exception. When 'options.retryInterval' is an array, the first element must be strictly greater than the second. Current: [${retryInterval.join(', ')}].`)
 
+										return math.randomNumber(retryInterval[0], retryInterval[1])
+									})()
+									: (explicitRetryInterval || retryInterval)
 
+								const delayMs = Math.round(delayFactor*i)
+
+								return delay(delayMs).then(() => failureFn 
+									? retry(fn, successFn, failureFn, merge(options, { startTime, attemptsCount:attemptsCount+1 }))
+									: retry(fn, successFn, merge(options, { startTime, attemptsCount:attemptsCount+1 })))
+							} else
+								throw new Error('timeout')
+						} else if (attemptsCount < retryAttempts) {
+							const delayMs = Math.round(delayFactor*retryInterval)
+							return delay(delayMs).then(() => failureFn
+								? retry(fn, successFn, failureFn, merge(options, { attemptsCount:attemptsCount+1 }))
+								: retry(fn, successFn, merge(options, { attemptsCount:attemptsCount+1 })))
+						} else if (options.ignoreError)
+							return data
+						else 
+							throw new Error(options.errorMsg ? options.errorMsg : `${retryAttempts} attempts to retry the procedure failed to pass the test`)
+					} else 
+						throw error
+				}))
+	})
+
+
 module.exports = {
@@ -95,0 +130,0 @@ delay,

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

21103260

increased by31386.59%

Number of package files

23

increased by43.75%

Lines of code

1252

increased by10.89%

Number of lines in readme file

333

increased by5.05%

Worsened metrics

Number of low supply chain risk alerts

3

increased by50%

No dependency changes