documentdb - npm Package Compare versions

Comparing version 1.8.0 to 1.9.0

changelog.md

@@ -0,1 +1,11 @@
+## Changes in 1.9.0 : ##
+
+- Added the retry policy support for throttle requests(Request Rate too large - Error code 429). By default, we now retry 9 times for each request when we get a 429, honoring the retryAfter time in the response header.
+  We also allow setting a fixed retry interval time as part of the RetryOptions property on the ConnectionPolicy object if you want to ignore the retryAfter time returned by server between the retries. We will also 
+  now wait for max 30 sec for each request that is being throttled(irrespective of retry count) and will return the request with a 429. This time can also be overriden in the RetryOptions property on ConnectionPolicy object.
+
+- We also return x-ms-throttle-retry-count and x-ms-throttle-retry-wait-time-ms as the response headers in every request to denote the throttle retry count and the cummulative time the request waited between the retries.
+
+- Introducing a RetryOptions class and exposing the RetryOptions property on ConnectionPolicy class that can be used to override some of the default retry options we set.
+
 ## Changes in 1.8.0 : ##
@@ -2,0 +12,0 @@

lib/auth.js

@@ -67,3 +67,3 @@ /*
             } else {
-                var pathParts = path.split("/");
+				var pathParts = path && path.split("/") || [];
                 var resourceTypes = ["dbs", "colls", "docs", "sprocs", "udfs", "triggers", "users", "permissions", "attachments", "media", "conflicts", "offers"];
@@ -70,0 +70,0 @@ // Get the last resource id from the path and get it's token from resourceTokens

lib/constants.js

@@ -161,5 +161,9 @@ /*
     
+    // Client generated retry count response header
+    ThrottleRetryCount: "x-ms-throttle-retry-count",
+    ThrottleRetryWaitTimeInMs: "x-ms-throttle-retry-wait-time-ms",
+    
     CurrentVersion: "2016-05-30",
     
-    UserAgent: "documentdb-nodejs-sdk-1.8.0",
+    UserAgent: "documentdb-nodejs-sdk-1.9.0",
     
@@ -166,0 +170,0 @@ DefaultPrecisions: {

lib/documents.js

@@ -26,3 +26,4 @@ /*
 
-var Base = require("./base");
+var Base = require("./base"),
+    RetryOptions = require("./retryOptions");
 //SCRIPT START
@@ -269,6 +270,9 @@
          * @global
-         * Represents the Connection policy assocated with a DocumentClient.
-         * @property {string} MediaReadMode         -  Attachment content (aka media) download mode. Should be one of the values of {@link MediaReadMode}
-         * @property {number} MediaRequestTimeout    - Time to wait for response from network peer for attachment content (aka media) operations. Represented in milliseconds.
-         * @property {number} RequestTimeout         - Request timeout (time to wait for response from network peer). Represented in milliseconds.
+         * Represents the Connection policy associated with a DocumentClient.
+         * @property {string} MediaReadMode                - Attachment content (aka media) download mode. Should be one of the values of {@link MediaReadMode}
+         * @property {number} MediaRequestTimeout          - Time to wait for response from network peer for attachment content (aka media) operations. Represented in milliseconds.
+         * @property {number} RequestTimeout               - Request timeout (time to wait for response from network peer). Represented in milliseconds.
+         * @property {bool} EnableEndpointDiscovery        - Flag to enable/disable automatic redirecting of requests based on read/write operations.
+         * @property {Array} PreferredLocations            - List of azure regions to be used as preferred locations for read requests.
+         * @property {RetryOptions} RetryOptions           - RetryOptions instance which defines several configurable properties used during retry.
         */
@@ -297,2 +301,3 @@ ConnectionPolicy: Base.defineClass(function() {
             this.PreferredLocations = [];
+            this.RetryOptions = new RetryOptions();
         })
@@ -299,0 +304,0 @@ }

lib/endpointDiscoveryRetryPolicy.js

@@ -31,6 +31,6 @@ /*
      * This class implements the retry policy for endpoint discovery.
+     * @property {int} _maxRetryAttemptCount                           - Max number of retry attempts to perform.
+     * @property {int} currentRetryAttemptCount                        - Current retry attempt count.
      * @property {object} globalEndpointManager                        - The GlobalEndpointManager instance.
-     * @property {int} _maxRetryAttemptCount                           - Max number of retry attempts to perform.
-     * @property {int} _currentRetryAttemptCount                       - Current retry attempt count.
-     * @property {int} _retryAfterInMilliseconds                       - Retry interval in milliseconds.
+     * @property {int} retryAfterInMilliseconds                        - Retry interval in milliseconds.
 */
@@ -43,62 +43,28 @@ var EndpointDiscoveryRetryPolicy = Base.defineClass(
     function (globalEndpointManager) {
+        this._maxRetryAttemptCount = EndpointDiscoveryRetryPolicy.maxRetryAttemptCount;
+        this.currentRetryAttemptCount = 0;
         this.globalEndpointManager = globalEndpointManager;
-        this._maxRetryAttemptCount = EndpointDiscoveryRetryPolicy.maxRetryAttemptCount;
-        this._currentRetryAttemptCount = 0;
-        this._retryAfterInMilliseconds = EndpointDiscoveryRetryPolicy.retryAfterInMilliseconds;
+        this.retryAfterInMilliseconds = EndpointDiscoveryRetryPolicy.retryAfterInMilliseconds;
     }, 
     {
         /**
-         * Applies the retry policy for the created request object.
-         * @param {object} body - a dictionary containing 'buffer' and 'stream' keys to hold corresponding buffer or stream body, null otherwise.
-         * @param {function} createRequestObjectFunc - function that creates the request object.
-         * @param {object} connectionPolicy - an instance of ConnectionPolicy that has the connection configs.
-         * @param {RequestOptions} requestOptions - The request options.
-         * @param {function} callback - the callback that will be called when the response is retrieved and processed.
+         * Determines whether the request should be retried or not.
+         * @param {object} err - Error returned by the request.
+         * @param {function} callback - The callback function which takes bool argument which specifies whether the request will be retried or not.
         */
-        apply: function (body, createRequestObjectFunc, connectionPolicy, requestOptions, callback) {
-            var that = this;
-            var httpsRequest = createRequestObjectFunc(connectionPolicy, requestOptions, function (err, response, headers) {
-                // Check if it 's a write-forbidden exception, which has StatusCode=403 and SubStatus=3 and whether EnableEndpointDiscovery is set to True
-                if (err) {
-                    if (that._currentRetryAttemptCount < that._maxRetryAttemptCount && err.code === 403 && err.substatus === 3 && that.globalEndpointManager.enableEndpointDiscovery) {
-                        that._currentRetryAttemptCount++;
-                        console.log("Write region was changed, refreshing the regions list from database account and will retry the request.");
-                        that.globalEndpointManager.refreshEndpointList(function (writeEndpoint, readEndpoint) {
-                            that.globalEndpointManager.setWriteEndpoint(writeEndpoint);
-                            that.globalEndpointManager.setReadEndpoint(readEndpoint);
-                            
-                            setTimeout(function () {
-                                that.apply(body, createRequestObjectFunc, connectionPolicy, requestOptions, callback);
-                            }, that._retryAfterInMilliseconds);
-                        });
-                    }
-                    else {
-                        console.log("Operation will NOT be retried or has maxed out the retry count.", err);
-                        // This is a test hook to call a callback after the retry counts have been exhausted
-                        if (EndpointDiscoveryRetryPolicy.retryFinishCallback) {
-                            EndpointDiscoveryRetryPolicy.retryFinishCallback(that._currentRetryAttemptCount, that._maxRetryAttemptCount, function () {
-                                callback(err, response, headers);
-                                return;
-                            });
-                        } else {
-                            callback(err, response, headers);
-                            return;
-                        }
-                    }
-                } else {
-                    callback(undefined, response, headers);
+        shouldRetry: function (err, callback) {
+            if (err) {
+                if (this.currentRetryAttemptCount < this._maxRetryAttemptCount && this.globalEndpointManager.enableEndpointDiscovery) {
+                    this.currentRetryAttemptCount++;
+                    console.log("Write region was changed, refreshing the regions list from database account and will retry the request.");
+                    var that = this;
+                    this.globalEndpointManager.refreshEndpointList(function (writeEndpoint, readEndpoint) {
+                        that.globalEndpointManager.setWriteEndpoint(writeEndpoint);
+                        that.globalEndpointManager.setReadEndpoint(readEndpoint);
+                        callback(true);
+                    });
                     return;
                 }
-            });
-            
-            if (httpsRequest) {
-                if (body["stream"] !== null) {
-                    body["stream"].pipe(httpsRequest);
-                } else if (body["buffer"] !== null) {
-                    httpsRequest.write(body["buffer"]);
-                    httpsRequest.end();
-                } else {
-                    httpsRequest.end();
-                }
             }
+            return callback(false);
         }
@@ -109,3 +75,4 @@ },
         retryAfterInMilliseconds : 1000,
-        retryFinishCallback: undefined
+        FORBIDDEN_STATUS_CODE : 403,
+        WRITE_FORBIDDEN_SUB_STATUS_CODE : 3
     }
@@ -112,0 +79,0 @@ );

lib/globalEndpointManager.js

@@ -37,3 +37,3 @@ /*
      * @property {bool} enableEndpointDiscovery        - Flag to enable/disable automatic redirecting of requests based on read/write operations.
-     * @property {Array} preferredLocations            - List of azure regions to be used as preferred locations for any requests.
+     * @property {Array} preferredLocations            - List of azure regions to be used as preferred locations for read requests.
      * @property {bool} isEndpointCacheInitialized     - Flag to determine whether the endpoint cache is initialized or not.
@@ -40,0 +40,0 @@ */

lib/index.js

@@ -35,2 +35,3 @@ /*
     exports.Constants = Client.Constants;
+    exports.RetryOptions = Client.RetryOptions;
     exports.Range = Range.Range;
@@ -37,0 +38,0 @@ exports.RangePartitionResolver = Range.RangePartitionResolver;

lib/request.js

@@ -59,4 +59,3 @@ /*
         if (isMedia && connectionPolicy.MediaReadMode === Documents.MediaReadMode.Streamed) {
-            callback(undefined, response, response.headers);
-            return;
+            return callback(undefined, response, response.headers);
         }
@@ -70,10 +69,4 @@
             if (response.statusCode >= 400) {
-                if (Constants.HttpHeaders.SubStatus in response.headers) {
-                    var subStatus = parseInt(response.headers[Constants.HttpHeaders.SubStatus]);
-                    callback({ code: response.statusCode, substatus: subStatus, body: data }, undefined, response.headers);
-                } else {
-                    callback({ code: response.statusCode, body: data }, undefined, response.headers);
+                return callback(getErrorBody(response, data), undefined, response.headers);
                 }
-                return;
-            }
 
@@ -88,4 +81,3 @@ var result;
             } catch (exception) {
-                callback(exception);
-                return;
+                return callback(exception);
             }
@@ -107,3 +99,3 @@
         httpsRequest.once("response", function () {
-          socket.removeListener("timeout", onTimeout);
+            socket.removeListener("timeout", onTimeout);
         });
@@ -116,2 +108,21 @@ });
 
+/**
+*  Constructs the error body from the response and the data returned from the request.
+* @param {object} response - response object returned from the executon of a request.
+* @param {object} data - the data body returned from the executon of a request.
+*/
+function getErrorBody(response, data) {
+    var errorBody = { code: response.statusCode, body: data };
+    
+    if (Constants.HttpHeaders.SubStatus in response.headers) {
+        errorBody.substatus = parseInt(response.headers[Constants.HttpHeaders.SubStatus]);
+    }
+    
+    if (Constants.HttpHeaders.RetryAfterInMilliseconds in response.headers) {
+        errorBody.retryAfterInMilliseconds = parseInt(response.headers[Constants.HttpHeaders.RetryAfterInMilliseconds]);
+    }
+
+    return errorBody;
+}
+
 var RequestHandler = {
@@ -118,0 +129,0 @@ _createRequestObjectStub: function (connectionPolicy, requestOptions, callback) {

lib/retryUtility.js

@@ -27,3 +27,5 @@ /*
 var Base = require("./base"),
-    EndpointDiscoveryRetryPolicy = require("./endpointDiscoveryRetryPolicy");
+    Constants = require("./constants"),
+    EndpointDiscoveryRetryPolicy = require("./endpointDiscoveryRetryPolicy"),
+    ResourceThrottleRetryPolicy = require("./resourceThrottleRetryPolicy");
 
@@ -41,5 +43,63 @@ //SCRIPT START
     */
-    execute: function (globalEndpointManager, body, createRequestObjectStub, connectionPolicy, requestOptions, callback) {
+    execute: function (globalEndpointManager, body, createRequestObjectFunc, connectionPolicy, requestOptions, callback) {
         var endpointDiscoveryRetryPolicy = new EndpointDiscoveryRetryPolicy(globalEndpointManager);
-        endpointDiscoveryRetryPolicy.apply(body, createRequestObjectStub, connectionPolicy, requestOptions, callback);
+        var resourceThrottleRetryPolicy = new ResourceThrottleRetryPolicy(connectionPolicy.RetryOptions.MaxRetryAttemptCount, 
+                                                connectionPolicy.RetryOptions.FixedRetryIntervalInMilliseconds,
+                                                connectionPolicy.RetryOptions.MaxWaitTimeInSeconds);
+
+        this.apply(body, createRequestObjectFunc, connectionPolicy, requestOptions, endpointDiscoveryRetryPolicy, resourceThrottleRetryPolicy, callback);
+    },
+    
+    /**
+    * Applies the retry policy for the created request object.
+    * @param {object} body - a dictionary containing 'buffer' and 'stream' keys to hold corresponding buffer or stream body, null otherwise.
+    * @param {function} createRequestObjectFunc - function that creates the request object.
+    * @param {object} connectionPolicy - an instance of ConnectionPolicy that has the connection configs.
+    * @param {RequestOptions} requestOptions - The request options.
+    * @param {EndpointDiscoveryRetryPolicy} endpointDiscoveryRetryPolicy - The endpoint discovery retry policy instance.
+    * @param {ResourceThrottleRetryPolicy} resourceThrottleRetryPolicy - The resource throttle retry policy instance.
+    * @param {function} callback - the callback that will be called when the response is retrieved and processed.
+    */
+    apply: function (body, createRequestObjectFunc, connectionPolicy, requestOptions, endpointDiscoveryRetryPolicy, resourceThrottleRetryPolicy, callback) {
+        var that = this;
+        var httpsRequest = createRequestObjectFunc(connectionPolicy, requestOptions, function (err, response, headers) {
+            if (err) {
+                var retryPolicy = null;
+                headers = headers || {};
+                if (err.code === EndpointDiscoveryRetryPolicy.FORBIDDEN_STATUS_CODE && err.substatus === EndpointDiscoveryRetryPolicy.WRITE_FORBIDDEN_SUB_STATUS_CODE) {
+                    retryPolicy = endpointDiscoveryRetryPolicy;
+                } else if (err.code === ResourceThrottleRetryPolicy.THROTTLE_STATUS_CODE) {
+                    retryPolicy = resourceThrottleRetryPolicy;
+                }
+                if (retryPolicy) {
+                    retryPolicy.shouldRetry(err, function (shouldRetry) {
+                        if (!shouldRetry) {
+                            headers[Constants.ThrottleRetryCount] = resourceThrottleRetryPolicy.currentRetryAttemptCount;
+                            headers[Constants.ThrottleRetryWaitTimeInMs] = resourceThrottleRetryPolicy.cummulativeWaitTimeinMilliseconds;
+                            return callback(err, response, headers);
+                        } else {
+                            setTimeout(function () {
+                                that.apply(body, createRequestObjectFunc, connectionPolicy, requestOptions, endpointDiscoveryRetryPolicy, resourceThrottleRetryPolicy, callback);
+                            }, retryPolicy.retryAfterInMilliseconds);
+                            return;
+                        }
+                    });
+                    return;
+                }
+            }
+            headers[Constants.ThrottleRetryCount] = resourceThrottleRetryPolicy.currentRetryAttemptCount;
+            headers[Constants.ThrottleRetryWaitTimeInMs] = resourceThrottleRetryPolicy.cummulativeWaitTimeinMilliseconds;
+            return callback(err, response, headers);
+        });
+        
+        if (httpsRequest) {
+            if (body["stream"] !== null) {
+                body["stream"].pipe(httpsRequest);
+            } else if (body["buffer"] !== null) {
+                httpsRequest.write(body["buffer"]);
+                httpsRequest.end();
+            } else {
+                httpsRequest.end();
+            }
+        }
     }
@@ -46,0 +106,0 @@ }

package.json

@@ -12,3 +12,3 @@ {
   ],
-  "version": "1.8.0",
+  "version": "1.9.0",
   "author": "Microsoft Corporation",
@@ -15,0 +15,0 @@ "main": "./index.js",

test/BaselineTest.PathParser.json

@@ -0,0 +0,0 @@ [

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

586303

increased by3.52%

Number of package files

39

increased by8.33%

Lines of code

9958

increased by2.76%

No dependency changes