ghrequestor - npm Package Compare versions

Comparing version 0.1.5 to 0.1.6

lib/ghrequestor.js

 const extend = require('extend');
+const parse = require('parse-link-header');
 const Q = require('q');
+const qlimit = require('qlimit');
 const request = require('requestretry');
 
 class GHRequestor {
+  /**
+   * Attempt to get the GitHub resource at the given target URL.  The callback supplied, if any,
+   * is called when the resource has been retrieved or an irrecoverable problem has been encountered.
+   * If a callback is not supplied, a promise is returned. The promise will be resolved with the
+   * response on success or rejected with an error object. Note that responses with statusCode >=300 are not
+   * errors -- the promise will be resolved with such a response.
+   *
+   * Note that the options can include an etags property that is an array of etags to use for the GET requests.
+   * Element N-1 of the array will be used for page N of a multi page request.
+   *
+   * @param {string} target URL to fetch
+   * @param {object} [options] Options to use through the retry and request process.
+   * @param {function} [callback] Function to call on completion of the retrieval.
+   * @returns {null|promise} null if a callback is supplied. A promise otherwise.
+   */
+  static get(target, options = {}, callback = null) {
+    if (typeof options == 'function') {
+      callback = options;
+      options = null;
+    }
+    return new RequestorAction(options).get(target, callback);
+  }
 
-  constructor(givenOptions) {
-    this.result = [];
-    const defaultOptions = GHRequestor._defaultOptions;
+  /**
+   * Attempt to get all pages related to the given target URL.  The callback supplied, if any,
+   * is called when all pages have been retrieved or an irrecoverable problem has been encountered.
+   * If a callback is not supplied, a promise is returned. Either way, on success the returned value is the
+   * collected elements from all the pages.  The result is truncated at the point of the first non-200
+   * responses. In the error case, the error object may have a response property containing the
+   * response that caused the failure.
+   *
+   * @param {string} target URL to fetch and paginate
+   * @param {object} [options] Options to use through the retry and request process.
+   * @param {function} [callback] Function to call on completion of the retrieval.
+   * @returns {null|promise} null if a callback is supplied. A promise otherwise.
+   */
+  static getAll(target, options = {}, callback = null) {
+    return GHRequestor.getAllResponses(target, options, callback).then(GHRequestor.flattenResponses);
+  }
+
+  /**
+   * Attempt to get all pages related to the given target URL.  The callback supplied, if any,
+   * is called when all pages have been retrieved or an irrecoverable problem has been encountered.
+   * If a callback is not supplied, a promise is returned. Either way, on success the returned value is the
+   * collected responses.  Note that this may be a mixed bag of 200 OK and 304 Not Modified (if etags were
+   * supplied). Each response's "body" will contain that page of data. In the error case, the error object
+   * may have a response property containing the response that caused the failure.
+   *
+   * Note that the options can include an etags property that is an array of etags to use for the GET requests.
+   * Element N-1 of the array will be used for page N of a multi page request.
+   *
+   * @param {string} target URL to fetch and paginate
+   * @param {object} [options] Options to use through the retry and request process.
+   * @param {function} [callback] Function to call on completion of the retrieval.
+   * @returns {null|promise} null if a callback is supplied. A promise otherwise.
+   */
+  static getAllResponses(target, options = {}, callback = null) {
+    if (typeof options === 'function') {
+      callback = options;
+      options = null;
+    }
+    return new RequestorAction(options).getAll(target, callback);
+  }
+
+  /**
+   * Get a requestor pre-configured with the given options.
+   * @param {object} options The set of options with which to configure the result.
+   * @returns {requestor} A requestor configured with the given options.
+   */
+  static defaults(options) {
+    const newOptions = GHRequestor.mergeOptions(this.defaultOptions, options);
+    return new RequestorTemplate(newOptions);
+  }
+
+  /**
+   * Helper function used to merge options. Care is taken to merge headers as well as properties.
+   * @param {object} defaultOptions The base set of options.
+   * @param {object} givenOptions The set of options to overlay on the defaults.
+   * @returns {object} The two sets of options merged.
+   */
+  static mergeOptions(defaultOptions, givenOptions) {
+    if (!givenOptions) {
+      return defaultOptions;
+    } else if (!defaultOptions) {
+      return givenOptions;
+    }
     const headers = extend({}, defaultOptions.headers, givenOptions.headers);
-    this.options = extend({}, defaultOptions, givenOptions, { headers: headers });
-    this.options.retryStrategy = GHRequestor._retryStrategy.bind(this);
-    this.options.delayStrategy = GHRequestor._retryDelayStrategy.bind(this);
-    this.activity = [];
+    return extend({}, defaultOptions, givenOptions, { headers: headers });
+  }
+
+  /**
+   * Flatten a set of responses (e.g., pages) into one array of values.  If any
+   * repsonse is a 304 then ask the given supplier to resolve response to an array of
+   * values to be included in the result.  The supplier is given the response and
+   * should return a value or a promise.  A supplier may, for example, look in a local
+   * cache for the previously fetched response.
+   * @param {array} repsonses The GET responses to flatten
+   * @param {function} [supplier] A function that will resolve a response to a set of values
+   * @returns {array} The flattened set of response bodies
+   */
+  static flattenResponses(responses, supplier) {
+    const chunks = responses.map(qlimit(10)(response => {
+      if (response.statusCode === 200) {
+        return Q(response.body);
+      }
+      if (response.statusCode === 304) {
+        if (!supplier) {
+          return Q.reject(new Error(`304 response encountered but no content supplier found`));
+        }
+        return Q(supplier(response.url));
+      }
+      return Q.reject(new Error(`Cannot flatten response with status code: ${response.statusCode}`));
+    }));
+    return Q.all(chunks).then(resolvedChunks => {
+      const result = resolvedChunks.reduce((result, element) => {
+        return result.concat(element);
+      }, []);
+      result.activity = responses.activity;
+      return result;
+    });
+  }
+}
+
+module.exports = GHRequestor;
+
+class RequestorTemplate {
+  constructor(options = {}) {
+    this.defaultOptions = options;
+  }
+
+  get(target, options, callback) {
+    if (typeof options === 'function') {
+      callback = options;
+      options = null;
+    }
+    return GHRequestor.get(target, GHRequestor.mergeOptions(this.defaultOptions, options), callback);
+  }
+
+  getAll(target, options, callback) {
+    if (typeof options === 'function') {
+      callback = options;
+      options = null;
+    }
+    return GHRequestor.getAll(target, GHRequestor.mergeOptions(this.defaultOptions, options), callback);
+  }
+
+  getAllResponses(target, options, callback) {
+    if (typeof options === 'function') {
+      callback = options;
+      options = null;
+    }
+    return GHRequestor.getAllResponses(target, GHRequestor.mergeOptions(this.defaultOptions, options), callback);
+  }
+
+  mergeOptions(defaultOptions, givenOptions) {
+    return GHRequestor.mergeOptions(defaultOptions, givenOptions);
+  }
+
+  defaults(options) {
+    const newOptions = GHRequestor.mergeOptions(this.defaultOptions, options);
+    return new RequestorTemplate(newOptions);
+  }
+
+  flattenResponses(responses, supplier) {
+    return GHRequestor.flattenResponses(responses, supplier);
+  }
+}
+
+class RequestorAction {
+  constructor(givenOptions = {}) {
+    this.options = GHRequestor.mergeOptions(RequestorAction._defaultOptions, givenOptions);
+    this.options.retryStrategy = RequestorAction._retryStrategy.bind(this);
+    this.options.delayStrategy = RequestorAction._retryDelayStrategy.bind(this);
+    this._initialize();
     return this;
   }
 
+  _initialize() {
+    this.result = [];
+    this.activity = [];
+  }
+
   static get _defaultOptions() {
@@ -25,2 +197,3 @@ return {
       forbiddenDelay: 3 * 60 * 1000,
+      delayOnThrottle: true,
       tokenLowerBound: 500
@@ -40,39 +213,8 @@ };
 
-  static _parseLinkHeader(header) {
-    if (header.length === 0) {
-      throw new Error("input must not be of zero length");
-    }
-
-    // Split parts by comma
-    const parts = header.split(',');
-    const links = {};
-    // Parse each part into a named link
-    for (var i = 0; i < parts.length; i++) {
-      const section = parts[i].split(';');
-      if (section.length !== 2) {
-        throw new Error("section could not be split on ';'");
-      }
-      const url = section[0].replace(/<(.*)>/, '$1').trim();
-      const name = section[1].replace(/rel="(.*)"/, '$1').trim();
-      links[name] = url;
-    }
-    return links;
-  }
-
-  /**
-   * Attempt to get all pages related to the given target URL.  The callback supplied, if any,
-   * is called when all pages have been retrieved or an irrecoverable problem has been encountered.
-   * If a callback is not supplied, a promise is returned. The promise will be resolved with the
-   * collected bodies on success or rejected with an error object.  The error may have a response property
-   * containing the response that caused the failure.
-   * @param {string} target URL to fetch and paginate
-   * @param {Array} [result] Array into which results are put.
-   * @param {function} [callback] Function to call on completion of the retrieval.
-   * @returns {null|promise} null if a callback is supplied. A promise otherwise.
-   */
-  static getAll(target, options = {}, callback = null) {
-    const requestor = new GHRequestor(options);
-    return requestor._getAll(target, callback).then(result => {
-      requestor.result.activity = requestor.activity;
-      return requestor.result;
+  getAll(target, callback = null) {
+    this._initialize();
+    const self = this;
+    return this._getAll(target, callback).then(result => {
+      self.result.activity = self.activity;
+      return self.result;
     });
@@ -90,8 +232,5 @@ }
     const self = this;
-    target = GHRequestor._ensureMaxPerPage(target);
     self._get(target, (err, response, body) => {
-      // if we get an error or a non-200 response, return an Error object that has the
-      // error and response (if any).
-      if (err || response && response.statusCode >= 300) {
-        err = err || new Error(response.statusMessage);
+      // if we get an error, stash the repsonse (if any) and activity, and pass the error along.
+      if (err) {
         err.response = response;
@@ -102,14 +241,14 @@ err.activity = self.activity;
 
-      // merge or add the body to the result
-      if (Array.isArray(body)) {
-        Array.prototype.push.apply(self.result, body);
-      } else {
-        self.result.push(body);
+      self.result.push(response);
+
+      // if the response is not great (basically anything other than 200 and 304), bail out
+      if (response && (response.statusCode >= 300 && response.statusCode !== 304)) {
+        return realCallback(null, self.result);
       }
 
+      // if there is a next page, go for it.
       if (response.headers.link) {
-        // if there is a next page, go for it.
-        const links = GHRequestor._parseLinkHeader(response.headers.link);
+        const links = parse(response.headers.link);
         if (links.next) {
-          return self._getAll(links.next, realCallback);
+          return self._getAll(links.next.url, realCallback);
         }
@@ -123,15 +262,5 @@ }
 
-  /**
-   * Attempt to get the GitHub resource at the given target URL.  The callback supplied, if any,
-   * is called when the resource has been retrieved or an irrecoverable problem has been encountered.
-   * If a callback is not supplied, a promise is returned. The promise will be resolved with the
-   * response on success or rejected with an error object. Note that responses with statusCode >=300 are not
-   * errors -- the promise will be resolved with such a response.
-   * @param {string} target URL to fetch
-   * @param {function} [callback] Function to call on completion of the retrieval.
-   * @returns {null|promise} null if a callback is supplied. A promise otherwise.
-   */
-  static get(target, options = {}, callback = null) {
-    const requestor = new GHRequestor(options);
-    return requestor._get(target, callback);
+  get(target, callback = null) {
+    this._initialize();
+    return this._get(target, callback);
   }
@@ -141,7 +270,14 @@
     const deferred = Q.defer();
-    target = GHRequestor._ensureMaxPerPage(target);
+    target = RequestorAction._ensureMaxPerPage(target);
     const self = this;
     const activity = {};
     this.activity.push(activity);
-    request.get(target, this.options, (err, response, body) => {
+    let options = this.options;
+    if (this.options.etags) {
+      const etag = this.options.etags[this.activity.length - 1];
+      if (etag) {
+        options = GHRequestor.mergeOptions(this.options, { headers: { 'If-None-Match': etag } });
+      }
+    }
+    request.get(target, options, (err, response, body) => {
       if (response) {
@@ -151,6 +287,4 @@ activity.attempts = response.attempts;
       }
-      if (err || response && response.statusCode >= 300) {
-        if (!err) {
-          err = new Error(response.statusMessage);
-        }
+      if (err || !response) {
+        err = err || new Error(response.statusMessage);
         err.response = response;
@@ -160,2 +294,7 @@ err.activity = self.activity;
       }
+      // Failed here so resolve with the same response
+      if (response.statusCode >= 300) {
+        return callback ? callback(err, response, body) : deferred.resolve(response);
+      }
+
       // If we hit the low water mark for requests, proactively sleep until the next ratelimit reset
@@ -165,3 +304,3 @@ // This code is not designed to handle the 403 scenarios.  That is handled by the retry logic.
       const reset = parseInt(response.headers['x-ratelimit-reset']) || 0;
-      if (remaining < self.options.tokenLowerBound) {
+      if (self.options.delayOnThrottle && remaining < self.options.tokenLowerBound) {
         const toSleep = Math.max(reset * 1000 - Date.now(), 2000);
@@ -186,3 +325,3 @@ activity.rateLimitDelay = toSleep;
     // All others, do not retry as it won't help
-    if (response && response.statusCode === 403) {
+    if (response && response.statusCode === 403 && this.options.forbiddenDelay) {
       response._forbiddenDelay = this.options.forbiddenDelay;
@@ -209,3 +348,1 @@ return true;
 }
-
-module.exports = GHRequestor;

package.json

 {
   "name": "ghrequestor",
-  "version": "0.1.5",
+  "version": "0.1.6",
   "description": "A simple, resilient GitHub API client for bulk retrieval of GitHub resources",
   "main": "./lib/ghrequestor.js",
-  "bin": {
-    "ghrequestor": "./lib/ghrequestor.js"
-  },
   "scripts": {
@@ -34,3 +31,5 @@ "test": "echo \"Error: no test specified\" && exit 1"
   "dependencies": {
+    "parse-link-header": "^0.4.1",
     "q": "^1.4.1",
+    "qlimit": "^0.1.1",
     "requestretry": "^1.12.0"
@@ -42,4 +41,5 @@ },
     "grunt-mocha-test": "^0.13.2",
+    "istanbul": "^0.4.5",
     "mocha": "^3.1.2"
   }
 }

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

16818

49.85%

Lines of code

307

68.68%

Worsened metrics

Dependency count

4

100%

Dev dependency count

5

25%

Number of medium supply chain risk alerts

1

Infinity%

Dependency changes

• + Added

  parse-link-header@^0.4.1

• + Added

  qlimit@^0.1.1

• + Added

  parse-link-header@0.4.1
  (transitive)

• + Added

  qlimit@0.1.1
  (transitive)

• + Added

  xtend@4.0.2
  (transitive)