@nerdwallet/shepherd - npm Package Compare versions

Comparing version 1.0.1 to 1.0.2

CHANGELOG.md

@@ -7,2 +7,6 @@ # Changelog
 
+## v1.0.2
+* Fix for issue hitting GitHub rate limiting when operating on very large result sets by abiding by
+the retry-after response header on failure
+
 ## v1.0.1
@@ -9,0 +13,0 @@

lib/adapters/base.d.ts

 export interface IRepo {
     [key: string]: any;
 }
+export declare type RetryMethod = (opts: number) => any;
 interface IRepoAdapter {
-    getCandidateRepos(): Promise<IRepo[]>;
+    getCandidateRepos(onRetry: RetryMethod): Promise<IRepo[]>;
     parseRepo(repo: string): IRepo;
@@ -7,0 +8,0 @@ reposEqual(repo1: IRepo, repo2: IRepo): boolean;

lib/adapters/git.d.ts

 import { SimpleGit } from 'simple-git/promise';
 import { IMigrationContext } from '../migration-context';
-import IRepoAdapter, { IRepo } from './base';
+import IRepoAdapter, { IRepo, RetryMethod } from './base';
 declare abstract class GitAdapter implements IRepoAdapter {
@@ -8,3 +8,3 @@ protected migrationContext: IMigrationContext;
     constructor(migrationContext: IMigrationContext);
-    abstract getCandidateRepos(): Promise<IRepo[]>;
+    abstract getCandidateRepos(onRetry: RetryMethod): Promise<IRepo[]>;
     abstract parseRepo(repo: string): IRepo;
@@ -11,0 +11,0 @@ abstract reposEqual(repo1: IRepo, repo2: IRepo): boolean;

lib/adapters/github.d.ts

 import Octokit from '@octokit/rest';
 import { IMigrationContext } from '../migration-context';
-import { IRepo } from './base';
+import { IRepo, RetryMethod } from './base';
 import GitAdapter from './git';
@@ -13,3 +13,3 @@ declare class GithubAdapter extends GitAdapter {
     constructor(migrationContext: IMigrationContext, octokit?: Octokit);
-    getCandidateRepos(): Promise<IRepo[]>;
+    getCandidateRepos(onRetry: RetryMethod): Promise<IRepo[]>;
     mapRepoAfterCheckout(repo: Readonly<IRepo>): Promise<IRepo>;
@@ -16,0 +16,0 @@ parseRepo(repo: string): IRepo;

lib/adapters/github.js

@@ -17,2 +17,3 @@ "use strict";
 const chalk_1 = __importDefault(require("chalk"));
+const lodash_1 = __importDefault(require("lodash"));
 const netrc_1 = __importDefault(require("netrc"));
@@ -61,9 +62,9 @@ const path_1 = __importDefault(require("path"));
     }
-    getCandidateRepos() {
+    getCandidateRepos(onRetry) {
         return __awaiter(this, void 0, void 0, function* () {
-            const searchResults = yield octokit_1.paginateSearch(this.octokit, this.octokit.search.code)({
+            const searchResults = yield octokit_1.paginateSearch(this.octokit, this.octokit.search.code, onRetry)({
                 q: this.migrationContext.migration.spec.adapter.search_query,
             });
             const repoNames = searchResults.map((r) => r.repository.full_name).sort();
-            return repoNames.map((r) => this.parseRepo(r));
+            return lodash_1.default.uniq(repoNames).map((r) => this.parseRepo(r));
         });
@@ -196,2 +197,3 @@ }
                     status.push(`PR was merged at ${pullRequest.merged_at}`);
+                    // @ts-ignore: mergeable_state is not included in @octokit/rest type definition
                 }
@@ -198,0 +200,0 @@ else if (pullRequest.mergeable && pullRequest.mergeable_state === 'clean') {

lib/commands/checkout.js

@@ -24,2 +24,5 @@ "use strict";
     const { migration: { selectedRepos }, adapter, logger, } = context;
+    function onRetry(numSeconds) {
+        logger.info(`Hit rate limit; waiting ${numSeconds} seconds and retrying.`);
+    }
     let repos;
@@ -32,3 +35,3 @@ if (selectedRepos) {
         const spinner = logger.spinner('Loading candidate repos');
-        repos = yield adapter.getCandidateRepos();
+        repos = yield adapter.getCandidateRepos(onRetry);
         spinner.succeed(`Loaded ${repos.length} repos`);
@@ -35,0 +38,0 @@ }

lib/util/octokit.d.ts

 import Octokit from '@octokit/rest';
+import { RetryMethod } from '../adapters/base';
 declare type DataExtractor = (d: any) => any[];
 declare type Method = (opts: any) => Promise<any>;
-export declare const paginate: (octokit: Octokit, method: Method, extractItems?: DataExtractor) => (options: any) => Promise<any[]>;
-export declare const paginateSearch: (octokit: Octokit, method: Method) => (options: any) => Promise<any[]>;
+export declare const paginate: (octokit: Octokit, method: Method, extractItems: DataExtractor | undefined, onRetry: RetryMethod) => (options: any) => Promise<any[]>;
+export declare const paginateSearch: (octokit: Octokit, method: Method, onRetry: Method) => (options: any) => Promise<any[]>;
 export {};

lib/util/octokit.js

@@ -12,3 +12,3 @@ "use strict";
 const wait = (timeout) => new Promise((resolve) => setTimeout(resolve, timeout));
-exports.paginate = (octokit, method, extractItems = (d) => d) => (options) => __awaiter(this, void 0, void 0, function* () {
+exports.paginate = (octokit, method, extractItems = (d) => d, onRetry) => (options) => __awaiter(this, void 0, void 0, function* () {
     let response = yield method(Object.assign({}, options, { per_page: 100 }));
@@ -18,5 +18,21 @@ let data = extractItems(response.data);
         // Avoid GitHub's "abuse detection mechanisms"
-        yield wait(2000);
-        response = yield octokit.getNextPage(response); // eslint-disable-line no-await-in-loop
-        data = data.concat(extractItems(response.data));
+        yield wait(500);
+        try {
+            response = yield octokit.getNextPage(response); // eslint-disable-line no-await-in-loop
+            data = data.concat(extractItems(response.data));
+        }
+        catch (e) {
+            if (e.headers && e.headers['retry-after']) {
+                const retryAfter = Number(e.headers['retry-after']);
+                if (Number.isNaN(retryAfter)) {
+                    throw e;
+                }
+                onRetry(retryAfter);
+                yield wait(retryAfter * 1000);
+                continue;
+            }
+            else {
+                throw e;
+            }
+        }
     }
@@ -28,3 +44,3 @@ return data;
 const extractSearch = (data) => data.items;
-exports.paginateSearch = (octokit, method) => exports.paginate(octokit, method, extractSearch);
+exports.paginateSearch = (octokit, method, onRetry) => exports.paginate(octokit, method, extractSearch, onRetry);
 //# sourceMappingURL=octokit.js.map

package.json

 {
   "name": "@nerdwallet/shepherd",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "description": "A utility for applying code changes across many repositories",
@@ -5,0 +5,0 @@ "keywords": [

README.md

 # Shepherd
 
+<img alt="Illustration of a sheep" width=320 align=right src="https://user-images.githubusercontent.com/332258/46430732-c7bf5c80-c6fe-11e8-9cc8-2abc4e68dce2.jpg">
+
 ![Travis status](https://img.shields.io/travis/NerdWalletOSS/shepherd/master.svg?style=flat-square)
@@ -4,0 +6,0 @@ ![npm version](https://img.shields.io/npm/v/@nerdwallet/shepherd.svg?style=flat-square)

No alert changes

Improved metrics

Total package byte prevSize

162132

increased by1.22%

Lines of code

1952

increased by1.19%

Number of lines in readme file

133

increased by1.53%

No dependency changes