carp-streamer - npm Package Compare versions

Comparing version 0.2.9 to 0.3.0

dist/app.js

@@ -52,4 +52,4 @@ "use strict";
 const debug = util_1.default.debuglog('carp-streamer:app');
-class File {
-    constructor(root, relativePath, dirent, remoteRoot, remoteFile) {
+class Entry {
+    constructor(root, relativePath, dirent, remoteRoot) {
         this.root = root;
@@ -59,3 +59,2 @@ this.relativePath = relativePath;
         this.remoteRoot = remoteRoot;
-        this.remoteFile = remoteFile;
     }
@@ -65,31 +64,40 @@ get absolutePath() {
     }
-    _synchronize(client, pretend = false, retryTimes, delay) {
+    synchronize(client, pretend = false) {
+        return this._synchronize(client, pretend, INIT_RETRY_TIMES, 0);
+    }
+    _synchronize(client, pretend, retryTimes, delay) {
         return __awaiter(this, void 0, void 0, function* () {
             yield sleep(delay);
             try {
-                if (!this.dirent) {
-                    // client.files.getReadStream()
-                    return ResultStatus.DOWNLOADED;
+                return yield this.sync(client, pretend);
+            }
+            catch (error) {
+                if (!isBoxAPIResponseError(error))
+                    throw error;
+                debug('API Response Error: %s', error.message);
+                if (!(error.statusCode === 429 && retryTimes > 0))
+                    throw error;
+                debug('Retries %d more times.', retryTimes);
+                const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 10 * (1 / retryTimes))) * 1000;
+                debug('Tries again in %d milliseconds.', retryAfter);
+                return this._synchronize(client, pretend, retryTimes - 1, retryAfter);
+            }
+        });
+    }
+    static create(dirent, root, relativePath, remoteRoot, client) {
+        return Entry._create(dirent, root, relativePath, remoteRoot, client, INIT_RETRY_TIMES, 0);
+    }
+    static _create(dirent, root, relativePath, remoteRoot, client, retryTimes, delay) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield sleep(delay);
+            try {
+                if (!dirent) {
+                    // TODO:
+                    throw new Error('Not Implemented Error');
                 }
-                else if (!this.remoteFile) {
-                    if (!pretend) {
-                        const { dir, base } = path_1.default.parse(this.relativePath);
-                        const folder = yield createRemoteFolderUnlessItExists(dir, this.remoteRoot, client);
-                        debug('Uploading `%s`...', this.relativePath);
-                        yield client.files.uploadFile(folder.id, base, this.createReadStream());
-                    }
-                    return ResultStatus.UPLOADED;
+                else if (dirent.isDirectory()) {
+                    return new Directory(root, relativePath, dirent, remoteRoot, yield findRemoteFolderByPath(relativePath, remoteRoot, client));
                 }
                 else {
-                    const sha1 = yield this.digest();
-                    if (sha1 === this.remoteFile.sha1) {
-                        return ResultStatus.SYNCHRONIZED;
-                    }
-                    else {
-                        if (!pretend) {
-                            debug('Upgrading `%s`...', this.relativePath);
-                            yield client.files.uploadNewFileVersion(this.remoteFile.id, this.createReadStream());
-                        }
-                        return ResultStatus.UPGRADED;
-                    }
+                    return new File(root, relativePath, dirent, remoteRoot, yield findRemoteFileByPath(relativePath, remoteRoot, client));
                 }
@@ -104,11 +112,59 @@ }
                 debug('Retries %d more times.', retryTimes);
-                const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 100) * (1 / retryTimes)) * 1000;
+                const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 10 * (1 / retryTimes))) * 1000;
                 debug('Tries again in %d milliseconds.', retryAfter);
-                return yield this._synchronize(client, pretend, retryTimes - 1, retryAfter);
+                return this._create(dirent, root, relativePath, remoteRoot, client, retryTimes - 1, retryAfter);
             }
         });
     }
-    synchronize(client, pretend = false) {
-        return this._synchronize(client, pretend, INIT_RETRY_TIMES, 0);
+}
+exports.Entry = Entry;
+class Directory extends Entry {
+    constructor(root, relativePath, dirent, remoteRoot, remoteFile) {
+        super(root, relativePath, dirent, remoteRoot);
+        this.remoteFile = remoteFile;
     }
+    sync(client, pretend = false) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!pretend)
+                yield createRemoteFolderUnlessItExists(this.relativePath, this.remoteRoot, client);
+            return ResultStatus.SYNCHRONIZED;
+        });
+    }
+}
+exports.Directory = Directory;
+class File extends Entry {
+    constructor(root, relativePath, dirent, remoteRoot, remoteFile) {
+        super(root, relativePath, dirent, remoteRoot);
+        this.remoteFile = remoteFile;
+    }
+    sync(client, pretend = false) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.dirent) {
+                // client.files.getReadStream()
+                return ResultStatus.DOWNLOADED;
+            }
+            else if (!this.remoteFile) {
+                if (!pretend) {
+                    const { dir, base } = path_1.default.parse(this.relativePath);
+                    const folder = yield createRemoteFolderUnlessItExists(dir, this.remoteRoot, client);
+                    debug('Uploading `%s`...', this.relativePath);
+                    yield client.files.uploadFile(folder.id, base, this.createReadStream());
+                }
+                return ResultStatus.UPLOADED;
+            }
+            else {
+                const sha1 = yield this.digest();
+                if (sha1 === this.remoteFile.sha1) {
+                    return ResultStatus.SYNCHRONIZED;
+                }
+                else {
+                    if (!pretend) {
+                        debug('Upgrading `%s`...', this.relativePath);
+                        yield client.files.uploadNewFileVersion(this.remoteFile.id, this.createReadStream());
+                    }
+                    return ResultStatus.UPGRADED;
+                }
+            }
+        });
+    }
     digest() {
@@ -131,27 +187,10 @@ return new Promise((resolve, reject) => {
 function findRemoteFileByPath(relativePath, rootFolder, client) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const { dir, base } = path_1.default.parse(relativePath);
-        const dirs = dir === '' ? [] : dir.split(path_1.default.sep);
-        return yield _findRemoteFileByPath(dirs, base, rootFolder, client, INIT_RETRY_TIMES, 0);
-    });
+    const { dir, base } = path_1.default.parse(relativePath);
+    const dirs = dir === '' ? [] : dir.split(path_1.default.sep);
+    return _findRemoteFileByPath(dirs, base, rootFolder, client);
 }
-exports.findRemoteFileByPath = findRemoteFileByPath;
-function _findRemoteFileByPath(folderPath, filename, rootFolder, client, retryTimes, delay) {
+function _findRemoteFileByPath(folderPath, filename, rootFolder, client) {
     return __awaiter(this, void 0, void 0, function* () {
-        yield sleep(delay);
-        try {
-            const folder = yield findRemoteFolderByPath(folderPath, rootFolder, client);
-            return !folder ? folder : yield findRemoteFileByName(filename, client, folder.id);
-        }
-        catch (error) {
-            if (!isBoxAPIResponseError(error))
-                throw error;
-            debug('API Response Error: %s', error.message);
-            if (!(error.statusCode === 429 && retryTimes > 0))
-                throw error;
-            debug('Retries %d more times.', retryTimes);
-            const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 100) * (1 / retryTimes)) * 1000;
-            debug('Tries again in %d milliseconds.', retryAfter);
-            return yield _findRemoteFileByPath(folderPath, filename, rootFolder, client, retryTimes - 1, retryAfter);
-        }
+        const folder = yield _findRemoteFolderByPath(folderPath, rootFolder, client);
+        return !folder ? folder : yield findRemoteFileByName(filename, client, folder.id);
     });
@@ -161,3 +200,8 @@ }
 const isMiniFolder = (item) => item.type === 'folder';
-function findRemoteFolderByPath(folderPath, rootFolder, client) {
+function findRemoteFolderByPath(relativePath, rootFolder, client) {
+    const { dir, base } = path_1.default.parse(relativePath);
+    const dirs = dir === '' ? [] : dir.split(path_1.default.sep);
+    return _findRemoteFolderByPath(dirs, rootFolder, client);
+}
+function _findRemoteFolderByPath(folderPath, rootFolder, client) {
     return __awaiter(this, void 0, void 0, function* () {
@@ -169,3 +213,3 @@ if (folderPath.length === 0 || rootFolder === undefined) {
         const subFolder = yield findRemoteFolderByName(folderName, client, rootFolder.id);
-        return yield findRemoteFolderByPath(folderPath.slice(1), subFolder, client);
+        return _findRemoteFolderByPath(folderPath.slice(1), subFolder, client);
     });
@@ -181,3 +225,3 @@ }
     const folder = subFolder || (yield createRemoteFolder(client, rootFolder.id, folderName, INIT_RETRY_TIMES, 0));
-    return yield createRemoteFolderByPath(folderPath.slice(1), folder, client);
+    return createRemoteFolderByPath(folderPath.slice(1), folder, client);
 });
@@ -248,3 +292,3 @@ function sleep(delay) {
             if (folder) {
-                return yield client.folders.get(folder.id);
+                return client.folders.get(folder.id);
             }
@@ -254,3 +298,3 @@ else {
                 debug(`Waiting time is %d milliseconds.`, retryAfter);
-                return yield createRemoteFolder(client, parentFolderId, folderName, retryTimes - 1, retryAfter);
+                return createRemoteFolder(client, parentFolderId, folderName, retryTimes - 1, retryAfter);
             }
@@ -262,27 +306,7 @@ }
     return __awaiter(this, void 0, void 0, function* () {
-        return yield _createRemoteFolderUnlessItExists(relativePath, rootFolder, client, INIT_RETRY_TIMES, 0);
+        const dirs = !relativePath ? [] : relativePath.split(path_1.default.sep);
+        const foundFolder = yield _findRemoteFolderByPath(dirs, rootFolder, client);
+        return foundFolder || (yield createRemoteFolderByPath(dirs, rootFolder, client));
     });
 }
-exports.createRemoteFolderUnlessItExists = createRemoteFolderUnlessItExists;
-function _createRemoteFolderUnlessItExists(relativePath, rootFolder, client, retryTimes, delay) {
-    return __awaiter(this, void 0, void 0, function* () {
-        yield sleep(delay);
-        try {
-            const dirs = !relativePath ? [] : relativePath.split(path_1.default.sep);
-            const foundFolder = yield findRemoteFolderByPath(dirs, rootFolder, client);
-            return foundFolder || (yield createRemoteFolderByPath(dirs, rootFolder, client));
-        }
-        catch (error) {
-            if (!isBoxAPIResponseError(error))
-                throw error;
-            debug('API Response Error: %s', error.message);
-            if (!(error.statusCode === 429 && retryTimes > 0))
-                throw error;
-            debug('Retries %d more times.', retryTimes);
-            const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 100) * (1 / retryTimes)) * 1000;
-            debug('Tries again in %d milliseconds.', retryAfter);
-            return yield _createRemoteFolderUnlessItExists(relativePath, rootFolder, client, retryTimes - 1, retryAfter);
-        }
-    });
-}
 const readdir = util_1.default.promisify(fs_1.default.readdir);
@@ -289,0 +313,0 @@ function listDirectoryEntriesRecursively(root) {

dist/index.js

@@ -24,2 +24,3 @@ #!/usr/bin/env node
 const minimist_1 = __importDefault(require("minimist"));
+const async_1 = __importDefault(require("async"));
 const fs_1 = __importDefault(require("fs"));
@@ -32,6 +33,7 @@ const path_1 = __importDefault(require("path"));
 const argsOption = {
-    'alias': { t: 'token', v: 'version' },
+    'alias': { t: 'token', v: 'version', c: 'concurrency' },
     'string': ['t', 'as-user'],
     'boolean': ['v', 'dry-run'],
-    'default': { 'dry-run': false }
+    'number': ['c'],
+    'default': { 'dry-run': false, concurrency: 10 }
 };
@@ -49,2 +51,3 @@ const args = minimist_1.default(process.argv.slice(2), argsOption);
 const pretend = args['dry-run'];
+const concurrency = args['concurrency'];
 const appConfig = process.env.BOX_APP_CONFIG && JSON.parse(fs_1.default.readFileSync(process.env.BOX_APP_CONFIG).toString());
@@ -64,48 +67,38 @@ const createBoxClient = (params) => {
     var e_1, _a;
-    const promises = [];
+    const q = async_1.default.queue(({ path: absolutePath, dirent }, done) => __awaiter(this, void 0, void 0, function* () {
+        const relativePath = path_1.default.relative(rootPath, absolutePath);
+        try {
+            const entry = yield app_1.Entry.create(dirent, rootPath, relativePath, rootFolder, client);
+            const status = yield entry.synchronize(client, pretend);
+            switch (status) {
+                case app_1.ResultStatus.DOWNLOADED:
+                    console.log(`'${relativePath}' only exists remotely.`);
+                    break;
+                case app_1.ResultStatus.SYNCHRONIZED:
+                    console.log(`'${relativePath}' is synchronized.`);
+                    break;
+                case app_1.ResultStatus.UPLOADED:
+                    console.log(`'${relativePath}' is newly uploaded.`);
+                    break;
+                case app_1.ResultStatus.UPGRADED:
+                    console.log(`A new version of '${relativePath}' has been uploaded.`);
+                    break;
+                default:
+                    throw new Error('unknown result status');
+            }
+            done();
+        }
+        catch (error) {
+            debug('%s: %s', error.name, error.message);
+            debug('%s', error.stack);
+            console.log(`Failed to synchronize '${relativePath}'.`);
+            done(error);
+        }
+    }), concurrency);
+    let count = 0;
     try {
         for (var _b = __asyncValues(app_1.listDirectoryEntriesRecursively(rootPath)), _c; _c = yield _b.next(), !_c.done;) {
-            let { path: absolutePath, dirent } = _c.value;
-            const relativePath = path_1.default.relative(rootPath, absolutePath);
-            if (dirent.isDirectory()) {
-                try {
-                    if (!pretend)
-                        yield app_1.createRemoteFolderUnlessItExists(relativePath, rootFolder, client);
-                }
-                catch (error) {
-                    debug('%s: %s', error.name, error.message);
-                    debug('%s', error.stack);
-                    console.log(`Failed to synchronize '${relativePath}'.`);
-                    throw error;
-                }
-                continue;
-            }
-            const remoteFile = yield app_1.findRemoteFileByPath(relativePath, rootFolder, client);
-            debug('%o', { relativePath, dirent, remoteFile });
-            const file = new app_1.File(rootPath, relativePath, dirent, rootFolder, remoteFile);
-            const promise = file.synchronize(client, pretend).then(status => {
-                switch (status) {
-                    case app_1.ResultStatus.DOWNLOADED:
-                        console.log(`'${file.relativePath}' only exists remotely.`);
-                        break;
-                    case app_1.ResultStatus.SYNCHRONIZED:
-                        console.log(`'${file.relativePath}' is synchronized.`);
-                        break;
-                    case app_1.ResultStatus.UPLOADED:
-                        console.log(`'${file.relativePath}' is newly uploaded.`);
-                        break;
-                    case app_1.ResultStatus.UPGRADED:
-                        console.log(`A new version of '${file.relativePath}' has been uploaded.`);
-                        break;
-                    default:
-                        throw new Error('unknown result status');
-                }
-                return status;
-            }).catch(error => {
-                debug('%s: %s', error.name, error.message);
-                debug('%s', error.stack);
-                console.log(`Failed to synchronize '${file.relativePath}'.`);
-                throw error;
-            });
-            promises.push(promise);
+            let entry = _c.value;
+            q.push(entry);
+            count++;
         }
@@ -120,6 +113,6 @@ }
     }
-    console.log(`${promises.length} entries were found.`);
-    return Promise.all(promises);
+    console.log(`${count} entries were found.`);
+    return yield q.drain();
 })).then(results => {
-    console.log('Successful! %d entries were synchronized.', results.length);
+    console.log('Successful!');
     process.exit(0);
@@ -126,0 +119,0 @@ }).catch(reason => {

package.json

 {
   "name": "carp-streamer",
-  "version": "0.2.9",
+  "version": "0.3.0",
   "description": "Carp streamer",
@@ -26,2 +26,3 @@ "bin": "dist/index.js",
   "devDependencies": {
+    "@types/async": "^3.0.0",
     "@types/chai": "^4.1.7",
@@ -42,2 +43,3 @@ "@types/lodash": "^4.14.135",
   "dependencies": {
+    "async": "^3.1.0",
     "box-node-sdk": "^1.29.0",
@@ -44,0 +46,0 @@ "lodash": "^4.17.11",

src/app.ts

@@ -19,5 +19,4 @@ import BoxSDK from 'box-node-sdk';
 
-export class File {
-
-  constructor(private root: string, readonly relativePath: string, private dirent: fs.Dirent | undefined, private remoteRoot: BoxSDK.Folder, private remoteFile: BoxSDK.MiniFile | undefined) {
+export abstract class Entry {
+  constructor(protected root: string, readonly relativePath: string, protected dirent: fs.Dirent | undefined, protected remoteRoot: BoxSDK.Folder) {
   }
@@ -29,27 +28,36 @@
 
-  private async _synchronize(client: BoxSDK.BoxClient, pretend: boolean = false, retryTimes: number, delay: number): Promise<ResultStatus> {
+  synchronize(client: BoxSDK.BoxClient, pretend: boolean = false) {
+    return this._synchronize(client, pretend, INIT_RETRY_TIMES, 0);
+  }
+
+  private async _synchronize(client: BoxSDK.BoxClient, pretend: boolean, retryTimes: number, delay: number): Promise<ResultStatus> {
     await sleep(delay);
     try {
-      if (!this.dirent) {
-        // client.files.getReadStream()
-        return ResultStatus.DOWNLOADED;
-      } else if (!this.remoteFile) {
-        if (!pretend) {
-          const { dir, base } = path.parse(this.relativePath);
-          const folder = await createRemoteFolderUnlessItExists(dir, this.remoteRoot, client);
-          debug('Uploading `%s`...', this.relativePath);
-          await client.files.uploadFile(folder.id, base, this.createReadStream());  
-        }
-        return ResultStatus.UPLOADED;
+      return await this.sync(client, pretend);
+    } catch (error) {
+      if (!isBoxAPIResponseError(error)) throw error;
+      debug('API Response Error: %s', error.message);
+      if (!(error.statusCode === 429 && retryTimes > 0)) throw error;
+
+      debug('Retries %d more times.', retryTimes);
+      const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 10 * (1 / retryTimes))) * 1000;
+      debug('Tries again in %d milliseconds.', retryAfter);
+      return this._synchronize(client, pretend, retryTimes - 1, retryAfter);
+    }
+  }
+
+  static create(dirent: fs.Dirent, root: string, relativePath: string, remoteRoot: BoxSDK.Folder, client: BoxSDK.BoxClient): Promise<Entry> {
+    return Entry._create(dirent, root, relativePath, remoteRoot, client, INIT_RETRY_TIMES, 0);
+  }
+
+  private static async _create(dirent: fs.Dirent, root: string, relativePath: string, remoteRoot: BoxSDK.Folder, client: BoxSDK.BoxClient, retryTimes: number, delay: number): Promise<Entry> {
+    await sleep(delay);
+    try {
+      if (!dirent) {
+        // TODO:
+        throw new Error('Not Implemented Error');
+      } else if (dirent.isDirectory()) {
+        return new Directory(root, relativePath, dirent, remoteRoot, await findRemoteFolderByPath(relativePath, remoteRoot, client));
       } else {
-        const sha1 = await this.digest();
-        if (sha1 === this.remoteFile!.sha1) {
-          return ResultStatus.SYNCHRONIZED;
-        } else {
-          if (!pretend) {
-            debug('Upgrading `%s`...', this.relativePath);
-            await client.files.uploadNewFileVersion(this.remoteFile!.id, this.createReadStream());
-          }
-          return ResultStatus.UPGRADED;
-        }
+        return new File(root, relativePath, dirent, remoteRoot, await findRemoteFileByPath(relativePath, remoteRoot, client));
       }
@@ -62,12 +70,53 @@ } catch (error) {
       debug('Retries %d more times.', retryTimes);
-      const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 100) * (1 / retryTimes)) * 1000;
+      const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 10 * (1 / retryTimes))) * 1000;
       debug('Tries again in %d milliseconds.', retryAfter);
-      return await this._synchronize(client, pretend, retryTimes - 1, retryAfter);
+      return this._create(dirent, root, relativePath, remoteRoot, client, retryTimes - 1, retryAfter);
     }
   }
 
-  synchronize(client: BoxSDK.BoxClient, pretend: boolean = false) {
-    return this._synchronize(client, pretend, INIT_RETRY_TIMES, 0);
+  protected abstract sync(client: BoxSDK.BoxClient, pretend: boolean): Promise<ResultStatus>;
+}
+
+export class Directory extends Entry {
+  constructor(root: string, relativePath: string, dirent: fs.Dirent | undefined, remoteRoot: BoxSDK.Folder, private remoteFile: BoxSDK.MiniFolder | undefined) {
+    super(root, relativePath, dirent, remoteRoot);
   }
 
+  protected async sync(client: BoxSDK.BoxClient, pretend: boolean = false): Promise<ResultStatus> {
+    if (!pretend) await createRemoteFolderUnlessItExists(this.relativePath, this.remoteRoot, client);
+    return ResultStatus.SYNCHRONIZED;
+  }
+}
+
+export class File extends Entry {
+  constructor(root: string, relativePath: string, dirent: fs.Dirent | undefined, remoteRoot: BoxSDK.Folder, private remoteFile: BoxSDK.MiniFile | undefined) {
+    super(root, relativePath, dirent, remoteRoot);
+  }
+
+  protected async sync(client: BoxSDK.BoxClient, pretend: boolean = false): Promise<ResultStatus> {
+    if (!this.dirent) {
+      // client.files.getReadStream()
+      return ResultStatus.DOWNLOADED;
+    } else if (!this.remoteFile) {
+      if (!pretend) {
+        const { dir, base } = path.parse(this.relativePath);
+        const folder = await createRemoteFolderUnlessItExists(dir, this.remoteRoot, client);
+        debug('Uploading `%s`...', this.relativePath);
+        await client.files.uploadFile(folder.id, base, this.createReadStream());  
+      }
+      return ResultStatus.UPLOADED;
+    } else {
+      const sha1 = await this.digest();
+      if (sha1 === this.remoteFile!.sha1) {
+        return ResultStatus.SYNCHRONIZED;
+      } else {
+        if (!pretend) {
+          debug('Upgrading `%s`...', this.relativePath);
+          await client.files.uploadNewFileVersion(this.remoteFile!.id, this.createReadStream());
+        }
+        return ResultStatus.UPGRADED;
+      }
+    }
+  }
+
   private digest() {
@@ -97,23 +146,11 @@ return new Promise<string>((resolve, reject) => {
 
-export async function findRemoteFileByPath(relativePath: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient): Promise<BoxSDK.File | undefined> {
+function findRemoteFileByPath(relativePath: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient): Promise<BoxSDK.File | undefined> {
   const { dir, base } = path.parse(relativePath);
   const dirs = dir === '' ? [] : dir.split(path.sep);
-  return await _findRemoteFileByPath(dirs, base, rootFolder, client, INIT_RETRY_TIMES, 0);
+  return _findRemoteFileByPath(dirs, base, rootFolder, client);
 }
 
-async function _findRemoteFileByPath(folderPath: string[], filename: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient, retryTimes: number, delay: number): Promise<BoxSDK.MiniFile | undefined> {
-  await sleep(delay);
-  try {
-    const folder = await findRemoteFolderByPath(folderPath, rootFolder, client);
-    return !folder ? folder : await findRemoteFileByName(filename, client, folder.id);
-  } catch (error) {
-    if (!isBoxAPIResponseError(error)) throw error;
-    debug('API Response Error: %s', error.message);
-    if (!(error.statusCode === 429 && retryTimes > 0)) throw error;
-
-    debug('Retries %d more times.', retryTimes);
-    const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 100) * (1 / retryTimes)) * 1000;
-    debug('Tries again in %d milliseconds.', retryAfter);
-    return await _findRemoteFileByPath(folderPath, filename, rootFolder, client, retryTimes - 1, retryAfter);
-  }
+async function _findRemoteFileByPath(folderPath: string[], filename: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient): Promise<BoxSDK.MiniFile | undefined> {
+  const folder = await _findRemoteFolderByPath(folderPath, rootFolder, client);
+  return !folder ? folder : await findRemoteFileByName(filename, client, folder.id);
 }
@@ -124,3 +161,9 @@
 
-async function findRemoteFolderByPath(folderPath: string[], rootFolder: BoxSDK.MiniFolder | undefined, client: BoxSDK.BoxClient): Promise<BoxSDK.Folder | undefined> {
+function findRemoteFolderByPath(relativePath: string, rootFolder: BoxSDK.MiniFolder | undefined, client: BoxSDK.BoxClient): Promise<BoxSDK.Folder | undefined> {
+  const { dir, base } = path.parse(relativePath);
+  const dirs = dir === '' ? [] : dir.split(path.sep);
+  return _findRemoteFolderByPath(dirs, rootFolder, client);
+}
+
+async function _findRemoteFolderByPath(folderPath: string[], rootFolder: BoxSDK.MiniFolder | undefined, client: BoxSDK.BoxClient): Promise<BoxSDK.Folder | undefined> {
   if (folderPath.length === 0 || rootFolder === undefined) {
@@ -132,3 +175,3 @@ return rootFolder === undefined || isFolder(rootFolder) ? rootFolder : await client.folders.get(rootFolder.id);
   const subFolder = await findRemoteFolderByName(folderName, client, rootFolder.id);
-  return await findRemoteFolderByPath(folderPath.slice(1), subFolder, client);
+  return _findRemoteFolderByPath(folderPath.slice(1), subFolder, client);
 };
@@ -144,3 +187,3 @@
   const folder = subFolder || await createRemoteFolder(client, rootFolder.id, folderName, INIT_RETRY_TIMES, 0);
-  return await createRemoteFolderByPath(folderPath.slice(1), folder, client);
+  return createRemoteFolderByPath(folderPath.slice(1), folder, client);
 };
@@ -182,7 +225,7 @@
     if (folder) {
-      return await client.folders.get(folder.id);
+      return client.folders.get(folder.id);
     } else {
       const retryAfter = (Math.floor(Math.random() * 100) * (1 / retryTimes)) * 1000;
       debug(`Waiting time is %d milliseconds.`, retryAfter);
-      return await createRemoteFolder(client, parentFolderId, folderName, retryTimes - 1, retryAfter);
+      return createRemoteFolder(client, parentFolderId, folderName, retryTimes - 1, retryAfter);
     }
@@ -192,24 +235,8 @@ }
 
-export async function createRemoteFolderUnlessItExists(relativePath: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient): Promise<BoxSDK.Folder> {
-  return await _createRemoteFolderUnlessItExists(relativePath, rootFolder, client, INIT_RETRY_TIMES, 0);
+async function createRemoteFolderUnlessItExists(relativePath: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient): Promise<BoxSDK.Folder> {
+  const dirs = !relativePath ? [] : relativePath.split(path.sep);
+  const foundFolder = await _findRemoteFolderByPath(dirs, rootFolder, client);
+  return foundFolder || await createRemoteFolderByPath(dirs, rootFolder, client)
 }
 
-async function _createRemoteFolderUnlessItExists(relativePath: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient, retryTimes: number, delay: number): Promise<BoxSDK.Folder> {
-  await sleep(delay);
-  try {
-    const dirs = !relativePath ? [] : relativePath.split(path.sep);
-    const foundFolder = await findRemoteFolderByPath(dirs, rootFolder, client);
-    return foundFolder || await createRemoteFolderByPath(dirs, rootFolder, client)
-  } catch (error) {
-    if (!isBoxAPIResponseError(error)) throw error;
-    debug('API Response Error: %s', error.message);
-    if (!(error.statusCode === 429 && retryTimes > 0)) throw error;
-
-    debug('Retries %d more times.', retryTimes);
-    const retryAfter = (Number(error.response.headers['retry-after'] || 0) + Math.floor(Math.random() * 100) * (1 / retryTimes)) * 1000;
-    debug('Tries again in %d milliseconds.', retryAfter);
-    return await _createRemoteFolderUnlessItExists(relativePath, rootFolder, client, retryTimes - 1, retryAfter);
-  }
-}
-
 const readdir = util.promisify(fs.readdir);
@@ -216,0 +243,0 @@ export async function* listDirectoryEntriesRecursively(root: string): AsyncIterableIterator<{path: string, dirent: fs.Dirent}> {

src/index.ts

@@ -6,6 +6,7 @@ #!/usr/bin/env node
 import minimist from 'minimist';
+import async from 'async';
 import fs from 'fs';
 import path from 'path';
 import util from 'util';
-import {File, ResultStatus, listDirectoryEntriesRecursively, findRemoteFileByPath, createRemoteFolderUnlessItExists} from './app'
+import {Entry, ResultStatus, listDirectoryEntriesRecursively} from './app'
 
@@ -16,6 +17,7 @@ const npmPackage = require('../package.json');
 const argsOption = {
-  'alias': { t: 'token', v: 'version' },
+  'alias': { t: 'token', v: 'version', c: 'concurrency' },
   'string': ['t', 'as-user'],
   'boolean': ['v', 'dry-run'],
-  'default': { 'dry-run': false }
+  'number': ['c'],
+  'default': { 'dry-run': false, concurrency: 10 }
 };
@@ -37,2 +39,3 @@ const args = minimist(process.argv.slice(2), argsOption);
 const pretend: boolean = args['dry-run'];
+const concurrency: number = args['concurrency'];
 
@@ -53,32 +56,19 @@ const appConfig = process.env.BOX_APP_CONFIG && JSON.parse(fs.readFileSync(process.env.BOX_APP_CONFIG).toString());
 client.folders.get(destination).then(async (rootFolder) => {
-  const promises = [];
-  for await (let { path: absolutePath, dirent } of listDirectoryEntriesRecursively(rootPath)) {
+  const q = async.queue(async ({ path: absolutePath, dirent }, done) => {
     const relativePath = path.relative(rootPath, absolutePath);
-    if (dirent.isDirectory()) {
-      try {
-        if (!pretend) await createRemoteFolderUnlessItExists(relativePath, rootFolder, client);
-      } catch (error) {
-        debug('%s: %s', error.name, error.message);
-        debug('%s', error.stack);
-        console.log(`Failed to synchronize '${relativePath}'.`);
-        throw error;
-      }
-      continue;
-    }
-    const remoteFile = await findRemoteFileByPath(relativePath, rootFolder, client);
-    debug('%o', { relativePath, dirent, remoteFile });
-    const file = new File(rootPath, relativePath, dirent, rootFolder, remoteFile);
-    const promise = file.synchronize(client, pretend).then(status => {
+    try {
+      const entry = await Entry.create(dirent, rootPath, relativePath, rootFolder, client);
+      const status = await entry.synchronize(client, pretend);
       switch (status) {
         case ResultStatus.DOWNLOADED:
-          console.log(`'${file.relativePath}' only exists remotely.`);
+          console.log(`'${relativePath}' only exists remotely.`);
           break;
         case ResultStatus.SYNCHRONIZED:
-          console.log(`'${file.relativePath}' is synchronized.`);
+          console.log(`'${relativePath}' is synchronized.`);
           break;
         case ResultStatus.UPLOADED:
-          console.log(`'${file.relativePath}' is newly uploaded.`);
+          console.log(`'${relativePath}' is newly uploaded.`);
           break;
         case ResultStatus.UPGRADED:
-          console.log(`A new version of '${file.relativePath}' has been uploaded.`);
+          console.log(`A new version of '${relativePath}' has been uploaded.`);
           break;
@@ -88,15 +78,19 @@ default:
       }
-      return status;
-    }).catch(error => {
+      done();
+    } catch (error) {
       debug('%s: %s', error.name, error.message);
       debug('%s', error.stack);
-      console.log(`Failed to synchronize '${file.relativePath}'.`);
-      throw error;
-    });
-    promises.push(promise);
+      console.log(`Failed to synchronize '${relativePath}'.`);
+      done(error);
+    }
+  }, concurrency);
+  let count = 0;
+  for await (let entry of listDirectoryEntriesRecursively(rootPath)) {
+    q.push(entry);
+    count++;
   }
-  console.log(`${promises.length} entries were found.`);
-  return Promise.all(promises);
+  console.log(`${count} entries were found.`);
+  return await q.drain();
 }).then(results => {
-  console.log('Successful! %d entries were synchronized.', results.length);
+  console.log('Successful!');
   process.exit(0);
@@ -103,0 +97,0 @@ }).catch(reason => {

No alert changes

Improved metrics

Total package byte prevSize

64026

increased by1.8%

Lines of code

1023

increased by3.33%

Worsened metrics

Dependency count

4

increased by33.33%

Dev dependency count

14

increased by7.69%

Dependency changes

• + Addedasync@^3.1.0

• + Addedasync@3.2.5(transitive)