carp-streamer - npm Package Compare versions

Comparing version 0.2.6 to 0.2.7

dist/app.js

@@ -126,8 +126,3 @@ "use strict";
         const folder = yield findRemoteFolderByPath(folderPath, rootFolder, client);
-        if (!folder)
-            return folder;
-        const items = yield client.folders.getItems(folder.id);
-        return lodash_1.default.first(items.entries
-            .filter(isMiniFile)
-            .filter(item => item.name.normalize() === filename.normalize()));
+        return !folder ? folder : yield findRemoteFileByName(filename, client, folder.id);
     });
@@ -142,5 +137,4 @@ }
         }
-        const folderName = lodash_1.default.first(folderPath);
-        const items = yield client.folders.getItems(rootFolder.id);
-        const subFolder = lodash_1.default.first(items.entries.filter(isMiniFolder).filter(item_2 => item_2.name === folderName));
+        const folderName = lodash_1.default.first(folderPath) || '';
+        const subFolder = yield findRemoteFolderByName(folderName, client, rootFolder.id);
         return yield findRemoteFolderByPath(folderPath.slice(1), subFolder, client);
@@ -156,7 +150,82 @@ });
     const folderName = lodash_1.default.first(folderPath) || '';
-    const items = yield client.folders.getItems(rootFolder.id);
-    const subFolder = lodash_1.default.first(items.entries.filter(isMiniFolder).filter(item => item.name === folderName));
-    const folder = subFolder || (yield client.folders.create(rootFolder.id, folderName));
+    const subFolder = yield findRemoteFolderByName(folderName, client, rootFolder.id);
+    const folder = subFolder || (yield createRemoteFolder(client, rootFolder.id, folderName, 3));
     return yield createRemoteFolderByPath(folderPath.slice(1), folder, client);
 });
+function sleep(delay) {
+    return new Promise(resolve => setTimeout(resolve, delay));
+}
+function fetchRemoteFolderItems(client, parentFolderId, marker) {
+    return __asyncGenerator(this, arguments, function* fetchRemoteFolderItems_1() {
+        const items = yield __await(client.folders.getItems(parentFolderId, { usemarker: true, marker }));
+        yield __await(yield* __asyncDelegator(__asyncValues(items.entries)));
+        if (items.next_marker)
+            yield __await(yield* __asyncDelegator(__asyncValues(fetchRemoteFolderItems(client, parentFolderId, items.next_marker))));
+    });
+}
+function findRemoteFolderByName(folderName, client, parentFolderId) {
+    var e_1, _a;
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            for (var _b = __asyncValues(fetchRemoteFolderItems(client, parentFolderId)), _c; _c = yield _b.next(), !_c.done;) {
+                let item = _c.value;
+                if (isMiniFolder(item) && item.name == folderName)
+                    return item;
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+    });
+}
+function findRemoteFileByName(fileName, client, parentFolderId) {
+    var e_2, _a;
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            for (var _b = __asyncValues(fetchRemoteFolderItems(client, parentFolderId)), _c; _c = yield _b.next(), !_c.done;) {
+                let item = _c.value;
+                if (isMiniFile(item) && item.name.normalize() == fileName.normalize())
+                    return item;
+            }
+        }
+        catch (e_2_1) { e_2 = { error: e_2_1 }; }
+        finally {
+            try {
+                if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b);
+            }
+            finally { if (e_2) throw e_2.error; }
+        }
+    });
+}
+function createRemoteFolder(client, parentFolderId, folderName, retryTimes) {
+    return __awaiter(this, void 0, void 0, function* () {
+        try {
+            return yield client.folders.create(parentFolderId, folderName);
+        }
+        catch (error) {
+            debug('%s: %s', error.name, error.message);
+            debug(`Failed to create folder '%s' (parent folder id: %s). Retries %d more times.`, folderName, parentFolderId, retryTimes);
+            const waitingTime = Math.floor(Math.random() * 100);
+            debug(`Waiting time is %d milliseconds.`, waitingTime);
+            const startTimestamp = Date.now();
+            yield sleep(waitingTime);
+            const elapsedTime = Date.now() - startTimestamp;
+            debug(`Waited for %d milliseconds.`, elapsedTime);
+            const folder = yield findRemoteFolderByName(folderName, client, parentFolderId);
+            if (folder) {
+                return yield client.folders.get(folder.id);
+            }
+            else if (retryTimes > 0) {
+                return yield createRemoteFolder(client, parentFolderId, folderName, retryTimes - 1);
+            }
+            else {
+                throw error;
+            }
+        }
+    });
+}
 function createRemoteFolderUnlessItExists(relativePath, rootFolder, client) {
@@ -163,0 +232,0 @@ return __awaiter(this, void 0, void 0, function* () {

package.json

 {
   "name": "carp-streamer",
-  "version": "0.2.6",
+  "version": "0.2.7",
   "description": "Carp streamer",
@@ -5,0 +5,0 @@ "bin": "dist/index.js",

src/@types/box-node-sdk/index.d.ts

@@ -29,6 +29,16 @@ declare module 'box-node-sdk' {
     get(folderId: string): Promise<Folder>;
-    getItems(folderId: string): Promise<Items>;
+    getItems(folderId: string, options?: GetItemsOptions): Promise<Items>;
     create(folderId: string, folderName: string): Promise<Folder>;
   }
 
+  export interface GetItemsOptions {
+    fields?: string;
+    usemarker? : boolean;
+    marker?: string;
+    offset?: number;
+    sort?: string;
+    direction?: 'ASC' | 'DESC';
+    limit?: number;
+  }
+
   export interface Files {
@@ -52,5 +62,6 @@ uploadFile(folderId: string, fileName: string, stream: ReadStream): Promise<File>;
   export interface Items {
-    total_count: number;
+    next_marker?: string;
+    total_count?: number;
+    offset?: number;
     entries: Item[];
-    offset: number;
     limit: number;
@@ -57,0 +68,0 @@ order: Order[];

src/app.ts

@@ -84,11 +84,5 @@ import BoxSDK from 'box-node-sdk';
 
-async function _findRemoteFileByPath(folderPath: string[], filename: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient): Promise<BoxSDK.File | undefined> {
+async function _findRemoteFileByPath(folderPath: string[], filename: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient): Promise<BoxSDK.MiniFile | undefined> {
   const folder = await findRemoteFolderByPath(folderPath, rootFolder, client);
-  if (!folder) return folder;
-  const items = await client.folders.getItems(folder.id);
-  return _.first(
-    items.entries
-    .filter(isMiniFile)
-    .filter(item => item.name.normalize() === filename.normalize())
-  );
+  return !folder ? folder : await findRemoteFileByName(filename, client, folder.id);
 }
@@ -104,5 +98,4 @@
 
-  const folderName = _.first(folderPath);
-  const items = await client.folders.getItems(rootFolder.id);
-  const subFolder = _.first(items.entries.filter(isMiniFolder).filter(item_2 => item_2.name === folderName));
+  const folderName = _.first(folderPath) || '';
+  const subFolder = await findRemoteFolderByName(folderName, client, rootFolder.id);
   return await findRemoteFolderByPath(folderPath.slice(1), subFolder, client);
@@ -117,8 +110,52 @@ };
   const folderName = _.first(folderPath) || '';
-  const items = await client.folders.getItems(rootFolder.id);
-  const subFolder = _.first(items.entries.filter(isMiniFolder).filter(item => item.name === folderName));
-  const folder = subFolder || await client.folders.create(rootFolder.id, folderName);
+  const subFolder = await findRemoteFolderByName(folderName, client, rootFolder.id);
+  const folder = subFolder || await createRemoteFolder(client, rootFolder.id, folderName, 3);
   return await createRemoteFolderByPath(folderPath.slice(1), folder, client);
 };
 
+function sleep(delay: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, delay));
+}
+
+async function* fetchRemoteFolderItems(client: BoxSDK.BoxClient, parentFolderId: string, marker?: string): AsyncIterableIterator<BoxSDK.Item> {
+  const items = await client.folders.getItems(parentFolderId, { usemarker: true, marker });
+  yield* items.entries;
+  if (items.next_marker) yield* fetchRemoteFolderItems(client, parentFolderId, items.next_marker);
+}
+
+async function findRemoteFolderByName(folderName: string, client: BoxSDK.BoxClient, parentFolderId: string): Promise<BoxSDK.MiniFolder | undefined> {
+  for await (let item of fetchRemoteFolderItems(client, parentFolderId)) {
+    if (isMiniFolder(item) && item.name == folderName) return item;
+  }
+}
+
+async function findRemoteFileByName(fileName: string, client: BoxSDK.BoxClient, parentFolderId: string): Promise<BoxSDK.MiniFile | undefined> {
+  for await (let item of fetchRemoteFolderItems(client, parentFolderId)) {
+    if (isMiniFile(item) && item.name.normalize() == fileName.normalize()) return item;
+  }
+}
+
+async function createRemoteFolder(client: BoxSDK.BoxClient, parentFolderId: string, folderName: string, retryTimes: number): Promise<BoxSDK.Folder> {
+  try {
+    return await client.folders.create(parentFolderId, folderName);
+  } catch (error) {
+    debug('%s: %s', error.name, error.message);
+    debug(`Failed to create folder '%s' (parent folder id: %s). Retries %d more times.`, folderName, parentFolderId, retryTimes);
+    const waitingTime = Math.floor(Math.random() * 100);
+    debug(`Waiting time is %d milliseconds.`, waitingTime);
+    const startTimestamp = Date.now();
+    await sleep(waitingTime);
+    const elapsedTime = Date.now() - startTimestamp;
+    debug(`Waited for %d milliseconds.`, elapsedTime);
+    const folder = await findRemoteFolderByName(folderName, client, parentFolderId);
+    if (folder) {
+      return await client.folders.get(folder.id);
+    } else if (retryTimes > 0) {
+      return await createRemoteFolder(client, parentFolderId, folderName, retryTimes - 1);
+    } else {
+      throw error;
+    }
+  }
+}
+
 export async function createRemoteFolderUnlessItExists(relativePath: string, rootFolder: BoxSDK.MiniFolder, client: BoxSDK.BoxClient): Promise<BoxSDK.Folder> {
@@ -125,0 +162,0 @@ const dirs = !relativePath ? [] : relativePath.split(path.sep);

No alert changes

Improved metrics

Total package byte prevSize

56373

increased by12.94%

Lines of code

914

increased by13.82%

No dependency changes