@alwaysmeticulous/downloading-helpers - npm Package Compare versions

Comparing version 2.77.0 to 2.78.0

dist/file-downloads/download-file.d.ts

 export declare const downloadFile: (fileUrl: string, path: string) => Promise<void>;
+export declare const downloadAndExtractFile: (fileUrl: string, filePath: string, extractPath: string) => Promise<void>;

dist/file-downloads/download-file.js

@@ -6,4 +6,5 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.downloadFile = void 0;
+exports.downloadAndExtractFile = exports.downloadFile = void 0;
 const fs_1 = require("fs");
+const promises_1 = require("fs/promises");
 const stream_1 = require("stream");
@@ -13,2 +14,3 @@ const util_1 = require("util");
 const axios_retry_1 = __importDefault(require("axios-retry"));
+const extract_zip_1 = __importDefault(require("extract-zip"));
 const promisifiedFinished = (0, util_1.promisify)(stream_1.finished);
@@ -28,2 +30,8 @@ const downloadFile = async (fileUrl, path) => {
 exports.downloadFile = downloadFile;
+const downloadAndExtractFile = async (fileUrl, filePath, extractPath) => {
+    await (0, exports.downloadFile)(fileUrl, filePath);
+    await (0, extract_zip_1.default)(filePath, { dir: extractPath });
+    await (0, promises_1.rm)(filePath);
+};
+exports.downloadAndExtractFile = downloadAndExtractFile;
 //# sourceMappingURL=download-file.js.map

dist/scripts/replays.js

@@ -13,4 +13,6 @@ "use strict";
 const loglevel_1 = __importDefault(require("loglevel"));
+const p_limit_1 = __importDefault(require("p-limit"));
 const download_file_1 = require("../file-downloads/download-file");
 const local_data_utils_1 = require("../file-downloads/local-data.utils");
+const MAX_DOWNLOAD_CONCURRENCY = 20;
 const getOrFetchReplay = async (client, replayId) => {
@@ -37,3 +39,2 @@ const logger = loglevel_1.default.getLogger(common_1.METICULOUS_LOGGER_NAME);
     try {
-        const replayArchiveFile = (0, path_1.join)(replayDir, `${replayId}.zip`);
         const paramsFile = (0, path_1.join)(replayDir, "metadata.json");
@@ -46,9 +47,12 @@ // Check if "metadata.json" exists. If yes, we assume the replay
         }
-        const downloadUrlData = await (0, client_1.getReplayDownloadUrl)(client, replayId);
-        if (!downloadUrlData) {
-            throw new Error("Error: Could not retrieve replay archive URL. This may be an invalid replay");
+        const replay = await (0, client_1.getReplay)(client, replayId);
+        if (["v1", "v2"].includes(replay.version)) {
+            await downloadReplayV2Archive(client, replayId, replayDir);
         }
-        await (0, download_file_1.downloadFile)(downloadUrlData.dowloadUrl, replayArchiveFile);
-        await (0, extract_zip_1.default)(replayArchiveFile, { dir: replayDir });
-        await (0, promises_1.rm)(replayArchiveFile);
+        else if (replay.version === "v3") {
+            await downloadReplayV3Files(client, replayId, replayDir);
+        }
+        else {
+            throw new Error(`Error: Unknown replay version "${replay.version}". This may be an invalid replay`);
+        }
         logger.debug(`Extracted replay archive in ${replayDir}`);
@@ -62,4 +66,46 @@ return { fileName: replayDir };
 exports.getOrFetchReplayArchive = getOrFetchReplayArchive;
+const downloadReplayV2Archive = async (client, replayId, replayDir) => {
+    const replayArchiveFile = (0, path_1.join)(replayDir, `${replayId}.zip`);
+    const downloadUrlData = await (0, client_1.getReplayDownloadUrl)(client, replayId);
+    if (!downloadUrlData) {
+        throw new Error("Error: Could not retrieve replay archive URL. This may be an invalid replay");
+    }
+    await (0, download_file_1.downloadFile)(downloadUrlData.dowloadUrl, replayArchiveFile);
+    await (0, extract_zip_1.default)(replayArchiveFile, { dir: replayDir });
+    await (0, promises_1.rm)(replayArchiveFile);
+};
+const downloadReplayV3Files = async (client, replayId, replayDir) => {
+    const downloadUrls = await (0, client_1.getReplayV3DownloadUrls)(client, replayId);
+    if (!downloadUrls) {
+        throw new Error("Error: Could not retrieve replay download URLs. This may be an invalid replay");
+    }
+    const { screenshots, snapshottedAssets, ...rest } = downloadUrls;
+    await (0, promises_1.mkdir)((0, path_1.join)(replayDir, "screenshots"), { recursive: true });
+    const filePromises = Object.entries(rest).map(([fileName, data]) => {
+        const filePath = (0, path_1.join)(replayDir, fileName);
+        return () => (0, download_file_1.downloadAndExtractFile)(data.signedUrl, filePath, replayDir);
+    });
+    const screenshotPromises = Object.values(screenshots).flatMap((data) => {
+        const imageFilePath = (0, path_1.join)(replayDir, data.image.filePath);
+        const metadataFilePath = (0, path_1.join)(replayDir, data.metadata.filePath);
+        return [
+            () => (0, download_file_1.downloadFile)(data.image.signedUrl, imageFilePath),
+            () => (0, download_file_1.downloadAndExtractFile)(data.metadata.signedUrl, metadataFilePath, (0, path_1.join)(replayDir, (0, path_1.dirname)(data.metadata.filePath))),
+        ];
+    });
+    const snapshottedAssetsPromise = async () => {
+        if (!snapshottedAssets) {
+            return;
+        }
+        const snapshottedAssetsDir = (0, path_1.join)(replayDir, "snapshotted-assets");
+        await (0, promises_1.mkdir)(snapshottedAssetsDir, {
+            recursive: true,
+        });
+        await (0, download_file_1.downloadAndExtractFile)(snapshottedAssets.signedUrl, (0, path_1.join)(replayDir, snapshottedAssets.filePath), snapshottedAssetsDir);
+    };
+    const limited = (0, p_limit_1.default)(MAX_DOWNLOAD_CONCURRENCY);
+    await Promise.all([...filePromises, ...screenshotPromises, snapshottedAssetsPromise].map((p) => limited(p)));
+};
 const getReplayDir = (replayId) => (0, path_1.join)((0, common_1.getMeticulousLocalDataDir)(), "replays", replayId);
 exports.getReplayDir = getReplayDir;
 //# sourceMappingURL=replays.js.map

package.json

 {
   "name": "@alwaysmeticulous/downloading-helpers",
-  "version": "2.77.0",
+  "version": "2.78.0",
   "description": "Helper utilities for downloading files & scripts required to execute replays",
@@ -22,5 +22,5 @@ "license": "ISC",
   "dependencies": {
-    "@alwaysmeticulous/api": "^2.77.0",
-    "@alwaysmeticulous/client": "^2.77.0",
-    "@alwaysmeticulous/common": "^2.77.0",
+    "@alwaysmeticulous/api": "^2.78.0",
+    "@alwaysmeticulous/client": "^2.78.0",
+    "@alwaysmeticulous/common": "^2.78.0",
     "axios": "^1.2.6",
@@ -31,2 +31,3 @@ "axios-retry": "^3.5.0",
     "luxon": "^3.2.1",
+    "p-limit": "^3.1.0",
     "proper-lockfile": "^4.1.2"
@@ -55,3 +56,3 @@ },
   },
-  "gitHead": "0e434accc42eceb323078977344cb6298dfb5b4e"
+  "gitHead": "586f71337fc4cc82f56753970cf0259c54c71c8b"
 }

No alert changes

Improved metrics

Total package byte prevSize

38972

increased by17.17%

Lines of code

443

increased by14.18%

Worsened metrics

Dependency count

10

increased by11.11%

Number of low supply chain risk alerts

6

increased by20%

Dependency changes

• + Addedp-limit@^3.1.0

• + Addedp-limit@3.1.0(transitive)

• + Addedyocto-queue@0.1.0(transitive)

• Updated@alwaysmeticulous/api@^2.78.0

• Updated@alwaysmeticulous/client@^2.78.0

• Updated@alwaysmeticulous/common@^2.78.0