amp-ext - npm Package Compare versions

Comparing version 1.0.6 to 1.0.7

index.js

@@ -89,5 +89,9 @@ #! /usr/bin/env node
 		const logPair = `${dayjs.utc(dates.start).format(logFormat)} → ${dayjs.utc(dates.end).format(logFormat)}`;
-		try {
-			await pipeline(
-				got.stream({
+		const download = () => new Promise(async (resolve, reject) => {
+			let retryStream = null;
+			let retry = false;
+			let error = null;
+	
+			while(true) {
+				const stream = retryStream ?? got.stream({
 					url,
@@ -100,8 +104,6 @@ searchParams: new URLSearchParams([['start', dates.start], ['end', dates.end]]),
 					retry: {
-						limit: 1000,
+						limit: 100,
 						statusCodes: [429, 500, 501, 503, 504, 502],
-						errorCodes: [],
 						methods: ['GET'],
 						noise: 100
-
 					},
@@ -112,23 +114,43 @@ agent: {
 					hooks: {
-						beforeRetry: [(req, resp, count) => {
-							try {
-								l(`got ${resp?.statusCode}...retrying request...#${count}`);
-							}
-							catch (e) {
-								//noop
-							}
+						beforeRetry: [(err, count) => {
+							l(`${logPair}: ERROR ${err?.response?.statusCode || "unknown"}...retrying request...#${count}`);
 						}]
 					}
-
-				}),
-				fs.createWriteStream(path.resolve(`${TEMP_DIR}/${dates.start}--${dates.end}.zip`))
-			);
-			l(`${logPair}: SUCCESS 200`);
-
-		}
-		catch (e) {
-			l(`${logPair}: ${e?.name || ""} ${e?.response?.statusCode || "unknown code"}; ${e?.message || "unknown error"}`);
+				});
+	
+				let writeStream = fs.createWriteStream(path.resolve(`${TEMP_DIR}/${dates.start}--${dates.end}.zip`));
+				
+				stream.once('retry', (retryCount, err, createRetryStream) => {
+					error = err;
+					retryStream = createRetryStream();
+					retry = true;
+				});
+	
+				try {
+					await pipeline(stream, writeStream);
+					l(`${logPair}: SUCCESS 200`);
+					resolve('success');
+					break;
+				} catch (err) {
+					error = err;
+					if (!retry) {
+						break;
+					}
+					retry = false;
+				}
+			}
+	
+			if (error) {
+				reject(error);
+			}
+		});
+	
+		try {
+			await download();
+		} catch (err) {			
+			l(`${logPair}: ERROR ${err?.response?.statusCode || "unknown"}... WILL NOT RETRY!`);
+			l(`\t${err?.response?.body}\n\n`)
 			continue consumeData;
 		}
-	}
+	};
 
@@ -143,3 +165,3 @@ const allFiles = (await u.ls(TEMP_DIR)).filter(path => path.endsWith('.zip'));
 	for (const empty of emptyFiles) {
-		u.rm(empty);
+		await u.rm(empty);
 	}
@@ -155,3 +177,3 @@ const downloadedFiles = (await u.ls(TEMP_DIR)).filter(path => path.endsWith('.zip'));
 		try {
-			execSync(`unzip -j ${escapeForShell(zipFile)} -d ${escapeForShell(dir)}`);
+			execSync(`unzip -o -j ${escapeForShell(zipFile)} -d ${escapeForShell(dir)}`);
 			if (cleanup) await u.rm(zipFile);
@@ -158,0 +180,0 @@ continue unzip;

package.json

 {
 	"name": "amp-ext",
-	"version": "1.0.6",
+	"version": "1.0.7",
 	"description": "pull data out of amplitude",
@@ -5,0 +5,0 @@ "main": "index.js",

No alert changes

Improved metrics

Total package byte prevSize

14474

increased by4.09%

Lines of code

483

increased by4.09%

No dependency changes