ipull - npm Package Compare versions

Comparing version 3.2.2 to 3.3.0

dist/download/download-engine/download-file/download-engine-file.js

@@ -169,3 +169,3 @@ import ProgressStatusFile, { DownloadFlags, DownloadStatus } from "./progress-status-file.js";
                 this._progress.chunks[index] = ChunkStatus.COMPLETE;
-                lastChunkSize = this._activeStreamBytes[startChunk];
+                lastChunkSize = chunks.reduce((last, current) => last + current.length, 0);
                 delete this._activeStreamBytes[startChunk];
@@ -172,0 +172,0 @@ void this._saveProgress();

dist/download/download-engine/engine/download-engine-multi-download.d.ts

@@ -10,10 +10,14 @@ import { EventEmitter } from "eventemitter3";
 };
+export type DownloadEngineMultiDownloadOptions = {
+    parallelDownloads?: number;
+};
 export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMultiAllowedEngines = DownloadEngineMultiAllowedEngines> extends EventEmitter<DownloadEngineMultiDownloadEvents> {
     readonly downloads: Engine[];
+    readonly options: DownloadEngineMultiDownloadOptions;
     protected _aborted: boolean;
-    protected _activeEngine?: Engine;
+    protected _activeEngines: Set<Engine>;
     protected _progressStatisticsBuilder: ProgressStatisticsBuilder;
     protected _downloadStatues: (ProgressStatusWithIndex | FormattedStatus)[];
     protected _closeFiles: (() => Promise<void>)[];
-    protected constructor(engines: (DownloadEngineMultiAllowedEngines | DownloadEngineMultiDownload)[]);
+    protected constructor(engines: (DownloadEngineMultiAllowedEngines | DownloadEngineMultiDownload)[], options: DownloadEngineMultiDownloadOptions);
     get downloadStatues(): (FormattedStatus | ProgressStatusWithIndex)[];
@@ -29,4 +33,4 @@ get downloadSize(): number;
     protected static _extractEngines<Engine>(engines: Engine[]): any[];
-    static fromEngines<Engine extends DownloadEngineMultiAllowedEngines>(engines: (Engine | Promise<Engine>)[]): Promise<DownloadEngineMultiDownload<BaseDownloadEngine>>;
+    static fromEngines<Engine extends DownloadEngineMultiAllowedEngines>(engines: (Engine | Promise<Engine>)[], options?: DownloadEngineMultiDownloadOptions): Promise<DownloadEngineMultiDownload<BaseDownloadEngine>>;
 }
 export {};

dist/download/download-engine/engine/download-engine-multi-download.js

 import { EventEmitter } from "eventemitter3";
 import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js";
 import DownloadAlreadyStartedError from "./error/download-already-started-error.js";
+import { PromisePool } from "@supercharge/promise-pool";
+const DEFAULT_PARALLEL_DOWNLOADS = 1;
 export default class DownloadEngineMultiDownload extends EventEmitter {
     downloads;
+    options;
     _aborted = false;
-    _activeEngine;
+    _activeEngines = new Set();
     _progressStatisticsBuilder = new ProgressStatisticsBuilder();
     _downloadStatues = [];
     _closeFiles = [];
-    constructor(engines) {
+    constructor(engines, options) {
         super();
         this.downloads = DownloadEngineMultiDownload._extractEngines(engines);
+        this.options = options;
         this._init();
@@ -37,14 +41,18 @@ }
     async download() {
-        if (this._activeEngine) {
+        if (this._activeEngines.size) {
             throw new DownloadAlreadyStartedError();
         }
         this.emit("start");
-        for (const engine of this.downloads) {
+        await PromisePool
+            .withConcurrency(this.options.parallelDownloads ?? DEFAULT_PARALLEL_DOWNLOADS)
+            .for(this.downloads)
+            .process(async (engine) => {
             if (this._aborted)
                 return;
-            this._activeEngine = engine;
+            this._activeEngines.add(engine);
             this.emit("childDownloadStarted", engine);
             await engine.download();
             this.emit("childDownloadClosed", engine);
-        }
+            this._activeEngines.delete(engine);
+        });
         this.emit("finished");
@@ -72,6 +80,6 @@ await this._finishEnginesDownload();
     pause() {
-        this._activeEngine?.pause();
+        this._activeEngines.forEach(engine => engine.pause());
     }
     resume() {
-        this._activeEngine?.resume();
+        this._activeEngines.forEach(engine => engine.resume());
     }
@@ -82,3 +90,5 @@ async close() {
         this._aborted = true;
-        await this._activeEngine?.close();
+        const closePromises = Array.from(this._activeEngines)
+            .map(engine => engine.close());
+        await Promise.all(closePromises);
         this.emit("closed");
@@ -95,6 +105,6 @@ }
     }
-    static async fromEngines(engines) {
-        return new DownloadEngineMultiDownload(await Promise.all(engines));
+    static async fromEngines(engines, options = {}) {
+        return new DownloadEngineMultiDownload(await Promise.all(engines), options);
     }
 }
 //# sourceMappingURL=download-engine-multi-download.js.map

dist/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.d.ts

 import retry from "async-retry";
 import { EventEmitter } from "eventemitter3";
 import { AvailablePrograms } from "../../download-file/download-programs/switch-program.js";
+export declare const MIN_LENGTH_FOR_MORE_INFO_REQUEST: number;
 export type BaseDownloadEngineFetchStreamOptions = {
@@ -5,0 +6,0 @@ retry?: retry.Options;

dist/download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js

@@ -5,2 +5,3 @@ import retry from "async-retry";
 import HttpError from "./errors/http-error.js";
+export const MIN_LENGTH_FOR_MORE_INFO_REQUEST = 1024 * 1024 * 3; // 3MB
 const DEFAULT_OPTIONS = {
@@ -7,0 +8,0 @@ retry: {

dist/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.d.ts

@@ -8,4 +8,5 @@ import BaseDownloadEngineFetchStream, { DownloadInfoResponse, FetchSubState, WriteCallback } from "./base-download-engine-fetch-stream.js";
     protected fetchDownloadInfoWithoutRetry(url: string): Promise<DownloadInfoResponse>;
+    protected fetchDownloadInfoWithoutRetryContentRange(url: string): Promise<number>;
     chunkGenerator(callback: WriteCallback, getNextChunk: GetNextChunk): Promise<void>;
 }
 export {};

dist/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js

@@ -1,2 +0,2 @@
-import BaseDownloadEngineFetchStream from "./base-download-engine-fetch-stream.js";
+import BaseDownloadEngineFetchStream, { MIN_LENGTH_FOR_MORE_INFO_REQUEST } from "./base-download-engine-fetch-stream.js";
 import InvalidContentLengthError from "./errors/invalid-content-length-error.js";
@@ -6,2 +6,4 @@ import SmartChunkSplit from "./utils/smart-chunk-split.js";
 import StatusCodeError from "./errors/status-code-error.js";
+import { parseHttpContentRange } from "./utils/httpRange.js";
+import { browserCheck } from "./utils/browserCheck.js";
 export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream {
@@ -16,3 +18,2 @@ transferAction = "Downloading";
             accept: "*/*",
-            "Accept-Encoding": "identity",
             ...this.options.headers
@@ -31,3 +32,3 @@ };
         }
-        const contentLength = parseInt(response.headers.get("content-length"));
+        const contentLength = parseHttpContentRange(response.headers.get("content-range"))?.length ?? parseInt(response.headers.get("content-length"));
         const expectedContentLength = this._endSize - this._startSize;
@@ -54,8 +55,8 @@ if (this.state.rangeSupport && contentLength !== expectedContentLength) {
         }
+        const acceptRange = this.options.acceptRangeIsKnown ?? response.headers.get("accept-ranges") === "bytes";
+        const fileName = parseContentDisposition(response.headers.get("content-disposition"));
         let length = parseInt(response.headers.get("content-length"));
-        if (response.headers.get("content-encoding")) {
-            length = 0;
+        if (response.headers.get("content-encoding") || browserCheck() && MIN_LENGTH_FOR_MORE_INFO_REQUEST < length) {
+            length = acceptRange ? await this.fetchDownloadInfoWithoutRetryContentRange(url) : 0;
         }
-        const acceptRange = this.options.acceptRangeIsKnown ?? response.headers.get("accept-ranges") === "bytes";
-        const fileName = parseContentDisposition(response.headers.get("content-disposition"));
         return {
@@ -68,2 +69,14 @@ length,
     }
+    async fetchDownloadInfoWithoutRetryContentRange(url) {
+        const responseGet = await fetch(url, {
+            method: "GET",
+            headers: {
+                accept: "*/*",
+                ...this.options.headers,
+                range: "bytes=0-0"
+            }
+        });
+        const contentRange = responseGet.headers.get("content-range");
+        return parseHttpContentRange(contentRange)?.size || 0;
+    }
     async chunkGenerator(callback, getNextChunk) {
@@ -70,0 +83,0 @@ const smartSplit = new SmartChunkSplit(callback, this.state);

dist/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.d.ts

@@ -14,2 +14,3 @@ import BaseDownloadEngineFetchStream, { DownloadInfoResponse, FetchSubState, WriteCallback } from "./base-download-engine-fetch-stream.js";
     protected fetchDownloadInfoWithoutRetry(url: string): Promise<DownloadInfoResponse>;
+    protected fetchDownloadInfoWithoutRetryContentRange(url: string): Promise<number>;
 }

dist/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.js

@@ -1,2 +0,2 @@
-import BaseDownloadEngineFetchStream from "./base-download-engine-fetch-stream.js";
+import BaseDownloadEngineFetchStream, { MIN_LENGTH_FOR_MORE_INFO_REQUEST } from "./base-download-engine-fetch-stream.js";
 import EmptyResponseError from "./errors/empty-response-error.js";
@@ -8,2 +8,3 @@ import StatusCodeError from "./errors/status-code-error.js";
 import { parseContentDisposition } from "./utils/content-disposition.js";
+import { parseHttpContentRange } from "./utils/httpRange.js";
 export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream {
@@ -25,3 +26,2 @@ programType = "chunks";
                 accept: "*/*",
-                "Accept-Encoding": "identity",
                 ...this.options.headers
@@ -91,3 +91,3 @@ };
             const result = await this.fetchBytes(this.state.url, 0, this._endSize, this.state.onProgress);
-            return result.slice(this._startSize, this._endSize);
+            return result.slice(this._startSize, this._endSize || result.length);
         })();
@@ -112,3 +112,2 @@ let totalReceivedLength = 0;
             const allHeaders = {
-                "Accept-Encoding": "identity",
                 ...this.options.headers
@@ -119,12 +118,10 @@ };
             }
-            xhr.onload = () => {
+            xhr.onload = async () => {
                 if (xhr.status >= 200 && xhr.status < 300) {
-                    let length = xhr.getResponseHeader("Content-Length") || "-";
+                    const contentLength = parseInt(xhr.getResponseHeader("content-length"));
+                    const length = MIN_LENGTH_FOR_MORE_INFO_REQUEST < contentLength ? await this.fetchDownloadInfoWithoutRetryContentRange(url) : 0;
                     const fileName = parseContentDisposition(xhr.getResponseHeader("content-disposition"));
-                    if (xhr.getResponseHeader("Content-Encoding")) {
-                        length = "0";
-                    }
                     const acceptRange = this.options.acceptRangeIsKnown ?? xhr.getResponseHeader("Accept-Ranges") === "bytes";
                     resolve({
-                        length: parseInt(length),
+                        length,
                         acceptRange,
@@ -145,3 +142,25 @@ newURL: xhr.responseURL,
     }
+    fetchDownloadInfoWithoutRetryContentRange(url) {
+        return new Promise((resolve, reject) => {
+            const xhr = new XMLHttpRequest();
+            xhr.open("GET", url, true);
+            const allHeaders = {
+                accept: "*/*",
+                ...this.options.headers,
+                range: "bytes=0-0"
+            };
+            for (const [key, value] of Object.entries(allHeaders)) {
+                xhr.setRequestHeader(key, value);
+            }
+            xhr.onload = () => {
+                const contentRange = xhr.getResponseHeader("Content-Range");
+                resolve(parseHttpContentRange(contentRange)?.size || 0);
+            };
+            xhr.onerror = () => {
+                reject(new XhrError(`Failed to fetch ${url}`));
+            };
+            xhr.send();
+        });
+    }
 }
 //# sourceMappingURL=download-engine-fetch-stream-xhr.js.map

dist/download/download-engine/streams/download-engine-fetch-stream/errors/invalid-content-length-error.d.ts

 import HttpError from "./http-error.js";
 export default class InvalidContentLengthError extends HttpError {
-    constructor(expectedLength: number, gotLength: number);
+    constructor(expectedLength: number, gotLength: number | string);
 }

dist/download/node-download.d.ts

 import DownloadEngineNodejs, { DownloadEngineOptionsNodejs } from "./download-engine/engine/download-engine-nodejs.js";
 import BaseDownloadEngine from "./download-engine/engine/base-download-engine.js";
-import DownloadEngineMultiDownload from "./download-engine/engine/download-engine-multi-download.js";
+import DownloadEngineMultiDownload, { DownloadEngineMultiDownloadOptions } from "./download-engine/engine/download-engine-multi-download.js";
 import { CliProgressDownloadEngineOptions } from "./transfer-visualize/transfer-cli/cli-animation-wrapper.js";
@@ -13,3 +13,3 @@ export type DownloadFileOptions = DownloadEngineOptionsNodejs & CliProgressDownloadEngineOptions & {
 export declare function downloadFile(options: DownloadFileOptions): Promise<DownloadEngineNodejs<import("./download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.js").default>>;
-export type DownloadSequenceOptions = CliProgressDownloadEngineOptions & {
+export type DownloadSequenceOptions = CliProgressDownloadEngineOptions & DownloadEngineMultiDownloadOptions & {
     fetchStrategy?: "localFile" | "fetch";
@@ -16,0 +16,0 @@ };

dist/download/node-download.js

@@ -33,3 +33,3 @@ import DownloadEngineNodejs from "./download-engine/engine/download-engine-nodejs.js";
     downloadOptions.cliLevel = CLI_LEVEL.HIGH;
-    const downloader = DownloadEngineMultiDownload.fromEngines(downloads);
+    const downloader = DownloadEngineMultiDownload.fromEngines(downloads, downloadOptions);
     const wrapper = new CliAnimationWrapper(downloader, downloadOptions);
@@ -36,0 +36,0 @@ await wrapper.attachAnimation();

package.json

 {
   "name": "ipull",
-  "version": "3.2.2",
+  "version": "3.3.0",
   "description": "The only file downloader you'll ever need. For node.js and the browser, CLI and library for fast and reliable file downloads.",
@@ -127,2 +127,3 @@ "main": "dist/index.js",
   "dependencies": {
+    "@supercharge/promise-pool": "^3.2.0",
     "@tinyhttp/content-disposition": "^2.2.0",
@@ -129,0 +130,0 @@ "async-retry": "^1.3.3",

README.md

@@ -282,2 +282,3 @@ <div align="center">
         cliProgress: true,
+        // parallelDownloads: 2, download 2 files in parallel, default is 1
     },
@@ -284,0 +285,0 @@ downloadFile({

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

258583

increased by2.85%

Number of package files

195

increased by3.17%

Lines of code

3442

increased by2.41%

Number of lines in readme file

325

increased by0.31%

Worsened metrics

Dependency count

19

increased by5.56%

Number of medium supply chain risk alerts

3

increased by50%

Dependency changes

• + Added@supercharge/promise-pool@^3.2.0

• + Added@supercharge/promise-pool@3.2.0(transitive)