ipull - npm Package Compare versions

Comparing version 3.0.12 to 3.1.0

dist/cli/cli.js

 #!/usr/bin/env node
 import path from "path";
-import { Command } from "commander";
+import { Command, Option } from "commander";
 import { packageJson } from "../const.js";
@@ -14,4 +14,5 @@ import { downloadFile, downloadSequence } from "../download/node-download.js";
     .option("-c --connections [number]", "Number of parallel connections", "4")
+    .addOption(new Option("-p --program [type]", "The download strategy").choices(["stream", "chunks"]))
     .option("-t --truncate-name", "Truncate file names in the CLI status to make them appear shorter")
-    .action(async (files = [], { save: saveLocation, truncateName, number }) => {
+    .action(async (files = [], { save: saveLocation, truncateName, number, program }) => {
     if (files.length === 0) {
@@ -30,3 +31,4 @@ pullCommand.outputHelp();
             truncateName,
-            parallelStreams: Number(number) || 4
+            parallelStreams: Number(number) || 4,
+            programType: program
         });
@@ -33,0 +35,0 @@ }));

dist/download/download-engine/download-file/download-engine-file.d.ts

@@ -53,2 +53,3 @@ import ProgressStatusFile, { DownloadStatus, ProgressStatus } from "./progress-status-file.js";
     private get _downloadedPartsSize();
+    private get _activeDownloadedChunkSize();
     get transferredBytes(): number;
@@ -59,3 +60,3 @@ protected _emptyChunksForPart(part: number): any[];
     protected _downloadSlice(startChunk: number, endChunk: number): Promise<void>;
-    protected _saveProgress(): Promise<void>;
+    protected _saveProgress(): Promise<void> | undefined;
     protected _sendProgressDownloadPart(): void;
@@ -62,0 +63,0 @@ pause(): void;

dist/download/download-engine/download-file/download-engine-file.js

@@ -28,3 +28,3 @@ import ProgressStatusFile, { DownloadStatus } from "./progress-status-file.js";
         this.file = file;
-        this._progressStatus = new ProgressStatusFile(file.totalSize, file.parts.length, file.localFileName, options.comment, options.fetchStream.transferAction);
+        this._progressStatus = new ProgressStatusFile(file.parts.length, file.localFileName, options.comment, options.fetchStream.transferAction);
         this.options = { ...DEFAULT_OPTIONS, ...options };
@@ -34,3 +34,3 @@ this._initProgress();
     get downloadSize() {
-        return this.file.totalSize;
+        return this.file.parts.reduce((acc, part) => acc + part.size, 0);
     }
@@ -41,3 +41,3 @@ get fileName() {
     get status() {
-        return this._progressStatus.createStatus(this._progress.part + 1, this.transferredBytes, this._downloadStatus);
+        return this._progressStatus.createStatus(this._progress.part + 1, this.transferredBytes, this.downloadSize, this._downloadStatus);
     }
@@ -51,12 +51,21 @@ get _activePart() {
     }
+    get _activeDownloadedChunkSize() {
+        return this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE).length * this._progress.chunkSize;
+    }
     get transferredBytes() {
-        const activeDownloadBytes = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE).length * this._progress.chunkSize;
         const streamingBytes = Object.values(this._activeStreamBytes)
             .reduce((acc, bytes) => acc + bytes, 0);
         const partNotFinishedYet = this._downloadStatus !== DownloadStatus.Finished;
-        const chunksBytes = (partNotFinishedYet ? activeDownloadBytes : 0) + this._downloadedPartsSize;
-        return Math.min(chunksBytes + streamingBytes, this.downloadSize);
+        const chunksBytes = (partNotFinishedYet ? this._activeDownloadedChunkSize : 0) + this._downloadedPartsSize;
+        const allBytes = chunksBytes + streamingBytes;
+        if (this._activePart.size === 0) {
+            return allBytes;
+        }
+        return Math.min(allBytes, this.downloadSize);
     }
     _emptyChunksForPart(part) {
         const partInfo = this.file.parts[part];
+        if (!partInfo.acceptRange) {
+            return [ChunkStatus.NOT_STARTED];
+        }
         const chunksCount = Math.ceil(partInfo.size / this.options.chunkSize);
@@ -91,6 +100,2 @@ return new Array(chunksCount).fill(ChunkStatus.NOT_STARTED);
             }
-            // If the part does not support range, we can only download it with a single stream
-            if (!this._activePart.acceptRange) {
-                this._progress.parallelStreams = 1;
-            }
             // Reset in progress chunks
@@ -100,4 +105,9 @@ this._progress.chunks = this._progress.chunks.map(chunk => (chunk === ChunkStatus.COMPLETE ? ChunkStatus.COMPLETE : ChunkStatus.NOT_STARTED));
             this._activeStreamBytes = {};
-            this._activeProgram = switchProgram(this._progress, this._downloadSlice.bind(this), this.options.fetchStream.programType || this.options.programType);
-            await this._activeProgram.download();
+            if (this._activePart.acceptRange) {
+                this._activeProgram = switchProgram(this._progress, this._downloadSlice.bind(this), this.options.fetchStream.programType || this.options.programType);
+                await this._activeProgram.download();
+            }
+            else {
+                await this._downloadSlice(0, Infinity);
+            }
         }
@@ -145,2 +155,6 @@ // All parts are downloaded, we can clear the progress
                 }
+                // if content length is 0, we do not know how many chunks we should have
+                if (this._activePart.size === 0) {
+                    this._progress.chunks.push(ChunkStatus.NOT_STARTED);
+                }
                 this._progress.chunks[index] = ChunkStatus.COMPLETE;
@@ -158,2 +172,6 @@ delete this._activeStreamBytes[startChunk];
             });
+            if (this._activePart.size === 0) {
+                this._activePart.size = this._activeDownloadedChunkSize;
+                this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE);
+            }
             delete this._activeStreamBytes[startChunk];
@@ -166,2 +184,4 @@ await Promise.all(allWrites);
         this._sendProgressDownloadPart();
+        if (!this._activePart.acceptRange)
+            return;
         this.emit("save", this._progress);
@@ -168,0 +188,0 @@ return withLock(this, "_saveLock", async () => {

dist/download/download-engine/download-file/progress-status-file.d.ts

@@ -21,3 +21,2 @@ export type ProgressStatus = {
 export default class ProgressStatusFile {
-    readonly totalBytes: number;
     readonly totalDownloadParts: number;
@@ -30,8 +29,9 @@ readonly fileName: string;
     readonly downloadStatus: DownloadStatus;
+    totalBytes: number;
     startTime: number;
     endTime: number;
-    constructor(totalBytes: number, totalDownloadParts: number, fileName: string, comment?: string, transferAction?: string, downloadPart?: number, transferredBytes?: number, downloadStatus?: DownloadStatus);
+    constructor(totalDownloadParts: number, fileName: string, comment?: string, transferAction?: string, downloadPart?: number, transferredBytes?: number, downloadStatus?: DownloadStatus);
     started(): void;
     finished(): void;
-    createStatus(downloadPart: number, transferredBytes: number, downloadStatus?: DownloadStatus): ProgressStatusFile;
+    createStatus(downloadPart: number, transferredBytes: number, totalBytes?: number, downloadStatus?: DownloadStatus): ProgressStatusFile;
 }

dist/download/download-engine/download-file/progress-status-file.js

@@ -10,3 +10,2 @@ export var DownloadStatus;
 export default class ProgressStatusFile {
-    totalBytes;
     totalDownloadParts;
@@ -19,5 +18,6 @@ fileName;
     downloadStatus = DownloadStatus.Active;
+    totalBytes = 0;
     startTime = 0;
     endTime = 0;
-    constructor(totalBytes, totalDownloadParts, fileName, comment, transferAction = "Transferring", downloadPart = 0, transferredBytes = 0, downloadStatus = DownloadStatus.Active) {
+    constructor(totalDownloadParts, fileName, comment, transferAction = "Transferring", downloadPart = 0, transferredBytes = 0, downloadStatus = DownloadStatus.Active) {
         this.transferAction = transferAction;
@@ -29,3 +29,2 @@ this.transferredBytes = transferredBytes;
         this.totalDownloadParts = totalDownloadParts;
-        this.totalBytes = totalBytes;
         this.downloadStatus = downloadStatus;
@@ -39,4 +38,5 @@ }
     }
-    createStatus(downloadPart, transferredBytes, downloadStatus = DownloadStatus.Active) {
-        const newStatus = new ProgressStatusFile(this.totalBytes, this.totalDownloadParts, this.fileName, this.comment, this.transferAction, downloadPart, transferredBytes, downloadStatus);
+    createStatus(downloadPart, transferredBytes, totalBytes = this.totalBytes, downloadStatus = DownloadStatus.Active) {
+        const newStatus = new ProgressStatusFile(this.totalDownloadParts, this.fileName, this.comment, this.transferAction, downloadPart, transferredBytes, downloadStatus);
+        newStatus.totalBytes = totalBytes;
         newStatus.startTime = this.startTime;
@@ -43,0 +43,0 @@ newStatus.endTime = this.endTime;

dist/download/download-engine/engine/base-download-engine.js

@@ -7,2 +7,4 @@ import UrlInputError from "./error/url-input-error.js";
 import InvalidContentLengthError from "./error/invalid-content-length-error.js";
+import StatusCodeError from "../streams/download-engine-fetch-stream/errors/status-code-error.js";
+const IGNORE_HEAD_STATUS_CODES = [405, 501, 404];
 export default class BaseDownloadEngine extends EventEmitter {
@@ -25,3 +27,3 @@ options;
     get downloadSize() {
-        return this.file.totalSize;
+        return this._engine.downloadSize;
     }
@@ -98,15 +100,29 @@ get fileName() {
         for (const part of parts) {
-            const { length, acceptRange, newURL, fileName } = await fetchStream.fetchDownloadInfo(part);
-            const downloadURL = newURL ?? part;
-            if (isNaN(length)) {
-                throw new InvalidContentLengthError(downloadURL);
+            try {
+                const { length, acceptRange, newURL, fileName } = await fetchStream.fetchDownloadInfo(part);
+                const downloadURL = newURL ?? part;
+                if (acceptRange && isNaN(length)) {
+                    throw new InvalidContentLengthError(downloadURL);
+                }
+                downloadFile.totalSize += length || 0;
+                downloadFile.parts.push({
+                    downloadURL,
+                    size: length || 0,
+                    acceptRange
+                });
+                if (counter++ === 0 && fileName) {
+                    downloadFile.localFileName = fileName;
+                }
             }
-            downloadFile.totalSize += length;
-            downloadFile.parts.push({
-                downloadURL,
-                size: length,
-                acceptRange
-            });
-            if (counter++ === 0 && fileName) {
-                downloadFile.localFileName = fileName;
+            catch (error) {
+                if (error instanceof StatusCodeError && IGNORE_HEAD_STATUS_CODES.includes(error.statusCode)) {
+                    // if the server does not support HEAD request, we will skip that step
+                    downloadFile.parts.push({
+                        downloadURL: part,
+                        size: 0,
+                        acceptRange: false
+                    });
+                    continue;
+                }
+                throw error;
             }
@@ -113,0 +129,0 @@ }

dist/download/download-engine/engine/download-engine-nodejs.js

@@ -26,3 +26,3 @@ import path from "path";
         this._engine.options.onFinishAsync = async () => {
-            await this.options.writeStream.ftruncate();
+            await this.options.writeStream.ftruncate(this.downloadSize);
         };
@@ -29,0 +29,0 @@ this._engine.options.onCloseAsync = async () => {

dist/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js

@@ -15,5 +15,7 @@ import BaseDownloadEngineFetchStream from "./base-download-engine-fetch-stream.js";
             accept: "*/*",
-            ...this.options.headers,
-            range: `bytes=${this._startSize}-${this._endSize - 1}`
+            ...this.options.headers
         };
+        if (this.state.rangeSupport) {
+            headers.range = `bytes=${this._startSize}-${this._endSize - 1}`;
+        }
         const controller = new AbortController();
@@ -29,3 +31,3 @@ const response = await fetch(this.appendToURL(this.state.url), {
         const expectedContentLength = this._endSize - this._startSize;
-        if (contentLength !== expectedContentLength) {
+        if (this.state.rangeSupport && contentLength !== expectedContentLength) {
             throw new InvalidContentLengthError(expectedContentLength, contentLength);
@@ -32,0 +34,0 @@ }

dist/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.js

@@ -24,5 +24,7 @@ import BaseDownloadEngineFetchStream from "./base-download-engine-fetch-stream.js";
                 accept: "*/*",
-                ...this.options.headers,
-                range: `bytes=${start}-${end - 1}` // get the range up to end-1. Length 2: 0-1
+                ...this.options.headers
             };
+            if (this.state.rangeSupport) {
+                headers.range = `bytes=${start}-${end - 1}`;
+            }
             const xhr = new XMLHttpRequest();
@@ -34,5 +36,5 @@ xhr.responseType = "arraybuffer";
             }
-            xhr.onload = function () {
+            xhr.onload = () => {
                 const contentLength = parseInt(xhr.getResponseHeader("content-length"));
-                if (contentLength !== end - start) {
+                if (this.state.rangeSupport && contentLength !== end - start) {
                     throw new InvalidContentLengthError(end - start, contentLength);
@@ -53,6 +55,6 @@ }
             };
-            xhr.onerror = function () {
+            xhr.onerror = () => {
                 reject(new XhrError(`Failed to fetch ${url}`));
             };
-            xhr.onprogress = function (event) {
+            xhr.onprogress = (event) => {
                 if (event.lengthComputable) {
@@ -59,0 +61,0 @@ onProgress?.(event.loaded);

dist/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.d.ts

@@ -14,5 +14,5 @@ import retry from "async-retry";
     protected _bytesWritten: number;
-    get writerClosed(): boolean;
+    get writerClosed(): boolean | 0 | undefined;
     constructor(_writer?: DownloadEngineWriteStreamBrowserWriter, options?: DownloadEngineWriteStreamOptionsBrowser);
-    protected _ensureBuffer(): Uint8Array;
+    protected _ensureBuffer(length: number): Uint8Array;
     write(cursor: number, buffer: Uint8Array): void | Promise<void>;
@@ -19,0 +19,0 @@ get result(): Uint8Array;

dist/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.js

@@ -10,3 +10,3 @@ import BaseDownloadEngineWriteStream from "./base-download-engine-write-stream.js";
     get writerClosed() {
-        return this._bytesWritten === this.options.file?.totalSize;
+        return this.options.file?.totalSize && this._bytesWritten === this.options.file.totalSize;
     }
@@ -18,4 +18,4 @@ constructor(_writer, options = {}) {
     }
-    _ensureBuffer() {
-        if (this._memory.length > 0) {
+    _ensureBuffer(length) {
+        if (this._memory.length >= length) {
             return this._memory;
@@ -26,3 +26,6 @@ }
         }
-        return this._memory = new Uint8Array(this.options.file.totalSize);
+        const newSize = Math.max(length, this.options.file.totalSize);
+        const newMemory = new Uint8Array(newSize);
+        newMemory.set(this._memory);
+        return this._memory = newMemory;
     }
@@ -34,3 +37,3 @@ write(cursor, buffer) {
         if (!this._writer) {
-            this._ensureBuffer()
+            this._ensureBuffer(cursor + buffer.byteLength)
                 .set(buffer, cursor);
@@ -37,0 +40,0 @@ this._bytesWritten += buffer.byteLength;

dist/download/transfer-visualize/format-transfer-status.js

@@ -35,5 +35,5 @@ import prettyBytes from "pretty-bytes";
     const formatTransferred = prettyBytes(fullStatus.transferredBytes, PRETTY_BYTES_OPTIONS);
-    const formatTotal = prettyBytes(fullStatus.totalBytes, PRETTY_BYTES_OPTIONS);
+    const formatTotal = fullStatus.totalBytes === 0 ? "???" : prettyBytes(fullStatus.totalBytes, PRETTY_BYTES_OPTIONS);
     const formatTransferredOfTotal = `${formatTransferred}/${formatTotal}`;
-    const formatTimeLeft = prettyMilliseconds(fullStatus.timeLeft, PRETTY_MS_OPTIONS);
+    const formatTimeLeft = fullStatus.totalBytes === 0 ? "unknown time" : prettyMilliseconds(fullStatus.timeLeft, PRETTY_MS_OPTIONS);
     const formattedPercentage = fullStatus.percentage.toLocaleString(DEFAULT_LOCALIZATION, {
@@ -40,0 +40,0 @@ minimumIntegerDigits: 1,

dist/download/transfer-visualize/transfer-statistics.js

@@ -38,3 +38,3 @@ import { clamp } from "./utils/numbers.js";
     updateProgress(transferred, total) {
-        const speed = clamp(this._calculateSpeed(transferred), 0);
+        const speed = clamp(this._calculateSpeed(transferred));
         const timeLeft = (total - transferred) / speed;
@@ -41,0 +41,0 @@ const timeLeftFinalNumber = clamp((timeLeft || 0) * 1000, 0, MAX_TIME_LEFT);

dist/download/transfer-visualize/utils/numbers.js

@@ -1,4 +0,4 @@
-export function clamp(value, min = Number.MIN_VALUE, max = Number.MAX_VALUE) {
+export function clamp(value, min = 0, max = Number.MAX_VALUE) {
     return Math.min(Math.max(value, min), max);
 }
 //# sourceMappingURL=numbers.js.map

package.json

 {
   "name": "ipull",
-  "version": "3.0.12",
+  "version": "3.1.0",
   "description": "The only file downloader you'll ever need. For node.js and the browser, CLI and library for fast and reliable file downloads.",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

README.md

@@ -96,5 +96,7 @@ <div align="center">
 Options:
+  -s --save [path]              Save location (directory/file)
+  -c --connections [number]     Number of parallel connections (default: "4")
+  -p --program [type]           The download strategy (choices: "stream", "chunks")
+  -t --truncate-name            Truncate file names in the CLI status to make them appear shorter
   -V, --version                 output the version number
-  -s --save [path]              Save location (directory/file)
-  -f --full-name                Show full name of the file while downloading, even if it long
   -h, --help                    display help for command
@@ -104,2 +106,3 @@
   set [options] [path] <value>  Set download locations
+
 ```
@@ -106,0 +109,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

244928

increased by1.76%

Lines of code

3266

increased by1.43%

Number of lines in readme file

309

increased by0.98%

No dependency changes