ipull - npm Package Compare versions

Comparing version 3.1.1 to 3.2.0

dist/download/browser-download.d.ts

@@ -14,2 +14,2 @@ import DownloadEngineBrowser, { DownloadEngineOptionsBrowser } from "./download-engine/engine/download-engine-browser.js";
  */
-export declare function downloadSequenceBrowser(...downloads: (DownloadEngineBrowser | Promise<DownloadEngineBrowser>)[]): Promise<DownloadEngineMultiDownload<import("./download-engine/download-file/download-engine-file.js").default | import("./download-engine/engine/base-download-engine.js").default>>;
+export declare function downloadSequenceBrowser(...downloads: (DownloadEngineBrowser | Promise<DownloadEngineBrowser>)[]): Promise<DownloadEngineMultiDownload<import("./download-engine/engine/base-download-engine.js").default>>;

dist/download/download-engine/download-file/download-engine-file.d.ts

@@ -20,2 +20,4 @@ import ProgressStatusFile, { DownloadStatus, ProgressStatus } from "./progress-status-file.js";
     programType?: AvailablePrograms;
+    /** @internal */
+    skipExisting?: boolean;
 };
@@ -49,2 +51,3 @@ export type DownloadEngineFileOptionsWithDefaults = DownloadEngineFileOptions & {
     constructor(file: DownloadFile, options: DownloadEngineFileOptions);
+    private _createProgressFlags;
     get downloadSize(): number;
@@ -51,0 +54,0 @@ get fileName(): string;

dist/download/download-engine/download-file/download-engine-file.js

@@ -1,2 +0,2 @@
-import ProgressStatusFile, { DownloadStatus } from "./progress-status-file.js";
+import ProgressStatusFile, { DownloadFlags, DownloadStatus } from "./progress-status-file.js";
 import { ChunkStatus } from "../types.js";
@@ -6,2 +6,3 @@ import { EventEmitter } from "eventemitter3";
 import switchProgram from "./download-programs/switch-program.js";
+import { pushComment } from "./utils/push-comment.js";
 const DEFAULT_OPTIONS = {
@@ -29,6 +30,13 @@ chunkSize: 1024 * 1024 * 5,
         this.file = file;
-        this._progressStatus = new ProgressStatusFile(file.parts.length, file.localFileName, options.comment, options.fetchStream.transferAction);
         this.options = { ...DEFAULT_OPTIONS, ...options };
+        this._progressStatus = new ProgressStatusFile(file.parts.length, file.localFileName, options.fetchStream.transferAction, this._createProgressFlags());
         this._initProgress();
     }
+    _createProgressFlags() {
+        const flags = [];
+        if (this.options.skipExisting) {
+            flags.push(DownloadFlags.Existing);
+        }
+        return flags;
+    }
     get downloadSize() {
@@ -41,3 +49,3 @@ return this.file.parts.reduce((acc, part) => acc + part.size, 0);
     get status() {
-        return this._progressStatus.createStatus(this._progress.part + 1, this.transferredBytes, this.downloadSize, this._downloadStatus);
+        return this._progressStatus.createStatus(this._progress.part + 1, this.transferredBytes, this.downloadSize, this._downloadStatus, this.options.comment);
     }
@@ -55,11 +63,11 @@ get _activePart() {
     get transferredBytes() {
+        if (this._downloadStatus === DownloadStatus.Finished) {
+            return this.downloadSize;
+        }
         const streamingBytes = Object.values(this._activeStreamBytes)
             .reduce((acc, bytes) => acc + bytes, 0);
-        const partNotFinishedYet = this._downloadStatus !== DownloadStatus.Finished;
-        const chunksBytes = (partNotFinishedYet ? this._activeDownloadedChunkSize : 0) + this._downloadedPartsSize;
-        const allBytes = chunksBytes + streamingBytes;
-        if (this._activePart.size === 0) {
-            return allBytes;
-        }
-        return Math.min(allBytes, this.downloadSize);
+        const streamBytes = this._activeDownloadedChunkSize + streamingBytes;
+        const streamBytesMin = Math.min(streamBytes, this._activePart.size || streamBytes);
+        const allBytes = streamBytesMin + this._downloadedPartsSize;
+        return Math.min(allBytes, this.downloadSize || allBytes);
     }
@@ -75,3 +83,8 @@ _emptyChunksForPart(part) {
     _initProgress() {
-        if (this.file.downloadProgress) {
+        if (this.options.skipExisting) {
+            this._progress.part = this.file.parts.length - 1;
+            this._downloadStatus = DownloadStatus.Finished;
+            this.options.comment = pushComment("Skipping existing", this.options.comment);
+        }
+        else if (this.file.downloadProgress) {
             this._progress = this.file.downloadProgress;
@@ -89,5 +102,5 @@ }
     async download() {
+        this._progressStatus.started();
         this.emit("start");
-        this._progressStatus.started();
-        for (let i = this._progress.part; i < this.file.parts.length; i++) {
+        for (let i = this._progress.part; i < this.file.parts.length && this._downloadStatus !== DownloadStatus.Finished; i++) {
             if (this._closed)
@@ -141,2 +154,3 @@ return;
             const allWrites = new Set();
+            let lastChunkSize = 0;
             await fetchState.fetchChunks((chunks, writePosition, index) => {
@@ -161,2 +175,3 @@ if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) {
                 this._progress.chunks[index] = ChunkStatus.COMPLETE;
+                lastChunkSize = this._activeStreamBytes[startChunk];
                 delete this._activeStreamBytes[startChunk];
@@ -173,4 +188,5 @@ void this._saveProgress();
             });
+            // On dynamic content length, we need to adjust the last chunk size
             if (this._activePart.size === 0) {
-                this._activePart.size = this._activeDownloadedChunkSize;
+                this._activePart.size = this._activeDownloadedChunkSize - this.options.chunkSize + lastChunkSize;
                 this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE);
@@ -177,0 +193,0 @@ }

dist/download/download-engine/download-file/progress-status-file.d.ts

@@ -12,2 +12,3 @@ export type ProgressStatus = {
     downloadStatus: DownloadStatus;
+    downloadFlags: DownloadFlags[];
 };
@@ -21,2 +22,5 @@ export declare enum DownloadStatus {
 }
+export declare enum DownloadFlags {
+    Existing = "Existing"
+}
 export default class ProgressStatusFile {
@@ -30,9 +34,10 @@ readonly totalDownloadParts: number;
     readonly downloadStatus: DownloadStatus;
+    downloadFlags: DownloadFlags[];
     totalBytes: number;
     startTime: number;
     endTime: number;
-    constructor(totalDownloadParts: number, fileName: string, comment?: string, transferAction?: string, downloadPart?: number, transferredBytes?: number, downloadStatus?: DownloadStatus);
+    constructor(totalDownloadParts: number, fileName: string, transferAction?: string, downloadFlags?: DownloadFlags[], comment?: string, downloadPart?: number, transferredBytes?: number, downloadStatus?: DownloadStatus);
     started(): void;
     finished(): void;
-    createStatus(downloadPart: number, transferredBytes: number, totalBytes?: number, downloadStatus?: DownloadStatus): ProgressStatusFile;
+    createStatus(downloadPart: number, transferredBytes: number, totalBytes?: number, downloadStatus?: DownloadStatus, comment?: string | undefined): ProgressStatusFile;
 }

dist/download/download-engine/download-file/progress-status-file.js

@@ -9,2 +9,6 @@ export var DownloadStatus;
 })(DownloadStatus || (DownloadStatus = {}));
+export var DownloadFlags;
+(function (DownloadFlags) {
+    DownloadFlags["Existing"] = "Existing";
+})(DownloadFlags || (DownloadFlags = {}));
 export default class ProgressStatusFile {
@@ -18,6 +22,7 @@ totalDownloadParts;
     downloadStatus = DownloadStatus.Active;
+    downloadFlags = [];
     totalBytes = 0;
     startTime = 0;
     endTime = 0;
-    constructor(totalDownloadParts, fileName, comment, transferAction = "Transferring", downloadPart = 0, transferredBytes = 0, downloadStatus = DownloadStatus.Active) {
+    constructor(totalDownloadParts, fileName, transferAction = "Transferring", downloadFlags = [], comment, downloadPart = 0, transferredBytes = 0, downloadStatus = DownloadStatus.Active) {
         this.transferAction = transferAction;
@@ -29,2 +34,3 @@ this.transferredBytes = transferredBytes;
         this.totalDownloadParts = totalDownloadParts;
+        this.downloadFlags = downloadFlags;
         this.downloadStatus = downloadStatus;
@@ -38,4 +44,4 @@ }
     }
-    createStatus(downloadPart, transferredBytes, totalBytes = this.totalBytes, downloadStatus = DownloadStatus.Active) {
-        const newStatus = new ProgressStatusFile(this.totalDownloadParts, this.fileName, this.comment, this.transferAction, downloadPart, transferredBytes, downloadStatus);
+    createStatus(downloadPart, transferredBytes, totalBytes = this.totalBytes, downloadStatus = DownloadStatus.Active, comment = this.comment) {
+        const newStatus = new ProgressStatusFile(this.totalDownloadParts, this.fileName, this.transferAction, this.downloadFlags, comment, downloadPart, transferredBytes, downloadStatus);
         newStatus.totalBytes = totalBytes;
@@ -42,0 +48,0 @@ newStatus.startTime = this.startTime;

dist/download/download-engine/engine/base-download-engine.d.ts

@@ -35,3 +35,2 @@ import { DownloadFile, SaveProgressInfo } from "../types.js";
     protected _downloadStarted: boolean;
-    protected _latestStatus?: ProgressStatusWithIndex;
     protected constructor(engine: DownloadEngineFile, options: DownloadEngineFileOptions);
@@ -38,0 +37,0 @@ get file(): DownloadFile;

dist/download/download-engine/engine/base-download-engine.js

@@ -5,3 +5,2 @@ import UrlInputError from "./error/url-input-error.js";
 import DownloadAlreadyStartedError from "./error/download-already-started-error.js";
-import { createFormattedStatus } from "../../transfer-visualize/format-transfer-status.js";
 import StatusCodeError from "../streams/download-engine-fetch-stream/errors/status-code-error.js";
@@ -14,3 +13,2 @@ const IGNORE_HEAD_STATUS_CODES = [405, 501, 404];
     _downloadStarted = false;
-    _latestStatus;
     constructor(engine, options) {
@@ -33,3 +31,3 @@ super();
     get status() {
-        return this._latestStatus ?? createFormattedStatus(this._engine.status);
+        return ProgressStatisticsBuilder.oneStatistics(this._engine);
     }
@@ -65,3 +63,2 @@ get downloadStatues() {
         this._progressStatisticsBuilder.on("progress", (status) => {
-            this._latestStatus = status;
             return this.emit("progress", status);
@@ -105,3 +102,3 @@ });
                 const size = length || 0;
-                downloadFile.totalSize += length;
+                downloadFile.totalSize += size;
                 downloadFile.parts.push({
@@ -108,0 +105,0 @@ downloadURL,

dist/download/download-engine/engine/download-engine-multi-download.d.ts

 import BaseDownloadEngine, { BaseDownloadEngineEvents } from "./base-download-engine.js";
 import { EventEmitter } from "eventemitter3";
 import ProgressStatisticsBuilder, { ProgressStatusWithIndex } from "../../transfer-visualize/progress-statistics-builder.js";
-import DownloadEngineFile from "../download-file/download-engine-file.js";
 import { FormattedStatus } from "../../transfer-visualize/format-transfer-status.js";
-type DownloadEngineMultiAllowedEngines = BaseDownloadEngine | DownloadEngineFile;
+type DownloadEngineMultiAllowedEngines = BaseDownloadEngine;
 type DownloadEngineMultiDownloadEvents<Engine = DownloadEngineMultiAllowedEngines> = BaseDownloadEngineEvents & {
@@ -12,3 +11,3 @@ childDownloadStarted: (engine: Engine) => void;
 export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMultiAllowedEngines = DownloadEngineMultiAllowedEngines> extends EventEmitter<DownloadEngineMultiDownloadEvents> {
-    protected readonly _engines: Engine[];
+    readonly downloads: Engine[];
     protected _aborted: boolean;
@@ -30,4 +29,4 @@ protected _activeEngine?: Engine;
     protected static _extractEngines<Engine>(engines: Engine[]): any[];
-    static fromEngines<Engine extends DownloadEngineMultiAllowedEngines>(engines: (Engine | Promise<Engine>)[]): Promise<DownloadEngineMultiDownload<DownloadEngineMultiAllowedEngines>>;
+    static fromEngines<Engine extends DownloadEngineMultiAllowedEngines>(engines: (Engine | Promise<Engine>)[]): Promise<DownloadEngineMultiDownload<BaseDownloadEngine>>;
 }
 export {};

dist/download/download-engine/engine/download-engine-multi-download.js

@@ -1,2 +0,1 @@
-import BaseDownloadEngine from "./base-download-engine.js";
 import { EventEmitter } from "eventemitter3";
@@ -7,3 +6,3 @@ import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js";
 export default class DownloadEngineMultiDownload extends EventEmitter {
-    _engines;
+    downloads;
     _aborted = false;
@@ -16,3 +15,3 @@ _activeEngine;
         super();
-        this._engines = DownloadEngineMultiDownload._extractEngines(engines);
+        this.downloads = DownloadEngineMultiDownload._extractEngines(engines);
         this._init();
@@ -24,7 +23,7 @@ }
     get downloadSize() {
-        return this._engines.reduce((acc, engine) => acc + engine.downloadSize, 0);
+        return this.downloads.reduce((acc, engine) => acc + engine.downloadSize, 0);
     }
     _init() {
         this._changeEngineFinishDownload();
-        for (const [index, engine] of Object.entries(this._engines)) {
+        for (const [index, engine] of Object.entries(this.downloads)) {
             const numberIndex = Number(index);
@@ -36,3 +35,3 @@ this._downloadStatues[numberIndex] = createFormattedStatus(engine.status);
         }
-        this._progressStatisticsBuilder.add(...this._engines);
+        this._progressStatisticsBuilder.add(...this.downloads);
         this._progressStatisticsBuilder.on("progress", progress => {
@@ -47,3 +46,3 @@ this.emit("progress", progress);
         this.emit("start");
-        for (const engine of this._engines) {
+        for (const engine of this.downloads) {
             if (this._aborted)
@@ -61,4 +60,4 @@ return;
     _changeEngineFinishDownload() {
-        for (const engine of this._engines) {
-            const options = engine instanceof BaseDownloadEngine ? engine._fileEngineOptions : engine.options;
+        for (const engine of this.downloads) {
+            const options = engine._fileEngineOptions;
             const onFinishAsync = options.onFinishAsync;
@@ -94,3 +93,3 @@ const onCloseAsync = options.onCloseAsync;
             if (engine instanceof DownloadEngineMultiDownload) {
-                return engine._engines;
+                return engine.downloads;
             }
@@ -97,0 +96,0 @@ return engine;

dist/download/download-engine/engine/download-engine-nodejs.d.ts

@@ -15,2 +15,3 @@ import { DownloadFile } from "../types.js";
     fetchStrategy?: "localFile" | "fetch";
+    skipExisting?: boolean;
 };
@@ -31,2 +32,3 @@ export type DownloadEngineOptionsNodejsCustomFetch = DownloadEngineOptionsNodejs & {
     protected _initEvents(): void;
+    get fileAbsolutePath(): string;
     /**
@@ -33,0 +35,0 @@ * Abort the download & delete the file (if it exists)

dist/download/download-engine/engine/download-engine-nodejs.js

@@ -23,14 +23,25 @@ import path from "path";
         this._engine.options.onSaveProgressAsync = async (progress) => {
+            if (this.options.skipExisting)
+                return;
             await this.options.writeStream.saveMedataAfterFile(progress);
         };
         this._engine.options.onFinishAsync = async () => {
+            if (this.options.skipExisting)
+                return;
             await this.options.writeStream.ftruncate(this.downloadSize);
         };
         this._engine.options.onCloseAsync = async () => {
-            if (this.status.ended) {
+            if (this.status.ended && !this.options.skipExisting) {
                 const closedFileName = this.options.writeStream.path.slice(0, -PROGRESS_FILE_EXTENSION.length);
                 await fs.rename(this.options.writeStream.path, closedFileName);
+                this.options.writeStream.path = closedFileName;
             }
         };
+        if (this.options.skipExisting) {
+            this.options.writeStream.path = this.options.writeStream.path.slice(0, -PROGRESS_FILE_EXTENSION.length);
+        }
     }
+    get fileAbsolutePath() {
+        return path.resolve(this.options.writeStream.path);
+    }
     /**
@@ -42,3 +53,3 @@ * Abort the download & delete the file (if it exists)
         try {
-            await fs.unlink(this.options.writeStream.path);
+            await fs.unlink(this.fileAbsolutePath);
         }
@@ -69,2 +80,14 @@ catch { }
         downloadFile.downloadProgress = await writeStream.loadMetadataAfterFileWithoutRetry();
+        if (options.skipExisting && !downloadFile.downloadProgress) {
+            options.skipExisting = false;
+            if (downloadFile.totalSize > 0) {
+                try {
+                    const stat = await fs.stat(downloadLocation);
+                    if (stat.isFile() && stat.size === downloadFile.totalSize) {
+                        options.skipExisting = true;
+                    }
+                }
+                catch { }
+            }
+        }
         const allOptions = { ...options, writeStream };
@@ -71,0 +94,0 @@ const engine = new DownloadEngineFile(downloadFile, allOptions);

dist/download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.d.ts

@@ -8,3 +8,3 @@ import retry from "async-retry";
 export default class DownloadEngineWriteStreamNodejs extends BaseDownloadEngineWriteStream {
-    readonly path: string;
+    path: string;
     private _fd;
@@ -11,0 +11,0 @@ private _fileWriteFinished;

dist/download/node-download.d.ts

@@ -19,2 +19,2 @@ import DownloadEngineNodejs, { DownloadEngineOptionsNodejs } from "./download-engine/engine/download-engine-nodejs.js";
  */
-export declare function downloadSequence(options: DownloadSequenceOptions | DownloadEngineNodejs | Promise<DownloadEngineNodejs>, ...downloads: (DownloadEngineNodejs | Promise<DownloadEngineNodejs>)[]): Promise<DownloadEngineMultiDownload<import("./download-engine/download-file/download-engine-file.js").default | BaseDownloadEngine>>;
+export declare function downloadSequence(options: DownloadSequenceOptions | DownloadEngineNodejs | Promise<DownloadEngineNodejs>, ...downloads: (DownloadEngineNodejs | Promise<DownloadEngineNodejs>)[]): Promise<DownloadEngineMultiDownload<BaseDownloadEngine>>;

dist/download/transfer-visualize/progress-statistics-builder.d.ts

@@ -26,3 +26,4 @@ import BaseDownloadEngine from "../download-engine/engine/base-download-engine.js";
     protected _initEvents(engine: AnyEngine): void;
+    static oneStatistics(engine: DownloadEngineFile): FormattedStatus;
 }
 export {};

dist/download/transfer-visualize/progress-statistics-builder.js

@@ -42,3 +42,11 @@ import { EventEmitter } from "eventemitter3";
     }
+    static oneStatistics(engine) {
+        const progress = engine.status;
+        const statistics = TransferStatistics.oneStatistics(progress.transferredBytes, progress.totalBytes);
+        return createFormattedStatus({
+            ...progress,
+            ...statistics
+        });
+    }
 }
 //# sourceMappingURL=progress-statistics-builder.js.map

dist/download/transfer-visualize/transfer-statistics.d.ts

@@ -27,2 +27,3 @@ export type TransferProgressInfo = {
     updateProgress(transferred: number, total: number): TransferProgressInfo;
+    static oneStatistics(transferred: number, total: number): TransferProgressInfo;
 }

dist/download/transfer-visualize/transfer-statistics.js

@@ -51,3 +51,6 @@ import { clamp } from "./utils/numbers.js";
     }
+    static oneStatistics(transferred, total) {
+        return new TransferStatistics().updateProgress(transferred, total);
+    }
 }
 //# sourceMappingURL=transfer-statistics.js.map

dist/download/transfer-visualize/utils/data-line.d.ts

@@ -15,3 +15,3 @@ export type DataPart = {
 export declare function resizeDataLine(dataLine: DataLine, lineLength: number): {
-    type: "status" | "name" | "nameComment" | "progressBar" | "speed" | "timeLeft" | "spacer" | "description";
+    type: "speed" | "timeLeft" | "status" | "name" | "nameComment" | "progressBar" | "spacer" | "description";
     fullText: string;
@@ -18,0 +18,0 @@ size: number;

package.json

 {
   "name": "ipull",
-  "version": "3.1.1",
+  "version": "3.2.0",
   "description": "The only file downloader you'll ever need. For node.js and the browser, CLI and library for fast and reliable file downloads.",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

README.md

@@ -120,2 +120,16 @@ <div align="center">
 
+### Skip existing files
+
+Skip downloading files that already exist in the save location and have the same size.
+
+```ts
+import {downloadFile} from 'ipull';
+
+const downloader = await downloadFile({
+    url: 'https://example.com/file.large',
+    directory: './this/path',
+    skipExisting: true
+});
+```
+
 ### Download file from parts
@@ -280,2 +294,3 @@
 
+console.log(`Downloading ${downloader.downloads.length} files...`);
 await downloader.download();
@@ -282,0 +297,0 @@ ```

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

249931

increased by2.15%

Number of package files

189

increased by1.61%

Lines of code

3330

increased by2.05%

Number of lines in readme file

324

increased by4.85%

No dependency changes