Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More
Socket
Sign inDemoInstall
Socket

ipull

Package Overview
Dependencies
Maintainers
1
Versions
50
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

ipull - npm Package Compare versions

Comparing version 1.2.1 to 2.0.0

dist/browser.d.ts

1

dist/cli/cli.d.ts

@@ -1,2 +0,1 @@

#!/usr/bin/env node
export {};

@@ -1,11 +0,7 @@

#!/usr/bin/env node
import path from "path";
import { pathToFileURL } from "url";
import { Command } from "commander";
import { packageJson } from "../const.js";
import pullFileCLI from "../download/index.js";
import { truncateText } from "../utils/truncate-text.js";
import { FastDownload } from "../index.js";
import findDownloadDir, { downloadToDirectory } from "./utils/find-download-dir.js";
import { copyFile, downloadFile } from "../download/node-download.js";
import { setCommand } from "./commands/set.js";
import findDownloadDir, { downloadToDirectory, findFileName } from "./utils/find-download-dir.js";
const pullCommand = new Command();

@@ -17,5 +13,5 @@ pullCommand

.option("-s --save [path]", "Save location (directory/file)")
.option("-c --connections [number]", "Number of parallel connections", "4")
.option("-f --full-name", "Show full name of the file while downloading, even if it long")
.action(async (files = [], { save, fullName }) => {
let specificFileName = null;
.action(async (files = [], { save: saveLocation, fullName, number }) => {
if (files.length === 0) {

@@ -25,24 +21,19 @@ pullCommand.outputHelp();

}
if (save && !await downloadToDirectory(save)) {
specificFileName = path.basename(save);
save = path.dirname(save);
let counter = 1;
for (const file of files) {
const isDirectory = saveLocation && await downloadToDirectory(saveLocation);
const directory = isDirectory ? saveLocation : await findDownloadDir(findFileName(file));
const fileName = isDirectory || !saveLocation ? "" : path.basename(saveLocation);
const objectType = files.length > 1 ? `${counter++}/${files.length}` : "";
const options = {
directory,
fileName,
truncateName: !fullName,
objectType,
parallelStreams: Number(number) || 4
};
const downloadStrategy = file.startsWith("http") ? downloadFile : copyFile;
const downloader = await downloadStrategy(file, options);
await downloader.download();
}
const pullLogs = [];
for (const [index, file] of Object.entries(files)) {
let fileName = path.basename(file);
if (specificFileName) {
fileName = files.length > 1 ? specificFileName + index : specificFileName;
}
else if (file.startsWith("http")) {
fileName = await FastDownload.fetchFilename(file);
}
const downloadTag = fullName ? fileName : truncateText(fileName);
const downloadPath = path.join(save || await findDownloadDir(fileName), fileName);
const fileFullPath = new URL(file, pathToFileURL(process.cwd()));
await pullFileCLI(fileFullPath.href, downloadPath, downloadTag);
pullLogs.push(`${fileFullPath.href} -> ${downloadPath}`);
}
console.log();
console.log(pullLogs.join("\n"));
process.exit(0);
});

@@ -49,0 +40,0 @@ pullCommand.addCommand(setCommand);

@@ -1,2 +0,3 @@

export default function findDownloadDir(fileName: string): Promise<string>;
export default function findDownloadDir(fileName?: string): Promise<string>;
export declare function findFileName(url: string): string;
export declare function downloadToDirectory(path: string): Promise<boolean>;

@@ -6,6 +6,14 @@ import path from "path";

export default async function findDownloadDir(fileName) {
const downloadLocation = await getWithDefault(path.extname(fileName));
const downloadLocation = await getWithDefault(path.extname(fileName || ""));
const defaultLocation = await getWithDefault("default");
return downloadLocation || defaultLocation || DEFAULT_DOWNLOAD_DIR;
}
export function findFileName(url) {
try {
return path.basename(new URL(url).pathname);
}
catch {
return path.basename(url);
}
}
export async function downloadToDirectory(path) {

@@ -12,0 +20,0 @@ try {

export declare const __dirname: string;
export declare const packageJson: any;
export declare const TRUNCATE_TEXT_MAX_LENGTH = 30;
export declare const DB_PATH: string;

@@ -6,4 +6,3 @@ import { fileURLToPath } from "url";

export const packageJson = await fs.readJSON(path.join(__dirname, "..", "package.json"));
export const TRUNCATE_TEXT_MAX_LENGTH = 30;
export const DB_PATH = path.join(__dirname, "..", "db.json");
//# sourceMappingURL=const.js.map

@@ -1,9 +0,14 @@

import pullFileCLI from "./download/index.js";
import CLIPullProgress from "./download/cli-pull-progress.js";
import { truncateText } from "./utils/truncate-text.js";
import CopyProgress from "./download/stream-progress/copy-progress.js";
import FastDownload from "./download/stream-progress/fast-download.js";
import { IStreamProgress } from "./download/stream-progress/istream-progress.js";
import PullProgress, { PullProgressCallback, TransferProgressInfo } from "./download/pull-progress.js";
export { pullFileCLI, CLIPullProgress, PullProgress, truncateText, CopyProgress, FastDownload, IStreamProgress };
export type { TransferProgressInfo, PullProgressCallback };
import TransferStatistics, { TransferProgressInfo } from "./download/transfer-visualize/transfer-statistics.js";
import TransferCli, { TransferCliOptions, TransferCliStatus } from "./download/transfer-visualize/transfer-cli.js";
import DownloadEngineNodejs, { DownloadEngineOptionsNodejs } from "./download/download-engine/engine/download-engine-nodejs.js";
import DownloadEngineFile, { DownloadEngineFileOptionsWithDefaults } from "./download/download-engine/download-engine-file.js";
import { copyFile, downloadFile } from "./download/node-download.js";
import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js";
import BaseDownloadEngineFetchStream, { DownloadEngineFetchStreamOptions } from "./download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js";
import DownloadEngineFetchStreamFetch from "./download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js";
import DownloadEngineFetchStreamLocalFile from "./download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.js";
import BaseDownloadEngineWriteStream from "./download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.js";
import DownloadEngineWriteStreamNodejs from "./download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.js";
export { TransferStatistics, TransferCli, DownloadEngineNodejs, BaseDownloadEngine, DownloadEngineFile, truncateText, downloadFile, copyFile, BaseDownloadEngineFetchStream, DownloadEngineFetchStreamFetch, DownloadEngineFetchStreamLocalFile, BaseDownloadEngineWriteStream, DownloadEngineWriteStreamNodejs };
export type { TransferProgressInfo, TransferCliOptions, TransferCliStatus, DownloadEngineOptionsNodejs, DownloadEngineFileOptionsWithDefaults as DownloadEngineFileOptions, DownloadEngineFetchStreamOptions };

@@ -1,9 +0,14 @@

import pullFileCLI from "./download/index.js";
import CLIPullProgress from "./download/cli-pull-progress.js";
import { truncateText } from "./utils/truncate-text.js";
import CopyProgress from "./download/stream-progress/copy-progress.js";
import FastDownload from "./download/stream-progress/fast-download.js";
import { IStreamProgress } from "./download/stream-progress/istream-progress.js";
import PullProgress from "./download/pull-progress.js";
export { pullFileCLI, CLIPullProgress, PullProgress, truncateText, CopyProgress, FastDownload, IStreamProgress };
import TransferStatistics from "./download/transfer-visualize/transfer-statistics.js";
import TransferCli from "./download/transfer-visualize/transfer-cli.js";
import DownloadEngineNodejs from "./download/download-engine/engine/download-engine-nodejs.js";
import DownloadEngineFile from "./download/download-engine/download-engine-file.js";
import { copyFile, downloadFile } from "./download/node-download.js";
import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js";
import BaseDownloadEngineFetchStream from "./download/download-engine/streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js";
import DownloadEngineFetchStreamFetch from "./download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js";
import DownloadEngineFetchStreamLocalFile from "./download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.js";
import BaseDownloadEngineWriteStream from "./download/download-engine/streams/download-engine-write-stream/base-download-engine-write-stream.js";
import DownloadEngineWriteStreamNodejs from "./download/download-engine/streams/download-engine-write-stream/download-engine-write-stream-nodejs.js";
export { TransferStatistics, TransferCli, DownloadEngineNodejs, BaseDownloadEngine, DownloadEngineFile, truncateText, downloadFile, copyFile, BaseDownloadEngineFetchStream, DownloadEngineFetchStreamFetch, DownloadEngineFetchStreamLocalFile, BaseDownloadEngineWriteStream, DownloadEngineWriteStreamNodejs };
//# sourceMappingURL=index.js.map

@@ -0,1 +1,2 @@

export declare const TRUNCATE_TEXT_MAX_LENGTH = 30;
export declare function truncateText(text: string, maxLength?: number): string;

@@ -1,2 +0,2 @@

import { TRUNCATE_TEXT_MAX_LENGTH } from "../const.js";
export const TRUNCATE_TEXT_MAX_LENGTH = 30;
export function truncateText(text, maxLength = TRUNCATE_TEXT_MAX_LENGTH) {

@@ -3,0 +3,0 @@ if (text.length <= maxLength) {

{
"name": "ipull",
"version": "1.2.1",
"description": "Super fast cli file downloader",
"version": "2.0.0",
"description": "The only file downloader you'll ever need. Node.js & Browser, CLI and library for fast and reliable file downloads.",
"main": "dist/index.js",

@@ -18,4 +18,4 @@ "type": "module",

".": "./dist/index.js",
"./pull-progress": "./dist/download/pull-progress.js",
"./pull-progress.js": "./dist/download/pull-progress.js"
"./browser": "./dist/browser.js",
"./dist/browser.js": "./dist/browser.js"
},

@@ -26,9 +26,8 @@ "engines": {

"scripts": {
"generate-docs": "typedoc --plugin typedoc-plugin-missing-exports",
"build": "tsc --build tsconfig.json --force",
"generate-docs": "typedoc",
"build": "tsc --build tsconfig.json",
"cli": "npm run build && node -r dotenv/config ./dist/cli/cli.js",
"format": "npm run lint:eslint -- --fix",
"prepack": "npm run build",
"test": "npm run test:typescript && npm run lint:eslint",
"test:typescript": "tsc --build tsconfig.json --dry --force",
"test": "vitest",
"lint": "npm run lint:eslint",

@@ -44,6 +43,15 @@ "lint:eslint": "eslint --ext .js --ext .ts ."

"partial-download",
"parallel-connections",
"browser-download",
"node-download",
"copy-file",
"stream-download",
"io",
"retry-download",
"resume-download",
"file-downloader",
"transfer-speed",
"transfer-statistics",
"cli-progress",
"cli",
"turbo-downloader",
"file-copy",

@@ -61,2 +69,6 @@ "downloader",

"homepage": "https://github.com/ido-pluto/ipull#readme",
"funding": {
"type": "github",
"url": "https://github.com/ido-pluto/ipull?sponsor=1"
},
"devDependencies": {

@@ -66,34 +78,38 @@ "@commitlint/cli": "^17.7.1",

"@semantic-release/exec": "^6.0.3",
"@types/cli-progress": "^3.11.0",
"@types/content-disposition": "^0.5.8",
"@types/async-retry": "^1.4.8",
"@types/fs-extra": "^11.0.1",
"@types/lodash.debounce": "^4.0.9",
"@types/node": "^20.4.9",
"@types/progress-stream": "^2.0.2",
"@typescript-eslint/eslint-plugin": "^6.3.0",
"@typescript-eslint/parser": "^6.3.0",
"@vitest/ui": "^1.3.1",
"dotenv": "^16.3.1",
"eslint": "^8.46.0",
"eslint-plugin-import": "^2.28.0",
"eslint-plugin-node": "github:giladgd/eslint-plugin-node#dev/giladgd/fixImportExtentionFixingInTypeScript",
"eslint-plugin-jsdoc": "^48.2.0",
"eslint-plugin-n": "^16.6.2",
"eslint-plugin-node": "^11.1.0",
"husky": "^8.0.3",
"semantic-release": "^21.0.7",
"semantic-release": "^21.1.2",
"tslib": "^2.6.1",
"typedoc": "^0.24.8",
"typedoc-plugin-missing-exports": "^2.0.1",
"typescript": "^5.1.6"
"typedoc": "^0.25.8",
"typedoc-material-theme": "^1.0.2",
"typedoc-plugin-missing-exports": "^2.2.0",
"typescript": "^5.1.6",
"vitest": "^1.3.1",
"xmlhttprequest-ssl": "^2.1.1"
},
"dependencies": {
"@supercharge/promise-pool": "^3.1.1",
"async-retry": "^1.3.3",
"chalk": "^5.3.0",
"cli-progress": "^3.12.0",
"commander": "^10.0.0",
"content-disposition": "^0.5.4",
"execa": "^7.2.0",
"fs-extra": "^11.1.1",
"level": "^8.0.0",
"lifecycle-utils": "^1.3.1",
"lodash.debounce": "^4.0.8",
"log-update": "^6.0.0",
"pretty-bytes": "^6.1.0",
"pretty-ms": "^8.0.0",
"progress-stream": "^2.0.0",
"turbo-downloader": "1.3.1",
"wretch": "^2.6.0"
"pretty-ms": "^8.0.0"
}
}
<div align="center">
<h1>IPULL</h1>
<h1>iPull</h1>
<img src="https://github.com/ido-pluto/ipull/blob/main/assets/ipull-logo-rounded.png" height="200px" />
</div>

@@ -7,5 +8,7 @@

[![npm version](https://badge.fury.io/js/ipull.svg)](https://badge.fury.io/js/ipull)
[![Build](https://github.com/ido-pluto/ipull/actions/workflows/build.yml/badge.svg)](https://github.com/ido-pluto/ipull/actions/workflows/build.yml)
[![License](https://badgen.net/badge/color/MIT/green?label=license)](https://www.npmjs.com/package/ipull)
[![Types](https://badgen.net/badge/color/TypeScript/blue?label=types)](https://www.npmjs.com/package/ipull)
[![npm downloads](https://img.shields.io/npm/dt/ipull.svg)](https://www.npmjs.com/package/ipull)
[![GitHub license](https://img.shields.io/github/license/ido-pluto/ipull)](./LICENSE)
[![Version](https://badgen.net/npm/v/ipull)](https://www.npmjs.com/package/ipull)

@@ -15,16 +18,61 @@ </div>

> Simple CLI to pull files from the internet **super fast**!
> Super fast file downloader with multiple connections
```bash
npx ipull http://example.com/file.txt
npx ipull http://example.com/file.large
```
![pull-example](https://github.com/ido-pluto/ipull/blob/main/demo/pull-file.gif)
![pull-example](https://github.com/ido-pluto/ipull/blob/main/assets/pull-file.gif)
## Features
- Download using multiple connections
- Download using parallels connections
- Pausing and resuming downloads
- Node.js and browser support
- Smart retry on fail
- Resuming after fails
- CLI Progress bar
- Download statistics (speed, time left, etc.)
### NodeJS API
```ts
import {downloadFile} from 'ipull';
const downloader = downloadFile('https://example.com/file.large', {
directory: './this/path',
cliProgress: true // Show progress bar in the CLI (default: true)
});
await downloader.download();
```
## Browser support
Download a file in the browser using multiple connections
```ts
import {downloadFileBrowserMemory} from "ipull/dist/browser.js";
const {downloader, memory} = await downloadFileBrowserMemory(DOWNLOAD_URL, {
acceptRangeAlwaysTrue: true // cors origin request will not return the range header, but we can force it to be true (multipart download)
});
await downloader.download();
image.src = memory.createBlobURL();
```
### Custom stream
```ts
import {downloadFileBrowser} from "ipull/dist/browser.js";
const downloader = await downloadFileBrowser(DOWNLOAD_URL, {
onWrite: (cursor: number, buffer: Uint8Array, options) => {
console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`);
}
});
await downloader.download();
```
## CLI

@@ -60,63 +108,168 @@

### NodeJS API
## Advanced usage
### Download file from parts
Download a file from multiple parts, and merge them into a single file.
Beneficial for downloading large files from servers that limit file size. (e.g. HuggingFace models)
```ts
class PullProgress {
constructor(progress: IStreamProgress, onProgress: PullProgressCallback);
import {downloadFile} from 'ipull';
startPull(): Promise<void>;
}
const downloadParts = [
"https://example.com/file.large1",
"https://example.com/file.large2",
"https://example.com/file.large3",
];
class CLIPullProgress {
constructor(progress: IStreamProgress, name: string);
const downloader = downloadFile(downloadParts, {
directory: './this/path',
filename: 'file.large'
});
startPull(): Promise<void>;
}
await downloader.download();
```
interface IStreamProgress {
init(): Promise<void>;
** The split must be binary and not a zip-split
progress(callback: (progressBytes: number, totalBytes: number) => void): Promise<any>;
}
### Custom headers
You can set custom headers for the download request
class FastDownload implements IStreamProgress {
constructor(url: string, savePath: string, options?: TurboDownloaderOptions);
```ts
import {downloadFile} from 'ipull';
static async fetchFilename(url: string);
}
const downloader = downloadFile('https://example.com/file.large', {
directory: './this/path',
headers: {
'Authorization': 'Bearer token'
}
});
class CopyProgress implements IStreamProgress {
constructor(fromPath: string, toPath: string);
}
await downloader.download();
```
Example:
### Copy file
Copy file from local directory to another directory with CLI progress bar
```ts
import {FastDownload, CLIPullProgress} from 'ipull';
import {copyFile} from 'ipull';
const download = new FastDownload('http://example.com/file.txt', './file.txt');
await download.init();
const downloader = await copyFile('path/to/file', {
directory: './this/path'
});
await downloader.download();
const progress = new CLIPullProgress(download, 'file.txt');
await progress.startPull();
```
## Browser support
You can also use IPull without the CLI, just for download metadata
### Abort download
You can cancel the download by calling the `abort` method
```ts
import PullProgress from 'ipull/pull-progress.js';
import {downloadFile} from 'ipull';
const pull = new PullProgress(download, (info) => {
console.log(info.speed);
const downloader = downloadFile('https://example.com/file.large', {
directory: './this/path'
});
pull.startPull();
setTimeout(() => {
downloader.abort();
}, 5_000);
await downloader.download();
```
- Make sure you import the exact file when using, so Node.js only modules will not be imported
## Credits
### Pause & Resume download
[Turbo-Downloader](https://www.npmjs.com/package/turbo-downloader) - The download engine used in this package
```ts
import {downloadFile} from 'ipull';
const downloader = downloadFile('https://example.com/file.large', {
directory: './this/path'
});
setInterval(() => {
downloader.pause();
setTimeout(() => {
downloader.resume();
}, 5_000);
}, 10_000);
await downloader.download();
```
** The pause may take a few seconds to actually pause the download, because it waits for the current connections to
finish
### Error handling
If a network/file-system error occurs, the download will automatically retry
with [async-retry](https://www.npmjs.com/package/async-retry)
```ts
import {downloadFile} from 'ipull';
const downloader = downloadFile('https://example.com/file.large', {
directory: './this/path'
});
try {
await downloader.download();
} catch (error) {
console.error(`Download failed: ${error.message}`);
}
```
### Custom Downloader
In this example, there will be one progress bar for all the files
```ts
import {TransferCli, DownloadEngineNodejs, TransferStatistics} from "ipull";
const cli = new TransferCli();
const statistics = new TransferStatistics();
const filesToDownload = ["https://example.com/file1.large", "https://example.com/file2.large", "https://example.com/file3.large"];
let totalSize = 0;
let bytesDownloaded = 0;
const downloadsPromise = filesToDownload.map((url, index) => {
return DownloadEngineNodejs.fromParts(url, {
onInit(engine) {
totalSize += engine.file.totalSize;
},
onProgress(progress) {
const status = statistics.updateProgress(bytesDownloaded + progress.bytesDownloaded, totalSize);
cli.updateProgress({
...status,
...progress,
objectType: `${index}/${filesToDownload.length}`
});
},
onClosed(engine) {
bytesDownloaded += engine.file.totalSize
}
});
});
for (const downloader of await Promise.all(downloadsPromise)) {
await downloader.download();
}
```
![custom-progress-bar](https://github.com/ido-pluto/ipull/blob/main/assets/custom-progress.png)
<br />
<div align="center" width="360">
<img alt="Star please" src="https://github.com/ido-pluto/ipull/blob/main/assets/star-please.png" width="360" margin="auto" />
<br/>
<p align="right">
<i>If you like this repo, star it ✨</i>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</p>
</div>

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc