New Case Study:See how Anthropic automated 95% of dependency reviews with Socket.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

@thirdweb-dev/storage

Package Overview
Dependencies
Maintainers
8
Versions
801
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@thirdweb-dev/storage - npm Package Compare versions

Comparing version

0.0.0-dev-7cfa22e-20230329230519
to
0.0.0-dev-812a24f-20231130143736

dist/declarations/src/common/process.d.ts
dist/declarations/src/common/process.d.ts.map
dist/thirdweb-dev-storage.cjs.d.ts.map

1

dist/declarations/src/common/index.d.ts
export * from "./urls";
export * from "./utils";
export * from "./cid";
//# sourceMappingURL=index.d.ts.map

18

dist/declarations/src/common/urls.d.ts
import { GatewayUrls } from "../types";
/**
* @internal
* @param url - the url to check
* @returns
*/
export declare function isTwGatewayUrl(url: string): boolean;
/**
* @internal
*/
export declare const DEFAULT_GATEWAY_URLS: GatewayUrls;

@@ -9,3 +15,3 @@ /**

*/
export declare const TW_IPFS_SERVER_URL = "https://upload.nftlabs.co";
export declare const TW_UPLOAD_SERVER_URL: string;
/**

@@ -22,3 +28,11 @@ * @internal

*/
export declare function prepareGatewayUrls(gatewayUrls?: GatewayUrls): GatewayUrls;
export declare function getGatewayUrlForCid(gatewayUrl: string, cid: string, clientId?: string): string;
/**
* @internal
*/
export declare function prepareGatewayUrls(gatewayUrls: GatewayUrls, clientId?: string, secretKey?: string): GatewayUrls;
/**
* @internal
*/
export declare function convertCidToV1(cid: string): string;
//# sourceMappingURL=urls.d.ts.map

4

dist/declarations/src/common/utils.d.ts

@@ -31,3 +31,3 @@ /// <reference types="node" />

*/
export declare function replaceSchemeWithGatewayUrl(uri: string, gatewayUrls: GatewayUrls, index?: number): string | undefined;
export declare function replaceSchemeWithGatewayUrl(uri: string, gatewayUrls: GatewayUrls, index?: number, clientId?: string): string | undefined;
/**

@@ -40,3 +40,3 @@ * @internal

*/
export declare function replaceObjectSchemesWithGatewayUrls<TData = unknown>(data: TData, gatewayUrls: GatewayUrls): TData;
export declare function replaceObjectSchemesWithGatewayUrls<TData = unknown>(data: TData, gatewayUrls: GatewayUrls, clientId?: string): TData;
/**

@@ -43,0 +43,0 @@ * @internal

19

dist/declarations/src/core/downloaders/storage-downloader.d.ts

@@ -1,2 +0,2 @@

import { GatewayUrls, IStorageDownloader } from "../../types";
import { GatewayUrls, IStorageDownloader, IpfsDownloaderOptions, SingleDownloadOptions } from "../../types";
/**

@@ -9,3 +9,10 @@ * Default downloader used - handles downloading from all schemes specified in the gateway URLs configuration.

* const downloader = new StorageDownloader();
* const storage = new ThirdwebStorage({ downloader });
*
* // client id if used in client-side applications
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, downloader });
*
* // secret key if used in server-side applications
* const secretKey = "your-secret-key";
* const storage = new ThirdwebStorage({ secretKey, downloader });
* ```

@@ -16,4 +23,10 @@ *

export declare class StorageDownloader implements IStorageDownloader {
download(uri: string, gatewayUrls: GatewayUrls, attempts?: number): Promise<Response>;
DEFAULT_TIMEOUT_IN_SECONDS: number;
DEFAULT_MAX_RETRIES: number;
private secretKey?;
private clientId?;
private defaultTimeout;
constructor(options: IpfsDownloaderOptions);
download(uri: string, gatewayUrls: GatewayUrls, options?: SingleDownloadOptions, attempts?: number): Promise<Response>;
}
//# sourceMappingURL=storage-downloader.d.ts.map

24

dist/declarations/src/core/storage.d.ts

@@ -1,2 +0,2 @@

import { GatewayUrls, IpfsUploadBatchOptions, ThirdwebStorageOptions, UploadOptions } from "../types";
import { GatewayUrls, IThirdwebStorage, IpfsUploadBatchOptions, SingleDownloadOptions, ThirdwebStorageOptions, UploadOptions } from "../types";
/**

@@ -7,5 +7,10 @@ * Upload and download files from decentralized storage systems.

* ```jsx
* // Create a default storage class without any configuration
* const storage = new ThirdwebStorage();
* // Create a default storage class with a client ID when used in client-side applications
* const storage = new ThirdwebStorage({ clientId: "your-client-id" });
*
* // Create a default storage class with a secret key when used in server-side applications
* const storage = new ThirdwebStorage({ secretKey: "your-secret-key" });
*
* You can get a clientId and secretKey from https://thirdweb.com/create-api-key
*
* // Upload any file or JSON object

@@ -26,3 +31,4 @@ * const uri = await storage.upload(data);

* const uploader = new IpfsUploader();
* const storage = new ThirdwebStorage({ uploader, downloader, gatewayUrls });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader, downloader, gatewayUrls });
* ```

@@ -32,6 +38,7 @@ *

*/
export declare class ThirdwebStorage<T extends UploadOptions = IpfsUploadBatchOptions> {
export declare class ThirdwebStorage<T extends UploadOptions = IpfsUploadBatchOptions> implements IThirdwebStorage {
private uploader;
private downloader;
gatewayUrls: GatewayUrls;
private gatewayUrls;
private clientId?;
constructor(options?: ThirdwebStorageOptions<T>);

@@ -64,3 +71,3 @@ /**

*/
download(url: string): Promise<Response>;
download(url: string, options?: SingleDownloadOptions): Promise<Response>;
/**

@@ -79,3 +86,3 @@ * Downloads JSON data from any URL scheme.

*/
downloadJSON<TJSON = any>(url: string): Promise<TJSON>;
downloadJSON<TJSON = any>(url: string, options?: SingleDownloadOptions): Promise<TJSON>;
/**

@@ -127,4 +134,5 @@ * Upload arbitrary file or JSON data using the configured decentralized storage system.

uploadBatch(data: unknown[], options?: T): Promise<string[]>;
getGatewayUrls(): GatewayUrls;
private uploadAndReplaceFilesWithHashes;
}
//# sourceMappingURL=storage.d.ts.map

23

dist/declarations/src/core/uploaders/ipfs-uploader.d.ts

@@ -7,6 +7,12 @@ import { FileOrBufferOrString, IpfsUploadBatchOptions, IpfsUploaderOptions, IStorageUploader } from "../../types";

* ```jsx
* // Can instantiate the uploader with default configuration
* // Can instantiate the uploader with default configuration and your client ID when used in client-side applications
* const uploader = new StorageUploader();
* const storage = new ThirdwebStorage({ uploader });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader });
*
* // Can instantiate the uploader with default configuration and your secret key when used in server-side applications
* const uploader = new StorageUploader();
* const secretKey = "your-secret-key";
* const storage = new ThirdwebStorage({ secretKey, uploader });
*
* // Or optionally, can pass configuration

@@ -18,3 +24,4 @@ * const options = {

* const uploader = new StorageUploader(options);
* const storage = new ThirdwebStorage({ uploader });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader });
* ```

@@ -26,11 +33,7 @@ *

uploadWithGatewayUrl: boolean;
private uploadServerUrl;
private clientId?;
private secretKey?;
constructor(options?: IpfsUploaderOptions);
uploadBatch(data: FileOrBufferOrString[], options?: IpfsUploadBatchOptions): Promise<string[]>;
/**
* Fetches a one-time-use upload token that can used to upload
* a file to storage.
*
* @returns - The one time use token that can be passed to the Pinata API.
*/
private getUploadToken;
private buildFormData;

@@ -37,0 +40,0 @@ private uploadBatchBrowser;

31

dist/declarations/src/types/download.d.ts

@@ -12,3 +12,3 @@ /**

*/
download(url: string, gatewayUrls?: GatewayUrls): Promise<Response>;
download(url: string, gatewayUrls?: GatewayUrls, options?: SingleDownloadOptions): Promise<Response>;
}

@@ -18,2 +18,31 @@ /**

*/
export type IpfsDownloaderOptions = {
/**
* Optional secretKey to associate with the IpfsDownloader - when used from the backend.
* You can get an secretKey here: https://thirdweb.com/create-api-key
*/
secretKey?: string;
/**
* Optional clientId to associate with the IpfsDownloader - when used from the frontend.
* You can get a clientId here: https://thirdweb.com/create-api-key
*/
clientId?: string;
/**
* Optional timeout in seconds for the download request, overrides the default timeout
*/
timeoutInSeconds?: number;
};
export type SingleDownloadOptions = {
/**
* Optional timeout in seconds for the download request, overrides the default timeout
*/
timeoutInSeconds?: number;
/**
* Number of different gateways to attempt on failure
*/
maxRetries?: number;
};
/**
* @public
*/
export type GatewayUrls = {

@@ -20,0 +49,0 @@ [key: string]: string[];

17

dist/declarations/src/types/index.d.ts

@@ -1,2 +0,2 @@

import { GatewayUrls, IStorageDownloader } from "./download";
import { GatewayUrls, IStorageDownloader, SingleDownloadOptions } from "./download";
import { IStorageUploader, UploadOptions } from "./upload";

@@ -7,3 +7,18 @@ export type ThirdwebStorageOptions<T extends UploadOptions> = {

gatewayUrls?: GatewayUrls | string[];
uploadServerUrl?: string;
clientId?: string;
secretKey?: string;
};
export interface IThirdwebStorage {
resolveScheme(url: string): string;
download(url: string, options?: SingleDownloadOptions): Promise<Response>;
downloadJSON<TJSON = any>(url: string): Promise<TJSON>;
upload(data: any, options?: {
[key: string]: any;
}): Promise<string>;
uploadBatch(data: any[], options?: {
[key: string]: any;
}): Promise<string[]>;
getGatewayUrls(): GatewayUrls;
}
export * from "./download";

@@ -10,0 +25,0 @@ export * from "./upload";

14

dist/declarations/src/types/upload.d.ts

@@ -46,2 +46,16 @@ import { FileOrBufferOrString } from "./data";

uploadWithGatewayUrl?: boolean;
/**
* Optional clientId to associate with the IpfsUploader.
* You can get an clientId here: https://thirdweb.com/create-api-key
*/
clientId?: string;
/**
* Optional secretKey to associate with the IpfsUploader - when used from the backend.
* You can get an secretKey here: https://thirdweb.com/create-api-key
*/
secretKey?: string;
/**
* Optional upload server url to use instead of the default. (Advanced Usage)
*/
uploadServerUrl?: string;
};

@@ -48,0 +62,0 @@ /**

1

dist/thirdweb-dev-storage.cjs.d.ts
export * from "./declarations/src/index";
//# sourceMappingURL=thirdweb-dev-storage.cjs.d.ts.map

653

dist/thirdweb-dev-storage.cjs.dev.js

@@ -5,4 +5,4 @@ 'use strict';

var fetch = require('cross-fetch');
var ipfsUnixfsImporter = require('ipfs-unixfs-importer');
var CIDTool = require('cid-tool');
var crypto = require('@thirdweb-dev/crypto');
var FormData = require('form-data');

@@ -13,35 +13,34 @@ var uuid = require('uuid');

var fetch__default = /*#__PURE__*/_interopDefault(fetch);
var CIDTool__default = /*#__PURE__*/_interopDefault(CIDTool);
var FormData__default = /*#__PURE__*/_interopDefault(FormData);
function _toPrimitive(input, hint) {
if (typeof input !== "object" || input === null) return input;
var prim = input[Symbol.toPrimitive];
if (prim !== undefined) {
var res = prim.call(input, hint || "default");
if (typeof res !== "object") return res;
throw new TypeError("@@toPrimitive must return a primitive value.");
function getProcessEnv(key) {
let defaultValue = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : "";
if (typeof process !== "undefined") {
if (process.env[key]) {
return process.env[key];
}
}
return (hint === "string" ? String : Number)(input);
return defaultValue;
}
function _toPropertyKey(arg) {
var key = _toPrimitive(arg, "string");
return typeof key === "symbol" ? key : String(key);
}
const TW_HOSTNAME_SUFFIX = ".ipfscdn.io";
const TW_STAGINGHOSTNAME_SUFFIX = ".thirdwebstorage-staging.com";
const TW_GATEWAY_URLS = [`https://{clientId}${TW_HOSTNAME_SUFFIX}/ipfs/{cid}/{path}`];
function _defineProperty(obj, key, value) {
key = _toPropertyKey(key);
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
/**
* @internal
* @param url - the url to check
* @returns
*/
function isTwGatewayUrl(url) {
const hostname = new URL(url).hostname;
const isProd = hostname.endsWith(TW_HOSTNAME_SUFFIX);
if (isProd) {
return true;
}
return obj;
// fall back to also handle staging urls
return hostname.endsWith(TW_STAGINGHOSTNAME_SUFFIX);
}
const PUBLIC_GATEWAY_URLS = ["https://{cid}.ipfs.cf-ipfs.com/{path}", "https://{cid}.ipfs.dweb.link/{path}", "https://ipfs.io/ipfs/{cid}/{path}", "https://cloudflare-ipfs.com/ipfs/{cid}/{path}", "https://{cid}.ipfs.w3s.link/{path}", "https://w3s.link/ipfs/{cid}/{path}", "https://nftstorage.link/ipfs/{cid}/{path}", "https://gateway.pinata.cloud/ipfs/{cid}/{path}"];

@@ -53,3 +52,3 @@ /**

// Note: Gateway URLs should have trailing slashes (we clean this on user input)
"ipfs://": ["https://ipfs.thirdwebcdn.com/ipfs/", "https://cloudflare-ipfs.com/ipfs/", "https://ipfs.io/ipfs/"]
"ipfs://": [...TW_GATEWAY_URLS, ...PUBLIC_GATEWAY_URLS]
};

@@ -60,3 +59,3 @@

*/
const TW_IPFS_SERVER_URL = "https://upload.nftlabs.co";
const TW_UPLOAD_SERVER_URL = getProcessEnv("CUSTOM_UPLOAD_SERVER_URL", "https://storage.thirdweb.com");

@@ -83,13 +82,60 @@ /**

*/
function prepareGatewayUrls(gatewayUrls) {
function getGatewayUrlForCid(gatewayUrl, cid, clientId) {
const parts = cid.split("/");
const hash = convertCidToV1(parts[0]);
const filePath = parts.slice(1).join("/");
let url = gatewayUrl;
// If the URL contains {cid} or {path} tokens, replace them with the CID and path
// Both tokens must be present for the URL to be valid
if (gatewayUrl.includes("{cid}") && gatewayUrl.includes("{path}")) {
url = url.replace("{cid}", hash).replace("{path}", filePath);
}
// If the URL contains only the {cid} token, replace it with the CID
else if (gatewayUrl.includes("{cid}")) {
url = url.replace("{cid}", hash);
}
// If those tokens don't exist, use the canonical gateway URL format
else {
url += `${hash}/${filePath}`;
}
// if the URL contains the {clientId} token, replace it with the client ID
if (gatewayUrl.includes("{clientId}")) {
if (!clientId) {
throw new Error("Cannot use {clientId} in gateway URL without providing a client ID");
}
url = url.replace("{clientId}", clientId);
}
return url;
}
/**
* @internal
*/
function prepareGatewayUrls(gatewayUrls, clientId, secretKey) {
const allGatewayUrls = {
...gatewayUrls,
...DEFAULT_GATEWAY_URLS
...DEFAULT_GATEWAY_URLS,
...gatewayUrls
};
for (const key of Object.keys(DEFAULT_GATEWAY_URLS)) {
if (gatewayUrls && gatewayUrls[key]) {
// Make sure that all user gateway URLs have trailing slashes
const cleanedGatewayUrls = gatewayUrls[key].map(url => url.replace(/\/$/, "") + "/");
allGatewayUrls[key] = [...cleanedGatewayUrls, ...DEFAULT_GATEWAY_URLS[key]];
}
for (const key of Object.keys(allGatewayUrls)) {
const cleanedGatewayUrls = allGatewayUrls[key].map(url => {
// inject clientId when present
if (clientId && url.includes("{clientId}")) {
return url.replace("{clientId}", clientId);
} else if (secretKey && url.includes("{clientId}")) {
// should only be used on Node.js in a backend/script context
if (typeof window !== "undefined") {
throw new Error("Cannot use secretKey in browser context");
}
const hashedSecretKey = crypto.sha256HexSync(secretKey);
const derivedClientId = hashedSecretKey.slice(0, 32);
return url.replace("{clientId}", derivedClientId);
} else if (url.includes("{clientId}")) {
// if no client id passed, filter out the url
return undefined;
} else {
return url;
}
}).filter(url => url !== undefined);
allGatewayUrls[key] = cleanedGatewayUrls;
}

@@ -102,2 +148,16 @@ return allGatewayUrls;

*/
function convertCidToV1(cid) {
let normalized;
try {
const hash = cid.split("/")[0];
normalized = CIDTool__default["default"].base32(hash);
} catch (e) {
throw new Error(`The CID ${cid} is not valid.`);
}
return normalized;
}
/**
* @internal
*/
function isBrowser() {

@@ -163,7 +223,38 @@ return typeof window !== "undefined";

*/
function parseCidAndPath(gatewayUrl, uri) {
const regexString = gatewayUrl.replace("{cid}", "(?<hash>[^/]+)").replace("{path}", "(?<path>[^?#]+)");
const regex = new RegExp(regexString);
const match = uri.match(regex);
if (match) {
const hash = match.groups?.hash;
const path = match.groups?.path;
const queryString = uri.includes("?") ? uri.substring(uri.indexOf("?") + 1) : "";
return {
hash,
path,
query: queryString
};
}
}
/**
* @internal
*/
function replaceGatewayUrlWithScheme(uri, gatewayUrls) {
for (const scheme of Object.keys(gatewayUrls)) {
for (const url of gatewayUrls[scheme]) {
if (uri.startsWith(url)) {
return uri.replace(url, scheme);
for (const gatewayUrl of gatewayUrls[scheme]) {
// If the url is a tokenized url, we need to convert it to a canonical url
// Otherwise, we just need to check if the url is a prefix of the uri
if (gatewayUrl.includes("{cid}")) {
// Given the url is a tokenized url, we need to lift the cid and the path from the uri
const parsed = parseCidAndPath(gatewayUrl, uri);
if (parsed?.hash && parsed?.path) {
const queryString = parsed?.query ? `?${parsed?.query}` : "";
return `${scheme}${parsed?.hash}/${parsed?.path}${queryString}`;
} else {
// If we can't lift the cid and path from the uri, we can't replace the gateway url, return the orig string
return uri;
}
} else if (uri.startsWith(gatewayUrl)) {
return uri.replace(gatewayUrl, scheme);
}

@@ -180,2 +271,3 @@ }

let index = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
let clientId = arguments.length > 3 ? arguments[3] : undefined;
const scheme = Object.keys(gatewayUrls).find(s => uri.startsWith(s));

@@ -189,3 +281,10 @@ const schemeGatewayUrls = scheme ? gatewayUrls[scheme] : [];

}
return uri.replace(scheme, schemeGatewayUrls[index]);
const path = uri.replace(scheme, "");
try {
const gatewayUrl = getGatewayUrlForCid(schemeGatewayUrls[index], path, clientId);
return gatewayUrl;
} catch (err) {
console.warn(`The IPFS uri: ${path} is not valid.`);
return undefined;
}
}

@@ -221,5 +320,5 @@

*/
function replaceObjectSchemesWithGatewayUrls(data, gatewayUrls) {
function replaceObjectSchemesWithGatewayUrls(data, gatewayUrls, clientId) {
if (typeof data === "string") {
return replaceSchemeWithGatewayUrl(data, gatewayUrls);
return replaceSchemeWithGatewayUrl(data, gatewayUrls, 0, clientId);
}

@@ -234,7 +333,7 @@ if (typeof data === "object") {

if (Array.isArray(data)) {
return data.map(entry => replaceObjectSchemesWithGatewayUrls(entry, gatewayUrls));
return data.map(entry => replaceObjectSchemesWithGatewayUrls(entry, gatewayUrls, clientId));
}
return Object.fromEntries(Object.entries(data).map(_ref2 => {
let [key, value] = _ref2;
return [key, replaceObjectSchemesWithGatewayUrls(value, gatewayUrls)];
return [key, replaceObjectSchemesWithGatewayUrls(value, gatewayUrls, clientId)];
}));

@@ -296,54 +395,71 @@ }

async function getCIDForUpload(data, fileNames) {
let wrapWithDirectory = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true;
let cidVersion = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 0;
const contentWithPath = await Promise.all(data.map(async (file, i) => {
const path = fileNames[i];
let content;
if (typeof file === "string") {
content = new TextEncoder().encode(file);
} else if (isBufferOrStringWithName(file)) {
if (typeof file.data === "string") {
content = new TextEncoder().encode(file.data);
} else {
content = file.data;
}
} else if (Buffer.isBuffer(file)) {
content = file;
} else {
const buffer = await file.arrayBuffer();
content = new Uint8Array(buffer);
}
return {
path,
content
};
}));
return getCID(contentWithPath, wrapWithDirectory, cidVersion);
}
async function getCID(content) {
let wrapWithDirectory = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;
let cidVersion = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
const options = {
onlyHash: true,
wrapWithDirectory,
cidVersion
};
const dummyBlockstore = {
put: async () => {}
};
let lastCid;
for await (const {
cid
} of ipfsUnixfsImporter.importer(content, dummyBlockstore, options)) {
lastCid = cid;
}
return `${lastCid}`;
}
async function isUploaded(cid) {
const res = await fetch__default["default"](`${DEFAULT_GATEWAY_URLS["ipfs://"][0]}${cid}`, {
method: "HEAD"
});
return res.ok;
}
var pkg = {
name: "@thirdweb-dev/storage",
version: "2.0.5",
main: "dist/thirdweb-dev-storage.cjs.js",
module: "dist/thirdweb-dev-storage.esm.js",
exports: {
".": {
module: "./dist/thirdweb-dev-storage.esm.js",
"default": "./dist/thirdweb-dev-storage.cjs.js"
},
"./package.json": "./package.json"
},
repository: "https://github.com/thirdweb-dev/js/tree/main/packages/storage",
author: "thirdweb eng <eng@thirdweb.com>",
license: "Apache-2.0",
sideEffects: false,
scripts: {
format: "prettier --write 'src/**/*'",
lint: "eslint src/ && bunx publint --strict --level warning",
fix: "eslint src/ --fix",
"generate-docs": "api-extractor run --local && api-documenter markdown -i ./temp -o ./docs",
clean: "rm -rf dist/",
build: "tsc && preconstruct build",
"test:all": "NODE_ENV=test SWC_NODE_PROJECT=./tsconfig.test.json mocha --timeout 30000 --parallel './test/**/*.test.ts'",
test: "pnpm test:all",
"test:single": "NODE_ENV=test SWC_NODE_PROJECT=./tsconfig.test.json mocha --timeout 30000",
push: "yalc push",
typedoc: "rimraf typedoc && node typedoc.js"
},
files: [
"dist/"
],
preconstruct: {
exports: true
},
devDependencies: {
"@babel/preset-env": "^7.22.9",
"@babel/preset-typescript": "^7.22.5",
"@microsoft/api-documenter": "^7.22.30",
"@microsoft/api-extractor": "^7.36.3",
"@microsoft/tsdoc": "^0.14.1",
"@preconstruct/cli": "2.7.0",
"@swc-node/register": "^1.6.6",
"@thirdweb-dev/tsconfig": "workspace:*",
"@types/chai": "^4.3.5",
"@types/mocha": "^10.0.0",
"@types/uuid": "^9.0.5",
"@typescript-eslint/eslint-plugin": "^6.2.0",
"@typescript-eslint/parser": "^6.2.0",
chai: "^4.3.6",
eslint: "^8.45.0",
"eslint-config-thirdweb": "workspace:*",
"eslint-plugin-tsdoc": "^0.2.16",
esm: "^3.2.25",
mocha: "^10.2.0",
rimraf: "^3.0.2",
typedoc: "^0.25.2",
typescript: "^5.1.6"
},
dependencies: {
"cid-tool": "^3.0.0",
"form-data": "^4.0.0",
uuid: "^9.0.1",
"@thirdweb-dev/crypto": "workspace:*"
},
engines: {
node: ">=18"
}
};

@@ -357,3 +473,10 @@ /**

* const downloader = new StorageDownloader();
* const storage = new ThirdwebStorage({ downloader });
*
* // client id if used in client-side applications
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, downloader });
*
* // secret key if used in server-side applications
* const secretKey = "your-secret-key";
* const storage = new ThirdwebStorage({ secretKey, downloader });
* ```

@@ -364,25 +487,150 @@ *

class StorageDownloader {
async download(uri, gatewayUrls) {
let attempts = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
if (attempts > 3) {
throw new Error("[FAILED_TO_DOWNLOAD_ERROR] Failed to download from URI - too many attempts failed.");
DEFAULT_TIMEOUT_IN_SECONDS = 60;
DEFAULT_MAX_RETRIES = 3;
constructor(options) {
this.secretKey = options.secretKey;
this.clientId = options.clientId;
this.defaultTimeout = options.timeoutInSeconds || this.DEFAULT_TIMEOUT_IN_SECONDS;
}
async download(uri, gatewayUrls, options) {
let attempts = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 0;
const maxRetries = options?.maxRetries || this.DEFAULT_MAX_RETRIES;
if (attempts > maxRetries) {
console.error("[FAILED_TO_DOWNLOAD_ERROR] Failed to download from URI - too many attempts failed.");
// return a 404 response to avoid retrying
return new Response(JSON.stringify({
error: "Not Found"
}), {
status: 404,
headers: {
"Content-Type": "application/json"
}
});
}
// Replace recognized scheme with the highest priority gateway URL that hasn't already been attempted
const resolvedUri = replaceSchemeWithGatewayUrl(uri, gatewayUrls, attempts);
let resolvedUri = replaceSchemeWithGatewayUrl(uri, gatewayUrls, attempts, this.clientId);
// If every gateway URL we know about for the designated scheme has been tried (via recursion) and failed, throw an error
if (!resolvedUri) {
throw new Error("[FAILED_TO_DOWNLOAD_ERROR] Unable to download from URI - all gateway URLs failed to respond.");
console.error("[FAILED_TO_DOWNLOAD_ERROR] Unable to download from URI - all gateway URLs failed to respond.");
return new Response(JSON.stringify({
error: "Not Found"
}), {
status: 404,
headers: {
"Content-Type": "application/json"
}
});
} else if (attempts > 0) {
console.warn(`Retrying download with backup gateway URL: ${resolvedUri}`);
}
const res = await fetch__default["default"](resolvedUri);
let headers = {};
if (isTwGatewayUrl(resolvedUri)) {
const bundleId = typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis ? globalThis.APP_BUNDLE_ID : undefined;
if (this.secretKey) {
headers = {
"x-secret-key": this.secretKey
};
} else if (this.clientId) {
if (!resolvedUri.includes("bundleId")) {
resolvedUri = resolvedUri + (bundleId ? `?bundleId=${bundleId}` : "");
}
headers["x-client-Id"] = this.clientId;
}
// if we have a authorization token on global context then add that to the headers
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
headers = {
...headers,
authorization: `Bearer ${globalThis.TW_AUTH_TOKEN}`
};
}
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
headers = {
...headers,
authorization: `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`
};
headers["x-authorize-wallet"] = "true";
}
headers["x-sdk-version"] = pkg.version;
headers["x-sdk-name"] = pkg.name;
headers["x-sdk-platform"] = bundleId ? "react-native" : isBrowser() ? window.bridge !== undefined ? "webGL" : "browser" : "node";
}
if (isTooManyRequests(resolvedUri)) {
// skip the request if we're getting too many request error from the gateway
return this.download(uri, gatewayUrls, options, attempts + 1);
}
const controller = new AbortController();
const timeoutInSeconds = options?.timeoutInSeconds || this.defaultTimeout;
const timeout = setTimeout(() => controller.abort(), timeoutInSeconds * 1000);
const resOrErr = await fetch(resolvedUri, {
headers,
signal: controller.signal
}).catch(err => err);
// if we get here clear the timeout
if (timeout) {
clearTimeout(timeout);
}
if (!("status" in resOrErr)) {
// early exit if we don't have a status code
throw new Error(`Request timed out after ${timeoutInSeconds} seconds. ${isTwGatewayUrl(resolvedUri) ? "You can update the timeoutInSeconds option to increase the timeout." : "You're using a public IPFS gateway, pass in a clientId or secretKey for a reliable IPFS gateway."}`);
}
// If request to the current gateway fails, recursively try the next one we know about
if (res.status >= 500 || res.status === 403 || res.status === 408) {
console.warn(`Request to ${resolvedUri} failed with status ${res.status} - ${res.statusText}`);
return this.download(uri, gatewayUrls, attempts + 1);
// if the request is good we can skip everything else
if (resOrErr.ok) {
return resOrErr;
}
return res;
if (resOrErr.status === 429) {
// track that we got a too many requests error
tooManyRequestsBackOff(resolvedUri, resOrErr);
// Since the current gateway failed, recursively try the next one we know about
return this.download(uri, gatewayUrls, options, attempts + 1);
}
if (resOrErr.status === 410) {
// Don't retry if the content is blocklisted
console.error(`Request to ${resolvedUri} failed because this content seems to be blocklisted. Search VirusTotal for this URL to confirm: ${resolvedUri} `);
return resOrErr;
}
console.warn(`Request to ${resolvedUri} failed with status ${resOrErr.status} - ${resOrErr.statusText}`);
// if the status is 404 and we're using a thirdweb gateway url, return the response as is
if (resOrErr.status === 404 && isTwGatewayUrl(resolvedUri)) {
return resOrErr;
}
// these are the only errors that we want to retry, everything else we should just return the error as is
// 408 - Request Timeout
// 429 - Too Many Requests
// 5xx - Server Errors
if (resOrErr.status !== 408 && resOrErr.status !== 429 && resOrErr.status < 500) {
return resOrErr;
}
// Since the current gateway failed, recursively try the next one we know about
return this.download(uri, gatewayUrls, options, attempts + 1);
}
}
const TOO_MANY_REQUESTS_TRACKER = new Map();
function isTooManyRequests(gatewayUrl) {
return TOO_MANY_REQUESTS_TRACKER.has(gatewayUrl);
}
const TIMEOUT_MAP = new Map();
function tooManyRequestsBackOff(gatewayUrl, response) {
// if we already have a timeout for this gateway url, clear it
if (TIMEOUT_MAP.has(gatewayUrl)) {
clearTimeout(TIMEOUT_MAP.get(gatewayUrl));
}
const retryAfter = response.headers.get("Retry-After");
let backOff = 5000;
if (retryAfter) {
const retryAfterSeconds = parseInt(retryAfter);
if (!isNaN(retryAfterSeconds)) {
backOff = retryAfterSeconds * 1000;
}
}
// track that we got a too many requests error
TOO_MANY_REQUESTS_TRACKER.set(gatewayUrl, true);
TIMEOUT_MAP.set(gatewayUrl, setTimeout(() => TOO_MANY_REQUESTS_TRACKER.delete(gatewayUrl), backOff));
}
/**

@@ -393,6 +641,12 @@ * Default uploader used - handles uploading arbitrary data to IPFS

* ```jsx
* // Can instantiate the uploader with default configuration
* // Can instantiate the uploader with default configuration and your client ID when used in client-side applications
* const uploader = new StorageUploader();
* const storage = new ThirdwebStorage({ uploader });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader });
*
* // Can instantiate the uploader with default configuration and your secret key when used in server-side applications
* const uploader = new StorageUploader();
* const secretKey = "your-secret-key";
* const storage = new ThirdwebStorage({ secretKey, uploader });
*
* // Or optionally, can pass configuration

@@ -404,3 +658,4 @@ * const options = {

* const uploader = new StorageUploader(options);
* const storage = new ThirdwebStorage({ uploader });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader });
* ```

@@ -412,4 +667,6 @@ *

constructor(options) {
_defineProperty(this, "uploadWithGatewayUrl", void 0);
this.uploadWithGatewayUrl = options?.uploadWithGatewayUrl || false;
this.uploadServerUrl = options?.uploadServerUrl || TW_UPLOAD_SERVER_URL;
this.clientId = options?.clientId;
this.secretKey = options?.secretKey;
}

@@ -425,20 +682,2 @@ async uploadBatch(data, options) {

} = this.buildFormData(formData, data, options);
try {
const cid = await getCIDForUpload(data, fileNames.map(name => decodeURIComponent(name)), !options?.uploadWithoutDirectory);
if ((await isUploaded(cid)) && !options?.alwaysUpload) {
if (options?.onProgress) {
options?.onProgress({
progress: 100,
total: 100
});
}
if (options?.uploadWithoutDirectory) {
return [`ipfs://${cid}`];
} else {
return fileNames.map(name => `ipfs://${cid}/${name}`);
}
}
} catch {
// no-op
}
if (isBrowser()) {

@@ -450,24 +689,2 @@ return this.uploadBatchBrowser(form, fileNames, options);

}
/**
* Fetches a one-time-use upload token that can used to upload
* a file to storage.
*
* @returns - The one time use token that can be passed to the Pinata API.
*/
async getUploadToken() {
const res = await fetch__default["default"](`${TW_IPFS_SERVER_URL}/grant`, {
method: "GET",
headers: {
"X-APP-NAME":
// eslint-disable-next-line turbo/no-undeclared-env-vars
process.env.NODE_ENV === "test" || !!process.env.CI ? "Storage SDK CI" : "Storage SDK"
}
});
if (!res.ok) {
throw new Error(`Failed to get upload token`);
}
const body = await res.text();
return body;
}
buildFormData(form, files, options) {

@@ -555,3 +772,2 @@ const fileNameToFileMap = new Map();

async uploadBatchBrowser(form, fileNames, options) {
const token = await this.getUploadToken();
return new Promise((resolve, reject) => {

@@ -601,3 +817,3 @@ const xhr = new XMLHttpRequest();

} else {
return resolve(fileNames.map(name => `ipfs://${cid}/${name}`));
return resolve(fileNames.map(n => `ipfs://${cid}/${n}`));
}

@@ -611,8 +827,30 @@ }

if (xhr.readyState !== 0 && xhr.readyState !== 4 || xhr.status === 0) {
return reject(new Error("This looks like a network error, the endpoint might be blocked by an internet provider or a firewall."));
return reject(new Error("Upload failed due to a network error."));
}
return reject(new Error("Unknown upload error occured"));
});
xhr.open("POST", PINATA_IPFS_URL);
xhr.setRequestHeader("Authorization", `Bearer ${token}`);
xhr.open("POST", `${this.uploadServerUrl}/ipfs/upload`);
if (this.secretKey) {
xhr.setRequestHeader("x-secret-key", this.secretKey);
} else if (this.clientId) {
xhr.setRequestHeader("x-client-id", this.clientId);
}
const bundleId = typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis ? globalThis.APP_BUNDLE_ID : undefined;
if (bundleId) {
xhr.setRequestHeader("x-bundle-id", bundleId);
}
xhr.setRequestHeader("x-sdk-version", pkg.version);
xhr.setRequestHeader("x-sdk-name", pkg.name);
xhr.setRequestHeader("x-sdk-platform", bundleId ? "react-native" : isBrowser() ? window.bridge !== undefined ? "webGL" : "browser" : "node");
// if we have a authorization token on global context then add that to the headers, this is for the dashboard.
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
xhr.setRequestHeader("authorization", `Bearer ${globalThis.TW_AUTH_TOKEN}`);
}
// CLI auth token
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
xhr.setRequestHeader("authorization", `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`);
xhr.setRequestHeader("x-authorize-wallet", `true`);
}
xhr.send(form);

@@ -622,10 +860,31 @@ });

async uploadBatchNode(form, fileNames, options) {
const token = await this.getUploadToken();
if (options?.onProgress) {
console.warn("The onProgress option is only supported in the browser");
}
const res = await fetch__default["default"](PINATA_IPFS_URL, {
const headers = {};
if (this.secretKey) {
headers["x-secret-key"] = this.secretKey;
} else if (this.clientId) {
headers["x-client-id"] = this.clientId;
}
// if we have a bundle id on global context then add that to the headers
if (typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis) {
headers["x-bundle-id"] = globalThis.APP_BUNDLE_ID;
}
// if we have a authorization token on global context then add that to the headers, this is for the dashboard.
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
headers["authorization"] = `Bearer ${globalThis.TW_AUTH_TOKEN}`;
}
// CLI auth token
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
headers["authorization"] = `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`;
headers["x-authorize-wallet"] = "true";
}
const res = await fetch(`${this.uploadServerUrl}/ipfs/upload`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
...headers,
...form.getHeaders()

@@ -635,10 +894,12 @@ },

});
const body = await res.json();
if (!res.ok) {
console.warn(body);
throw new Error("Failed to upload files to IPFS");
if (res.status === 401) {
throw new Error("Unauthorized - You don't have permission to use this service.");
}
throw new Error(`Failed to upload files to IPFS - ${res.status} - ${res.statusText} - ${await res.text()}`);
}
const body = await res.json();
const cid = body.IpfsHash;
if (!cid) {
throw new Error("Failed to upload files to IPFS");
throw new Error("Failed to upload files to IPFS - Bad CID");
}

@@ -658,5 +919,10 @@ if (options?.uploadWithoutDirectory) {

* ```jsx
* // Create a default storage class without any configuration
* const storage = new ThirdwebStorage();
* // Create a default storage class with a client ID when used in client-side applications
* const storage = new ThirdwebStorage({ clientId: "your-client-id" });
*
* // Create a default storage class with a secret key when used in server-side applications
* const storage = new ThirdwebStorage({ secretKey: "your-secret-key" });
*
* You can get a clientId and secretKey from https://thirdweb.com/create-api-key
*
* // Upload any file or JSON object

@@ -677,3 +943,4 @@ * const uri = await storage.upload(data);

* const uploader = new IpfsUploader();
* const storage = new ThirdwebStorage({ uploader, downloader, gatewayUrls });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader, downloader, gatewayUrls });
* ```

@@ -685,8 +952,13 @@ *

constructor(options) {
_defineProperty(this, "uploader", void 0);
_defineProperty(this, "downloader", void 0);
_defineProperty(this, "gatewayUrls", void 0);
this.uploader = options?.uploader || new IpfsUploader();
this.downloader = options?.downloader || new StorageDownloader();
this.gatewayUrls = prepareGatewayUrls(parseGatewayUrls(options?.gatewayUrls));
this.uploader = options?.uploader || new IpfsUploader({
clientId: options?.clientId,
secretKey: options?.secretKey,
uploadServerUrl: options?.uploadServerUrl
});
this.downloader = options?.downloader || new StorageDownloader({
secretKey: options?.secretKey,
clientId: options?.clientId
});
this.gatewayUrls = prepareGatewayUrls(parseGatewayUrls(options?.gatewayUrls), options?.clientId, options?.secretKey);
this.clientId = options?.clientId;
}

@@ -708,3 +980,3 @@

resolveScheme(url) {
return replaceSchemeWithGatewayUrl(url, this.gatewayUrls);
return replaceSchemeWithGatewayUrl(url, this.gatewayUrls, 0, this.clientId);
}

@@ -724,4 +996,4 @@

*/
async download(url) {
return this.downloader.download(url, this.gatewayUrls);
async download(url, options) {
return this.downloader.download(url, this.gatewayUrls, options);
}

@@ -742,8 +1014,8 @@

*/
async downloadJSON(url) {
const res = await this.download(url);
async downloadJSON(url, options) {
const res = await this.download(url, options);
// If we get a JSON object, recursively replace any schemes with gatewayUrls
const json = await res.json();
return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls);
return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls, this.clientId);
}

@@ -827,2 +1099,5 @@

}
getGatewayUrls() {
return this.gatewayUrls;
}
async uploadAndReplaceFilesWithHashes(data, options) {

@@ -845,3 +1120,3 @@ let cleaned = data;

// Ex: used for Solana, where services don't resolve schemes for you, so URLs must be usable by default
cleaned = replaceObjectSchemesWithGatewayUrls(cleaned, this.gatewayUrls);
cleaned = replaceObjectSchemesWithGatewayUrls(cleaned, this.gatewayUrls, this.clientId);
}

@@ -856,5 +1131,4 @@ return cleaned;

class MockDownloader {
gatewayUrls = DEFAULT_GATEWAY_URLS;
constructor(storage) {
_defineProperty(this, "gatewayUrls", DEFAULT_GATEWAY_URLS);
_defineProperty(this, "storage", void 0);
this.storage = storage;

@@ -881,3 +1155,2 @@ }

constructor(storage) {
_defineProperty(this, "storage", void 0);
this.storage = storage;

@@ -919,7 +1192,7 @@ }

exports.StorageDownloader = StorageDownloader;
exports.TW_IPFS_SERVER_URL = TW_IPFS_SERVER_URL;
exports.TW_UPLOAD_SERVER_URL = TW_UPLOAD_SERVER_URL;
exports.ThirdwebStorage = ThirdwebStorage;
exports.convertCidToV1 = convertCidToV1;
exports.extractObjectFiles = extractObjectFiles;
exports.getCID = getCID;
exports.getCIDForUpload = getCIDForUpload;
exports.getGatewayUrlForCid = getGatewayUrlForCid;
exports.isBrowser = isBrowser;

@@ -931,3 +1204,3 @@ exports.isBufferInstance = isBufferInstance;

exports.isFileOrBuffer = isFileOrBuffer;
exports.isUploaded = isUploaded;
exports.isTwGatewayUrl = isTwGatewayUrl;
exports.parseGatewayUrls = parseGatewayUrls;

@@ -934,0 +1207,0 @@ exports.prepareGatewayUrls = prepareGatewayUrls;

653

dist/thirdweb-dev-storage.cjs.prod.js

@@ -5,4 +5,4 @@ 'use strict';

var fetch = require('cross-fetch');
var ipfsUnixfsImporter = require('ipfs-unixfs-importer');
var CIDTool = require('cid-tool');
var crypto = require('@thirdweb-dev/crypto');
var FormData = require('form-data');

@@ -13,35 +13,34 @@ var uuid = require('uuid');

var fetch__default = /*#__PURE__*/_interopDefault(fetch);
var CIDTool__default = /*#__PURE__*/_interopDefault(CIDTool);
var FormData__default = /*#__PURE__*/_interopDefault(FormData);
function _toPrimitive(input, hint) {
if (typeof input !== "object" || input === null) return input;
var prim = input[Symbol.toPrimitive];
if (prim !== undefined) {
var res = prim.call(input, hint || "default");
if (typeof res !== "object") return res;
throw new TypeError("@@toPrimitive must return a primitive value.");
function getProcessEnv(key) {
let defaultValue = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : "";
if (typeof process !== "undefined") {
if (process.env[key]) {
return process.env[key];
}
}
return (hint === "string" ? String : Number)(input);
return defaultValue;
}
function _toPropertyKey(arg) {
var key = _toPrimitive(arg, "string");
return typeof key === "symbol" ? key : String(key);
}
const TW_HOSTNAME_SUFFIX = ".ipfscdn.io";
const TW_STAGINGHOSTNAME_SUFFIX = ".thirdwebstorage-staging.com";
const TW_GATEWAY_URLS = [`https://{clientId}${TW_HOSTNAME_SUFFIX}/ipfs/{cid}/{path}`];
function _defineProperty(obj, key, value) {
key = _toPropertyKey(key);
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
/**
* @internal
* @param url - the url to check
* @returns
*/
function isTwGatewayUrl(url) {
const hostname = new URL(url).hostname;
const isProd = hostname.endsWith(TW_HOSTNAME_SUFFIX);
if (isProd) {
return true;
}
return obj;
// fall back to also handle staging urls
return hostname.endsWith(TW_STAGINGHOSTNAME_SUFFIX);
}
const PUBLIC_GATEWAY_URLS = ["https://{cid}.ipfs.cf-ipfs.com/{path}", "https://{cid}.ipfs.dweb.link/{path}", "https://ipfs.io/ipfs/{cid}/{path}", "https://cloudflare-ipfs.com/ipfs/{cid}/{path}", "https://{cid}.ipfs.w3s.link/{path}", "https://w3s.link/ipfs/{cid}/{path}", "https://nftstorage.link/ipfs/{cid}/{path}", "https://gateway.pinata.cloud/ipfs/{cid}/{path}"];

@@ -53,3 +52,3 @@ /**

// Note: Gateway URLs should have trailing slashes (we clean this on user input)
"ipfs://": ["https://ipfs.thirdwebcdn.com/ipfs/", "https://cloudflare-ipfs.com/ipfs/", "https://ipfs.io/ipfs/"]
"ipfs://": [...TW_GATEWAY_URLS, ...PUBLIC_GATEWAY_URLS]
};

@@ -60,3 +59,3 @@

*/
const TW_IPFS_SERVER_URL = "https://upload.nftlabs.co";
const TW_UPLOAD_SERVER_URL = getProcessEnv("CUSTOM_UPLOAD_SERVER_URL", "https://storage.thirdweb.com");

@@ -83,13 +82,60 @@ /**

*/
function prepareGatewayUrls(gatewayUrls) {
function getGatewayUrlForCid(gatewayUrl, cid, clientId) {
const parts = cid.split("/");
const hash = convertCidToV1(parts[0]);
const filePath = parts.slice(1).join("/");
let url = gatewayUrl;
// If the URL contains {cid} or {path} tokens, replace them with the CID and path
// Both tokens must be present for the URL to be valid
if (gatewayUrl.includes("{cid}") && gatewayUrl.includes("{path}")) {
url = url.replace("{cid}", hash).replace("{path}", filePath);
}
// If the URL contains only the {cid} token, replace it with the CID
else if (gatewayUrl.includes("{cid}")) {
url = url.replace("{cid}", hash);
}
// If those tokens don't exist, use the canonical gateway URL format
else {
url += `${hash}/${filePath}`;
}
// if the URL contains the {clientId} token, replace it with the client ID
if (gatewayUrl.includes("{clientId}")) {
if (!clientId) {
throw new Error("Cannot use {clientId} in gateway URL without providing a client ID");
}
url = url.replace("{clientId}", clientId);
}
return url;
}
/**
* @internal
*/
function prepareGatewayUrls(gatewayUrls, clientId, secretKey) {
const allGatewayUrls = {
...gatewayUrls,
...DEFAULT_GATEWAY_URLS
...DEFAULT_GATEWAY_URLS,
...gatewayUrls
};
for (const key of Object.keys(DEFAULT_GATEWAY_URLS)) {
if (gatewayUrls && gatewayUrls[key]) {
// Make sure that all user gateway URLs have trailing slashes
const cleanedGatewayUrls = gatewayUrls[key].map(url => url.replace(/\/$/, "") + "/");
allGatewayUrls[key] = [...cleanedGatewayUrls, ...DEFAULT_GATEWAY_URLS[key]];
}
for (const key of Object.keys(allGatewayUrls)) {
const cleanedGatewayUrls = allGatewayUrls[key].map(url => {
// inject clientId when present
if (clientId && url.includes("{clientId}")) {
return url.replace("{clientId}", clientId);
} else if (secretKey && url.includes("{clientId}")) {
// should only be used on Node.js in a backend/script context
if (typeof window !== "undefined") {
throw new Error("Cannot use secretKey in browser context");
}
const hashedSecretKey = crypto.sha256HexSync(secretKey);
const derivedClientId = hashedSecretKey.slice(0, 32);
return url.replace("{clientId}", derivedClientId);
} else if (url.includes("{clientId}")) {
// if no client id passed, filter out the url
return undefined;
} else {
return url;
}
}).filter(url => url !== undefined);
allGatewayUrls[key] = cleanedGatewayUrls;
}

@@ -102,2 +148,16 @@ return allGatewayUrls;

*/
function convertCidToV1(cid) {
let normalized;
try {
const hash = cid.split("/")[0];
normalized = CIDTool__default["default"].base32(hash);
} catch (e) {
throw new Error(`The CID ${cid} is not valid.`);
}
return normalized;
}
/**
* @internal
*/
function isBrowser() {

@@ -163,7 +223,38 @@ return typeof window !== "undefined";

*/
function parseCidAndPath(gatewayUrl, uri) {
const regexString = gatewayUrl.replace("{cid}", "(?<hash>[^/]+)").replace("{path}", "(?<path>[^?#]+)");
const regex = new RegExp(regexString);
const match = uri.match(regex);
if (match) {
const hash = match.groups?.hash;
const path = match.groups?.path;
const queryString = uri.includes("?") ? uri.substring(uri.indexOf("?") + 1) : "";
return {
hash,
path,
query: queryString
};
}
}
/**
* @internal
*/
function replaceGatewayUrlWithScheme(uri, gatewayUrls) {
for (const scheme of Object.keys(gatewayUrls)) {
for (const url of gatewayUrls[scheme]) {
if (uri.startsWith(url)) {
return uri.replace(url, scheme);
for (const gatewayUrl of gatewayUrls[scheme]) {
// If the url is a tokenized url, we need to convert it to a canonical url
// Otherwise, we just need to check if the url is a prefix of the uri
if (gatewayUrl.includes("{cid}")) {
// Given the url is a tokenized url, we need to lift the cid and the path from the uri
const parsed = parseCidAndPath(gatewayUrl, uri);
if (parsed?.hash && parsed?.path) {
const queryString = parsed?.query ? `?${parsed?.query}` : "";
return `${scheme}${parsed?.hash}/${parsed?.path}${queryString}`;
} else {
// If we can't lift the cid and path from the uri, we can't replace the gateway url, return the orig string
return uri;
}
} else if (uri.startsWith(gatewayUrl)) {
return uri.replace(gatewayUrl, scheme);
}

@@ -180,2 +271,3 @@ }

let index = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
let clientId = arguments.length > 3 ? arguments[3] : undefined;
const scheme = Object.keys(gatewayUrls).find(s => uri.startsWith(s));

@@ -189,3 +281,10 @@ const schemeGatewayUrls = scheme ? gatewayUrls[scheme] : [];

}
return uri.replace(scheme, schemeGatewayUrls[index]);
const path = uri.replace(scheme, "");
try {
const gatewayUrl = getGatewayUrlForCid(schemeGatewayUrls[index], path, clientId);
return gatewayUrl;
} catch (err) {
console.warn(`The IPFS uri: ${path} is not valid.`);
return undefined;
}
}

@@ -221,5 +320,5 @@

*/
function replaceObjectSchemesWithGatewayUrls(data, gatewayUrls) {
function replaceObjectSchemesWithGatewayUrls(data, gatewayUrls, clientId) {
if (typeof data === "string") {
return replaceSchemeWithGatewayUrl(data, gatewayUrls);
return replaceSchemeWithGatewayUrl(data, gatewayUrls, 0, clientId);
}

@@ -234,7 +333,7 @@ if (typeof data === "object") {

if (Array.isArray(data)) {
return data.map(entry => replaceObjectSchemesWithGatewayUrls(entry, gatewayUrls));
return data.map(entry => replaceObjectSchemesWithGatewayUrls(entry, gatewayUrls, clientId));
}
return Object.fromEntries(Object.entries(data).map(_ref2 => {
let [key, value] = _ref2;
return [key, replaceObjectSchemesWithGatewayUrls(value, gatewayUrls)];
return [key, replaceObjectSchemesWithGatewayUrls(value, gatewayUrls, clientId)];
}));

@@ -296,54 +395,71 @@ }

async function getCIDForUpload(data, fileNames) {
let wrapWithDirectory = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true;
let cidVersion = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 0;
const contentWithPath = await Promise.all(data.map(async (file, i) => {
const path = fileNames[i];
let content;
if (typeof file === "string") {
content = new TextEncoder().encode(file);
} else if (isBufferOrStringWithName(file)) {
if (typeof file.data === "string") {
content = new TextEncoder().encode(file.data);
} else {
content = file.data;
}
} else if (Buffer.isBuffer(file)) {
content = file;
} else {
const buffer = await file.arrayBuffer();
content = new Uint8Array(buffer);
}
return {
path,
content
};
}));
return getCID(contentWithPath, wrapWithDirectory, cidVersion);
}
async function getCID(content) {
let wrapWithDirectory = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;
let cidVersion = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
const options = {
onlyHash: true,
wrapWithDirectory,
cidVersion
};
const dummyBlockstore = {
put: async () => {}
};
let lastCid;
for await (const {
cid
} of ipfsUnixfsImporter.importer(content, dummyBlockstore, options)) {
lastCid = cid;
}
return `${lastCid}`;
}
async function isUploaded(cid) {
const res = await fetch__default["default"](`${DEFAULT_GATEWAY_URLS["ipfs://"][0]}${cid}`, {
method: "HEAD"
});
return res.ok;
}
var pkg = {
name: "@thirdweb-dev/storage",
version: "2.0.5",
main: "dist/thirdweb-dev-storage.cjs.js",
module: "dist/thirdweb-dev-storage.esm.js",
exports: {
".": {
module: "./dist/thirdweb-dev-storage.esm.js",
"default": "./dist/thirdweb-dev-storage.cjs.js"
},
"./package.json": "./package.json"
},
repository: "https://github.com/thirdweb-dev/js/tree/main/packages/storage",
author: "thirdweb eng <eng@thirdweb.com>",
license: "Apache-2.0",
sideEffects: false,
scripts: {
format: "prettier --write 'src/**/*'",
lint: "eslint src/ && bunx publint --strict --level warning",
fix: "eslint src/ --fix",
"generate-docs": "api-extractor run --local && api-documenter markdown -i ./temp -o ./docs",
clean: "rm -rf dist/",
build: "tsc && preconstruct build",
"test:all": "NODE_ENV=test SWC_NODE_PROJECT=./tsconfig.test.json mocha --timeout 30000 --parallel './test/**/*.test.ts'",
test: "pnpm test:all",
"test:single": "NODE_ENV=test SWC_NODE_PROJECT=./tsconfig.test.json mocha --timeout 30000",
push: "yalc push",
typedoc: "rimraf typedoc && node typedoc.js"
},
files: [
"dist/"
],
preconstruct: {
exports: true
},
devDependencies: {
"@babel/preset-env": "^7.22.9",
"@babel/preset-typescript": "^7.22.5",
"@microsoft/api-documenter": "^7.22.30",
"@microsoft/api-extractor": "^7.36.3",
"@microsoft/tsdoc": "^0.14.1",
"@preconstruct/cli": "2.7.0",
"@swc-node/register": "^1.6.6",
"@thirdweb-dev/tsconfig": "workspace:*",
"@types/chai": "^4.3.5",
"@types/mocha": "^10.0.0",
"@types/uuid": "^9.0.5",
"@typescript-eslint/eslint-plugin": "^6.2.0",
"@typescript-eslint/parser": "^6.2.0",
chai: "^4.3.6",
eslint: "^8.45.0",
"eslint-config-thirdweb": "workspace:*",
"eslint-plugin-tsdoc": "^0.2.16",
esm: "^3.2.25",
mocha: "^10.2.0",
rimraf: "^3.0.2",
typedoc: "^0.25.2",
typescript: "^5.1.6"
},
dependencies: {
"cid-tool": "^3.0.0",
"form-data": "^4.0.0",
uuid: "^9.0.1",
"@thirdweb-dev/crypto": "workspace:*"
},
engines: {
node: ">=18"
}
};

@@ -357,3 +473,10 @@ /**

* const downloader = new StorageDownloader();
* const storage = new ThirdwebStorage({ downloader });
*
* // client id if used in client-side applications
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, downloader });
*
* // secret key if used in server-side applications
* const secretKey = "your-secret-key";
* const storage = new ThirdwebStorage({ secretKey, downloader });
* ```

@@ -364,25 +487,150 @@ *

class StorageDownloader {
async download(uri, gatewayUrls) {
let attempts = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
if (attempts > 3) {
throw new Error("[FAILED_TO_DOWNLOAD_ERROR] Failed to download from URI - too many attempts failed.");
DEFAULT_TIMEOUT_IN_SECONDS = 60;
DEFAULT_MAX_RETRIES = 3;
constructor(options) {
this.secretKey = options.secretKey;
this.clientId = options.clientId;
this.defaultTimeout = options.timeoutInSeconds || this.DEFAULT_TIMEOUT_IN_SECONDS;
}
async download(uri, gatewayUrls, options) {
let attempts = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 0;
const maxRetries = options?.maxRetries || this.DEFAULT_MAX_RETRIES;
if (attempts > maxRetries) {
console.error("[FAILED_TO_DOWNLOAD_ERROR] Failed to download from URI - too many attempts failed.");
// return a 404 response to avoid retrying
return new Response(JSON.stringify({
error: "Not Found"
}), {
status: 404,
headers: {
"Content-Type": "application/json"
}
});
}
// Replace recognized scheme with the highest priority gateway URL that hasn't already been attempted
const resolvedUri = replaceSchemeWithGatewayUrl(uri, gatewayUrls, attempts);
let resolvedUri = replaceSchemeWithGatewayUrl(uri, gatewayUrls, attempts, this.clientId);
// If every gateway URL we know about for the designated scheme has been tried (via recursion) and failed, throw an error
if (!resolvedUri) {
throw new Error("[FAILED_TO_DOWNLOAD_ERROR] Unable to download from URI - all gateway URLs failed to respond.");
console.error("[FAILED_TO_DOWNLOAD_ERROR] Unable to download from URI - all gateway URLs failed to respond.");
return new Response(JSON.stringify({
error: "Not Found"
}), {
status: 404,
headers: {
"Content-Type": "application/json"
}
});
} else if (attempts > 0) {
console.warn(`Retrying download with backup gateway URL: ${resolvedUri}`);
}
const res = await fetch__default["default"](resolvedUri);
let headers = {};
if (isTwGatewayUrl(resolvedUri)) {
const bundleId = typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis ? globalThis.APP_BUNDLE_ID : undefined;
if (this.secretKey) {
headers = {
"x-secret-key": this.secretKey
};
} else if (this.clientId) {
if (!resolvedUri.includes("bundleId")) {
resolvedUri = resolvedUri + (bundleId ? `?bundleId=${bundleId}` : "");
}
headers["x-client-Id"] = this.clientId;
}
// if we have a authorization token on global context then add that to the headers
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
headers = {
...headers,
authorization: `Bearer ${globalThis.TW_AUTH_TOKEN}`
};
}
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
headers = {
...headers,
authorization: `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`
};
headers["x-authorize-wallet"] = "true";
}
headers["x-sdk-version"] = pkg.version;
headers["x-sdk-name"] = pkg.name;
headers["x-sdk-platform"] = bundleId ? "react-native" : isBrowser() ? window.bridge !== undefined ? "webGL" : "browser" : "node";
}
if (isTooManyRequests(resolvedUri)) {
// skip the request if we're getting too many request error from the gateway
return this.download(uri, gatewayUrls, options, attempts + 1);
}
const controller = new AbortController();
const timeoutInSeconds = options?.timeoutInSeconds || this.defaultTimeout;
const timeout = setTimeout(() => controller.abort(), timeoutInSeconds * 1000);
const resOrErr = await fetch(resolvedUri, {
headers,
signal: controller.signal
}).catch(err => err);
// if we get here clear the timeout
if (timeout) {
clearTimeout(timeout);
}
if (!("status" in resOrErr)) {
// early exit if we don't have a status code
throw new Error(`Request timed out after ${timeoutInSeconds} seconds. ${isTwGatewayUrl(resolvedUri) ? "You can update the timeoutInSeconds option to increase the timeout." : "You're using a public IPFS gateway, pass in a clientId or secretKey for a reliable IPFS gateway."}`);
}
// If request to the current gateway fails, recursively try the next one we know about
if (res.status >= 500 || res.status === 403 || res.status === 408) {
console.warn(`Request to ${resolvedUri} failed with status ${res.status} - ${res.statusText}`);
return this.download(uri, gatewayUrls, attempts + 1);
// if the request is good we can skip everything else
if (resOrErr.ok) {
return resOrErr;
}
return res;
if (resOrErr.status === 429) {
// track that we got a too many requests error
tooManyRequestsBackOff(resolvedUri, resOrErr);
// Since the current gateway failed, recursively try the next one we know about
return this.download(uri, gatewayUrls, options, attempts + 1);
}
if (resOrErr.status === 410) {
// Don't retry if the content is blocklisted
console.error(`Request to ${resolvedUri} failed because this content seems to be blocklisted. Search VirusTotal for this URL to confirm: ${resolvedUri} `);
return resOrErr;
}
console.warn(`Request to ${resolvedUri} failed with status ${resOrErr.status} - ${resOrErr.statusText}`);
// if the status is 404 and we're using a thirdweb gateway url, return the response as is
if (resOrErr.status === 404 && isTwGatewayUrl(resolvedUri)) {
return resOrErr;
}
// these are the only errors that we want to retry, everything else we should just return the error as is
// 408 - Request Timeout
// 429 - Too Many Requests
// 5xx - Server Errors
if (resOrErr.status !== 408 && resOrErr.status !== 429 && resOrErr.status < 500) {
return resOrErr;
}
// Since the current gateway failed, recursively try the next one we know about
return this.download(uri, gatewayUrls, options, attempts + 1);
}
}
const TOO_MANY_REQUESTS_TRACKER = new Map();
function isTooManyRequests(gatewayUrl) {
return TOO_MANY_REQUESTS_TRACKER.has(gatewayUrl);
}
const TIMEOUT_MAP = new Map();
function tooManyRequestsBackOff(gatewayUrl, response) {
// if we already have a timeout for this gateway url, clear it
if (TIMEOUT_MAP.has(gatewayUrl)) {
clearTimeout(TIMEOUT_MAP.get(gatewayUrl));
}
const retryAfter = response.headers.get("Retry-After");
let backOff = 5000;
if (retryAfter) {
const retryAfterSeconds = parseInt(retryAfter);
if (!isNaN(retryAfterSeconds)) {
backOff = retryAfterSeconds * 1000;
}
}
// track that we got a too many requests error
TOO_MANY_REQUESTS_TRACKER.set(gatewayUrl, true);
TIMEOUT_MAP.set(gatewayUrl, setTimeout(() => TOO_MANY_REQUESTS_TRACKER.delete(gatewayUrl), backOff));
}
/**

@@ -393,6 +641,12 @@ * Default uploader used - handles uploading arbitrary data to IPFS

* ```jsx
* // Can instantiate the uploader with default configuration
* // Can instantiate the uploader with default configuration and your client ID when used in client-side applications
* const uploader = new StorageUploader();
* const storage = new ThirdwebStorage({ uploader });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader });
*
* // Can instantiate the uploader with default configuration and your secret key when used in server-side applications
* const uploader = new StorageUploader();
* const secretKey = "your-secret-key";
* const storage = new ThirdwebStorage({ secretKey, uploader });
*
* // Or optionally, can pass configuration

@@ -404,3 +658,4 @@ * const options = {

* const uploader = new StorageUploader(options);
* const storage = new ThirdwebStorage({ uploader });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader });
* ```

@@ -412,4 +667,6 @@ *

constructor(options) {
_defineProperty(this, "uploadWithGatewayUrl", void 0);
this.uploadWithGatewayUrl = options?.uploadWithGatewayUrl || false;
this.uploadServerUrl = options?.uploadServerUrl || TW_UPLOAD_SERVER_URL;
this.clientId = options?.clientId;
this.secretKey = options?.secretKey;
}

@@ -425,20 +682,2 @@ async uploadBatch(data, options) {

} = this.buildFormData(formData, data, options);
try {
const cid = await getCIDForUpload(data, fileNames.map(name => decodeURIComponent(name)), !options?.uploadWithoutDirectory);
if ((await isUploaded(cid)) && !options?.alwaysUpload) {
if (options?.onProgress) {
options?.onProgress({
progress: 100,
total: 100
});
}
if (options?.uploadWithoutDirectory) {
return [`ipfs://${cid}`];
} else {
return fileNames.map(name => `ipfs://${cid}/${name}`);
}
}
} catch {
// no-op
}
if (isBrowser()) {

@@ -450,24 +689,2 @@ return this.uploadBatchBrowser(form, fileNames, options);

}
/**
* Fetches a one-time-use upload token that can used to upload
* a file to storage.
*
* @returns - The one time use token that can be passed to the Pinata API.
*/
async getUploadToken() {
const res = await fetch__default["default"](`${TW_IPFS_SERVER_URL}/grant`, {
method: "GET",
headers: {
"X-APP-NAME":
// eslint-disable-next-line turbo/no-undeclared-env-vars
!!process.env.CI ? "Storage SDK CI" : "Storage SDK"
}
});
if (!res.ok) {
throw new Error(`Failed to get upload token`);
}
const body = await res.text();
return body;
}
buildFormData(form, files, options) {

@@ -555,3 +772,2 @@ const fileNameToFileMap = new Map();

async uploadBatchBrowser(form, fileNames, options) {
const token = await this.getUploadToken();
return new Promise((resolve, reject) => {

@@ -601,3 +817,3 @@ const xhr = new XMLHttpRequest();

} else {
return resolve(fileNames.map(name => `ipfs://${cid}/${name}`));
return resolve(fileNames.map(n => `ipfs://${cid}/${n}`));
}

@@ -611,8 +827,30 @@ }

if (xhr.readyState !== 0 && xhr.readyState !== 4 || xhr.status === 0) {
return reject(new Error("This looks like a network error, the endpoint might be blocked by an internet provider or a firewall."));
return reject(new Error("Upload failed due to a network error."));
}
return reject(new Error("Unknown upload error occured"));
});
xhr.open("POST", PINATA_IPFS_URL);
xhr.setRequestHeader("Authorization", `Bearer ${token}`);
xhr.open("POST", `${this.uploadServerUrl}/ipfs/upload`);
if (this.secretKey) {
xhr.setRequestHeader("x-secret-key", this.secretKey);
} else if (this.clientId) {
xhr.setRequestHeader("x-client-id", this.clientId);
}
const bundleId = typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis ? globalThis.APP_BUNDLE_ID : undefined;
if (bundleId) {
xhr.setRequestHeader("x-bundle-id", bundleId);
}
xhr.setRequestHeader("x-sdk-version", pkg.version);
xhr.setRequestHeader("x-sdk-name", pkg.name);
xhr.setRequestHeader("x-sdk-platform", bundleId ? "react-native" : isBrowser() ? window.bridge !== undefined ? "webGL" : "browser" : "node");
// if we have a authorization token on global context then add that to the headers, this is for the dashboard.
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
xhr.setRequestHeader("authorization", `Bearer ${globalThis.TW_AUTH_TOKEN}`);
}
// CLI auth token
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
xhr.setRequestHeader("authorization", `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`);
xhr.setRequestHeader("x-authorize-wallet", `true`);
}
xhr.send(form);

@@ -622,10 +860,31 @@ });

async uploadBatchNode(form, fileNames, options) {
const token = await this.getUploadToken();
if (options?.onProgress) {
console.warn("The onProgress option is only supported in the browser");
}
const res = await fetch__default["default"](PINATA_IPFS_URL, {
const headers = {};
if (this.secretKey) {
headers["x-secret-key"] = this.secretKey;
} else if (this.clientId) {
headers["x-client-id"] = this.clientId;
}
// if we have a bundle id on global context then add that to the headers
if (typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis) {
headers["x-bundle-id"] = globalThis.APP_BUNDLE_ID;
}
// if we have a authorization token on global context then add that to the headers, this is for the dashboard.
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
headers["authorization"] = `Bearer ${globalThis.TW_AUTH_TOKEN}`;
}
// CLI auth token
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
headers["authorization"] = `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`;
headers["x-authorize-wallet"] = "true";
}
const res = await fetch(`${this.uploadServerUrl}/ipfs/upload`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
...headers,
...form.getHeaders()

@@ -635,10 +894,12 @@ },

});
const body = await res.json();
if (!res.ok) {
console.warn(body);
throw new Error("Failed to upload files to IPFS");
if (res.status === 401) {
throw new Error("Unauthorized - You don't have permission to use this service.");
}
throw new Error(`Failed to upload files to IPFS - ${res.status} - ${res.statusText} - ${await res.text()}`);
}
const body = await res.json();
const cid = body.IpfsHash;
if (!cid) {
throw new Error("Failed to upload files to IPFS");
throw new Error("Failed to upload files to IPFS - Bad CID");
}

@@ -658,5 +919,10 @@ if (options?.uploadWithoutDirectory) {

* ```jsx
* // Create a default storage class without any configuration
* const storage = new ThirdwebStorage();
* // Create a default storage class with a client ID when used in client-side applications
* const storage = new ThirdwebStorage({ clientId: "your-client-id" });
*
* // Create a default storage class with a secret key when used in server-side applications
* const storage = new ThirdwebStorage({ secretKey: "your-secret-key" });
*
* You can get a clientId and secretKey from https://thirdweb.com/create-api-key
*
* // Upload any file or JSON object

@@ -677,3 +943,4 @@ * const uri = await storage.upload(data);

* const uploader = new IpfsUploader();
* const storage = new ThirdwebStorage({ uploader, downloader, gatewayUrls });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader, downloader, gatewayUrls });
* ```

@@ -685,8 +952,13 @@ *

constructor(options) {
_defineProperty(this, "uploader", void 0);
_defineProperty(this, "downloader", void 0);
_defineProperty(this, "gatewayUrls", void 0);
this.uploader = options?.uploader || new IpfsUploader();
this.downloader = options?.downloader || new StorageDownloader();
this.gatewayUrls = prepareGatewayUrls(parseGatewayUrls(options?.gatewayUrls));
this.uploader = options?.uploader || new IpfsUploader({
clientId: options?.clientId,
secretKey: options?.secretKey,
uploadServerUrl: options?.uploadServerUrl
});
this.downloader = options?.downloader || new StorageDownloader({
secretKey: options?.secretKey,
clientId: options?.clientId
});
this.gatewayUrls = prepareGatewayUrls(parseGatewayUrls(options?.gatewayUrls), options?.clientId, options?.secretKey);
this.clientId = options?.clientId;
}

@@ -708,3 +980,3 @@

resolveScheme(url) {
return replaceSchemeWithGatewayUrl(url, this.gatewayUrls);
return replaceSchemeWithGatewayUrl(url, this.gatewayUrls, 0, this.clientId);
}

@@ -724,4 +996,4 @@

*/
async download(url) {
return this.downloader.download(url, this.gatewayUrls);
async download(url, options) {
return this.downloader.download(url, this.gatewayUrls, options);
}

@@ -742,8 +1014,8 @@

*/
async downloadJSON(url) {
const res = await this.download(url);
async downloadJSON(url, options) {
const res = await this.download(url, options);
// If we get a JSON object, recursively replace any schemes with gatewayUrls
const json = await res.json();
return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls);
return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls, this.clientId);
}

@@ -827,2 +1099,5 @@

}
getGatewayUrls() {
return this.gatewayUrls;
}
async uploadAndReplaceFilesWithHashes(data, options) {

@@ -845,3 +1120,3 @@ let cleaned = data;

// Ex: used for Solana, where services don't resolve schemes for you, so URLs must be usable by default
cleaned = replaceObjectSchemesWithGatewayUrls(cleaned, this.gatewayUrls);
cleaned = replaceObjectSchemesWithGatewayUrls(cleaned, this.gatewayUrls, this.clientId);
}

@@ -856,5 +1131,4 @@ return cleaned;

class MockDownloader {
gatewayUrls = DEFAULT_GATEWAY_URLS;
constructor(storage) {
_defineProperty(this, "gatewayUrls", DEFAULT_GATEWAY_URLS);
_defineProperty(this, "storage", void 0);
this.storage = storage;

@@ -881,3 +1155,2 @@ }

constructor(storage) {
_defineProperty(this, "storage", void 0);
this.storage = storage;

@@ -919,7 +1192,7 @@ }

exports.StorageDownloader = StorageDownloader;
exports.TW_IPFS_SERVER_URL = TW_IPFS_SERVER_URL;
exports.TW_UPLOAD_SERVER_URL = TW_UPLOAD_SERVER_URL;
exports.ThirdwebStorage = ThirdwebStorage;
exports.convertCidToV1 = convertCidToV1;
exports.extractObjectFiles = extractObjectFiles;
exports.getCID = getCID;
exports.getCIDForUpload = getCIDForUpload;
exports.getGatewayUrlForCid = getGatewayUrlForCid;
exports.isBrowser = isBrowser;

@@ -931,3 +1204,3 @@ exports.isBufferInstance = isBufferInstance;

exports.isFileOrBuffer = isFileOrBuffer;
exports.isUploaded = isUploaded;
exports.isTwGatewayUrl = isTwGatewayUrl;
exports.parseGatewayUrls = parseGatewayUrls;

@@ -934,0 +1207,0 @@ exports.prepareGatewayUrls = prepareGatewayUrls;

645

dist/thirdweb-dev-storage.esm.js

@@ -1,36 +0,35 @@

import fetch from 'cross-fetch';
import { importer } from 'ipfs-unixfs-importer';
import CIDTool from 'cid-tool';
import { sha256HexSync } from '@thirdweb-dev/crypto';
import FormData from 'form-data';
import { v4 } from 'uuid';
function _toPrimitive(input, hint) {
if (typeof input !== "object" || input === null) return input;
var prim = input[Symbol.toPrimitive];
if (prim !== undefined) {
var res = prim.call(input, hint || "default");
if (typeof res !== "object") return res;
throw new TypeError("@@toPrimitive must return a primitive value.");
function getProcessEnv(key) {
let defaultValue = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : "";
if (typeof process !== "undefined") {
if (process.env[key]) {
return process.env[key];
}
}
return (hint === "string" ? String : Number)(input);
return defaultValue;
}
function _toPropertyKey(arg) {
var key = _toPrimitive(arg, "string");
return typeof key === "symbol" ? key : String(key);
}
const TW_HOSTNAME_SUFFIX = ".ipfscdn.io";
const TW_STAGINGHOSTNAME_SUFFIX = ".thirdwebstorage-staging.com";
const TW_GATEWAY_URLS = [`https://{clientId}${TW_HOSTNAME_SUFFIX}/ipfs/{cid}/{path}`];
function _defineProperty(obj, key, value) {
key = _toPropertyKey(key);
if (key in obj) {
Object.defineProperty(obj, key, {
value: value,
enumerable: true,
configurable: true,
writable: true
});
} else {
obj[key] = value;
/**
* @internal
* @param url - the url to check
* @returns
*/
function isTwGatewayUrl(url) {
const hostname = new URL(url).hostname;
const isProd = hostname.endsWith(TW_HOSTNAME_SUFFIX);
if (isProd) {
return true;
}
return obj;
// fall back to also handle staging urls
return hostname.endsWith(TW_STAGINGHOSTNAME_SUFFIX);
}
const PUBLIC_GATEWAY_URLS = ["https://{cid}.ipfs.cf-ipfs.com/{path}", "https://{cid}.ipfs.dweb.link/{path}", "https://ipfs.io/ipfs/{cid}/{path}", "https://cloudflare-ipfs.com/ipfs/{cid}/{path}", "https://{cid}.ipfs.w3s.link/{path}", "https://w3s.link/ipfs/{cid}/{path}", "https://nftstorage.link/ipfs/{cid}/{path}", "https://gateway.pinata.cloud/ipfs/{cid}/{path}"];

@@ -42,3 +41,3 @@ /**

// Note: Gateway URLs should have trailing slashes (we clean this on user input)
"ipfs://": ["https://ipfs.thirdwebcdn.com/ipfs/", "https://cloudflare-ipfs.com/ipfs/", "https://ipfs.io/ipfs/"]
"ipfs://": [...TW_GATEWAY_URLS, ...PUBLIC_GATEWAY_URLS]
};

@@ -49,3 +48,3 @@

*/
const TW_IPFS_SERVER_URL = "https://upload.nftlabs.co";
const TW_UPLOAD_SERVER_URL = getProcessEnv("CUSTOM_UPLOAD_SERVER_URL", "https://storage.thirdweb.com");

@@ -72,13 +71,60 @@ /**

*/
function prepareGatewayUrls(gatewayUrls) {
function getGatewayUrlForCid(gatewayUrl, cid, clientId) {
const parts = cid.split("/");
const hash = convertCidToV1(parts[0]);
const filePath = parts.slice(1).join("/");
let url = gatewayUrl;
// If the URL contains {cid} or {path} tokens, replace them with the CID and path
// Both tokens must be present for the URL to be valid
if (gatewayUrl.includes("{cid}") && gatewayUrl.includes("{path}")) {
url = url.replace("{cid}", hash).replace("{path}", filePath);
}
// If the URL contains only the {cid} token, replace it with the CID
else if (gatewayUrl.includes("{cid}")) {
url = url.replace("{cid}", hash);
}
// If those tokens don't exist, use the canonical gateway URL format
else {
url += `${hash}/${filePath}`;
}
// if the URL contains the {clientId} token, replace it with the client ID
if (gatewayUrl.includes("{clientId}")) {
if (!clientId) {
throw new Error("Cannot use {clientId} in gateway URL without providing a client ID");
}
url = url.replace("{clientId}", clientId);
}
return url;
}
/**
* @internal
*/
function prepareGatewayUrls(gatewayUrls, clientId, secretKey) {
const allGatewayUrls = {
...gatewayUrls,
...DEFAULT_GATEWAY_URLS
...DEFAULT_GATEWAY_URLS,
...gatewayUrls
};
for (const key of Object.keys(DEFAULT_GATEWAY_URLS)) {
if (gatewayUrls && gatewayUrls[key]) {
// Make sure that all user gateway URLs have trailing slashes
const cleanedGatewayUrls = gatewayUrls[key].map(url => url.replace(/\/$/, "") + "/");
allGatewayUrls[key] = [...cleanedGatewayUrls, ...DEFAULT_GATEWAY_URLS[key]];
}
for (const key of Object.keys(allGatewayUrls)) {
const cleanedGatewayUrls = allGatewayUrls[key].map(url => {
// inject clientId when present
if (clientId && url.includes("{clientId}")) {
return url.replace("{clientId}", clientId);
} else if (secretKey && url.includes("{clientId}")) {
// should only be used on Node.js in a backend/script context
if (typeof window !== "undefined") {
throw new Error("Cannot use secretKey in browser context");
}
const hashedSecretKey = sha256HexSync(secretKey);
const derivedClientId = hashedSecretKey.slice(0, 32);
return url.replace("{clientId}", derivedClientId);
} else if (url.includes("{clientId}")) {
// if no client id passed, filter out the url
return undefined;
} else {
return url;
}
}).filter(url => url !== undefined);
allGatewayUrls[key] = cleanedGatewayUrls;
}

@@ -91,2 +137,16 @@ return allGatewayUrls;

*/
function convertCidToV1(cid) {
let normalized;
try {
const hash = cid.split("/")[0];
normalized = CIDTool.base32(hash);
} catch (e) {
throw new Error(`The CID ${cid} is not valid.`);
}
return normalized;
}
/**
* @internal
*/
function isBrowser() {

@@ -152,7 +212,38 @@ return typeof window !== "undefined";

*/
function parseCidAndPath(gatewayUrl, uri) {
const regexString = gatewayUrl.replace("{cid}", "(?<hash>[^/]+)").replace("{path}", "(?<path>[^?#]+)");
const regex = new RegExp(regexString);
const match = uri.match(regex);
if (match) {
const hash = match.groups?.hash;
const path = match.groups?.path;
const queryString = uri.includes("?") ? uri.substring(uri.indexOf("?") + 1) : "";
return {
hash,
path,
query: queryString
};
}
}
/**
* @internal
*/
function replaceGatewayUrlWithScheme(uri, gatewayUrls) {
for (const scheme of Object.keys(gatewayUrls)) {
for (const url of gatewayUrls[scheme]) {
if (uri.startsWith(url)) {
return uri.replace(url, scheme);
for (const gatewayUrl of gatewayUrls[scheme]) {
// If the url is a tokenized url, we need to convert it to a canonical url
// Otherwise, we just need to check if the url is a prefix of the uri
if (gatewayUrl.includes("{cid}")) {
// Given the url is a tokenized url, we need to lift the cid and the path from the uri
const parsed = parseCidAndPath(gatewayUrl, uri);
if (parsed?.hash && parsed?.path) {
const queryString = parsed?.query ? `?${parsed?.query}` : "";
return `${scheme}${parsed?.hash}/${parsed?.path}${queryString}`;
} else {
// If we can't lift the cid and path from the uri, we can't replace the gateway url, return the orig string
return uri;
}
} else if (uri.startsWith(gatewayUrl)) {
return uri.replace(gatewayUrl, scheme);
}

@@ -169,2 +260,3 @@ }

let index = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
let clientId = arguments.length > 3 ? arguments[3] : undefined;
const scheme = Object.keys(gatewayUrls).find(s => uri.startsWith(s));

@@ -178,3 +270,10 @@ const schemeGatewayUrls = scheme ? gatewayUrls[scheme] : [];

}
return uri.replace(scheme, schemeGatewayUrls[index]);
const path = uri.replace(scheme, "");
try {
const gatewayUrl = getGatewayUrlForCid(schemeGatewayUrls[index], path, clientId);
return gatewayUrl;
} catch (err) {
console.warn(`The IPFS uri: ${path} is not valid.`);
return undefined;
}
}

@@ -210,5 +309,5 @@

*/
function replaceObjectSchemesWithGatewayUrls(data, gatewayUrls) {
function replaceObjectSchemesWithGatewayUrls(data, gatewayUrls, clientId) {
if (typeof data === "string") {
return replaceSchemeWithGatewayUrl(data, gatewayUrls);
return replaceSchemeWithGatewayUrl(data, gatewayUrls, 0, clientId);
}

@@ -223,7 +322,7 @@ if (typeof data === "object") {

if (Array.isArray(data)) {
return data.map(entry => replaceObjectSchemesWithGatewayUrls(entry, gatewayUrls));
return data.map(entry => replaceObjectSchemesWithGatewayUrls(entry, gatewayUrls, clientId));
}
return Object.fromEntries(Object.entries(data).map(_ref2 => {
let [key, value] = _ref2;
return [key, replaceObjectSchemesWithGatewayUrls(value, gatewayUrls)];
return [key, replaceObjectSchemesWithGatewayUrls(value, gatewayUrls, clientId)];
}));

@@ -285,54 +384,71 @@ }

async function getCIDForUpload(data, fileNames) {
let wrapWithDirectory = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true;
let cidVersion = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 0;
const contentWithPath = await Promise.all(data.map(async (file, i) => {
const path = fileNames[i];
let content;
if (typeof file === "string") {
content = new TextEncoder().encode(file);
} else if (isBufferOrStringWithName(file)) {
if (typeof file.data === "string") {
content = new TextEncoder().encode(file.data);
} else {
content = file.data;
}
} else if (Buffer.isBuffer(file)) {
content = file;
} else {
const buffer = await file.arrayBuffer();
content = new Uint8Array(buffer);
}
return {
path,
content
};
}));
return getCID(contentWithPath, wrapWithDirectory, cidVersion);
}
async function getCID(content) {
let wrapWithDirectory = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;
let cidVersion = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
const options = {
onlyHash: true,
wrapWithDirectory,
cidVersion
};
const dummyBlockstore = {
put: async () => {}
};
let lastCid;
for await (const {
cid
} of importer(content, dummyBlockstore, options)) {
lastCid = cid;
}
return `${lastCid}`;
}
async function isUploaded(cid) {
const res = await fetch(`${DEFAULT_GATEWAY_URLS["ipfs://"][0]}${cid}`, {
method: "HEAD"
});
return res.ok;
}
var pkg = {
name: "@thirdweb-dev/storage",
version: "2.0.5",
main: "dist/thirdweb-dev-storage.cjs.js",
module: "dist/thirdweb-dev-storage.esm.js",
exports: {
".": {
module: "./dist/thirdweb-dev-storage.esm.js",
"default": "./dist/thirdweb-dev-storage.cjs.js"
},
"./package.json": "./package.json"
},
repository: "https://github.com/thirdweb-dev/js/tree/main/packages/storage",
author: "thirdweb eng <eng@thirdweb.com>",
license: "Apache-2.0",
sideEffects: false,
scripts: {
format: "prettier --write 'src/**/*'",
lint: "eslint src/ && bunx publint --strict --level warning",
fix: "eslint src/ --fix",
"generate-docs": "api-extractor run --local && api-documenter markdown -i ./temp -o ./docs",
clean: "rm -rf dist/",
build: "tsc && preconstruct build",
"test:all": "NODE_ENV=test SWC_NODE_PROJECT=./tsconfig.test.json mocha --timeout 30000 --parallel './test/**/*.test.ts'",
test: "pnpm test:all",
"test:single": "NODE_ENV=test SWC_NODE_PROJECT=./tsconfig.test.json mocha --timeout 30000",
push: "yalc push",
typedoc: "rimraf typedoc && node typedoc.js"
},
files: [
"dist/"
],
preconstruct: {
exports: true
},
devDependencies: {
"@babel/preset-env": "^7.22.9",
"@babel/preset-typescript": "^7.22.5",
"@microsoft/api-documenter": "^7.22.30",
"@microsoft/api-extractor": "^7.36.3",
"@microsoft/tsdoc": "^0.14.1",
"@preconstruct/cli": "2.7.0",
"@swc-node/register": "^1.6.6",
"@thirdweb-dev/tsconfig": "workspace:*",
"@types/chai": "^4.3.5",
"@types/mocha": "^10.0.0",
"@types/uuid": "^9.0.5",
"@typescript-eslint/eslint-plugin": "^6.2.0",
"@typescript-eslint/parser": "^6.2.0",
chai: "^4.3.6",
eslint: "^8.45.0",
"eslint-config-thirdweb": "workspace:*",
"eslint-plugin-tsdoc": "^0.2.16",
esm: "^3.2.25",
mocha: "^10.2.0",
rimraf: "^3.0.2",
typedoc: "^0.25.2",
typescript: "^5.1.6"
},
dependencies: {
"cid-tool": "^3.0.0",
"form-data": "^4.0.0",
uuid: "^9.0.1",
"@thirdweb-dev/crypto": "workspace:*"
},
engines: {
node: ">=18"
}
};

@@ -346,3 +462,10 @@ /**

* const downloader = new StorageDownloader();
* const storage = new ThirdwebStorage({ downloader });
*
* // client id if used in client-side applications
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, downloader });
*
* // secret key if used in server-side applications
* const secretKey = "your-secret-key";
* const storage = new ThirdwebStorage({ secretKey, downloader });
* ```

@@ -353,25 +476,150 @@ *

class StorageDownloader {
async download(uri, gatewayUrls) {
let attempts = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : 0;
if (attempts > 3) {
throw new Error("[FAILED_TO_DOWNLOAD_ERROR] Failed to download from URI - too many attempts failed.");
DEFAULT_TIMEOUT_IN_SECONDS = 60;
DEFAULT_MAX_RETRIES = 3;
constructor(options) {
this.secretKey = options.secretKey;
this.clientId = options.clientId;
this.defaultTimeout = options.timeoutInSeconds || this.DEFAULT_TIMEOUT_IN_SECONDS;
}
async download(uri, gatewayUrls, options) {
let attempts = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : 0;
const maxRetries = options?.maxRetries || this.DEFAULT_MAX_RETRIES;
if (attempts > maxRetries) {
console.error("[FAILED_TO_DOWNLOAD_ERROR] Failed to download from URI - too many attempts failed.");
// return a 404 response to avoid retrying
return new Response(JSON.stringify({
error: "Not Found"
}), {
status: 404,
headers: {
"Content-Type": "application/json"
}
});
}
// Replace recognized scheme with the highest priority gateway URL that hasn't already been attempted
const resolvedUri = replaceSchemeWithGatewayUrl(uri, gatewayUrls, attempts);
let resolvedUri = replaceSchemeWithGatewayUrl(uri, gatewayUrls, attempts, this.clientId);
// If every gateway URL we know about for the designated scheme has been tried (via recursion) and failed, throw an error
if (!resolvedUri) {
throw new Error("[FAILED_TO_DOWNLOAD_ERROR] Unable to download from URI - all gateway URLs failed to respond.");
console.error("[FAILED_TO_DOWNLOAD_ERROR] Unable to download from URI - all gateway URLs failed to respond.");
return new Response(JSON.stringify({
error: "Not Found"
}), {
status: 404,
headers: {
"Content-Type": "application/json"
}
});
} else if (attempts > 0) {
console.warn(`Retrying download with backup gateway URL: ${resolvedUri}`);
}
const res = await fetch(resolvedUri);
let headers = {};
if (isTwGatewayUrl(resolvedUri)) {
const bundleId = typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis ? globalThis.APP_BUNDLE_ID : undefined;
if (this.secretKey) {
headers = {
"x-secret-key": this.secretKey
};
} else if (this.clientId) {
if (!resolvedUri.includes("bundleId")) {
resolvedUri = resolvedUri + (bundleId ? `?bundleId=${bundleId}` : "");
}
headers["x-client-Id"] = this.clientId;
}
// if we have a authorization token on global context then add that to the headers
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
headers = {
...headers,
authorization: `Bearer ${globalThis.TW_AUTH_TOKEN}`
};
}
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
headers = {
...headers,
authorization: `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`
};
headers["x-authorize-wallet"] = "true";
}
headers["x-sdk-version"] = pkg.version;
headers["x-sdk-name"] = pkg.name;
headers["x-sdk-platform"] = bundleId ? "react-native" : isBrowser() ? window.bridge !== undefined ? "webGL" : "browser" : "node";
}
if (isTooManyRequests(resolvedUri)) {
// skip the request if we're getting too many request error from the gateway
return this.download(uri, gatewayUrls, options, attempts + 1);
}
const controller = new AbortController();
const timeoutInSeconds = options?.timeoutInSeconds || this.defaultTimeout;
const timeout = setTimeout(() => controller.abort(), timeoutInSeconds * 1000);
const resOrErr = await fetch(resolvedUri, {
headers,
signal: controller.signal
}).catch(err => err);
// if we get here clear the timeout
if (timeout) {
clearTimeout(timeout);
}
if (!("status" in resOrErr)) {
// early exit if we don't have a status code
throw new Error(`Request timed out after ${timeoutInSeconds} seconds. ${isTwGatewayUrl(resolvedUri) ? "You can update the timeoutInSeconds option to increase the timeout." : "You're using a public IPFS gateway, pass in a clientId or secretKey for a reliable IPFS gateway."}`);
}
// If request to the current gateway fails, recursively try the next one we know about
if (res.status >= 500 || res.status === 403 || res.status === 408) {
console.warn(`Request to ${resolvedUri} failed with status ${res.status} - ${res.statusText}`);
return this.download(uri, gatewayUrls, attempts + 1);
// if the request is good we can skip everything else
if (resOrErr.ok) {
return resOrErr;
}
return res;
if (resOrErr.status === 429) {
// track that we got a too many requests error
tooManyRequestsBackOff(resolvedUri, resOrErr);
// Since the current gateway failed, recursively try the next one we know about
return this.download(uri, gatewayUrls, options, attempts + 1);
}
if (resOrErr.status === 410) {
// Don't retry if the content is blocklisted
console.error(`Request to ${resolvedUri} failed because this content seems to be blocklisted. Search VirusTotal for this URL to confirm: ${resolvedUri} `);
return resOrErr;
}
console.warn(`Request to ${resolvedUri} failed with status ${resOrErr.status} - ${resOrErr.statusText}`);
// if the status is 404 and we're using a thirdweb gateway url, return the response as is
if (resOrErr.status === 404 && isTwGatewayUrl(resolvedUri)) {
return resOrErr;
}
// these are the only errors that we want to retry, everything else we should just return the error as is
// 408 - Request Timeout
// 429 - Too Many Requests
// 5xx - Server Errors
if (resOrErr.status !== 408 && resOrErr.status !== 429 && resOrErr.status < 500) {
return resOrErr;
}
// Since the current gateway failed, recursively try the next one we know about
return this.download(uri, gatewayUrls, options, attempts + 1);
}
}
const TOO_MANY_REQUESTS_TRACKER = new Map();
function isTooManyRequests(gatewayUrl) {
return TOO_MANY_REQUESTS_TRACKER.has(gatewayUrl);
}
const TIMEOUT_MAP = new Map();
function tooManyRequestsBackOff(gatewayUrl, response) {
// if we already have a timeout for this gateway url, clear it
if (TIMEOUT_MAP.has(gatewayUrl)) {
clearTimeout(TIMEOUT_MAP.get(gatewayUrl));
}
const retryAfter = response.headers.get("Retry-After");
let backOff = 5000;
if (retryAfter) {
const retryAfterSeconds = parseInt(retryAfter);
if (!isNaN(retryAfterSeconds)) {
backOff = retryAfterSeconds * 1000;
}
}
// track that we got a too many requests error
TOO_MANY_REQUESTS_TRACKER.set(gatewayUrl, true);
TIMEOUT_MAP.set(gatewayUrl, setTimeout(() => TOO_MANY_REQUESTS_TRACKER.delete(gatewayUrl), backOff));
}
/**

@@ -382,6 +630,12 @@ * Default uploader used - handles uploading arbitrary data to IPFS

* ```jsx
* // Can instantiate the uploader with default configuration
* // Can instantiate the uploader with default configuration and your client ID when used in client-side applications
* const uploader = new StorageUploader();
* const storage = new ThirdwebStorage({ uploader });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader });
*
* // Can instantiate the uploader with default configuration and your secret key when used in server-side applications
* const uploader = new StorageUploader();
* const secretKey = "your-secret-key";
* const storage = new ThirdwebStorage({ secretKey, uploader });
*
* // Or optionally, can pass configuration

@@ -393,3 +647,4 @@ * const options = {

* const uploader = new StorageUploader(options);
* const storage = new ThirdwebStorage({ uploader });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader });
* ```

@@ -401,4 +656,6 @@ *

constructor(options) {
_defineProperty(this, "uploadWithGatewayUrl", void 0);
this.uploadWithGatewayUrl = options?.uploadWithGatewayUrl || false;
this.uploadServerUrl = options?.uploadServerUrl || TW_UPLOAD_SERVER_URL;
this.clientId = options?.clientId;
this.secretKey = options?.secretKey;
}

@@ -414,20 +671,2 @@ async uploadBatch(data, options) {

} = this.buildFormData(formData, data, options);
try {
const cid = await getCIDForUpload(data, fileNames.map(name => decodeURIComponent(name)), !options?.uploadWithoutDirectory);
if ((await isUploaded(cid)) && !options?.alwaysUpload) {
if (options?.onProgress) {
options?.onProgress({
progress: 100,
total: 100
});
}
if (options?.uploadWithoutDirectory) {
return [`ipfs://${cid}`];
} else {
return fileNames.map(name => `ipfs://${cid}/${name}`);
}
}
} catch {
// no-op
}
if (isBrowser()) {

@@ -439,24 +678,2 @@ return this.uploadBatchBrowser(form, fileNames, options);

}
/**
* Fetches a one-time-use upload token that can used to upload
* a file to storage.
*
* @returns - The one time use token that can be passed to the Pinata API.
*/
async getUploadToken() {
const res = await fetch(`${TW_IPFS_SERVER_URL}/grant`, {
method: "GET",
headers: {
"X-APP-NAME":
// eslint-disable-next-line turbo/no-undeclared-env-vars
process.env.NODE_ENV === "test" || !!process.env.CI ? "Storage SDK CI" : "Storage SDK"
}
});
if (!res.ok) {
throw new Error(`Failed to get upload token`);
}
const body = await res.text();
return body;
}
buildFormData(form, files, options) {

@@ -544,3 +761,2 @@ const fileNameToFileMap = new Map();

async uploadBatchBrowser(form, fileNames, options) {
const token = await this.getUploadToken();
return new Promise((resolve, reject) => {

@@ -590,3 +806,3 @@ const xhr = new XMLHttpRequest();

} else {
return resolve(fileNames.map(name => `ipfs://${cid}/${name}`));
return resolve(fileNames.map(n => `ipfs://${cid}/${n}`));
}

@@ -600,8 +816,30 @@ }

if (xhr.readyState !== 0 && xhr.readyState !== 4 || xhr.status === 0) {
return reject(new Error("This looks like a network error, the endpoint might be blocked by an internet provider or a firewall."));
return reject(new Error("Upload failed due to a network error."));
}
return reject(new Error("Unknown upload error occured"));
});
xhr.open("POST", PINATA_IPFS_URL);
xhr.setRequestHeader("Authorization", `Bearer ${token}`);
xhr.open("POST", `${this.uploadServerUrl}/ipfs/upload`);
if (this.secretKey) {
xhr.setRequestHeader("x-secret-key", this.secretKey);
} else if (this.clientId) {
xhr.setRequestHeader("x-client-id", this.clientId);
}
const bundleId = typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis ? globalThis.APP_BUNDLE_ID : undefined;
if (bundleId) {
xhr.setRequestHeader("x-bundle-id", bundleId);
}
xhr.setRequestHeader("x-sdk-version", pkg.version);
xhr.setRequestHeader("x-sdk-name", pkg.name);
xhr.setRequestHeader("x-sdk-platform", bundleId ? "react-native" : isBrowser() ? window.bridge !== undefined ? "webGL" : "browser" : "node");
// if we have a authorization token on global context then add that to the headers, this is for the dashboard.
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
xhr.setRequestHeader("authorization", `Bearer ${globalThis.TW_AUTH_TOKEN}`);
}
// CLI auth token
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
xhr.setRequestHeader("authorization", `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`);
xhr.setRequestHeader("x-authorize-wallet", `true`);
}
xhr.send(form);

@@ -611,10 +849,31 @@ });

async uploadBatchNode(form, fileNames, options) {
const token = await this.getUploadToken();
if (options?.onProgress) {
console.warn("The onProgress option is only supported in the browser");
}
const res = await fetch(PINATA_IPFS_URL, {
const headers = {};
if (this.secretKey) {
headers["x-secret-key"] = this.secretKey;
} else if (this.clientId) {
headers["x-client-id"] = this.clientId;
}
// if we have a bundle id on global context then add that to the headers
if (typeof globalThis !== "undefined" && "APP_BUNDLE_ID" in globalThis) {
headers["x-bundle-id"] = globalThis.APP_BUNDLE_ID;
}
// if we have a authorization token on global context then add that to the headers, this is for the dashboard.
if (typeof globalThis !== "undefined" && "TW_AUTH_TOKEN" in globalThis && typeof globalThis.TW_AUTH_TOKEN === "string") {
headers["authorization"] = `Bearer ${globalThis.TW_AUTH_TOKEN}`;
}
// CLI auth token
if (typeof globalThis !== "undefined" && "TW_CLI_AUTH_TOKEN" in globalThis && typeof globalThis.TW_CLI_AUTH_TOKEN === "string") {
headers["authorization"] = `Bearer ${globalThis.TW_CLI_AUTH_TOKEN}`;
headers["x-authorize-wallet"] = "true";
}
const res = await fetch(`${this.uploadServerUrl}/ipfs/upload`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
...headers,
...form.getHeaders()

@@ -624,10 +883,12 @@ },

});
const body = await res.json();
if (!res.ok) {
console.warn(body);
throw new Error("Failed to upload files to IPFS");
if (res.status === 401) {
throw new Error("Unauthorized - You don't have permission to use this service.");
}
throw new Error(`Failed to upload files to IPFS - ${res.status} - ${res.statusText} - ${await res.text()}`);
}
const body = await res.json();
const cid = body.IpfsHash;
if (!cid) {
throw new Error("Failed to upload files to IPFS");
throw new Error("Failed to upload files to IPFS - Bad CID");
}

@@ -647,5 +908,10 @@ if (options?.uploadWithoutDirectory) {

* ```jsx
* // Create a default storage class without any configuration
* const storage = new ThirdwebStorage();
* // Create a default storage class with a client ID when used in client-side applications
* const storage = new ThirdwebStorage({ clientId: "your-client-id" });
*
* // Create a default storage class with a secret key when used in server-side applications
* const storage = new ThirdwebStorage({ secretKey: "your-secret-key" });
*
* You can get a clientId and secretKey from https://thirdweb.com/create-api-key
*
* // Upload any file or JSON object

@@ -666,3 +932,4 @@ * const uri = await storage.upload(data);

* const uploader = new IpfsUploader();
* const storage = new ThirdwebStorage({ uploader, downloader, gatewayUrls });
* const clientId = "your-client-id";
* const storage = new ThirdwebStorage({ clientId, uploader, downloader, gatewayUrls });
* ```

@@ -674,8 +941,13 @@ *

constructor(options) {
_defineProperty(this, "uploader", void 0);
_defineProperty(this, "downloader", void 0);
_defineProperty(this, "gatewayUrls", void 0);
this.uploader = options?.uploader || new IpfsUploader();
this.downloader = options?.downloader || new StorageDownloader();
this.gatewayUrls = prepareGatewayUrls(parseGatewayUrls(options?.gatewayUrls));
this.uploader = options?.uploader || new IpfsUploader({
clientId: options?.clientId,
secretKey: options?.secretKey,
uploadServerUrl: options?.uploadServerUrl
});
this.downloader = options?.downloader || new StorageDownloader({
secretKey: options?.secretKey,
clientId: options?.clientId
});
this.gatewayUrls = prepareGatewayUrls(parseGatewayUrls(options?.gatewayUrls), options?.clientId, options?.secretKey);
this.clientId = options?.clientId;
}

@@ -697,3 +969,3 @@

resolveScheme(url) {
return replaceSchemeWithGatewayUrl(url, this.gatewayUrls);
return replaceSchemeWithGatewayUrl(url, this.gatewayUrls, 0, this.clientId);
}

@@ -713,4 +985,4 @@

*/
async download(url) {
return this.downloader.download(url, this.gatewayUrls);
async download(url, options) {
return this.downloader.download(url, this.gatewayUrls, options);
}

@@ -731,8 +1003,8 @@

*/
async downloadJSON(url) {
const res = await this.download(url);
async downloadJSON(url, options) {
const res = await this.download(url, options);
// If we get a JSON object, recursively replace any schemes with gatewayUrls
const json = await res.json();
return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls);
return replaceObjectSchemesWithGatewayUrls(json, this.gatewayUrls, this.clientId);
}

@@ -816,2 +1088,5 @@

}
getGatewayUrls() {
return this.gatewayUrls;
}
async uploadAndReplaceFilesWithHashes(data, options) {

@@ -834,3 +1109,3 @@ let cleaned = data;

// Ex: used for Solana, where services don't resolve schemes for you, so URLs must be usable by default
cleaned = replaceObjectSchemesWithGatewayUrls(cleaned, this.gatewayUrls);
cleaned = replaceObjectSchemesWithGatewayUrls(cleaned, this.gatewayUrls, this.clientId);
}

@@ -845,5 +1120,4 @@ return cleaned;

class MockDownloader {
gatewayUrls = DEFAULT_GATEWAY_URLS;
constructor(storage) {
_defineProperty(this, "gatewayUrls", DEFAULT_GATEWAY_URLS);
_defineProperty(this, "storage", void 0);
this.storage = storage;

@@ -870,3 +1144,2 @@ }

constructor(storage) {
_defineProperty(this, "storage", void 0);
this.storage = storage;

@@ -902,2 +1175,2 @@ }

export { DEFAULT_GATEWAY_URLS, IpfsUploader, MockDownloader, MockUploader, PINATA_IPFS_URL, StorageDownloader, TW_IPFS_SERVER_URL, ThirdwebStorage, extractObjectFiles, getCID, getCIDForUpload, isBrowser, isBufferInstance, isBufferOrStringWithName, isFileBufferOrStringEqual, isFileInstance, isFileOrBuffer, isUploaded, parseGatewayUrls, prepareGatewayUrls, replaceGatewayUrlWithScheme, replaceObjectFilesWithUris, replaceObjectGatewayUrlsWithSchemes, replaceObjectSchemesWithGatewayUrls, replaceSchemeWithGatewayUrl };
export { DEFAULT_GATEWAY_URLS, IpfsUploader, MockDownloader, MockUploader, PINATA_IPFS_URL, StorageDownloader, TW_UPLOAD_SERVER_URL, ThirdwebStorage, convertCidToV1, extractObjectFiles, getGatewayUrlForCid, isBrowser, isBufferInstance, isBufferOrStringWithName, isFileBufferOrStringEqual, isFileInstance, isFileOrBuffer, isTwGatewayUrl, parseGatewayUrls, prepareGatewayUrls, replaceGatewayUrlWithScheme, replaceObjectFilesWithUris, replaceObjectGatewayUrlsWithSchemes, replaceObjectSchemesWithGatewayUrls, replaceSchemeWithGatewayUrl };

62

package.json
{
"name": "@thirdweb-dev/storage",
"version": "0.0.0-dev-7cfa22e-20230329230519",
"version": "0.0.0-dev-812a24f-20231130143736",
"main": "dist/thirdweb-dev-storage.cjs.js",
"module": "dist/thirdweb-dev-storage.esm.js",
"browser": {
"./dist/thirdweb-dev-storage.esm.js": "./dist/thirdweb-dev-storage.browser.esm.js"
},
"exports": {
".": {
"module": {
"browser": "./dist/thirdweb-dev-storage.browser.esm.js",
"default": "./dist/thirdweb-dev-storage.esm.js"
},
"module": "./dist/thirdweb-dev-storage.esm.js",
"default": "./dist/thirdweb-dev-storage.cjs.js"

@@ -22,2 +16,3 @@ },

"license": "Apache-2.0",
"sideEffects": false,
"files": [

@@ -27,38 +22,40 @@ "dist/"

"preconstruct": {
"exports": {
"envConditions": [
"browser"
]
}
"exports": true
},
"devDependencies": {
"@babel/preset-env": "^7.18.10",
"@babel/preset-typescript": "^7.18.6",
"@microsoft/api-documenter": "^7.19.4",
"@microsoft/api-extractor": "^7.29.2",
"@babel/preset-env": "^7.22.9",
"@babel/preset-typescript": "^7.22.5",
"@microsoft/api-documenter": "^7.22.30",
"@microsoft/api-extractor": "^7.36.3",
"@microsoft/tsdoc": "^0.14.1",
"@preconstruct/cli": "^2.2.1",
"@swc-node/register": "^1.5.1",
"@thirdweb-dev/tsconfig": "^0.0.0-dev-7cfa22e-20230329230519",
"@types/chai": "^4.3.3",
"@preconstruct/cli": "2.7.0",
"@swc-node/register": "^1.6.6",
"@thirdweb-dev/tsconfig": "^0.1.7",
"@types/chai": "^4.3.5",
"@types/mocha": "^10.0.0",
"@types/uuid": "^9.0.0",
"@typescript-eslint/eslint-plugin": "^5.33.0",
"@typescript-eslint/parser": "^5.33.0",
"@types/uuid": "^9.0.5",
"@typescript-eslint/eslint-plugin": "^6.2.0",
"@typescript-eslint/parser": "^6.2.0",
"chai": "^4.3.6",
"eslint": "^8.21.0",
"eslint-config-thirdweb": "^0.1.4",
"eslint": "^8.45.0",
"eslint-config-thirdweb": "^0.1.6",
"eslint-plugin-tsdoc": "^0.2.16",
"esm": "^3.2.25",
"mocha": "^10.2.0",
"typescript": "^4.7.4"
"rimraf": "^3.0.2",
"typedoc": "^0.25.2",
"typescript": "^5.1.6"
},
"dependencies": {
"cross-fetch": "^3.1.5",
"cid-tool": "^3.0.0",
"form-data": "^4.0.0",
"ipfs-unixfs-importer": "^7.0.1",
"uuid": "^9.0.0"
"uuid": "^9.0.1",
"@thirdweb-dev/crypto": "0.2.0"
},
"engines": {
"node": ">=18"
},
"scripts": {
"format": "prettier --write 'src/**/*'",
"lint": "eslint src/",
"lint": "eslint src/ && bunx publint --strict --level warning",
"fix": "eslint src/ --fix",

@@ -71,4 +68,5 @@ "generate-docs": "api-extractor run --local && api-documenter markdown -i ./temp -o ./docs",

"test:single": "NODE_ENV=test SWC_NODE_PROJECT=./tsconfig.test.json mocha --timeout 30000",
"push": "yalc push"
"push": "yalc push",
"typedoc": "rimraf typedoc && node typedoc.js"
}
}
dist/declarations/src/common/cid.d.ts
dist/declarations/src/common/cid.d.ts.map
dist/thirdweb-dev-storage.browser.esm.js
dist/declarations/src/common/index.d.ts.map

Sorry, the diff of this file is not supported yet

dist/declarations/src/common/urls.d.ts.map

Sorry, the diff of this file is not supported yet

dist/declarations/src/common/utils.d.ts.map

Sorry, the diff of this file is not supported yet

dist/declarations/src/core/downloaders/storage-downloader.d.ts.map

Sorry, the diff of this file is not supported yet

dist/declarations/src/core/storage.d.ts.map

Sorry, the diff of this file is not supported yet

dist/declarations/src/core/uploaders/ipfs-uploader.d.ts.map

Sorry, the diff of this file is not supported yet

dist/declarations/src/types/download.d.ts.map

Sorry, the diff of this file is not supported yet

dist/declarations/src/types/index.d.ts.map

Sorry, the diff of this file is not supported yet

dist/declarations/src/types/upload.d.ts.map

Sorry, the diff of this file is not supported yet