fetch-extras - npm Package Compare versions

Comparing version

1.0.0

1.1.0

+428

source/index.d.ts

		/**
		Custom error class for HTTP errors that should be thrown when the response has a non-2xx status code.
		*/
		export class HttpError extends Error {
		readonly name: 'HttpError';
		readonly code: 'ERR_HTTP_RESPONSE_NOT_OK';
		response: Response;

		/**
		Constructs a new `HttpError` instance.

		@param response - The `Response` object that caused the error.
		*/
		constructor(response: Response);
		}

		/**
		Throws an `HttpError` if the response is not ok (non-2xx status code).

		@param response - The `Response` object to check.
		@returns The same `Response` object if it is ok.
		@throws {HttpError} If the response is not ok.

		@example
		```
		import {throwIfHttpError} from 'fetch-extras';

		const response = await fetch('/api');
		throwIfHttpError(response);
		const data = await response.json();
		```
		*/
		export function throwIfHttpError(response: Response): Response;

		/**
		Throws an `HttpError` if the response is not ok (non-2xx status code).

		@param responsePromise - A promise that resolves to a `Response` object to check.
		@returns A promise that resolves to the same `Response` object if it is ok.
		@throws {HttpError} If the response is not ok.

		@example
		```
		import {throwIfHttpError} from 'fetch-extras';

		const response = await throwIfHttpError(fetch('/api'));
		const data = await response.json();
		```
		*/
		export function throwIfHttpError(responsePromise: Promise<Response>): Promise<Response>;

		/**
		Wraps a fetch function to automatically throw `HttpError` for non-2xx responses.

		Can be combined with other `with*` methods.

		@param fetchFunction - The fetch function to wrap (usually the global `fetch`).
		@returns A wrapped fetch function that will throw HttpError for non-2xx responses.

		@example
		```
		import {withHttpError} from 'fetch-extras';

		const fetchWithError = withHttpError(fetch);
		const response = await fetchWithError('/api'); // Throws HttpError for non-2xx responses
		const data = await response.json();
		```
		*/
		export function withHttpError(
		fetchFunction: typeof fetch
		): typeof fetch;

		/**
		Wraps a fetch function with timeout functionality.

		Can be combined with other `with*` methods.

		@param fetchFunction - The fetch function to wrap (usually the global `fetch`).
		@param timeout - Timeout in milliseconds.
		@returns A wrapped fetch function that will abort if the request takes longer than the specified timeout.

		@example
		```
		import {withTimeout} from 'fetch-extras';

		const fetchWithTimeout = withTimeout(fetch, 5000);
		const response = await fetchWithTimeout('/api');
		const data = await response.json();
		```
		*/
		export function withTimeout(
		fetchFunction: typeof fetch,
		timeout: number
		): typeof fetch;

		/**
		Pagination options for customizing how pagination works.
		*/
		export type PaginationOptions<ItemType = unknown> = {
		/**
		Transform the response into an array of items.

		By default, it calls `response.json()` and expects an array.

		@param response - The Response object from the fetch request.
		@returns An array of items to yield, or a Promise that resolves to an array.

		@example
		```
		import {paginate} from 'fetch-extras';

		const items = await paginate.all('https://api.example.com/users', {
		pagination: {
		transform: async response => {
		const data = await response.json();
		return data.users; // Extract items from nested property
		}
		}
		});
		```
		*/
		transform?: (response: Response) => ItemType[] \| Promise<ItemType[]>;

		/**
		Determine the next page to fetch.

		Return an object with fetch options for the next request, or `false` to stop pagination.

		By default, it parses the `Link` header and follows the `rel="next"` link.

		Important: The response body has already been consumed by the `transform` function. Do NOT call `response.json()` or other body methods here. Instead, extract pagination info from headers, the URL, or share data from the transform function through closure.

		Note: Returning `headers` replaces all inherited headers, consistent with standard Fetch API behavior. Setting `body` to `undefined` will strip body-related headers (`Content-Type`, `Content-Length`, etc.) from the request.

		@param data - Context object with response, current URL, and items.
		@returns Options for the next fetch request, or `false` to stop pagination.

		@example
		```
		import {paginate} from 'fetch-extras';

		// Cursor-based pagination using headers (recommended)
		for await (const item of paginate('https://api.example.com/items', {
		pagination: {
		paginate: ({response}) => {
		const nextCursor = response.headers.get('X-Next-Cursor');
		if (!nextCursor) return false;

		return {
		url: new URL(`https://api.example.com/items?cursor=${nextCursor}`)
		};
		}
		}
		})) {
		console.log(item);
		}
		```

		@example
		```
		import {paginate} from 'fetch-extras';

		// Sharing data between transform and paginate via closure
		let nextCursor;
		for await (const item of paginate('https://api.example.com/items', {
		pagination: {
		transform: async (response) => {
		const data = await response.json();
		// Store pagination info in closure
		nextCursor = data.nextCursor;
		return data.items;
		},
		paginate: () => {
		if (!nextCursor) return false;
		return {
		url: new URL(`https://api.example.com/items?cursor=${nextCursor}`)
		};
		}
		}
		})) {
		console.log(item);
		}
		```
		*/
		paginate?: (data: {
		response: Response;
		currentUrl: URL \| string;
		currentItems: ItemType[];
		allItems: ItemType[];
		}) => PaginationNextPage \| false \| Promise<PaginationNextPage \| false>;

		/**
		Filter items before yielding them.

		@param data - Context object with the current item and item arrays.
		@returns `true` to yield the item, `false` to skip it.

		@example
		```
		import {paginate} from 'fetch-extras';

		// Only get active users
		for await (const user of paginate('https://api.example.com/users', {
		pagination: {
		filter: ({item}) => item.status === 'active'
		}
		})) {
		console.log(user);
		}
		```
		*/
		filter?: (data: {
		item: ItemType;
		currentItems: ItemType[];
		allItems: ItemType[];
		}) => boolean;

		/**
		Check if pagination should continue after yielding an item.

		This is called after `filter` returns `true`. Useful for stopping pagination based on item values.

		@param data - Context object with the current item and item arrays.
		@returns `true` to continue pagination, `false` to stop.

		@example
		```
		import {paginate} from 'fetch-extras';

		// Stop when we reach items older than one week
		const oneWeekAgo = Date.now() - (7 * 24 * 60 * 60 * 1000);
		for await (const commit of paginate('https://api.github.com/repos/user/repo/commits', {
		pagination: {
		shouldContinue: ({item}) => new Date(item.date).getTime() >= oneWeekAgo
		}
		})) {
		console.log(commit);
		}
		```
		*/
		shouldContinue?: (data: {
		item: ItemType;
		currentItems: ItemType[];
		allItems: ItemType[];
		}) => boolean;

		/**
		Maximum number of items to yield.

		@default Infinity
		*/
		countLimit?: number;

		/**
		Delay in milliseconds between requests.

		Useful for rate limiting.

		@default 0
		*/
		backoff?: number;

		/**
		Maximum number of requests to make.

		This prevents infinite loops if your `paginate` function has bugs. Ensure your `paginate` function eventually returns `false` or the iteration will continue until this limit is reached.

		@default 10000
		*/
		requestLimit?: number;

		/**
		Whether to keep all yielded items in memory.

		When `true`, the `allItems` array passed to callbacks will contain all previously yielded items. When `false` (default), `allItems` will always be empty to save memory.

		@default false
		*/
		stackAllItems?: boolean;
		};

		/**
		Options for the next page request.
		*/
		export type PaginationNextPage = {
		/**
		URL for the next page.

		Must be a URL instance, not a string.
		*/
		url?: URL;
		} & RequestInit;

		/**
		A function with the same signature as the global `fetch`.

		This allows you to use a custom fetch implementation, such as [`ky`](https://github.com/sindresorhus/ky).
		*/
		export type FetchFunction = (input: RequestInfo \| URL, init?: any) => Promise<Response>;

		/**
		Options for the `paginate` function.
		*/
		export type PaginateOptions<ItemType = unknown> = RequestInit & {
		/**
		Pagination-specific options.
		*/
		pagination?: PaginationOptions<ItemType>;

		/**
		Custom fetch function to use for requests.

		This allows you to use a custom fetch implementation, such as [`ky`](https://github.com/sindresorhus/ky), or a fetch function wrapped with `withHttpError` or `withTimeout`.

		@default globalThis.fetch

		@example
		```
		import {paginate} from 'fetch-extras';
		import ky from 'ky';

		const url = 'https://api.github.com/repos/sindresorhus/ky/commits';

		for await (const commit of paginate(url, {fetchFunction: ky})) {
		console.log(commit.sha);
		}
		```
		*/
		fetchFunction?: FetchFunction;
		};

		/**
		Paginate through API responses using async iteration.

		By default, it automatically follows RFC 5988 `Link` headers with `rel="next"`.

		Note: This function does not check response status codes. If you need error handling for non-2xx responses, wrap fetch with `withHttpError()` or handle errors in your `transform` function.

		@param input - The URL to fetch. Can be a string or URL instance.
		@param options - Fetch options plus pagination options.
		@returns An async iterator that yields items from each page.

		@example
		```
		import {paginate} from 'fetch-extras';

		// Basic usage with Link headers (GitHub API)
		for await (const commit of paginate('https://api.github.com/repos/sindresorhus/ky/commits')) {
		console.log(commit.sha);
		}
		```

		@example
		```
		import {paginate} from 'fetch-extras';

		// With error handling for non-2xx responses
		for await (const item of paginate('https://api.example.com/items', {
		pagination: {
		transform: async (response) => {
		if (!response.ok) {
		throw new Error(`HTTP ${response.status}`);
		}
		return response.json();
		},
		countLimit: 100,
		backoff: 1000
		}
		})) {
		console.log(item);
		}
		```

		@example
		```
		import {paginate} from 'fetch-extras';

		// Cursor-based pagination using headers
		for await (const item of paginate('https://api.example.com/items', {
		pagination: {
		transform: async response => {
		const data = await response.json();
		return data.items;
		},
		paginate: ({response}) => {
		const cursor = response.headers.get('X-Next-Cursor');
		return cursor
		? {url: new URL(`https://api.example.com/items?cursor=${cursor}`)}
		: false;
		}
		}
		})) {
		console.log(item);
		}
		```
		*/
		export function paginate<ItemType = unknown>(
		input: RequestInfo \| URL,
		options?: PaginateOptions<ItemType>
		): AsyncIterableIterator<ItemType>;

		export namespace paginate {
		/**
		Get all paginated items as an array.

		This is a convenience method that collects all items into memory. For large datasets, prefer using the async iterator directly.

		@param input - The URL to fetch. Can be a string or URL instance.
		@param options - Fetch options plus pagination options.
		@returns A promise that resolves to an array of all items.

		@example
		```
		import {paginate} from 'fetch-extras';

		const commits = await paginate.all('https://api.github.com/repos/sindresorhus/ky/commits', {
		pagination: {
		countLimit: 50
		}
		});
		console.log(`Fetched ${commits.length} commits`);
		```
		*/
		export function all<ItemType = unknown>(
		input: RequestInfo \| URL,
		options?: PaginateOptions<ItemType>
		): Promise<ItemType[]>;
		}

+43

source/index.js

		export {paginate} from './paginate.js';

		export class HttpError extends Error {
		constructor(response) {
		const status = `${response.status} ${response.statusText}`.trim();
		const reason = status ? `status code ${status}` : 'an unknown error';

		super(`Request failed with ${reason}: ${response.url}`);
		Error.captureStackTrace?.(this, this.constructor);

		this.name = 'HttpError';
		this.code = 'ERR_HTTP_RESPONSE_NOT_OK';
		this.response = response;
		}
		}

		export async function throwIfHttpError(responseOrPromise) {
		if (!(responseOrPromise instanceof Response)) {
		responseOrPromise = await responseOrPromise;
		}

		if (!responseOrPromise.ok) {
		throw new HttpError(responseOrPromise);
		}

		return responseOrPromise;
		}

		export function withHttpError(fetchFunction) {
		return async (urlOrRequest, options = {}) => {
		const response = await fetchFunction(urlOrRequest, options);
		return throwIfHttpError(response);
		};
		}

		export function withTimeout(fetchFunction, timeout) {
		return async (urlOrRequest, options = {}) => {
		const providedSignal = options.signal ?? (urlOrRequest instanceof Request && urlOrRequest.signal);
		const timeoutSignal = AbortSignal.timeout(timeout);
		const signal = providedSignal ? AbortSignal.any([providedSignal, timeoutSignal]) : timeoutSignal;
		return fetchFunction(urlOrRequest, {...options, signal});
		};
		}

+430

source/paginate.js

		import parseLinkHeader from './parse-link-header.js';
		import {delay} from './utilities.js';

		const defaultPaginationOptions = {
		async transform(response) {
		return response.json();
		},
		async paginate({response, currentUrl}) {
		const linkHeader = response.headers.get('Link');

		if (!linkHeader?.trim()) {
		return false;
		}

		const links = parseLinkHeader(linkHeader);
		const next = links.find(link => link.parameters.rel?.split(/\s+/).some(relationType => relationType.toLowerCase() === 'next'));

		if (next) {
		return {url: toUrl(next.url, response.url \|\| absoluteUrl(currentUrl)?.href)};
		}

		return false;
		},
		filter: () => true,
		shouldContinue: () => true,
		countLimit: Number.POSITIVE_INFINITY,
		backoff: 0,
		requestLimit: 10_000,
		stackAllItems: false,
		};

		const requestBodyHeaderNames = [
		'content-encoding',
		'content-language',
		'content-location',
		'content-type',
		];
		const sensitiveHeaderNames = [
		'authorization',
		'cookie',
		'proxy-authorization',
		];

		const stripBodyHeaders = headers => {
		const cleanedHeaders = new Headers(headers);
		cleanedHeaders.delete('content-length');

		for (const headerName of requestBodyHeaderNames) {
		cleanedHeaders.delete(headerName);
		}

		return cleanedHeaders;
		};

		const stripSensitiveHeaders = headers => {
		const cleanedHeaders = new Headers(headers);

		for (const headerName of sensitiveHeaderNames) {
		cleanedHeaders.delete(headerName);
		}

		return cleanedHeaders;
		};

		const methodCanHaveBody = method => method === undefined \|\| !['get', 'head'].includes(method.toLowerCase());

		const requestSnapshot = request => ({
		method: request.method,
		referrer: request.referrer,
		referrerPolicy: request.referrerPolicy,
		mode: request.mode,
		credentials: request.credentials,
		cache: request.cache,
		redirect: request.redirect,
		integrity: request.integrity,
		keepalive: request.keepalive,
		signal: request.signal,
		duplex: request.duplex,
		priority: request.priority,
		});

		const requestWithoutBody = request => ({
		...requestSnapshot(request),
		headers: stripBodyHeaders(request.headers),
		});

		const requestWithoutSensitiveState = request => ({
		...requestWithoutBody(request),
		headers: stripSensitiveHeaders(stripBodyHeaders(request.headers)),
		});

		const stripBodyHeadersFromFetchOptions = fetchOptions => {
		if (!('headers' in fetchOptions)) {
		return fetchOptions;
		}

		return {
		...fetchOptions,
		headers: stripBodyHeaders(fetchOptions.headers),
		};
		};

		const stripSensitiveHeadersFromFetchOptions = fetchOptions => {
		if (!('headers' in fetchOptions)) {
		return fetchOptions;
		}

		return {
		...fetchOptions,
		headers: stripSensitiveHeaders(fetchOptions.headers),
		};
		};

		const normalizeBodylessFetchOptions = fetchOptions => {
		const {body: _body, ...restFetchOptions} = fetchOptions;

		if (!('headers' in restFetchOptions)) {
		return restFetchOptions;
		}

		return {
		...restFetchOptions,
		headers: stripBodyHeaders(restFetchOptions.headers),
		};
		};

		// URL/string inputs stay on the `fetch(input, init)` path, so body cleanup must happen on the init object itself.
		const normalizeFetchOptions = fetchOptions => shouldStripBodyHeaders(fetchOptions) ? normalizeBodylessFetchOptions(fetchOptions) : fetchOptions;

		const shouldStripBodyHeaders = fetchOptions => (Object.hasOwn(fetchOptions, 'body') && fetchOptions.body === undefined) \|\| (!methodCanHaveBody(fetchOptions.method) && !Object.hasOwn(fetchOptions, 'body'));
		const shouldResetBodyHeaders = fetchOptions => shouldStripBodyHeaders(fetchOptions) \|\| Object.hasOwn(fetchOptions, 'body');
		const integerOrInfinity = value => value === Number.POSITIVE_INFINITY \|\| Number.isInteger(value);
		const isPaginationFetchOptions = value => typeof value === 'object' && value !== null && !Array.isArray(value) && !(value instanceof URL);
		const isCrossOrigin = (currentUrl, nextUrl) => currentUrl.origin !== nextUrl.origin;
		const shouldStripInheritedSensitiveState = (inheritedStateOrigin, requestUrl) => inheritedStateOrigin && requestUrl instanceof URL && isCrossOrigin(inheritedStateOrigin, requestUrl);

		const toUrl = (input, baseUrl = globalThis.location?.href) => {
		const url = input instanceof Request ? input.url : input;

		if (url instanceof URL) {
		return url;
		}

		if (baseUrl === undefined) {
		return new URL(url);
		}

		return new URL(url, baseUrl);
		};

		// Custom fetch functions are expected to follow Fetch semantics here: `response.url` is absolute after resolution, or the empty string when unavailable.
		const responseUrl = (response, currentUrl) => response.url ? new URL(response.url) : currentUrl;
		const absoluteUrl = input => {
		// Custom fetch wrappers can leave `response.url` empty or keep relative inputs unresolved.
		try {
		return toUrl(input);
		} catch {}

		return undefined;
		};

		// Template-backed requests keep their headers on the Request itself to match `new Request(input, init)` semantics.
		const createTemplateFetchOptions = ({body: _body, headers: _headers, ...rest}) => rest;

		const createRequestTemplate = (input, fetchOptions) => {
		if (input instanceof Request && shouldResetBodyHeaders(fetchOptions)) {
		// Replacing only the body should keep the inherited request headers, so defer to the platform behavior.
		if (!shouldStripBodyHeaders(fetchOptions)) {
		return new Request(input, fetchOptions);
		}

		// Bodyless requests need explicit cleanup because Fetch only removes the request-body-header names.
		const requestInit = requestWithoutBody(input);
		const nextFetchOptions = stripBodyHeadersFromFetchOptions(fetchOptions);

		if ('headers' in nextFetchOptions) {
		return new Request(input.url, {
		...requestInit,
		...nextFetchOptions,
		headers: new Headers(nextFetchOptions.headers),
		});
		}

		return new Request(input.url, {...requestInit, ...nextFetchOptions});
		}

		return new Request(input, fetchOptions);
		};

		const currentSignal = (requestTemplate, fetchOptions) => fetchOptions.signal ?? requestTemplate?.signal;
		const requestWithoutSensitiveHeaders = request => ({
		...requestSnapshot(request),
		headers: stripSensitiveHeaders(request.headers),
		});

		const stripInheritedSensitiveState = (requestTemplate, fetchOptions) => ({
		requestTemplate: requestTemplate && new Request(requestTemplate.url, requestTemplate.body === null ? requestWithoutSensitiveHeaders(requestTemplate) : requestWithoutSensitiveState(requestTemplate)),
		fetchOptions: Object.hasOwn(fetchOptions, 'body') && fetchOptions.body !== undefined ? normalizeBodylessFetchOptions(stripSensitiveHeadersFromFetchOptions(fetchOptions)) : stripSensitiveHeadersFromFetchOptions(fetchOptions),
		});

		const hasSensitiveHeaders = headers => {
		const normalizedHeaders = new Headers(headers);

		for (const headerName of sensitiveHeaderNames) {
		if (normalizedHeaders.has(headerName)) {
		return true;
		}
		}

		return false;
		};

		const hasExplicitSensitiveState = fetchOptions => ('body' in fetchOptions && fetchOptions.body !== undefined) \|\| ('headers' in fetchOptions && hasSensitiveHeaders(fetchOptions.headers));
		const shouldClearInheritedBody = (fetchOptions, nextPageOptions) => !methodCanHaveBody(fetchOptions.method) && !Object.hasOwn(nextPageOptions, 'body') && Object.hasOwn(fetchOptions, 'body');

		/**
		Paginate through API responses using async iteration.

		By default, it automatically follows RFC 5988 Link headers with rel="next".

		@param {RequestInfo \| URL} input - The URL to fetch.
		@param {RequestInit & {pagination?: PaginationOptions, fetchFunction?: Function}} options - Fetch options plus pagination options.
		@yields {*} Items from each page.
		@returns {AsyncIterableIterator} An async iterator that yields items from each page.

		@example
		```
		import {paginate} from 'fetch-extras';

		// Basic usage with Link headers
		for await (const item of paginate('https://api.example.com/items')) {
		console.log(item);
		}
		```
		*/
		// eslint-disable-next-line complexity
		export async function * paginate(input, options = {}) {
		const {pagination = {}, fetchFunction = fetch, ...fetchOptions} = options;
		const paginationOptions = {...defaultPaginationOptions, ...pagination};

		if (typeof paginationOptions.transform !== 'function') {
		throw new TypeError('pagination.transform must be a function');
		}

		if (typeof paginationOptions.paginate !== 'function') {
		throw new TypeError('pagination.paginate must be a function');
		}

		if (typeof paginationOptions.filter !== 'function') {
		throw new TypeError('pagination.filter must be a function');
		}

		if (typeof paginationOptions.shouldContinue !== 'function') {
		throw new TypeError('pagination.shouldContinue must be a function');
		}

		if (typeof paginationOptions.countLimit !== 'number') {
		throw new TypeError('pagination.countLimit must be a number');
		}

		if (Number.isNaN(paginationOptions.countLimit)) {
		throw new TypeError('pagination.countLimit must not be NaN');
		}

		if (paginationOptions.countLimit < 0) {
		throw new TypeError('pagination.countLimit must be non-negative');
		}

		if (!integerOrInfinity(paginationOptions.countLimit)) {
		throw new TypeError('pagination.countLimit must be an integer');
		}

		if (typeof paginationOptions.requestLimit !== 'number') {
		throw new TypeError('pagination.requestLimit must be a number');
		}

		if (Number.isNaN(paginationOptions.requestLimit)) {
		throw new TypeError('pagination.requestLimit must not be NaN');
		}

		if (paginationOptions.requestLimit < 0) {
		throw new TypeError('pagination.requestLimit must be non-negative');
		}

		if (!integerOrInfinity(paginationOptions.requestLimit)) {
		throw new TypeError('pagination.requestLimit must be an integer');
		}

		if (typeof paginationOptions.backoff !== 'number') {
		throw new TypeError('pagination.backoff must be a number');
		}

		if (Number.isNaN(paginationOptions.backoff)) {
		throw new TypeError('pagination.backoff must not be NaN');
		}

		if (!Number.isFinite(paginationOptions.backoff)) {
		throw new TypeError('pagination.backoff must be finite');
		}

		if (paginationOptions.backoff < 0) {
		throw new TypeError('pagination.backoff must be non-negative');
		}

		if (typeof paginationOptions.stackAllItems !== 'boolean') {
		throw new TypeError('pagination.stackAllItems must be a boolean');
		}

		const allItems = [];
		let {countLimit} = paginationOptions;
		let numberOfRequests = 0;
		const absoluteInputUrl = input instanceof Request ? undefined : absoluteUrl(input);
		let requestTemplate = input instanceof Request \|\| ('body' in fetchOptions && absoluteInputUrl) ? createRequestTemplate(input instanceof Request ? input : absoluteInputUrl, fetchOptions) : undefined;
		let currentUrl = requestTemplate ? new URL(requestTemplate.url) : input;
		let currentFetchOptions = requestTemplate ? createTemplateFetchOptions(fetchOptions) : normalizeFetchOptions(fetchOptions);
		let inheritedStateOrigin = absoluteUrl(requestTemplate ? requestTemplate.url : input);

		while (numberOfRequests < paginationOptions.requestLimit && countLimit > 0) {
		if (numberOfRequests !== 0 && paginationOptions.backoff > 0) {
		// eslint-disable-next-line no-await-in-loop
		await delay(paginationOptions.backoff, {signal: currentSignal(requestTemplate, currentFetchOptions)});
		}

		const currentInput = requestTemplate ? new Request(currentUrl, requestTemplate.clone()) : currentUrl;

		// eslint-disable-next-line no-await-in-loop
		const response = await fetchFunction(currentInput, currentFetchOptions);
		const currentResponseUrl = responseUrl(response, currentUrl);

		currentUrl = currentResponseUrl;

		if (shouldStripInheritedSensitiveState(inheritedStateOrigin, currentResponseUrl)) {
		const strippedState = stripInheritedSensitiveState(requestTemplate, currentFetchOptions);
		requestTemplate = strippedState.requestTemplate;
		currentFetchOptions = strippedState.fetchOptions;
		inheritedStateOrigin = undefined;
		}

		// eslint-disable-next-line no-await-in-loop
		const parsed = await paginationOptions.transform(response);

		if (!Array.isArray(parsed)) {
		throw new TypeError('pagination.transform must return an array');
		}

		const currentItems = [];

		for (const item of parsed) {
		if (paginationOptions.filter({item, currentItems, allItems})) {
		yield item;

		if (paginationOptions.stackAllItems) {
		allItems.push(item);
		}

		currentItems.push(item);
		countLimit--;

		if (!paginationOptions.shouldContinue({item, currentItems, allItems})) {
		return;
		}

		if (countLimit === 0) {
		return;
		}
		}
		}

		// eslint-disable-next-line no-await-in-loop
		const nextPageOptions = await paginationOptions.paginate({
		response,
		currentUrl,
		currentItems,
		allItems,
		});

		if (nextPageOptions === false) {
		return;
		}

		if (!isPaginationFetchOptions(nextPageOptions)) {
		throw new TypeError('pagination.paginate must return an object or false');
		}

		if ('url' in nextPageOptions) {
		if (!(nextPageOptions.url instanceof URL)) {
		throw new TypeError('pagination.paginate must return an object with url as a URL instance');
		}

		currentUrl = nextPageOptions.url;
		}

		const nextRequestUrl = absoluteUrl(nextPageOptions.url ?? currentUrl);

		if (shouldStripInheritedSensitiveState(inheritedStateOrigin, nextRequestUrl)) {
		const strippedState = stripInheritedSensitiveState(requestTemplate, currentFetchOptions);
		requestTemplate = strippedState.requestTemplate;
		currentFetchOptions = strippedState.fetchOptions;
		inheritedStateOrigin = undefined;
		}

		const nextPageKeys = Object.keys(nextPageOptions);
		if (nextPageKeys.length > 1 \|\| (nextPageKeys.length === 1 && !nextPageOptions.url)) {
		const {url: _, ...restNextPageOptions} = nextPageOptions;
		const nextFetchOptions = {...currentFetchOptions, ...restNextPageOptions};

		if (requestTemplate) {
		requestTemplate = createRequestTemplate(requestTemplate, nextFetchOptions);
		currentFetchOptions = createTemplateFetchOptions(nextFetchOptions);
		} else {
		currentFetchOptions = normalizeFetchOptions(shouldClearInheritedBody(nextFetchOptions, restNextPageOptions) ? {...nextFetchOptions, body: undefined} : nextFetchOptions);
		}

		if (hasExplicitSensitiveState(restNextPageOptions)) {
		inheritedStateOrigin = nextRequestUrl ?? inheritedStateOrigin;
		}
		}

		numberOfRequests++;
		}
		}

		paginate.all = async (input, options) => {
		const items = [];
		for await (const item of paginate(input, options)) {
		items.push(item);
		}

		return items;
		};

+115

source/parse-link-header.js

		function splitHeaderValue(value, separator) {
		const parts = [];
		let startIndex = 0;
		let insideQuotes;
		let isEscaped = false;
		let insideUrl = false;

		for (let index = 0; index < value.length; index++) {
		const character = value[index];

		// RFC 8288 treats the URI reference inside `<...>` as opaque for parameter parsing.
		if (insideUrl) {
		if (character === '>') {
		insideUrl = false;
		}

		continue;
		}

		if (insideQuotes) {
		if (isEscaped) {
		isEscaped = false;
		continue;
		}

		if (character === '\\') {
		isEscaped = true;
		continue;
		}

		if (character === insideQuotes) {
		insideQuotes = undefined;
		}

		continue;
		}

		if (character === '"') {
		insideQuotes = character;
		continue;
		}

		if (character === '<') {
		insideUrl = true;
		continue;
		}

		if (character === separator) {
		parts.push(value.slice(startIndex, index));
		startIndex = index + 1;
		}
		}

		parts.push(value.slice(startIndex));

		return parts;
		}

		/**
		Parses an RFC 5988 Link header into an array of link objects.

		@param {string} linkHeader - The Link header value.
		@returns {Array<{url: string, parameters: Object}>} Parsed links with normalized parameter values.
		@throws {Error} If the Link header format is invalid.
		*/
		export default function parseLinkHeader(linkHeader) {
		const links = [];
		const parts = splitHeaderValue(linkHeader, ',');

		for (const part of parts) {
		const [rawUrlReference, ...rawLinkParameters] = splitHeaderValue(part, ';');
		const trimmedUrlReference = rawUrlReference.trim();

		if (trimmedUrlReference[0] !== '<' \|\| !trimmedUrlReference.endsWith('>')) {
		throw new Error(`Invalid Link header format: ${trimmedUrlReference}`);
		}

		const url = trimmedUrlReference.slice(1, -1);
		const parameters = {};

		for (const parameter of rawLinkParameters) {
		const trimmedParameter = parameter.trim();

		if (!trimmedParameter) {
		continue;
		}

		const equalIndex = trimmedParameter.indexOf('=');
		let name;
		let value;

		if (equalIndex === -1) {
		name = trimmedParameter.toLowerCase();
		value = '';
		} else {
		name = trimmedParameter.slice(0, equalIndex).trim().toLowerCase();
		value = trimmedParameter.slice(equalIndex + 1).trim();
		}

		if (value.startsWith('"') && value.endsWith('"')) {
		value = value.slice(1, -1).replaceAll(/\\(.)/g, '$1');
		}

		if (name in parameters) {
		continue;
		}

		parameters[name] = value;
		}

		links.push({url, parameters});
		}

		return links;
		}

+37

source/utilities.js

		/**
		Creates a promise that resolves after the specified delay.

		@param {number} milliseconds - The delay duration in milliseconds.
		@param {{signal?: AbortSignal}} [options] - Options for the delay.
		@returns {Promise<void>} A promise that resolves after the delay.
		*/
		export function delay(milliseconds, {signal} = {}) {
		return new Promise((resolve, reject) => {
		signal?.throwIfAborted();

		const rejectWithAbortReason = () => {
		try {
		signal.throwIfAborted();
		} catch (error) {
		reject(error);
		}
		};

		const timeout = setTimeout(() => {
		cleanup();
		resolve();
		}, milliseconds);

		const cleanup = () => {
		signal?.removeEventListener('abort', onAbort);
		};

		const onAbort = () => {
		clearTimeout(timeout);
		cleanup();
		rejectWithAbortReason();
		};

		signal?.addEventListener('abort', onAbort, {once: true});
		});
		}

+5

-6

package.json

		{
		"name": "fetch-extras",
		"version": "1.0.0",
		"version": "1.1.0",
		"description": "Useful utilities for working with Fetch",
		@@ -15,4 +15,4 @@ "license": "MIT",
		"exports": {
		"types": "./index.d.ts",
		"default": "./index.js"
		"types": "./source/index.d.ts",
		"default": "./source/index.js"
		},
		@@ -24,7 +24,6 @@ "sideEffects": false,
		"scripts": {
		"test": "xo && ava && tsc index.d.ts"
		"test": "xo && ava && tsc source/index.d.ts"
		},
		"files": [
		"index.js",
		"index.d.ts"
		"source"
		],
		@@ -31,0 +30,0 @@ "keywords": [

+263

-2

readme.md

		@@ -21,3 +21,3 @@ <h1 align="center" title="fetch-extras">
		// Create an enhanced reusable fetch function that:
		// - Throws errors for non-200 responses
		// - Throws errors for non-2xx responses
		// - Times out after 5 seconds
		@@ -32,4 +32,264 @@ const enhancedFetch = withHttpError(withTimeout(fetch, 5000));

		See the [types](index.d.ts) for now.
		### HttpError

		Error class thrown when a response has a non-2xx status code.

		```js
		import {HttpError, throwIfHttpError} from 'fetch-extras';

		try {
		await throwIfHttpError(fetch('/api'));
		} catch (error) {
		if (error instanceof HttpError) {
		console.log(error.response.status); // 404
		}
		}
		```

		### throwIfHttpError(response)

		Throws an `HttpError` if the response is not ok. Can also accept a promise that resolves to a response.

		```js
		import {throwIfHttpError} from 'fetch-extras';

		const response = await throwIfHttpError(fetch('/api'));
		const data = await response.json();
		```

		### withHttpError(fetchFunction)

		Returns a wrapped fetch function that automatically throws `HttpError` for non-2xx responses.

		```js
		import {withHttpError} from 'fetch-extras';

		const fetchWithError = withHttpError(fetch);
		const response = await fetchWithError('/api');
		```

		### withTimeout(fetchFunction, timeout)

		Returns a wrapped fetch function with timeout functionality.

		```js
		import {withTimeout} from 'fetch-extras';

		const fetchWithTimeout = withTimeout(fetch, 5000);
		const response = await fetchWithTimeout('/api');
		```

		### paginate(input, options?)

		Paginate through API responses using async iteration. By default, it automatically follows RFC 5988 `Link` headers with `rel="next"`.

		Returns an async iterator that yields items from each page.

		```js
		import {paginate} from 'fetch-extras';

		// Basic usage with Link headers (GitHub API)
		for await (const commit of paginate('https://api.github.com/repos/sindresorhus/ky/commits')) {
		console.log(commit.sha);
		}
		```

		#### options

		Type: `object`

		##### pagination

		Type: `object`

		###### transform

		Type: `(response: Response) => Promise<unknown[]>`\
		Default: `response => response.json()`

		Transform the response into an array of items.

		```js
		for await (const user of paginate('https://api.example.com/users', {
		pagination: {
		transform: async response => {
		const data = await response.json();
		return data.users; // Extract from nested property
		}
		}
		})) {
		console.log(user);
		}
		```

		###### paginate

		Type: `(data: {response, currentUrl, currentItems, allItems}) => Promise<PaginationNextPage \| false>`\
		Default: Parses RFC 5988 `Link` header

		Determine the next page to fetch. Return an object with fetch options for the next request, or `false` to stop pagination.

		> [!IMPORTANT]
		> The response body has already been consumed by the `transform` function. Do NOT call `response.json()` or other body methods here. Extract pagination info from headers, the URL, or share data from the transform function through closure.

		> [!NOTE]
		> Returning `headers` replaces all inherited headers, consistent with standard Fetch API behavior. If you need to add headers while keeping existing ones, read them from the response and include them in the returned object.
		> Setting `body` to `undefined` will strip body-related headers (`Content-Type`, `Content-Length`, etc.) from the request, consistent with HTTP semantics for bodyless requests.

		```js
		// Cursor-based pagination using headers (recommended)
		for await (const item of paginate('https://api.example.com/items', {
		pagination: {
		paginate: ({response}) => {
		const cursor = response.headers.get('X-Next-Cursor');
		return cursor
		? {url: new URL(`https://api.example.com/items?cursor=${cursor}`)}
		: false;
		}
		}
		})) {
		console.log(item);
		}
		```

		```js
		// Sharing data between transform and paginate via closure
		let nextCursor;

		for await (const item of paginate('https://api.example.com/items', {
		pagination: {
		transform: async (response) => {
		const data = await response.json();
		nextCursor = data.nextCursor;
		return data.items;
		},
		paginate: () => {
		return nextCursor
		? {url: new URL(`https://api.example.com/items?cursor=${nextCursor}`)}
		: false;
		}
		}
		})) {
		console.log(item);
		}
		```

		###### filter

		Type: `(data: {item, currentItems, allItems}) => boolean`\
		Default: `() => true`

		Filter items before yielding them.

		```js
		// Only get active users
		for await (const user of paginate('https://api.example.com/users', {
		pagination: {
		filter: ({item}) => item.status === 'active'
		}
		})) {
		console.log(user);
		}
		```

		###### shouldContinue

		Type: `(data: {item, currentItems, allItems}) => boolean`\
		Default: `() => true`

		Check if pagination should continue after yielding an item. This is called after `filter` returns `true`. Useful for stopping pagination based on item values.

		```js
		// Stop when we reach items older than one week
		const oneWeekAgo = Date.now() - (7 * 24 * 60 * 60 * 1000);

		for await (const commit of paginate('https://api.github.com/repos/user/repo/commits', {
		pagination: {
		shouldContinue: ({item}) => new Date(item.date).getTime() >= oneWeekAgo
		}
		})) {
		console.log(commit);
		}
		```

		###### countLimit

		Type: `number`\
		Default: `Infinity`

		Maximum number of items to yield.

		```js
		const items = await paginate.all('https://api.example.com/items', {
		pagination: {
		countLimit: 100 // Stop after 100 items
		}
		});
		```

		###### requestLimit

		Type: `number`\
		Default: `10000`

		Maximum number of requests to make. This prevents infinite loops if your `paginate` function has bugs. Ensure your `paginate` function eventually returns `false` or the iteration will continue until this limit is reached.

		###### backoff

		Type: `number`\
		Default: `0`

		Delay in milliseconds between requests. Useful for rate limiting.

		```js
		for await (const item of paginate('https://api.example.com/items', {
		pagination: {
		backoff: 1000 // Wait 1 second between requests
		}
		})) {
		console.log(item);
		}
		```

		###### stackAllItems

		Type: `boolean`\
		Default: `false`

		Whether to keep all yielded items in memory. When `true`, the `allItems` array passed to callbacks will contain all previously yielded items. When `false`, `allItems` will always be empty to save memory.

		##### fetchFunction

		Type: `(input: RequestInfo \| URL, init?: any) => Promise<Response>`\
		Default: `globalThis.fetch`

		Custom fetch function to use for requests. This allows you to use a custom fetch implementation, such as [`ky`](https://github.com/sindresorhus/ky), or a fetch function wrapped with `withHttpError` or `withTimeout`.

		```js
		import {paginate} from 'fetch-extras';
		import ky from 'ky';

		const url = 'https://api.github.com/repos/sindresorhus/ky/commits';

		for await (const commit of paginate(url, {fetchFunction: ky})) {
		console.log(commit.sha);
		}
		```

		### paginate.all(input, options?)

		Get all paginated items as an array. This is a convenience method that collects all items into memory. For large datasets, prefer using the async iterator directly.

		```js
		import {paginate} from 'fetch-extras';

		const commits = await paginate.all('https://api.github.com/repos/sindresorhus/ky/commits', {
		pagination: {
		countLimit: 50
		}
		});

		console.log(`Fetched ${commits.length} commits`);
		```

		## Related
		@@ -39,1 +299,2 @@
		- [ky](https://github.com/sindresorhus/ky) - HTTP client based on Fetch
		- [parse-sse](https://www.npmjs.com/package/parse-sse) - Parse Server-Sent Events (SSE) from a Response

-94

index.d.ts

		/**
		Custom error class for HTTP errors that should be thrown when the response has a non-200 status code.
		*/
		export class HttpError extends Error {
		readonly name: 'HttpError';
		readonly code: 'ERR_HTTP_RESPONSE_NOT_OK';
		response: Response;

		/**
		Constructs a new `HttpError` instance.

		@param response - The `Response` object that caused the error.
		*/
		constructor(response: Response);
		}

		/**
		Throws an `HttpError` if the response is not ok (non-200 status code).

		@param response - The `Response` object to check.
		@returns The same `Response` object if it is ok.
		@throws {HttpError} If the response is not ok.

		@example
		```
		import {throwIfHttpError} from 'fetch-extras';

		const response = await fetch('/api');
		throwIfHttpError(response);
		const data = await response.json();
		```
		*/
		export function throwIfHttpError(response: Response): Response;

		/**
		Throws an `HttpError` if the response is not ok (non-200 status code).

		@param responsePromise - A promise that resolves to a `Response` object to check.
		@returns A promise that resolves to the same `Response` object if it is ok.
		@throws {HttpError} If the response is not ok.

		@example
		```
		import {throwIfHttpError} from 'fetch-extras';

		const response = await throwIfHttpError(fetch('/api'));
		const data = await response.json();
		```
		*/
		export function throwIfHttpError(responsePromise: Promise<Response>): Promise<Response>;

		/**
		Wraps a fetch function to automatically throw `HttpError` for non-200 responses.

		Can be combined with other `with*` methods.

		@param fetchFunction - The fetch function to wrap (usually the global `fetch`).
		@returns A wrapped fetch function that will throw HttpError for non-200 responses.

		@example
		```
		import {withHttpError} from 'fetch-extras';

		const fetchWithError = withHttpError(fetch);
		const response = await fetchWithError('/api'); // Throws HttpError if status is not 200-299
		const data = await response.json();
		```
		*/
		export function withHttpError(
		fetchFunction: typeof fetch
		): typeof fetch;

		/**
		Wraps a fetch function with timeout functionality.

		Can be combined with other `with*` methods.

		@param fetchFunction - The fetch function to wrap (usually the global `fetch`).
		@param timeout - Timeout in milliseconds.
		@returns A wrapped fetch function that will abort if the request takes longer than the specified timeout.

		@example
		```
		import {withTimeout} from 'fetch-extras';

		const fetchWithTimeout = withTimeout(fetch, 5000);
		const response = await fetchWithTimeout('/api');
		const data = await response.json();
		```
		*/
		export function withTimeout(
		fetchFunction: typeof fetch,
		timeout: number
		): typeof fetch;

-41

index.js

		export class HttpError extends Error {
		constructor(response) {
		const status = `${response.status} ${response.statusText}`.trim();
		const reason = status ? `status code ${status}` : 'an unknown error';

		super(`Request failed with ${reason}: ${response.url}`);
		Error.captureStackTrace?.(this, this.constructor);

		this.name = 'HttpError';
		this.code = 'ERR_HTTP_RESPONSE_NOT_OK';
		this.response = response;
		}
		}

		export async function throwIfHttpError(responseOrPromise) {
		if (!(responseOrPromise instanceof Response)) {
		responseOrPromise = await responseOrPromise;
		}

		if (!responseOrPromise.ok) {
		throw new HttpError(responseOrPromise);
		}

		return responseOrPromise;
		}

		export function withHttpError(fetchFunction) {
		return async (urlOrRequest, options = {}) => {
		const response = await fetchFunction(urlOrRequest, options);
		return throwIfHttpError(response);
		};
		}

		export function withTimeout(fetchFunction, timeout) {
		return async (urlOrRequest, options = {}) => {
		const providedSignal = options.signal ?? (urlOrRequest instanceof Request && urlOrRequest.signal);
		const timeoutSignal = AbortSignal.timeout(timeout);
		const signal = providedSignal ? AbortSignal.any([providedSignal, timeoutSignal]) : timeoutSignal;
		return fetchFunction(urlOrRequest, {...options, signal});
		};
		}

fetch-extras - npm Package Compare versions

New alerts

Improved metrics

Worsened metrics