@datastream/fetch - npm Package Compare versions

Comparing version

0.1.5

0.1.6

+1

-1

index.node.mjs

		@@ -96,3 +96,3 @@ import {
		const { dataPath, nextPath } = options;
		let { url } = options;
		let url;
		while (options.url) {
		@@ -99,0 +99,0 @@ const response = options.prefetchResponse ?? await fetchRateLimit(options, streamOptions);

+2

-2

index.node.mjs.map

		{
		"version": 3,
		"sources": ["index.js"],
		"sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global fetch /\nimport {\n\tcreateReadableStream,\n\tcreateWritableStream,\n\ttimeout,\n} from \"@datastream/core\";\n\nconst defaults = {\n\t// custom\n\trateLimit: 0.01, // 100 per sec\n\tdataPath: undefined, // for json response, where the data is to return form body root\n\tnextPath: undefined, // for json pagination, body root\n\tqs: {}, // object to convert to query string\n\toffsetParam: undefined, // offset query parameter to use for pagination\n\toffsetAmount: undefined, // offset amount to use for pagination\n\n\t// fetch\n\tmethod: \"GET\",\n\theaders: {\n\t\tAccept: \"application/json\",\n\t\t\"Accept-Encoding\": \"br, gzip, deflate\",\n\t},\n};\n\nconst mergeOptions = (options = {}) => {\n\treturn {\n\t\t...defaults,\n\t\t...options,\n\t\theaders: { ...defaults.headers, ...options.headers },\n\t\tqs: { ...defaults.qs, ...options.qs },\n\t};\n};\n\nexport const fetchSetDefaults = (options) => {\n\tObject.assign(defaults, mergeOptions(options));\n};\n\n// Note: requires EncodeStream to ensure it's Uint8Array\n// Poor browser support - https://github.com/Fyrd/caniuse/issues/6375\nexport const fetchWritableStream = async (options, streamOptions = {}) => {\n\tconst body = createReadableStream();\n\t// Duplex: half - For browser compatibility - https://developer.chrome.com/articles/fetch-streaming-requests/#half-duplex\n\toptions = mergeOptions(options);\n\tconst value = await fetchRateLimit({\n\t\t...options,\n\t\tbody,\n\t\tduplex: \"half\",\n\t\tsignal: streamOptions.signal,\n\t});\n\tconst write = (chunk) => {\n\t\tbody.push(chunk);\n\t};\n\tconst stream = createWritableStream(write, streamOptions);\n\tstream.result = () => ({ key: \"output\", value });\n\treturn stream;\n};\nexport const fetchRequestStream = fetchWritableStream;\n\nexport const fetchReadableStream = (fetchOptions, streamOptions = {}) => {\n\treturn createReadableStream(\n\t\tfetchGenerator(fetchOptions, streamOptions),\n\t\tstreamOptions,\n\t);\n};\nexport const fetchResponseStream = fetchReadableStream;\n\nasync function fetchGenerator(fetchOptions, streamOptions) {\n\tlet rateLimitTimestamp = 0;\n\tif (!Array.isArray(fetchOptions)) fetchOptions = [fetchOptions];\n\tfor (let options of fetchOptions) {\n\t\toptions = mergeOptions(options);\n\t\toptions.rateLimitTimestamp ??= rateLimitTimestamp;\n\n\t\tif (options.offsetParam) {\n\t\t\toptions.qs[options.offsetParam] ??= 0;\n\t\t}\n\n\t\tif (Object.keys(options.qs).length) {\n\t\t\toptions.url += `?${new URLSearchParams(options.qs)}`.replaceAll(\n\t\t\t\t\"+\",\n\t\t\t\t\"%20\",\n\t\t\t);\n\t\t}\n\t\tconst response = await fetchUnknown(options, streamOptions);\n\t\tfor await (const chunk of response) {\n\t\t\tyield chunk;\n\t\t}\n\t\t// ensure there is rate limiting between req with different options\n\t\trateLimitTimestamp = options.rateLimitTimestamp;\n\t}\n}\n\nconst jsonContentTypeRegExp = /^application\\/(.+\\+)?json($\|;.+)/;\nconst fetchUnknown = async (options, streamOptions) => {\n\tconst response = await fetchRateLimit(options, streamOptions);\n\tif (jsonContentTypeRegExp.test(response.headers.get(\"Content-Type\"))) {\n\t\toptions.prefetchResponse = response; // hack\n\t\treturn fetchJson(options, streamOptions);\n\t}\n\treturn response.body;\n};\n\nconst nextLinkRegExp = /<(.?)>; rel=\"next\"/;\n\nasync function fetchJson(options, streamOptions) {\n\tconst { dataPath, nextPath } = options;\n\tlet { url } = options;\n\n\twhile (options.url) {\n\t\tconst response =\n\t\t\toptions.prefetchResponse ??\n\t\t\t(await fetchRateLimit(options, streamOptions));\n\t\tdelete options.prefetchResponse;\n\t\tconst body = await response.json();\n\t\turl = parseLinkFromHeader(response.headers);\n\t\turl ??= parseNextPath(body, nextPath);\n\t\turl ??= paginateUsingQuery(options);\n\t\toptions.url = url;\n\t\tconst data = pickPath(body, dataPath);\n\t\tif (Array.isArray(data)) {\n\t\t\tfor (const item of data) {\n\t\t\t\tyield item;\n\t\t\t}\n\n\t\t\tif (options.offsetParam && !data.length) break;\n\t\t} else {\n\t\t\tyield data;\n\t\t}\n\t}\n}\n\nconst paginateUsingQuery = (options) => {\n\tif (!options.offsetParam \|\| !options.offsetAmount) return undefined;\n\n\tconst url = new URL(options.url);\n\tlet offset = url.searchParams.get(options.offsetParam);\n\tif (!offset) return null;\n\n\toffset = Number.parseInt(offset, 10) + options.offsetAmount;\n\turl.searchParams.delete(options.offsetParam);\n\turl.searchParams.set(options.offsetParam, offset);\n\treturn url.toString();\n};\n\nconst parseNextPath = (body, nextPath) => {\n\treturn nextPath ? pickPath(body, nextPath) : undefined;\n};\n\nconst parseLinkFromHeader = (headers) => {\n\tconst link = headers.get(\"Link\");\n\treturn link?.match(nextLinkRegExp)?.[1];\n};\n\nexport const fetchRateLimit = async (options, streamOptions = {}) => {\n\tconst now = Date.now();\n\tif (now < (options.rateLimitTimestamp ?? 0)) {\n\t\tawait timeout(options.rateLimitTimestamp - now, streamOptions);\n\t}\n\toptions.rateLimitTimestamp = Date.now() + 1000 * options.rateLimit;\n\toptions = mergeOptions(options); // for when called directly\n\n\tconst {\n\t\tmethod,\n\t\theaders,\n\t\tbody,\n\t\tmode,\n\t\tcredentials,\n\t\tcache,\n\t\tredirect,\n\t\treferrer,\n\t\treferrerPolicy,\n\t\tintegrity,\n\t\tkeepalive,\n\t\tduplex,\n\t} = options;\n\tconst fetchInit = {\n\t\tmethod,\n\t\theaders,\n\t\tbody,\n\t\tmode,\n\t\tcredentials,\n\t\tcache,\n\t\tredirect,\n\t\treferrer,\n\t\treferrerPolicy,\n\t\tintegrity,\n\t\tkeepalive,\n\t\tduplex,\n\t\tsignal: streamOptions.signal,\n\t};\n\tconst response = await fetch(options.url, fetchInit);\n\tif (!response.ok) {\n\t\t// 429 Too Many Requests\n\t\tif (response.status === 429) {\n\t\t\treturn fetchRateLimit(options, streamOptions);\n\t\t}\n\t\tthrow new Error(\n\t\t\t`fetch ${response.status} ${options.method} ${options.url}`,\n\t\t\t{\n\t\t\t\tcause: {\n\t\t\t\t\tstatus: response.status,\n\t\t\t\t\turl: options.url,\n\t\t\t\t\tmethod: options.method,\n\t\t\t\t},\n\t\t\t},\n\t\t);\n\t}\n\treturn response;\n};\n\nconst pickPath = (obj, path = \"\") => {\n\tif (path === \"\") return obj;\n\tif (!Array.isArray(path)) path = path.split(\".\");\n\treturn path.reduce((a, b) => a?.[b], obj);\n};\n\nexport default {\n\tsetDefaults: fetchSetDefaults,\n\treadableStream: fetchReadableStream,\n\tresponseStream: fetchReadableStream,\n};\n"],
		"mappings": "AAGA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;AAEP,MAAM,WAAW;AAAA;AAAA,EAEhB,WAAW;AAAA;AAAA,EACX,UAAU;AAAA;AAAA,EACV,UAAU;AAAA;AAAA,EACV,IAAI,CAAC;AAAA;AAAA,EACL,aAAa;AAAA;AAAA,EACb,cAAc;AAAA;AAAA;AAAA,EAGd,QAAQ;AAAA,EACR,SAAS;AAAA,IACR,QAAQ;AAAA,IACR,mBAAmB;AAAA,EACpB;AACD;AAEA,MAAM,eAAe,CAAC,UAAU,CAAC,MAAM;AACtC,SAAO;AAAA,IACN,GAAG;AAAA,IACH,GAAG;AAAA,IACH,SAAS,EAAE,GAAG,SAAS,SAAS,GAAG,QAAQ,QAAQ;AAAA,IACnD,IAAI,EAAE,GAAG,SAAS,IAAI,GAAG,QAAQ,GAAG;AAAA,EACrC;AACD;AAEO,MAAM,mBAAmB,CAAC,YAAY;AAC5C,SAAO,OAAO,UAAU,aAAa,OAAO,CAAC;AAC9C;AAIO,MAAM,sBAAsB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AACzE,QAAM,OAAO,qBAAqB;AAElC,YAAU,aAAa,OAAO;AAC9B,QAAM,QAAQ,MAAM,eAAe;AAAA,IAClC,GAAG;AAAA,IACH;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ,cAAc;AAAA,EACvB,CAAC;AACD,QAAM,QAAQ,CAAC,UAAU;AACxB,SAAK,KAAK,KAAK;AAAA,EAChB;AACA,QAAM,SAAS,qBAAqB,OAAO,aAAa;AACxD,SAAO,SAAS,OAAO,EAAE,KAAK,UAAU,MAAM;AAC9C,SAAO;AACR;AACO,MAAM,qBAAqB;AAE3B,MAAM,sBAAsB,CAAC,cAAc,gBAAgB,CAAC,MAAM;AACxE,SAAO;AAAA,IACN,eAAe,cAAc,aAAa;AAAA,IAC1C;AAAA,EACD;AACD;AACO,MAAM,sBAAsB;AAEnC,gBAAgB,eAAe,cAAc,eAAe;AAC3D,MAAI,qBAAqB;AACzB,MAAI,CAAC,MAAM,QAAQ,YAAY,EAAG,gBAAe,CAAC,YAAY;AAC9D,WAAS,WAAW,cAAc;AACjC,cAAU,aAAa,OAAO;AAC9B,YAAQ,uBAAuB;AAE/B,QAAI,QAAQ,aAAa;AACxB,cAAQ,GAAG,QAAQ,WAAW,MAAM;AAAA,IACrC;AAEA,QAAI,OAAO,KAAK,QAAQ,EAAE,EAAE,QAAQ;AACnC,cAAQ,OAAO,IAAI,IAAI,gBAAgB,QAAQ,EAAE,CAAC,GAAG;AAAA,QACpD;AAAA,QACA;AAAA,MACD;AAAA,IACD;AACA,UAAM,WAAW,MAAM,aAAa,SAAS,aAAa;AAC1D,qBAAiB,SAAS,UAAU;AACnC,YAAM;AAAA,IACP;AAEA,yBAAqB,QAAQ;AAAA,EAC9B;AACD;AAEA,MAAM,wBAAwB;AAC9B,MAAM,eAAe,OAAO,SAAS,kBAAkB;AACtD,QAAM,WAAW,MAAM,eAAe,SAAS,aAAa;AAC5D,MAAI,sBAAsB,KAAK,SAAS,QAAQ,IAAI,cAAc,CAAC,GAAG;AACrE,YAAQ,mBAAmB;AAC3B,WAAO,UAAU,SAAS,aAAa;AAAA,EACxC;AACA,SAAO,SAAS;AACjB;AAEA,MAAM,iBAAiB;AAEvB,gBAAgB,UAAU,SAAS,eAAe;AACjD,QAAM,EAAE,UAAU,SAAS,IAAI;AAC/B,MAAI,EAAE,IAAI,IAAI;AAEd,SAAO,QAAQ,KAAK;AACnB,UAAM,WACL,QAAQ,oBACP,MAAM,eAAe,SAAS,aAAa;AAC7C,WAAO,QAAQ;AACf,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,oBAAoB,SAAS,OAAO;AAC1C,YAAQ,cAAc,MAAM,QAAQ;AACpC,YAAQ,mBAAmB,OAAO;AAClC,YAAQ,MAAM;AACd,UAAM,OAAO,SAAS,MAAM,QAAQ;AACpC,QAAI,MAAM,QAAQ,IAAI,GAAG;AACxB,iBAAW,QAAQ,MAAM;AACxB,cAAM;AAAA,MACP;AAEA,UAAI,QAAQ,eAAe,CAAC,KAAK,OAAQ;AAAA,IAC1C,OAAO;AACN,YAAM;AAAA,IACP;AAAA,EACD;AACD;AAEA,MAAM,qBAAqB,CAAC,YAAY;AACvC,MAAI,CAAC,QAAQ,eAAe,CAAC,QAAQ,aAAc,QAAO;AAE1D,QAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,MAAI,SAAS,IAAI,aAAa,IAAI,QAAQ,WAAW;AACrD,MAAI,CAAC,OAAQ,QAAO;AAEpB,WAAS,OAAO,SAAS,QAAQ,EAAE,IAAI,QAAQ;AAC/C,MAAI,aAAa,OAAO,QAAQ,WAAW;AAC3C,MAAI,aAAa,IAAI,QAAQ,aAAa,MAAM;AAChD,SAAO,IAAI,SAAS;AACrB;AAEA,MAAM,gBAAgB,CAAC,MAAM,aAAa;AACzC,SAAO,WAAW,SAAS,MAAM,QAAQ,IAAI;AAC9C;AAEA,MAAM,sBAAsB,CAAC,YAAY;AACxC,QAAM,OAAO,QAAQ,IAAI,MAAM;AAC/B,SAAO,MAAM,MAAM,cAAc,IAAI,CAAC;AACvC;AAEO,MAAM,iBAAiB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,OAAO,QAAQ,sBAAsB,IAAI;AAC5C,UAAM,QAAQ,QAAQ,qBAAqB,KAAK,aAAa;AAAA,EAC9D;AACA,UAAQ,qBAAqB,KAAK,IAAI,IAAI,MAAO,QAAQ;AACzD,YAAU,aAAa,OAAO;AAE9B,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,IAAI;AACJ,QAAM,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,cAAc;AAAA,EACvB;AACA,QAAM,WAAW,MAAM,MAAM,QAAQ,KAAK,SAAS;AACnD,MAAI,CAAC,SAAS,IAAI;AAEjB,QAAI,SAAS,WAAW,KAAK;AAC5B,aAAO,eAAe,SAAS,aAAa;AAAA,IAC7C;AACA,UAAM,IAAI;AAAA,MACT,SAAS,SAAS,MAAM,IAAI,QAAQ,MAAM,IAAI,QAAQ,GAAG;AAAA,MACzD;AAAA,QACC,OAAO;AAAA,UACN,QAAQ,SAAS;AAAA,UACjB,KAAK,QAAQ;AAAA,UACb,QAAQ,QAAQ;AAAA,QACjB;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,WAAW,CAAC,KAAK,OAAO,OAAO;AACpC,MAAI,SAAS,GAAI,QAAO;AACxB,MAAI,CAAC,MAAM,QAAQ,IAAI,EAAG,QAAO,KAAK,MAAM,GAAG;AAC/C,SAAO,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,GAAG;AACzC;AAEA,IAAO,gBAAQ;AAAA,EACd,aAAa;AAAA,EACb,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;",
		"sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global fetch /\nimport {\n\tcreateReadableStream,\n\tcreateWritableStream,\n\ttimeout,\n} from \"@datastream/core\";\n\nconst defaults = {\n\t// custom\n\trateLimit: 0.01, // 100 per sec\n\tdataPath: undefined, // for json response, where the data is to return form body root\n\tnextPath: undefined, // for json pagination, body root\n\tqs: {}, // object to convert to query string\n\toffsetParam: undefined, // offset query parameter to use for pagination\n\toffsetAmount: undefined, // offset amount to use for pagination\n\n\t// fetch\n\tmethod: \"GET\",\n\theaders: {\n\t\tAccept: \"application/json\",\n\t\t\"Accept-Encoding\": \"br, gzip, deflate\",\n\t},\n};\n\nconst mergeOptions = (options = {}) => {\n\treturn {\n\t\t...defaults,\n\t\t...options,\n\t\theaders: { ...defaults.headers, ...options.headers },\n\t\tqs: { ...defaults.qs, ...options.qs },\n\t};\n};\n\nexport const fetchSetDefaults = (options) => {\n\tObject.assign(defaults, mergeOptions(options));\n};\n\n// Note: requires EncodeStream to ensure it's Uint8Array\n// Poor browser support - https://github.com/Fyrd/caniuse/issues/6375\nexport const fetchWritableStream = async (options, streamOptions = {}) => {\n\tconst body = createReadableStream();\n\t// Duplex: half - For browser compatibility - https://developer.chrome.com/articles/fetch-streaming-requests/#half-duplex\n\toptions = mergeOptions(options);\n\tconst value = await fetchRateLimit({\n\t\t...options,\n\t\tbody,\n\t\tduplex: \"half\",\n\t\tsignal: streamOptions.signal,\n\t});\n\tconst write = (chunk) => {\n\t\tbody.push(chunk);\n\t};\n\tconst stream = createWritableStream(write, streamOptions);\n\tstream.result = () => ({ key: \"output\", value });\n\treturn stream;\n};\nexport const fetchRequestStream = fetchWritableStream;\n\nexport const fetchReadableStream = (fetchOptions, streamOptions = {}) => {\n\treturn createReadableStream(\n\t\tfetchGenerator(fetchOptions, streamOptions),\n\t\tstreamOptions,\n\t);\n};\nexport const fetchResponseStream = fetchReadableStream;\n\nasync function fetchGenerator(fetchOptions, streamOptions) {\n\tlet rateLimitTimestamp = 0;\n\tif (!Array.isArray(fetchOptions)) fetchOptions = [fetchOptions];\n\tfor (let options of fetchOptions) {\n\t\toptions = mergeOptions(options);\n\t\toptions.rateLimitTimestamp ??= rateLimitTimestamp;\n\n\t\tif (options.offsetParam) {\n\t\t\toptions.qs[options.offsetParam] ??= 0;\n\t\t}\n\n\t\tif (Object.keys(options.qs).length) {\n\t\t\toptions.url += `?${new URLSearchParams(options.qs)}`.replaceAll(\n\t\t\t\t\"+\",\n\t\t\t\t\"%20\",\n\t\t\t);\n\t\t}\n\t\tconst response = await fetchUnknown(options, streamOptions);\n\t\tfor await (const chunk of response) {\n\t\t\tyield chunk;\n\t\t}\n\t\t// ensure there is rate limiting between req with different options\n\t\trateLimitTimestamp = options.rateLimitTimestamp;\n\t}\n}\n\nconst jsonContentTypeRegExp = /^application\\/(.+\\+)?json($\|;.+)/;\nconst fetchUnknown = async (options, streamOptions) => {\n\tconst response = await fetchRateLimit(options, streamOptions);\n\tif (jsonContentTypeRegExp.test(response.headers.get(\"Content-Type\"))) {\n\t\toptions.prefetchResponse = response; // hack\n\t\treturn fetchJson(options, streamOptions);\n\t}\n\treturn response.body;\n};\n\nconst nextLinkRegExp = /<(.?)>; rel=\"next\"/;\n\nasync function fetchJson(options, streamOptions) {\n\tconst { dataPath, nextPath } = options;\n\tlet url;\n\n\twhile (options.url) {\n\t\tconst response =\n\t\t\toptions.prefetchResponse ??\n\t\t\t(await fetchRateLimit(options, streamOptions));\n\t\tdelete options.prefetchResponse;\n\t\tconst body = await response.json();\n\t\turl = parseLinkFromHeader(response.headers);\n\t\turl ??= parseNextPath(body, nextPath);\n\t\turl ??= paginateUsingQuery(options);\n\t\toptions.url = url;\n\t\tconst data = pickPath(body, dataPath);\n\t\tif (Array.isArray(data)) {\n\t\t\tfor (const item of data) {\n\t\t\t\tyield item;\n\t\t\t}\n\n\t\t\tif (options.offsetParam && !data.length) break;\n\t\t} else {\n\t\t\tyield data;\n\t\t}\n\t}\n}\n\nconst paginateUsingQuery = (options) => {\n\tif (!options.offsetParam \|\| !options.offsetAmount) return undefined;\n\n\tconst url = new URL(options.url);\n\tlet offset = url.searchParams.get(options.offsetParam);\n\tif (!offset) return null;\n\n\toffset = Number.parseInt(offset, 10) + options.offsetAmount;\n\turl.searchParams.delete(options.offsetParam);\n\turl.searchParams.set(options.offsetParam, offset);\n\treturn url.toString();\n};\n\nconst parseNextPath = (body, nextPath) => {\n\treturn nextPath ? pickPath(body, nextPath) : undefined;\n};\n\nconst parseLinkFromHeader = (headers) => {\n\tconst link = headers.get(\"Link\");\n\treturn link?.match(nextLinkRegExp)?.[1];\n};\n\nexport const fetchRateLimit = async (options, streamOptions = {}) => {\n\tconst now = Date.now();\n\tif (now < (options.rateLimitTimestamp ?? 0)) {\n\t\tawait timeout(options.rateLimitTimestamp - now, streamOptions);\n\t}\n\toptions.rateLimitTimestamp = Date.now() + 1000 * options.rateLimit;\n\toptions = mergeOptions(options); // for when called directly\n\n\tconst {\n\t\tmethod,\n\t\theaders,\n\t\tbody,\n\t\tmode,\n\t\tcredentials,\n\t\tcache,\n\t\tredirect,\n\t\treferrer,\n\t\treferrerPolicy,\n\t\tintegrity,\n\t\tkeepalive,\n\t\tduplex,\n\t} = options;\n\tconst fetchInit = {\n\t\tmethod,\n\t\theaders,\n\t\tbody,\n\t\tmode,\n\t\tcredentials,\n\t\tcache,\n\t\tredirect,\n\t\treferrer,\n\t\treferrerPolicy,\n\t\tintegrity,\n\t\tkeepalive,\n\t\tduplex,\n\t\tsignal: streamOptions.signal,\n\t};\n\tconst response = await fetch(options.url, fetchInit);\n\tif (!response.ok) {\n\t\t// 429 Too Many Requests\n\t\tif (response.status === 429) {\n\t\t\treturn fetchRateLimit(options, streamOptions);\n\t\t}\n\t\tthrow new Error(\n\t\t\t`fetch ${response.status} ${options.method} ${options.url}`,\n\t\t\t{\n\t\t\t\tcause: {\n\t\t\t\t\tstatus: response.status,\n\t\t\t\t\turl: options.url,\n\t\t\t\t\tmethod: options.method,\n\t\t\t\t},\n\t\t\t},\n\t\t);\n\t}\n\treturn response;\n};\n\nconst pickPath = (obj, path = \"\") => {\n\tif (path === \"\") return obj;\n\tif (!Array.isArray(path)) path = path.split(\".\");\n\treturn path.reduce((a, b) => a?.[b], obj);\n};\n\nexport default {\n\tsetDefaults: fetchSetDefaults,\n\treadableStream: fetchReadableStream,\n\tresponseStream: fetchReadableStream,\n};\n"],
		"mappings": "AAGA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;AAEP,MAAM,WAAW;AAAA;AAAA,EAEhB,WAAW;AAAA;AAAA,EACX,UAAU;AAAA;AAAA,EACV,UAAU;AAAA;AAAA,EACV,IAAI,CAAC;AAAA;AAAA,EACL,aAAa;AAAA;AAAA,EACb,cAAc;AAAA;AAAA;AAAA,EAGd,QAAQ;AAAA,EACR,SAAS;AAAA,IACR,QAAQ;AAAA,IACR,mBAAmB;AAAA,EACpB;AACD;AAEA,MAAM,eAAe,CAAC,UAAU,CAAC,MAAM;AACtC,SAAO;AAAA,IACN,GAAG;AAAA,IACH,GAAG;AAAA,IACH,SAAS,EAAE,GAAG,SAAS,SAAS,GAAG,QAAQ,QAAQ;AAAA,IACnD,IAAI,EAAE,GAAG,SAAS,IAAI,GAAG,QAAQ,GAAG;AAAA,EACrC;AACD;AAEO,MAAM,mBAAmB,CAAC,YAAY;AAC5C,SAAO,OAAO,UAAU,aAAa,OAAO,CAAC;AAC9C;AAIO,MAAM,sBAAsB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AACzE,QAAM,OAAO,qBAAqB;AAElC,YAAU,aAAa,OAAO;AAC9B,QAAM,QAAQ,MAAM,eAAe;AAAA,IAClC,GAAG;AAAA,IACH;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ,cAAc;AAAA,EACvB,CAAC;AACD,QAAM,QAAQ,CAAC,UAAU;AACxB,SAAK,KAAK,KAAK;AAAA,EAChB;AACA,QAAM,SAAS,qBAAqB,OAAO,aAAa;AACxD,SAAO,SAAS,OAAO,EAAE,KAAK,UAAU,MAAM;AAC9C,SAAO;AACR;AACO,MAAM,qBAAqB;AAE3B,MAAM,sBAAsB,CAAC,cAAc,gBAAgB,CAAC,MAAM;AACxE,SAAO;AAAA,IACN,eAAe,cAAc,aAAa;AAAA,IAC1C;AAAA,EACD;AACD;AACO,MAAM,sBAAsB;AAEnC,gBAAgB,eAAe,cAAc,eAAe;AAC3D,MAAI,qBAAqB;AACzB,MAAI,CAAC,MAAM,QAAQ,YAAY,EAAG,gBAAe,CAAC,YAAY;AAC9D,WAAS,WAAW,cAAc;AACjC,cAAU,aAAa,OAAO;AAC9B,YAAQ,uBAAuB;AAE/B,QAAI,QAAQ,aAAa;AACxB,cAAQ,GAAG,QAAQ,WAAW,MAAM;AAAA,IACrC;AAEA,QAAI,OAAO,KAAK,QAAQ,EAAE,EAAE,QAAQ;AACnC,cAAQ,OAAO,IAAI,IAAI,gBAAgB,QAAQ,EAAE,CAAC,GAAG;AAAA,QACpD;AAAA,QACA;AAAA,MACD;AAAA,IACD;AACA,UAAM,WAAW,MAAM,aAAa,SAAS,aAAa;AAC1D,qBAAiB,SAAS,UAAU;AACnC,YAAM;AAAA,IACP;AAEA,yBAAqB,QAAQ;AAAA,EAC9B;AACD;AAEA,MAAM,wBAAwB;AAC9B,MAAM,eAAe,OAAO,SAAS,kBAAkB;AACtD,QAAM,WAAW,MAAM,eAAe,SAAS,aAAa;AAC5D,MAAI,sBAAsB,KAAK,SAAS,QAAQ,IAAI,cAAc,CAAC,GAAG;AACrE,YAAQ,mBAAmB;AAC3B,WAAO,UAAU,SAAS,aAAa;AAAA,EACxC;AACA,SAAO,SAAS;AACjB;AAEA,MAAM,iBAAiB;AAEvB,gBAAgB,UAAU,SAAS,eAAe;AACjD,QAAM,EAAE,UAAU,SAAS,IAAI;AAC/B,MAAI;AAEJ,SAAO,QAAQ,KAAK;AACnB,UAAM,WACL,QAAQ,oBACP,MAAM,eAAe,SAAS,aAAa;AAC7C,WAAO,QAAQ;AACf,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,oBAAoB,SAAS,OAAO;AAC1C,YAAQ,cAAc,MAAM,QAAQ;AACpC,YAAQ,mBAAmB,OAAO;AAClC,YAAQ,MAAM;AACd,UAAM,OAAO,SAAS,MAAM,QAAQ;AACpC,QAAI,MAAM,QAAQ,IAAI,GAAG;AACxB,iBAAW,QAAQ,MAAM;AACxB,cAAM;AAAA,MACP;AAEA,UAAI,QAAQ,eAAe,CAAC,KAAK,OAAQ;AAAA,IAC1C,OAAO;AACN,YAAM;AAAA,IACP;AAAA,EACD;AACD;AAEA,MAAM,qBAAqB,CAAC,YAAY;AACvC,MAAI,CAAC,QAAQ,eAAe,CAAC,QAAQ,aAAc,QAAO;AAE1D,QAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,MAAI,SAAS,IAAI,aAAa,IAAI,QAAQ,WAAW;AACrD,MAAI,CAAC,OAAQ,QAAO;AAEpB,WAAS,OAAO,SAAS,QAAQ,EAAE,IAAI,QAAQ;AAC/C,MAAI,aAAa,OAAO,QAAQ,WAAW;AAC3C,MAAI,aAAa,IAAI,QAAQ,aAAa,MAAM;AAChD,SAAO,IAAI,SAAS;AACrB;AAEA,MAAM,gBAAgB,CAAC,MAAM,aAAa;AACzC,SAAO,WAAW,SAAS,MAAM,QAAQ,IAAI;AAC9C;AAEA,MAAM,sBAAsB,CAAC,YAAY;AACxC,QAAM,OAAO,QAAQ,IAAI,MAAM;AAC/B,SAAO,MAAM,MAAM,cAAc,IAAI,CAAC;AACvC;AAEO,MAAM,iBAAiB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,OAAO,QAAQ,sBAAsB,IAAI;AAC5C,UAAM,QAAQ,QAAQ,qBAAqB,KAAK,aAAa;AAAA,EAC9D;AACA,UAAQ,qBAAqB,KAAK,IAAI,IAAI,MAAO,QAAQ;AACzD,YAAU,aAAa,OAAO;AAE9B,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,IAAI;AACJ,QAAM,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,cAAc;AAAA,EACvB;AACA,QAAM,WAAW,MAAM,MAAM,QAAQ,KAAK,SAAS;AACnD,MAAI,CAAC,SAAS,IAAI;AAEjB,QAAI,SAAS,WAAW,KAAK;AAC5B,aAAO,eAAe,SAAS,aAAa;AAAA,IAC7C;AACA,UAAM,IAAI;AAAA,MACT,SAAS,SAAS,MAAM,IAAI,QAAQ,MAAM,IAAI,QAAQ,GAAG;AAAA,MACzD;AAAA,QACC,OAAO;AAAA,UACN,QAAQ,SAAS;AAAA,UACjB,KAAK,QAAQ;AAAA,UACb,QAAQ,QAAQ;AAAA,QACjB;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,WAAW,CAAC,KAAK,OAAO,OAAO;AACpC,MAAI,SAAS,GAAI,QAAO;AACxB,MAAI,CAAC,MAAM,QAAQ,IAAI,EAAG,QAAO,KAAK,MAAM,GAAG;AAC/C,SAAO,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,GAAG;AACzC;AAEA,IAAO,gBAAQ;AAAA,EACd,aAAa;AAAA,EACb,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;",
		"names": []
		}

+1

-1

index.web.mjs

		@@ -96,3 +96,3 @@ import {
		const { dataPath, nextPath } = options;
		let { url } = options;
		let url;
		while (options.url) {
		@@ -99,0 +99,0 @@ const response = options.prefetchResponse ?? await fetchRateLimit(options, streamOptions);

+2

-2

index.web.mjs.map

		{
		"version": 3,
		"sources": ["index.js"],
		"sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global fetch /\nimport {\n\tcreateReadableStream,\n\tcreateWritableStream,\n\ttimeout,\n} from \"@datastream/core\";\n\nconst defaults = {\n\t// custom\n\trateLimit: 0.01, // 100 per sec\n\tdataPath: undefined, // for json response, where the data is to return form body root\n\tnextPath: undefined, // for json pagination, body root\n\tqs: {}, // object to convert to query string\n\toffsetParam: undefined, // offset query parameter to use for pagination\n\toffsetAmount: undefined, // offset amount to use for pagination\n\n\t// fetch\n\tmethod: \"GET\",\n\theaders: {\n\t\tAccept: \"application/json\",\n\t\t\"Accept-Encoding\": \"br, gzip, deflate\",\n\t},\n};\n\nconst mergeOptions = (options = {}) => {\n\treturn {\n\t\t...defaults,\n\t\t...options,\n\t\theaders: { ...defaults.headers, ...options.headers },\n\t\tqs: { ...defaults.qs, ...options.qs },\n\t};\n};\n\nexport const fetchSetDefaults = (options) => {\n\tObject.assign(defaults, mergeOptions(options));\n};\n\n// Note: requires EncodeStream to ensure it's Uint8Array\n// Poor browser support - https://github.com/Fyrd/caniuse/issues/6375\nexport const fetchWritableStream = async (options, streamOptions = {}) => {\n\tconst body = createReadableStream();\n\t// Duplex: half - For browser compatibility - https://developer.chrome.com/articles/fetch-streaming-requests/#half-duplex\n\toptions = mergeOptions(options);\n\tconst value = await fetchRateLimit({\n\t\t...options,\n\t\tbody,\n\t\tduplex: \"half\",\n\t\tsignal: streamOptions.signal,\n\t});\n\tconst write = (chunk) => {\n\t\tbody.push(chunk);\n\t};\n\tconst stream = createWritableStream(write, streamOptions);\n\tstream.result = () => ({ key: \"output\", value });\n\treturn stream;\n};\nexport const fetchRequestStream = fetchWritableStream;\n\nexport const fetchReadableStream = (fetchOptions, streamOptions = {}) => {\n\treturn createReadableStream(\n\t\tfetchGenerator(fetchOptions, streamOptions),\n\t\tstreamOptions,\n\t);\n};\nexport const fetchResponseStream = fetchReadableStream;\n\nasync function fetchGenerator(fetchOptions, streamOptions) {\n\tlet rateLimitTimestamp = 0;\n\tif (!Array.isArray(fetchOptions)) fetchOptions = [fetchOptions];\n\tfor (let options of fetchOptions) {\n\t\toptions = mergeOptions(options);\n\t\toptions.rateLimitTimestamp ??= rateLimitTimestamp;\n\n\t\tif (options.offsetParam) {\n\t\t\toptions.qs[options.offsetParam] ??= 0;\n\t\t}\n\n\t\tif (Object.keys(options.qs).length) {\n\t\t\toptions.url += `?${new URLSearchParams(options.qs)}`.replaceAll(\n\t\t\t\t\"+\",\n\t\t\t\t\"%20\",\n\t\t\t);\n\t\t}\n\t\tconst response = await fetchUnknown(options, streamOptions);\n\t\tfor await (const chunk of response) {\n\t\t\tyield chunk;\n\t\t}\n\t\t// ensure there is rate limiting between req with different options\n\t\trateLimitTimestamp = options.rateLimitTimestamp;\n\t}\n}\n\nconst jsonContentTypeRegExp = /^application\\/(.+\\+)?json($\|;.+)/;\nconst fetchUnknown = async (options, streamOptions) => {\n\tconst response = await fetchRateLimit(options, streamOptions);\n\tif (jsonContentTypeRegExp.test(response.headers.get(\"Content-Type\"))) {\n\t\toptions.prefetchResponse = response; // hack\n\t\treturn fetchJson(options, streamOptions);\n\t}\n\treturn response.body;\n};\n\nconst nextLinkRegExp = /<(.?)>; rel=\"next\"/;\n\nasync function fetchJson(options, streamOptions) {\n\tconst { dataPath, nextPath } = options;\n\tlet { url } = options;\n\n\twhile (options.url) {\n\t\tconst response =\n\t\t\toptions.prefetchResponse ??\n\t\t\t(await fetchRateLimit(options, streamOptions));\n\t\tdelete options.prefetchResponse;\n\t\tconst body = await response.json();\n\t\turl = parseLinkFromHeader(response.headers);\n\t\turl ??= parseNextPath(body, nextPath);\n\t\turl ??= paginateUsingQuery(options);\n\t\toptions.url = url;\n\t\tconst data = pickPath(body, dataPath);\n\t\tif (Array.isArray(data)) {\n\t\t\tfor (const item of data) {\n\t\t\t\tyield item;\n\t\t\t}\n\n\t\t\tif (options.offsetParam && !data.length) break;\n\t\t} else {\n\t\t\tyield data;\n\t\t}\n\t}\n}\n\nconst paginateUsingQuery = (options) => {\n\tif (!options.offsetParam \|\| !options.offsetAmount) return undefined;\n\n\tconst url = new URL(options.url);\n\tlet offset = url.searchParams.get(options.offsetParam);\n\tif (!offset) return null;\n\n\toffset = Number.parseInt(offset, 10) + options.offsetAmount;\n\turl.searchParams.delete(options.offsetParam);\n\turl.searchParams.set(options.offsetParam, offset);\n\treturn url.toString();\n};\n\nconst parseNextPath = (body, nextPath) => {\n\treturn nextPath ? pickPath(body, nextPath) : undefined;\n};\n\nconst parseLinkFromHeader = (headers) => {\n\tconst link = headers.get(\"Link\");\n\treturn link?.match(nextLinkRegExp)?.[1];\n};\n\nexport const fetchRateLimit = async (options, streamOptions = {}) => {\n\tconst now = Date.now();\n\tif (now < (options.rateLimitTimestamp ?? 0)) {\n\t\tawait timeout(options.rateLimitTimestamp - now, streamOptions);\n\t}\n\toptions.rateLimitTimestamp = Date.now() + 1000 * options.rateLimit;\n\toptions = mergeOptions(options); // for when called directly\n\n\tconst {\n\t\tmethod,\n\t\theaders,\n\t\tbody,\n\t\tmode,\n\t\tcredentials,\n\t\tcache,\n\t\tredirect,\n\t\treferrer,\n\t\treferrerPolicy,\n\t\tintegrity,\n\t\tkeepalive,\n\t\tduplex,\n\t} = options;\n\tconst fetchInit = {\n\t\tmethod,\n\t\theaders,\n\t\tbody,\n\t\tmode,\n\t\tcredentials,\n\t\tcache,\n\t\tredirect,\n\t\treferrer,\n\t\treferrerPolicy,\n\t\tintegrity,\n\t\tkeepalive,\n\t\tduplex,\n\t\tsignal: streamOptions.signal,\n\t};\n\tconst response = await fetch(options.url, fetchInit);\n\tif (!response.ok) {\n\t\t// 429 Too Many Requests\n\t\tif (response.status === 429) {\n\t\t\treturn fetchRateLimit(options, streamOptions);\n\t\t}\n\t\tthrow new Error(\n\t\t\t`fetch ${response.status} ${options.method} ${options.url}`,\n\t\t\t{\n\t\t\t\tcause: {\n\t\t\t\t\tstatus: response.status,\n\t\t\t\t\turl: options.url,\n\t\t\t\t\tmethod: options.method,\n\t\t\t\t},\n\t\t\t},\n\t\t);\n\t}\n\treturn response;\n};\n\nconst pickPath = (obj, path = \"\") => {\n\tif (path === \"\") return obj;\n\tif (!Array.isArray(path)) path = path.split(\".\");\n\treturn path.reduce((a, b) => a?.[b], obj);\n};\n\nexport default {\n\tsetDefaults: fetchSetDefaults,\n\treadableStream: fetchReadableStream,\n\tresponseStream: fetchReadableStream,\n};\n"],
		"mappings": "AAGA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;AAEP,MAAM,WAAW;AAAA;AAAA,EAEhB,WAAW;AAAA;AAAA,EACX,UAAU;AAAA;AAAA,EACV,UAAU;AAAA;AAAA,EACV,IAAI,CAAC;AAAA;AAAA,EACL,aAAa;AAAA;AAAA,EACb,cAAc;AAAA;AAAA;AAAA,EAGd,QAAQ;AAAA,EACR,SAAS;AAAA,IACR,QAAQ;AAAA,IACR,mBAAmB;AAAA,EACpB;AACD;AAEA,MAAM,eAAe,CAAC,UAAU,CAAC,MAAM;AACtC,SAAO;AAAA,IACN,GAAG;AAAA,IACH,GAAG;AAAA,IACH,SAAS,EAAE,GAAG,SAAS,SAAS,GAAG,QAAQ,QAAQ;AAAA,IACnD,IAAI,EAAE,GAAG,SAAS,IAAI,GAAG,QAAQ,GAAG;AAAA,EACrC;AACD;AAEO,MAAM,mBAAmB,CAAC,YAAY;AAC5C,SAAO,OAAO,UAAU,aAAa,OAAO,CAAC;AAC9C;AAIO,MAAM,sBAAsB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AACzE,QAAM,OAAO,qBAAqB;AAElC,YAAU,aAAa,OAAO;AAC9B,QAAM,QAAQ,MAAM,eAAe;AAAA,IAClC,GAAG;AAAA,IACH;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ,cAAc;AAAA,EACvB,CAAC;AACD,QAAM,QAAQ,CAAC,UAAU;AACxB,SAAK,KAAK,KAAK;AAAA,EAChB;AACA,QAAM,SAAS,qBAAqB,OAAO,aAAa;AACxD,SAAO,SAAS,OAAO,EAAE,KAAK,UAAU,MAAM;AAC9C,SAAO;AACR;AACO,MAAM,qBAAqB;AAE3B,MAAM,sBAAsB,CAAC,cAAc,gBAAgB,CAAC,MAAM;AACxE,SAAO;AAAA,IACN,eAAe,cAAc,aAAa;AAAA,IAC1C;AAAA,EACD;AACD;AACO,MAAM,sBAAsB;AAEnC,gBAAgB,eAAe,cAAc,eAAe;AAC3D,MAAI,qBAAqB;AACzB,MAAI,CAAC,MAAM,QAAQ,YAAY,EAAG,gBAAe,CAAC,YAAY;AAC9D,WAAS,WAAW,cAAc;AACjC,cAAU,aAAa,OAAO;AAC9B,YAAQ,uBAAuB;AAE/B,QAAI,QAAQ,aAAa;AACxB,cAAQ,GAAG,QAAQ,WAAW,MAAM;AAAA,IACrC;AAEA,QAAI,OAAO,KAAK,QAAQ,EAAE,EAAE,QAAQ;AACnC,cAAQ,OAAO,IAAI,IAAI,gBAAgB,QAAQ,EAAE,CAAC,GAAG;AAAA,QACpD;AAAA,QACA;AAAA,MACD;AAAA,IACD;AACA,UAAM,WAAW,MAAM,aAAa,SAAS,aAAa;AAC1D,qBAAiB,SAAS,UAAU;AACnC,YAAM;AAAA,IACP;AAEA,yBAAqB,QAAQ;AAAA,EAC9B;AACD;AAEA,MAAM,wBAAwB;AAC9B,MAAM,eAAe,OAAO,SAAS,kBAAkB;AACtD,QAAM,WAAW,MAAM,eAAe,SAAS,aAAa;AAC5D,MAAI,sBAAsB,KAAK,SAAS,QAAQ,IAAI,cAAc,CAAC,GAAG;AACrE,YAAQ,mBAAmB;AAC3B,WAAO,UAAU,SAAS,aAAa;AAAA,EACxC;AACA,SAAO,SAAS;AACjB;AAEA,MAAM,iBAAiB;AAEvB,gBAAgB,UAAU,SAAS,eAAe;AACjD,QAAM,EAAE,UAAU,SAAS,IAAI;AAC/B,MAAI,EAAE,IAAI,IAAI;AAEd,SAAO,QAAQ,KAAK;AACnB,UAAM,WACL,QAAQ,oBACP,MAAM,eAAe,SAAS,aAAa;AAC7C,WAAO,QAAQ;AACf,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,oBAAoB,SAAS,OAAO;AAC1C,YAAQ,cAAc,MAAM,QAAQ;AACpC,YAAQ,mBAAmB,OAAO;AAClC,YAAQ,MAAM;AACd,UAAM,OAAO,SAAS,MAAM,QAAQ;AACpC,QAAI,MAAM,QAAQ,IAAI,GAAG;AACxB,iBAAW,QAAQ,MAAM;AACxB,cAAM;AAAA,MACP;AAEA,UAAI,QAAQ,eAAe,CAAC,KAAK,OAAQ;AAAA,IAC1C,OAAO;AACN,YAAM;AAAA,IACP;AAAA,EACD;AACD;AAEA,MAAM,qBAAqB,CAAC,YAAY;AACvC,MAAI,CAAC,QAAQ,eAAe,CAAC,QAAQ,aAAc,QAAO;AAE1D,QAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,MAAI,SAAS,IAAI,aAAa,IAAI,QAAQ,WAAW;AACrD,MAAI,CAAC,OAAQ,QAAO;AAEpB,WAAS,OAAO,SAAS,QAAQ,EAAE,IAAI,QAAQ;AAC/C,MAAI,aAAa,OAAO,QAAQ,WAAW;AAC3C,MAAI,aAAa,IAAI,QAAQ,aAAa,MAAM;AAChD,SAAO,IAAI,SAAS;AACrB;AAEA,MAAM,gBAAgB,CAAC,MAAM,aAAa;AACzC,SAAO,WAAW,SAAS,MAAM,QAAQ,IAAI;AAC9C;AAEA,MAAM,sBAAsB,CAAC,YAAY;AACxC,QAAM,OAAO,QAAQ,IAAI,MAAM;AAC/B,SAAO,MAAM,MAAM,cAAc,IAAI,CAAC;AACvC;AAEO,MAAM,iBAAiB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,OAAO,QAAQ,sBAAsB,IAAI;AAC5C,UAAM,QAAQ,QAAQ,qBAAqB,KAAK,aAAa;AAAA,EAC9D;AACA,UAAQ,qBAAqB,KAAK,IAAI,IAAI,MAAO,QAAQ;AACzD,YAAU,aAAa,OAAO;AAE9B,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,IAAI;AACJ,QAAM,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,cAAc;AAAA,EACvB;AACA,QAAM,WAAW,MAAM,MAAM,QAAQ,KAAK,SAAS;AACnD,MAAI,CAAC,SAAS,IAAI;AAEjB,QAAI,SAAS,WAAW,KAAK;AAC5B,aAAO,eAAe,SAAS,aAAa;AAAA,IAC7C;AACA,UAAM,IAAI;AAAA,MACT,SAAS,SAAS,MAAM,IAAI,QAAQ,MAAM,IAAI,QAAQ,GAAG;AAAA,MACzD;AAAA,QACC,OAAO;AAAA,UACN,QAAQ,SAAS;AAAA,UACjB,KAAK,QAAQ;AAAA,UACb,QAAQ,QAAQ;AAAA,QACjB;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,WAAW,CAAC,KAAK,OAAO,OAAO;AACpC,MAAI,SAAS,GAAI,QAAO;AACxB,MAAI,CAAC,MAAM,QAAQ,IAAI,EAAG,QAAO,KAAK,MAAM,GAAG;AAC/C,SAAO,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,GAAG;AACzC;AAEA,IAAO,gBAAQ;AAAA,EACd,aAAa;AAAA,EACb,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;",
		"sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global fetch /\nimport {\n\tcreateReadableStream,\n\tcreateWritableStream,\n\ttimeout,\n} from \"@datastream/core\";\n\nconst defaults = {\n\t// custom\n\trateLimit: 0.01, // 100 per sec\n\tdataPath: undefined, // for json response, where the data is to return form body root\n\tnextPath: undefined, // for json pagination, body root\n\tqs: {}, // object to convert to query string\n\toffsetParam: undefined, // offset query parameter to use for pagination\n\toffsetAmount: undefined, // offset amount to use for pagination\n\n\t// fetch\n\tmethod: \"GET\",\n\theaders: {\n\t\tAccept: \"application/json\",\n\t\t\"Accept-Encoding\": \"br, gzip, deflate\",\n\t},\n};\n\nconst mergeOptions = (options = {}) => {\n\treturn {\n\t\t...defaults,\n\t\t...options,\n\t\theaders: { ...defaults.headers, ...options.headers },\n\t\tqs: { ...defaults.qs, ...options.qs },\n\t};\n};\n\nexport const fetchSetDefaults = (options) => {\n\tObject.assign(defaults, mergeOptions(options));\n};\n\n// Note: requires EncodeStream to ensure it's Uint8Array\n// Poor browser support - https://github.com/Fyrd/caniuse/issues/6375\nexport const fetchWritableStream = async (options, streamOptions = {}) => {\n\tconst body = createReadableStream();\n\t// Duplex: half - For browser compatibility - https://developer.chrome.com/articles/fetch-streaming-requests/#half-duplex\n\toptions = mergeOptions(options);\n\tconst value = await fetchRateLimit({\n\t\t...options,\n\t\tbody,\n\t\tduplex: \"half\",\n\t\tsignal: streamOptions.signal,\n\t});\n\tconst write = (chunk) => {\n\t\tbody.push(chunk);\n\t};\n\tconst stream = createWritableStream(write, streamOptions);\n\tstream.result = () => ({ key: \"output\", value });\n\treturn stream;\n};\nexport const fetchRequestStream = fetchWritableStream;\n\nexport const fetchReadableStream = (fetchOptions, streamOptions = {}) => {\n\treturn createReadableStream(\n\t\tfetchGenerator(fetchOptions, streamOptions),\n\t\tstreamOptions,\n\t);\n};\nexport const fetchResponseStream = fetchReadableStream;\n\nasync function fetchGenerator(fetchOptions, streamOptions) {\n\tlet rateLimitTimestamp = 0;\n\tif (!Array.isArray(fetchOptions)) fetchOptions = [fetchOptions];\n\tfor (let options of fetchOptions) {\n\t\toptions = mergeOptions(options);\n\t\toptions.rateLimitTimestamp ??= rateLimitTimestamp;\n\n\t\tif (options.offsetParam) {\n\t\t\toptions.qs[options.offsetParam] ??= 0;\n\t\t}\n\n\t\tif (Object.keys(options.qs).length) {\n\t\t\toptions.url += `?${new URLSearchParams(options.qs)}`.replaceAll(\n\t\t\t\t\"+\",\n\t\t\t\t\"%20\",\n\t\t\t);\n\t\t}\n\t\tconst response = await fetchUnknown(options, streamOptions);\n\t\tfor await (const chunk of response) {\n\t\t\tyield chunk;\n\t\t}\n\t\t// ensure there is rate limiting between req with different options\n\t\trateLimitTimestamp = options.rateLimitTimestamp;\n\t}\n}\n\nconst jsonContentTypeRegExp = /^application\\/(.+\\+)?json($\|;.+)/;\nconst fetchUnknown = async (options, streamOptions) => {\n\tconst response = await fetchRateLimit(options, streamOptions);\n\tif (jsonContentTypeRegExp.test(response.headers.get(\"Content-Type\"))) {\n\t\toptions.prefetchResponse = response; // hack\n\t\treturn fetchJson(options, streamOptions);\n\t}\n\treturn response.body;\n};\n\nconst nextLinkRegExp = /<(.?)>; rel=\"next\"/;\n\nasync function fetchJson(options, streamOptions) {\n\tconst { dataPath, nextPath } = options;\n\tlet url;\n\n\twhile (options.url) {\n\t\tconst response =\n\t\t\toptions.prefetchResponse ??\n\t\t\t(await fetchRateLimit(options, streamOptions));\n\t\tdelete options.prefetchResponse;\n\t\tconst body = await response.json();\n\t\turl = parseLinkFromHeader(response.headers);\n\t\turl ??= parseNextPath(body, nextPath);\n\t\turl ??= paginateUsingQuery(options);\n\t\toptions.url = url;\n\t\tconst data = pickPath(body, dataPath);\n\t\tif (Array.isArray(data)) {\n\t\t\tfor (const item of data) {\n\t\t\t\tyield item;\n\t\t\t}\n\n\t\t\tif (options.offsetParam && !data.length) break;\n\t\t} else {\n\t\t\tyield data;\n\t\t}\n\t}\n}\n\nconst paginateUsingQuery = (options) => {\n\tif (!options.offsetParam \|\| !options.offsetAmount) return undefined;\n\n\tconst url = new URL(options.url);\n\tlet offset = url.searchParams.get(options.offsetParam);\n\tif (!offset) return null;\n\n\toffset = Number.parseInt(offset, 10) + options.offsetAmount;\n\turl.searchParams.delete(options.offsetParam);\n\turl.searchParams.set(options.offsetParam, offset);\n\treturn url.toString();\n};\n\nconst parseNextPath = (body, nextPath) => {\n\treturn nextPath ? pickPath(body, nextPath) : undefined;\n};\n\nconst parseLinkFromHeader = (headers) => {\n\tconst link = headers.get(\"Link\");\n\treturn link?.match(nextLinkRegExp)?.[1];\n};\n\nexport const fetchRateLimit = async (options, streamOptions = {}) => {\n\tconst now = Date.now();\n\tif (now < (options.rateLimitTimestamp ?? 0)) {\n\t\tawait timeout(options.rateLimitTimestamp - now, streamOptions);\n\t}\n\toptions.rateLimitTimestamp = Date.now() + 1000 * options.rateLimit;\n\toptions = mergeOptions(options); // for when called directly\n\n\tconst {\n\t\tmethod,\n\t\theaders,\n\t\tbody,\n\t\tmode,\n\t\tcredentials,\n\t\tcache,\n\t\tredirect,\n\t\treferrer,\n\t\treferrerPolicy,\n\t\tintegrity,\n\t\tkeepalive,\n\t\tduplex,\n\t} = options;\n\tconst fetchInit = {\n\t\tmethod,\n\t\theaders,\n\t\tbody,\n\t\tmode,\n\t\tcredentials,\n\t\tcache,\n\t\tredirect,\n\t\treferrer,\n\t\treferrerPolicy,\n\t\tintegrity,\n\t\tkeepalive,\n\t\tduplex,\n\t\tsignal: streamOptions.signal,\n\t};\n\tconst response = await fetch(options.url, fetchInit);\n\tif (!response.ok) {\n\t\t// 429 Too Many Requests\n\t\tif (response.status === 429) {\n\t\t\treturn fetchRateLimit(options, streamOptions);\n\t\t}\n\t\tthrow new Error(\n\t\t\t`fetch ${response.status} ${options.method} ${options.url}`,\n\t\t\t{\n\t\t\t\tcause: {\n\t\t\t\t\tstatus: response.status,\n\t\t\t\t\turl: options.url,\n\t\t\t\t\tmethod: options.method,\n\t\t\t\t},\n\t\t\t},\n\t\t);\n\t}\n\treturn response;\n};\n\nconst pickPath = (obj, path = \"\") => {\n\tif (path === \"\") return obj;\n\tif (!Array.isArray(path)) path = path.split(\".\");\n\treturn path.reduce((a, b) => a?.[b], obj);\n};\n\nexport default {\n\tsetDefaults: fetchSetDefaults,\n\treadableStream: fetchReadableStream,\n\tresponseStream: fetchReadableStream,\n};\n"],
		"mappings": "AAGA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;AAEP,MAAM,WAAW;AAAA;AAAA,EAEhB,WAAW;AAAA;AAAA,EACX,UAAU;AAAA;AAAA,EACV,UAAU;AAAA;AAAA,EACV,IAAI,CAAC;AAAA;AAAA,EACL,aAAa;AAAA;AAAA,EACb,cAAc;AAAA;AAAA;AAAA,EAGd,QAAQ;AAAA,EACR,SAAS;AAAA,IACR,QAAQ;AAAA,IACR,mBAAmB;AAAA,EACpB;AACD;AAEA,MAAM,eAAe,CAAC,UAAU,CAAC,MAAM;AACtC,SAAO;AAAA,IACN,GAAG;AAAA,IACH,GAAG;AAAA,IACH,SAAS,EAAE,GAAG,SAAS,SAAS,GAAG,QAAQ,QAAQ;AAAA,IACnD,IAAI,EAAE,GAAG,SAAS,IAAI,GAAG,QAAQ,GAAG;AAAA,EACrC;AACD;AAEO,MAAM,mBAAmB,CAAC,YAAY;AAC5C,SAAO,OAAO,UAAU,aAAa,OAAO,CAAC;AAC9C;AAIO,MAAM,sBAAsB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AACzE,QAAM,OAAO,qBAAqB;AAElC,YAAU,aAAa,OAAO;AAC9B,QAAM,QAAQ,MAAM,eAAe;AAAA,IAClC,GAAG;AAAA,IACH;AAAA,IACA,QAAQ;AAAA,IACR,QAAQ,cAAc;AAAA,EACvB,CAAC;AACD,QAAM,QAAQ,CAAC,UAAU;AACxB,SAAK,KAAK,KAAK;AAAA,EAChB;AACA,QAAM,SAAS,qBAAqB,OAAO,aAAa;AACxD,SAAO,SAAS,OAAO,EAAE,KAAK,UAAU,MAAM;AAC9C,SAAO;AACR;AACO,MAAM,qBAAqB;AAE3B,MAAM,sBAAsB,CAAC,cAAc,gBAAgB,CAAC,MAAM;AACxE,SAAO;AAAA,IACN,eAAe,cAAc,aAAa;AAAA,IAC1C;AAAA,EACD;AACD;AACO,MAAM,sBAAsB;AAEnC,gBAAgB,eAAe,cAAc,eAAe;AAC3D,MAAI,qBAAqB;AACzB,MAAI,CAAC,MAAM,QAAQ,YAAY,EAAG,gBAAe,CAAC,YAAY;AAC9D,WAAS,WAAW,cAAc;AACjC,cAAU,aAAa,OAAO;AAC9B,YAAQ,uBAAuB;AAE/B,QAAI,QAAQ,aAAa;AACxB,cAAQ,GAAG,QAAQ,WAAW,MAAM;AAAA,IACrC;AAEA,QAAI,OAAO,KAAK,QAAQ,EAAE,EAAE,QAAQ;AACnC,cAAQ,OAAO,IAAI,IAAI,gBAAgB,QAAQ,EAAE,CAAC,GAAG;AAAA,QACpD;AAAA,QACA;AAAA,MACD;AAAA,IACD;AACA,UAAM,WAAW,MAAM,aAAa,SAAS,aAAa;AAC1D,qBAAiB,SAAS,UAAU;AACnC,YAAM;AAAA,IACP;AAEA,yBAAqB,QAAQ;AAAA,EAC9B;AACD;AAEA,MAAM,wBAAwB;AAC9B,MAAM,eAAe,OAAO,SAAS,kBAAkB;AACtD,QAAM,WAAW,MAAM,eAAe,SAAS,aAAa;AAC5D,MAAI,sBAAsB,KAAK,SAAS,QAAQ,IAAI,cAAc,CAAC,GAAG;AACrE,YAAQ,mBAAmB;AAC3B,WAAO,UAAU,SAAS,aAAa;AAAA,EACxC;AACA,SAAO,SAAS;AACjB;AAEA,MAAM,iBAAiB;AAEvB,gBAAgB,UAAU,SAAS,eAAe;AACjD,QAAM,EAAE,UAAU,SAAS,IAAI;AAC/B,MAAI;AAEJ,SAAO,QAAQ,KAAK;AACnB,UAAM,WACL,QAAQ,oBACP,MAAM,eAAe,SAAS,aAAa;AAC7C,WAAO,QAAQ;AACf,UAAM,OAAO,MAAM,SAAS,KAAK;AACjC,UAAM,oBAAoB,SAAS,OAAO;AAC1C,YAAQ,cAAc,MAAM,QAAQ;AACpC,YAAQ,mBAAmB,OAAO;AAClC,YAAQ,MAAM;AACd,UAAM,OAAO,SAAS,MAAM,QAAQ;AACpC,QAAI,MAAM,QAAQ,IAAI,GAAG;AACxB,iBAAW,QAAQ,MAAM;AACxB,cAAM;AAAA,MACP;AAEA,UAAI,QAAQ,eAAe,CAAC,KAAK,OAAQ;AAAA,IAC1C,OAAO;AACN,YAAM;AAAA,IACP;AAAA,EACD;AACD;AAEA,MAAM,qBAAqB,CAAC,YAAY;AACvC,MAAI,CAAC,QAAQ,eAAe,CAAC,QAAQ,aAAc,QAAO;AAE1D,QAAM,MAAM,IAAI,IAAI,QAAQ,GAAG;AAC/B,MAAI,SAAS,IAAI,aAAa,IAAI,QAAQ,WAAW;AACrD,MAAI,CAAC,OAAQ,QAAO;AAEpB,WAAS,OAAO,SAAS,QAAQ,EAAE,IAAI,QAAQ;AAC/C,MAAI,aAAa,OAAO,QAAQ,WAAW;AAC3C,MAAI,aAAa,IAAI,QAAQ,aAAa,MAAM;AAChD,SAAO,IAAI,SAAS;AACrB;AAEA,MAAM,gBAAgB,CAAC,MAAM,aAAa;AACzC,SAAO,WAAW,SAAS,MAAM,QAAQ,IAAI;AAC9C;AAEA,MAAM,sBAAsB,CAAC,YAAY;AACxC,QAAM,OAAO,QAAQ,IAAI,MAAM;AAC/B,SAAO,MAAM,MAAM,cAAc,IAAI,CAAC;AACvC;AAEO,MAAM,iBAAiB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,MAAM,KAAK,IAAI;AACrB,MAAI,OAAO,QAAQ,sBAAsB,IAAI;AAC5C,UAAM,QAAQ,QAAQ,qBAAqB,KAAK,aAAa;AAAA,EAC9D;AACA,UAAQ,qBAAqB,KAAK,IAAI,IAAI,MAAO,QAAQ;AACzD,YAAU,aAAa,OAAO;AAE9B,QAAM;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,IAAI;AACJ,QAAM,YAAY;AAAA,IACjB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,QAAQ,cAAc;AAAA,EACvB;AACA,QAAM,WAAW,MAAM,MAAM,QAAQ,KAAK,SAAS;AACnD,MAAI,CAAC,SAAS,IAAI;AAEjB,QAAI,SAAS,WAAW,KAAK;AAC5B,aAAO,eAAe,SAAS,aAAa;AAAA,IAC7C;AACA,UAAM,IAAI;AAAA,MACT,SAAS,SAAS,MAAM,IAAI,QAAQ,MAAM,IAAI,QAAQ,GAAG;AAAA,MACzD;AAAA,QACC,OAAO;AAAA,UACN,QAAQ,SAAS;AAAA,UACjB,KAAK,QAAQ;AAAA,UACb,QAAQ,QAAQ;AAAA,QACjB;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,WAAW,CAAC,KAAK,OAAO,OAAO;AACpC,MAAI,SAAS,GAAI,QAAO;AACxB,MAAI,CAAC,MAAM,QAAQ,IAAI,EAAG,QAAO,KAAK,MAAM,GAAG;AAC/C,SAAO,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,GAAG,GAAG;AACzC;AAEA,IAAO,gBAAQ;AAAA,EACd,aAAa;AAAA,EACb,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;",
		"names": []
		}

+2

-2

package.json

		{
		"name": "@datastream/fetch",
		"version": "0.1.5",
		"version": "0.1.6",
		"description": "HTTP fetch-based readable and writable streams with pagination and rate limiting",
		@@ -63,4 +63,4 @@ "type": "module",
		"dependencies": {
		"@datastream/core": "0.1.5"
		"@datastream/core": "0.1.6"
		}
		}

@datastream/fetch - npm Package Compare versions

New alerts

Fixed alerts

Worsened metrics

Dependency changes