@datastream/aws
Advanced tools
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| import type { StreamOptions } from "@datastream/core"; | ||
| export function awsCloudWatchLogsSetClient(cwlClient: unknown): void; | ||
| export function awsCloudWatchLogsGetLogEventsStream( | ||
| options: { | ||
| client?: unknown; | ||
| logGroupName?: string; | ||
| logGroupIdentifier?: string; | ||
| logStreamName?: string; | ||
| startTime?: number; | ||
| endTime?: number; | ||
| startFromHead?: boolean; | ||
| pollingActive?: boolean; | ||
| pollingDelay?: number; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; | ||
| export function awsCloudWatchLogsFilterLogEventsStream( | ||
| options: { | ||
| client?: unknown; | ||
| logGroupName?: string; | ||
| logGroupIdentifier?: string; | ||
| filterPattern?: string; | ||
| startTime?: number; | ||
| endTime?: number; | ||
| logStreamNames?: string[]; | ||
| logStreamNamePrefix?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; |
| import { | ||
| CloudWatchLogsClient, | ||
| FilterLogEventsCommand, | ||
| GetLogEventsCommand | ||
| } from "@aws-sdk/client-cloudwatch-logs"; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new CloudWatchLogsClient(awsClientDefaults); | ||
| const awsCloudWatchLogsSetClient = (cwlClient) => { | ||
| client = cwlClient; | ||
| }; | ||
| const awsCloudWatchLogsGetLogEventsStream = async (options, _streamOptions = {}) => { | ||
| const { pollingActive, pollingDelay = 1e3, ...cwlOptions } = options; | ||
| cwlOptions.startFromHead ??= true; | ||
| async function* command(options2) { | ||
| let previousToken; | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new GetLogEventsCommand(options2)); | ||
| const events = response.events ?? []; | ||
| for (const item of events) { | ||
| yield item; | ||
| } | ||
| const tokenUnchanged = response.nextForwardToken === previousToken || response.nextForwardToken === options2.nextToken; | ||
| previousToken = response.nextForwardToken; | ||
| options2.nextToken = response.nextForwardToken; | ||
| if (tokenUnchanged) { | ||
| if (pollingActive) { | ||
| if (pollingDelay > 0) { | ||
| await new Promise((resolve) => setTimeout(resolve, pollingDelay)); | ||
| } | ||
| } else { | ||
| expectMore = false; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| return command(cwlOptions); | ||
| }; | ||
| const awsCloudWatchLogsFilterLogEventsStream = async (options, _streamOptions = {}) => { | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new FilterLogEventsCommand(options2)); | ||
| for (const item of response.events ?? []) { | ||
| yield item; | ||
| } | ||
| options2.nextToken = response.nextToken; | ||
| expectMore = !!response.nextToken; | ||
| } | ||
| } | ||
| return command(options); | ||
| }; | ||
| var cloudwatch_logs_default = { | ||
| setClient: awsCloudWatchLogsSetClient, | ||
| getLogEventsStream: awsCloudWatchLogsGetLogEventsStream, | ||
| filterLogEventsStream: awsCloudWatchLogsFilterLogEventsStream | ||
| }; | ||
| export { | ||
| awsCloudWatchLogsFilterLogEventsStream, | ||
| awsCloudWatchLogsGetLogEventsStream, | ||
| awsCloudWatchLogsSetClient, | ||
| cloudwatch_logs_default as default | ||
| }; |
| { | ||
| "version": 3, | ||
| "sources": ["cloudwatch-logs.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tCloudWatchLogsClient,\n\tFilterLogEventsCommand,\n\tGetLogEventsCommand,\n} from \"@aws-sdk/client-cloudwatch-logs\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new CloudWatchLogsClient(awsClientDefaults);\nexport const awsCloudWatchLogsSetClient = (cwlClient) => {\n\tclient = cwlClient;\n};\n\nexport const awsCloudWatchLogsGetLogEventsStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tconst { pollingActive, pollingDelay = 1000, ...cwlOptions } = options;\n\tcwlOptions.startFromHead ??= true;\n\tasync function* command(options) {\n\t\tlet previousToken;\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new GetLogEventsCommand(options));\n\t\t\tconst events = response.events ?? [];\n\t\t\tfor (const item of events) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst tokenUnchanged =\n\t\t\t\tresponse.nextForwardToken === previousToken ||\n\t\t\t\tresponse.nextForwardToken === options.nextToken;\n\t\t\tpreviousToken = response.nextForwardToken;\n\t\t\toptions.nextToken = response.nextForwardToken;\n\n\t\t\tif (tokenUnchanged) {\n\t\t\t\tif (pollingActive) {\n\t\t\t\t\tif (pollingDelay > 0) {\n\t\t\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, pollingDelay));\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\texpectMore = false;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn command(cwlOptions);\n};\n\nexport const awsCloudWatchLogsFilterLogEventsStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new FilterLogEventsCommand(options));\n\t\t\tfor (const item of response.events ?? []) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.nextToken = response.nextToken;\n\t\t\texpectMore = !!response.nextToken;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport default {\n\tsetClient: awsCloudWatchLogsSetClient,\n\tgetLogEventsStream: awsCloudWatchLogsGetLogEventsStream,\n\tfilterLogEventsStream: awsCloudWatchLogsFilterLogEventsStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,qBAAqB,iBAAiB;AAChD,MAAM,6BAA6B,CAAC,cAAc;AACxD,WAAS;AACV;AAEO,MAAM,sCAAsC,OAClD,SACA,iBAAiB,CAAC,MACd;AACJ,QAAM,EAAE,eAAe,eAAe,KAAM,GAAG,WAAW,IAAI;AAC9D,aAAW,kBAAkB;AAC7B,kBAAgB,QAAQA,UAAS;AAChC,QAAI;AACJ,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,oBAAoBA,QAAO,CAAC;AACnE,YAAM,SAAS,SAAS,UAAU,CAAC;AACnC,iBAAW,QAAQ,QAAQ;AAC1B,cAAM;AAAA,MACP;AACA,YAAM,iBACL,SAAS,qBAAqB,iBAC9B,SAAS,qBAAqBA,SAAQ;AACvC,sBAAgB,SAAS;AACzB,MAAAA,SAAQ,YAAY,SAAS;AAE7B,UAAI,gBAAgB;AACnB,YAAI,eAAe;AAClB,cAAI,eAAe,GAAG;AACrB,kBAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,YAAY,CAAC;AAAA,UACjE;AAAA,QACD,OAAO;AACN,uBAAa;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACA,SAAO,QAAQ,UAAU;AAC1B;AAEO,MAAM,yCAAyC,OACrD,SACA,iBAAiB,CAAC,MACd;AACJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,uBAAuBA,QAAO,CAAC;AACtE,iBAAW,QAAQ,SAAS,UAAU,CAAC,GAAG;AACzC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,YAAY,SAAS;AAC7B,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEA,IAAO,0BAAQ;AAAA,EACd,WAAW;AAAA,EACX,oBAAoB;AAAA,EACpB,uBAAuB;AACxB;", | ||
| "names": ["options"] | ||
| } |
| import { | ||
| CloudWatchLogsClient, | ||
| FilterLogEventsCommand, | ||
| GetLogEventsCommand | ||
| } from "@aws-sdk/client-cloudwatch-logs"; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new CloudWatchLogsClient(awsClientDefaults); | ||
| const awsCloudWatchLogsSetClient = (cwlClient) => { | ||
| client = cwlClient; | ||
| }; | ||
| const awsCloudWatchLogsGetLogEventsStream = async (options, _streamOptions = {}) => { | ||
| const { pollingActive, pollingDelay = 1e3, ...cwlOptions } = options; | ||
| cwlOptions.startFromHead ??= true; | ||
| async function* command(options2) { | ||
| let previousToken; | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new GetLogEventsCommand(options2)); | ||
| const events = response.events ?? []; | ||
| for (const item of events) { | ||
| yield item; | ||
| } | ||
| const tokenUnchanged = response.nextForwardToken === previousToken || response.nextForwardToken === options2.nextToken; | ||
| previousToken = response.nextForwardToken; | ||
| options2.nextToken = response.nextForwardToken; | ||
| if (tokenUnchanged) { | ||
| if (pollingActive) { | ||
| if (pollingDelay > 0) { | ||
| await new Promise((resolve) => setTimeout(resolve, pollingDelay)); | ||
| } | ||
| } else { | ||
| expectMore = false; | ||
| } | ||
| } | ||
| } | ||
| } | ||
| return command(cwlOptions); | ||
| }; | ||
| const awsCloudWatchLogsFilterLogEventsStream = async (options, _streamOptions = {}) => { | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new FilterLogEventsCommand(options2)); | ||
| for (const item of response.events ?? []) { | ||
| yield item; | ||
| } | ||
| options2.nextToken = response.nextToken; | ||
| expectMore = !!response.nextToken; | ||
| } | ||
| } | ||
| return command(options); | ||
| }; | ||
| var cloudwatch_logs_default = { | ||
| setClient: awsCloudWatchLogsSetClient, | ||
| getLogEventsStream: awsCloudWatchLogsGetLogEventsStream, | ||
| filterLogEventsStream: awsCloudWatchLogsFilterLogEventsStream | ||
| }; | ||
| export { | ||
| awsCloudWatchLogsFilterLogEventsStream, | ||
| awsCloudWatchLogsGetLogEventsStream, | ||
| awsCloudWatchLogsSetClient, | ||
| cloudwatch_logs_default as default | ||
| }; |
| { | ||
| "version": 3, | ||
| "sources": ["cloudwatch-logs.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tCloudWatchLogsClient,\n\tFilterLogEventsCommand,\n\tGetLogEventsCommand,\n} from \"@aws-sdk/client-cloudwatch-logs\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new CloudWatchLogsClient(awsClientDefaults);\nexport const awsCloudWatchLogsSetClient = (cwlClient) => {\n\tclient = cwlClient;\n};\n\nexport const awsCloudWatchLogsGetLogEventsStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tconst { pollingActive, pollingDelay = 1000, ...cwlOptions } = options;\n\tcwlOptions.startFromHead ??= true;\n\tasync function* command(options) {\n\t\tlet previousToken;\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new GetLogEventsCommand(options));\n\t\t\tconst events = response.events ?? [];\n\t\t\tfor (const item of events) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst tokenUnchanged =\n\t\t\t\tresponse.nextForwardToken === previousToken ||\n\t\t\t\tresponse.nextForwardToken === options.nextToken;\n\t\t\tpreviousToken = response.nextForwardToken;\n\t\t\toptions.nextToken = response.nextForwardToken;\n\n\t\t\tif (tokenUnchanged) {\n\t\t\t\tif (pollingActive) {\n\t\t\t\t\tif (pollingDelay > 0) {\n\t\t\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, pollingDelay));\n\t\t\t\t\t}\n\t\t\t\t} else {\n\t\t\t\t\texpectMore = false;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t}\n\treturn command(cwlOptions);\n};\n\nexport const awsCloudWatchLogsFilterLogEventsStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new FilterLogEventsCommand(options));\n\t\t\tfor (const item of response.events ?? []) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.nextToken = response.nextToken;\n\t\t\texpectMore = !!response.nextToken;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport default {\n\tsetClient: awsCloudWatchLogsSetClient,\n\tgetLogEventsStream: awsCloudWatchLogsGetLogEventsStream,\n\tfilterLogEventsStream: awsCloudWatchLogsFilterLogEventsStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,qBAAqB,iBAAiB;AAChD,MAAM,6BAA6B,CAAC,cAAc;AACxD,WAAS;AACV;AAEO,MAAM,sCAAsC,OAClD,SACA,iBAAiB,CAAC,MACd;AACJ,QAAM,EAAE,eAAe,eAAe,KAAM,GAAG,WAAW,IAAI;AAC9D,aAAW,kBAAkB;AAC7B,kBAAgB,QAAQA,UAAS;AAChC,QAAI;AACJ,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,oBAAoBA,QAAO,CAAC;AACnE,YAAM,SAAS,SAAS,UAAU,CAAC;AACnC,iBAAW,QAAQ,QAAQ;AAC1B,cAAM;AAAA,MACP;AACA,YAAM,iBACL,SAAS,qBAAqB,iBAC9B,SAAS,qBAAqBA,SAAQ;AACvC,sBAAgB,SAAS;AACzB,MAAAA,SAAQ,YAAY,SAAS;AAE7B,UAAI,gBAAgB;AACnB,YAAI,eAAe;AAClB,cAAI,eAAe,GAAG;AACrB,kBAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,YAAY,CAAC;AAAA,UACjE;AAAA,QACD,OAAO;AACN,uBAAa;AAAA,QACd;AAAA,MACD;AAAA,IACD;AAAA,EACD;AACA,SAAO,QAAQ,UAAU;AAC1B;AAEO,MAAM,yCAAyC,OACrD,SACA,iBAAiB,CAAC,MACd;AACJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,uBAAuBA,QAAO,CAAC;AACtE,iBAAW,QAAQ,SAAS,UAAU,CAAC,GAAG;AACzC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,YAAY,SAAS;AAC7B,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEA,IAAO,0BAAQ;AAAA,EACd,WAAW;AAAA,EACX,oBAAoB;AAAA,EACpB,uBAAuB;AACxB;", | ||
| "names": ["options"] | ||
| } |
+26
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| import type { StreamOptions } from "@datastream/core"; | ||
| export function awsKinesisSetClient(kinesisClient: unknown): void; | ||
| export function awsKinesisGetRecordsStream( | ||
| options: { | ||
| client?: unknown; | ||
| ShardIterator?: string; | ||
| pollingActive?: boolean; | ||
| pollingDelay?: number; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; | ||
| export function awsKinesisPutRecordsStream( | ||
| options: { | ||
| client?: unknown; | ||
| StreamName?: string; | ||
| StreamARN?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): unknown; |
| import { | ||
| GetRecordsCommand, | ||
| KinesisClient, | ||
| PutRecordsCommand | ||
| } from "@aws-sdk/client-kinesis"; | ||
| import { createWritableStream } from "@datastream/core"; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new KinesisClient(awsClientDefaults); | ||
| const awsKinesisSetClient = (kinesisClient) => { | ||
| client = kinesisClient; | ||
| }; | ||
| const awsKinesisGetRecordsStream = async (options, _streamOptions = {}) => { | ||
| const { pollingActive, pollingDelay = 1e3, ...kinesisOptions } = options; | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new GetRecordsCommand(options2)); | ||
| const records = response.Records ?? []; | ||
| for (const item of records) { | ||
| yield item; | ||
| } | ||
| options2.ShardIterator = response.NextShardIterator; | ||
| expectMore = pollingActive || records.length > 0; | ||
| if (pollingActive && records.length === 0 && pollingDelay > 0) { | ||
| await new Promise((resolve) => setTimeout(resolve, pollingDelay)); | ||
| } | ||
| } | ||
| } | ||
| return command(kinesisOptions); | ||
| }; | ||
| const awsKinesisPutRecordsStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
| const send = () => { | ||
| options.Records = batch; | ||
| batch = []; | ||
| return client.send(new PutRecordsCommand(options)); | ||
| }; | ||
| const write = async (chunk) => { | ||
| if (batch.length === 500) { | ||
| await send(); | ||
| } | ||
| batch.push(chunk); | ||
| }; | ||
| const final = () => batch.length ? send() : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
| }; | ||
| var kinesis_default = { | ||
| setClient: awsKinesisSetClient, | ||
| getRecordsStream: awsKinesisGetRecordsStream, | ||
| putRecordsStream: awsKinesisPutRecordsStream | ||
| }; | ||
| export { | ||
| awsKinesisGetRecordsStream, | ||
| awsKinesisPutRecordsStream, | ||
| awsKinesisSetClient, | ||
| kinesis_default as default | ||
| }; |
| { | ||
| "version": 3, | ||
| "sources": ["kinesis.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tGetRecordsCommand,\n\tKinesisClient,\n\tPutRecordsCommand,\n} from \"@aws-sdk/client-kinesis\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new KinesisClient(awsClientDefaults);\nexport const awsKinesisSetClient = (kinesisClient) => {\n\tclient = kinesisClient;\n};\n\nexport const awsKinesisGetRecordsStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tconst { pollingActive, pollingDelay = 1000, ...kinesisOptions } = options;\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new GetRecordsCommand(options));\n\t\t\tconst records = response.Records ?? [];\n\t\t\tfor (const item of records) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ShardIterator = response.NextShardIterator;\n\t\t\texpectMore = pollingActive || records.length > 0;\n\t\t\tif (pollingActive && records.length === 0 && pollingDelay > 0) {\n\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, pollingDelay));\n\t\t\t}\n\t\t}\n\t}\n\treturn command(kinesisOptions);\n};\n\nexport const awsKinesisPutRecordsStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Records = batch;\n\t\tbatch = [];\n\t\treturn client.send(new PutRecordsCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 500) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsKinesisSetClient,\n\tgetRecordsStream: awsKinesisGetRecordsStream,\n\tputRecordsStream: awsKinesisPutRecordsStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,cAAc,iBAAiB;AACzC,MAAM,sBAAsB,CAAC,kBAAkB;AACrD,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,iBAAiB,CAAC,MACd;AACJ,QAAM,EAAE,eAAe,eAAe,KAAM,GAAG,eAAe,IAAI;AAClE,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,kBAAkBA,QAAO,CAAC;AACjE,YAAM,UAAU,SAAS,WAAW,CAAC;AACrC,iBAAW,QAAQ,SAAS;AAC3B,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,gBAAgB,SAAS;AACjC,mBAAa,iBAAiB,QAAQ,SAAS;AAC/C,UAAI,iBAAiB,QAAQ,WAAW,KAAK,eAAe,GAAG;AAC9D,cAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,YAAY,CAAC;AAAA,MACjE;AAAA,IACD;AAAA,EACD;AACA,SAAO,QAAQ,cAAc;AAC9B;AAEO,MAAM,6BAA6B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC1E,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,kBAAkB,OAAO,CAAC;AAAA,EAClD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,KAAK;AACzB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,kBAAQ;AAAA,EACd,WAAW;AAAA,EACX,kBAAkB;AAAA,EAClB,kBAAkB;AACnB;", | ||
| "names": ["options"] | ||
| } |
| import { | ||
| GetRecordsCommand, | ||
| KinesisClient, | ||
| PutRecordsCommand | ||
| } from "@aws-sdk/client-kinesis"; | ||
| import { createWritableStream } from "@datastream/core"; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new KinesisClient(awsClientDefaults); | ||
| const awsKinesisSetClient = (kinesisClient) => { | ||
| client = kinesisClient; | ||
| }; | ||
| const awsKinesisGetRecordsStream = async (options, _streamOptions = {}) => { | ||
| const { pollingActive, pollingDelay = 1e3, ...kinesisOptions } = options; | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new GetRecordsCommand(options2)); | ||
| const records = response.Records ?? []; | ||
| for (const item of records) { | ||
| yield item; | ||
| } | ||
| options2.ShardIterator = response.NextShardIterator; | ||
| expectMore = pollingActive || records.length > 0; | ||
| if (pollingActive && records.length === 0 && pollingDelay > 0) { | ||
| await new Promise((resolve) => setTimeout(resolve, pollingDelay)); | ||
| } | ||
| } | ||
| } | ||
| return command(kinesisOptions); | ||
| }; | ||
| const awsKinesisPutRecordsStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
| const send = () => { | ||
| options.Records = batch; | ||
| batch = []; | ||
| return client.send(new PutRecordsCommand(options)); | ||
| }; | ||
| const write = async (chunk) => { | ||
| if (batch.length === 500) { | ||
| await send(); | ||
| } | ||
| batch.push(chunk); | ||
| }; | ||
| const final = () => batch.length ? send() : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
| }; | ||
| var kinesis_default = { | ||
| setClient: awsKinesisSetClient, | ||
| getRecordsStream: awsKinesisGetRecordsStream, | ||
| putRecordsStream: awsKinesisPutRecordsStream | ||
| }; | ||
| export { | ||
| awsKinesisGetRecordsStream, | ||
| awsKinesisPutRecordsStream, | ||
| awsKinesisSetClient, | ||
| kinesis_default as default | ||
| }; |
| { | ||
| "version": 3, | ||
| "sources": ["kinesis.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tGetRecordsCommand,\n\tKinesisClient,\n\tPutRecordsCommand,\n} from \"@aws-sdk/client-kinesis\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new KinesisClient(awsClientDefaults);\nexport const awsKinesisSetClient = (kinesisClient) => {\n\tclient = kinesisClient;\n};\n\nexport const awsKinesisGetRecordsStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tconst { pollingActive, pollingDelay = 1000, ...kinesisOptions } = options;\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new GetRecordsCommand(options));\n\t\t\tconst records = response.Records ?? [];\n\t\t\tfor (const item of records) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ShardIterator = response.NextShardIterator;\n\t\t\texpectMore = pollingActive || records.length > 0;\n\t\t\tif (pollingActive && records.length === 0 && pollingDelay > 0) {\n\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, pollingDelay));\n\t\t\t}\n\t\t}\n\t}\n\treturn command(kinesisOptions);\n};\n\nexport const awsKinesisPutRecordsStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Records = batch;\n\t\tbatch = [];\n\t\treturn client.send(new PutRecordsCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 500) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsKinesisSetClient,\n\tgetRecordsStream: awsKinesisGetRecordsStream,\n\tputRecordsStream: awsKinesisPutRecordsStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,cAAc,iBAAiB;AACzC,MAAM,sBAAsB,CAAC,kBAAkB;AACrD,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,iBAAiB,CAAC,MACd;AACJ,QAAM,EAAE,eAAe,eAAe,KAAM,GAAG,eAAe,IAAI;AAClE,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,kBAAkBA,QAAO,CAAC;AACjE,YAAM,UAAU,SAAS,WAAW,CAAC;AACrC,iBAAW,QAAQ,SAAS;AAC3B,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,gBAAgB,SAAS;AACjC,mBAAa,iBAAiB,QAAQ,SAAS;AAC/C,UAAI,iBAAiB,QAAQ,WAAW,KAAK,eAAe,GAAG;AAC9D,cAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,YAAY,CAAC;AAAA,MACjE;AAAA,IACD;AAAA,EACD;AACA,SAAO,QAAQ,cAAc;AAC9B;AAEO,MAAM,6BAA6B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC1E,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,kBAAkB,OAAO,CAAC;AAAA,EAClD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,KAAK;AACzB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,kBAAQ;AAAA,EACd,WAAW;AAAA,EACX,kBAAkB;AAAA,EAClB,kBAAkB;AACnB;", | ||
| "names": ["options"] | ||
| } |
+10
-0
@@ -28,2 +28,12 @@ // Copyright 2026 will Farrell, and datastream contributors. | ||
| export function awsDynamoDBExecuteStatementStream( | ||
| options: { | ||
| client?: unknown; | ||
| Statement?: string; | ||
| NextToken?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; | ||
| export function awsDynamoDBGetItemStream( | ||
@@ -30,0 +40,0 @@ options: { |
+46
-20
@@ -5,2 +5,3 @@ import { | ||
| DynamoDBClient, | ||
| ExecuteStatementCommand, | ||
| QueryCommand, | ||
@@ -16,7 +17,9 @@ ScanCommand | ||
| awsDynamoDBSetClient(client); | ||
| const awsDynamoDBQueryStream = async (options, _streamOptions = {}) => { | ||
| const awsDynamoDBQueryStream = async (options, streamOptions = {}) => { | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new QueryCommand(options2)); | ||
| const response = await client.send(new QueryCommand(options2), { | ||
| abortSignal: streamOptions.signal | ||
| }); | ||
| for (const item of response.Items) { | ||
@@ -31,7 +34,9 @@ yield item; | ||
| }; | ||
| const awsDynamoDBScanStream = async (options, _streamOptions = {}) => { | ||
| const awsDynamoDBScanStream = async (options, streamOptions = {}) => { | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new ScanCommand(options2)); | ||
| const response = await client.send(new ScanCommand(options2), { | ||
| abortSignal: streamOptions.signal | ||
| }); | ||
| for (const item of response.Items) { | ||
@@ -46,3 +51,19 @@ yield item; | ||
| }; | ||
| const awsDynamoDBGetItemStream = async (options, _streamOptions = {}) => { | ||
| const awsDynamoDBExecuteStatementStream = async (options, streamOptions = {}) => { | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new ExecuteStatementCommand(options2), { | ||
| abortSignal: streamOptions.signal | ||
| }); | ||
| for (const item of response.Items ?? []) { | ||
| yield item; | ||
| } | ||
| options2.NextToken = response.NextToken; | ||
| expectMore = !!response.NextToken; | ||
| } | ||
| } | ||
| return command(options); | ||
| }; | ||
| const awsDynamoDBGetItemStream = async (options, streamOptions = {}) => { | ||
| if (options.Keys?.length > 100) { | ||
@@ -53,5 +74,6 @@ throw new Error( | ||
| } | ||
| options.retryCount ??= 0; | ||
| options.retryMaxCount ??= 10; | ||
| async function* command(options2) { | ||
| let keys = options2.Keys; | ||
| let retryCount = options2.retryCount ?? 0; | ||
| const retryMaxCount = options2.retryMaxCount ?? 10; | ||
| while (true) { | ||
@@ -61,5 +83,6 @@ const response = await client.send( | ||
| RequestItems: { | ||
| [options2.TableName]: { Keys: options2.Keys } | ||
| [options2.TableName]: { Keys: keys } | ||
| } | ||
| }) | ||
| }), | ||
| { abortSignal: streamOptions.signal } | ||
| ); | ||
@@ -73,3 +96,3 @@ for (const item of response.Responses[options2.TableName]) { | ||
| } | ||
| if (options2.retryCount >= options2.retryMaxCount) { | ||
| if (retryCount >= retryMaxCount) { | ||
| throw new Error("awsDynamoDBBatchGetItem has UnprocessedKeys", { | ||
@@ -82,4 +105,5 @@ cause: { | ||
| } | ||
| await timeout(3 ** options2.retryCount++); | ||
| options2.Keys = UnprocessedKeys; | ||
| await timeout(3 ** retryCount); | ||
| retryCount++; | ||
| keys = UnprocessedKeys; | ||
| } | ||
@@ -121,5 +145,4 @@ } | ||
| }; | ||
| const dynamodbBatchWrite = async (options, batch, streamOptions) => { | ||
| options.retryCount ??= 0; | ||
| options.retryMaxCount ??= 10; | ||
| const dynamodbBatchWrite = async (options, batch, streamOptions, retryCount = 0) => { | ||
| const retryMaxCount = options.retryMaxCount ?? 10; | ||
| const { UnprocessedItems } = await client.send( | ||
@@ -130,6 +153,7 @@ new BatchWriteItemCommand({ | ||
| } | ||
| }) | ||
| }), | ||
| { abortSignal: streamOptions?.signal } | ||
| ); | ||
| if (UnprocessedItems?.[options.TableName]?.length) { | ||
| if (options.retryCount >= options.retryMaxCount) { | ||
| if (retryCount >= retryMaxCount) { | ||
| throw new Error("awsDynamoDBBatchWriteItem has UnprocessedItems", { | ||
@@ -142,10 +166,10 @@ cause: { | ||
| } | ||
| await timeout(3 ** options.retryCount++); | ||
| await timeout(3 ** retryCount); | ||
| return dynamodbBatchWrite( | ||
| options, | ||
| UnprocessedItems[options.TableName], | ||
| streamOptions | ||
| streamOptions, | ||
| retryCount + 1 | ||
| ); | ||
| } | ||
| options.retryCount = 0; | ||
| }; | ||
@@ -156,2 +180,3 @@ var dynamodb_default = { | ||
| scanStream: awsDynamoDBScanStream, | ||
| executeStatementStream: awsDynamoDBExecuteStatementStream, | ||
| getItemStream: awsDynamoDBGetItemStream, | ||
@@ -163,2 +188,3 @@ putItemStream: awsDynamoDBPutItemStream, | ||
| awsDynamoDBDeleteItemStream, | ||
| awsDynamoDBExecuteStatementStream, | ||
| awsDynamoDBGetItemStream, | ||
@@ -165,0 +191,0 @@ awsDynamoDBPutItemStream, |
| { | ||
| "version": 3, | ||
| "sources": ["dynamodb.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tBatchGetItemCommand,\n\tBatchWriteItemCommand,\n\tDynamoDBClient,\n\tQueryCommand,\n\tScanCommand,\n} from \"@aws-sdk/client-dynamodb\";\nimport { createWritableStream, timeout } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new DynamoDBClient(awsClientDefaults);\nexport const awsDynamoDBSetClient = (ddbClient, _translateConfig) => {\n\tclient = ddbClient;\n};\nawsDynamoDBSetClient(client);\n\n// options = {TableName, ...}\n\nexport const awsDynamoDBQueryStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new QueryCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBScanStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ScanCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\n// TODO awsDynamoDBExecuteStatementStream\n\nexport const awsDynamoDBGetItemStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tif (options.Keys?.length > 100) {\n\t\tthrow new Error(\n\t\t\t`awsDynamoDBGetItemStream Keys.length (${options.Keys.length}) exceeds BatchGetItem limit of 100`,\n\t\t);\n\t}\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tasync function* command(options) {\n\t\twhile (true) {\n\t\t\tconst response = await client.send(\n\t\t\t\tnew BatchGetItemCommand({\n\t\t\t\t\tRequestItems: {\n\t\t\t\t\t\t[options.TableName]: { Keys: options.Keys },\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t);\n\t\t\tfor (const item of response.Responses[options.TableName]) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst UnprocessedKeys =\n\t\t\t\tresponse?.UnprocessedKeys?.[options.TableName]?.Keys ?? [];\n\t\t\tif (!UnprocessedKeys.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\t\tthrow new Error(\"awsDynamoDBBatchGetItem has UnprocessedKeys\", {\n\t\t\t\t\tcause: {\n\t\t\t\t\t\t...options,\n\t\t\t\t\t\tUnprocessedKeysCount: UnprocessedKeys.length,\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\n\t\t\toptions.Keys = UnprocessedKeys;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBPutItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tPutRequest: {\n\t\t\t\tItem: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsDynamoDBDeleteItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tDeleteRequest: {\n\t\t\t\tKey: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nconst dynamodbBatchWrite = async (options, batch, streamOptions) => {\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tconst { UnprocessedItems } = await client.send(\n\t\tnew BatchWriteItemCommand({\n\t\t\tRequestItems: {\n\t\t\t\t[options.TableName]: batch,\n\t\t\t},\n\t\t}),\n\t);\n\tif (UnprocessedItems?.[options.TableName]?.length) {\n\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\tthrow new Error(\"awsDynamoDBBatchWriteItem has UnprocessedItems\", {\n\t\t\t\tcause: {\n\t\t\t\t\t...options,\n\t\t\t\t\tUnprocessedItemsCount: UnprocessedItems[options.TableName].length,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\t\treturn dynamodbBatchWrite(\n\t\t\toptions,\n\t\t\tUnprocessedItems[options.TableName],\n\t\t\tstreamOptions,\n\t\t);\n\t}\n\toptions.retryCount = 0; // reset for next batch\n};\n\nexport default {\n\tsetClient: awsDynamoDBSetClient,\n\tqueryStream: awsDynamoDBQueryStream,\n\tscanStream: awsDynamoDBScanStream,\n\tgetItemStream: awsDynamoDBGetItemStream,\n\tputItemStream: awsDynamoDBPutItemStream,\n\tdeleteItemStream: awsDynamoDBDeleteItemStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,sBAAsB,eAAe;AAC9C,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,eAAe,iBAAiB;AAC1C,MAAM,uBAAuB,CAAC,WAAW,qBAAqB;AACpE,WAAS;AACV;AACA,qBAAqB,MAAM;AAIpB,MAAM,yBAAyB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC7E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,aAAaA,QAAO,CAAC;AAC5D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,wBAAwB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC5E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,YAAYA,QAAO,CAAC;AAC3D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAIO,MAAM,2BAA2B,OACvC,SACA,iBAAiB,CAAC,MACd;AACJ,MAAI,QAAQ,MAAM,SAAS,KAAK;AAC/B,UAAM,IAAI;AAAA,MACT,yCAAyC,QAAQ,KAAK,MAAM;AAAA,IAC7D;AAAA,EACD;AACA,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,kBAAgB,QAAQA,UAAS;AAChC,WAAO,MAAM;AACZ,YAAM,WAAW,MAAM,OAAO;AAAA,QAC7B,IAAI,oBAAoB;AAAA,UACvB,cAAc;AAAA,YACb,CAACA,SAAQ,SAAS,GAAG,EAAE,MAAMA,SAAQ,KAAK;AAAA,UAC3C;AAAA,QACD,CAAC;AAAA,MACF;AACA,iBAAW,QAAQ,SAAS,UAAUA,SAAQ,SAAS,GAAG;AACzD,cAAM;AAAA,MACP;AACA,YAAM,kBACL,UAAU,kBAAkBA,SAAQ,SAAS,GAAG,QAAQ,CAAC;AAC1D,UAAI,CAAC,gBAAgB,QAAQ;AAC5B;AAAA,MACD;AAEA,UAAIA,SAAQ,cAAcA,SAAQ,eAAe;AAChD,cAAM,IAAI,MAAM,+CAA+C;AAAA,UAC9D,OAAO;AAAA,YACN,GAAGA;AAAA,YACH,sBAAsB,gBAAgB;AAAA,UACvC;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,QAAQ,KAAKA,SAAQ,YAAY;AAEvC,MAAAA,SAAQ,OAAO;AAAA,IAChB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACxE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,YAAY;AAAA,QACX,MAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,8BAA8B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC3E,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,eAAe;AAAA,QACd,KAAK;AAAA,MACN;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,MAAM,qBAAqB,OAAO,SAAS,OAAO,kBAAkB;AACnE,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,QAAM,EAAE,iBAAiB,IAAI,MAAM,OAAO;AAAA,IACzC,IAAI,sBAAsB;AAAA,MACzB,cAAc;AAAA,QACb,CAAC,QAAQ,SAAS,GAAG;AAAA,MACtB;AAAA,IACD,CAAC;AAAA,EACF;AACA,MAAI,mBAAmB,QAAQ,SAAS,GAAG,QAAQ;AAClD,QAAI,QAAQ,cAAc,QAAQ,eAAe;AAChD,YAAM,IAAI,MAAM,kDAAkD;AAAA,QACjE,OAAO;AAAA,UACN,GAAG;AAAA,UACH,uBAAuB,iBAAiB,QAAQ,SAAS,EAAE;AAAA,QAC5D;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,QAAQ,YAAY;AACvC,WAAO;AAAA,MACN;AAAA,MACA,iBAAiB,QAAQ,SAAS;AAAA,MAClC;AAAA,IACD;AAAA,EACD;AACA,UAAQ,aAAa;AACtB;AAEA,IAAO,mBAAQ;AAAA,EACd,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,eAAe;AAAA,EACf,kBAAkB;AACnB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tBatchGetItemCommand,\n\tBatchWriteItemCommand,\n\tDynamoDBClient,\n\tExecuteStatementCommand,\n\tQueryCommand,\n\tScanCommand,\n} from \"@aws-sdk/client-dynamodb\";\nimport { createWritableStream, timeout } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new DynamoDBClient(awsClientDefaults);\nexport const awsDynamoDBSetClient = (ddbClient, _translateConfig) => {\n\tclient = ddbClient;\n};\nawsDynamoDBSetClient(client);\n\n// options = {TableName, ...}\n\nexport const awsDynamoDBQueryStream = async (options, streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new QueryCommand(options), {\n\t\t\t\tabortSignal: streamOptions.signal,\n\t\t\t});\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBScanStream = async (options, streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ScanCommand(options), {\n\t\t\t\tabortSignal: streamOptions.signal,\n\t\t\t});\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBExecuteStatementStream = async (\n\toptions,\n\tstreamOptions = {},\n) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ExecuteStatementCommand(options), {\n\t\t\t\tabortSignal: streamOptions.signal,\n\t\t\t});\n\t\t\tfor (const item of response.Items ?? []) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.NextToken = response.NextToken;\n\t\t\texpectMore = !!response.NextToken;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBGetItemStream = async (options, streamOptions = {}) => {\n\tif (options.Keys?.length > 100) {\n\t\tthrow new Error(\n\t\t\t`awsDynamoDBGetItemStream Keys.length (${options.Keys.length}) exceeds BatchGetItem limit of 100`,\n\t\t);\n\t}\n\tasync function* command(options) {\n\t\tlet keys = options.Keys;\n\t\tlet retryCount = options.retryCount ?? 0;\n\t\tconst retryMaxCount = options.retryMaxCount ?? 10;\n\t\twhile (true) {\n\t\t\tconst response = await client.send(\n\t\t\t\tnew BatchGetItemCommand({\n\t\t\t\t\tRequestItems: {\n\t\t\t\t\t\t[options.TableName]: { Keys: keys },\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\t{ abortSignal: streamOptions.signal },\n\t\t\t);\n\t\t\tfor (const item of response.Responses[options.TableName]) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst UnprocessedKeys =\n\t\t\t\tresponse?.UnprocessedKeys?.[options.TableName]?.Keys ?? [];\n\t\t\tif (!UnprocessedKeys.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (retryCount >= retryMaxCount) {\n\t\t\t\tthrow new Error(\"awsDynamoDBBatchGetItem has UnprocessedKeys\", {\n\t\t\t\t\tcause: {\n\t\t\t\t\t\t...options,\n\t\t\t\t\t\tUnprocessedKeysCount: UnprocessedKeys.length,\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tawait timeout(3 ** retryCount); // 3^10 == 59sec\n\t\t\tretryCount++;\n\t\t\tkeys = UnprocessedKeys;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBPutItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tPutRequest: {\n\t\t\t\tItem: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsDynamoDBDeleteItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tDeleteRequest: {\n\t\t\t\tKey: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nconst dynamodbBatchWrite = async (\n\toptions,\n\tbatch,\n\tstreamOptions,\n\tretryCount = 0,\n) => {\n\tconst retryMaxCount = options.retryMaxCount ?? 10;\n\tconst { UnprocessedItems } = await client.send(\n\t\tnew BatchWriteItemCommand({\n\t\t\tRequestItems: {\n\t\t\t\t[options.TableName]: batch,\n\t\t\t},\n\t\t}),\n\t\t{ abortSignal: streamOptions?.signal },\n\t);\n\tif (UnprocessedItems?.[options.TableName]?.length) {\n\t\tif (retryCount >= retryMaxCount) {\n\t\t\tthrow new Error(\"awsDynamoDBBatchWriteItem has UnprocessedItems\", {\n\t\t\t\tcause: {\n\t\t\t\t\t...options,\n\t\t\t\t\tUnprocessedItemsCount: UnprocessedItems[options.TableName].length,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\tawait timeout(3 ** retryCount); // 3^10 == 59sec\n\t\treturn dynamodbBatchWrite(\n\t\t\toptions,\n\t\t\tUnprocessedItems[options.TableName],\n\t\t\tstreamOptions,\n\t\t\tretryCount + 1,\n\t\t);\n\t}\n};\n\nexport default {\n\tsetClient: awsDynamoDBSetClient,\n\tqueryStream: awsDynamoDBQueryStream,\n\tscanStream: awsDynamoDBScanStream,\n\texecuteStatementStream: awsDynamoDBExecuteStatementStream,\n\tgetItemStream: awsDynamoDBGetItemStream,\n\tputItemStream: awsDynamoDBPutItemStream,\n\tdeleteItemStream: awsDynamoDBDeleteItemStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,sBAAsB,eAAe;AAC9C,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,eAAe,iBAAiB;AAC1C,MAAM,uBAAuB,CAAC,WAAW,qBAAqB;AACpE,WAAS;AACV;AACA,qBAAqB,MAAM;AAIpB,MAAM,yBAAyB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC5E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,aAAaA,QAAO,GAAG;AAAA,QAC7D,aAAa,cAAc;AAAA,MAC5B,CAAC;AACD,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,wBAAwB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC3E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,YAAYA,QAAO,GAAG;AAAA,QAC5D,aAAa,cAAc;AAAA,MAC5B,CAAC;AACD,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,oCAAoC,OAChD,SACA,gBAAgB,CAAC,MACb;AACJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,wBAAwBA,QAAO,GAAG;AAAA,QACxE,aAAa,cAAc;AAAA,MAC5B,CAAC;AACD,iBAAW,QAAQ,SAAS,SAAS,CAAC,GAAG;AACxC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,YAAY,SAAS;AAC7B,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC9E,MAAI,QAAQ,MAAM,SAAS,KAAK;AAC/B,UAAM,IAAI;AAAA,MACT,yCAAyC,QAAQ,KAAK,MAAM;AAAA,IAC7D;AAAA,EACD;AACA,kBAAgB,QAAQA,UAAS;AAChC,QAAI,OAAOA,SAAQ;AACnB,QAAI,aAAaA,SAAQ,cAAc;AACvC,UAAM,gBAAgBA,SAAQ,iBAAiB;AAC/C,WAAO,MAAM;AACZ,YAAM,WAAW,MAAM,OAAO;AAAA,QAC7B,IAAI,oBAAoB;AAAA,UACvB,cAAc;AAAA,YACb,CAACA,SAAQ,SAAS,GAAG,EAAE,MAAM,KAAK;AAAA,UACnC;AAAA,QACD,CAAC;AAAA,QACD,EAAE,aAAa,cAAc,OAAO;AAAA,MACrC;AACA,iBAAW,QAAQ,SAAS,UAAUA,SAAQ,SAAS,GAAG;AACzD,cAAM;AAAA,MACP;AACA,YAAM,kBACL,UAAU,kBAAkBA,SAAQ,SAAS,GAAG,QAAQ,CAAC;AAC1D,UAAI,CAAC,gBAAgB,QAAQ;AAC5B;AAAA,MACD;AAEA,UAAI,cAAc,eAAe;AAChC,cAAM,IAAI,MAAM,+CAA+C;AAAA,UAC9D,OAAO;AAAA,YACN,GAAGA;AAAA,YACH,sBAAsB,gBAAgB;AAAA,UACvC;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,QAAQ,KAAK,UAAU;AAC7B;AACA,aAAO;AAAA,IACR;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACxE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,YAAY;AAAA,QACX,MAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,8BAA8B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC3E,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,eAAe;AAAA,QACd,KAAK;AAAA,MACN;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,MAAM,qBAAqB,OAC1B,SACA,OACA,eACA,aAAa,MACT;AACJ,QAAM,gBAAgB,QAAQ,iBAAiB;AAC/C,QAAM,EAAE,iBAAiB,IAAI,MAAM,OAAO;AAAA,IACzC,IAAI,sBAAsB;AAAA,MACzB,cAAc;AAAA,QACb,CAAC,QAAQ,SAAS,GAAG;AAAA,MACtB;AAAA,IACD,CAAC;AAAA,IACD,EAAE,aAAa,eAAe,OAAO;AAAA,EACtC;AACA,MAAI,mBAAmB,QAAQ,SAAS,GAAG,QAAQ;AAClD,QAAI,cAAc,eAAe;AAChC,YAAM,IAAI,MAAM,kDAAkD;AAAA,QACjE,OAAO;AAAA,UACN,GAAG;AAAA,UACH,uBAAuB,iBAAiB,QAAQ,SAAS,EAAE;AAAA,QAC5D;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,UAAU;AAC7B,WAAO;AAAA,MACN;AAAA,MACA,iBAAiB,QAAQ,SAAS;AAAA,MAClC;AAAA,MACA,aAAa;AAAA,IACd;AAAA,EACD;AACD;AAEA,IAAO,mBAAQ;AAAA,EACd,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,wBAAwB;AAAA,EACxB,eAAe;AAAA,EACf,eAAe;AAAA,EACf,kBAAkB;AACnB;", | ||
| "names": ["options"] | ||
| } |
+46
-20
@@ -5,2 +5,3 @@ import { | ||
| DynamoDBClient, | ||
| ExecuteStatementCommand, | ||
| QueryCommand, | ||
@@ -16,7 +17,9 @@ ScanCommand | ||
| awsDynamoDBSetClient(client); | ||
| const awsDynamoDBQueryStream = async (options, _streamOptions = {}) => { | ||
| const awsDynamoDBQueryStream = async (options, streamOptions = {}) => { | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new QueryCommand(options2)); | ||
| const response = await client.send(new QueryCommand(options2), { | ||
| abortSignal: streamOptions.signal | ||
| }); | ||
| for (const item of response.Items) { | ||
@@ -31,7 +34,9 @@ yield item; | ||
| }; | ||
| const awsDynamoDBScanStream = async (options, _streamOptions = {}) => { | ||
| const awsDynamoDBScanStream = async (options, streamOptions = {}) => { | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new ScanCommand(options2)); | ||
| const response = await client.send(new ScanCommand(options2), { | ||
| abortSignal: streamOptions.signal | ||
| }); | ||
| for (const item of response.Items) { | ||
@@ -46,3 +51,19 @@ yield item; | ||
| }; | ||
| const awsDynamoDBGetItemStream = async (options, _streamOptions = {}) => { | ||
| const awsDynamoDBExecuteStatementStream = async (options, streamOptions = {}) => { | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new ExecuteStatementCommand(options2), { | ||
| abortSignal: streamOptions.signal | ||
| }); | ||
| for (const item of response.Items ?? []) { | ||
| yield item; | ||
| } | ||
| options2.NextToken = response.NextToken; | ||
| expectMore = !!response.NextToken; | ||
| } | ||
| } | ||
| return command(options); | ||
| }; | ||
| const awsDynamoDBGetItemStream = async (options, streamOptions = {}) => { | ||
| if (options.Keys?.length > 100) { | ||
@@ -53,5 +74,6 @@ throw new Error( | ||
| } | ||
| options.retryCount ??= 0; | ||
| options.retryMaxCount ??= 10; | ||
| async function* command(options2) { | ||
| let keys = options2.Keys; | ||
| let retryCount = options2.retryCount ?? 0; | ||
| const retryMaxCount = options2.retryMaxCount ?? 10; | ||
| while (true) { | ||
@@ -61,5 +83,6 @@ const response = await client.send( | ||
| RequestItems: { | ||
| [options2.TableName]: { Keys: options2.Keys } | ||
| [options2.TableName]: { Keys: keys } | ||
| } | ||
| }) | ||
| }), | ||
| { abortSignal: streamOptions.signal } | ||
| ); | ||
@@ -73,3 +96,3 @@ for (const item of response.Responses[options2.TableName]) { | ||
| } | ||
| if (options2.retryCount >= options2.retryMaxCount) { | ||
| if (retryCount >= retryMaxCount) { | ||
| throw new Error("awsDynamoDBBatchGetItem has UnprocessedKeys", { | ||
@@ -82,4 +105,5 @@ cause: { | ||
| } | ||
| await timeout(3 ** options2.retryCount++); | ||
| options2.Keys = UnprocessedKeys; | ||
| await timeout(3 ** retryCount); | ||
| retryCount++; | ||
| keys = UnprocessedKeys; | ||
| } | ||
@@ -121,5 +145,4 @@ } | ||
| }; | ||
| const dynamodbBatchWrite = async (options, batch, streamOptions) => { | ||
| options.retryCount ??= 0; | ||
| options.retryMaxCount ??= 10; | ||
| const dynamodbBatchWrite = async (options, batch, streamOptions, retryCount = 0) => { | ||
| const retryMaxCount = options.retryMaxCount ?? 10; | ||
| const { UnprocessedItems } = await client.send( | ||
@@ -130,6 +153,7 @@ new BatchWriteItemCommand({ | ||
| } | ||
| }) | ||
| }), | ||
| { abortSignal: streamOptions?.signal } | ||
| ); | ||
| if (UnprocessedItems?.[options.TableName]?.length) { | ||
| if (options.retryCount >= options.retryMaxCount) { | ||
| if (retryCount >= retryMaxCount) { | ||
| throw new Error("awsDynamoDBBatchWriteItem has UnprocessedItems", { | ||
@@ -142,10 +166,10 @@ cause: { | ||
| } | ||
| await timeout(3 ** options.retryCount++); | ||
| await timeout(3 ** retryCount); | ||
| return dynamodbBatchWrite( | ||
| options, | ||
| UnprocessedItems[options.TableName], | ||
| streamOptions | ||
| streamOptions, | ||
| retryCount + 1 | ||
| ); | ||
| } | ||
| options.retryCount = 0; | ||
| }; | ||
@@ -156,2 +180,3 @@ var dynamodb_default = { | ||
| scanStream: awsDynamoDBScanStream, | ||
| executeStatementStream: awsDynamoDBExecuteStatementStream, | ||
| getItemStream: awsDynamoDBGetItemStream, | ||
@@ -163,2 +188,3 @@ putItemStream: awsDynamoDBPutItemStream, | ||
| awsDynamoDBDeleteItemStream, | ||
| awsDynamoDBExecuteStatementStream, | ||
| awsDynamoDBGetItemStream, | ||
@@ -165,0 +191,0 @@ awsDynamoDBPutItemStream, |
| { | ||
| "version": 3, | ||
| "sources": ["dynamodb.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tBatchGetItemCommand,\n\tBatchWriteItemCommand,\n\tDynamoDBClient,\n\tQueryCommand,\n\tScanCommand,\n} from \"@aws-sdk/client-dynamodb\";\nimport { createWritableStream, timeout } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new DynamoDBClient(awsClientDefaults);\nexport const awsDynamoDBSetClient = (ddbClient, _translateConfig) => {\n\tclient = ddbClient;\n};\nawsDynamoDBSetClient(client);\n\n// options = {TableName, ...}\n\nexport const awsDynamoDBQueryStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new QueryCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBScanStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ScanCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\n// TODO awsDynamoDBExecuteStatementStream\n\nexport const awsDynamoDBGetItemStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tif (options.Keys?.length > 100) {\n\t\tthrow new Error(\n\t\t\t`awsDynamoDBGetItemStream Keys.length (${options.Keys.length}) exceeds BatchGetItem limit of 100`,\n\t\t);\n\t}\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tasync function* command(options) {\n\t\twhile (true) {\n\t\t\tconst response = await client.send(\n\t\t\t\tnew BatchGetItemCommand({\n\t\t\t\t\tRequestItems: {\n\t\t\t\t\t\t[options.TableName]: { Keys: options.Keys },\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t);\n\t\t\tfor (const item of response.Responses[options.TableName]) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst UnprocessedKeys =\n\t\t\t\tresponse?.UnprocessedKeys?.[options.TableName]?.Keys ?? [];\n\t\t\tif (!UnprocessedKeys.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\t\tthrow new Error(\"awsDynamoDBBatchGetItem has UnprocessedKeys\", {\n\t\t\t\t\tcause: {\n\t\t\t\t\t\t...options,\n\t\t\t\t\t\tUnprocessedKeysCount: UnprocessedKeys.length,\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\n\t\t\toptions.Keys = UnprocessedKeys;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBPutItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tPutRequest: {\n\t\t\t\tItem: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsDynamoDBDeleteItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tDeleteRequest: {\n\t\t\t\tKey: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nconst dynamodbBatchWrite = async (options, batch, streamOptions) => {\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tconst { UnprocessedItems } = await client.send(\n\t\tnew BatchWriteItemCommand({\n\t\t\tRequestItems: {\n\t\t\t\t[options.TableName]: batch,\n\t\t\t},\n\t\t}),\n\t);\n\tif (UnprocessedItems?.[options.TableName]?.length) {\n\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\tthrow new Error(\"awsDynamoDBBatchWriteItem has UnprocessedItems\", {\n\t\t\t\tcause: {\n\t\t\t\t\t...options,\n\t\t\t\t\tUnprocessedItemsCount: UnprocessedItems[options.TableName].length,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\t\treturn dynamodbBatchWrite(\n\t\t\toptions,\n\t\t\tUnprocessedItems[options.TableName],\n\t\t\tstreamOptions,\n\t\t);\n\t}\n\toptions.retryCount = 0; // reset for next batch\n};\n\nexport default {\n\tsetClient: awsDynamoDBSetClient,\n\tqueryStream: awsDynamoDBQueryStream,\n\tscanStream: awsDynamoDBScanStream,\n\tgetItemStream: awsDynamoDBGetItemStream,\n\tputItemStream: awsDynamoDBPutItemStream,\n\tdeleteItemStream: awsDynamoDBDeleteItemStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,sBAAsB,eAAe;AAC9C,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,eAAe,iBAAiB;AAC1C,MAAM,uBAAuB,CAAC,WAAW,qBAAqB;AACpE,WAAS;AACV;AACA,qBAAqB,MAAM;AAIpB,MAAM,yBAAyB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC7E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,aAAaA,QAAO,CAAC;AAC5D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,wBAAwB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC5E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,YAAYA,QAAO,CAAC;AAC3D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAIO,MAAM,2BAA2B,OACvC,SACA,iBAAiB,CAAC,MACd;AACJ,MAAI,QAAQ,MAAM,SAAS,KAAK;AAC/B,UAAM,IAAI;AAAA,MACT,yCAAyC,QAAQ,KAAK,MAAM;AAAA,IAC7D;AAAA,EACD;AACA,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,kBAAgB,QAAQA,UAAS;AAChC,WAAO,MAAM;AACZ,YAAM,WAAW,MAAM,OAAO;AAAA,QAC7B,IAAI,oBAAoB;AAAA,UACvB,cAAc;AAAA,YACb,CAACA,SAAQ,SAAS,GAAG,EAAE,MAAMA,SAAQ,KAAK;AAAA,UAC3C;AAAA,QACD,CAAC;AAAA,MACF;AACA,iBAAW,QAAQ,SAAS,UAAUA,SAAQ,SAAS,GAAG;AACzD,cAAM;AAAA,MACP;AACA,YAAM,kBACL,UAAU,kBAAkBA,SAAQ,SAAS,GAAG,QAAQ,CAAC;AAC1D,UAAI,CAAC,gBAAgB,QAAQ;AAC5B;AAAA,MACD;AAEA,UAAIA,SAAQ,cAAcA,SAAQ,eAAe;AAChD,cAAM,IAAI,MAAM,+CAA+C;AAAA,UAC9D,OAAO;AAAA,YACN,GAAGA;AAAA,YACH,sBAAsB,gBAAgB;AAAA,UACvC;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,QAAQ,KAAKA,SAAQ,YAAY;AAEvC,MAAAA,SAAQ,OAAO;AAAA,IAChB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACxE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,YAAY;AAAA,QACX,MAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,8BAA8B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC3E,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,eAAe;AAAA,QACd,KAAK;AAAA,MACN;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,MAAM,qBAAqB,OAAO,SAAS,OAAO,kBAAkB;AACnE,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,QAAM,EAAE,iBAAiB,IAAI,MAAM,OAAO;AAAA,IACzC,IAAI,sBAAsB;AAAA,MACzB,cAAc;AAAA,QACb,CAAC,QAAQ,SAAS,GAAG;AAAA,MACtB;AAAA,IACD,CAAC;AAAA,EACF;AACA,MAAI,mBAAmB,QAAQ,SAAS,GAAG,QAAQ;AAClD,QAAI,QAAQ,cAAc,QAAQ,eAAe;AAChD,YAAM,IAAI,MAAM,kDAAkD;AAAA,QACjE,OAAO;AAAA,UACN,GAAG;AAAA,UACH,uBAAuB,iBAAiB,QAAQ,SAAS,EAAE;AAAA,QAC5D;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,QAAQ,YAAY;AACvC,WAAO;AAAA,MACN;AAAA,MACA,iBAAiB,QAAQ,SAAS;AAAA,MAClC;AAAA,IACD;AAAA,EACD;AACA,UAAQ,aAAa;AACtB;AAEA,IAAO,mBAAQ;AAAA,EACd,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,eAAe;AAAA,EACf,kBAAkB;AACnB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tBatchGetItemCommand,\n\tBatchWriteItemCommand,\n\tDynamoDBClient,\n\tExecuteStatementCommand,\n\tQueryCommand,\n\tScanCommand,\n} from \"@aws-sdk/client-dynamodb\";\nimport { createWritableStream, timeout } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new DynamoDBClient(awsClientDefaults);\nexport const awsDynamoDBSetClient = (ddbClient, _translateConfig) => {\n\tclient = ddbClient;\n};\nawsDynamoDBSetClient(client);\n\n// options = {TableName, ...}\n\nexport const awsDynamoDBQueryStream = async (options, streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new QueryCommand(options), {\n\t\t\t\tabortSignal: streamOptions.signal,\n\t\t\t});\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBScanStream = async (options, streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ScanCommand(options), {\n\t\t\t\tabortSignal: streamOptions.signal,\n\t\t\t});\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBExecuteStatementStream = async (\n\toptions,\n\tstreamOptions = {},\n) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ExecuteStatementCommand(options), {\n\t\t\t\tabortSignal: streamOptions.signal,\n\t\t\t});\n\t\t\tfor (const item of response.Items ?? []) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.NextToken = response.NextToken;\n\t\t\texpectMore = !!response.NextToken;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBGetItemStream = async (options, streamOptions = {}) => {\n\tif (options.Keys?.length > 100) {\n\t\tthrow new Error(\n\t\t\t`awsDynamoDBGetItemStream Keys.length (${options.Keys.length}) exceeds BatchGetItem limit of 100`,\n\t\t);\n\t}\n\tasync function* command(options) {\n\t\tlet keys = options.Keys;\n\t\tlet retryCount = options.retryCount ?? 0;\n\t\tconst retryMaxCount = options.retryMaxCount ?? 10;\n\t\twhile (true) {\n\t\t\tconst response = await client.send(\n\t\t\t\tnew BatchGetItemCommand({\n\t\t\t\t\tRequestItems: {\n\t\t\t\t\t\t[options.TableName]: { Keys: keys },\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t\t{ abortSignal: streamOptions.signal },\n\t\t\t);\n\t\t\tfor (const item of response.Responses[options.TableName]) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst UnprocessedKeys =\n\t\t\t\tresponse?.UnprocessedKeys?.[options.TableName]?.Keys ?? [];\n\t\t\tif (!UnprocessedKeys.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (retryCount >= retryMaxCount) {\n\t\t\t\tthrow new Error(\"awsDynamoDBBatchGetItem has UnprocessedKeys\", {\n\t\t\t\t\tcause: {\n\t\t\t\t\t\t...options,\n\t\t\t\t\t\tUnprocessedKeysCount: UnprocessedKeys.length,\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tawait timeout(3 ** retryCount); // 3^10 == 59sec\n\t\t\tretryCount++;\n\t\t\tkeys = UnprocessedKeys;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBPutItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tPutRequest: {\n\t\t\t\tItem: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsDynamoDBDeleteItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tDeleteRequest: {\n\t\t\t\tKey: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nconst dynamodbBatchWrite = async (\n\toptions,\n\tbatch,\n\tstreamOptions,\n\tretryCount = 0,\n) => {\n\tconst retryMaxCount = options.retryMaxCount ?? 10;\n\tconst { UnprocessedItems } = await client.send(\n\t\tnew BatchWriteItemCommand({\n\t\t\tRequestItems: {\n\t\t\t\t[options.TableName]: batch,\n\t\t\t},\n\t\t}),\n\t\t{ abortSignal: streamOptions?.signal },\n\t);\n\tif (UnprocessedItems?.[options.TableName]?.length) {\n\t\tif (retryCount >= retryMaxCount) {\n\t\t\tthrow new Error(\"awsDynamoDBBatchWriteItem has UnprocessedItems\", {\n\t\t\t\tcause: {\n\t\t\t\t\t...options,\n\t\t\t\t\tUnprocessedItemsCount: UnprocessedItems[options.TableName].length,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\tawait timeout(3 ** retryCount); // 3^10 == 59sec\n\t\treturn dynamodbBatchWrite(\n\t\t\toptions,\n\t\t\tUnprocessedItems[options.TableName],\n\t\t\tstreamOptions,\n\t\t\tretryCount + 1,\n\t\t);\n\t}\n};\n\nexport default {\n\tsetClient: awsDynamoDBSetClient,\n\tqueryStream: awsDynamoDBQueryStream,\n\tscanStream: awsDynamoDBScanStream,\n\texecuteStatementStream: awsDynamoDBExecuteStatementStream,\n\tgetItemStream: awsDynamoDBGetItemStream,\n\tputItemStream: awsDynamoDBPutItemStream,\n\tdeleteItemStream: awsDynamoDBDeleteItemStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,sBAAsB,eAAe;AAC9C,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,eAAe,iBAAiB;AAC1C,MAAM,uBAAuB,CAAC,WAAW,qBAAqB;AACpE,WAAS;AACV;AACA,qBAAqB,MAAM;AAIpB,MAAM,yBAAyB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC5E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,aAAaA,QAAO,GAAG;AAAA,QAC7D,aAAa,cAAc;AAAA,MAC5B,CAAC;AACD,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,wBAAwB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC3E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,YAAYA,QAAO,GAAG;AAAA,QAC5D,aAAa,cAAc;AAAA,MAC5B,CAAC;AACD,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,oCAAoC,OAChD,SACA,gBAAgB,CAAC,MACb;AACJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,wBAAwBA,QAAO,GAAG;AAAA,QACxE,aAAa,cAAc;AAAA,MAC5B,CAAC;AACD,iBAAW,QAAQ,SAAS,SAAS,CAAC,GAAG;AACxC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,YAAY,SAAS;AAC7B,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC9E,MAAI,QAAQ,MAAM,SAAS,KAAK;AAC/B,UAAM,IAAI;AAAA,MACT,yCAAyC,QAAQ,KAAK,MAAM;AAAA,IAC7D;AAAA,EACD;AACA,kBAAgB,QAAQA,UAAS;AAChC,QAAI,OAAOA,SAAQ;AACnB,QAAI,aAAaA,SAAQ,cAAc;AACvC,UAAM,gBAAgBA,SAAQ,iBAAiB;AAC/C,WAAO,MAAM;AACZ,YAAM,WAAW,MAAM,OAAO;AAAA,QAC7B,IAAI,oBAAoB;AAAA,UACvB,cAAc;AAAA,YACb,CAACA,SAAQ,SAAS,GAAG,EAAE,MAAM,KAAK;AAAA,UACnC;AAAA,QACD,CAAC;AAAA,QACD,EAAE,aAAa,cAAc,OAAO;AAAA,MACrC;AACA,iBAAW,QAAQ,SAAS,UAAUA,SAAQ,SAAS,GAAG;AACzD,cAAM;AAAA,MACP;AACA,YAAM,kBACL,UAAU,kBAAkBA,SAAQ,SAAS,GAAG,QAAQ,CAAC;AAC1D,UAAI,CAAC,gBAAgB,QAAQ;AAC5B;AAAA,MACD;AAEA,UAAI,cAAc,eAAe;AAChC,cAAM,IAAI,MAAM,+CAA+C;AAAA,UAC9D,OAAO;AAAA,YACN,GAAGA;AAAA,YACH,sBAAsB,gBAAgB;AAAA,UACvC;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,QAAQ,KAAK,UAAU;AAC7B;AACA,aAAO;AAAA,IACR;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACxE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,YAAY;AAAA,QACX,MAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,8BAA8B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC3E,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,eAAe;AAAA,QACd,KAAK;AAAA,MACN;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,MAAM,qBAAqB,OAC1B,SACA,OACA,eACA,aAAa,MACT;AACJ,QAAM,gBAAgB,QAAQ,iBAAiB;AAC/C,QAAM,EAAE,iBAAiB,IAAI,MAAM,OAAO;AAAA,IACzC,IAAI,sBAAsB;AAAA,MACzB,cAAc;AAAA,QACb,CAAC,QAAQ,SAAS,GAAG;AAAA,MACtB;AAAA,IACD,CAAC;AAAA,IACD,EAAE,aAAa,eAAe,OAAO;AAAA,EACtC;AACA,MAAI,mBAAmB,QAAQ,SAAS,GAAG,QAAQ;AAClD,QAAI,cAAc,eAAe;AAChC,YAAM,IAAI,MAAM,kDAAkD;AAAA,QACjE,OAAO;AAAA,UACN,GAAG;AAAA,UACH,uBAAuB,iBAAiB,QAAQ,SAAS,EAAE;AAAA,QAC5D;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,UAAU;AAC7B,WAAO;AAAA,MACN;AAAA,MACA,iBAAiB,QAAQ,SAAS;AAAA,MAClC;AAAA,MACA,aAAa;AAAA,IACd;AAAA,EACD;AACD;AAEA,IAAO,mBAAQ;AAAA,EACd,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,wBAAwB;AAAA,EACxB,eAAe;AAAA,EACf,eAAe;AAAA,EACf,kBAAkB;AACnB;", | ||
| "names": ["options"] | ||
| } |
+11
-0
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| export { | ||
| awsCloudWatchLogsGetLogEventsStream, | ||
| awsCloudWatchLogsFilterLogEventsStream, | ||
| awsCloudWatchLogsSetClient, | ||
| } from "@datastream/aws/cloudwatch-logs"; | ||
| export { | ||
| awsKinesisGetRecordsStream, | ||
| awsKinesisPutRecordsStream, | ||
| awsKinesisSetClient, | ||
| } from "@datastream/aws/kinesis"; | ||
| export { | ||
| awsS3GetObjectStream, | ||
@@ -12,2 +22,3 @@ awsS3PutObjectStream, | ||
| awsDynamoDBScanStream, | ||
| awsDynamoDBExecuteStatementStream, | ||
| awsDynamoDBGetItemStream, | ||
@@ -14,0 +25,0 @@ awsDynamoDBPutItemStream, |
+2
-0
@@ -0,2 +1,4 @@ | ||
| export * from "@datastream/aws/cloudwatch-logs"; | ||
| export * from "@datastream/aws/dynamodb"; | ||
| export * from "@datastream/aws/kinesis"; | ||
| export * from "@datastream/aws/lambda"; | ||
@@ -3,0 +5,0 @@ export * from "@datastream/aws/s3"; |
| { | ||
| "version": 3, | ||
| "sources": ["index.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nexport * from \"@datastream/aws/dynamodb\";\nexport * from \"@datastream/aws/lambda\";\nexport * from \"@datastream/aws/s3\";\nexport * from \"@datastream/aws/sns\";\nexport * from \"@datastream/aws/sqs\";\n"], | ||
| "mappings": "AAEA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nexport * from \"@datastream/aws/cloudwatch-logs\";\nexport * from \"@datastream/aws/dynamodb\";\nexport * from \"@datastream/aws/kinesis\";\nexport * from \"@datastream/aws/lambda\";\nexport * from \"@datastream/aws/s3\";\nexport * from \"@datastream/aws/sns\";\nexport * from \"@datastream/aws/sqs\";\n"], | ||
| "mappings": "AAEA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;", | ||
| "names": [] | ||
| } |
+2
-0
@@ -0,2 +1,4 @@ | ||
| export * from "@datastream/aws/cloudwatch-logs"; | ||
| export * from "@datastream/aws/dynamodb"; | ||
| export * from "@datastream/aws/kinesis"; | ||
| export * from "@datastream/aws/lambda"; | ||
@@ -3,0 +5,0 @@ export * from "@datastream/aws/s3"; |
| { | ||
| "version": 3, | ||
| "sources": ["index.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nexport * from \"@datastream/aws/dynamodb\";\nexport * from \"@datastream/aws/lambda\";\nexport * from \"@datastream/aws/s3\";\nexport * from \"@datastream/aws/sns\";\nexport * from \"@datastream/aws/sqs\";\n"], | ||
| "mappings": "AAEA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nexport * from \"@datastream/aws/cloudwatch-logs\";\nexport * from \"@datastream/aws/dynamodb\";\nexport * from \"@datastream/aws/kinesis\";\nexport * from \"@datastream/aws/lambda\";\nexport * from \"@datastream/aws/s3\";\nexport * from \"@datastream/aws/sns\";\nexport * from \"@datastream/aws/sqs\";\n"], | ||
| "mappings": "AAEA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;", | ||
| "names": [] | ||
| } |
+7
-3
@@ -12,6 +12,9 @@ import { | ||
| const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => { | ||
| return createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions); | ||
| return createReadableStream( | ||
| awsLambdaGenerator(lambdaOptions, streamOptions), | ||
| streamOptions | ||
| ); | ||
| }; | ||
| const awsLambdaResponseStream = awsLambdaReadableStream; | ||
| async function* awsLambdaGenerator(lambdaOptions, _streamOptions = {}) { | ||
| async function* awsLambdaGenerator(lambdaOptions, streamOptions = {}) { | ||
| if (!Array.isArray(lambdaOptions)) { | ||
@@ -22,3 +25,4 @@ lambdaOptions = [lambdaOptions]; | ||
| const response = await defaultClient.send( | ||
| new InvokeWithResponseStreamCommand(options) | ||
| new InvokeWithResponseStreamCommand(options), | ||
| { abortSignal: streamOptions.signal } | ||
| ); | ||
@@ -25,0 +29,0 @@ for await (const chunk of response.EventStream) { |
| { | ||
| "version": 3, | ||
| "sources": ["lambda.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tInvokeWithResponseStreamCommand,\n\tLambdaClient,\n} from \"@aws-sdk/client-lambda\";\nimport { createReadableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new LambdaClient(awsClientDefaults);\nexport const awsLambdaSetClient = (lambdaClient) => {\n\tdefaultClient = lambdaClient;\n};\n\nexport const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => {\n\treturn createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions);\n};\nexport const awsLambdaResponseStream = awsLambdaReadableStream;\n\nasync function* awsLambdaGenerator(lambdaOptions, _streamOptions = {}) {\n\tif (!Array.isArray(lambdaOptions)) {\n\t\tlambdaOptions = [lambdaOptions];\n\t}\n\tfor (const options of lambdaOptions) {\n\t\tconst response = await defaultClient.send(\n\t\t\tnew InvokeWithResponseStreamCommand(options),\n\t\t);\n\t\tfor await (const chunk of response.EventStream) {\n\t\t\tif (chunk?.PayloadChunk?.Payload) {\n\t\t\t\tyield chunk.PayloadChunk.Payload;\n\t\t\t} else if (chunk?.InvokeComplete?.ErrorCode) {\n\t\t\t\tthrow new Error(chunk.InvokeComplete.ErrorCode, {\n\t\t\t\t\tcause: chunk.InvokeComplete.ErrorDetails,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n}\n\nexport default {\n\tsetClient: awsLambdaSetClient,\n\treadableStream: awsLambdaReadableStream,\n\tresponseStream: awsLambdaReadableStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,aAAa,iBAAiB;AAC/C,MAAM,qBAAqB,CAAC,iBAAiB;AACnD,kBAAgB;AACjB;AAEO,MAAM,0BAA0B,CAAC,eAAe,gBAAgB,CAAC,MAAM;AAC7E,SAAO,qBAAqB,mBAAmB,aAAa,GAAG,aAAa;AAC7E;AACO,MAAM,0BAA0B;AAEvC,gBAAgB,mBAAmB,eAAe,iBAAiB,CAAC,GAAG;AACtE,MAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AAClC,oBAAgB,CAAC,aAAa;AAAA,EAC/B;AACA,aAAW,WAAW,eAAe;AACpC,UAAM,WAAW,MAAM,cAAc;AAAA,MACpC,IAAI,gCAAgC,OAAO;AAAA,IAC5C;AACA,qBAAiB,SAAS,SAAS,aAAa;AAC/C,UAAI,OAAO,cAAc,SAAS;AACjC,cAAM,MAAM,aAAa;AAAA,MAC1B,WAAW,OAAO,gBAAgB,WAAW;AAC5C,cAAM,IAAI,MAAM,MAAM,eAAe,WAAW;AAAA,UAC/C,OAAO,MAAM,eAAe;AAAA,QAC7B,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AACD;AAEA,IAAO,iBAAQ;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tInvokeWithResponseStreamCommand,\n\tLambdaClient,\n} from \"@aws-sdk/client-lambda\";\nimport { createReadableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new LambdaClient(awsClientDefaults);\nexport const awsLambdaSetClient = (lambdaClient) => {\n\tdefaultClient = lambdaClient;\n};\n\nexport const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => {\n\treturn createReadableStream(\n\t\tawsLambdaGenerator(lambdaOptions, streamOptions),\n\t\tstreamOptions,\n\t);\n};\nexport const awsLambdaResponseStream = awsLambdaReadableStream;\n\nasync function* awsLambdaGenerator(lambdaOptions, streamOptions = {}) {\n\tif (!Array.isArray(lambdaOptions)) {\n\t\tlambdaOptions = [lambdaOptions];\n\t}\n\tfor (const options of lambdaOptions) {\n\t\tconst response = await defaultClient.send(\n\t\t\tnew InvokeWithResponseStreamCommand(options),\n\t\t\t{ abortSignal: streamOptions.signal },\n\t\t);\n\t\tfor await (const chunk of response.EventStream) {\n\t\t\tif (chunk?.PayloadChunk?.Payload) {\n\t\t\t\tyield chunk.PayloadChunk.Payload;\n\t\t\t} else if (chunk?.InvokeComplete?.ErrorCode) {\n\t\t\t\tthrow new Error(chunk.InvokeComplete.ErrorCode, {\n\t\t\t\t\tcause: chunk.InvokeComplete.ErrorDetails,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n}\n\nexport default {\n\tsetClient: awsLambdaSetClient,\n\treadableStream: awsLambdaReadableStream,\n\tresponseStream: awsLambdaReadableStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,aAAa,iBAAiB;AAC/C,MAAM,qBAAqB,CAAC,iBAAiB;AACnD,kBAAgB;AACjB;AAEO,MAAM,0BAA0B,CAAC,eAAe,gBAAgB,CAAC,MAAM;AAC7E,SAAO;AAAA,IACN,mBAAmB,eAAe,aAAa;AAAA,IAC/C;AAAA,EACD;AACD;AACO,MAAM,0BAA0B;AAEvC,gBAAgB,mBAAmB,eAAe,gBAAgB,CAAC,GAAG;AACrE,MAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AAClC,oBAAgB,CAAC,aAAa;AAAA,EAC/B;AACA,aAAW,WAAW,eAAe;AACpC,UAAM,WAAW,MAAM,cAAc;AAAA,MACpC,IAAI,gCAAgC,OAAO;AAAA,MAC3C,EAAE,aAAa,cAAc,OAAO;AAAA,IACrC;AACA,qBAAiB,SAAS,SAAS,aAAa;AAC/C,UAAI,OAAO,cAAc,SAAS;AACjC,cAAM,MAAM,aAAa;AAAA,MAC1B,WAAW,OAAO,gBAAgB,WAAW;AAC5C,cAAM,IAAI,MAAM,MAAM,eAAe,WAAW;AAAA,UAC/C,OAAO,MAAM,eAAe;AAAA,QAC7B,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AACD;AAEA,IAAO,iBAAQ;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;", | ||
| "names": [] | ||
| } |
+7
-3
@@ -12,6 +12,9 @@ import { | ||
| const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => { | ||
| return createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions); | ||
| return createReadableStream( | ||
| awsLambdaGenerator(lambdaOptions, streamOptions), | ||
| streamOptions | ||
| ); | ||
| }; | ||
| const awsLambdaResponseStream = awsLambdaReadableStream; | ||
| async function* awsLambdaGenerator(lambdaOptions, _streamOptions = {}) { | ||
| async function* awsLambdaGenerator(lambdaOptions, streamOptions = {}) { | ||
| if (!Array.isArray(lambdaOptions)) { | ||
@@ -22,3 +25,4 @@ lambdaOptions = [lambdaOptions]; | ||
| const response = await defaultClient.send( | ||
| new InvokeWithResponseStreamCommand(options) | ||
| new InvokeWithResponseStreamCommand(options), | ||
| { abortSignal: streamOptions.signal } | ||
| ); | ||
@@ -25,0 +29,0 @@ for await (const chunk of response.EventStream) { |
| { | ||
| "version": 3, | ||
| "sources": ["lambda.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tInvokeWithResponseStreamCommand,\n\tLambdaClient,\n} from \"@aws-sdk/client-lambda\";\nimport { createReadableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new LambdaClient(awsClientDefaults);\nexport const awsLambdaSetClient = (lambdaClient) => {\n\tdefaultClient = lambdaClient;\n};\n\nexport const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => {\n\treturn createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions);\n};\nexport const awsLambdaResponseStream = awsLambdaReadableStream;\n\nasync function* awsLambdaGenerator(lambdaOptions, _streamOptions = {}) {\n\tif (!Array.isArray(lambdaOptions)) {\n\t\tlambdaOptions = [lambdaOptions];\n\t}\n\tfor (const options of lambdaOptions) {\n\t\tconst response = await defaultClient.send(\n\t\t\tnew InvokeWithResponseStreamCommand(options),\n\t\t);\n\t\tfor await (const chunk of response.EventStream) {\n\t\t\tif (chunk?.PayloadChunk?.Payload) {\n\t\t\t\tyield chunk.PayloadChunk.Payload;\n\t\t\t} else if (chunk?.InvokeComplete?.ErrorCode) {\n\t\t\t\tthrow new Error(chunk.InvokeComplete.ErrorCode, {\n\t\t\t\t\tcause: chunk.InvokeComplete.ErrorDetails,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n}\n\nexport default {\n\tsetClient: awsLambdaSetClient,\n\treadableStream: awsLambdaReadableStream,\n\tresponseStream: awsLambdaReadableStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,aAAa,iBAAiB;AAC/C,MAAM,qBAAqB,CAAC,iBAAiB;AACnD,kBAAgB;AACjB;AAEO,MAAM,0BAA0B,CAAC,eAAe,gBAAgB,CAAC,MAAM;AAC7E,SAAO,qBAAqB,mBAAmB,aAAa,GAAG,aAAa;AAC7E;AACO,MAAM,0BAA0B;AAEvC,gBAAgB,mBAAmB,eAAe,iBAAiB,CAAC,GAAG;AACtE,MAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AAClC,oBAAgB,CAAC,aAAa;AAAA,EAC/B;AACA,aAAW,WAAW,eAAe;AACpC,UAAM,WAAW,MAAM,cAAc;AAAA,MACpC,IAAI,gCAAgC,OAAO;AAAA,IAC5C;AACA,qBAAiB,SAAS,SAAS,aAAa;AAC/C,UAAI,OAAO,cAAc,SAAS;AACjC,cAAM,MAAM,aAAa;AAAA,MAC1B,WAAW,OAAO,gBAAgB,WAAW;AAC5C,cAAM,IAAI,MAAM,MAAM,eAAe,WAAW;AAAA,UAC/C,OAAO,MAAM,eAAe;AAAA,QAC7B,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AACD;AAEA,IAAO,iBAAQ;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tInvokeWithResponseStreamCommand,\n\tLambdaClient,\n} from \"@aws-sdk/client-lambda\";\nimport { createReadableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new LambdaClient(awsClientDefaults);\nexport const awsLambdaSetClient = (lambdaClient) => {\n\tdefaultClient = lambdaClient;\n};\n\nexport const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => {\n\treturn createReadableStream(\n\t\tawsLambdaGenerator(lambdaOptions, streamOptions),\n\t\tstreamOptions,\n\t);\n};\nexport const awsLambdaResponseStream = awsLambdaReadableStream;\n\nasync function* awsLambdaGenerator(lambdaOptions, streamOptions = {}) {\n\tif (!Array.isArray(lambdaOptions)) {\n\t\tlambdaOptions = [lambdaOptions];\n\t}\n\tfor (const options of lambdaOptions) {\n\t\tconst response = await defaultClient.send(\n\t\t\tnew InvokeWithResponseStreamCommand(options),\n\t\t\t{ abortSignal: streamOptions.signal },\n\t\t);\n\t\tfor await (const chunk of response.EventStream) {\n\t\t\tif (chunk?.PayloadChunk?.Payload) {\n\t\t\t\tyield chunk.PayloadChunk.Payload;\n\t\t\t} else if (chunk?.InvokeComplete?.ErrorCode) {\n\t\t\t\tthrow new Error(chunk.InvokeComplete.ErrorCode, {\n\t\t\t\t\tcause: chunk.InvokeComplete.ErrorDetails,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n}\n\nexport default {\n\tsetClient: awsLambdaSetClient,\n\treadableStream: awsLambdaReadableStream,\n\tresponseStream: awsLambdaReadableStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,aAAa,iBAAiB;AAC/C,MAAM,qBAAqB,CAAC,iBAAiB;AACnD,kBAAgB;AACjB;AAEO,MAAM,0BAA0B,CAAC,eAAe,gBAAgB,CAAC,MAAM;AAC7E,SAAO;AAAA,IACN,mBAAmB,eAAe,aAAa;AAAA,IAC/C;AAAA,EACD;AACD;AACO,MAAM,0BAA0B;AAEvC,gBAAgB,mBAAmB,eAAe,gBAAgB,CAAC,GAAG;AACrE,MAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AAClC,oBAAgB,CAAC,aAAa;AAAA,EAC/B;AACA,aAAW,WAAW,eAAe;AACpC,UAAM,WAAW,MAAM,cAAc;AAAA,MACpC,IAAI,gCAAgC,OAAO;AAAA,MAC3C,EAAE,aAAa,cAAc,OAAO;AAAA,IACrC;AACA,qBAAiB,SAAS,SAAS,aAAa;AAC/C,UAAI,OAAO,cAAc,SAAS;AACjC,cAAM,MAAM,aAAa;AAAA,MAC1B,WAAW,OAAO,gBAAgB,WAAW;AAC5C,cAAM,IAAI,MAAM,MAAM,eAAe,WAAW;AAAA,UAC/C,OAAO,MAAM,eAAe;AAAA,QAC7B,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AACD;AAEA,IAAO,iBAAQ;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;", | ||
| "names": [] | ||
| } |
+39
-3
| { | ||
| "name": "@datastream/aws", | ||
| "version": "0.1.6", | ||
| "description": "AWS service streaming integrations for S3, DynamoDB, Lambda, SNS, and SQS", | ||
| "version": "0.2.0", | ||
| "description": "AWS service streaming integrations for CloudWatch Logs, DynamoDB, Kinesis, Lambda, S3, SNS, and SQS", | ||
| "type": "module", | ||
@@ -76,2 +76,26 @@ "engines": { | ||
| }, | ||
| "./cloudwatch-logs": { | ||
| "node": { | ||
| "import": { | ||
| "types": "./cloudwatch-logs.d.ts", | ||
| "default": "./cloudwatch-logs.node.mjs" | ||
| } | ||
| }, | ||
| "import": { | ||
| "types": "./cloudwatch-logs.d.ts", | ||
| "default": "./cloudwatch-logs.web.mjs" | ||
| } | ||
| }, | ||
| "./kinesis": { | ||
| "node": { | ||
| "import": { | ||
| "types": "./kinesis.d.ts", | ||
| "default": "./kinesis.node.mjs" | ||
| } | ||
| }, | ||
| "import": { | ||
| "types": "./kinesis.d.ts", | ||
| "default": "./kinesis.web.mjs" | ||
| } | ||
| }, | ||
| "./sqs": { | ||
@@ -104,3 +128,5 @@ "node": { | ||
| "AWS", | ||
| "CloudWatch", | ||
| "DynamoDB", | ||
| "Kinesis", | ||
| "Lambda", | ||
@@ -127,6 +153,8 @@ "S3", | ||
| "dependencies": { | ||
| "@datastream/core": "0.1.6" | ||
| "@datastream/core": "0.2.0" | ||
| }, | ||
| "peerDependencies": { | ||
| "@aws-sdk/client-cloudwatch-logs": "^3.0.0", | ||
| "@aws-sdk/client-dynamodb": "^3.0.0", | ||
| "@aws-sdk/client-kinesis": "^3.0.0", | ||
| "@aws-sdk/client-lambda": "^3.0.0", | ||
@@ -140,5 +168,11 @@ "@aws-sdk/client-s3": "^3.0.0", | ||
| "peerDependenciesMeta": { | ||
| "@aws-sdk/client-cloudwatch-logs": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/client-dynamodb": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/client-kinesis": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/client-lambda": { | ||
@@ -164,3 +198,5 @@ "optional": true | ||
| "devDependencies": { | ||
| "@aws-sdk/client-cloudwatch-logs": "^3.0.0", | ||
| "@aws-sdk/client-dynamodb": "^3.0.0", | ||
| "@aws-sdk/client-kinesis": "^3.0.0", | ||
| "@aws-sdk/client-lambda": "^3.0.0", | ||
@@ -167,0 +203,0 @@ "@aws-sdk/client-s3": "^3.0.0", |
+1
-1
| <div align="center"> | ||
| <h1><datastream> `aws`</h1> | ||
| <img alt="datastream logo" src="https://raw.githubusercontent.com/willfarrell/datastream/main/docs/img/datastream-logo.svg"/> | ||
| <p><strong>AWS service streams for S3, SQS, SNS, Lambda, and DynamoDB.</strong></p> | ||
| <p><strong>AWS service streams for CloudWatch Logs, DynamoDB, Kinesis, Lambda, S3, SNS, and SQS.</strong></p> | ||
| <p> | ||
@@ -6,0 +6,0 @@ <a href="https://github.com/willfarrell/datastream/actions/workflows/test-unit.yml"><img src="https://github.com/willfarrell/datastream/actions/workflows/test-unit.yml/badge.svg" alt="GitHub Actions unit test status"></a> |
+2
-1
@@ -15,3 +15,4 @@ import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; | ||
| const { Body } = await (client ?? defaultClient).send( | ||
| new GetObjectCommand(params) | ||
| new GetObjectCommand(params), | ||
| { abortSignal: streamOptions.signal } | ||
| ); | ||
@@ -18,0 +19,0 @@ if (!Body) { |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["s3.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global crypto */\n\nimport { GetObjectCommand, S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport {\n\tcreatePassThroughStream,\n\tcreateReadableStream,\n} from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new S3Client(awsClientDefaults);\nexport const awsS3SetClient = (s3Client) => {\n\tdefaultClient = s3Client;\n};\n\nexport const awsS3GetObjectStream = async (options, streamOptions = {}) => {\n\tconst { client, ...params } = options;\n\tconst { Body } = await (client ?? defaultClient).send(\n\t\tnew GetObjectCommand(params),\n\t);\n\tif (!Body) {\n\t\tthrow new Error(\"S3.GetObject not found\", { cause: params });\n\t}\n\treturn createReadableStream(Body, streamOptions);\n};\n\nexport const awsS3PutObjectStream = (options, streamOptions = {}) => {\n\tconst { onProgress, client, tags, ...params } = options;\n\tconst stream = createPassThroughStream(() => {}, streamOptions);\n\tconst upload = new Upload({\n\t\tclient: client ?? defaultClient,\n\t\tparams: {\n\t\t\t...params,\n\t\t\tBody: stream,\n\t\t},\n\t\ttags,\n\t});\n\tif (onProgress) {\n\t\tstream.on(\"httpUploadProgress\", onProgress);\n\t}\n\tconst result = upload.done();\n\n\tstream.result = async () => {\n\t\tawait result;\n\t\treturn {};\n\t};\n\treturn stream;\n};\n\n// This is designed to be used in the browser on a file that you want to upload via a presigned URL\n// partSize; magic number, no 16MB mentioned in the docs\nexport const awsS3ChecksumStream = (\n\t{ ChecksumAlgorithm, partSize, resultKey } = {},\n\tstreamOptions = {},\n) => {\n\tChecksumAlgorithm ??= \"SHA256\";\n\tpartSize ??= 17_179_870;\n\tconst algorithm = _algorithms[ChecksumAlgorithm];\n\tif (!algorithm)\n\t\tthrow new Error(`Unsupported ChecksumAlgorithm: ${ChecksumAlgorithm}`);\n\tlet checksums = [];\n\tlet bytes = new Uint8Array(0);\n\tconst passThrough = async (chunk) => {\n\t\tif (typeof chunk === \"string\") {\n\t\t\tchunk = new TextEncoder().encode(chunk);\n\t\t}\n\t\twhile (bytes.byteLength + chunk.byteLength > partSize) {\n\t\t\tchunk = _concatBuffers([bytes, chunk]);\n\t\t\tconst prefixChunk = chunk.slice(0, partSize);\n\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, prefixChunk);\n\t\t\tchecksums.push(checksum);\n\t\t\tchunk = chunk.slice(prefixChunk.byteLength);\n\n\t\t\tbytes = new Uint8Array(0);\n\t\t}\n\t\tbytes = _concatBuffers([bytes, chunk]);\n\t};\n\tconst flush = async () => {\n\t\tif (bytes.byteLength) {\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, bytes);\n\t\t\tchecksums.push(checksum);\n\t\t}\n\t};\n\tconst stream = createPassThroughStream(passThrough, flush, streamOptions);\n\tlet checksum;\n\tstream.result = async () => {\n\t\tif (!checksum) {\n\t\t\tif (checksums.length > 1) {\n\t\t\t\tchecksum = await crypto.subtle.digest(\n\t\t\t\t\talgorithm,\n\t\t\t\t\t_concatBuffers(checksums),\n\t\t\t\t);\n\t\t\t\tchecksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`;\n\t\t\t} else if (checksums.length === 1) {\n\t\t\t\tchecksum = _arrayBufferToBase64(checksums[0]);\n\t\t\t} else {\n\t\t\t\tchecksum = \"\";\n\t\t\t}\n\t\t\tchecksums = checksums.map(_arrayBufferToBase64);\n\t\t}\n\t\treturn {\n\t\t\tkey: resultKey ?? \"s3\",\n\t\t\tvalue: { checksum, checksums, partSize },\n\t\t};\n\t};\n\treturn stream;\n};\n\nconst _algorithms = {\n\t// AWS_NAME: NODE_NAME\n\tSHA1: \"SHA-1\",\n\tSHA256: \"SHA-256\",\n\t// CRC32: '',\n\t// CRC32C: '',\n};\nconst _concatBuffers = (buffers) => {\n\tconst tmp = new Uint8Array(\n\t\tbuffers.reduce((byteLength, buffer) => byteLength + buffer.byteLength, 0),\n\t);\n\tlet byteLength = 0;\n\tfor (let i = 0, l = buffers.length; i < l; i++) {\n\t\ttmp.set(new Uint8Array(buffers[i]), byteLength);\n\t\tbyteLength += buffers[i].byteLength;\n\t}\n\treturn tmp.buffer;\n};\nconst _arrayBufferToBase64 = (buffer) => {\n\tlet binary = \"\";\n\tconst bytes = new Uint8Array(buffer);\n\tconst len = bytes.byteLength;\n\tfor (let i = 0; i < len; i++) {\n\t\tbinary += String.fromCharCode(bytes[i]);\n\t}\n\treturn btoa(binary);\n};\n\nexport default {\n\tsetClient: awsS3SetClient,\n\tgetObjectStream: awsS3GetObjectStream,\n\tputObjectStream: awsS3PutObjectStream,\n\tchecksumStream: awsS3ChecksumStream,\n};\n"], | ||
| "mappings": "AAIA,SAAS,kBAAkB,gBAAgB;AAC3C,SAAS,cAAc;AACvB;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,SAAS,iBAAiB;AAC3C,MAAM,iBAAiB,CAAC,aAAa;AAC3C,kBAAgB;AACjB;AAEO,MAAM,uBAAuB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC1E,QAAM,EAAE,QAAQ,GAAG,OAAO,IAAI;AAC9B,QAAM,EAAE,KAAK,IAAI,OAAO,UAAU,eAAe;AAAA,IAChD,IAAI,iBAAiB,MAAM;AAAA,EAC5B;AACA,MAAI,CAAC,MAAM;AACV,UAAM,IAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,CAAC;AAAA,EAC5D;AACA,SAAO,qBAAqB,MAAM,aAAa;AAChD;AAEO,MAAM,uBAAuB,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,EAAE,YAAY,QAAQ,MAAM,GAAG,OAAO,IAAI;AAChD,QAAM,SAAS,wBAAwB,MAAM;AAAA,EAAC,GAAG,aAAa;AAC9D,QAAM,SAAS,IAAI,OAAO;AAAA,IACzB,QAAQ,UAAU;AAAA,IAClB,QAAQ;AAAA,MACP,GAAG;AAAA,MACH,MAAM;AAAA,IACP;AAAA,IACA;AAAA,EACD,CAAC;AACD,MAAI,YAAY;AACf,WAAO,GAAG,sBAAsB,UAAU;AAAA,EAC3C;AACA,QAAM,SAAS,OAAO,KAAK;AAE3B,SAAO,SAAS,YAAY;AAC3B,UAAM;AACN,WAAO,CAAC;AAAA,EACT;AACA,SAAO;AACR;AAIO,MAAM,sBAAsB,CAClC,EAAE,mBAAmB,UAAU,UAAU,IAAI,CAAC,GAC9C,gBAAgB,CAAC,MACb;AACJ,wBAAsB;AACtB,eAAa;AACb,QAAM,YAAY,YAAY,iBAAiB;AAC/C,MAAI,CAAC;AACJ,UAAM,IAAI,MAAM,kCAAkC,iBAAiB,EAAE;AACtE,MAAI,YAAY,CAAC;AACjB,MAAI,QAAQ,IAAI,WAAW,CAAC;AAC5B,QAAM,cAAc,OAAO,UAAU;AACpC,QAAI,OAAO,UAAU,UAAU;AAC9B,cAAQ,IAAI,YAAY,EAAE,OAAO,KAAK;AAAA,IACvC;AACA,WAAO,MAAM,aAAa,MAAM,aAAa,UAAU;AACtD,cAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AACrC,YAAM,cAAc,MAAM,MAAM,GAAG,QAAQ;AAE3C,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,WAAW;AAClE,gBAAU,KAAKA,SAAQ;AACvB,cAAQ,MAAM,MAAM,YAAY,UAAU;AAE1C,cAAQ,IAAI,WAAW,CAAC;AAAA,IACzB;AACA,YAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AAAA,EACtC;AACA,QAAM,QAAQ,YAAY;AACzB,QAAI,MAAM,YAAY;AACrB,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AAC5D,gBAAU,KAAKA,SAAQ;AAAA,IACxB;AAAA,EACD;AACA,QAAM,SAAS,wBAAwB,aAAa,OAAO,aAAa;AACxE,MAAI;AACJ,SAAO,SAAS,YAAY;AAC3B,QAAI,CAAC,UAAU;AACd,UAAI,UAAU,SAAS,GAAG;AACzB,mBAAW,MAAM,OAAO,OAAO;AAAA,UAC9B;AAAA,UACA,eAAe,SAAS;AAAA,QACzB;AACA,mBAAW,GAAG,qBAAqB,QAAQ,CAAC,IAAI,UAAU,MAAM;AAAA,MACjE,WAAW,UAAU,WAAW,GAAG;AAClC,mBAAW,qBAAqB,UAAU,CAAC,CAAC;AAAA,MAC7C,OAAO;AACN,mBAAW;AAAA,MACZ;AACA,kBAAY,UAAU,IAAI,oBAAoB;AAAA,IAC/C;AACA,WAAO;AAAA,MACN,KAAK,aAAa;AAAA,MAClB,OAAO,EAAE,UAAU,WAAW,SAAS;AAAA,IACxC;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,cAAc;AAAA;AAAA,EAEnB,MAAM;AAAA,EACN,QAAQ;AAAA;AAAA;AAGT;AACA,MAAM,iBAAiB,CAAC,YAAY;AACnC,QAAM,MAAM,IAAI;AAAA,IACf,QAAQ,OAAO,CAACC,aAAY,WAAWA,cAAa,OAAO,YAAY,CAAC;AAAA,EACzE;AACA,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAAI,GAAG,KAAK;AAC/C,QAAI,IAAI,IAAI,WAAW,QAAQ,CAAC,CAAC,GAAG,UAAU;AAC9C,kBAAc,QAAQ,CAAC,EAAE;AAAA,EAC1B;AACA,SAAO,IAAI;AACZ;AACA,MAAM,uBAAuB,CAAC,WAAW;AACxC,MAAI,SAAS;AACb,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,MAAM,MAAM;AAClB,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAU,OAAO,aAAa,MAAM,CAAC,CAAC;AAAA,EACvC;AACA,SAAO,KAAK,MAAM;AACnB;AAEA,IAAO,aAAQ;AAAA,EACd,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,gBAAgB;AACjB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global crypto */\n\nimport { GetObjectCommand, S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport {\n\tcreatePassThroughStream,\n\tcreateReadableStream,\n} from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new S3Client(awsClientDefaults);\nexport const awsS3SetClient = (s3Client) => {\n\tdefaultClient = s3Client;\n};\n\nexport const awsS3GetObjectStream = async (options, streamOptions = {}) => {\n\tconst { client, ...params } = options;\n\tconst { Body } = await (client ?? defaultClient).send(\n\t\tnew GetObjectCommand(params),\n\t\t{ abortSignal: streamOptions.signal },\n\t);\n\tif (!Body) {\n\t\tthrow new Error(\"S3.GetObject not found\", { cause: params });\n\t}\n\treturn createReadableStream(Body, streamOptions);\n};\n\nexport const awsS3PutObjectStream = (options, streamOptions = {}) => {\n\tconst { onProgress, client, tags, ...params } = options;\n\tconst stream = createPassThroughStream(() => {}, streamOptions);\n\tconst upload = new Upload({\n\t\tclient: client ?? defaultClient,\n\t\tparams: {\n\t\t\t...params,\n\t\t\tBody: stream,\n\t\t},\n\t\ttags,\n\t});\n\tif (onProgress) {\n\t\tstream.on(\"httpUploadProgress\", onProgress);\n\t}\n\tconst result = upload.done();\n\n\tstream.result = async () => {\n\t\tawait result;\n\t\treturn {};\n\t};\n\treturn stream;\n};\n\n// This is designed to be used in the browser on a file that you want to upload via a presigned URL\n// partSize; magic number, no 16MB mentioned in the docs\nexport const awsS3ChecksumStream = (\n\t{ ChecksumAlgorithm, partSize, resultKey } = {},\n\tstreamOptions = {},\n) => {\n\tChecksumAlgorithm ??= \"SHA256\";\n\tpartSize ??= 17_179_870; // ~16MB, just under S3 multipart minimum\n\tconst algorithm = _algorithms[ChecksumAlgorithm];\n\tif (!algorithm)\n\t\tthrow new Error(`Unsupported ChecksumAlgorithm: ${ChecksumAlgorithm}`);\n\tlet checksums = [];\n\tlet bytes = new Uint8Array(0);\n\tconst passThrough = async (chunk) => {\n\t\tif (typeof chunk === \"string\") {\n\t\t\tchunk = new TextEncoder().encode(chunk);\n\t\t}\n\t\twhile (bytes.byteLength + chunk.byteLength > partSize) {\n\t\t\tchunk = _concatBuffers([bytes, chunk]);\n\t\t\tconst prefixChunk = chunk.slice(0, partSize);\n\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, prefixChunk);\n\t\t\tchecksums.push(checksum);\n\t\t\tchunk = chunk.slice(prefixChunk.byteLength);\n\n\t\t\tbytes = new Uint8Array(0);\n\t\t}\n\t\tbytes = _concatBuffers([bytes, chunk]);\n\t};\n\tconst flush = async () => {\n\t\tif (bytes.byteLength) {\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, bytes);\n\t\t\tchecksums.push(checksum);\n\t\t}\n\t};\n\tconst stream = createPassThroughStream(passThrough, flush, streamOptions);\n\tlet checksum;\n\tstream.result = async () => {\n\t\tif (!checksum) {\n\t\t\tif (checksums.length > 1) {\n\t\t\t\tchecksum = await crypto.subtle.digest(\n\t\t\t\t\talgorithm,\n\t\t\t\t\t_concatBuffers(checksums),\n\t\t\t\t);\n\t\t\t\tchecksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`;\n\t\t\t} else if (checksums.length === 1) {\n\t\t\t\tchecksum = _arrayBufferToBase64(checksums[0]);\n\t\t\t} else {\n\t\t\t\tchecksum = \"\";\n\t\t\t}\n\t\t\tchecksums = checksums.map(_arrayBufferToBase64);\n\t\t}\n\t\treturn {\n\t\t\tkey: resultKey ?? \"s3\",\n\t\t\tvalue: { checksum, checksums, partSize },\n\t\t};\n\t};\n\treturn stream;\n};\n\nconst _algorithms = {\n\t// AWS_NAME: NODE_NAME\n\tSHA1: \"SHA-1\",\n\tSHA256: \"SHA-256\",\n\t// CRC32: '',\n\t// CRC32C: '',\n};\nconst _concatBuffers = (buffers) => {\n\tconst tmp = new Uint8Array(\n\t\tbuffers.reduce((byteLength, buffer) => byteLength + buffer.byteLength, 0),\n\t);\n\tlet byteLength = 0;\n\tfor (let i = 0, l = buffers.length; i < l; i++) {\n\t\ttmp.set(new Uint8Array(buffers[i]), byteLength);\n\t\tbyteLength += buffers[i].byteLength;\n\t}\n\treturn tmp.buffer;\n};\nconst _arrayBufferToBase64 = (buffer) => {\n\tlet binary = \"\";\n\tconst bytes = new Uint8Array(buffer);\n\tconst len = bytes.byteLength;\n\tfor (let i = 0; i < len; i++) {\n\t\tbinary += String.fromCharCode(bytes[i]);\n\t}\n\treturn btoa(binary);\n};\n\nexport default {\n\tsetClient: awsS3SetClient,\n\tgetObjectStream: awsS3GetObjectStream,\n\tputObjectStream: awsS3PutObjectStream,\n\tchecksumStream: awsS3ChecksumStream,\n};\n"], | ||
| "mappings": "AAIA,SAAS,kBAAkB,gBAAgB;AAC3C,SAAS,cAAc;AACvB;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,SAAS,iBAAiB;AAC3C,MAAM,iBAAiB,CAAC,aAAa;AAC3C,kBAAgB;AACjB;AAEO,MAAM,uBAAuB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC1E,QAAM,EAAE,QAAQ,GAAG,OAAO,IAAI;AAC9B,QAAM,EAAE,KAAK,IAAI,OAAO,UAAU,eAAe;AAAA,IAChD,IAAI,iBAAiB,MAAM;AAAA,IAC3B,EAAE,aAAa,cAAc,OAAO;AAAA,EACrC;AACA,MAAI,CAAC,MAAM;AACV,UAAM,IAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,CAAC;AAAA,EAC5D;AACA,SAAO,qBAAqB,MAAM,aAAa;AAChD;AAEO,MAAM,uBAAuB,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,EAAE,YAAY,QAAQ,MAAM,GAAG,OAAO,IAAI;AAChD,QAAM,SAAS,wBAAwB,MAAM;AAAA,EAAC,GAAG,aAAa;AAC9D,QAAM,SAAS,IAAI,OAAO;AAAA,IACzB,QAAQ,UAAU;AAAA,IAClB,QAAQ;AAAA,MACP,GAAG;AAAA,MACH,MAAM;AAAA,IACP;AAAA,IACA;AAAA,EACD,CAAC;AACD,MAAI,YAAY;AACf,WAAO,GAAG,sBAAsB,UAAU;AAAA,EAC3C;AACA,QAAM,SAAS,OAAO,KAAK;AAE3B,SAAO,SAAS,YAAY;AAC3B,UAAM;AACN,WAAO,CAAC;AAAA,EACT;AACA,SAAO;AACR;AAIO,MAAM,sBAAsB,CAClC,EAAE,mBAAmB,UAAU,UAAU,IAAI,CAAC,GAC9C,gBAAgB,CAAC,MACb;AACJ,wBAAsB;AACtB,eAAa;AACb,QAAM,YAAY,YAAY,iBAAiB;AAC/C,MAAI,CAAC;AACJ,UAAM,IAAI,MAAM,kCAAkC,iBAAiB,EAAE;AACtE,MAAI,YAAY,CAAC;AACjB,MAAI,QAAQ,IAAI,WAAW,CAAC;AAC5B,QAAM,cAAc,OAAO,UAAU;AACpC,QAAI,OAAO,UAAU,UAAU;AAC9B,cAAQ,IAAI,YAAY,EAAE,OAAO,KAAK;AAAA,IACvC;AACA,WAAO,MAAM,aAAa,MAAM,aAAa,UAAU;AACtD,cAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AACrC,YAAM,cAAc,MAAM,MAAM,GAAG,QAAQ;AAE3C,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,WAAW;AAClE,gBAAU,KAAKA,SAAQ;AACvB,cAAQ,MAAM,MAAM,YAAY,UAAU;AAE1C,cAAQ,IAAI,WAAW,CAAC;AAAA,IACzB;AACA,YAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AAAA,EACtC;AACA,QAAM,QAAQ,YAAY;AACzB,QAAI,MAAM,YAAY;AACrB,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AAC5D,gBAAU,KAAKA,SAAQ;AAAA,IACxB;AAAA,EACD;AACA,QAAM,SAAS,wBAAwB,aAAa,OAAO,aAAa;AACxE,MAAI;AACJ,SAAO,SAAS,YAAY;AAC3B,QAAI,CAAC,UAAU;AACd,UAAI,UAAU,SAAS,GAAG;AACzB,mBAAW,MAAM,OAAO,OAAO;AAAA,UAC9B;AAAA,UACA,eAAe,SAAS;AAAA,QACzB;AACA,mBAAW,GAAG,qBAAqB,QAAQ,CAAC,IAAI,UAAU,MAAM;AAAA,MACjE,WAAW,UAAU,WAAW,GAAG;AAClC,mBAAW,qBAAqB,UAAU,CAAC,CAAC;AAAA,MAC7C,OAAO;AACN,mBAAW;AAAA,MACZ;AACA,kBAAY,UAAU,IAAI,oBAAoB;AAAA,IAC/C;AACA,WAAO;AAAA,MACN,KAAK,aAAa;AAAA,MAClB,OAAO,EAAE,UAAU,WAAW,SAAS;AAAA,IACxC;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,cAAc;AAAA;AAAA,EAEnB,MAAM;AAAA,EACN,QAAQ;AAAA;AAAA;AAGT;AACA,MAAM,iBAAiB,CAAC,YAAY;AACnC,QAAM,MAAM,IAAI;AAAA,IACf,QAAQ,OAAO,CAACC,aAAY,WAAWA,cAAa,OAAO,YAAY,CAAC;AAAA,EACzE;AACA,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAAI,GAAG,KAAK;AAC/C,QAAI,IAAI,IAAI,WAAW,QAAQ,CAAC,CAAC,GAAG,UAAU;AAC9C,kBAAc,QAAQ,CAAC,EAAE;AAAA,EAC1B;AACA,SAAO,IAAI;AACZ;AACA,MAAM,uBAAuB,CAAC,WAAW;AACxC,MAAI,SAAS;AACb,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,MAAM,MAAM;AAClB,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAU,OAAO,aAAa,MAAM,CAAC,CAAC;AAAA,EACvC;AACA,SAAO,KAAK,MAAM;AACnB;AAEA,IAAO,aAAQ;AAAA,EACd,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,gBAAgB;AACjB;", | ||
| "names": ["checksum", "byteLength"] | ||
| } |
+2
-1
@@ -15,3 +15,4 @@ import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; | ||
| const { Body } = await (client ?? defaultClient).send( | ||
| new GetObjectCommand(params) | ||
| new GetObjectCommand(params), | ||
| { abortSignal: streamOptions.signal } | ||
| ); | ||
@@ -18,0 +19,0 @@ if (!Body) { |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["s3.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global crypto */\n\nimport { GetObjectCommand, S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport {\n\tcreatePassThroughStream,\n\tcreateReadableStream,\n} from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new S3Client(awsClientDefaults);\nexport const awsS3SetClient = (s3Client) => {\n\tdefaultClient = s3Client;\n};\n\nexport const awsS3GetObjectStream = async (options, streamOptions = {}) => {\n\tconst { client, ...params } = options;\n\tconst { Body } = await (client ?? defaultClient).send(\n\t\tnew GetObjectCommand(params),\n\t);\n\tif (!Body) {\n\t\tthrow new Error(\"S3.GetObject not found\", { cause: params });\n\t}\n\treturn createReadableStream(Body, streamOptions);\n};\n\nexport const awsS3PutObjectStream = (options, streamOptions = {}) => {\n\tconst { onProgress, client, tags, ...params } = options;\n\tconst stream = createPassThroughStream(() => {}, streamOptions);\n\tconst upload = new Upload({\n\t\tclient: client ?? defaultClient,\n\t\tparams: {\n\t\t\t...params,\n\t\t\tBody: stream,\n\t\t},\n\t\ttags,\n\t});\n\tif (onProgress) {\n\t\tstream.on(\"httpUploadProgress\", onProgress);\n\t}\n\tconst result = upload.done();\n\n\tstream.result = async () => {\n\t\tawait result;\n\t\treturn {};\n\t};\n\treturn stream;\n};\n\n// This is designed to be used in the browser on a file that you want to upload via a presigned URL\n// partSize; magic number, no 16MB mentioned in the docs\nexport const awsS3ChecksumStream = (\n\t{ ChecksumAlgorithm, partSize, resultKey } = {},\n\tstreamOptions = {},\n) => {\n\tChecksumAlgorithm ??= \"SHA256\";\n\tpartSize ??= 17_179_870;\n\tconst algorithm = _algorithms[ChecksumAlgorithm];\n\tif (!algorithm)\n\t\tthrow new Error(`Unsupported ChecksumAlgorithm: ${ChecksumAlgorithm}`);\n\tlet checksums = [];\n\tlet bytes = new Uint8Array(0);\n\tconst passThrough = async (chunk) => {\n\t\tif (typeof chunk === \"string\") {\n\t\t\tchunk = new TextEncoder().encode(chunk);\n\t\t}\n\t\twhile (bytes.byteLength + chunk.byteLength > partSize) {\n\t\t\tchunk = _concatBuffers([bytes, chunk]);\n\t\t\tconst prefixChunk = chunk.slice(0, partSize);\n\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, prefixChunk);\n\t\t\tchecksums.push(checksum);\n\t\t\tchunk = chunk.slice(prefixChunk.byteLength);\n\n\t\t\tbytes = new Uint8Array(0);\n\t\t}\n\t\tbytes = _concatBuffers([bytes, chunk]);\n\t};\n\tconst flush = async () => {\n\t\tif (bytes.byteLength) {\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, bytes);\n\t\t\tchecksums.push(checksum);\n\t\t}\n\t};\n\tconst stream = createPassThroughStream(passThrough, flush, streamOptions);\n\tlet checksum;\n\tstream.result = async () => {\n\t\tif (!checksum) {\n\t\t\tif (checksums.length > 1) {\n\t\t\t\tchecksum = await crypto.subtle.digest(\n\t\t\t\t\talgorithm,\n\t\t\t\t\t_concatBuffers(checksums),\n\t\t\t\t);\n\t\t\t\tchecksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`;\n\t\t\t} else if (checksums.length === 1) {\n\t\t\t\tchecksum = _arrayBufferToBase64(checksums[0]);\n\t\t\t} else {\n\t\t\t\tchecksum = \"\";\n\t\t\t}\n\t\t\tchecksums = checksums.map(_arrayBufferToBase64);\n\t\t}\n\t\treturn {\n\t\t\tkey: resultKey ?? \"s3\",\n\t\t\tvalue: { checksum, checksums, partSize },\n\t\t};\n\t};\n\treturn stream;\n};\n\nconst _algorithms = {\n\t// AWS_NAME: NODE_NAME\n\tSHA1: \"SHA-1\",\n\tSHA256: \"SHA-256\",\n\t// CRC32: '',\n\t// CRC32C: '',\n};\nconst _concatBuffers = (buffers) => {\n\tconst tmp = new Uint8Array(\n\t\tbuffers.reduce((byteLength, buffer) => byteLength + buffer.byteLength, 0),\n\t);\n\tlet byteLength = 0;\n\tfor (let i = 0, l = buffers.length; i < l; i++) {\n\t\ttmp.set(new Uint8Array(buffers[i]), byteLength);\n\t\tbyteLength += buffers[i].byteLength;\n\t}\n\treturn tmp.buffer;\n};\nconst _arrayBufferToBase64 = (buffer) => {\n\tlet binary = \"\";\n\tconst bytes = new Uint8Array(buffer);\n\tconst len = bytes.byteLength;\n\tfor (let i = 0; i < len; i++) {\n\t\tbinary += String.fromCharCode(bytes[i]);\n\t}\n\treturn btoa(binary);\n};\n\nexport default {\n\tsetClient: awsS3SetClient,\n\tgetObjectStream: awsS3GetObjectStream,\n\tputObjectStream: awsS3PutObjectStream,\n\tchecksumStream: awsS3ChecksumStream,\n};\n"], | ||
| "mappings": "AAIA,SAAS,kBAAkB,gBAAgB;AAC3C,SAAS,cAAc;AACvB;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,SAAS,iBAAiB;AAC3C,MAAM,iBAAiB,CAAC,aAAa;AAC3C,kBAAgB;AACjB;AAEO,MAAM,uBAAuB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC1E,QAAM,EAAE,QAAQ,GAAG,OAAO,IAAI;AAC9B,QAAM,EAAE,KAAK,IAAI,OAAO,UAAU,eAAe;AAAA,IAChD,IAAI,iBAAiB,MAAM;AAAA,EAC5B;AACA,MAAI,CAAC,MAAM;AACV,UAAM,IAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,CAAC;AAAA,EAC5D;AACA,SAAO,qBAAqB,MAAM,aAAa;AAChD;AAEO,MAAM,uBAAuB,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,EAAE,YAAY,QAAQ,MAAM,GAAG,OAAO,IAAI;AAChD,QAAM,SAAS,wBAAwB,MAAM;AAAA,EAAC,GAAG,aAAa;AAC9D,QAAM,SAAS,IAAI,OAAO;AAAA,IACzB,QAAQ,UAAU;AAAA,IAClB,QAAQ;AAAA,MACP,GAAG;AAAA,MACH,MAAM;AAAA,IACP;AAAA,IACA;AAAA,EACD,CAAC;AACD,MAAI,YAAY;AACf,WAAO,GAAG,sBAAsB,UAAU;AAAA,EAC3C;AACA,QAAM,SAAS,OAAO,KAAK;AAE3B,SAAO,SAAS,YAAY;AAC3B,UAAM;AACN,WAAO,CAAC;AAAA,EACT;AACA,SAAO;AACR;AAIO,MAAM,sBAAsB,CAClC,EAAE,mBAAmB,UAAU,UAAU,IAAI,CAAC,GAC9C,gBAAgB,CAAC,MACb;AACJ,wBAAsB;AACtB,eAAa;AACb,QAAM,YAAY,YAAY,iBAAiB;AAC/C,MAAI,CAAC;AACJ,UAAM,IAAI,MAAM,kCAAkC,iBAAiB,EAAE;AACtE,MAAI,YAAY,CAAC;AACjB,MAAI,QAAQ,IAAI,WAAW,CAAC;AAC5B,QAAM,cAAc,OAAO,UAAU;AACpC,QAAI,OAAO,UAAU,UAAU;AAC9B,cAAQ,IAAI,YAAY,EAAE,OAAO,KAAK;AAAA,IACvC;AACA,WAAO,MAAM,aAAa,MAAM,aAAa,UAAU;AACtD,cAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AACrC,YAAM,cAAc,MAAM,MAAM,GAAG,QAAQ;AAE3C,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,WAAW;AAClE,gBAAU,KAAKA,SAAQ;AACvB,cAAQ,MAAM,MAAM,YAAY,UAAU;AAE1C,cAAQ,IAAI,WAAW,CAAC;AAAA,IACzB;AACA,YAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AAAA,EACtC;AACA,QAAM,QAAQ,YAAY;AACzB,QAAI,MAAM,YAAY;AACrB,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AAC5D,gBAAU,KAAKA,SAAQ;AAAA,IACxB;AAAA,EACD;AACA,QAAM,SAAS,wBAAwB,aAAa,OAAO,aAAa;AACxE,MAAI;AACJ,SAAO,SAAS,YAAY;AAC3B,QAAI,CAAC,UAAU;AACd,UAAI,UAAU,SAAS,GAAG;AACzB,mBAAW,MAAM,OAAO,OAAO;AAAA,UAC9B;AAAA,UACA,eAAe,SAAS;AAAA,QACzB;AACA,mBAAW,GAAG,qBAAqB,QAAQ,CAAC,IAAI,UAAU,MAAM;AAAA,MACjE,WAAW,UAAU,WAAW,GAAG;AAClC,mBAAW,qBAAqB,UAAU,CAAC,CAAC;AAAA,MAC7C,OAAO;AACN,mBAAW;AAAA,MACZ;AACA,kBAAY,UAAU,IAAI,oBAAoB;AAAA,IAC/C;AACA,WAAO;AAAA,MACN,KAAK,aAAa;AAAA,MAClB,OAAO,EAAE,UAAU,WAAW,SAAS;AAAA,IACxC;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,cAAc;AAAA;AAAA,EAEnB,MAAM;AAAA,EACN,QAAQ;AAAA;AAAA;AAGT;AACA,MAAM,iBAAiB,CAAC,YAAY;AACnC,QAAM,MAAM,IAAI;AAAA,IACf,QAAQ,OAAO,CAACC,aAAY,WAAWA,cAAa,OAAO,YAAY,CAAC;AAAA,EACzE;AACA,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAAI,GAAG,KAAK;AAC/C,QAAI,IAAI,IAAI,WAAW,QAAQ,CAAC,CAAC,GAAG,UAAU;AAC9C,kBAAc,QAAQ,CAAC,EAAE;AAAA,EAC1B;AACA,SAAO,IAAI;AACZ;AACA,MAAM,uBAAuB,CAAC,WAAW;AACxC,MAAI,SAAS;AACb,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,MAAM,MAAM;AAClB,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAU,OAAO,aAAa,MAAM,CAAC,CAAC;AAAA,EACvC;AACA,SAAO,KAAK,MAAM;AACnB;AAEA,IAAO,aAAQ;AAAA,EACd,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,gBAAgB;AACjB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global crypto */\n\nimport { GetObjectCommand, S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport {\n\tcreatePassThroughStream,\n\tcreateReadableStream,\n} from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new S3Client(awsClientDefaults);\nexport const awsS3SetClient = (s3Client) => {\n\tdefaultClient = s3Client;\n};\n\nexport const awsS3GetObjectStream = async (options, streamOptions = {}) => {\n\tconst { client, ...params } = options;\n\tconst { Body } = await (client ?? defaultClient).send(\n\t\tnew GetObjectCommand(params),\n\t\t{ abortSignal: streamOptions.signal },\n\t);\n\tif (!Body) {\n\t\tthrow new Error(\"S3.GetObject not found\", { cause: params });\n\t}\n\treturn createReadableStream(Body, streamOptions);\n};\n\nexport const awsS3PutObjectStream = (options, streamOptions = {}) => {\n\tconst { onProgress, client, tags, ...params } = options;\n\tconst stream = createPassThroughStream(() => {}, streamOptions);\n\tconst upload = new Upload({\n\t\tclient: client ?? defaultClient,\n\t\tparams: {\n\t\t\t...params,\n\t\t\tBody: stream,\n\t\t},\n\t\ttags,\n\t});\n\tif (onProgress) {\n\t\tstream.on(\"httpUploadProgress\", onProgress);\n\t}\n\tconst result = upload.done();\n\n\tstream.result = async () => {\n\t\tawait result;\n\t\treturn {};\n\t};\n\treturn stream;\n};\n\n// This is designed to be used in the browser on a file that you want to upload via a presigned URL\n// partSize; magic number, no 16MB mentioned in the docs\nexport const awsS3ChecksumStream = (\n\t{ ChecksumAlgorithm, partSize, resultKey } = {},\n\tstreamOptions = {},\n) => {\n\tChecksumAlgorithm ??= \"SHA256\";\n\tpartSize ??= 17_179_870; // ~16MB, just under S3 multipart minimum\n\tconst algorithm = _algorithms[ChecksumAlgorithm];\n\tif (!algorithm)\n\t\tthrow new Error(`Unsupported ChecksumAlgorithm: ${ChecksumAlgorithm}`);\n\tlet checksums = [];\n\tlet bytes = new Uint8Array(0);\n\tconst passThrough = async (chunk) => {\n\t\tif (typeof chunk === \"string\") {\n\t\t\tchunk = new TextEncoder().encode(chunk);\n\t\t}\n\t\twhile (bytes.byteLength + chunk.byteLength > partSize) {\n\t\t\tchunk = _concatBuffers([bytes, chunk]);\n\t\t\tconst prefixChunk = chunk.slice(0, partSize);\n\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, prefixChunk);\n\t\t\tchecksums.push(checksum);\n\t\t\tchunk = chunk.slice(prefixChunk.byteLength);\n\n\t\t\tbytes = new Uint8Array(0);\n\t\t}\n\t\tbytes = _concatBuffers([bytes, chunk]);\n\t};\n\tconst flush = async () => {\n\t\tif (bytes.byteLength) {\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, bytes);\n\t\t\tchecksums.push(checksum);\n\t\t}\n\t};\n\tconst stream = createPassThroughStream(passThrough, flush, streamOptions);\n\tlet checksum;\n\tstream.result = async () => {\n\t\tif (!checksum) {\n\t\t\tif (checksums.length > 1) {\n\t\t\t\tchecksum = await crypto.subtle.digest(\n\t\t\t\t\talgorithm,\n\t\t\t\t\t_concatBuffers(checksums),\n\t\t\t\t);\n\t\t\t\tchecksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`;\n\t\t\t} else if (checksums.length === 1) {\n\t\t\t\tchecksum = _arrayBufferToBase64(checksums[0]);\n\t\t\t} else {\n\t\t\t\tchecksum = \"\";\n\t\t\t}\n\t\t\tchecksums = checksums.map(_arrayBufferToBase64);\n\t\t}\n\t\treturn {\n\t\t\tkey: resultKey ?? \"s3\",\n\t\t\tvalue: { checksum, checksums, partSize },\n\t\t};\n\t};\n\treturn stream;\n};\n\nconst _algorithms = {\n\t// AWS_NAME: NODE_NAME\n\tSHA1: \"SHA-1\",\n\tSHA256: \"SHA-256\",\n\t// CRC32: '',\n\t// CRC32C: '',\n};\nconst _concatBuffers = (buffers) => {\n\tconst tmp = new Uint8Array(\n\t\tbuffers.reduce((byteLength, buffer) => byteLength + buffer.byteLength, 0),\n\t);\n\tlet byteLength = 0;\n\tfor (let i = 0, l = buffers.length; i < l; i++) {\n\t\ttmp.set(new Uint8Array(buffers[i]), byteLength);\n\t\tbyteLength += buffers[i].byteLength;\n\t}\n\treturn tmp.buffer;\n};\nconst _arrayBufferToBase64 = (buffer) => {\n\tlet binary = \"\";\n\tconst bytes = new Uint8Array(buffer);\n\tconst len = bytes.byteLength;\n\tfor (let i = 0; i < len; i++) {\n\t\tbinary += String.fromCharCode(bytes[i]);\n\t}\n\treturn btoa(binary);\n};\n\nexport default {\n\tsetClient: awsS3SetClient,\n\tgetObjectStream: awsS3GetObjectStream,\n\tputObjectStream: awsS3PutObjectStream,\n\tchecksumStream: awsS3ChecksumStream,\n};\n"], | ||
| "mappings": "AAIA,SAAS,kBAAkB,gBAAgB;AAC3C,SAAS,cAAc;AACvB;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,SAAS,iBAAiB;AAC3C,MAAM,iBAAiB,CAAC,aAAa;AAC3C,kBAAgB;AACjB;AAEO,MAAM,uBAAuB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC1E,QAAM,EAAE,QAAQ,GAAG,OAAO,IAAI;AAC9B,QAAM,EAAE,KAAK,IAAI,OAAO,UAAU,eAAe;AAAA,IAChD,IAAI,iBAAiB,MAAM;AAAA,IAC3B,EAAE,aAAa,cAAc,OAAO;AAAA,EACrC;AACA,MAAI,CAAC,MAAM;AACV,UAAM,IAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,CAAC;AAAA,EAC5D;AACA,SAAO,qBAAqB,MAAM,aAAa;AAChD;AAEO,MAAM,uBAAuB,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,EAAE,YAAY,QAAQ,MAAM,GAAG,OAAO,IAAI;AAChD,QAAM,SAAS,wBAAwB,MAAM;AAAA,EAAC,GAAG,aAAa;AAC9D,QAAM,SAAS,IAAI,OAAO;AAAA,IACzB,QAAQ,UAAU;AAAA,IAClB,QAAQ;AAAA,MACP,GAAG;AAAA,MACH,MAAM;AAAA,IACP;AAAA,IACA;AAAA,EACD,CAAC;AACD,MAAI,YAAY;AACf,WAAO,GAAG,sBAAsB,UAAU;AAAA,EAC3C;AACA,QAAM,SAAS,OAAO,KAAK;AAE3B,SAAO,SAAS,YAAY;AAC3B,UAAM;AACN,WAAO,CAAC;AAAA,EACT;AACA,SAAO;AACR;AAIO,MAAM,sBAAsB,CAClC,EAAE,mBAAmB,UAAU,UAAU,IAAI,CAAC,GAC9C,gBAAgB,CAAC,MACb;AACJ,wBAAsB;AACtB,eAAa;AACb,QAAM,YAAY,YAAY,iBAAiB;AAC/C,MAAI,CAAC;AACJ,UAAM,IAAI,MAAM,kCAAkC,iBAAiB,EAAE;AACtE,MAAI,YAAY,CAAC;AACjB,MAAI,QAAQ,IAAI,WAAW,CAAC;AAC5B,QAAM,cAAc,OAAO,UAAU;AACpC,QAAI,OAAO,UAAU,UAAU;AAC9B,cAAQ,IAAI,YAAY,EAAE,OAAO,KAAK;AAAA,IACvC;AACA,WAAO,MAAM,aAAa,MAAM,aAAa,UAAU;AACtD,cAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AACrC,YAAM,cAAc,MAAM,MAAM,GAAG,QAAQ;AAE3C,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,WAAW;AAClE,gBAAU,KAAKA,SAAQ;AACvB,cAAQ,MAAM,MAAM,YAAY,UAAU;AAE1C,cAAQ,IAAI,WAAW,CAAC;AAAA,IACzB;AACA,YAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AAAA,EACtC;AACA,QAAM,QAAQ,YAAY;AACzB,QAAI,MAAM,YAAY;AACrB,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AAC5D,gBAAU,KAAKA,SAAQ;AAAA,IACxB;AAAA,EACD;AACA,QAAM,SAAS,wBAAwB,aAAa,OAAO,aAAa;AACxE,MAAI;AACJ,SAAO,SAAS,YAAY;AAC3B,QAAI,CAAC,UAAU;AACd,UAAI,UAAU,SAAS,GAAG;AACzB,mBAAW,MAAM,OAAO,OAAO;AAAA,UAC9B;AAAA,UACA,eAAe,SAAS;AAAA,QACzB;AACA,mBAAW,GAAG,qBAAqB,QAAQ,CAAC,IAAI,UAAU,MAAM;AAAA,MACjE,WAAW,UAAU,WAAW,GAAG;AAClC,mBAAW,qBAAqB,UAAU,CAAC,CAAC;AAAA,MAC7C,OAAO;AACN,mBAAW;AAAA,MACZ;AACA,kBAAY,UAAU,IAAI,oBAAoB;AAAA,IAC/C;AACA,WAAO;AAAA,MACN,KAAK,aAAa;AAAA,MAClB,OAAO,EAAE,UAAU,WAAW,SAAS;AAAA,IACxC;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,cAAc;AAAA;AAAA,EAEnB,MAAM;AAAA,EACN,QAAQ;AAAA;AAAA;AAGT;AACA,MAAM,iBAAiB,CAAC,YAAY;AACnC,QAAM,MAAM,IAAI;AAAA,IACf,QAAQ,OAAO,CAACC,aAAY,WAAWA,cAAa,OAAO,YAAY,CAAC;AAAA,EACzE;AACA,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAAI,GAAG,KAAK;AAC/C,QAAI,IAAI,IAAI,WAAW,QAAQ,CAAC,CAAC,GAAG,UAAU;AAC9C,kBAAc,QAAQ,CAAC,EAAE;AAAA,EAC1B;AACA,SAAO,IAAI;AACZ;AACA,MAAM,uBAAuB,CAAC,WAAW;AACxC,MAAI,SAAS;AACb,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,MAAM,MAAM;AAClB,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAU,OAAO,aAAa,MAAM,CAAC,CAAC;AAAA,EACvC;AACA,SAAO,KAAK,MAAM;AACnB;AAEA,IAAO,aAAQ;AAAA,EACd,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,gBAAgB;AACjB;", | ||
| "names": ["checksum", "byteLength"] | ||
| } |
+2
-0
@@ -11,2 +11,4 @@ // Copyright 2026 will Farrell, and datastream contributors. | ||
| QueueUrl?: string; | ||
| pollingActive?: boolean; | ||
| pollingDelay?: number; | ||
| [key: string]: unknown; | ||
@@ -13,0 +15,0 @@ }, |
+10
-4
@@ -13,7 +13,10 @@ import { | ||
| }; | ||
| const awsSQSReceiveMessageStream = async (options, _streamOptions = {}) => { | ||
| const awsSQSReceiveMessageStream = async (options, streamOptions = {}) => { | ||
| const { pollingActive, pollingDelay = 1e3, ...sqsOptions } = options; | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new ReceiveMessageCommand(options2)); | ||
| const response = await client.send(new ReceiveMessageCommand(options2), { | ||
| abortSignal: streamOptions.signal | ||
| }); | ||
| const messages = response.Messages ?? []; | ||
@@ -23,6 +26,9 @@ for (const item of messages) { | ||
| } | ||
| expectMore = messages.length; | ||
| expectMore = pollingActive || messages.length > 0; | ||
| if (pollingActive && messages.length === 0 && pollingDelay > 0) { | ||
| await new Promise((resolve) => setTimeout(resolve, pollingDelay)); | ||
| } | ||
| } | ||
| } | ||
| return command(options); | ||
| return command(sqsOptions); | ||
| }; | ||
@@ -29,0 +35,0 @@ const awsSQSDeleteMessageStream = (options, streamOptions = {}) => { |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["sqs.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tDeleteMessageBatchCommand,\n\tReceiveMessageCommand,\n\tSendMessageBatchCommand,\n\tSQSClient,\n} from \"@aws-sdk/client-sqs\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new SQSClient(awsClientDefaults);\nexport const awsSQSSetClient = (sqsClient) => {\n\tclient = sqsClient;\n};\n\nexport const awsSQSReceiveMessageStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\t// TODO needs option to keep polling or not\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ReceiveMessageCommand(options));\n\t\t\tconst messages = response.Messages ?? [];\n\t\t\tfor (const item of messages) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\texpectMore = messages.length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsSQSDeleteMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new DeleteMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsSQSSendMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new SendMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSQSSetClient,\n\tsendMessageStream: awsSQSSendMessageStream,\n\treceiveMessageStream: awsSQSReceiveMessageStream,\n\tdeleteMessageStream: awsSQSDeleteMessageStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,iBAAiB,CAAC,MACd;AAEJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,sBAAsBA,QAAO,CAAC;AACrE,YAAM,WAAW,SAAS,YAAY,CAAC;AACvC,iBAAW,QAAQ,UAAU;AAC5B,cAAM;AAAA,MACP;AACA,mBAAa,SAAS;AAAA,IACvB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,4BAA4B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACzE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,0BAA0B,OAAO,CAAC;AAAA,EAC1D;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,0BAA0B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACvE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,wBAAwB,OAAO,CAAC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,qBAAqB;AACtB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tDeleteMessageBatchCommand,\n\tReceiveMessageCommand,\n\tSendMessageBatchCommand,\n\tSQSClient,\n} from \"@aws-sdk/client-sqs\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new SQSClient(awsClientDefaults);\nexport const awsSQSSetClient = (sqsClient) => {\n\tclient = sqsClient;\n};\n\nexport const awsSQSReceiveMessageStream = async (\n\toptions,\n\tstreamOptions = {},\n) => {\n\tconst { pollingActive, pollingDelay = 1000, ...sqsOptions } = options;\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ReceiveMessageCommand(options), {\n\t\t\t\tabortSignal: streamOptions.signal,\n\t\t\t});\n\t\t\tconst messages = response.Messages ?? [];\n\t\t\tfor (const item of messages) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\texpectMore = pollingActive || messages.length > 0;\n\t\t\tif (pollingActive && messages.length === 0 && pollingDelay > 0) {\n\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, pollingDelay));\n\t\t\t}\n\t\t}\n\t}\n\treturn command(sqsOptions);\n};\n\nexport const awsSQSDeleteMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new DeleteMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsSQSSendMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new SendMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSQSSetClient,\n\tsendMessageStream: awsSQSSendMessageStream,\n\treceiveMessageStream: awsSQSReceiveMessageStream,\n\tdeleteMessageStream: awsSQSDeleteMessageStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,gBAAgB,CAAC,MACb;AACJ,QAAM,EAAE,eAAe,eAAe,KAAM,GAAG,WAAW,IAAI;AAC9D,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,sBAAsBA,QAAO,GAAG;AAAA,QACtE,aAAa,cAAc;AAAA,MAC5B,CAAC;AACD,YAAM,WAAW,SAAS,YAAY,CAAC;AACvC,iBAAW,QAAQ,UAAU;AAC5B,cAAM;AAAA,MACP;AACA,mBAAa,iBAAiB,SAAS,SAAS;AAChD,UAAI,iBAAiB,SAAS,WAAW,KAAK,eAAe,GAAG;AAC/D,cAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,YAAY,CAAC;AAAA,MACjE;AAAA,IACD;AAAA,EACD;AACA,SAAO,QAAQ,UAAU;AAC1B;AAEO,MAAM,4BAA4B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACzE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,0BAA0B,OAAO,CAAC;AAAA,EAC1D;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,0BAA0B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACvE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,wBAAwB,OAAO,CAAC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,qBAAqB;AACtB;", | ||
| "names": ["options"] | ||
| } |
+10
-4
@@ -13,7 +13,10 @@ import { | ||
| }; | ||
| const awsSQSReceiveMessageStream = async (options, _streamOptions = {}) => { | ||
| const awsSQSReceiveMessageStream = async (options, streamOptions = {}) => { | ||
| const { pollingActive, pollingDelay = 1e3, ...sqsOptions } = options; | ||
| async function* command(options2) { | ||
| let expectMore = true; | ||
| while (expectMore) { | ||
| const response = await client.send(new ReceiveMessageCommand(options2)); | ||
| const response = await client.send(new ReceiveMessageCommand(options2), { | ||
| abortSignal: streamOptions.signal | ||
| }); | ||
| const messages = response.Messages ?? []; | ||
@@ -23,6 +26,9 @@ for (const item of messages) { | ||
| } | ||
| expectMore = messages.length; | ||
| expectMore = pollingActive || messages.length > 0; | ||
| if (pollingActive && messages.length === 0 && pollingDelay > 0) { | ||
| await new Promise((resolve) => setTimeout(resolve, pollingDelay)); | ||
| } | ||
| } | ||
| } | ||
| return command(options); | ||
| return command(sqsOptions); | ||
| }; | ||
@@ -29,0 +35,0 @@ const awsSQSDeleteMessageStream = (options, streamOptions = {}) => { |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["sqs.js"], | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tDeleteMessageBatchCommand,\n\tReceiveMessageCommand,\n\tSendMessageBatchCommand,\n\tSQSClient,\n} from \"@aws-sdk/client-sqs\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new SQSClient(awsClientDefaults);\nexport const awsSQSSetClient = (sqsClient) => {\n\tclient = sqsClient;\n};\n\nexport const awsSQSReceiveMessageStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\t// TODO needs option to keep polling or not\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ReceiveMessageCommand(options));\n\t\t\tconst messages = response.Messages ?? [];\n\t\t\tfor (const item of messages) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\texpectMore = messages.length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsSQSDeleteMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new DeleteMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsSQSSendMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new SendMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSQSSetClient,\n\tsendMessageStream: awsSQSSendMessageStream,\n\treceiveMessageStream: awsSQSReceiveMessageStream,\n\tdeleteMessageStream: awsSQSDeleteMessageStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,iBAAiB,CAAC,MACd;AAEJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,sBAAsBA,QAAO,CAAC;AACrE,YAAM,WAAW,SAAS,YAAY,CAAC;AACvC,iBAAW,QAAQ,UAAU;AAC5B,cAAM;AAAA,MACP;AACA,mBAAa,SAAS;AAAA,IACvB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,4BAA4B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACzE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,0BAA0B,OAAO,CAAC;AAAA,EAC1D;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,0BAA0B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACvE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,wBAAwB,OAAO,CAAC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,qBAAqB;AACtB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tDeleteMessageBatchCommand,\n\tReceiveMessageCommand,\n\tSendMessageBatchCommand,\n\tSQSClient,\n} from \"@aws-sdk/client-sqs\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new SQSClient(awsClientDefaults);\nexport const awsSQSSetClient = (sqsClient) => {\n\tclient = sqsClient;\n};\n\nexport const awsSQSReceiveMessageStream = async (\n\toptions,\n\tstreamOptions = {},\n) => {\n\tconst { pollingActive, pollingDelay = 1000, ...sqsOptions } = options;\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ReceiveMessageCommand(options), {\n\t\t\t\tabortSignal: streamOptions.signal,\n\t\t\t});\n\t\t\tconst messages = response.Messages ?? [];\n\t\t\tfor (const item of messages) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\texpectMore = pollingActive || messages.length > 0;\n\t\t\tif (pollingActive && messages.length === 0 && pollingDelay > 0) {\n\t\t\t\tawait new Promise((resolve) => setTimeout(resolve, pollingDelay));\n\t\t\t}\n\t\t}\n\t}\n\treturn command(sqsOptions);\n};\n\nexport const awsSQSDeleteMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new DeleteMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsSQSSendMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new SendMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSQSSetClient,\n\tsendMessageStream: awsSQSSendMessageStream,\n\treceiveMessageStream: awsSQSReceiveMessageStream,\n\tdeleteMessageStream: awsSQSDeleteMessageStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,gBAAgB,CAAC,MACb;AACJ,QAAM,EAAE,eAAe,eAAe,KAAM,GAAG,WAAW,IAAI;AAC9D,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,sBAAsBA,QAAO,GAAG;AAAA,QACtE,aAAa,cAAc;AAAA,MAC5B,CAAC;AACD,YAAM,WAAW,SAAS,YAAY,CAAC;AACvC,iBAAW,QAAQ,UAAU;AAC5B,cAAM;AAAA,MACP;AACA,mBAAa,iBAAiB,SAAS,SAAS;AAChD,UAAI,iBAAiB,SAAS,WAAW,KAAK,eAAe,GAAG;AAC/D,cAAM,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,YAAY,CAAC;AAAA,MACjE;AAAA,IACD;AAAA,EACD;AACA,SAAO,QAAQ,UAAU;AAC1B;AAEO,MAAM,4BAA4B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACzE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,0BAA0B,OAAO,CAAC;AAAA,EAC1D;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,0BAA0B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACvE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,wBAAwB,OAAO,CAAC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,qBAAqB;AACtB;", | ||
| "names": ["options"] | ||
| } |
116406
36.23%42
31.25%1454
37.43%10
25%10
25%+ Added
+ Added
+ Added
- Removed
Updated