@datastream/aws
Advanced tools
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| import type { StreamOptions } from "@datastream/core"; | ||
| export function awsDynamoDBSetClient( | ||
| ddbClient: unknown, | ||
| translateConfig?: unknown, | ||
| ): void; | ||
| export function awsDynamoDBQueryStream( | ||
| options: { | ||
| client?: unknown; | ||
| TableName?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; | ||
| export function awsDynamoDBScanStream( | ||
| options: { | ||
| client?: unknown; | ||
| TableName?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; | ||
| export function awsDynamoDBGetItemStream( | ||
| options: { | ||
| client?: unknown; | ||
| Keys?: unknown[]; | ||
| TableName?: string; | ||
| retryCount?: number; | ||
| retryMaxCount?: number; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; | ||
| export function awsDynamoDBPutItemStream( | ||
| options: { | ||
| client?: unknown; | ||
| TableName?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): unknown; | ||
| export function awsDynamoDBDeleteItemStream( | ||
| options: { | ||
| client?: unknown; | ||
| TableName?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): unknown; |
+31
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| export { | ||
| awsS3GetObjectStream, | ||
| awsS3PutObjectStream, | ||
| awsS3ChecksumStream, | ||
| awsS3SetClient, | ||
| } from "@datastream/aws/s3"; | ||
| export { | ||
| awsDynamoDBQueryStream, | ||
| awsDynamoDBScanStream, | ||
| awsDynamoDBGetItemStream, | ||
| awsDynamoDBPutItemStream, | ||
| awsDynamoDBDeleteItemStream, | ||
| awsDynamoDBSetClient, | ||
| } from "@datastream/aws/dynamodb"; | ||
| export { | ||
| awsLambdaReadableStream, | ||
| awsLambdaResponseStream, | ||
| awsLambdaSetClient, | ||
| } from "@datastream/aws/lambda"; | ||
| export { | ||
| awsSNSPublishMessageStream, | ||
| awsSNSSetClient, | ||
| } from "@datastream/aws/sns"; | ||
| export { | ||
| awsSQSReceiveMessageStream, | ||
| awsSQSSendMessageStream, | ||
| awsSQSDeleteMessageStream, | ||
| awsSQSSetClient, | ||
| } from "@datastream/aws/sqs"; |
+11
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| import type { StreamOptions } from "@datastream/core"; | ||
| export function awsLambdaSetClient(lambdaClient: unknown): void; | ||
| export function awsLambdaReadableStream( | ||
| lambdaOptions: Record<string, unknown> | Record<string, unknown>[], | ||
| streamOptions?: StreamOptions, | ||
| ): unknown; | ||
| export { awsLambdaReadableStream as awsLambdaResponseStream }; |
+52
| <div align="center"> | ||
| <h1><datastream> `aws`</h1> | ||
| <img alt="datastream logo" src="https://raw.githubusercontent.com/willfarrell/datastream/main/docs/img/datastream-logo.svg"/> | ||
| <p><strong>AWS service streams for S3, SQS, SNS, Lambda, and DynamoDB.</strong></p> | ||
| <p> | ||
| <a href="https://github.com/willfarrell/datastream/actions/workflows/test-unit.yml"><img src="https://github.com/willfarrell/datastream/actions/workflows/test-unit.yml/badge.svg" alt="GitHub Actions unit test status"></a> | ||
| <a href="https://github.com/willfarrell/datastream/actions/workflows/test-dast.yml"><img src="https://github.com/willfarrell/datastream/actions/workflows/test-dast.yml/badge.svg" alt="GitHub Actions dast test status"></a> | ||
| <a href="https://github.com/willfarrell/datastream/actions/workflows/test-perf.yml"><img src="https://github.com/willfarrell/datastream/actions/workflows/test-perf.yml/badge.svg" alt="GitHub Actions perf test status"></a> | ||
| <a href="https://github.com/willfarrell/datastream/actions/workflows/test-sast.yml"><img src="https://github.com/willfarrell/datastream/actions/workflows/test-sast.yml/badge.svg" alt="GitHub Actions SAST test status"></a> | ||
| <a href="https://github.com/willfarrell/datastream/actions/workflows/test-lint.yml"><img src="https://github.com/willfarrell/datastream/actions/workflows/test-lint.yml/badge.svg" alt="GitHub Actions lint test status"></a> | ||
| <br/> | ||
| <a href="https://www.npmjs.com/package/@datastream/aws"><img alt="npm version" src="https://img.shields.io/npm/v/@datastream/aws.svg"></a> | ||
| <a href="https://packagephobia.com/result?p=@datastream/aws"><img src="https://packagephobia.com/badge?p=@datastream/aws" alt="npm install size"></a> | ||
| <a href="https://www.npmjs.com/package/@datastream/aws"> | ||
| <img alt="npm weekly downloads" src="https://img.shields.io/npm/dw/@datastream/aws.svg"></a> | ||
| <a href="https://www.npmjs.com/package/@datastream/aws#provenance"> | ||
| <img alt="npm provenance" src="https://img.shields.io/badge/provenance-Yes-brightgreen"></a> | ||
| <br/> | ||
| <a href="https://scorecard.dev/viewer/?uri=github.com/willfarrell/datastream"><img src="https://api.scorecard.dev/projects/github.com/willfarrell/datastream/badge" alt="Open Source Security Foundation (OpenSSF) Scorecard"></a> | ||
| <a href="https://slsa.dev"><img src="https://slsa.dev/images/gh-badge-level3.svg" alt="SLSA 3"></a> | ||
| <a href="https://github.com/willfarrell/datastream/blob/main/docs/CODE_OF_CONDUCT.md"><img src="https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg"></a> | ||
| <a href="https://biomejs.dev"><img alt="Checked with Biome" src="https://img.shields.io/badge/Checked_with-Biome-60a5fa?style=flat&logo=biome"></a> | ||
| <a href="https://conventionalcommits.org"><img alt="Conventional Commits" src="https://img.shields.io/badge/Conventional%20Commits-1.0.0-%23FE5196?logo=conventionalcommits&logoColor=white"></a> | ||
| <a href="https://github.com/willfarrell/datastream/blob/main/package.json#L32"> | ||
| <img alt="code coverage" src="https://img.shields.io/badge/code%20coverage-95%25-brightgreen"></a> | ||
| </p> | ||
| <p>You can read the documentation at: <a href="https://datastream.js.org">https://datastream.js.org</a></p> | ||
| </div> | ||
| ## Install | ||
| To install datastream you can use NPM: | ||
| ```bash | ||
| npm install --save @datastream/aws | ||
| ``` | ||
| ## Documentation and examples | ||
| For documentation and examples, refer to the main [datastream monorepo on GitHub](https://github.com/willfarrell/datastream) or [datastream official website](https://datastream.js.org). | ||
| ## Contributing | ||
| Everyone is very welcome to contribute to this repository. Feel free to [raise issues](https://github.com/willfarrell/datastream/issues) or to [submit Pull Requests](https://github.com/willfarrell/datastream/pulls). | ||
| ## License | ||
| Licensed under [MIT License](LICENSE). Copyright (c) 2026 [will Farrell](https://github.com/willfarrell), and [datastream contributors](https://github.com/willfarrell/datastream/graphs/contributors). |
+44
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| import type { StreamOptions, StreamResult } from "@datastream/core"; | ||
| export function awsS3SetClient(s3Client: unknown): void; | ||
| export function awsS3GetObjectStream( | ||
| options: { | ||
| client?: unknown; | ||
| Bucket?: string; | ||
| Key?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; | ||
| export function awsS3PutObjectStream( | ||
| options: { | ||
| client?: unknown; | ||
| onProgress?: (progress: unknown) => void; | ||
| tags?: Record<string, string>; | ||
| Bucket?: string; | ||
| Key?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): unknown & { | ||
| result: () => Promise<StreamResult<unknown>>; | ||
| }; | ||
| export function awsS3ChecksumStream( | ||
| options?: { | ||
| ChecksumAlgorithm?: string; | ||
| partSize?: number; | ||
| resultKey?: string; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): unknown & { | ||
| result: () => StreamResult<{ | ||
| checksum: string; | ||
| checksums: string[]; | ||
| partSize: number; | ||
| }>; | ||
| }; |
+14
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| import type { StreamOptions } from "@datastream/core"; | ||
| export function awsSNSSetClient(snsClient: unknown): void; | ||
| export function awsSNSPublishMessageStream( | ||
| options: { | ||
| client?: unknown; | ||
| TopicArn?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): unknown; |
+32
| // Copyright 2026 will Farrell, and datastream contributors. | ||
| // SPDX-License-Identifier: MIT | ||
| import type { StreamOptions } from "@datastream/core"; | ||
| export function awsSQSSetClient(sqsClient: unknown): void; | ||
| export function awsSQSReceiveMessageStream( | ||
| options: { | ||
| client?: unknown; | ||
| QueueUrl?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): Promise<unknown>; | ||
| export function awsSQSDeleteMessageStream( | ||
| options: { | ||
| client?: unknown; | ||
| QueueUrl?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): unknown; | ||
| export function awsSQSSendMessageStream( | ||
| options: { | ||
| client?: unknown; | ||
| QueueUrl?: string; | ||
| [key: string]: unknown; | ||
| }, | ||
| streamOptions?: StreamOptions, | ||
| ): unknown; |
+12
-16
@@ -9,12 +9,3 @@ import { | ||
| import { createWritableStream, timeout } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2", | ||
| "ca-central-1" | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new DynamoDBClient(awsClientDefaults); | ||
@@ -34,3 +25,3 @@ const awsDynamoDBSetClient = (ddbClient, _translateConfig) => { | ||
| options2.ExclusiveStartKey = response.LastEvaluatedKey; | ||
| expectMore = Object.keys(options2.ExclusiveStartKey).length; | ||
| expectMore = !!response.LastEvaluatedKey; | ||
| } | ||
@@ -49,3 +40,3 @@ } | ||
| options2.ExclusiveStartKey = response.LastEvaluatedKey; | ||
| expectMore = Object.keys(options2.ExclusiveStartKey).length; | ||
| expectMore = !!response.LastEvaluatedKey; | ||
| } | ||
@@ -56,2 +47,7 @@ } | ||
| const awsDynamoDBGetItemStream = async (options, _streamOptions = {}) => { | ||
| if (options.Keys?.length > 100) { | ||
| throw new Error( | ||
| `awsDynamoDBGetItemStream Keys.length (${options.Keys.length}) exceeds BatchGetItem limit of 100` | ||
| ); | ||
| } | ||
| options.retryCount ??= 0; | ||
@@ -89,3 +85,3 @@ options.retryMaxCount ??= 10; | ||
| }; | ||
| const awsDynamoDBPutItemStream = (options, streamOptions) => { | ||
| const awsDynamoDBPutItemStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -103,6 +99,6 @@ const write = async (chunk) => { | ||
| }; | ||
| const final = () => dynamodbBatchWrite(options, batch, streamOptions); | ||
| const final = () => batch.length ? dynamodbBatchWrite(options, batch, streamOptions) : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
| }; | ||
| const awsDynamoDBDeleteItemStream = (options, streamOptions) => { | ||
| const awsDynamoDBDeleteItemStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -120,3 +116,3 @@ const write = async (chunk) => { | ||
| }; | ||
| const final = () => dynamodbBatchWrite(options, batch, streamOptions); | ||
| const final = () => batch.length ? dynamodbBatchWrite(options, batch, streamOptions) : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
@@ -123,0 +119,0 @@ }; |
| { | ||
| "version": 3, | ||
| "sources": ["dynamodb.js"], | ||
| "sourcesContent": ["import {\n\tBatchGetItemCommand,\n\tBatchWriteItemCommand,\n\tDynamoDBClient,\n\tQueryCommand,\n\tScanCommand,\n} from \"@aws-sdk/client-dynamodb\";\nimport { createWritableStream, timeout } from \"@datastream/core\";\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t\t\"ca-central-1\",\n\t].includes(process.env.AWS_REGION),\n};\n\nlet client = new DynamoDBClient(awsClientDefaults);\nexport const awsDynamoDBSetClient = (ddbClient, _translateConfig) => {\n\tclient = ddbClient;\n};\nawsDynamoDBSetClient(client);\n\n// options = {TableName, ...}\n\nexport const awsDynamoDBQueryStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new QueryCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = Object.keys(options.ExclusiveStartKey).length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBScanStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ScanCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = Object.keys(options.ExclusiveStartKey).length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\n// TODO awsDynamoDBExecuteStatementStream\n\n// TODO max Keys.length = 100\nexport const awsDynamoDBGetItemStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tasync function* command(options) {\n\t\twhile (true) {\n\t\t\tconst response = await client.send(\n\t\t\t\tnew BatchGetItemCommand({\n\t\t\t\t\tRequestItems: {\n\t\t\t\t\t\t[options.TableName]: { Keys: options.Keys },\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t);\n\t\t\tfor (const item of response.Responses[options.TableName]) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst UnprocessedKeys =\n\t\t\t\tresponse?.UnprocessedKeys?.[options.TableName]?.Keys ?? [];\n\t\t\tif (!UnprocessedKeys.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\t\tthrow new Error(\"awsDynamoDBBatchGetItem has UnprocessedKeys\", {\n\t\t\t\t\tcause: {\n\t\t\t\t\t\t...options,\n\t\t\t\t\t\tUnprocessedKeysCount: UnprocessedKeys.length,\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\n\t\t\toptions.Keys = UnprocessedKeys;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBPutItemStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tPutRequest: {\n\t\t\t\tItem: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () => dynamodbBatchWrite(options, batch, streamOptions);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsDynamoDBDeleteItemStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tDeleteRequest: {\n\t\t\t\tKey: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () => dynamodbBatchWrite(options, batch, streamOptions);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nconst dynamodbBatchWrite = async (options, batch, streamOptions) => {\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tconst { UnprocessedItems } = await client.send(\n\t\tnew BatchWriteItemCommand({\n\t\t\tRequestItems: {\n\t\t\t\t[options.TableName]: batch,\n\t\t\t},\n\t\t}),\n\t);\n\tif (UnprocessedItems?.[options.TableName]?.length) {\n\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\tthrow new Error(\"awsDynamoDBBatchWriteItem has UnprocessedItems\", {\n\t\t\t\tcause: {\n\t\t\t\t\t...options,\n\t\t\t\t\tUnprocessedItemsCount: UnprocessedItems[options.TableName].length,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\t\treturn dynamodbBatchWrite(\n\t\t\toptions,\n\t\t\tUnprocessedItems[options.TableName],\n\t\t\tstreamOptions,\n\t\t);\n\t}\n\toptions.retryCount = 0; // reset for next batch\n};\n\nexport default {\n\tsetClient: awsDynamoDBSetClient,\n\tqueryStream: awsDynamoDBQueryStream,\n\tscanStream: awsDynamoDBScanStream,\n\tgetItemStream: awsDynamoDBGetItemStream,\n\tputItemStream: awsDynamoDBPutItemStream,\n\tdeleteItemStream: awsDynamoDBDeleteItemStream,\n};\n"], | ||
| "mappings": "AAAA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,sBAAsB,eAAe;AAE9C,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,SAAS,IAAI,eAAe,iBAAiB;AAC1C,MAAM,uBAAuB,CAAC,WAAW,qBAAqB;AACpE,WAAS;AACV;AACA,qBAAqB,MAAM;AAIpB,MAAM,yBAAyB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC7E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,aAAaA,QAAO,CAAC;AAC5D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,OAAO,KAAKA,SAAQ,iBAAiB,EAAE;AAAA,IACrD;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,wBAAwB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC5E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,YAAYA,QAAO,CAAC;AAC3D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,OAAO,KAAKA,SAAQ,iBAAiB,EAAE;AAAA,IACrD;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAKO,MAAM,2BAA2B,OACvC,SACA,iBAAiB,CAAC,MACd;AACJ,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,kBAAgB,QAAQA,UAAS;AAChC,WAAO,MAAM;AACZ,YAAM,WAAW,MAAM,OAAO;AAAA,QAC7B,IAAI,oBAAoB;AAAA,UACvB,cAAc;AAAA,YACb,CAACA,SAAQ,SAAS,GAAG,EAAE,MAAMA,SAAQ,KAAK;AAAA,UAC3C;AAAA,QACD,CAAC;AAAA,MACF;AACA,iBAAW,QAAQ,SAAS,UAAUA,SAAQ,SAAS,GAAG;AACzD,cAAM;AAAA,MACP;AACA,YAAM,kBACL,UAAU,kBAAkBA,SAAQ,SAAS,GAAG,QAAQ,CAAC;AAC1D,UAAI,CAAC,gBAAgB,QAAQ;AAC5B;AAAA,MACD;AAEA,UAAIA,SAAQ,cAAcA,SAAQ,eAAe;AAChD,cAAM,IAAI,MAAM,+CAA+C;AAAA,UAC9D,OAAO;AAAA,YACN,GAAGA;AAAA,YACH,sBAAsB,gBAAgB;AAAA,UACvC;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,QAAQ,KAAKA,SAAQ,YAAY;AAEvC,MAAAA,SAAQ,OAAO;AAAA,IAChB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,CAAC,SAAS,kBAAkB;AACnE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,YAAY;AAAA,QACX,MAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MAAM,mBAAmB,SAAS,OAAO,aAAa;AACpE,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,8BAA8B,CAAC,SAAS,kBAAkB;AACtE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,eAAe;AAAA,QACd,KAAK;AAAA,MACN;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MAAM,mBAAmB,SAAS,OAAO,aAAa;AACpE,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,MAAM,qBAAqB,OAAO,SAAS,OAAO,kBAAkB;AACnE,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,QAAM,EAAE,iBAAiB,IAAI,MAAM,OAAO;AAAA,IACzC,IAAI,sBAAsB;AAAA,MACzB,cAAc;AAAA,QACb,CAAC,QAAQ,SAAS,GAAG;AAAA,MACtB;AAAA,IACD,CAAC;AAAA,EACF;AACA,MAAI,mBAAmB,QAAQ,SAAS,GAAG,QAAQ;AAClD,QAAI,QAAQ,cAAc,QAAQ,eAAe;AAChD,YAAM,IAAI,MAAM,kDAAkD;AAAA,QACjE,OAAO;AAAA,UACN,GAAG;AAAA,UACH,uBAAuB,iBAAiB,QAAQ,SAAS,EAAE;AAAA,QAC5D;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,QAAQ,YAAY;AACvC,WAAO;AAAA,MACN;AAAA,MACA,iBAAiB,QAAQ,SAAS;AAAA,MAClC;AAAA,IACD;AAAA,EACD;AACA,UAAQ,aAAa;AACtB;AAEA,IAAO,mBAAQ;AAAA,EACd,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,eAAe;AAAA,EACf,kBAAkB;AACnB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tBatchGetItemCommand,\n\tBatchWriteItemCommand,\n\tDynamoDBClient,\n\tQueryCommand,\n\tScanCommand,\n} from \"@aws-sdk/client-dynamodb\";\nimport { createWritableStream, timeout } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new DynamoDBClient(awsClientDefaults);\nexport const awsDynamoDBSetClient = (ddbClient, _translateConfig) => {\n\tclient = ddbClient;\n};\nawsDynamoDBSetClient(client);\n\n// options = {TableName, ...}\n\nexport const awsDynamoDBQueryStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new QueryCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBScanStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ScanCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\n// TODO awsDynamoDBExecuteStatementStream\n\nexport const awsDynamoDBGetItemStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tif (options.Keys?.length > 100) {\n\t\tthrow new Error(\n\t\t\t`awsDynamoDBGetItemStream Keys.length (${options.Keys.length}) exceeds BatchGetItem limit of 100`,\n\t\t);\n\t}\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tasync function* command(options) {\n\t\twhile (true) {\n\t\t\tconst response = await client.send(\n\t\t\t\tnew BatchGetItemCommand({\n\t\t\t\t\tRequestItems: {\n\t\t\t\t\t\t[options.TableName]: { Keys: options.Keys },\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t);\n\t\t\tfor (const item of response.Responses[options.TableName]) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst UnprocessedKeys =\n\t\t\t\tresponse?.UnprocessedKeys?.[options.TableName]?.Keys ?? [];\n\t\t\tif (!UnprocessedKeys.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\t\tthrow new Error(\"awsDynamoDBBatchGetItem has UnprocessedKeys\", {\n\t\t\t\t\tcause: {\n\t\t\t\t\t\t...options,\n\t\t\t\t\t\tUnprocessedKeysCount: UnprocessedKeys.length,\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\n\t\t\toptions.Keys = UnprocessedKeys;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBPutItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tPutRequest: {\n\t\t\t\tItem: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsDynamoDBDeleteItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tDeleteRequest: {\n\t\t\t\tKey: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nconst dynamodbBatchWrite = async (options, batch, streamOptions) => {\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tconst { UnprocessedItems } = await client.send(\n\t\tnew BatchWriteItemCommand({\n\t\t\tRequestItems: {\n\t\t\t\t[options.TableName]: batch,\n\t\t\t},\n\t\t}),\n\t);\n\tif (UnprocessedItems?.[options.TableName]?.length) {\n\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\tthrow new Error(\"awsDynamoDBBatchWriteItem has UnprocessedItems\", {\n\t\t\t\tcause: {\n\t\t\t\t\t...options,\n\t\t\t\t\tUnprocessedItemsCount: UnprocessedItems[options.TableName].length,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\t\treturn dynamodbBatchWrite(\n\t\t\toptions,\n\t\t\tUnprocessedItems[options.TableName],\n\t\t\tstreamOptions,\n\t\t);\n\t}\n\toptions.retryCount = 0; // reset for next batch\n};\n\nexport default {\n\tsetClient: awsDynamoDBSetClient,\n\tqueryStream: awsDynamoDBQueryStream,\n\tscanStream: awsDynamoDBScanStream,\n\tgetItemStream: awsDynamoDBGetItemStream,\n\tputItemStream: awsDynamoDBPutItemStream,\n\tdeleteItemStream: awsDynamoDBDeleteItemStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,sBAAsB,eAAe;AAC9C,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,eAAe,iBAAiB;AAC1C,MAAM,uBAAuB,CAAC,WAAW,qBAAqB;AACpE,WAAS;AACV;AACA,qBAAqB,MAAM;AAIpB,MAAM,yBAAyB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC7E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,aAAaA,QAAO,CAAC;AAC5D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,wBAAwB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC5E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,YAAYA,QAAO,CAAC;AAC3D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAIO,MAAM,2BAA2B,OACvC,SACA,iBAAiB,CAAC,MACd;AACJ,MAAI,QAAQ,MAAM,SAAS,KAAK;AAC/B,UAAM,IAAI;AAAA,MACT,yCAAyC,QAAQ,KAAK,MAAM;AAAA,IAC7D;AAAA,EACD;AACA,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,kBAAgB,QAAQA,UAAS;AAChC,WAAO,MAAM;AACZ,YAAM,WAAW,MAAM,OAAO;AAAA,QAC7B,IAAI,oBAAoB;AAAA,UACvB,cAAc;AAAA,YACb,CAACA,SAAQ,SAAS,GAAG,EAAE,MAAMA,SAAQ,KAAK;AAAA,UAC3C;AAAA,QACD,CAAC;AAAA,MACF;AACA,iBAAW,QAAQ,SAAS,UAAUA,SAAQ,SAAS,GAAG;AACzD,cAAM;AAAA,MACP;AACA,YAAM,kBACL,UAAU,kBAAkBA,SAAQ,SAAS,GAAG,QAAQ,CAAC;AAC1D,UAAI,CAAC,gBAAgB,QAAQ;AAC5B;AAAA,MACD;AAEA,UAAIA,SAAQ,cAAcA,SAAQ,eAAe;AAChD,cAAM,IAAI,MAAM,+CAA+C;AAAA,UAC9D,OAAO;AAAA,YACN,GAAGA;AAAA,YACH,sBAAsB,gBAAgB;AAAA,UACvC;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,QAAQ,KAAKA,SAAQ,YAAY;AAEvC,MAAAA,SAAQ,OAAO;AAAA,IAChB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACxE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,YAAY;AAAA,QACX,MAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,8BAA8B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC3E,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,eAAe;AAAA,QACd,KAAK;AAAA,MACN;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,MAAM,qBAAqB,OAAO,SAAS,OAAO,kBAAkB;AACnE,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,QAAM,EAAE,iBAAiB,IAAI,MAAM,OAAO;AAAA,IACzC,IAAI,sBAAsB;AAAA,MACzB,cAAc;AAAA,QACb,CAAC,QAAQ,SAAS,GAAG;AAAA,MACtB;AAAA,IACD,CAAC;AAAA,EACF;AACA,MAAI,mBAAmB,QAAQ,SAAS,GAAG,QAAQ;AAClD,QAAI,QAAQ,cAAc,QAAQ,eAAe;AAChD,YAAM,IAAI,MAAM,kDAAkD;AAAA,QACjE,OAAO;AAAA,UACN,GAAG;AAAA,UACH,uBAAuB,iBAAiB,QAAQ,SAAS,EAAE;AAAA,QAC5D;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,QAAQ,YAAY;AACvC,WAAO;AAAA,MACN;AAAA,MACA,iBAAiB,QAAQ,SAAS;AAAA,MAClC;AAAA,IACD;AAAA,EACD;AACA,UAAQ,aAAa;AACtB;AAEA,IAAO,mBAAQ;AAAA,EACd,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,eAAe;AAAA,EACf,kBAAkB;AACnB;", | ||
| "names": ["options"] | ||
| } |
+12
-16
@@ -9,12 +9,3 @@ import { | ||
| import { createWritableStream, timeout } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2", | ||
| "ca-central-1" | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new DynamoDBClient(awsClientDefaults); | ||
@@ -34,3 +25,3 @@ const awsDynamoDBSetClient = (ddbClient, _translateConfig) => { | ||
| options2.ExclusiveStartKey = response.LastEvaluatedKey; | ||
| expectMore = Object.keys(options2.ExclusiveStartKey).length; | ||
| expectMore = !!response.LastEvaluatedKey; | ||
| } | ||
@@ -49,3 +40,3 @@ } | ||
| options2.ExclusiveStartKey = response.LastEvaluatedKey; | ||
| expectMore = Object.keys(options2.ExclusiveStartKey).length; | ||
| expectMore = !!response.LastEvaluatedKey; | ||
| } | ||
@@ -56,2 +47,7 @@ } | ||
| const awsDynamoDBGetItemStream = async (options, _streamOptions = {}) => { | ||
| if (options.Keys?.length > 100) { | ||
| throw new Error( | ||
| `awsDynamoDBGetItemStream Keys.length (${options.Keys.length}) exceeds BatchGetItem limit of 100` | ||
| ); | ||
| } | ||
| options.retryCount ??= 0; | ||
@@ -89,3 +85,3 @@ options.retryMaxCount ??= 10; | ||
| }; | ||
| const awsDynamoDBPutItemStream = (options, streamOptions) => { | ||
| const awsDynamoDBPutItemStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -103,6 +99,6 @@ const write = async (chunk) => { | ||
| }; | ||
| const final = () => dynamodbBatchWrite(options, batch, streamOptions); | ||
| const final = () => batch.length ? dynamodbBatchWrite(options, batch, streamOptions) : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
| }; | ||
| const awsDynamoDBDeleteItemStream = (options, streamOptions) => { | ||
| const awsDynamoDBDeleteItemStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -120,3 +116,3 @@ const write = async (chunk) => { | ||
| }; | ||
| const final = () => dynamodbBatchWrite(options, batch, streamOptions); | ||
| const final = () => batch.length ? dynamodbBatchWrite(options, batch, streamOptions) : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
@@ -123,0 +119,0 @@ }; |
| { | ||
| "version": 3, | ||
| "sources": ["dynamodb.js"], | ||
| "sourcesContent": ["import {\n\tBatchGetItemCommand,\n\tBatchWriteItemCommand,\n\tDynamoDBClient,\n\tQueryCommand,\n\tScanCommand,\n} from \"@aws-sdk/client-dynamodb\";\nimport { createWritableStream, timeout } from \"@datastream/core\";\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t\t\"ca-central-1\",\n\t].includes(process.env.AWS_REGION),\n};\n\nlet client = new DynamoDBClient(awsClientDefaults);\nexport const awsDynamoDBSetClient = (ddbClient, _translateConfig) => {\n\tclient = ddbClient;\n};\nawsDynamoDBSetClient(client);\n\n// options = {TableName, ...}\n\nexport const awsDynamoDBQueryStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new QueryCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = Object.keys(options.ExclusiveStartKey).length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBScanStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ScanCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = Object.keys(options.ExclusiveStartKey).length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\n// TODO awsDynamoDBExecuteStatementStream\n\n// TODO max Keys.length = 100\nexport const awsDynamoDBGetItemStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tasync function* command(options) {\n\t\twhile (true) {\n\t\t\tconst response = await client.send(\n\t\t\t\tnew BatchGetItemCommand({\n\t\t\t\t\tRequestItems: {\n\t\t\t\t\t\t[options.TableName]: { Keys: options.Keys },\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t);\n\t\t\tfor (const item of response.Responses[options.TableName]) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst UnprocessedKeys =\n\t\t\t\tresponse?.UnprocessedKeys?.[options.TableName]?.Keys ?? [];\n\t\t\tif (!UnprocessedKeys.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\t\tthrow new Error(\"awsDynamoDBBatchGetItem has UnprocessedKeys\", {\n\t\t\t\t\tcause: {\n\t\t\t\t\t\t...options,\n\t\t\t\t\t\tUnprocessedKeysCount: UnprocessedKeys.length,\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\n\t\t\toptions.Keys = UnprocessedKeys;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBPutItemStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tPutRequest: {\n\t\t\t\tItem: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () => dynamodbBatchWrite(options, batch, streamOptions);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsDynamoDBDeleteItemStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tDeleteRequest: {\n\t\t\t\tKey: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () => dynamodbBatchWrite(options, batch, streamOptions);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nconst dynamodbBatchWrite = async (options, batch, streamOptions) => {\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tconst { UnprocessedItems } = await client.send(\n\t\tnew BatchWriteItemCommand({\n\t\t\tRequestItems: {\n\t\t\t\t[options.TableName]: batch,\n\t\t\t},\n\t\t}),\n\t);\n\tif (UnprocessedItems?.[options.TableName]?.length) {\n\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\tthrow new Error(\"awsDynamoDBBatchWriteItem has UnprocessedItems\", {\n\t\t\t\tcause: {\n\t\t\t\t\t...options,\n\t\t\t\t\tUnprocessedItemsCount: UnprocessedItems[options.TableName].length,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\t\treturn dynamodbBatchWrite(\n\t\t\toptions,\n\t\t\tUnprocessedItems[options.TableName],\n\t\t\tstreamOptions,\n\t\t);\n\t}\n\toptions.retryCount = 0; // reset for next batch\n};\n\nexport default {\n\tsetClient: awsDynamoDBSetClient,\n\tqueryStream: awsDynamoDBQueryStream,\n\tscanStream: awsDynamoDBScanStream,\n\tgetItemStream: awsDynamoDBGetItemStream,\n\tputItemStream: awsDynamoDBPutItemStream,\n\tdeleteItemStream: awsDynamoDBDeleteItemStream,\n};\n"], | ||
| "mappings": "AAAA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,sBAAsB,eAAe;AAE9C,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,SAAS,IAAI,eAAe,iBAAiB;AAC1C,MAAM,uBAAuB,CAAC,WAAW,qBAAqB;AACpE,WAAS;AACV;AACA,qBAAqB,MAAM;AAIpB,MAAM,yBAAyB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC7E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,aAAaA,QAAO,CAAC;AAC5D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,OAAO,KAAKA,SAAQ,iBAAiB,EAAE;AAAA,IACrD;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,wBAAwB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC5E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,YAAYA,QAAO,CAAC;AAC3D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,OAAO,KAAKA,SAAQ,iBAAiB,EAAE;AAAA,IACrD;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAKO,MAAM,2BAA2B,OACvC,SACA,iBAAiB,CAAC,MACd;AACJ,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,kBAAgB,QAAQA,UAAS;AAChC,WAAO,MAAM;AACZ,YAAM,WAAW,MAAM,OAAO;AAAA,QAC7B,IAAI,oBAAoB;AAAA,UACvB,cAAc;AAAA,YACb,CAACA,SAAQ,SAAS,GAAG,EAAE,MAAMA,SAAQ,KAAK;AAAA,UAC3C;AAAA,QACD,CAAC;AAAA,MACF;AACA,iBAAW,QAAQ,SAAS,UAAUA,SAAQ,SAAS,GAAG;AACzD,cAAM;AAAA,MACP;AACA,YAAM,kBACL,UAAU,kBAAkBA,SAAQ,SAAS,GAAG,QAAQ,CAAC;AAC1D,UAAI,CAAC,gBAAgB,QAAQ;AAC5B;AAAA,MACD;AAEA,UAAIA,SAAQ,cAAcA,SAAQ,eAAe;AAChD,cAAM,IAAI,MAAM,+CAA+C;AAAA,UAC9D,OAAO;AAAA,YACN,GAAGA;AAAA,YACH,sBAAsB,gBAAgB;AAAA,UACvC;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,QAAQ,KAAKA,SAAQ,YAAY;AAEvC,MAAAA,SAAQ,OAAO;AAAA,IAChB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,CAAC,SAAS,kBAAkB;AACnE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,YAAY;AAAA,QACX,MAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MAAM,mBAAmB,SAAS,OAAO,aAAa;AACpE,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,8BAA8B,CAAC,SAAS,kBAAkB;AACtE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,eAAe;AAAA,QACd,KAAK;AAAA,MACN;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MAAM,mBAAmB,SAAS,OAAO,aAAa;AACpE,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,MAAM,qBAAqB,OAAO,SAAS,OAAO,kBAAkB;AACnE,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,QAAM,EAAE,iBAAiB,IAAI,MAAM,OAAO;AAAA,IACzC,IAAI,sBAAsB;AAAA,MACzB,cAAc;AAAA,QACb,CAAC,QAAQ,SAAS,GAAG;AAAA,MACtB;AAAA,IACD,CAAC;AAAA,EACF;AACA,MAAI,mBAAmB,QAAQ,SAAS,GAAG,QAAQ;AAClD,QAAI,QAAQ,cAAc,QAAQ,eAAe;AAChD,YAAM,IAAI,MAAM,kDAAkD;AAAA,QACjE,OAAO;AAAA,UACN,GAAG;AAAA,UACH,uBAAuB,iBAAiB,QAAQ,SAAS,EAAE;AAAA,QAC5D;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,QAAQ,YAAY;AACvC,WAAO;AAAA,MACN;AAAA,MACA,iBAAiB,QAAQ,SAAS;AAAA,MAClC;AAAA,IACD;AAAA,EACD;AACA,UAAQ,aAAa;AACtB;AAEA,IAAO,mBAAQ;AAAA,EACd,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,eAAe;AAAA,EACf,kBAAkB;AACnB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tBatchGetItemCommand,\n\tBatchWriteItemCommand,\n\tDynamoDBClient,\n\tQueryCommand,\n\tScanCommand,\n} from \"@aws-sdk/client-dynamodb\";\nimport { createWritableStream, timeout } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new DynamoDBClient(awsClientDefaults);\nexport const awsDynamoDBSetClient = (ddbClient, _translateConfig) => {\n\tclient = ddbClient;\n};\nawsDynamoDBSetClient(client);\n\n// options = {TableName, ...}\n\nexport const awsDynamoDBQueryStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new QueryCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBScanStream = async (options, _streamOptions = {}) => {\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ScanCommand(options));\n\t\t\tfor (const item of response.Items) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\toptions.ExclusiveStartKey = response.LastEvaluatedKey;\n\t\t\texpectMore = !!response.LastEvaluatedKey;\n\t\t}\n\t}\n\treturn command(options);\n};\n\n// TODO awsDynamoDBExecuteStatementStream\n\nexport const awsDynamoDBGetItemStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\tif (options.Keys?.length > 100) {\n\t\tthrow new Error(\n\t\t\t`awsDynamoDBGetItemStream Keys.length (${options.Keys.length}) exceeds BatchGetItem limit of 100`,\n\t\t);\n\t}\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tasync function* command(options) {\n\t\twhile (true) {\n\t\t\tconst response = await client.send(\n\t\t\t\tnew BatchGetItemCommand({\n\t\t\t\t\tRequestItems: {\n\t\t\t\t\t\t[options.TableName]: { Keys: options.Keys },\n\t\t\t\t\t},\n\t\t\t\t}),\n\t\t\t);\n\t\t\tfor (const item of response.Responses[options.TableName]) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\tconst UnprocessedKeys =\n\t\t\t\tresponse?.UnprocessedKeys?.[options.TableName]?.Keys ?? [];\n\t\t\tif (!UnprocessedKeys.length) {\n\t\t\t\tbreak;\n\t\t\t}\n\n\t\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\t\tthrow new Error(\"awsDynamoDBBatchGetItem has UnprocessedKeys\", {\n\t\t\t\t\tcause: {\n\t\t\t\t\t\t...options,\n\t\t\t\t\t\tUnprocessedKeysCount: UnprocessedKeys.length,\n\t\t\t\t\t},\n\t\t\t\t});\n\t\t\t}\n\n\t\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\n\t\t\toptions.Keys = UnprocessedKeys;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsDynamoDBPutItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tPutRequest: {\n\t\t\t\tItem: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsDynamoDBDeleteItemStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 25) {\n\t\t\tawait dynamodbBatchWrite(options, batch, streamOptions);\n\t\t\tbatch = [];\n\t\t}\n\t\tbatch.push({\n\t\t\tDeleteRequest: {\n\t\t\t\tKey: chunk,\n\t\t\t},\n\t\t});\n\t};\n\tconst final = () =>\n\t\tbatch.length\n\t\t\t? dynamodbBatchWrite(options, batch, streamOptions)\n\t\t\t: undefined;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nconst dynamodbBatchWrite = async (options, batch, streamOptions) => {\n\toptions.retryCount ??= 0;\n\toptions.retryMaxCount ??= 10;\n\tconst { UnprocessedItems } = await client.send(\n\t\tnew BatchWriteItemCommand({\n\t\t\tRequestItems: {\n\t\t\t\t[options.TableName]: batch,\n\t\t\t},\n\t\t}),\n\t);\n\tif (UnprocessedItems?.[options.TableName]?.length) {\n\t\tif (options.retryCount >= options.retryMaxCount) {\n\t\t\tthrow new Error(\"awsDynamoDBBatchWriteItem has UnprocessedItems\", {\n\t\t\t\tcause: {\n\t\t\t\t\t...options,\n\t\t\t\t\tUnprocessedItemsCount: UnprocessedItems[options.TableName].length,\n\t\t\t\t},\n\t\t\t});\n\t\t}\n\n\t\tawait timeout(3 ** options.retryCount++); // 3^10 == 59sec\n\t\treturn dynamodbBatchWrite(\n\t\t\toptions,\n\t\t\tUnprocessedItems[options.TableName],\n\t\t\tstreamOptions,\n\t\t);\n\t}\n\toptions.retryCount = 0; // reset for next batch\n};\n\nexport default {\n\tsetClient: awsDynamoDBSetClient,\n\tqueryStream: awsDynamoDBQueryStream,\n\tscanStream: awsDynamoDBScanStream,\n\tgetItemStream: awsDynamoDBGetItemStream,\n\tputItemStream: awsDynamoDBPutItemStream,\n\tdeleteItemStream: awsDynamoDBDeleteItemStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,sBAAsB,eAAe;AAC9C,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,eAAe,iBAAiB;AAC1C,MAAM,uBAAuB,CAAC,WAAW,qBAAqB;AACpE,WAAS;AACV;AACA,qBAAqB,MAAM;AAIpB,MAAM,yBAAyB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC7E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,aAAaA,QAAO,CAAC;AAC5D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,wBAAwB,OAAO,SAAS,iBAAiB,CAAC,MAAM;AAC5E,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,YAAYA,QAAO,CAAC;AAC3D,iBAAW,QAAQ,SAAS,OAAO;AAClC,cAAM;AAAA,MACP;AACA,MAAAA,SAAQ,oBAAoB,SAAS;AACrC,mBAAa,CAAC,CAAC,SAAS;AAAA,IACzB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAIO,MAAM,2BAA2B,OACvC,SACA,iBAAiB,CAAC,MACd;AACJ,MAAI,QAAQ,MAAM,SAAS,KAAK;AAC/B,UAAM,IAAI;AAAA,MACT,yCAAyC,QAAQ,KAAK,MAAM;AAAA,IAC7D;AAAA,EACD;AACA,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,kBAAgB,QAAQA,UAAS;AAChC,WAAO,MAAM;AACZ,YAAM,WAAW,MAAM,OAAO;AAAA,QAC7B,IAAI,oBAAoB;AAAA,UACvB,cAAc;AAAA,YACb,CAACA,SAAQ,SAAS,GAAG,EAAE,MAAMA,SAAQ,KAAK;AAAA,UAC3C;AAAA,QACD,CAAC;AAAA,MACF;AACA,iBAAW,QAAQ,SAAS,UAAUA,SAAQ,SAAS,GAAG;AACzD,cAAM;AAAA,MACP;AACA,YAAM,kBACL,UAAU,kBAAkBA,SAAQ,SAAS,GAAG,QAAQ,CAAC;AAC1D,UAAI,CAAC,gBAAgB,QAAQ;AAC5B;AAAA,MACD;AAEA,UAAIA,SAAQ,cAAcA,SAAQ,eAAe;AAChD,cAAM,IAAI,MAAM,+CAA+C;AAAA,UAC9D,OAAO;AAAA,YACN,GAAGA;AAAA,YACH,sBAAsB,gBAAgB;AAAA,UACvC;AAAA,QACD,CAAC;AAAA,MACF;AAEA,YAAM,QAAQ,KAAKA,SAAQ,YAAY;AAEvC,MAAAA,SAAQ,OAAO;AAAA,IAChB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,2BAA2B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACxE,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,YAAY;AAAA,QACX,MAAM;AAAA,MACP;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,8BAA8B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC3E,MAAI,QAAQ,CAAC;AACb,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,mBAAmB,SAAS,OAAO,aAAa;AACtD,cAAQ,CAAC;AAAA,IACV;AACA,UAAM,KAAK;AAAA,MACV,eAAe;AAAA,QACd,KAAK;AAAA,MACN;AAAA,IACD,CAAC;AAAA,EACF;AACA,QAAM,QAAQ,MACb,MAAM,SACH,mBAAmB,SAAS,OAAO,aAAa,IAChD;AACJ,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,MAAM,qBAAqB,OAAO,SAAS,OAAO,kBAAkB;AACnE,UAAQ,eAAe;AACvB,UAAQ,kBAAkB;AAC1B,QAAM,EAAE,iBAAiB,IAAI,MAAM,OAAO;AAAA,IACzC,IAAI,sBAAsB;AAAA,MACzB,cAAc;AAAA,QACb,CAAC,QAAQ,SAAS,GAAG;AAAA,MACtB;AAAA,IACD,CAAC;AAAA,EACF;AACA,MAAI,mBAAmB,QAAQ,SAAS,GAAG,QAAQ;AAClD,QAAI,QAAQ,cAAc,QAAQ,eAAe;AAChD,YAAM,IAAI,MAAM,kDAAkD;AAAA,QACjE,OAAO;AAAA,UACN,GAAG;AAAA,UACH,uBAAuB,iBAAiB,QAAQ,SAAS,EAAE;AAAA,QAC5D;AAAA,MACD,CAAC;AAAA,IACF;AAEA,UAAM,QAAQ,KAAK,QAAQ,YAAY;AACvC,WAAO;AAAA,MACN;AAAA,MACA,iBAAiB,QAAQ,SAAS;AAAA,MAClC;AAAA,IACD;AAAA,EACD;AACA,UAAQ,aAAa;AACtB;AAEA,IAAO,mBAAQ;AAAA,EACd,WAAW;AAAA,EACX,aAAa;AAAA,EACb,YAAY;AAAA,EACZ,eAAe;AAAA,EACf,eAAe;AAAA,EACf,kBAAkB;AACnB;", | ||
| "names": ["options"] | ||
| } |
| { | ||
| "version": 3, | ||
| "sources": ["index.js"], | ||
| "sourcesContent": ["export * from \"@datastream/aws/dynamodb\";\nexport * from \"@datastream/aws/lambda\";\nexport * from \"@datastream/aws/s3\";\nexport * from \"@datastream/aws/sns\";\nexport * from \"@datastream/aws/sqs\";\n"], | ||
| "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nexport * from \"@datastream/aws/dynamodb\";\nexport * from \"@datastream/aws/lambda\";\nexport * from \"@datastream/aws/s3\";\nexport * from \"@datastream/aws/sns\";\nexport * from \"@datastream/aws/sqs\";\n"], | ||
| "mappings": "AAEA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;", | ||
| "names": [] | ||
| } |
| { | ||
| "version": 3, | ||
| "sources": ["index.js"], | ||
| "sourcesContent": ["export * from \"@datastream/aws/dynamodb\";\nexport * from \"@datastream/aws/lambda\";\nexport * from \"@datastream/aws/s3\";\nexport * from \"@datastream/aws/sns\";\nexport * from \"@datastream/aws/sqs\";\n"], | ||
| "mappings": "AAAA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nexport * from \"@datastream/aws/dynamodb\";\nexport * from \"@datastream/aws/lambda\";\nexport * from \"@datastream/aws/s3\";\nexport * from \"@datastream/aws/sns\";\nexport * from \"@datastream/aws/sqs\";\n"], | ||
| "mappings": "AAEA,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;AACd,cAAc;", | ||
| "names": [] | ||
| } |
+2
-11
@@ -6,12 +6,3 @@ import { | ||
| import { createReadableStream } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2" | ||
| // 'ca-central-1' | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let defaultClient = new LambdaClient(awsClientDefaults); | ||
@@ -21,3 +12,3 @@ const awsLambdaSetClient = (lambdaClient) => { | ||
| }; | ||
| const awsLambdaReadableStream = (lambdaOptions, streamOptions) => { | ||
| const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => { | ||
| return createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions); | ||
@@ -24,0 +15,0 @@ }; |
| { | ||
| "version": 3, | ||
| "sources": ["lambda.js"], | ||
| "sourcesContent": ["import {\n\tInvokeWithResponseStreamCommand,\n\tLambdaClient,\n} from \"@aws-sdk/client-lambda\";\nimport { createReadableStream } from \"@datastream/core\";\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t\t// 'ca-central-1'\n\t].includes(process.env.AWS_REGION),\n};\n\nlet defaultClient = new LambdaClient(awsClientDefaults);\nexport const awsLambdaSetClient = (lambdaClient) => {\n\tdefaultClient = lambdaClient;\n};\n\nexport const awsLambdaReadableStream = (lambdaOptions, streamOptions) => {\n\treturn createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions);\n};\nexport const awsLambdaResponseStream = awsLambdaReadableStream;\n\nasync function* awsLambdaGenerator(lambdaOptions, _streamOptions = {}) {\n\tif (!Array.isArray(lambdaOptions)) {\n\t\tlambdaOptions = [lambdaOptions];\n\t}\n\tfor (const options of lambdaOptions) {\n\t\tconst response = await defaultClient.send(\n\t\t\tnew InvokeWithResponseStreamCommand(options),\n\t\t);\n\t\tfor await (const chunk of response.EventStream) {\n\t\t\tif (chunk?.PayloadChunk?.Payload) {\n\t\t\t\tyield chunk.PayloadChunk.Payload;\n\t\t\t} else if (chunk?.InvokeComplete?.ErrorCode) {\n\t\t\t\tthrow new Error(chunk.InvokeComplete.ErrorCode, {\n\t\t\t\t\tcause: chunk.InvokeComplete.ErrorDetails,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n}\n\nexport default {\n\tsetClient: awsLambdaSetClient,\n\treadableStream: awsLambdaReadableStream,\n\tresponseStream: awsLambdaReadableStream,\n};\n"], | ||
| "mappings": "AAAA;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AAErC,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,EAED,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,gBAAgB,IAAI,aAAa,iBAAiB;AAC/C,MAAM,qBAAqB,CAAC,iBAAiB;AACnD,kBAAgB;AACjB;AAEO,MAAM,0BAA0B,CAAC,eAAe,kBAAkB;AACxE,SAAO,qBAAqB,mBAAmB,aAAa,GAAG,aAAa;AAC7E;AACO,MAAM,0BAA0B;AAEvC,gBAAgB,mBAAmB,eAAe,iBAAiB,CAAC,GAAG;AACtE,MAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AAClC,oBAAgB,CAAC,aAAa;AAAA,EAC/B;AACA,aAAW,WAAW,eAAe;AACpC,UAAM,WAAW,MAAM,cAAc;AAAA,MACpC,IAAI,gCAAgC,OAAO;AAAA,IAC5C;AACA,qBAAiB,SAAS,SAAS,aAAa;AAC/C,UAAI,OAAO,cAAc,SAAS;AACjC,cAAM,MAAM,aAAa;AAAA,MAC1B,WAAW,OAAO,gBAAgB,WAAW;AAC5C,cAAM,IAAI,MAAM,MAAM,eAAe,WAAW;AAAA,UAC/C,OAAO,MAAM,eAAe;AAAA,QAC7B,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AACD;AAEA,IAAO,iBAAQ;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tInvokeWithResponseStreamCommand,\n\tLambdaClient,\n} from \"@aws-sdk/client-lambda\";\nimport { createReadableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new LambdaClient(awsClientDefaults);\nexport const awsLambdaSetClient = (lambdaClient) => {\n\tdefaultClient = lambdaClient;\n};\n\nexport const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => {\n\treturn createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions);\n};\nexport const awsLambdaResponseStream = awsLambdaReadableStream;\n\nasync function* awsLambdaGenerator(lambdaOptions, _streamOptions = {}) {\n\tif (!Array.isArray(lambdaOptions)) {\n\t\tlambdaOptions = [lambdaOptions];\n\t}\n\tfor (const options of lambdaOptions) {\n\t\tconst response = await defaultClient.send(\n\t\t\tnew InvokeWithResponseStreamCommand(options),\n\t\t);\n\t\tfor await (const chunk of response.EventStream) {\n\t\t\tif (chunk?.PayloadChunk?.Payload) {\n\t\t\t\tyield chunk.PayloadChunk.Payload;\n\t\t\t} else if (chunk?.InvokeComplete?.ErrorCode) {\n\t\t\t\tthrow new Error(chunk.InvokeComplete.ErrorCode, {\n\t\t\t\t\tcause: chunk.InvokeComplete.ErrorDetails,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n}\n\nexport default {\n\tsetClient: awsLambdaSetClient,\n\treadableStream: awsLambdaReadableStream,\n\tresponseStream: awsLambdaReadableStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,aAAa,iBAAiB;AAC/C,MAAM,qBAAqB,CAAC,iBAAiB;AACnD,kBAAgB;AACjB;AAEO,MAAM,0BAA0B,CAAC,eAAe,gBAAgB,CAAC,MAAM;AAC7E,SAAO,qBAAqB,mBAAmB,aAAa,GAAG,aAAa;AAC7E;AACO,MAAM,0BAA0B;AAEvC,gBAAgB,mBAAmB,eAAe,iBAAiB,CAAC,GAAG;AACtE,MAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AAClC,oBAAgB,CAAC,aAAa;AAAA,EAC/B;AACA,aAAW,WAAW,eAAe;AACpC,UAAM,WAAW,MAAM,cAAc;AAAA,MACpC,IAAI,gCAAgC,OAAO;AAAA,IAC5C;AACA,qBAAiB,SAAS,SAAS,aAAa;AAC/C,UAAI,OAAO,cAAc,SAAS;AACjC,cAAM,MAAM,aAAa;AAAA,MAC1B,WAAW,OAAO,gBAAgB,WAAW;AAC5C,cAAM,IAAI,MAAM,MAAM,eAAe,WAAW;AAAA,UAC/C,OAAO,MAAM,eAAe;AAAA,QAC7B,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AACD;AAEA,IAAO,iBAAQ;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;", | ||
| "names": [] | ||
| } |
+2
-11
@@ -6,12 +6,3 @@ import { | ||
| import { createReadableStream } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2" | ||
| // 'ca-central-1' | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let defaultClient = new LambdaClient(awsClientDefaults); | ||
@@ -21,3 +12,3 @@ const awsLambdaSetClient = (lambdaClient) => { | ||
| }; | ||
| const awsLambdaReadableStream = (lambdaOptions, streamOptions) => { | ||
| const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => { | ||
| return createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions); | ||
@@ -24,0 +15,0 @@ }; |
| { | ||
| "version": 3, | ||
| "sources": ["lambda.js"], | ||
| "sourcesContent": ["import {\n\tInvokeWithResponseStreamCommand,\n\tLambdaClient,\n} from \"@aws-sdk/client-lambda\";\nimport { createReadableStream } from \"@datastream/core\";\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t\t// 'ca-central-1'\n\t].includes(process.env.AWS_REGION),\n};\n\nlet defaultClient = new LambdaClient(awsClientDefaults);\nexport const awsLambdaSetClient = (lambdaClient) => {\n\tdefaultClient = lambdaClient;\n};\n\nexport const awsLambdaReadableStream = (lambdaOptions, streamOptions) => {\n\treturn createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions);\n};\nexport const awsLambdaResponseStream = awsLambdaReadableStream;\n\nasync function* awsLambdaGenerator(lambdaOptions, _streamOptions = {}) {\n\tif (!Array.isArray(lambdaOptions)) {\n\t\tlambdaOptions = [lambdaOptions];\n\t}\n\tfor (const options of lambdaOptions) {\n\t\tconst response = await defaultClient.send(\n\t\t\tnew InvokeWithResponseStreamCommand(options),\n\t\t);\n\t\tfor await (const chunk of response.EventStream) {\n\t\t\tif (chunk?.PayloadChunk?.Payload) {\n\t\t\t\tyield chunk.PayloadChunk.Payload;\n\t\t\t} else if (chunk?.InvokeComplete?.ErrorCode) {\n\t\t\t\tthrow new Error(chunk.InvokeComplete.ErrorCode, {\n\t\t\t\t\tcause: chunk.InvokeComplete.ErrorDetails,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n}\n\nexport default {\n\tsetClient: awsLambdaSetClient,\n\treadableStream: awsLambdaReadableStream,\n\tresponseStream: awsLambdaReadableStream,\n};\n"], | ||
| "mappings": "AAAA;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AAErC,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA;AAAA,EAED,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,gBAAgB,IAAI,aAAa,iBAAiB;AAC/C,MAAM,qBAAqB,CAAC,iBAAiB;AACnD,kBAAgB;AACjB;AAEO,MAAM,0BAA0B,CAAC,eAAe,kBAAkB;AACxE,SAAO,qBAAqB,mBAAmB,aAAa,GAAG,aAAa;AAC7E;AACO,MAAM,0BAA0B;AAEvC,gBAAgB,mBAAmB,eAAe,iBAAiB,CAAC,GAAG;AACtE,MAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AAClC,oBAAgB,CAAC,aAAa;AAAA,EAC/B;AACA,aAAW,WAAW,eAAe;AACpC,UAAM,WAAW,MAAM,cAAc;AAAA,MACpC,IAAI,gCAAgC,OAAO;AAAA,IAC5C;AACA,qBAAiB,SAAS,SAAS,aAAa;AAC/C,UAAI,OAAO,cAAc,SAAS;AACjC,cAAM,MAAM,aAAa;AAAA,MAC1B,WAAW,OAAO,gBAAgB,WAAW;AAC5C,cAAM,IAAI,MAAM,MAAM,eAAe,WAAW;AAAA,UAC/C,OAAO,MAAM,eAAe;AAAA,QAC7B,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AACD;AAEA,IAAO,iBAAQ;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tInvokeWithResponseStreamCommand,\n\tLambdaClient,\n} from \"@aws-sdk/client-lambda\";\nimport { createReadableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new LambdaClient(awsClientDefaults);\nexport const awsLambdaSetClient = (lambdaClient) => {\n\tdefaultClient = lambdaClient;\n};\n\nexport const awsLambdaReadableStream = (lambdaOptions, streamOptions = {}) => {\n\treturn createReadableStream(awsLambdaGenerator(lambdaOptions), streamOptions);\n};\nexport const awsLambdaResponseStream = awsLambdaReadableStream;\n\nasync function* awsLambdaGenerator(lambdaOptions, _streamOptions = {}) {\n\tif (!Array.isArray(lambdaOptions)) {\n\t\tlambdaOptions = [lambdaOptions];\n\t}\n\tfor (const options of lambdaOptions) {\n\t\tconst response = await defaultClient.send(\n\t\t\tnew InvokeWithResponseStreamCommand(options),\n\t\t);\n\t\tfor await (const chunk of response.EventStream) {\n\t\t\tif (chunk?.PayloadChunk?.Payload) {\n\t\t\t\tyield chunk.PayloadChunk.Payload;\n\t\t\t} else if (chunk?.InvokeComplete?.ErrorCode) {\n\t\t\t\tthrow new Error(chunk.InvokeComplete.ErrorCode, {\n\t\t\t\t\tcause: chunk.InvokeComplete.ErrorDetails,\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t}\n}\n\nexport default {\n\tsetClient: awsLambdaSetClient,\n\treadableStream: awsLambdaReadableStream,\n\tresponseStream: awsLambdaReadableStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,aAAa,iBAAiB;AAC/C,MAAM,qBAAqB,CAAC,iBAAiB;AACnD,kBAAgB;AACjB;AAEO,MAAM,0BAA0B,CAAC,eAAe,gBAAgB,CAAC,MAAM;AAC7E,SAAO,qBAAqB,mBAAmB,aAAa,GAAG,aAAa;AAC7E;AACO,MAAM,0BAA0B;AAEvC,gBAAgB,mBAAmB,eAAe,iBAAiB,CAAC,GAAG;AACtE,MAAI,CAAC,MAAM,QAAQ,aAAa,GAAG;AAClC,oBAAgB,CAAC,aAAa;AAAA,EAC/B;AACA,aAAW,WAAW,eAAe;AACpC,UAAM,WAAW,MAAM,cAAc;AAAA,MACpC,IAAI,gCAAgC,OAAO;AAAA,IAC5C;AACA,qBAAiB,SAAS,SAAS,aAAa;AAC/C,UAAI,OAAO,cAAc,SAAS;AACjC,cAAM,MAAM,aAAa;AAAA,MAC1B,WAAW,OAAO,gBAAgB,WAAW;AAC5C,cAAM,IAAI,MAAM,MAAM,eAAe,WAAW;AAAA,UAC/C,OAAO,MAAM,eAAe;AAAA,QAC7B,CAAC;AAAA,MACF;AAAA,IACD;AAAA,EACD;AACD;AAEA,IAAO,iBAAQ;AAAA,EACd,WAAW;AAAA,EACX,gBAAgB;AAAA,EAChB,gBAAgB;AACjB;", | ||
| "names": [] | ||
| } |
+36
-31
| { | ||
| "name": "@datastream/aws", | ||
| "version": "0.0.42", | ||
| "description": "", | ||
| "version": "0.1.4", | ||
| "description": "AWS service streaming integrations for S3, DynamoDB, Lambda, SNS, and SQS", | ||
| "type": "module", | ||
@@ -13,3 +13,3 @@ "engines": { | ||
| }, | ||
| "main": "./index.web.mjs", | ||
| "main": "./index.node.mjs", | ||
| "module": "./index.web.mjs", | ||
@@ -22,6 +22,2 @@ "exports": { | ||
| "default": "./index.node.mjs" | ||
| }, | ||
| "__require": { | ||
| "types": "./index.d.ts", | ||
| "default": "./index.node.cjs" | ||
| } | ||
@@ -39,6 +35,2 @@ }, | ||
| "default": "./dynamodb.node.mjs" | ||
| }, | ||
| "__require": { | ||
| "types": "./dynamodb.d.ts", | ||
| "default": "./dynamodb.node.cjs" | ||
| } | ||
@@ -56,6 +48,2 @@ }, | ||
| "default": "./lambda.node.mjs" | ||
| }, | ||
| "__require": { | ||
| "types": "./lambda.d.ts", | ||
| "default": "./lambda.node.cjs" | ||
| } | ||
@@ -73,6 +61,2 @@ }, | ||
| "default": "./s3.node.mjs" | ||
| }, | ||
| "__require": { | ||
| "types": "./s3.d.ts", | ||
| "default": "./s3.node.cjs" | ||
| } | ||
@@ -90,6 +74,2 @@ }, | ||
| "default": "./sns.node.mjs" | ||
| }, | ||
| "__require": { | ||
| "types": "./sns.d.ts", | ||
| "default": "./sns.node.cjs" | ||
| } | ||
@@ -107,6 +87,2 @@ }, | ||
| "default": "./sqs.node.mjs" | ||
| }, | ||
| "__require": { | ||
| "types": "./sqs.d.ts", | ||
| "default": "./sqs.node.cjs" | ||
| } | ||
@@ -123,3 +99,2 @@ }, | ||
| "*.mjs", | ||
| "*.cjs", | ||
| "*.map", | ||
@@ -158,6 +133,36 @@ "*.d.ts" | ||
| "dependencies": { | ||
| "@datastream/aws": "0.0.42", | ||
| "@datastream/core": "0.0.42" | ||
| "@datastream/core": "0.1.4" | ||
| }, | ||
| "gitHead": "69f6650ffe7b31dba8e4b96a3cc5d1e95a1fa971", | ||
| "peerDependencies": { | ||
| "@aws-sdk/client-dynamodb": "^3.0.0", | ||
| "@aws-sdk/client-lambda": "^3.0.0", | ||
| "@aws-sdk/client-s3": "^3.0.0", | ||
| "@aws-sdk/client-sns": "^3.0.0", | ||
| "@aws-sdk/client-sqs": "^3.0.0", | ||
| "@aws-sdk/client-ssm": "^3.0.0", | ||
| "@aws-sdk/lib-storage": "^3.0.0" | ||
| }, | ||
| "peerDependenciesMeta": { | ||
| "@aws-sdk/client-dynamodb": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/client-lambda": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/client-s3": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/client-sns": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/client-sqs": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/client-ssm": { | ||
| "optional": true | ||
| }, | ||
| "@aws-sdk/lib-storage": { | ||
| "optional": true | ||
| } | ||
| }, | ||
| "devDependencies": { | ||
@@ -164,0 +169,0 @@ "@aws-sdk/client-dynamodb": "^3.0.0", |
+10
-15
@@ -7,12 +7,3 @@ import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; | ||
| } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2", | ||
| "ca-central-1" | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let defaultClient = new S3Client(awsClientDefaults); | ||
@@ -22,3 +13,3 @@ const awsS3SetClient = (s3Client) => { | ||
| }; | ||
| const awsS3GetObjectStream = async (options, streamOptions) => { | ||
| const awsS3GetObjectStream = async (options, streamOptions = {}) => { | ||
| const { client, ...params } = options; | ||
@@ -29,7 +20,7 @@ const { Body } = await (client ?? defaultClient).send( | ||
| if (!Body) { | ||
| throw new Error("S3.GetObject not Found", { cause: params }); | ||
| throw new Error("S3.GetObject not found", { cause: params }); | ||
| } | ||
| return createReadableStream(Body, streamOptions); | ||
| }; | ||
| const awsS3PutObjectStream = (options, streamOptions) => { | ||
| const awsS3PutObjectStream = (options, streamOptions = {}) => { | ||
| const { onProgress, client, tags, ...params } = options; | ||
@@ -60,2 +51,4 @@ const stream = createPassThroughStream(() => { | ||
| const algorithm = _algorithms[ChecksumAlgorithm]; | ||
| if (!algorithm) | ||
| throw new Error(`Unsupported ChecksumAlgorithm: ${ChecksumAlgorithm}`); | ||
| let checksums = []; | ||
@@ -65,3 +58,3 @@ let bytes = new Uint8Array(0); | ||
| if (typeof chunk === "string") { | ||
| chunk = new TextEncoder("utf-8").encode(chunk); | ||
| chunk = new TextEncoder().encode(chunk); | ||
| } | ||
@@ -94,4 +87,6 @@ while (bytes.byteLength + chunk.byteLength > partSize) { | ||
| checksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`; | ||
| } else if (checksums.length === 1) { | ||
| checksum = _arrayBufferToBase64(checksums[0]); | ||
| } else { | ||
| checksum = _arrayBufferToBase64(checksums[0]); | ||
| checksum = ""; | ||
| } | ||
@@ -98,0 +93,0 @@ checksums = checksums.map(_arrayBufferToBase64); |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["s3.js"], | ||
| "sourcesContent": ["/* global crypto */\n\nimport { GetObjectCommand, S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport {\n\tcreatePassThroughStream,\n\tcreateReadableStream,\n} from \"@datastream/core\";\n\n// import { Crc32 } from '@aws-crypto/crc32';\n// import { Crc32c } from '@aws-crypto/crc32c';\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t\t\"ca-central-1\",\n\t].includes(process.env.AWS_REGION),\n};\n\nlet defaultClient = new S3Client(awsClientDefaults);\nexport const awsS3SetClient = (s3Client) => {\n\tdefaultClient = s3Client;\n};\n\nexport const awsS3GetObjectStream = async (options, streamOptions) => {\n\tconst { client, ...params } = options;\n\tconst { Body } = await (client ?? defaultClient).send(\n\t\tnew GetObjectCommand(params),\n\t);\n\tif (!Body) {\n\t\tthrow new Error(\"S3.GetObject not Found\", { cause: params });\n\t}\n\treturn createReadableStream(Body, streamOptions);\n};\n\nexport const awsS3PutObjectStream = (options, streamOptions) => {\n\tconst { onProgress, client, tags, ...params } = options;\n\tconst stream = createPassThroughStream(() => {}, streamOptions);\n\tconst upload = new Upload({\n\t\tclient: client ?? defaultClient,\n\t\tparams: {\n\t\t\t...params,\n\t\t\tBody: stream,\n\t\t},\n\t\ttags,\n\t});\n\tif (onProgress) {\n\t\tstream.on(\"httpUploadProgress\", onProgress);\n\t}\n\tconst result = upload.done();\n\n\tstream.result = async () => {\n\t\tawait result;\n\t\treturn {};\n\t};\n\treturn stream;\n};\n\n// This is designed to be used in the browser on a file that you want to upload via a presigned URL\n// partSize; magic number, no 16MB mentioned in the docs\nexport const awsS3ChecksumStream = (\n\t{ ChecksumAlgorithm, partSize, resultKey } = {},\n\tstreamOptions = {},\n) => {\n\tChecksumAlgorithm ??= \"SHA256\";\n\tpartSize ??= 17_179_870;\n\tconst algorithm = _algorithms[ChecksumAlgorithm];\n\tlet checksums = [];\n\tlet bytes = new Uint8Array(0);\n\tconst passThrough = async (chunk) => {\n\t\tif (typeof chunk === \"string\") {\n\t\t\tchunk = new TextEncoder(\"utf-8\").encode(chunk);\n\t\t}\n\t\twhile (bytes.byteLength + chunk.byteLength > partSize) {\n\t\t\tchunk = _concatBuffers([bytes, chunk]);\n\t\t\tconst prefixChunk = chunk.slice(0, partSize);\n\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, prefixChunk);\n\t\t\tchecksums.push(checksum);\n\t\t\tchunk = chunk.slice(prefixChunk.byteLength);\n\n\t\t\tbytes = new Uint8Array(0);\n\t\t}\n\t\tbytes = _concatBuffers([bytes, chunk]);\n\t};\n\tconst flush = async () => {\n\t\tif (bytes.byteLength) {\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, bytes);\n\t\t\tchecksums.push(checksum);\n\t\t}\n\t};\n\tconst stream = createPassThroughStream(passThrough, flush, streamOptions);\n\tlet checksum;\n\tstream.result = async () => {\n\t\tif (!checksum) {\n\t\t\tif (checksums.length > 1) {\n\t\t\t\tchecksum = await crypto.subtle.digest(\n\t\t\t\t\talgorithm,\n\t\t\t\t\t_concatBuffers(checksums),\n\t\t\t\t);\n\t\t\t\tchecksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`;\n\t\t\t} else {\n\t\t\t\tchecksum = _arrayBufferToBase64(checksums[0]);\n\t\t\t}\n\t\t\tchecksums = checksums.map(_arrayBufferToBase64);\n\t\t}\n\t\treturn {\n\t\t\tkey: resultKey ?? \"s3\",\n\t\t\tvalue: { checksum, checksums, partSize },\n\t\t};\n\t};\n\treturn stream;\n};\n\nconst _algorithms = {\n\t// AWS_NAME: NODE_NAME\n\tSHA1: \"SHA-1\",\n\tSHA256: \"SHA-256\",\n\t// CRC32: '',\n\t// CRC32C: '',\n};\nconst _concatBuffers = (buffers) => {\n\tconst tmp = new Uint8Array(\n\t\tbuffers.reduce((byteLength, buffer) => byteLength + buffer.byteLength, 0),\n\t);\n\tlet byteLength = 0;\n\tfor (let i = 0, l = buffers.length; i < l; i++) {\n\t\ttmp.set(new Uint8Array(buffers[i]), byteLength);\n\t\tbyteLength += buffers[i].byteLength;\n\t}\n\treturn tmp.buffer;\n};\nconst _arrayBufferToBase64 = (buffer) => {\n\tlet binary = \"\";\n\tconst bytes = new Uint8Array(buffer);\n\tconst len = bytes.byteLength;\n\tfor (let i = 0; i < len; i++) {\n\t\tbinary += String.fromCharCode(bytes[i]);\n\t}\n\treturn btoa(binary);\n};\n\nexport default {\n\tsetClient: awsS3SetClient,\n\tgetObjectStream: awsS3GetObjectStream,\n\tputObjectStream: awsS3PutObjectStream,\n\tchecksumStream: awsS3ChecksumStream,\n};\n"], | ||
| "mappings": "AAEA,SAAS,kBAAkB,gBAAgB;AAC3C,SAAS,cAAc;AACvB;AAAA,EACC;AAAA,EACA;AAAA,OACM;AAKP,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,gBAAgB,IAAI,SAAS,iBAAiB;AAC3C,MAAM,iBAAiB,CAAC,aAAa;AAC3C,kBAAgB;AACjB;AAEO,MAAM,uBAAuB,OAAO,SAAS,kBAAkB;AACrE,QAAM,EAAE,QAAQ,GAAG,OAAO,IAAI;AAC9B,QAAM,EAAE,KAAK,IAAI,OAAO,UAAU,eAAe;AAAA,IAChD,IAAI,iBAAiB,MAAM;AAAA,EAC5B;AACA,MAAI,CAAC,MAAM;AACV,UAAM,IAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,CAAC;AAAA,EAC5D;AACA,SAAO,qBAAqB,MAAM,aAAa;AAChD;AAEO,MAAM,uBAAuB,CAAC,SAAS,kBAAkB;AAC/D,QAAM,EAAE,YAAY,QAAQ,MAAM,GAAG,OAAO,IAAI;AAChD,QAAM,SAAS,wBAAwB,MAAM;AAAA,EAAC,GAAG,aAAa;AAC9D,QAAM,SAAS,IAAI,OAAO;AAAA,IACzB,QAAQ,UAAU;AAAA,IAClB,QAAQ;AAAA,MACP,GAAG;AAAA,MACH,MAAM;AAAA,IACP;AAAA,IACA;AAAA,EACD,CAAC;AACD,MAAI,YAAY;AACf,WAAO,GAAG,sBAAsB,UAAU;AAAA,EAC3C;AACA,QAAM,SAAS,OAAO,KAAK;AAE3B,SAAO,SAAS,YAAY;AAC3B,UAAM;AACN,WAAO,CAAC;AAAA,EACT;AACA,SAAO;AACR;AAIO,MAAM,sBAAsB,CAClC,EAAE,mBAAmB,UAAU,UAAU,IAAI,CAAC,GAC9C,gBAAgB,CAAC,MACb;AACJ,wBAAsB;AACtB,eAAa;AACb,QAAM,YAAY,YAAY,iBAAiB;AAC/C,MAAI,YAAY,CAAC;AACjB,MAAI,QAAQ,IAAI,WAAW,CAAC;AAC5B,QAAM,cAAc,OAAO,UAAU;AACpC,QAAI,OAAO,UAAU,UAAU;AAC9B,cAAQ,IAAI,YAAY,OAAO,EAAE,OAAO,KAAK;AAAA,IAC9C;AACA,WAAO,MAAM,aAAa,MAAM,aAAa,UAAU;AACtD,cAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AACrC,YAAM,cAAc,MAAM,MAAM,GAAG,QAAQ;AAE3C,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,WAAW;AAClE,gBAAU,KAAKA,SAAQ;AACvB,cAAQ,MAAM,MAAM,YAAY,UAAU;AAE1C,cAAQ,IAAI,WAAW,CAAC;AAAA,IACzB;AACA,YAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AAAA,EACtC;AACA,QAAM,QAAQ,YAAY;AACzB,QAAI,MAAM,YAAY;AACrB,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AAC5D,gBAAU,KAAKA,SAAQ;AAAA,IACxB;AAAA,EACD;AACA,QAAM,SAAS,wBAAwB,aAAa,OAAO,aAAa;AACxE,MAAI;AACJ,SAAO,SAAS,YAAY;AAC3B,QAAI,CAAC,UAAU;AACd,UAAI,UAAU,SAAS,GAAG;AACzB,mBAAW,MAAM,OAAO,OAAO;AAAA,UAC9B;AAAA,UACA,eAAe,SAAS;AAAA,QACzB;AACA,mBAAW,GAAG,qBAAqB,QAAQ,CAAC,IAAI,UAAU,MAAM;AAAA,MACjE,OAAO;AACN,mBAAW,qBAAqB,UAAU,CAAC,CAAC;AAAA,MAC7C;AACA,kBAAY,UAAU,IAAI,oBAAoB;AAAA,IAC/C;AACA,WAAO;AAAA,MACN,KAAK,aAAa;AAAA,MAClB,OAAO,EAAE,UAAU,WAAW,SAAS;AAAA,IACxC;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,cAAc;AAAA;AAAA,EAEnB,MAAM;AAAA,EACN,QAAQ;AAAA;AAAA;AAGT;AACA,MAAM,iBAAiB,CAAC,YAAY;AACnC,QAAM,MAAM,IAAI;AAAA,IACf,QAAQ,OAAO,CAACC,aAAY,WAAWA,cAAa,OAAO,YAAY,CAAC;AAAA,EACzE;AACA,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAAI,GAAG,KAAK;AAC/C,QAAI,IAAI,IAAI,WAAW,QAAQ,CAAC,CAAC,GAAG,UAAU;AAC9C,kBAAc,QAAQ,CAAC,EAAE;AAAA,EAC1B;AACA,SAAO,IAAI;AACZ;AACA,MAAM,uBAAuB,CAAC,WAAW;AACxC,MAAI,SAAS;AACb,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,MAAM,MAAM;AAClB,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAU,OAAO,aAAa,MAAM,CAAC,CAAC;AAAA,EACvC;AACA,SAAO,KAAK,MAAM;AACnB;AAEA,IAAO,aAAQ;AAAA,EACd,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,gBAAgB;AACjB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global crypto */\n\nimport { GetObjectCommand, S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport {\n\tcreatePassThroughStream,\n\tcreateReadableStream,\n} from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new S3Client(awsClientDefaults);\nexport const awsS3SetClient = (s3Client) => {\n\tdefaultClient = s3Client;\n};\n\nexport const awsS3GetObjectStream = async (options, streamOptions = {}) => {\n\tconst { client, ...params } = options;\n\tconst { Body } = await (client ?? defaultClient).send(\n\t\tnew GetObjectCommand(params),\n\t);\n\tif (!Body) {\n\t\tthrow new Error(\"S3.GetObject not found\", { cause: params });\n\t}\n\treturn createReadableStream(Body, streamOptions);\n};\n\nexport const awsS3PutObjectStream = (options, streamOptions = {}) => {\n\tconst { onProgress, client, tags, ...params } = options;\n\tconst stream = createPassThroughStream(() => {}, streamOptions);\n\tconst upload = new Upload({\n\t\tclient: client ?? defaultClient,\n\t\tparams: {\n\t\t\t...params,\n\t\t\tBody: stream,\n\t\t},\n\t\ttags,\n\t});\n\tif (onProgress) {\n\t\tstream.on(\"httpUploadProgress\", onProgress);\n\t}\n\tconst result = upload.done();\n\n\tstream.result = async () => {\n\t\tawait result;\n\t\treturn {};\n\t};\n\treturn stream;\n};\n\n// This is designed to be used in the browser on a file that you want to upload via a presigned URL\n// partSize; magic number, no 16MB mentioned in the docs\nexport const awsS3ChecksumStream = (\n\t{ ChecksumAlgorithm, partSize, resultKey } = {},\n\tstreamOptions = {},\n) => {\n\tChecksumAlgorithm ??= \"SHA256\";\n\tpartSize ??= 17_179_870;\n\tconst algorithm = _algorithms[ChecksumAlgorithm];\n\tif (!algorithm)\n\t\tthrow new Error(`Unsupported ChecksumAlgorithm: ${ChecksumAlgorithm}`);\n\tlet checksums = [];\n\tlet bytes = new Uint8Array(0);\n\tconst passThrough = async (chunk) => {\n\t\tif (typeof chunk === \"string\") {\n\t\t\tchunk = new TextEncoder().encode(chunk);\n\t\t}\n\t\twhile (bytes.byteLength + chunk.byteLength > partSize) {\n\t\t\tchunk = _concatBuffers([bytes, chunk]);\n\t\t\tconst prefixChunk = chunk.slice(0, partSize);\n\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, prefixChunk);\n\t\t\tchecksums.push(checksum);\n\t\t\tchunk = chunk.slice(prefixChunk.byteLength);\n\n\t\t\tbytes = new Uint8Array(0);\n\t\t}\n\t\tbytes = _concatBuffers([bytes, chunk]);\n\t};\n\tconst flush = async () => {\n\t\tif (bytes.byteLength) {\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, bytes);\n\t\t\tchecksums.push(checksum);\n\t\t}\n\t};\n\tconst stream = createPassThroughStream(passThrough, flush, streamOptions);\n\tlet checksum;\n\tstream.result = async () => {\n\t\tif (!checksum) {\n\t\t\tif (checksums.length > 1) {\n\t\t\t\tchecksum = await crypto.subtle.digest(\n\t\t\t\t\talgorithm,\n\t\t\t\t\t_concatBuffers(checksums),\n\t\t\t\t);\n\t\t\t\tchecksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`;\n\t\t\t} else if (checksums.length === 1) {\n\t\t\t\tchecksum = _arrayBufferToBase64(checksums[0]);\n\t\t\t} else {\n\t\t\t\tchecksum = \"\";\n\t\t\t}\n\t\t\tchecksums = checksums.map(_arrayBufferToBase64);\n\t\t}\n\t\treturn {\n\t\t\tkey: resultKey ?? \"s3\",\n\t\t\tvalue: { checksum, checksums, partSize },\n\t\t};\n\t};\n\treturn stream;\n};\n\nconst _algorithms = {\n\t// AWS_NAME: NODE_NAME\n\tSHA1: \"SHA-1\",\n\tSHA256: \"SHA-256\",\n\t// CRC32: '',\n\t// CRC32C: '',\n};\nconst _concatBuffers = (buffers) => {\n\tconst tmp = new Uint8Array(\n\t\tbuffers.reduce((byteLength, buffer) => byteLength + buffer.byteLength, 0),\n\t);\n\tlet byteLength = 0;\n\tfor (let i = 0, l = buffers.length; i < l; i++) {\n\t\ttmp.set(new Uint8Array(buffers[i]), byteLength);\n\t\tbyteLength += buffers[i].byteLength;\n\t}\n\treturn tmp.buffer;\n};\nconst _arrayBufferToBase64 = (buffer) => {\n\tlet binary = \"\";\n\tconst bytes = new Uint8Array(buffer);\n\tconst len = bytes.byteLength;\n\tfor (let i = 0; i < len; i++) {\n\t\tbinary += String.fromCharCode(bytes[i]);\n\t}\n\treturn btoa(binary);\n};\n\nexport default {\n\tsetClient: awsS3SetClient,\n\tgetObjectStream: awsS3GetObjectStream,\n\tputObjectStream: awsS3PutObjectStream,\n\tchecksumStream: awsS3ChecksumStream,\n};\n"], | ||
| "mappings": "AAIA,SAAS,kBAAkB,gBAAgB;AAC3C,SAAS,cAAc;AACvB;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,SAAS,iBAAiB;AAC3C,MAAM,iBAAiB,CAAC,aAAa;AAC3C,kBAAgB;AACjB;AAEO,MAAM,uBAAuB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC1E,QAAM,EAAE,QAAQ,GAAG,OAAO,IAAI;AAC9B,QAAM,EAAE,KAAK,IAAI,OAAO,UAAU,eAAe;AAAA,IAChD,IAAI,iBAAiB,MAAM;AAAA,EAC5B;AACA,MAAI,CAAC,MAAM;AACV,UAAM,IAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,CAAC;AAAA,EAC5D;AACA,SAAO,qBAAqB,MAAM,aAAa;AAChD;AAEO,MAAM,uBAAuB,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,EAAE,YAAY,QAAQ,MAAM,GAAG,OAAO,IAAI;AAChD,QAAM,SAAS,wBAAwB,MAAM;AAAA,EAAC,GAAG,aAAa;AAC9D,QAAM,SAAS,IAAI,OAAO;AAAA,IACzB,QAAQ,UAAU;AAAA,IAClB,QAAQ;AAAA,MACP,GAAG;AAAA,MACH,MAAM;AAAA,IACP;AAAA,IACA;AAAA,EACD,CAAC;AACD,MAAI,YAAY;AACf,WAAO,GAAG,sBAAsB,UAAU;AAAA,EAC3C;AACA,QAAM,SAAS,OAAO,KAAK;AAE3B,SAAO,SAAS,YAAY;AAC3B,UAAM;AACN,WAAO,CAAC;AAAA,EACT;AACA,SAAO;AACR;AAIO,MAAM,sBAAsB,CAClC,EAAE,mBAAmB,UAAU,UAAU,IAAI,CAAC,GAC9C,gBAAgB,CAAC,MACb;AACJ,wBAAsB;AACtB,eAAa;AACb,QAAM,YAAY,YAAY,iBAAiB;AAC/C,MAAI,CAAC;AACJ,UAAM,IAAI,MAAM,kCAAkC,iBAAiB,EAAE;AACtE,MAAI,YAAY,CAAC;AACjB,MAAI,QAAQ,IAAI,WAAW,CAAC;AAC5B,QAAM,cAAc,OAAO,UAAU;AACpC,QAAI,OAAO,UAAU,UAAU;AAC9B,cAAQ,IAAI,YAAY,EAAE,OAAO,KAAK;AAAA,IACvC;AACA,WAAO,MAAM,aAAa,MAAM,aAAa,UAAU;AACtD,cAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AACrC,YAAM,cAAc,MAAM,MAAM,GAAG,QAAQ;AAE3C,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,WAAW;AAClE,gBAAU,KAAKA,SAAQ;AACvB,cAAQ,MAAM,MAAM,YAAY,UAAU;AAE1C,cAAQ,IAAI,WAAW,CAAC;AAAA,IACzB;AACA,YAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AAAA,EACtC;AACA,QAAM,QAAQ,YAAY;AACzB,QAAI,MAAM,YAAY;AACrB,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AAC5D,gBAAU,KAAKA,SAAQ;AAAA,IACxB;AAAA,EACD;AACA,QAAM,SAAS,wBAAwB,aAAa,OAAO,aAAa;AACxE,MAAI;AACJ,SAAO,SAAS,YAAY;AAC3B,QAAI,CAAC,UAAU;AACd,UAAI,UAAU,SAAS,GAAG;AACzB,mBAAW,MAAM,OAAO,OAAO;AAAA,UAC9B;AAAA,UACA,eAAe,SAAS;AAAA,QACzB;AACA,mBAAW,GAAG,qBAAqB,QAAQ,CAAC,IAAI,UAAU,MAAM;AAAA,MACjE,WAAW,UAAU,WAAW,GAAG;AAClC,mBAAW,qBAAqB,UAAU,CAAC,CAAC;AAAA,MAC7C,OAAO;AACN,mBAAW;AAAA,MACZ;AACA,kBAAY,UAAU,IAAI,oBAAoB;AAAA,IAC/C;AACA,WAAO;AAAA,MACN,KAAK,aAAa;AAAA,MAClB,OAAO,EAAE,UAAU,WAAW,SAAS;AAAA,IACxC;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,cAAc;AAAA;AAAA,EAEnB,MAAM;AAAA,EACN,QAAQ;AAAA;AAAA;AAGT;AACA,MAAM,iBAAiB,CAAC,YAAY;AACnC,QAAM,MAAM,IAAI;AAAA,IACf,QAAQ,OAAO,CAACC,aAAY,WAAWA,cAAa,OAAO,YAAY,CAAC;AAAA,EACzE;AACA,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAAI,GAAG,KAAK;AAC/C,QAAI,IAAI,IAAI,WAAW,QAAQ,CAAC,CAAC,GAAG,UAAU;AAC9C,kBAAc,QAAQ,CAAC,EAAE;AAAA,EAC1B;AACA,SAAO,IAAI;AACZ;AACA,MAAM,uBAAuB,CAAC,WAAW;AACxC,MAAI,SAAS;AACb,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,MAAM,MAAM;AAClB,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAU,OAAO,aAAa,MAAM,CAAC,CAAC;AAAA,EACvC;AACA,SAAO,KAAK,MAAM;AACnB;AAEA,IAAO,aAAQ;AAAA,EACd,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,gBAAgB;AACjB;", | ||
| "names": ["checksum", "byteLength"] | ||
| } |
+10
-15
@@ -7,12 +7,3 @@ import { GetObjectCommand, S3Client } from "@aws-sdk/client-s3"; | ||
| } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2", | ||
| "ca-central-1" | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let defaultClient = new S3Client(awsClientDefaults); | ||
@@ -22,3 +13,3 @@ const awsS3SetClient = (s3Client) => { | ||
| }; | ||
| const awsS3GetObjectStream = async (options, streamOptions) => { | ||
| const awsS3GetObjectStream = async (options, streamOptions = {}) => { | ||
| const { client, ...params } = options; | ||
@@ -29,7 +20,7 @@ const { Body } = await (client ?? defaultClient).send( | ||
| if (!Body) { | ||
| throw new Error("S3.GetObject not Found", { cause: params }); | ||
| throw new Error("S3.GetObject not found", { cause: params }); | ||
| } | ||
| return createReadableStream(Body, streamOptions); | ||
| }; | ||
| const awsS3PutObjectStream = (options, streamOptions) => { | ||
| const awsS3PutObjectStream = (options, streamOptions = {}) => { | ||
| const { onProgress, client, tags, ...params } = options; | ||
@@ -60,2 +51,4 @@ const stream = createPassThroughStream(() => { | ||
| const algorithm = _algorithms[ChecksumAlgorithm]; | ||
| if (!algorithm) | ||
| throw new Error(`Unsupported ChecksumAlgorithm: ${ChecksumAlgorithm}`); | ||
| let checksums = []; | ||
@@ -65,3 +58,3 @@ let bytes = new Uint8Array(0); | ||
| if (typeof chunk === "string") { | ||
| chunk = new TextEncoder("utf-8").encode(chunk); | ||
| chunk = new TextEncoder().encode(chunk); | ||
| } | ||
@@ -94,4 +87,6 @@ while (bytes.byteLength + chunk.byteLength > partSize) { | ||
| checksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`; | ||
| } else if (checksums.length === 1) { | ||
| checksum = _arrayBufferToBase64(checksums[0]); | ||
| } else { | ||
| checksum = _arrayBufferToBase64(checksums[0]); | ||
| checksum = ""; | ||
| } | ||
@@ -98,0 +93,0 @@ checksums = checksums.map(_arrayBufferToBase64); |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["s3.js"], | ||
| "sourcesContent": ["/* global crypto */\n\nimport { GetObjectCommand, S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport {\n\tcreatePassThroughStream,\n\tcreateReadableStream,\n} from \"@datastream/core\";\n\n// import { Crc32 } from '@aws-crypto/crc32';\n// import { Crc32c } from '@aws-crypto/crc32c';\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t\t\"ca-central-1\",\n\t].includes(process.env.AWS_REGION),\n};\n\nlet defaultClient = new S3Client(awsClientDefaults);\nexport const awsS3SetClient = (s3Client) => {\n\tdefaultClient = s3Client;\n};\n\nexport const awsS3GetObjectStream = async (options, streamOptions) => {\n\tconst { client, ...params } = options;\n\tconst { Body } = await (client ?? defaultClient).send(\n\t\tnew GetObjectCommand(params),\n\t);\n\tif (!Body) {\n\t\tthrow new Error(\"S3.GetObject not Found\", { cause: params });\n\t}\n\treturn createReadableStream(Body, streamOptions);\n};\n\nexport const awsS3PutObjectStream = (options, streamOptions) => {\n\tconst { onProgress, client, tags, ...params } = options;\n\tconst stream = createPassThroughStream(() => {}, streamOptions);\n\tconst upload = new Upload({\n\t\tclient: client ?? defaultClient,\n\t\tparams: {\n\t\t\t...params,\n\t\t\tBody: stream,\n\t\t},\n\t\ttags,\n\t});\n\tif (onProgress) {\n\t\tstream.on(\"httpUploadProgress\", onProgress);\n\t}\n\tconst result = upload.done();\n\n\tstream.result = async () => {\n\t\tawait result;\n\t\treturn {};\n\t};\n\treturn stream;\n};\n\n// This is designed to be used in the browser on a file that you want to upload via a presigned URL\n// partSize; magic number, no 16MB mentioned in the docs\nexport const awsS3ChecksumStream = (\n\t{ ChecksumAlgorithm, partSize, resultKey } = {},\n\tstreamOptions = {},\n) => {\n\tChecksumAlgorithm ??= \"SHA256\";\n\tpartSize ??= 17_179_870;\n\tconst algorithm = _algorithms[ChecksumAlgorithm];\n\tlet checksums = [];\n\tlet bytes = new Uint8Array(0);\n\tconst passThrough = async (chunk) => {\n\t\tif (typeof chunk === \"string\") {\n\t\t\tchunk = new TextEncoder(\"utf-8\").encode(chunk);\n\t\t}\n\t\twhile (bytes.byteLength + chunk.byteLength > partSize) {\n\t\t\tchunk = _concatBuffers([bytes, chunk]);\n\t\t\tconst prefixChunk = chunk.slice(0, partSize);\n\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, prefixChunk);\n\t\t\tchecksums.push(checksum);\n\t\t\tchunk = chunk.slice(prefixChunk.byteLength);\n\n\t\t\tbytes = new Uint8Array(0);\n\t\t}\n\t\tbytes = _concatBuffers([bytes, chunk]);\n\t};\n\tconst flush = async () => {\n\t\tif (bytes.byteLength) {\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, bytes);\n\t\t\tchecksums.push(checksum);\n\t\t}\n\t};\n\tconst stream = createPassThroughStream(passThrough, flush, streamOptions);\n\tlet checksum;\n\tstream.result = async () => {\n\t\tif (!checksum) {\n\t\t\tif (checksums.length > 1) {\n\t\t\t\tchecksum = await crypto.subtle.digest(\n\t\t\t\t\talgorithm,\n\t\t\t\t\t_concatBuffers(checksums),\n\t\t\t\t);\n\t\t\t\tchecksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`;\n\t\t\t} else {\n\t\t\t\tchecksum = _arrayBufferToBase64(checksums[0]);\n\t\t\t}\n\t\t\tchecksums = checksums.map(_arrayBufferToBase64);\n\t\t}\n\t\treturn {\n\t\t\tkey: resultKey ?? \"s3\",\n\t\t\tvalue: { checksum, checksums, partSize },\n\t\t};\n\t};\n\treturn stream;\n};\n\nconst _algorithms = {\n\t// AWS_NAME: NODE_NAME\n\tSHA1: \"SHA-1\",\n\tSHA256: \"SHA-256\",\n\t// CRC32: '',\n\t// CRC32C: '',\n};\nconst _concatBuffers = (buffers) => {\n\tconst tmp = new Uint8Array(\n\t\tbuffers.reduce((byteLength, buffer) => byteLength + buffer.byteLength, 0),\n\t);\n\tlet byteLength = 0;\n\tfor (let i = 0, l = buffers.length; i < l; i++) {\n\t\ttmp.set(new Uint8Array(buffers[i]), byteLength);\n\t\tbyteLength += buffers[i].byteLength;\n\t}\n\treturn tmp.buffer;\n};\nconst _arrayBufferToBase64 = (buffer) => {\n\tlet binary = \"\";\n\tconst bytes = new Uint8Array(buffer);\n\tconst len = bytes.byteLength;\n\tfor (let i = 0; i < len; i++) {\n\t\tbinary += String.fromCharCode(bytes[i]);\n\t}\n\treturn btoa(binary);\n};\n\nexport default {\n\tsetClient: awsS3SetClient,\n\tgetObjectStream: awsS3GetObjectStream,\n\tputObjectStream: awsS3PutObjectStream,\n\tchecksumStream: awsS3ChecksumStream,\n};\n"], | ||
| "mappings": "AAEA,SAAS,kBAAkB,gBAAgB;AAC3C,SAAS,cAAc;AACvB;AAAA,EACC;AAAA,EACA;AAAA,OACM;AAKP,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,gBAAgB,IAAI,SAAS,iBAAiB;AAC3C,MAAM,iBAAiB,CAAC,aAAa;AAC3C,kBAAgB;AACjB;AAEO,MAAM,uBAAuB,OAAO,SAAS,kBAAkB;AACrE,QAAM,EAAE,QAAQ,GAAG,OAAO,IAAI;AAC9B,QAAM,EAAE,KAAK,IAAI,OAAO,UAAU,eAAe;AAAA,IAChD,IAAI,iBAAiB,MAAM;AAAA,EAC5B;AACA,MAAI,CAAC,MAAM;AACV,UAAM,IAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,CAAC;AAAA,EAC5D;AACA,SAAO,qBAAqB,MAAM,aAAa;AAChD;AAEO,MAAM,uBAAuB,CAAC,SAAS,kBAAkB;AAC/D,QAAM,EAAE,YAAY,QAAQ,MAAM,GAAG,OAAO,IAAI;AAChD,QAAM,SAAS,wBAAwB,MAAM;AAAA,EAAC,GAAG,aAAa;AAC9D,QAAM,SAAS,IAAI,OAAO;AAAA,IACzB,QAAQ,UAAU;AAAA,IAClB,QAAQ;AAAA,MACP,GAAG;AAAA,MACH,MAAM;AAAA,IACP;AAAA,IACA;AAAA,EACD,CAAC;AACD,MAAI,YAAY;AACf,WAAO,GAAG,sBAAsB,UAAU;AAAA,EAC3C;AACA,QAAM,SAAS,OAAO,KAAK;AAE3B,SAAO,SAAS,YAAY;AAC3B,UAAM;AACN,WAAO,CAAC;AAAA,EACT;AACA,SAAO;AACR;AAIO,MAAM,sBAAsB,CAClC,EAAE,mBAAmB,UAAU,UAAU,IAAI,CAAC,GAC9C,gBAAgB,CAAC,MACb;AACJ,wBAAsB;AACtB,eAAa;AACb,QAAM,YAAY,YAAY,iBAAiB;AAC/C,MAAI,YAAY,CAAC;AACjB,MAAI,QAAQ,IAAI,WAAW,CAAC;AAC5B,QAAM,cAAc,OAAO,UAAU;AACpC,QAAI,OAAO,UAAU,UAAU;AAC9B,cAAQ,IAAI,YAAY,OAAO,EAAE,OAAO,KAAK;AAAA,IAC9C;AACA,WAAO,MAAM,aAAa,MAAM,aAAa,UAAU;AACtD,cAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AACrC,YAAM,cAAc,MAAM,MAAM,GAAG,QAAQ;AAE3C,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,WAAW;AAClE,gBAAU,KAAKA,SAAQ;AACvB,cAAQ,MAAM,MAAM,YAAY,UAAU;AAE1C,cAAQ,IAAI,WAAW,CAAC;AAAA,IACzB;AACA,YAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AAAA,EACtC;AACA,QAAM,QAAQ,YAAY;AACzB,QAAI,MAAM,YAAY;AACrB,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AAC5D,gBAAU,KAAKA,SAAQ;AAAA,IACxB;AAAA,EACD;AACA,QAAM,SAAS,wBAAwB,aAAa,OAAO,aAAa;AACxE,MAAI;AACJ,SAAO,SAAS,YAAY;AAC3B,QAAI,CAAC,UAAU;AACd,UAAI,UAAU,SAAS,GAAG;AACzB,mBAAW,MAAM,OAAO,OAAO;AAAA,UAC9B;AAAA,UACA,eAAe,SAAS;AAAA,QACzB;AACA,mBAAW,GAAG,qBAAqB,QAAQ,CAAC,IAAI,UAAU,MAAM;AAAA,MACjE,OAAO;AACN,mBAAW,qBAAqB,UAAU,CAAC,CAAC;AAAA,MAC7C;AACA,kBAAY,UAAU,IAAI,oBAAoB;AAAA,IAC/C;AACA,WAAO;AAAA,MACN,KAAK,aAAa;AAAA,MAClB,OAAO,EAAE,UAAU,WAAW,SAAS;AAAA,IACxC;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,cAAc;AAAA;AAAA,EAEnB,MAAM;AAAA,EACN,QAAQ;AAAA;AAAA;AAGT;AACA,MAAM,iBAAiB,CAAC,YAAY;AACnC,QAAM,MAAM,IAAI;AAAA,IACf,QAAQ,OAAO,CAACC,aAAY,WAAWA,cAAa,OAAO,YAAY,CAAC;AAAA,EACzE;AACA,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAAI,GAAG,KAAK;AAC/C,QAAI,IAAI,IAAI,WAAW,QAAQ,CAAC,CAAC,GAAG,UAAU;AAC9C,kBAAc,QAAQ,CAAC,EAAE;AAAA,EAC1B;AACA,SAAO,IAAI;AACZ;AACA,MAAM,uBAAuB,CAAC,WAAW;AACxC,MAAI,SAAS;AACb,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,MAAM,MAAM;AAClB,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAU,OAAO,aAAa,MAAM,CAAC,CAAC;AAAA,EACvC;AACA,SAAO,KAAK,MAAM;AACnB;AAEA,IAAO,aAAQ;AAAA,EACd,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,gBAAgB;AACjB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\n/* global crypto */\n\nimport { GetObjectCommand, S3Client } from \"@aws-sdk/client-s3\";\nimport { Upload } from \"@aws-sdk/lib-storage\";\nimport {\n\tcreatePassThroughStream,\n\tcreateReadableStream,\n} from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet defaultClient = new S3Client(awsClientDefaults);\nexport const awsS3SetClient = (s3Client) => {\n\tdefaultClient = s3Client;\n};\n\nexport const awsS3GetObjectStream = async (options, streamOptions = {}) => {\n\tconst { client, ...params } = options;\n\tconst { Body } = await (client ?? defaultClient).send(\n\t\tnew GetObjectCommand(params),\n\t);\n\tif (!Body) {\n\t\tthrow new Error(\"S3.GetObject not found\", { cause: params });\n\t}\n\treturn createReadableStream(Body, streamOptions);\n};\n\nexport const awsS3PutObjectStream = (options, streamOptions = {}) => {\n\tconst { onProgress, client, tags, ...params } = options;\n\tconst stream = createPassThroughStream(() => {}, streamOptions);\n\tconst upload = new Upload({\n\t\tclient: client ?? defaultClient,\n\t\tparams: {\n\t\t\t...params,\n\t\t\tBody: stream,\n\t\t},\n\t\ttags,\n\t});\n\tif (onProgress) {\n\t\tstream.on(\"httpUploadProgress\", onProgress);\n\t}\n\tconst result = upload.done();\n\n\tstream.result = async () => {\n\t\tawait result;\n\t\treturn {};\n\t};\n\treturn stream;\n};\n\n// This is designed to be used in the browser on a file that you want to upload via a presigned URL\n// partSize; magic number, no 16MB mentioned in the docs\nexport const awsS3ChecksumStream = (\n\t{ ChecksumAlgorithm, partSize, resultKey } = {},\n\tstreamOptions = {},\n) => {\n\tChecksumAlgorithm ??= \"SHA256\";\n\tpartSize ??= 17_179_870;\n\tconst algorithm = _algorithms[ChecksumAlgorithm];\n\tif (!algorithm)\n\t\tthrow new Error(`Unsupported ChecksumAlgorithm: ${ChecksumAlgorithm}`);\n\tlet checksums = [];\n\tlet bytes = new Uint8Array(0);\n\tconst passThrough = async (chunk) => {\n\t\tif (typeof chunk === \"string\") {\n\t\t\tchunk = new TextEncoder().encode(chunk);\n\t\t}\n\t\twhile (bytes.byteLength + chunk.byteLength > partSize) {\n\t\t\tchunk = _concatBuffers([bytes, chunk]);\n\t\t\tconst prefixChunk = chunk.slice(0, partSize);\n\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, prefixChunk);\n\t\t\tchecksums.push(checksum);\n\t\t\tchunk = chunk.slice(prefixChunk.byteLength);\n\n\t\t\tbytes = new Uint8Array(0);\n\t\t}\n\t\tbytes = _concatBuffers([bytes, chunk]);\n\t};\n\tconst flush = async () => {\n\t\tif (bytes.byteLength) {\n\t\t\tconst checksum = await crypto.subtle.digest(algorithm, bytes);\n\t\t\tchecksums.push(checksum);\n\t\t}\n\t};\n\tconst stream = createPassThroughStream(passThrough, flush, streamOptions);\n\tlet checksum;\n\tstream.result = async () => {\n\t\tif (!checksum) {\n\t\t\tif (checksums.length > 1) {\n\t\t\t\tchecksum = await crypto.subtle.digest(\n\t\t\t\t\talgorithm,\n\t\t\t\t\t_concatBuffers(checksums),\n\t\t\t\t);\n\t\t\t\tchecksum = `${_arrayBufferToBase64(checksum)}-${checksums.length}`;\n\t\t\t} else if (checksums.length === 1) {\n\t\t\t\tchecksum = _arrayBufferToBase64(checksums[0]);\n\t\t\t} else {\n\t\t\t\tchecksum = \"\";\n\t\t\t}\n\t\t\tchecksums = checksums.map(_arrayBufferToBase64);\n\t\t}\n\t\treturn {\n\t\t\tkey: resultKey ?? \"s3\",\n\t\t\tvalue: { checksum, checksums, partSize },\n\t\t};\n\t};\n\treturn stream;\n};\n\nconst _algorithms = {\n\t// AWS_NAME: NODE_NAME\n\tSHA1: \"SHA-1\",\n\tSHA256: \"SHA-256\",\n\t// CRC32: '',\n\t// CRC32C: '',\n};\nconst _concatBuffers = (buffers) => {\n\tconst tmp = new Uint8Array(\n\t\tbuffers.reduce((byteLength, buffer) => byteLength + buffer.byteLength, 0),\n\t);\n\tlet byteLength = 0;\n\tfor (let i = 0, l = buffers.length; i < l; i++) {\n\t\ttmp.set(new Uint8Array(buffers[i]), byteLength);\n\t\tbyteLength += buffers[i].byteLength;\n\t}\n\treturn tmp.buffer;\n};\nconst _arrayBufferToBase64 = (buffer) => {\n\tlet binary = \"\";\n\tconst bytes = new Uint8Array(buffer);\n\tconst len = bytes.byteLength;\n\tfor (let i = 0; i < len; i++) {\n\t\tbinary += String.fromCharCode(bytes[i]);\n\t}\n\treturn btoa(binary);\n};\n\nexport default {\n\tsetClient: awsS3SetClient,\n\tgetObjectStream: awsS3GetObjectStream,\n\tputObjectStream: awsS3PutObjectStream,\n\tchecksumStream: awsS3ChecksumStream,\n};\n"], | ||
| "mappings": "AAIA,SAAS,kBAAkB,gBAAgB;AAC3C,SAAS,cAAc;AACvB;AAAA,EACC;AAAA,EACA;AAAA,OACM;AACP,SAAS,yBAAyB;AAElC,IAAI,gBAAgB,IAAI,SAAS,iBAAiB;AAC3C,MAAM,iBAAiB,CAAC,aAAa;AAC3C,kBAAgB;AACjB;AAEO,MAAM,uBAAuB,OAAO,SAAS,gBAAgB,CAAC,MAAM;AAC1E,QAAM,EAAE,QAAQ,GAAG,OAAO,IAAI;AAC9B,QAAM,EAAE,KAAK,IAAI,OAAO,UAAU,eAAe;AAAA,IAChD,IAAI,iBAAiB,MAAM;AAAA,EAC5B;AACA,MAAI,CAAC,MAAM;AACV,UAAM,IAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,CAAC;AAAA,EAC5D;AACA,SAAO,qBAAqB,MAAM,aAAa;AAChD;AAEO,MAAM,uBAAuB,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACpE,QAAM,EAAE,YAAY,QAAQ,MAAM,GAAG,OAAO,IAAI;AAChD,QAAM,SAAS,wBAAwB,MAAM;AAAA,EAAC,GAAG,aAAa;AAC9D,QAAM,SAAS,IAAI,OAAO;AAAA,IACzB,QAAQ,UAAU;AAAA,IAClB,QAAQ;AAAA,MACP,GAAG;AAAA,MACH,MAAM;AAAA,IACP;AAAA,IACA;AAAA,EACD,CAAC;AACD,MAAI,YAAY;AACf,WAAO,GAAG,sBAAsB,UAAU;AAAA,EAC3C;AACA,QAAM,SAAS,OAAO,KAAK;AAE3B,SAAO,SAAS,YAAY;AAC3B,UAAM;AACN,WAAO,CAAC;AAAA,EACT;AACA,SAAO;AACR;AAIO,MAAM,sBAAsB,CAClC,EAAE,mBAAmB,UAAU,UAAU,IAAI,CAAC,GAC9C,gBAAgB,CAAC,MACb;AACJ,wBAAsB;AACtB,eAAa;AACb,QAAM,YAAY,YAAY,iBAAiB;AAC/C,MAAI,CAAC;AACJ,UAAM,IAAI,MAAM,kCAAkC,iBAAiB,EAAE;AACtE,MAAI,YAAY,CAAC;AACjB,MAAI,QAAQ,IAAI,WAAW,CAAC;AAC5B,QAAM,cAAc,OAAO,UAAU;AACpC,QAAI,OAAO,UAAU,UAAU;AAC9B,cAAQ,IAAI,YAAY,EAAE,OAAO,KAAK;AAAA,IACvC;AACA,WAAO,MAAM,aAAa,MAAM,aAAa,UAAU;AACtD,cAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AACrC,YAAM,cAAc,MAAM,MAAM,GAAG,QAAQ;AAE3C,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,WAAW;AAClE,gBAAU,KAAKA,SAAQ;AACvB,cAAQ,MAAM,MAAM,YAAY,UAAU;AAE1C,cAAQ,IAAI,WAAW,CAAC;AAAA,IACzB;AACA,YAAQ,eAAe,CAAC,OAAO,KAAK,CAAC;AAAA,EACtC;AACA,QAAM,QAAQ,YAAY;AACzB,QAAI,MAAM,YAAY;AACrB,YAAMA,YAAW,MAAM,OAAO,OAAO,OAAO,WAAW,KAAK;AAC5D,gBAAU,KAAKA,SAAQ;AAAA,IACxB;AAAA,EACD;AACA,QAAM,SAAS,wBAAwB,aAAa,OAAO,aAAa;AACxE,MAAI;AACJ,SAAO,SAAS,YAAY;AAC3B,QAAI,CAAC,UAAU;AACd,UAAI,UAAU,SAAS,GAAG;AACzB,mBAAW,MAAM,OAAO,OAAO;AAAA,UAC9B;AAAA,UACA,eAAe,SAAS;AAAA,QACzB;AACA,mBAAW,GAAG,qBAAqB,QAAQ,CAAC,IAAI,UAAU,MAAM;AAAA,MACjE,WAAW,UAAU,WAAW,GAAG;AAClC,mBAAW,qBAAqB,UAAU,CAAC,CAAC;AAAA,MAC7C,OAAO;AACN,mBAAW;AAAA,MACZ;AACA,kBAAY,UAAU,IAAI,oBAAoB;AAAA,IAC/C;AACA,WAAO;AAAA,MACN,KAAK,aAAa;AAAA,MAClB,OAAO,EAAE,UAAU,WAAW,SAAS;AAAA,IACxC;AAAA,EACD;AACA,SAAO;AACR;AAEA,MAAM,cAAc;AAAA;AAAA,EAEnB,MAAM;AAAA,EACN,QAAQ;AAAA;AAAA;AAGT;AACA,MAAM,iBAAiB,CAAC,YAAY;AACnC,QAAM,MAAM,IAAI;AAAA,IACf,QAAQ,OAAO,CAACC,aAAY,WAAWA,cAAa,OAAO,YAAY,CAAC;AAAA,EACzE;AACA,MAAI,aAAa;AACjB,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,IAAI,GAAG,KAAK;AAC/C,QAAI,IAAI,IAAI,WAAW,QAAQ,CAAC,CAAC,GAAG,UAAU;AAC9C,kBAAc,QAAQ,CAAC,EAAE;AAAA,EAC1B;AACA,SAAO,IAAI;AACZ;AACA,MAAM,uBAAuB,CAAC,WAAW;AACxC,MAAI,SAAS;AACb,QAAM,QAAQ,IAAI,WAAW,MAAM;AACnC,QAAM,MAAM,MAAM;AAClB,WAAS,IAAI,GAAG,IAAI,KAAK,KAAK;AAC7B,cAAU,OAAO,aAAa,MAAM,CAAC,CAAC;AAAA,EACvC;AACA,SAAO,KAAK,MAAM;AACnB;AAEA,IAAO,aAAQ;AAAA,EACd,WAAW;AAAA,EACX,iBAAiB;AAAA,EACjB,iBAAiB;AAAA,EACjB,gBAAgB;AACjB;", | ||
| "names": ["checksum", "byteLength"] | ||
| } |
+3
-11
| import { PublishBatchCommand, SNSClient } from "@aws-sdk/client-sns"; | ||
| import { createWritableStream } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2" | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new SNSClient(awsClientDefaults); | ||
@@ -16,3 +8,3 @@ const awsSNSSetClient = (snsClient) => { | ||
| }; | ||
| const awsSNSPublishMessageStream = (options, streamOptions) => { | ||
| const awsSNSPublishMessageStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -30,3 +22,3 @@ const send = () => { | ||
| }; | ||
| const final = send; | ||
| const final = () => batch.length ? send() : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
@@ -33,0 +25,0 @@ }; |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["sns.js"], | ||
| "sourcesContent": ["import { PublishBatchCommand, SNSClient } from \"@aws-sdk/client-sns\";\nimport { createWritableStream } from \"@datastream/core\";\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t].includes(process.env.AWS_REGION),\n};\n\nlet client = new SNSClient(awsClientDefaults);\nexport const awsSNSSetClient = (snsClient) => {\n\tclient = snsClient;\n};\n\nexport const awsSNSPublishMessageStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.PublishBatchRequestEntries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new PublishBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = send;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSNSSetClient,\n\tpublishMessageStream: awsSNSPublishMessageStream,\n};\n"], | ||
| "mappings": "AAAA,SAAS,qBAAqB,iBAAiB;AAC/C,SAAS,4BAA4B;AAErC,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,CAAC,SAAS,kBAAkB;AACrE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,6BAA6B;AACrC,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,oBAAoB,OAAO,CAAC;AAAA,EACpD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ;AACd,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,sBAAsB;AACvB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport { PublishBatchCommand, SNSClient } from \"@aws-sdk/client-sns\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new SNSClient(awsClientDefaults);\nexport const awsSNSSetClient = (snsClient) => {\n\tclient = snsClient;\n};\n\nexport const awsSNSPublishMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.PublishBatchRequestEntries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new PublishBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSNSSetClient,\n\tpublishMessageStream: awsSNSPublishMessageStream,\n};\n"], | ||
| "mappings": "AAEA,SAAS,qBAAqB,iBAAiB;AAC/C,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC1E,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,6BAA6B;AACrC,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,oBAAoB,OAAO,CAAC;AAAA,EACpD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,sBAAsB;AACvB;", | ||
| "names": [] | ||
| } |
+3
-11
| import { PublishBatchCommand, SNSClient } from "@aws-sdk/client-sns"; | ||
| import { createWritableStream } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2" | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new SNSClient(awsClientDefaults); | ||
@@ -16,3 +8,3 @@ const awsSNSSetClient = (snsClient) => { | ||
| }; | ||
| const awsSNSPublishMessageStream = (options, streamOptions) => { | ||
| const awsSNSPublishMessageStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -30,3 +22,3 @@ const send = () => { | ||
| }; | ||
| const final = send; | ||
| const final = () => batch.length ? send() : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
@@ -33,0 +25,0 @@ }; |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["sns.js"], | ||
| "sourcesContent": ["import { PublishBatchCommand, SNSClient } from \"@aws-sdk/client-sns\";\nimport { createWritableStream } from \"@datastream/core\";\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t].includes(process.env.AWS_REGION),\n};\n\nlet client = new SNSClient(awsClientDefaults);\nexport const awsSNSSetClient = (snsClient) => {\n\tclient = snsClient;\n};\n\nexport const awsSNSPublishMessageStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.PublishBatchRequestEntries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new PublishBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = send;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSNSSetClient,\n\tpublishMessageStream: awsSNSPublishMessageStream,\n};\n"], | ||
| "mappings": "AAAA,SAAS,qBAAqB,iBAAiB;AAC/C,SAAS,4BAA4B;AAErC,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,CAAC,SAAS,kBAAkB;AACrE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,6BAA6B;AACrC,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,oBAAoB,OAAO,CAAC;AAAA,EACpD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ;AACd,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,sBAAsB;AACvB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport { PublishBatchCommand, SNSClient } from \"@aws-sdk/client-sns\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new SNSClient(awsClientDefaults);\nexport const awsSNSSetClient = (snsClient) => {\n\tclient = snsClient;\n};\n\nexport const awsSNSPublishMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.PublishBatchRequestEntries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new PublishBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSNSSetClient,\n\tpublishMessageStream: awsSNSPublishMessageStream,\n};\n"], | ||
| "mappings": "AAEA,SAAS,qBAAqB,iBAAiB;AAC/C,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AAC1E,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,6BAA6B;AACrC,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,oBAAoB,OAAO,CAAC;AAAA,EACpD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,sBAAsB;AACvB;", | ||
| "names": [] | ||
| } |
+8
-15
@@ -8,11 +8,3 @@ import { | ||
| import { createWritableStream } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2" | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new SQSClient(awsClientDefaults); | ||
@@ -27,6 +19,7 @@ const awsSQSSetClient = (sqsClient) => { | ||
| const response = await client.send(new ReceiveMessageCommand(options2)); | ||
| for (const item of response.Messages) { | ||
| const messages = response.Messages ?? []; | ||
| for (const item of messages) { | ||
| yield item; | ||
| } | ||
| expectMore = response.Messages.length; | ||
| expectMore = messages.length; | ||
| } | ||
@@ -36,3 +29,3 @@ } | ||
| }; | ||
| const awsSQSDeleteMessageStream = (options, streamOptions) => { | ||
| const awsSQSDeleteMessageStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -50,6 +43,6 @@ const send = () => { | ||
| }; | ||
| const final = send; | ||
| const final = () => batch.length ? send() : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
| }; | ||
| const awsSQSSendMessageStream = (options, streamOptions) => { | ||
| const awsSQSSendMessageStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -67,3 +60,3 @@ const send = () => { | ||
| }; | ||
| const final = send; | ||
| const final = () => batch.length ? send() : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
@@ -70,0 +63,0 @@ }; |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["sqs.js"], | ||
| "sourcesContent": ["import {\n\tDeleteMessageBatchCommand,\n\tReceiveMessageCommand,\n\tSendMessageBatchCommand,\n\tSQSClient,\n} from \"@aws-sdk/client-sqs\";\nimport { createWritableStream } from \"@datastream/core\";\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t].includes(process.env.AWS_REGION),\n};\n\nlet client = new SQSClient(awsClientDefaults);\nexport const awsSQSSetClient = (sqsClient) => {\n\tclient = sqsClient;\n};\n\nexport const awsSQSReceiveMessageStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\t// TODO needs option to keep polling or not\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ReceiveMessageCommand(options));\n\t\t\tfor (const item of response.Messages) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\texpectMore = response.Messages.length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsSQSDeleteMessageStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new DeleteMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = send;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsSQSSendMessageStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new SendMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = send;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSQSSetClient,\n\tsendMessageStream: awsSQSSendMessageStream,\n\treceiveMessageStream: awsSQSReceiveMessageStream,\n\tdeleteMessageStream: awsSQSDeleteMessageStream,\n};\n"], | ||
| "mappings": "AAAA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AAErC,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,iBAAiB,CAAC,MACd;AAEJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,sBAAsBA,QAAO,CAAC;AACrE,iBAAW,QAAQ,SAAS,UAAU;AACrC,cAAM;AAAA,MACP;AACA,mBAAa,SAAS,SAAS;AAAA,IAChC;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,4BAA4B,CAAC,SAAS,kBAAkB;AACpE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,0BAA0B,OAAO,CAAC;AAAA,EAC1D;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ;AACd,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,0BAA0B,CAAC,SAAS,kBAAkB;AAClE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,wBAAwB,OAAO,CAAC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ;AACd,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,qBAAqB;AACtB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tDeleteMessageBatchCommand,\n\tReceiveMessageCommand,\n\tSendMessageBatchCommand,\n\tSQSClient,\n} from \"@aws-sdk/client-sqs\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new SQSClient(awsClientDefaults);\nexport const awsSQSSetClient = (sqsClient) => {\n\tclient = sqsClient;\n};\n\nexport const awsSQSReceiveMessageStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\t// TODO needs option to keep polling or not\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ReceiveMessageCommand(options));\n\t\t\tconst messages = response.Messages ?? [];\n\t\t\tfor (const item of messages) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\texpectMore = messages.length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsSQSDeleteMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new DeleteMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsSQSSendMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new SendMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSQSSetClient,\n\tsendMessageStream: awsSQSSendMessageStream,\n\treceiveMessageStream: awsSQSReceiveMessageStream,\n\tdeleteMessageStream: awsSQSDeleteMessageStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,iBAAiB,CAAC,MACd;AAEJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,sBAAsBA,QAAO,CAAC;AACrE,YAAM,WAAW,SAAS,YAAY,CAAC;AACvC,iBAAW,QAAQ,UAAU;AAC5B,cAAM;AAAA,MACP;AACA,mBAAa,SAAS;AAAA,IACvB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,4BAA4B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACzE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,0BAA0B,OAAO,CAAC;AAAA,EAC1D;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,0BAA0B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACvE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,wBAAwB,OAAO,CAAC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,qBAAqB;AACtB;", | ||
| "names": ["options"] | ||
| } |
+8
-15
@@ -8,11 +8,3 @@ import { | ||
| import { createWritableStream } from "@datastream/core"; | ||
| const awsClientDefaults = { | ||
| // https://aws.amazon.com/compliance/fips/ | ||
| useFipsEndpoint: [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2" | ||
| ].includes(process.env.AWS_REGION) | ||
| }; | ||
| import { awsClientDefaults } from "./client.js"; | ||
| let client = new SQSClient(awsClientDefaults); | ||
@@ -27,6 +19,7 @@ const awsSQSSetClient = (sqsClient) => { | ||
| const response = await client.send(new ReceiveMessageCommand(options2)); | ||
| for (const item of response.Messages) { | ||
| const messages = response.Messages ?? []; | ||
| for (const item of messages) { | ||
| yield item; | ||
| } | ||
| expectMore = response.Messages.length; | ||
| expectMore = messages.length; | ||
| } | ||
@@ -36,3 +29,3 @@ } | ||
| }; | ||
| const awsSQSDeleteMessageStream = (options, streamOptions) => { | ||
| const awsSQSDeleteMessageStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -50,6 +43,6 @@ const send = () => { | ||
| }; | ||
| const final = send; | ||
| const final = () => batch.length ? send() : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
| }; | ||
| const awsSQSSendMessageStream = (options, streamOptions) => { | ||
| const awsSQSSendMessageStream = (options, streamOptions = {}) => { | ||
| let batch = []; | ||
@@ -67,3 +60,3 @@ const send = () => { | ||
| }; | ||
| const final = send; | ||
| const final = () => batch.length ? send() : void 0; | ||
| return createWritableStream(write, final, streamOptions); | ||
@@ -70,0 +63,0 @@ }; |
+2
-2
| { | ||
| "version": 3, | ||
| "sources": ["sqs.js"], | ||
| "sourcesContent": ["import {\n\tDeleteMessageBatchCommand,\n\tReceiveMessageCommand,\n\tSendMessageBatchCommand,\n\tSQSClient,\n} from \"@aws-sdk/client-sqs\";\nimport { createWritableStream } from \"@datastream/core\";\n\nconst awsClientDefaults = {\n\t// https://aws.amazon.com/compliance/fips/\n\tuseFipsEndpoint: [\n\t\t\"us-east-1\",\n\t\t\"us-east-2\",\n\t\t\"us-west-1\",\n\t\t\"us-west-2\",\n\t].includes(process.env.AWS_REGION),\n};\n\nlet client = new SQSClient(awsClientDefaults);\nexport const awsSQSSetClient = (sqsClient) => {\n\tclient = sqsClient;\n};\n\nexport const awsSQSReceiveMessageStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\t// TODO needs option to keep polling or not\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ReceiveMessageCommand(options));\n\t\t\tfor (const item of response.Messages) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\texpectMore = response.Messages.length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsSQSDeleteMessageStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new DeleteMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = send;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsSQSSendMessageStream = (options, streamOptions) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new SendMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = send;\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSQSSetClient,\n\tsendMessageStream: awsSQSSendMessageStream,\n\treceiveMessageStream: awsSQSReceiveMessageStream,\n\tdeleteMessageStream: awsSQSDeleteMessageStream,\n};\n"], | ||
| "mappings": "AAAA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AAErC,MAAM,oBAAoB;AAAA;AAAA,EAEzB,iBAAiB;AAAA,IAChB;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACD,EAAE,SAAS,QAAQ,IAAI,UAAU;AAClC;AAEA,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,iBAAiB,CAAC,MACd;AAEJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,sBAAsBA,QAAO,CAAC;AACrE,iBAAW,QAAQ,SAAS,UAAU;AACrC,cAAM;AAAA,MACP;AACA,mBAAa,SAAS,SAAS;AAAA,IAChC;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,4BAA4B,CAAC,SAAS,kBAAkB;AACpE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,0BAA0B,OAAO,CAAC;AAAA,EAC1D;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ;AACd,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,0BAA0B,CAAC,SAAS,kBAAkB;AAClE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,wBAAwB,OAAO,CAAC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ;AACd,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,qBAAqB;AACtB;", | ||
| "sourcesContent": ["// Copyright 2026 will Farrell, and datastream contributors.\n// SPDX-License-Identifier: MIT\nimport {\n\tDeleteMessageBatchCommand,\n\tReceiveMessageCommand,\n\tSendMessageBatchCommand,\n\tSQSClient,\n} from \"@aws-sdk/client-sqs\";\nimport { createWritableStream } from \"@datastream/core\";\nimport { awsClientDefaults } from \"./client.js\";\n\nlet client = new SQSClient(awsClientDefaults);\nexport const awsSQSSetClient = (sqsClient) => {\n\tclient = sqsClient;\n};\n\nexport const awsSQSReceiveMessageStream = async (\n\toptions,\n\t_streamOptions = {},\n) => {\n\t// TODO needs option to keep polling or not\n\tasync function* command(options) {\n\t\tlet expectMore = true;\n\t\twhile (expectMore) {\n\t\t\tconst response = await client.send(new ReceiveMessageCommand(options));\n\t\t\tconst messages = response.Messages ?? [];\n\t\t\tfor (const item of messages) {\n\t\t\t\tyield item;\n\t\t\t}\n\t\t\texpectMore = messages.length;\n\t\t}\n\t}\n\treturn command(options);\n};\n\nexport const awsSQSDeleteMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new DeleteMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport const awsSQSSendMessageStream = (options, streamOptions = {}) => {\n\tlet batch = [];\n\tconst send = () => {\n\t\toptions.Entries = batch;\n\t\tbatch = [];\n\t\treturn client.send(new SendMessageBatchCommand(options));\n\t};\n\tconst write = async (chunk) => {\n\t\tif (batch.length === 10) {\n\t\t\tawait send();\n\t\t}\n\t\tbatch.push(chunk);\n\t};\n\tconst final = () => (batch.length ? send() : undefined);\n\treturn createWritableStream(write, final, streamOptions);\n};\n\nexport default {\n\tsetClient: awsSQSSetClient,\n\tsendMessageStream: awsSQSSendMessageStream,\n\treceiveMessageStream: awsSQSReceiveMessageStream,\n\tdeleteMessageStream: awsSQSDeleteMessageStream,\n};\n"], | ||
| "mappings": "AAEA;AAAA,EACC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACM;AACP,SAAS,4BAA4B;AACrC,SAAS,yBAAyB;AAElC,IAAI,SAAS,IAAI,UAAU,iBAAiB;AACrC,MAAM,kBAAkB,CAAC,cAAc;AAC7C,WAAS;AACV;AAEO,MAAM,6BAA6B,OACzC,SACA,iBAAiB,CAAC,MACd;AAEJ,kBAAgB,QAAQA,UAAS;AAChC,QAAI,aAAa;AACjB,WAAO,YAAY;AAClB,YAAM,WAAW,MAAM,OAAO,KAAK,IAAI,sBAAsBA,QAAO,CAAC;AACrE,YAAM,WAAW,SAAS,YAAY,CAAC;AACvC,iBAAW,QAAQ,UAAU;AAC5B,cAAM;AAAA,MACP;AACA,mBAAa,SAAS;AAAA,IACvB;AAAA,EACD;AACA,SAAO,QAAQ,OAAO;AACvB;AAEO,MAAM,4BAA4B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACzE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,0BAA0B,OAAO,CAAC;AAAA,EAC1D;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEO,MAAM,0BAA0B,CAAC,SAAS,gBAAgB,CAAC,MAAM;AACvE,MAAI,QAAQ,CAAC;AACb,QAAM,OAAO,MAAM;AAClB,YAAQ,UAAU;AAClB,YAAQ,CAAC;AACT,WAAO,OAAO,KAAK,IAAI,wBAAwB,OAAO,CAAC;AAAA,EACxD;AACA,QAAM,QAAQ,OAAO,UAAU;AAC9B,QAAI,MAAM,WAAW,IAAI;AACxB,YAAM,KAAK;AAAA,IACZ;AACA,UAAM,KAAK,KAAK;AAAA,EACjB;AACA,QAAM,QAAQ,MAAO,MAAM,SAAS,KAAK,IAAI;AAC7C,SAAO,qBAAqB,OAAO,OAAO,aAAa;AACxD;AAEA,IAAO,cAAQ;AAAA,EACd,WAAW;AAAA,EACX,mBAAmB;AAAA,EACnB,sBAAsB;AAAA,EACtB,qBAAqB;AACtB;", | ||
| "names": ["options"] | ||
| } |
No README
QualityPackage does not have a README. This may indicate a failed publish or a low quality package.
Found 1 instance in 1 package
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 1 instance in 1 package
85446
9.95%32
28%1058
10.9%1
-50%52
Infinity%0
-100%8
300%+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
- Removed
- Removed
Updated