@ginger.io/beyonce - npm Package Compare versions

Comparing version 0.0.54 to 0.0.56

dist/dynamo/Beyonce.d.ts

@@ -5,3 +5,3 @@ import { JayZ } from "@ginger.io/jay-z";
 import { QueryBuilder } from "./QueryBuilder";
-import { ParallelScanConfig, ScanBuilder } from "./ScanBuilder";
+import { ScanBuilder, ParallelScanConfig } from "./ScanBuilder";
 import { Table } from "./Table";
@@ -8,0 +8,0 @@ import { ExtractKeyType, GroupedModels, TaggedModel } from "./types";

dist/dynamo/expressions/QueryExpressionBuilder.d.ts

@@ -23,5 +23,4 @@ import { KeysOf } from "../../typeUtils";
     protected addStatement(statement: string): void;
-    protected reset(): void;
     private addCondition;
 }
 //# sourceMappingURL=QueryExpressionBuilder.d.ts.map

dist/dynamo/expressions/QueryExpressionBuilder.js

@@ -62,3 +62,3 @@ "use strict";
             attributeNames,
-            attributeValues,
+            attributeValues
         };
@@ -75,7 +75,2 @@ }
     }
-    reset() {
-        this.attributeNames = new Attributes_1.Attributes();
-        this.attributeValues = new Variables_1.Variables();
-        this.statements = [];
-    }
     addCondition(params) {
@@ -82,0 +77,0 @@ const attributePlaceholder = this.addAttributeName(params.attribute);

dist/dynamo/pagedIterator.d.ts

@@ -22,3 +22,3 @@ import { JayZ } from "@ginger.io/jay-z";
     items: GroupedModels<T>;
-    errors?: Error[];
+    errors: Error[];
     cursor?: Cursor;
@@ -25,0 +25,0 @@ };

dist/dynamo/QueryBuilder.d.ts

@@ -32,3 +32,3 @@ import { JayZ } from "@ginger.io/jay-z";
     iterator(options?: IteratorOptions): PaginatedQueryResults<T>;
-    private buildQuery;
+    private createQueryInput;
     private buildKeyConditionForTable;
@@ -35,0 +35,0 @@ private buildKeyConditionForGSI;

dist/dynamo/QueryBuilder.js

@@ -49,3 +49,4 @@ "use strict";
         return __awaiter(this, void 0, void 0, function* () {
-            const iterator = pagedIterator_1.pagedIterator({}, (options) => this.buildQuery(options), (query) => this.config.db.query(query).promise(), this.config.jayz);
+            const query = this.createQueryInput({});
+            const iterator = pagedIterator_1.pagedIterator({}, ({ cursor, pageSize }) => (Object.assign(Object.assign({}, query), { ExclusiveStartKey: cursor, Limit: pageSize })), (query) => this.config.db.query(query).promise(), this.config.jayz);
             return pagedIterator_1.groupAllPages(iterator, this.modelTags);
@@ -57,3 +58,4 @@ });
             var e_1, _a;
-            const iterator = pagedIterator_1.pagedIterator(options, (options) => this.buildQuery(options), (query) => this.config.db.query(query).promise(), this.config.jayz);
+            const query = this.createQueryInput(options);
+            const iterator = pagedIterator_1.pagedIterator(options, ({ cursor, pageSize }) => (Object.assign(Object.assign({}, query), { ExclusiveStartKey: cursor, Limit: pageSize })), (query) => this.config.db.query(query).promise(), this.config.jayz);
             try {
@@ -64,3 +66,4 @@ for (var iterator_2 = __asyncValues(iterator), iterator_2_1; iterator_2_1 = yield __await(iterator_2.next()), !iterator_2_1.done;) {
                         items: groupModelsByType_1.groupModelsByType(response.items, this.modelTags),
-                        cursor: response.lastEvaluatedKey,
+                        errors: response.errors,
+                        cursor: response.lastEvaluatedKey
                     });
@@ -78,8 +81,8 @@ }
                 items: groupModelsByType_1.groupModelsByType([], this.modelTags),
-                cursor: undefined,
+                errors: [],
+                cursor: undefined
             });
         });
     }
-    buildQuery(options) {
-        let query;
+    createQueryInput(options) {
         if (isTableQuery(this.config)) {
@@ -90,3 +93,3 @@ const { table, consistentRead } = this.config;
             const filterExp = expression !== "" ? expression : undefined;
-            query = {
+            return {
                 TableName: table.tableName,
@@ -100,3 +103,3 @@ ConsistentRead: consistentRead,
                 ScanIndexForward: this.scanIndexForward,
-                Limit: options.pageSize,
+                Limit: options.pageSize
             };
@@ -109,3 +112,3 @@ }
             const filterExp = expression !== "" ? expression : undefined;
-            query = {
+            return {
                 TableName: table.tableName,
@@ -120,7 +123,5 @@ ConsistentRead: consistentRead,
                 ScanIndexForward: this.scanIndexForward,
-                Limit: options.pageSize,
+                Limit: options.pageSize
             };
         }
-        this.reset();
-        return query;
     }
@@ -127,0 +128,0 @@ buildKeyConditionForTable(config) {

dist/dynamo/ScanBuilder.d.ts

@@ -25,5 +25,5 @@ import { JayZ } from "@ginger.io/jay-z";
     iterator(options?: IteratorOptions): PaginatedQueryResults<T>;
-    private buildScan;
+    private createScanInput;
 }
 export {};
 //# sourceMappingURL=ScanBuilder.d.ts.map

dist/dynamo/ScanBuilder.js

@@ -44,3 +44,4 @@ "use strict";
         return __awaiter(this, void 0, void 0, function* () {
-            const iterator = pagedIterator_1.pagedIterator({}, (options) => this.buildScan(options), (input) => this.config.db.scan(input).promise(), this.config.jayz);
+            const scanInput = this.createScanInput();
+            const iterator = pagedIterator_1.pagedIterator({}, ({ cursor, pageSize }) => (Object.assign(Object.assign({}, scanInput), { ExclusiveStartKey: cursor, Limit: pageSize })), (input) => this.config.db.scan(input).promise(), this.config.jayz);
             return pagedIterator_1.groupAllPages(iterator, this.modelTags);
@@ -52,3 +53,4 @@ });
             var e_1, _a;
-            const iterator = pagedIterator_1.pagedIterator(options, (options) => this.buildScan(options), (input) => this.config.db.scan(input).promise(), this.config.jayz);
+            const scanInput = this.createScanInput(options);
+            const iterator = pagedIterator_1.pagedIterator(options, ({ cursor, pageSize }) => (Object.assign(Object.assign({}, scanInput), { ExclusiveStartKey: cursor, Limit: pageSize })), (input) => this.config.db.scan(input).promise(), this.config.jayz);
             try {
@@ -73,2 +75,3 @@ for (var iterator_2 = __asyncValues(iterator), iterator_2_1; iterator_2_1 = yield __await(iterator_2.next()), !iterator_2_1.done;) {
                 items: groupModelsByType_1.groupModelsByType([], this.modelTags),
+                errors: [],
                 cursor: undefined
@@ -78,3 +81,3 @@ });
     }
-    buildScan(options) {
+    createScanInput(iteratorOptions) {
         const { table, consistentRead, parallel } = this.config;
@@ -85,3 +88,3 @@ const { expression, attributeNames, attributeValues } = this.build();
         const includeAttributeValues = filterExp !== undefined && Object.values(attributeValues).length > 0;
-        const scan = {
+        return {
             TableName: table.tableName,
@@ -96,9 +99,7 @@ ConsistentRead: consistentRead,
             FilterExpression: filterExp,
-            ExclusiveStartKey: options.cursor,
-            Limit: options.pageSize,
+            ExclusiveStartKey: iteratorOptions === null || iteratorOptions === void 0 ? void 0 : iteratorOptions.cursor,
+            Limit: iteratorOptions === null || iteratorOptions === void 0 ? void 0 : iteratorOptions.pageSize,
             Segment: parallel === null || parallel === void 0 ? void 0 : parallel.segmentId,
             TotalSegments: parallel === null || parallel === void 0 ? void 0 : parallel.totalSegments
         };
-        this.reset();
-        return scan;
     }
@@ -105,0 +106,0 @@ }

package.json

 {
   "name": "@ginger.io/beyonce",
-  "version": "0.0.54",
+  "version": "0.0.56",
   "description": "Type-safe DynamoDB query builder for TypeScript. Designed with single-table architecture in mind.",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

src/main/dynamo/Beyonce.ts

@@ -12,3 +12,3 @@ import { JayZ } from "@ginger.io/jay-z"
 import { QueryBuilder } from "./QueryBuilder"
-import { ParallelScanConfig, ScanBuilder } from "./ScanBuilder"
+import { ScanBuilder, ParallelScanConfig } from "./ScanBuilder"
 import { Table } from "./Table"
@@ -15,0 +15,0 @@ import { ExtractKeyType, GroupedModels, TaggedModel } from "./types"

src/main/dynamo/expressions/QueryExpressionBuilder.ts

@@ -74,3 +74,3 @@ import { KeysOf } from "../../typeUtils"
       attributeNames,
-      attributeValues,
+      attributeValues
     }
@@ -90,8 +90,2 @@ }
 
-  protected reset(): void {
-    this.attributeNames = new Attributes()
-    this.attributeValues = new Variables()
-    this.statements = []
-  }
-
   private addCondition(params: {
@@ -98,0 +92,0 @@ attribute: string

src/main/dynamo/pagedIterator.ts

@@ -34,3 +34,3 @@ import { JayZ } from "@ginger.io/jay-z"
   items: GroupedModels<T>
-  errors?: Error[]
+  errors: Error[]
   cursor?: Cursor
@@ -37,0 +37,0 @@ }

src/main/dynamo/QueryBuilder.ts

@@ -11,3 +11,3 @@ import { JayZ } from "@ginger.io/jay-z"
   pagedIterator,
-  PaginatedQueryResults,
+  PaginatedQueryResults
 } from "./pagedIterator"
@@ -51,5 +51,10 @@ import { Table } from "./Table"
   async exec(): Promise<GroupedModels<T>> {
+    const query = this.createQueryInput({})
     const iterator = pagedIterator<DocumentClient.QueryInput, T>(
       {},
-      (options) => this.buildQuery(options),
+      ({ cursor, pageSize }) => ({
+        ...query,
+        ExclusiveStartKey: cursor,
+        Limit: pageSize
+      }),
       (query) => this.config.db.query(query).promise(),
@@ -63,5 +68,10 @@ this.config.jayz
   async *iterator(options: IteratorOptions = {}): PaginatedQueryResults<T> {
+    const query = this.createQueryInput(options)
     const iterator = pagedIterator<DocumentClient.QueryInput, T>(
       options,
-      (options) => this.buildQuery(options),
+      ({ cursor, pageSize }) => ({
+        ...query,
+        ExclusiveStartKey: cursor,
+        Limit: pageSize
+      }),
       (query) => this.config.db.query(query).promise(),
@@ -74,3 +84,4 @@ this.config.jayz
         items: groupModelsByType(response.items, this.modelTags),
-        cursor: response.lastEvaluatedKey,
+        errors: response.errors,
+        cursor: response.lastEvaluatedKey
       }
@@ -81,10 +92,10 @@ }
       items: groupModelsByType<T>([], this.modelTags),
-      cursor: undefined,
+      errors: [],
+      cursor: undefined
     }
   }
 
-  private buildQuery(
+  private createQueryInput(
     options: IteratorOptions
   ): DynamoDB.DocumentClient.QueryInput {
-    let query: DynamoDB.DocumentClient.QueryInput
     if (isTableQuery(this.config)) {
@@ -96,3 +107,3 @@ const { table, consistentRead } = this.config
 
-      query = {
+      return {
         TableName: table.tableName,
@@ -106,3 +117,3 @@ ConsistentRead: consistentRead,
         ScanIndexForward: this.scanIndexForward,
-        Limit: options.pageSize,
+        Limit: options.pageSize
       }
@@ -115,3 +126,3 @@ } else {
 
-      query = {
+      return {
         TableName: table.tableName,
@@ -126,8 +137,5 @@ ConsistentRead: consistentRead,
         ScanIndexForward: this.scanIndexForward,
-        Limit: options.pageSize,
+        Limit: options.pageSize
       }
     }
-
-    this.reset()
-    return query
   }
@@ -134,0 +142,0 @@

src/main/dynamo/ScanBuilder.ts

@@ -39,5 +39,10 @@ import { JayZ } from "@ginger.io/jay-z"
   async exec(): Promise<GroupedModels<T>> {
+    const scanInput = this.createScanInput()
     const iterator = pagedIterator<DocumentClient.ScanInput, T>(
       {},
-      (options) => this.buildScan(options),
+      ({ cursor, pageSize }) => ({
+        ...scanInput,
+        ExclusiveStartKey: cursor,
+        Limit: pageSize
+      }),
       (input) => this.config.db.scan(input).promise(),
@@ -51,5 +56,10 @@ this.config.jayz
   async *iterator(options: IteratorOptions = {}): PaginatedQueryResults<T> {
+    const scanInput = this.createScanInput(options)
     const iterator = pagedIterator<DocumentClient.ScanInput, T>(
       options,
-      (options) => this.buildScan(options),
+      ({ cursor, pageSize }) => ({
+        ...scanInput,
+        ExclusiveStartKey: cursor,
+        Limit: pageSize
+      }),
       (input) => this.config.db.scan(input).promise(),
@@ -69,2 +79,3 @@ this.config.jayz
       items: groupModelsByType<T>([], this.modelTags),
+      errors: [],
       cursor: undefined
@@ -74,9 +85,6 @@ }
 
-  private buildScan(
-    options: IteratorOptions
-  ): DynamoDB.DocumentClient.ScanInput {
+  private createScanInput(iteratorOptions?: IteratorOptions) {
     const { table, consistentRead, parallel } = this.config
     const { expression, attributeNames, attributeValues } = this.build()
     const filterExp = expression !== "" ? expression : undefined
-
     const includeAttributeNames = filterExp !== undefined
@@ -86,3 +94,3 @@ const includeAttributeValues =
 
-    const scan = {
+    return {
       TableName: table.tableName,
@@ -97,11 +105,8 @@ ConsistentRead: consistentRead,
       FilterExpression: filterExp,
-      ExclusiveStartKey: options.cursor,
-      Limit: options.pageSize,
+      ExclusiveStartKey: iteratorOptions?.cursor,
+      Limit: iteratorOptions?.pageSize,
       Segment: parallel?.segmentId,
       TotalSegments: parallel?.totalSegments
     }
-
-    this.reset()
-    return scan
   }
 }

src/test/dynamo/Beyonce.test.ts

@@ -126,3 +126,3 @@ import { JayZ } from "@ginger.io/jay-z"
       })
-      .buildQuery({})
+      .createQueryInput({})
 
@@ -129,0 +129,0 @@ expect(query).toMatchObject({

src/test/dynamo/query.test.ts

@@ -5,7 +5,9 @@ import { JayZ } from "@ginger.io/jay-z"
   ModelType,
+  Musician,
   MusicianModel,
   MusicianPartition,
+  Song,
   SongModel
 } from "./models"
-import { createJayZ, createSongs, setup } from "./util"
+import { createJayZ, create25Songs, setup } from "./util"
 
@@ -23,3 +25,3 @@ describe("Beyonce.query", () => {
       })
-      .buildQuery({})
+      .createQueryInput({})
 
@@ -43,2 +45,6 @@ expect(query).toMatchObject({
 
+  it("should filter and paginate query results", async () => {
+    await testPaginatedQueryWithFilter()
+  })
+
   it("should query with multiple attribute filters", async () => {
@@ -74,3 +80,3 @@ await testQueryWithCombinedAttributeFilters()
       })
-      .buildQuery({})
+      .createQueryInput({})
 
@@ -97,2 +103,7 @@ expect(query).toMatchObject({
 
+  it("should filter and paginate query results", async () => {
+    const jayz = await createJayZ()
+    await testPaginatedQueryWithFilter(jayz)
+  })
+
   it("should query with multiple attribute filters", async () => {
@@ -127,3 +138,3 @@ const jayz = await createJayZ()
   const db = await setup(jayZ)
-  const songs = await createSongs(db)
+  const songs = await create25Songs(db)
   const results = await db.query(MusicianPartition.key({ id: "1" })).exec()
@@ -146,2 +157,29 @@ expect(results.song.length).toEqual(songs.length)
 
+async function testPaginatedQueryWithFilter(jayZ?: JayZ) {
+  const db = await setup(jayZ)
+  await create25Songs(db)
+  const musician = MusicianModel.create({
+    id: "1",
+    name: "ZZ Top",
+    divaRating: 10,
+    details: {}
+  })
+  await db.put(musician)
+
+  const iterator = db
+    .query(MusicianPartition.key({ id: musician.id }))
+    .where("model", "=", ModelType.Musician)
+    .iterator()
+
+  const musiciansProcessed: Musician[] = []
+  const songsProcessed: Song[] = []
+  for await (const { items } of iterator) {
+    musiciansProcessed.push(...items.musician)
+    songsProcessed.push(...items.song)
+  }
+
+  expect(musiciansProcessed).toEqual([musician])
+  expect(songsProcessed.length).toEqual(0)
+}
+
 async function testQueryForSingleTypeOfModel(jayZ?: JayZ) {
@@ -148,0 +186,0 @@ const db = await setup(jayZ)

src/test/dynamo/scan.test.ts

 import { DataKeyProvider, JayZ } from "@ginger.io/jay-z"
 import crypto from "crypto"
 import { crypto_kdf_KEYBYTES, randombytes_buf } from "libsodium-wrappers"
-import { aMusicianWithTwoSongs, ModelType, Song, SongModel } from "./models"
 import {
+  aMusicianWithTwoSongs,
+  ModelType,
+  Musician,
+  MusicianModel,
+  Song,
+  SongModel
+} from "./models"
+import {
   createBeyonce,
   createDynamoDB,
   createJayZ,
-  createSongs,
+  create25Songs,
   setup
@@ -26,2 +33,6 @@ } from "./util"
 
+  it("should filter paginated scan results", async () => {
+    await testScanWithFilteredPaginatedResults()
+  })
+
   it("should scan with multiple attribute filters", async () => {
@@ -56,2 +67,7 @@ await testScanWithCombinedAttributeFilters()
 
+  it("should filter paginated scan results", async () => {
+    const jayz = await createJayZ()
+    await testScanWithFilteredPaginatedResults(jayz)
+  })
+
   it("should scan with multiple attribute filters", async () => {
@@ -76,3 +92,3 @@ const jayz = await createJayZ()
     const db = await setup(jayz)
-    await createSongs(db, 25)
+    await create25Songs(db)
 
@@ -128,3 +144,3 @@ // And one last song that is encrypted incorrectly (e.g. with a different key)
   const db = await setup(jayZ)
-  const songs = await createSongs(db)
+  const songs = await create25Songs(db)
   const results = await db.scan().exec()
@@ -134,5 +150,31 @@ expect(results.song.length).toEqual(songs.length)
 
+async function testScanWithFilteredPaginatedResults(jayZ?: JayZ) {
+  const db = await setup(jayZ)
+  const songs = await create25Songs(db)
+  const musician = MusicianModel.create({
+    id: "zz-top",
+    divaRating: 10,
+    name: "ZZ Top",
+    details: {}
+  })
+
+  await db.put(musician)
+
+  const musiciansProcessed: Musician[] = []
+  const songsProcessed: Song[] = []
+  for await (const { items } of db
+    .scan<Musician | Song>() // type-system hack so we can assert we don't get Songs in the results
+    .where("model", "=", ModelType.Musician)
+    .iterator()) {
+    musiciansProcessed.push(...items.musician)
+    songsProcessed.push(...items.song)
+  }
+
+  expect(musiciansProcessed).toEqual([musician])
+  expect(songsProcessed.length).toEqual(0)
+}
+
 async function testParallelScan(jayZ?: JayZ) {
   const db = await setup(jayZ)
-  const songs = await createSongs(db)
+  const songs = await create25Songs(db)
 
@@ -139,0 +181,0 @@ const segment1 = db

src/test/dynamo/util.ts

@@ -54,17 +54,21 @@ import { FixedDataKeyProvider, JayZ } from "@ginger.io/jay-z"
 // i.e. at least 3 pages. Note that data encrypted with JayZ is significantly larger
-export async function createSongs(
-  db: Beyonce,
-  n: number = 25
-): Promise<Song[]> {
-  const mp3 = crypto.randomBytes(100_000)
-  const songs: Song[] = [...Array(n).keys()].map((songId) =>
-    SongModel.create({
-      musicianId: "1",
-      id: songId.toString(),
-      title: `Song ${songId}`,
-      mp3
-    })
-  )
-  await Promise.all(songs.map((song) => db.put(song)))
+const mp3 = crypto.randomBytes(100_000)
+const songs: Song[] = [...Array(25).keys()].map((songId) =>
+  SongModel.create({
+    musicianId: "1",
+    id: songId.toString(),
+    title: `Song ${songId}`,
+    mp3
+  })
+)
+export async function create25Songs(db: Beyonce): Promise<Song[]> {
+  // Batch these to avoid DynamoDB local throwing errors about exceeding
+  // the max payload size
+  await Promise.all([
+    db.batchPutWithTransaction({ items: songs.slice(0, 5) }),
+    db.batchPutWithTransaction({ items: songs.slice(5, 10) }),
+    db.batchPutWithTransaction({ items: songs.slice(10, 15) }),
+    db.batchPutWithTransaction({ items: songs.slice(15) })
+  ])
   return songs
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

282389

increased by5.97%

Number of package files

178

increased by4.71%

Lines of code

5101

increased by4.85%

No dependency changes