convex-helpers - npm Package Compare versions

Comparing version 0.1.64-alpha.0 to 0.1.64-alpha.1

package.json

 {
   "name": "convex-helpers",
-  "version": "0.1.64-alpha.0",
+  "version": "0.1.64-alpha.1",
   "description": "A collection of useful code to complement the official convex package.",
@@ -5,0 +5,0 @@ "type": "module",

README.md

@@ -497,3 +497,3 @@ # convex-helpers
 
-See the [Stack post on Zod validation](https://stack.convex.dev/wrappers-as-middleware-zod-validation) to see how to validate your Convex functions using the [zod](https://www.npmjs.com/package/zod) library.
+See the [Stack post on Zod validation](https://stack.convex.dev/typescript-zod-function-validation) to see how to validate your Convex functions using the [zod](https://www.npmjs.com/package/zod) library.
 
@@ -781,2 +781,60 @@ Example:
 
+### `paginator`: manual pagination with familiar syntax
+
+In addition to `getPage`, convex-helpers provides a function
+`paginator` as an alternative to the built-in `db.query.paginate`.
+
+- The built-in `.paginate` is currently limited to one call per query, which allows
+  it to track the page's "end cursor" for contiguous reactive pagination client-side.
+- `paginator` can be called multiple times from a query,
+  but does not subscribe the query to the end cursor automatically.
+
+The syntax and interface for `paginator` is so similar to `.paginate` that it is
+nearly a drop-in replacement and can even be used with `usePaginatedQuery`.
+This makes it more suitable for non-reactive pagination usecases,
+such as iterating data in a mutation. Note: it supports `withIndex` but not `filter`.
+
+For more information on reactive pagination and end cursors, see
+https://stack.convex.dev/fully-reactive-pagination
+and
+https://stack.convex.dev/pagination
+
+As a basic example, consider replacing this query with `paginator`.
+It has the same behavior, except that the pages might not stay contiguous as
+items are added and removed from the list and the query updates reactively.
+
+```ts
+import { paginator } from "convex-helpers/server/pagination";
+import schema from "./schema";
+
+export const list = query({
+  args: { opts: paginationOptsValidator },
+  handler: async (ctx, { opts }) => {
+    // BEFORE:
+    return await ctx.db.query("messages").paginate(opts);
+    // AFTER:
+    return await paginator(ctx.db, schema).query("messages").paginate(opts);
+  },
+});
+```
+
+You can order by an index, restrict the pagination to a range of the index,
+and change the order to "desc", same as you would with a regular query.
+
+```ts
+import { paginator } from "convex-helpers/server/pagination";
+import schema from "./schema";
+
+export const list = query({
+  args: { opts: paginationOptsValidator, author: v.id("users") },
+  handler: async (ctx, { opts, author }) => {
+    return await paginator(ctx.db, schema)
+      .query("messages")
+      .withIndex("by_author", q=>q.eq("author", author))
+      .order("desc")
+      .paginate(opts);
+  },
+});
+```
+
 ## Query Caching
@@ -783,0 +841,0 @@

server/customFunctions.test.ts

@@ -524,2 +524,9 @@ import { Equals, assert } from "../index.js";
   });
+
+  test("still validates args", async () => {
+    const t = convexTest(schema, modules);
+    expect(() => t.query(testApi.redefine, { a: 3 as any })).rejects.toThrow(
+      "Validator error: Expected `string`",
+    );
+  });
 });
@@ -547,2 +554,9 @@
   });
+
+  test("still validates args", async () => {
+    const t = convexTest(schema, modules);
+    expect(() =>
+      t.query(testApi.outerAdds, { a: 3 as any, outer: "" }),
+    ).rejects.toThrow("Validator error: Expected `string`");
+  });
 });

server/pagination.d.ts

 import { Value } from "convex/values";
-import { DocumentByName, GenericDataModel, GenericDatabaseReader, IndexNames, NamedTableInfo, SchemaDefinition, TableNamesInDataModel } from "convex/server";
+import { DataModelFromSchemaDefinition, DocumentByInfo, DocumentByName, GenericDataModel, GenericDatabaseReader, IndexNames, IndexRange, IndexRangeBuilder, NamedIndex, NamedTableInfo, OrderedQuery, PaginationOptions, PaginationResult, Query, QueryInitializer, SchemaDefinition, TableNamesInDataModel } from "convex/server";
 export type IndexKey = Value[];
@@ -69,2 +69,123 @@ export type PageRequest<DataModel extends GenericDataModel, T extends TableNamesInDataModel<DataModel>> = {
 }, request: PageRequest<DataModel, T>): Promise<PageResponse<DataModel, T>>;
+/**
+ * Simpified version of `getPage` that you can use for one-off queries that
+ * don't need to be reactive.
+ *
+ * These two queries are roughly equivalent:
+ *
+ * ```ts
+ * await db.query(table)
+ *  .withIndex(index, q=>q.eq(field, value))
+ *  .order("desc")
+ *  .paginate(opts)
+ *
+ * await paginator(db, schema)
+ *   .query(table)
+ *   .withIndex(index, q=>q.eq(field, value))
+ *   .order("desc")
+ *   .paginate(opts)
+ * ```
+ *
+ * Differences:
+ *
+ * - `paginator` does not automatically track the end of the page for when
+ *   the query reruns. The standard `paginate` call will record the end of the page,
+ *   so a client can have seamless reactive pagination. To pin the end of the page,
+ *   you can use the `endCursor` option. This does not happen automatically.
+ *   Read more [here](https://stack.convex.dev/pagination#stitching-the-pages-together)
+ * - `paginator` can be called multiple times in a query or mutation,
+ *   and within Convex components.
+ * - Cursors are not encrypted.
+ * - `.filter()` and the `filter()` convex-helper are not supported.
+ *   Filter the returned `page` in TypeScript instead.
+ * - System tables like _storage and _scheduled_functions are not supported.
+ * - Having a schema is required.
+ *
+ * @argument opts.cursor Where to start the page. This should come from
+ * `continueCursor` in the previous page.
+ * @argument opts.endCursor Where to end the page. This should from from
+ * `continueCursor` in the *current* page.
+ * If not provided, the page will end when it reaches `options.opts.numItems`.
+ * @argument options.schema If you use an index that is not by_creation_time
+ * or by_id, you need to provide the schema.
+ */
+export declare function paginator<Schema extends SchemaDefinition<any, boolean>>(db: GenericDatabaseReader<DataModelFromSchemaDefinition<Schema>>, schema: Schema): PaginatorDatabaseReader<DataModelFromSchemaDefinition<Schema>>;
+export declare class PaginatorDatabaseReader<DataModel extends GenericDataModel> implements GenericDatabaseReader<DataModel> {
+    db: GenericDatabaseReader<DataModel>;
+    schema: SchemaDefinition<any, boolean>;
+    system: any;
+    constructor(db: GenericDatabaseReader<DataModel>, schema: SchemaDefinition<any, boolean>);
+    query<TableName extends TableNamesInDataModel<DataModel>>(tableName: TableName): PaginatorQueryInitializer<DataModel, TableName>;
+    get(_id: any): any;
+    normalizeId(_tableName: any, _id: any): any;
+}
+export declare class PaginatorQueryInitializer<DataModel extends GenericDataModel, T extends TableNamesInDataModel<DataModel>> implements QueryInitializer<NamedTableInfo<DataModel, T>> {
+    parent: PaginatorDatabaseReader<DataModel>;
+    table: T;
+    constructor(parent: PaginatorDatabaseReader<DataModel>, table: T);
+    fullTableScan(): PaginatorQuery<DataModel, T>;
+    withIndex<IndexName extends IndexNames<NamedTableInfo<DataModel, T>>>(indexName: IndexName, indexRange?: (q: IndexRangeBuilder<DocumentByInfo<NamedTableInfo<DataModel, T>>, NamedIndex<NamedTableInfo<DataModel, T>, IndexName>>) => IndexRange): PaginatorQuery<DataModel, T>;
+    withSearchIndex(_indexName: any, _searchFilter: any): any;
+    order(order: "asc" | "desc"): OrderedPaginatorQuery<DataModel, T>;
+    paginate(opts: PaginationOptions & {
+        endCursor?: string | null;
+    }): Promise<PaginationResult<DocumentByInfo<NamedTableInfo<DataModel, T>>>>;
+    filter(_predicate: any): any;
+    collect(): any;
+    first(): any;
+    unique(): any;
+    take(_n: number): any;
+    [Symbol.asyncIterator](): any;
+}
+export declare class PaginatorQuery<DataModel extends GenericDataModel, T extends TableNamesInDataModel<DataModel>> implements Query<NamedTableInfo<DataModel, T>> {
+    parent: PaginatorQueryInitializer<DataModel, T>;
+    index: IndexNames<NamedTableInfo<DataModel, T>>;
+    q: PaginatorIndexRange;
+    constructor(parent: PaginatorQueryInitializer<DataModel, T>, index: IndexNames<NamedTableInfo<DataModel, T>>, q: PaginatorIndexRange);
+    order(order: "asc" | "desc"): OrderedPaginatorQuery<DataModel, T>;
+    paginate(opts: PaginationOptions & {
+        endCursor?: string | null;
+    }): Promise<PaginationResult<DocumentByInfo<NamedTableInfo<DataModel, T>>>>;
+    filter(_predicate: any): this;
+    collect(): any;
+    first(): any;
+    unique(): any;
+    take(_n: number): any;
+    [Symbol.asyncIterator](): any;
+}
+export declare class OrderedPaginatorQuery<DataModel extends GenericDataModel, T extends TableNamesInDataModel<DataModel>> implements OrderedQuery<NamedTableInfo<DataModel, T>> {
+    parent: PaginatorQuery<DataModel, T>;
+    order: "asc" | "desc";
+    startIndexKey: IndexKey | undefined;
+    startInclusive: boolean;
+    endIndexKey: IndexKey | undefined;
+    endInclusive: boolean;
+    constructor(parent: PaginatorQuery<DataModel, T>, order: "asc" | "desc");
+    paginate(opts: PaginationOptions & {
+        endCursor?: string | null;
+    }): Promise<PaginationResult<DocumentByName<DataModel, T>>>;
+    filter(_predicate: any): any;
+    collect(): any;
+    first(): any;
+    unique(): any;
+    take(_n: number): any;
+    [Symbol.asyncIterator](): any;
+}
+declare class PaginatorIndexRange {
+    indexFields: string[];
+    private hasSuffix;
+    lowerBoundIndexKey: IndexKey | undefined;
+    lowerBoundInclusive: boolean;
+    upperBoundIndexKey: IndexKey | undefined;
+    upperBoundInclusive: boolean;
+    constructor(indexFields: string[]);
+    eq(field: string, value: Value): this;
+    lt(field: string, value: Value): this;
+    lte(field: string, value: Value): this;
+    gt(field: string, value: Value): this;
+    gte(field: string, value: Value): this;
+    private canLowerBound;
+    private canUpperBound;
+}
+export {};
 //# sourceMappingURL=pagination.d.ts.map

server/pagination.js

@@ -1,2 +0,2 @@
-import { convexToJson } from "convex/values";
+import { convexToJson, jsonToConvex } from "convex/values";
 /**
@@ -191,1 +191,320 @@ * Get a single page of documents from a table.
 }
+const END_CURSOR = "endcursor";
+/**
+ * Simpified version of `getPage` that you can use for one-off queries that
+ * don't need to be reactive.
+ *
+ * These two queries are roughly equivalent:
+ *
+ * ```ts
+ * await db.query(table)
+ *  .withIndex(index, q=>q.eq(field, value))
+ *  .order("desc")
+ *  .paginate(opts)
+ *
+ * await paginator(db, schema)
+ *   .query(table)
+ *   .withIndex(index, q=>q.eq(field, value))
+ *   .order("desc")
+ *   .paginate(opts)
+ * ```
+ *
+ * Differences:
+ *
+ * - `paginator` does not automatically track the end of the page for when
+ *   the query reruns. The standard `paginate` call will record the end of the page,
+ *   so a client can have seamless reactive pagination. To pin the end of the page,
+ *   you can use the `endCursor` option. This does not happen automatically.
+ *   Read more [here](https://stack.convex.dev/pagination#stitching-the-pages-together)
+ * - `paginator` can be called multiple times in a query or mutation,
+ *   and within Convex components.
+ * - Cursors are not encrypted.
+ * - `.filter()` and the `filter()` convex-helper are not supported.
+ *   Filter the returned `page` in TypeScript instead.
+ * - System tables like _storage and _scheduled_functions are not supported.
+ * - Having a schema is required.
+ *
+ * @argument opts.cursor Where to start the page. This should come from
+ * `continueCursor` in the previous page.
+ * @argument opts.endCursor Where to end the page. This should from from
+ * `continueCursor` in the *current* page.
+ * If not provided, the page will end when it reaches `options.opts.numItems`.
+ * @argument options.schema If you use an index that is not by_creation_time
+ * or by_id, you need to provide the schema.
+ */
+export function paginator(db, schema) {
+    return new PaginatorDatabaseReader(db, schema);
+}
+export class PaginatorDatabaseReader {
+    db;
+    schema;
+    // TODO: support system tables
+    system = null;
+    constructor(db, schema) {
+        this.db = db;
+        this.schema = schema;
+    }
+    query(tableName) {
+        return new PaginatorQueryInitializer(this, tableName);
+    }
+    get(_id) {
+        throw new Error("get() not supported for `paginator`");
+    }
+    normalizeId(_tableName, _id) {
+        throw new Error("normalizeId() not supported for `paginator`.");
+    }
+}
+export class PaginatorQueryInitializer {
+    parent;
+    table;
+    constructor(parent, table) {
+        this.parent = parent;
+        this.table = table;
+    }
+    fullTableScan() {
+        return this.withIndex("by_creation_time");
+    }
+    withIndex(indexName, indexRange) {
+        const indexFields = getIndexFields({
+            table: this.table,
+            index: indexName,
+            schema: this.parent.schema,
+        });
+        const q = new PaginatorIndexRange(indexFields);
+        if (indexRange) {
+            indexRange(q);
+        }
+        return new PaginatorQuery(this, indexName, q);
+    }
+    withSearchIndex(_indexName, _searchFilter) {
+        throw new Error("Cannot paginate withSearchIndex");
+    }
+    order(order) {
+        return this.fullTableScan().order(order);
+    }
+    paginate(opts) {
+        return this.fullTableScan().paginate(opts);
+    }
+    filter(_predicate) {
+        throw new Error(".filter() not supported for `paginator`. Filter the returned `page` instead.");
+    }
+    collect() {
+        throw new Error(".collect() not supported for `paginator`. Use .paginate() instead.");
+    }
+    first() {
+        throw new Error(".first() not supported for `paginator`. Use .paginate() instead.");
+    }
+    unique() {
+        throw new Error(".unique() not supported for `paginator`. Use .paginate() instead.");
+    }
+    take(_n) {
+        throw new Error(".take() not supported for `paginator`. Use .paginate() instead.");
+    }
+    [Symbol.asyncIterator]() {
+        throw new Error("[Symbol.asyncIterator]() not supported for `paginator`. Use .paginate() instead.");
+    }
+}
+export class PaginatorQuery {
+    parent;
+    index;
+    q;
+    constructor(parent, index, q) {
+        this.parent = parent;
+        this.index = index;
+        this.q = q;
+    }
+    order(order) {
+        return new OrderedPaginatorQuery(this, order);
+    }
+    paginate(opts) {
+        return this.order("asc").paginate(opts);
+    }
+    filter(_predicate) {
+        throw new Error(".filter() not supported for `paginator`. Filter the returned `page` instead.");
+    }
+    collect() {
+        throw new Error(".collect() not supported for `paginator`. Use .paginate() instead.");
+    }
+    first() {
+        throw new Error(".first() not supported for `paginator`. Use .paginate() instead.");
+    }
+    unique() {
+        throw new Error(".unique() not supported for `paginator`. Use .paginate() instead.");
+    }
+    take(_n) {
+        throw new Error(".take() not supported for `paginator`. Use .paginate() instead.");
+    }
+    [Symbol.asyncIterator]() {
+        throw new Error("[Symbol.asyncIterator]() not supported for `paginator`. Use .paginate() instead.");
+    }
+}
+export class OrderedPaginatorQuery {
+    parent;
+    order;
+    startIndexKey;
+    startInclusive;
+    endIndexKey;
+    endInclusive;
+    constructor(parent, order) {
+        this.parent = parent;
+        this.order = order;
+        this.startIndexKey = order === "asc" ? parent.q.lowerBoundIndexKey : parent.q.upperBoundIndexKey;
+        this.endIndexKey = order === "asc" ? parent.q.upperBoundIndexKey : parent.q.lowerBoundIndexKey;
+        this.startInclusive = order === "asc" ? parent.q.lowerBoundInclusive : parent.q.upperBoundInclusive;
+        this.endInclusive = order === "asc" ? parent.q.upperBoundInclusive : parent.q.lowerBoundInclusive;
+    }
+    async paginate(opts) {
+        if (opts.cursor === END_CURSOR) {
+            return {
+                page: [],
+                isDone: true,
+                continueCursor: END_CURSOR,
+            };
+        }
+        const schema = this.parent.parent.parent.schema;
+        let startIndexKey = this.startIndexKey;
+        let startInclusive = this.startInclusive;
+        if (opts.cursor !== null) {
+            startIndexKey = jsonToConvex(JSON.parse(opts.cursor));
+            startInclusive = false;
+        }
+        let endIndexKey = this.endIndexKey;
+        let endInclusive = this.endInclusive;
+        let absoluteMaxRows = opts.numItems;
+        if (opts.endCursor && opts.endCursor !== END_CURSOR) {
+            endIndexKey = jsonToConvex(JSON.parse(opts.endCursor));
+            endInclusive = true;
+            absoluteMaxRows = undefined;
+        }
+        const { page, hasMore, indexKeys, } = await getPage({ db: this.parent.parent.parent.db }, {
+            table: this.parent.parent.table,
+            startIndexKey,
+            startInclusive,
+            endIndexKey,
+            endInclusive,
+            targetMaxRows: opts.numItems,
+            absoluteMaxRows,
+            order: this.order,
+            index: this.parent.index,
+            schema,
+            indexFields: this.parent.q.indexFields,
+        });
+        let continueCursor = END_CURSOR;
+        let isDone = !hasMore;
+        if (opts.endCursor && opts.endCursor !== END_CURSOR) {
+            continueCursor = opts.endCursor;
+            isDone = false;
+        }
+        else if (indexKeys.length > 0 && hasMore) {
+            continueCursor = JSON.stringify(convexToJson(indexKeys[indexKeys.length - 1]));
+        }
+        return {
+            page,
+            isDone,
+            continueCursor,
+        };
+    }
+    filter(_predicate) {
+        throw new Error(".filter() not supported for `paginator`. Filter the returned `page` instead.");
+    }
+    collect() {
+        throw new Error(".collect() not supported for `paginator`. Use .paginate() instead.");
+    }
+    first() {
+        throw new Error(".first() not supported for `paginator`. Use .paginate() instead.");
+    }
+    unique() {
+        throw new Error(".unique() not supported for `paginator`. Use .paginate() instead.");
+    }
+    take(_n) {
+        throw new Error(".take() not supported for `paginator`. Use .paginate() instead.");
+    }
+    [Symbol.asyncIterator]() {
+        throw new Error("[Symbol.asyncIterator]() not supported for `paginator`. Use .paginate() instead.");
+    }
+}
+class PaginatorIndexRange {
+    indexFields;
+    hasSuffix = false;
+    lowerBoundIndexKey = undefined;
+    lowerBoundInclusive = true;
+    upperBoundIndexKey = undefined;
+    upperBoundInclusive = true;
+    constructor(indexFields) {
+        this.indexFields = indexFields;
+    }
+    eq(field, value) {
+        if (!this.canLowerBound(field) || !this.canUpperBound(field)) {
+            throw new Error(`Cannot use eq on field '${field}'`);
+        }
+        this.lowerBoundIndexKey = this.lowerBoundIndexKey ?? [];
+        this.lowerBoundIndexKey.push(value);
+        this.upperBoundIndexKey = this.upperBoundIndexKey ?? [];
+        this.upperBoundIndexKey.push(value);
+        return this;
+    }
+    lt(field, value) {
+        if (!this.canUpperBound(field)) {
+            throw new Error(`Cannot use lt on field '${field}'`);
+        }
+        this.upperBoundIndexKey = this.upperBoundIndexKey ?? [];
+        this.upperBoundIndexKey.push(value);
+        this.upperBoundInclusive = false;
+        this.hasSuffix = true;
+        return this;
+    }
+    lte(field, value) {
+        if (!this.canUpperBound(field)) {
+            throw new Error(`Cannot use lte on field '${field}'`);
+        }
+        this.upperBoundIndexKey = this.upperBoundIndexKey ?? [];
+        this.upperBoundIndexKey.push(value);
+        this.hasSuffix = true;
+        return this;
+    }
+    gt(field, value) {
+        if (!this.canLowerBound(field)) {
+            throw new Error(`Cannot use gt on field '${field}'`);
+        }
+        this.lowerBoundIndexKey = this.lowerBoundIndexKey ?? [];
+        this.lowerBoundIndexKey.push(value);
+        this.lowerBoundInclusive = false;
+        this.hasSuffix = true;
+        return this;
+    }
+    gte(field, value) {
+        if (!this.canLowerBound(field)) {
+            throw new Error(`Cannot use gte on field '${field}'`);
+        }
+        this.lowerBoundIndexKey = this.lowerBoundIndexKey ?? [];
+        this.lowerBoundIndexKey.push(value);
+        this.hasSuffix = true;
+        return this;
+    }
+    canLowerBound(field) {
+        const currentLowerBoundLength = this.lowerBoundIndexKey?.length ?? 0;
+        const currentUpperBoundLength = this.upperBoundIndexKey?.length ?? 0;
+        if (currentLowerBoundLength > currentUpperBoundLength) {
+            // Already have a lower bound.
+            return false;
+        }
+        if (currentLowerBoundLength === currentUpperBoundLength && this.hasSuffix) {
+            // Already have a lower bound and an upper bound.
+            return false;
+        }
+        return currentLowerBoundLength < this.indexFields.length && this.indexFields[currentLowerBoundLength] === field;
+    }
+    canUpperBound(field) {
+        const currentLowerBoundLength = this.lowerBoundIndexKey?.length ?? 0;
+        const currentUpperBoundLength = this.upperBoundIndexKey?.length ?? 0;
+        if (currentUpperBoundLength > currentLowerBoundLength) {
+            // Already have an upper bound.
+            return false;
+        }
+        if (currentLowerBoundLength === currentUpperBoundLength && this.hasSuffix) {
+            // Already have a lower bound and an upper bound.
+            return false;
+        }
+        return currentUpperBoundLength < this.indexFields.length && this.indexFields[currentUpperBoundLength] === field;
+    }
+}

server/pagination.test.ts

 import { defineTable, defineSchema, GenericDocument } from "convex/server";
 import { convexTest } from "convex-test";
 import { expect, test } from "vitest";
-import { IndexKey, getPage } from "./pagination.js";
+import { IndexKey, getPage, paginator } from "./pagination.js";
 import { modules } from "./setup.test.js";
@@ -246,1 +246,166 @@ import { GenericId, v } from "convex/values";
 });
+
+describe("paginator", () => {
+  test("full table scan", async () => {
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      await ctx.db.insert("foo", { a: 1, b: 2, c: 3 });
+      await ctx.db.insert("foo", { a: 1, b: 2, c: 4 });
+      await ctx.db.insert("foo", { a: 1, b: 2, c: 5 });
+      const result1 = await paginator(ctx.db, schema).query("foo")
+        .paginate({ numItems: 100, cursor: null });
+      expect(result1.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 2, c: 3 },
+        { a: 1, b: 2, c: 4 },
+        { a: 1, b: 2, c: 5 },
+      ]);
+      expect(result1.isDone).toBe(true);
+      expect(result1.continueCursor).toBe("endcursor");
+    });
+  });
+
+  test("paginated table scan", async () => {
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      await ctx.db.insert("foo", { a: 1, b: 2, c: 3 });
+      await ctx.db.insert("foo", { a: 1, b: 2, c: 4 });
+      await ctx.db.insert("foo", { a: 1, b: 2, c: 5 });
+      const result1 = await paginator(ctx.db, schema)
+        .query("foo")
+        .paginate({ numItems: 2, cursor: null });
+      expect(result1.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 2, c: 3 },
+        { a: 1, b: 2, c: 4 },
+      ]);
+      expect(result1.isDone).toBe(false);
+      
+      const result2 = await paginator(ctx.db, schema)
+        .query("foo")
+        .paginate({ numItems: 2, cursor: result1.continueCursor });
+      expect(result2.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 2, c: 5 },
+      ]);
+      expect(result2.isDone).toBe(true);
+    });
+  });
+
+  test("index range", async () => {
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      await ctx.db.insert("foo", { a: 1, b: 5, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 6, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 3, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 4, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 4, c: 2 });
+      const result1 = await paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5))
+        .paginate({ cursor: null, numItems: 100 });
+      expect(result1.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 4, c: 1 },
+        { a: 1, b: 4, c: 2 },
+        { a: 1, b: 5, c: 1 },
+      ]);
+      expect(result1.isDone).toBe(true);
+
+      // Descending.
+      const result2 = await paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)).order("desc")
+        .paginate({ cursor: null, numItems: 100 });
+      expect(result2.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 5, c: 1 },
+        { a: 1, b: 4, c: 2 },
+        { a: 1, b: 4, c: 1 },
+      ]);
+      expect(result2.isDone).toBe(true);
+    });
+  });
+
+  test("paginated index range desc", async () => {
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      await ctx.db.insert("foo", { a: 1, b: 5, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 6, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 3, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 4, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 4, c: 2 });
+      const result1 = await paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)).order("desc")
+        .paginate({ cursor: null, numItems: 2 });
+      expect(result1.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 5, c: 1 },
+        { a: 1, b: 4, c: 2 },
+      ]);
+      expect(result1.isDone).toBe(false);
+
+      const result2 = await paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5)).order("desc")
+        .paginate({ cursor: result1.continueCursor, numItems: 2 });
+      expect(result2.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 4, c: 1 },
+      ]);
+      expect(result2.isDone).toBe(true);
+    });
+  });
+
+  test("invalid index range", async () => {
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      expect(() => paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.gt("c" as any, 3))
+      ).toThrow("Cannot use gt on field 'c'");
+      expect(() => paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).eq("c" as any, 3))
+      ).toThrow("Cannot use eq on field 'c'");
+      expect(() => paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => (q.gt("a", 1) as any).gt("b", 3))
+      ).toThrow("Cannot use gt on field 'b'");
+      expect(() => paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => (q.gt("a", 1).lt("a", 3) as any).eq("b", 3))
+      ).toThrow("Cannot use eq on field 'b'");
+    });
+  });
+
+  test("endCursor", async () => {
+    const t = convexTest(schema, modules);
+    await t.run(async (ctx) => {
+      await ctx.db.insert("foo", { a: 1, b: 5, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 6, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 3, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 4, c: 1 });
+      await ctx.db.insert("foo", { a: 1, b: 4, c: 3 });
+      const result1 = await paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5))
+        .paginate({ cursor: null, numItems: 2 });
+      expect(result1.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 4, c: 1 },
+        { a: 1, b: 4, c: 3 },
+      ]);
+      expect(result1.isDone).toBe(false);
+      await ctx.db.insert("foo", { a: 1, b: 4, c: 2 });
+      const result2 = await paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5))
+        .paginate({ cursor: null, endCursor: result1.continueCursor, numItems: 2 });
+      expect(result2.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 4, c: 1 },
+        { a: 1, b: 4, c: 2 },
+        { a: 1, b: 4, c: 3 },
+      ]);
+      expect(result2.isDone).toBe(false);
+      expect(result1.continueCursor).toStrictEqual(result2.continueCursor);
+      const result3 = await paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5))
+        .paginate({ cursor: result2.continueCursor, numItems: 2 });
+      expect(result3.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 5, c: 1 },
+      ]);
+      expect(result3.isDone).toBe(true);
+      const result4 = await paginator(ctx.db, schema)
+        .query("foo").withIndex("abc", q => q.eq("a", 1).gt("b", 3).lte("b", 5))
+        .paginate({ cursor: result2.continueCursor, endCursor: result3.continueCursor, numItems: 2 });
+      expect(result4.page.map(stripSystemFields)).toEqual([
+        { a: 1, b: 5, c: 1 },
+      ]);
+      expect(result4.isDone).toBe(true);
+    });
+  });
+});

server/pagination.ts

@@ -1,3 +0,5 @@
-import { Value, convexToJson } from "convex/values";
+import { Value, convexToJson, jsonToConvex } from "convex/values";
 import {
+  DataModelFromSchemaDefinition,
+  DocumentByInfo,
   DocumentByName,
@@ -7,3 +9,11 @@ GenericDataModel,
   IndexNames,
+  IndexRange,
+  IndexRangeBuilder,
+  NamedIndex,
   NamedTableInfo,
+  OrderedQuery,
+  PaginationOptions,
+  PaginationResult,
+  Query,
+  QueryInitializer,
   SchemaDefinition,
@@ -318,1 +328,347 @@ TableNamesInDataModel,
 }
+
+const END_CURSOR = "endcursor";
+
+/**
+ * Simpified version of `getPage` that you can use for one-off queries that
+ * don't need to be reactive.
+ *
+ * These two queries are roughly equivalent:
+ *
+ * ```ts
+ * await db.query(table)
+ *  .withIndex(index, q=>q.eq(field, value))
+ *  .order("desc")
+ *  .paginate(opts)
+ *
+ * await paginator(db, schema)
+ *   .query(table)
+ *   .withIndex(index, q=>q.eq(field, value))
+ *   .order("desc")
+ *   .paginate(opts)
+ * ```
+ * 
+ * Differences:
+ *
+ * - `paginator` does not automatically track the end of the page for when
+ *   the query reruns. The standard `paginate` call will record the end of the page,
+ *   so a client can have seamless reactive pagination. To pin the end of the page,
+ *   you can use the `endCursor` option. This does not happen automatically.
+ *   Read more [here](https://stack.convex.dev/pagination#stitching-the-pages-together)
+ * - `paginator` can be called multiple times in a query or mutation,
+ *   and within Convex components.
+ * - Cursors are not encrypted.
+ * - `.filter()` and the `filter()` convex-helper are not supported.
+ *   Filter the returned `page` in TypeScript instead.
+ * - System tables like _storage and _scheduled_functions are not supported.
+ * - Having a schema is required.
+ * 
+ * @argument opts.cursor Where to start the page. This should come from
+ * `continueCursor` in the previous page.
+ * @argument opts.endCursor Where to end the page. This should from from
+ * `continueCursor` in the *current* page.
+ * If not provided, the page will end when it reaches `options.opts.numItems`.
+ * @argument options.schema If you use an index that is not by_creation_time
+ * or by_id, you need to provide the schema.
+ */
+export function paginator<
+  Schema extends SchemaDefinition<any, boolean>,
+>(
+  db: GenericDatabaseReader<DataModelFromSchemaDefinition<Schema>>,
+  schema: Schema,
+): PaginatorDatabaseReader<DataModelFromSchemaDefinition<Schema>> {
+  return new PaginatorDatabaseReader(db, schema);
+}
+
+export class PaginatorDatabaseReader<DataModel extends GenericDataModel>
+  implements GenericDatabaseReader<DataModel> {
+
+  // TODO: support system tables
+  public system: any = null;
+
+  constructor(
+    public db: GenericDatabaseReader<DataModel>,
+    public schema: SchemaDefinition<any, boolean>,
+  ) {}
+
+  query<TableName extends TableNamesInDataModel<DataModel>>(
+    tableName: TableName,
+  ): PaginatorQueryInitializer<DataModel, TableName> {
+    return new PaginatorQueryInitializer(this, tableName);
+  }
+  get(_id: any): any {
+    throw new Error("get() not supported for `paginator`");
+  }
+  normalizeId(_tableName: any, _id: any): any {
+    throw new Error("normalizeId() not supported for `paginator`.");
+  }
+}
+
+export class PaginatorQueryInitializer<
+  DataModel extends GenericDataModel,
+  T extends TableNamesInDataModel<DataModel>,
+> implements QueryInitializer<NamedTableInfo<DataModel, T>> {
+  constructor(
+    public parent: PaginatorDatabaseReader<DataModel>,
+    public table: T,
+  ) {}
+  fullTableScan(): PaginatorQuery<DataModel, T> {
+    return this.withIndex("by_creation_time");
+  }
+  withIndex<IndexName extends IndexNames<NamedTableInfo<DataModel, T>>>(
+    indexName: IndexName,
+    indexRange?: (
+      q: IndexRangeBuilder<
+        DocumentByInfo<NamedTableInfo<DataModel, T>>,
+        NamedIndex<NamedTableInfo<DataModel, T>, IndexName>
+      >,
+    ) => IndexRange,
+  ): PaginatorQuery<DataModel, T> {
+    const indexFields = getIndexFields<DataModel, T>({
+      table: this.table,
+      index: indexName,
+      schema: this.parent.schema,
+    });
+    const q = new PaginatorIndexRange(indexFields);
+    if (indexRange) {
+      indexRange(q as any);
+    }
+    return new PaginatorQuery(this, indexName, q);
+  }
+  withSearchIndex(_indexName: any, _searchFilter: any): any {
+    throw new Error("Cannot paginate withSearchIndex");
+  }
+  order(order: "asc" | "desc"): OrderedPaginatorQuery<DataModel, T> {
+    return this.fullTableScan().order(order);
+  }
+  paginate(opts: PaginationOptions & { endCursor?: string | null }): Promise<PaginationResult<DocumentByInfo<NamedTableInfo<DataModel, T>>>> {
+    return this.fullTableScan().paginate(opts);
+  }
+  filter(_predicate: any): any {
+    throw new Error(".filter() not supported for `paginator`. Filter the returned `page` instead.");
+  }
+  collect(): any {
+    throw new Error(".collect() not supported for `paginator`. Use .paginate() instead.");
+  }
+  first(): any {
+    throw new Error(".first() not supported for `paginator`. Use .paginate() instead.");
+  }
+  unique(): any {
+    throw new Error(".unique() not supported for `paginator`. Use .paginate() instead.");
+  }
+  take(_n: number): any {
+    throw new Error(".take() not supported for `paginator`. Use .paginate() instead.");
+  }
+  [Symbol.asyncIterator](): any {
+    throw new Error("[Symbol.asyncIterator]() not supported for `paginator`. Use .paginate() instead.");
+  }
+}
+
+export class PaginatorQuery<
+  DataModel extends GenericDataModel,
+  T extends TableNamesInDataModel<DataModel>,
+> implements Query<NamedTableInfo<DataModel, T>> {
+  constructor(
+    public parent: PaginatorQueryInitializer<DataModel, T>,
+    public index: IndexNames<NamedTableInfo<DataModel, T>>,
+    public q: PaginatorIndexRange,
+  ) {}
+  order(order: "asc" | "desc") {
+    return new OrderedPaginatorQuery(this, order);
+  }
+  paginate(opts: PaginationOptions & { endCursor?: string | null }): Promise<PaginationResult<DocumentByInfo<NamedTableInfo<DataModel, T>>>> {
+    return this.order("asc").paginate(opts);
+  }
+  filter(_predicate: any): this {
+    throw new Error(".filter() not supported for `paginator`. Filter the returned `page` instead.");
+  }
+  collect(): any {
+    throw new Error(".collect() not supported for `paginator`. Use .paginate() instead.");
+  }
+  first(): any {
+    throw new Error(".first() not supported for `paginator`. Use .paginate() instead.");
+  }
+  unique(): any {
+    throw new Error(".unique() not supported for `paginator`. Use .paginate() instead.");
+  }
+  take(_n: number): any {
+    throw new Error(".take() not supported for `paginator`. Use .paginate() instead.");
+  }
+  [Symbol.asyncIterator](): any {
+    throw new Error("[Symbol.asyncIterator]() not supported for `paginator`. Use .paginate() instead.");
+  }
+}
+
+export class OrderedPaginatorQuery<
+  DataModel extends GenericDataModel,
+  T extends TableNamesInDataModel<DataModel>,
+> implements OrderedQuery<NamedTableInfo<DataModel, T>> {
+  public startIndexKey: IndexKey | undefined;
+  public startInclusive: boolean;
+  public endIndexKey: IndexKey | undefined;
+  public endInclusive: boolean;
+  constructor(
+    public parent: PaginatorQuery<DataModel, T>,
+    public order: "asc" | "desc",
+  ) {
+    this.startIndexKey = order === "asc" ? parent.q.lowerBoundIndexKey : parent.q.upperBoundIndexKey;
+    this.endIndexKey = order === "asc" ? parent.q.upperBoundIndexKey : parent.q.lowerBoundIndexKey;
+    this.startInclusive = order === "asc" ? parent.q.lowerBoundInclusive : parent.q.upperBoundInclusive;
+    this.endInclusive = order === "asc" ? parent.q.upperBoundInclusive : parent.q.lowerBoundInclusive;
+  }
+  async paginate(opts: PaginationOptions & { endCursor?: string | null }): Promise<PaginationResult<DocumentByName<DataModel, T>>> {
+    if (opts.cursor === END_CURSOR) {
+      return {
+        page: [],
+        isDone: true,
+        continueCursor: END_CURSOR,
+      };
+    }
+    const schema = this.parent.parent.parent.schema;
+    let startIndexKey = this.startIndexKey;
+    let startInclusive = this.startInclusive;
+    if (opts.cursor !== null) {
+      startIndexKey = jsonToConvex(JSON.parse(opts.cursor)) as IndexKey;
+      startInclusive = false;
+    }
+    let endIndexKey = this.endIndexKey;
+    let endInclusive = this.endInclusive;
+    let absoluteMaxRows: number | undefined = opts.numItems;
+    if (opts.endCursor && opts.endCursor !== END_CURSOR) {
+      endIndexKey = jsonToConvex(JSON.parse(opts.endCursor)) as IndexKey;
+      endInclusive = true;
+      absoluteMaxRows = undefined;
+    }
+    const {
+      page, hasMore, indexKeys,
+    } = await getPage({ db: this.parent.parent.parent.db }, {
+      table: this.parent.parent.table,
+      startIndexKey,
+      startInclusive,
+      endIndexKey,
+      endInclusive,
+      targetMaxRows: opts.numItems,
+      absoluteMaxRows,
+      order: this.order,
+      index: this.parent.index,
+      schema,
+      indexFields: this.parent.q.indexFields,
+    });
+    let continueCursor = END_CURSOR;
+    let isDone = !hasMore;
+    if (opts.endCursor && opts.endCursor !== END_CURSOR) {
+      continueCursor = opts.endCursor;
+      isDone = false;
+    } else if (indexKeys.length > 0 && hasMore) {
+      continueCursor = JSON.stringify(convexToJson(indexKeys[indexKeys.length - 1] as Value));
+    }
+    return {
+      page,
+      isDone,
+      continueCursor,
+    };
+  }
+  filter(_predicate: any): any {
+    throw new Error(".filter() not supported for `paginator`. Filter the returned `page` instead.");
+  }
+  collect(): any {
+    throw new Error(".collect() not supported for `paginator`. Use .paginate() instead.");
+  }
+  first(): any {
+    throw new Error(".first() not supported for `paginator`. Use .paginate() instead.");
+  }
+  unique(): any {
+    throw new Error(".unique() not supported for `paginator`. Use .paginate() instead.");
+  }
+  take(_n: number): any {
+    throw new Error(".take() not supported for `paginator`. Use .paginate() instead.");
+  }
+  [Symbol.asyncIterator](): any {
+    throw new Error("[Symbol.asyncIterator]() not supported for `paginator`. Use .paginate() instead.");
+  }
+}
+
+class PaginatorIndexRange {
+  private hasSuffix = false;
+  public lowerBoundIndexKey: IndexKey | undefined = undefined;
+  public lowerBoundInclusive: boolean = true;
+  public upperBoundIndexKey: IndexKey | undefined = undefined;
+  public upperBoundInclusive: boolean = true;
+  constructor(
+    public indexFields: string[],
+  ) {}
+  eq(field: string, value: Value) {
+    if (!this.canLowerBound(field) || !this.canUpperBound(field)) {
+      throw new Error(`Cannot use eq on field '${field}'`);
+    }
+    this.lowerBoundIndexKey = this.lowerBoundIndexKey ?? [];
+    this.lowerBoundIndexKey.push(value);
+    this.upperBoundIndexKey = this.upperBoundIndexKey ?? [];
+    this.upperBoundIndexKey.push(value);
+    return this;
+  }
+  lt(field: string, value: Value) {
+    if (!this.canUpperBound(field)) {
+      throw new Error(`Cannot use lt on field '${field}'`);
+    }
+    this.upperBoundIndexKey = this.upperBoundIndexKey ?? [];
+    this.upperBoundIndexKey.push(value);
+    this.upperBoundInclusive = false;
+    this.hasSuffix = true;
+    return this;
+  }
+  lte(field: string, value: Value) {
+    if (!this.canUpperBound(field)) {
+      throw new Error(`Cannot use lte on field '${field}'`);
+    }
+    this.upperBoundIndexKey = this.upperBoundIndexKey ?? [];
+    this.upperBoundIndexKey.push(value);
+    this.hasSuffix = true;
+    return this;
+  }
+  gt(field: string, value: Value) {
+    if (!this.canLowerBound(field)) {
+      throw new Error(`Cannot use gt on field '${field}'`);
+    }
+    this.lowerBoundIndexKey = this.lowerBoundIndexKey ?? [];
+    this.lowerBoundIndexKey.push(value);
+    this.lowerBoundInclusive = false;
+    this.hasSuffix = true;
+    return this;
+  }
+  gte(field: string, value: Value) {
+    if (!this.canLowerBound(field)) {
+      throw new Error(`Cannot use gte on field '${field}'`);
+    }
+    this.lowerBoundIndexKey = this.lowerBoundIndexKey ?? [];
+    this.lowerBoundIndexKey.push(value);
+    this.hasSuffix = true;
+    return this;
+  }
+  private canLowerBound(field: string) {
+    const currentLowerBoundLength = this.lowerBoundIndexKey?.length ?? 0;
+    const currentUpperBoundLength = this.upperBoundIndexKey?.length ?? 0;
+    if (currentLowerBoundLength > currentUpperBoundLength) {
+      // Already have a lower bound.
+      return false;
+    }
+    if (currentLowerBoundLength === currentUpperBoundLength && this.hasSuffix) {
+      // Already have a lower bound and an upper bound.
+      return false;
+    }
+    return currentLowerBoundLength < this.indexFields.length && this.indexFields[currentLowerBoundLength] === field;
+  }
+  private canUpperBound(field: string) {
+    const currentLowerBoundLength = this.lowerBoundIndexKey?.length ?? 0;
+    const currentUpperBoundLength = this.upperBoundIndexKey?.length ?? 0;
+    if (currentUpperBoundLength > currentLowerBoundLength) {
+      // Already have an upper bound.
+      return false;
+    }
+    if (currentLowerBoundLength === currentUpperBoundLength && this.hasSuffix) {
+      // Already have a lower bound and an upper bound.
+      return false;
+    }
+    return currentUpperBoundLength < this.indexFields.length && this.indexFields[currentUpperBoundLength] === field;
+  }
+}

server/zod.d.ts

@@ -228,3 +228,3 @@ import { ZodTypeDef, z } from "zod";
     VNull
-], "optional", ConvexValidatorFromZod<Inner>["fieldPaths"]> : never : Z extends z.ZodBranded<infer Inner, infer Brand> ? Inner extends z.ZodString ? VString<string & z.BRAND<Brand>> : Inner extends z.ZodNumber ? VFloat64<number & z.BRAND<Brand>> : Inner extends z.ZodBigInt ? VInt64<bigint & z.BRAND<Brand>> : ConvexValidatorFromZod<Inner> : Z extends z.ZodDefault<infer Inner> ? ConvexValidatorFromZod<Inner> extends GenericValidator ? VOptional<ConvexValidatorFromZod<Inner>> : never : Z extends z.ZodRecord<infer K, infer V> ? K extends z.ZodString | Zid<string> | z.ZodUnion<[
+], "optional", ConvexValidatorFromZod<Inner>["fieldPaths"]> : never : Z extends z.ZodBranded<infer Inner, any> ? ConvexValidatorFromZod<Inner> : Z extends z.ZodDefault<infer Inner> ? ConvexValidatorFromZod<Inner> extends GenericValidator ? VOptional<ConvexValidatorFromZod<Inner>> : never : Z extends z.ZodRecord<infer K, infer V> ? K extends z.ZodString | Zid<string> | z.ZodUnion<[
     (z.ZodString | Zid<string>),
@@ -231,0 +231,0 @@ (z.ZodString | Zid<string>),

server/zod.test.ts

@@ -16,3 +16,3 @@ import {
 import { customCtx } from "./customFunctions.js";
-import { v, VString } from "convex/values";
+import { v } from "convex/values";
 import { z } from "zod";
@@ -321,3 +321,3 @@
     nullable: null,
-    branded: "branded" as string & z.BRAND<"branded">,
+    branded: "branded",
     default: undefined,
@@ -592,3 +592,3 @@ readonly: { a: "1", b: 2 },
       literal: v.literal("hi"),
-      branded: v.string() as VString<string & z.BRAND<"branded">>,
+      branded: v.string(),
     },
@@ -595,0 +595,0 @@ ),

server/zod.ts

@@ -576,15 +576,5 @@ import { ZodFirstPartyTypeKind, ZodTypeDef, z } from "zod";
                                                 infer Inner,
-                                                infer Brand
+                                                any
                                               >
-                                            ? Inner extends z.ZodString
-                                              ? VString<string & z.BRAND<Brand>>
-                                              : Inner extends z.ZodNumber
-                                                ? VFloat64<
-                                                    number & z.BRAND<Brand>
-                                                  >
-                                                : Inner extends z.ZodBigInt
-                                                  ? VInt64<
-                                                      bigint & z.BRAND<Brand>
-                                                    >
-                                                  : ConvexValidatorFromZod<Inner>
+                                            ? ConvexValidatorFromZod<Inner>
                                             : Z extends z.ZodDefault<
@@ -591,0 +581,0 @@ infer Inner

No alert changes

Improved metrics

Total package byte prevSize

847043

increased by5.4%

Lines of code

19496

increased by5.23%

Number of lines in readme file

1067

increased by5.75%

No dependency changes