vectordb - npm Package Compare versions

Comparing version 0.4.0 to 0.4.1

dist/index.d.ts

@@ -128,2 +128,51 @@ import { type Schema, Table as ArrowTable } from 'apache-arrow';
     /**
+     * Create a scalar index on this Table for the given column
+     *
+     * @param column The column to index
+     * @param replace If false, fail if an index already exists on the column
+     *
+     * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
+     * index can speed up scans that contain filter expressions on the indexed column.
+     * For example, the following scan will be faster if the column `my_col` has
+     * a scalar index:
+     *
+     * ```ts
+     * const con = await lancedb.connect('./.lancedb');
+     * const table = await con.openTable('images');
+     * const results = await table.where('my_col = 7').execute();
+     * ```
+     *
+     * Scalar indices can also speed up scans containing a vector search and a
+     * prefilter:
+     *
+     * ```ts
+     * const con = await lancedb.connect('././lancedb');
+     * const table = await con.openTable('images');
+     * const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true);
+     * ```
+     *
+     * Scalar indices can only speed up scans for basic filters using
+     * equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set
+     * membership (e.g. `my_col IN (0, 1, 2)`)
+     *
+     * Scalar indices can be used if the filter contains multiple indexed columns and
+     * the filter criteria are AND'd or OR'd together
+     * (e.g. `my_col < 0 AND other_col> 100`)
+     *
+     * Scalar indices may be used if the filter contains non-indexed columns but,
+     * depending on the structure of the filter, they may not be usable.  For example,
+     * if the column `not_indexed` does not have a scalar index then the filter
+     * `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on
+     * `my_col`.
+     *
+     * @examples
+     *
+     * ```ts
+     * const con = await lancedb.connect('././lancedb')
+     * const table = await con.openTable('images')
+     * await table.createScalarIndex('my_col')
+     * ```
+     */
+    createScalarIndex: (column: string, replace: boolean) => Promise<void>;
+    /**
      * Returns the number of rows in this table.
@@ -184,4 +233,4 @@ */
      * await tbl.update({
-     *   filter: "id = 2",
-     *   updates: { vector: [2, 2], name: "Michael" },
+     *   where: "id = 2",
+     *   values: { vector: [2, 2], name: "Michael" },
      * })
@@ -319,2 +368,3 @@ *
     createIndex(indexParams: VectorIndexParams): Promise<any>;
+    createScalarIndex(column: string, replace: boolean): Promise<void>;
     /**
@@ -469,2 +519,6 @@ * Returns the number of rows in this table.
     replace?: boolean;
+    /**
+     * Cache size of the index
+     */
+    index_cache_size?: number;
     type: 'ivf_pq';
@@ -471,0 +525,0 @@ }

dist/index.js

@@ -34,3 +34,3 @@ "use strict";
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js');
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateScalarIndex, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js');
 var openai_1 = require("./embedding/openai");
@@ -227,2 +227,7 @@ Object.defineProperty(exports, "OpenAIEmbeddingFunction", { enumerable: true, get: function () { return openai_1.OpenAIEmbeddingFunction; } });
     }
+    createScalarIndex(column, replace) {
+        return __awaiter(this, void 0, void 0, function* () {
+            return tableCreateScalarIndex.call(this._tbl, column, replace);
+        });
+    }
     /**
@@ -229,0 +234,0 @@ * Returns the number of rows in this table.

dist/remote/index.d.ts

@@ -12,3 +12,3 @@ import { type EmbeddingFunction, type Table, type VectorIndexParams, type Connection, type ConnectionOptions, type CreateTableOptions, type VectorIndex, type WriteOptions, type IndexStats, type UpdateArgs, type UpdateSqlArgs } from '../index';
     get uri(): string;
-    tableNames(): Promise<string[]>;
+    tableNames(pageToken?: string, limit?: number): Promise<string[]>;
     openTable(name: string): Promise<Table>;
@@ -32,6 +32,8 @@ openTable<T>(name: string, embeddings: EmbeddingFunction<T>): Promise<Table<T>>;
     get name(): string;
+    get schema(): Promise<any>;
     search(query: T): Query<T>;
     add(data: Array<Record<string, unknown>>): Promise<number>;
     overwrite(data: Array<Record<string, unknown>>): Promise<number>;
-    createIndex(indexParams: VectorIndexParams): Promise<any>;
+    createIndex(indexParams: VectorIndexParams): Promise<void>;
+    createScalarIndex(column: string, replace: boolean): Promise<void>;
     countRows(): Promise<number>;
@@ -38,0 +40,0 @@ delete(filter: string): Promise<void>;

dist/remote/index.js

@@ -57,5 +57,5 @@ "use strict";
     }
-    tableNames() {
+    tableNames(pageToken = '', limit = 10) {
         return __awaiter(this, void 0, void 0, function* () {
-            const response = yield this._client.get('/v1/table/');
+            const response = yield this._client.get('/v1/table/', { limit, page_token: pageToken });
             return response.data.tables;
@@ -178,2 +178,13 @@ });
     }
+    get schema() {
+        return this._client.post(`/v1/table/${this._name}/describe/`).then(res => {
+            var _a;
+            if (res.status !== 200) {
+                throw new Error(`Server Error, status: ${res.status}, ` +
+                    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+                    `message: ${res.statusText}: ${res.data}`);
+            }
+            return (_a = res.data) === null || _a === void 0 ? void 0 : _a.schema;
+        });
+    }
     search(query) {
@@ -211,3 +222,40 @@ return new RemoteQuery(query, this._client, this._name); //, this._embeddings_new)
     createIndex(indexParams) {
+        var _a, _b;
         return __awaiter(this, void 0, void 0, function* () {
+            const unsupportedParams = [
+                'index_name',
+                'num_partitions',
+                'max_iters',
+                'use_opq',
+                'num_sub_vectors',
+                'num_bits',
+                'max_opq_iters',
+                'replace'
+            ];
+            for (const param of unsupportedParams) {
+                // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+                if (indexParams[param]) {
+                    throw new Error(`${param} is not supported for remote connections`);
+                }
+            }
+            const column = (_a = indexParams.column) !== null && _a !== void 0 ? _a : 'vector';
+            const indexType = 'vector'; // only vector index is supported for remote connections
+            const metricType = (_b = indexParams.metric_type) !== null && _b !== void 0 ? _b : 'L2';
+            const indexCacheSize = indexParams !== null && indexParams !== void 0 ? indexParams : null;
+            const data = {
+                column,
+                index_type: indexType,
+                metric_type: metricType,
+                index_cache_size: indexCacheSize
+            };
+            const res = yield this._client.post(`/v1/table/${this._name}/create_index/`, data);
+            if (res.status !== 200) {
+                throw new Error(`Server Error, status: ${res.status}, ` +
+                    // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+                    `message: ${res.statusText}: ${res.data}`);
+            }
+        });
+    }
+    createScalarIndex(column, replace) {
+        return __awaiter(this, void 0, void 0, function* () {
             throw new Error('Not implemented');
@@ -214,0 +262,0 @@ });

dist/test/test.js

@@ -150,2 +150,13 @@ "use strict";
         });
+        it('should allow creation and use of scalar indices', function () {
+            return __awaiter(this, void 0, void 0, function* () {
+                const uri = yield createTestDB(16, 300);
+                const con = yield lancedb.connect(uri);
+                const table = yield con.openTable('vectors');
+                yield table.createScalarIndex('id', true);
+                // Prefiltering should still work the same
+                const results = yield table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute();
+                assert.isTrue(results.length === 10);
+            });
+        });
         it('select only a subset of columns', function () {
@@ -152,0 +163,0 @@ return __awaiter(this, void 0, void 0, function* () {

package.json

 {
   "name": "vectordb",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": " Serverless, low-latency vector database for AI applications",
@@ -84,8 +84,8 @@ "main": "dist/index.js",
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.0",
-    "@lancedb/vectordb-darwin-x64": "0.4.0",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.0",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.0",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.0"
+    "@lancedb/vectordb-darwin-arm64": "0.4.1",
+    "@lancedb/vectordb-darwin-x64": "0.4.1",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.1",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.1",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.1"
   }
 }

README.md

@@ -12,5 +12,10 @@ # LanceDB
 This will download the appropriate native library for your platform. We currently
-support x86_64 Linux, aarch64 Linux, Intel MacOS, and ARM (M1/M2) MacOS. We do not
-yet support musl-based Linux (such as Alpine Linux).
+support:
 
+* Linux (x86_64 and aarch64)
+* MacOS (Intel and ARM/M1/M2)
+* Windows (x86_64 only)
+
+We do not yet support musl-based Linux (such as Alpine Linux) or aarch64 Windows.
+
 ## Usage
@@ -17,0 +22,0 @@

src/index.ts

@@ -27,3 +27,3 @@ // Copyright 2023 Lance Developers.
 // eslint-disable-next-line @typescript-eslint/no-var-requires
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js')
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateScalarIndex, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats } = require('../native.js')
 
@@ -228,2 +228,52 @@ export { Query }
   /**
+   * Create a scalar index on this Table for the given column
+   *
+   * @param column The column to index
+   * @param replace If false, fail if an index already exists on the column
+   *
+   * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
+   * index can speed up scans that contain filter expressions on the indexed column.
+   * For example, the following scan will be faster if the column `my_col` has
+   * a scalar index:
+   *
+   * ```ts
+   * const con = await lancedb.connect('./.lancedb');
+   * const table = await con.openTable('images');
+   * const results = await table.where('my_col = 7').execute();
+   * ```
+   *
+   * Scalar indices can also speed up scans containing a vector search and a
+   * prefilter:
+   *
+   * ```ts
+   * const con = await lancedb.connect('././lancedb');
+   * const table = await con.openTable('images');
+   * const results = await table.search([1.0, 2.0]).where('my_col != 7').prefilter(true);
+   * ```
+   *
+   * Scalar indices can only speed up scans for basic filters using
+   * equality, comparison, range (e.g. `my_col BETWEEN 0 AND 100`), and set
+   * membership (e.g. `my_col IN (0, 1, 2)`)
+   *
+   * Scalar indices can be used if the filter contains multiple indexed columns and
+   * the filter criteria are AND'd or OR'd together
+   * (e.g. `my_col < 0 AND other_col> 100`)
+   *
+   * Scalar indices may be used if the filter contains non-indexed columns but,
+   * depending on the structure of the filter, they may not be usable.  For example,
+   * if the column `not_indexed` does not have a scalar index then the filter
+   * `my_col = 0 OR not_indexed = 1` will not be able to use any scalar index on
+   * `my_col`.
+   *
+   * @examples
+   *
+   * ```ts
+   * const con = await lancedb.connect('././lancedb')
+   * const table = await con.openTable('images')
+   * await table.createScalarIndex('my_col')
+   * ```
+   */
+  createScalarIndex: (column: string, replace: boolean) => Promise<void>
+
+  /**
    * Returns the number of rows in this table.
@@ -286,4 +336,4 @@ */
    * await tbl.update({
-   *   filter: "id = 2",
-   *   updates: { vector: [2, 2], name: "Michael" },
+   *   where: "id = 2",
+   *   values: { vector: [2, 2], name: "Michael" },
    * })
@@ -543,2 +593,6 @@ *
 
+  async createScalarIndex (column: string, replace: boolean): Promise<void> {
+    return tableCreateScalarIndex.call(this._tbl, column, replace)
+  }
+
   /**
@@ -751,2 +805,7 @@ * Returns the number of rows in this table.
 
+  /**
+   * Cache size of the index
+   */
+  index_cache_size?: number
+
   type: 'ivf_pq'
@@ -753,0 +812,0 @@ }

src/remote/index.ts

@@ -60,4 +60,4 @@ // Copyright 2023 LanceDB Developers.
 
-  async tableNames (): Promise<string[]> {
-    const response = await this._client.get('/v1/table/')
+  async tableNames (pageToken: string = '', limit: number = 10): Promise<string[]> {
+    const response = await this._client.get('/v1/table/', { limit, page_token: pageToken })
     return response.data.tables
@@ -199,2 +199,13 @@ }
 
+  get schema (): Promise<any> {
+    return this._client.post(`/v1/table/${this._name}/describe/`).then(res => {
+      if (res.status !== 200) {
+        throw new Error(`Server Error, status: ${res.status}, ` +
+        // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${res.data}`)
+      }
+      return res.data?.schema
+    })
+  }
+
   search (query: T): Query<T> {
@@ -240,3 +251,40 @@ return new RemoteQuery(query, this._client, this._name)//, this._embeddings_new)
 
-  async createIndex (indexParams: VectorIndexParams): Promise<any> {
+  async createIndex (indexParams: VectorIndexParams): Promise<void> {
+    const unsupportedParams = [
+      'index_name',
+      'num_partitions',
+      'max_iters',
+      'use_opq',
+      'num_sub_vectors',
+      'num_bits',
+      'max_opq_iters',
+      'replace'
+    ]
+    for (const param of unsupportedParams) {
+      // eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
+      if (indexParams[param as keyof VectorIndexParams]) {
+        throw new Error(`${param} is not supported for remote connections`)
+      }
+    }
+
+    const column = indexParams.column ?? 'vector'
+    const indexType = 'vector' // only vector index is supported for remote connections
+    const metricType = indexParams.metric_type ?? 'L2'
+    const indexCacheSize = indexParams ?? null
+
+    const data = {
+      column,
+      index_type: indexType,
+      metric_type: metricType,
+      index_cache_size: indexCacheSize
+    }
+    const res = await this._client.post(`/v1/table/${this._name}/create_index/`, data)
+    if (res.status !== 200) {
+      throw new Error(`Server Error, status: ${res.status}, ` +
+      // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+        `message: ${res.statusText}: ${res.data}`)
+    }
+  }
+
+  async createScalarIndex (column: string, replace: boolean): Promise<void> {
     throw new Error('Not implemented')
@@ -243,0 +291,0 @@ }

src/test/test.ts

@@ -138,2 +138,13 @@ // Copyright 2023 LanceDB Developers.
 
+    it('should allow creation and use of scalar indices', async function () {
+      const uri = await createTestDB(16, 300)
+      const con = await lancedb.connect(uri)
+      const table = await con.openTable('vectors')
+      await table.createScalarIndex('id', true)
+
+      // Prefiltering should still work the same
+      const results = await table.search(new Array(16).fill(0.1)).limit(10).filter('id >= 10').prefilter(true).execute()
+      assert.isTrue(results.length === 10)
+    })
+
     it('select only a subset of columns', async function () {
@@ -140,0 +151,0 @@ const uri = await createTestDB()

No alert changes

Improved metrics

Total package byte prevSize

237899

increased by4.38%

Lines of code

5595

increased by4.31%

Number of lines in readme file

77

increased by6.94%

No dependency changes