vectordb - npm Package Compare versions

Comparing version 0.4.11 to 0.4.12

.eslintrc.js

@@ -16,3 +16,8 @@ module.exports = {
     "@typescript-eslint/method-signature-style": "off",
+    "@typescript-eslint/quotes": "off",
+    "@typescript-eslint/semi": "off",
+    "@typescript-eslint/explicit-function-return-type": "off",
+    "@typescript-eslint/space-before-function-paren": "off",
+    "@typescript-eslint/indent": "off",
   }
 }

dist/arrow.d.ts

 /// <reference types="node" />
-import { type Schema, Table as ArrowTable, type Float } from 'apache-arrow';
+import { Schema, Table as ArrowTable, Float } from 'apache-arrow';
 import { type EmbeddingFunction } from './index';
@@ -4,0 +4,0 @@ export declare class VectorColumnOptions {

dist/arrow.js

@@ -18,2 +18,3 @@ "use strict";
 const apache_arrow_1 = require("apache-arrow");
+const sanitize_1 = require("./sanitize");
 /*
@@ -166,6 +167,9 @@ * Options to control how a column should be converted to a vector array
     const opt = new MakeArrowTableOptions(options !== undefined ? options : {});
+    if (opt.schema !== undefined && opt.schema !== null) {
+        opt.schema = (0, sanitize_1.sanitizeSchema)(opt.schema);
+    }
     const columns = {};
     // TODO: sample dataset to find missing columns
     // Prefer the field ordering of the schema, if present
-    const columnNames = ((options?.schema) != null) ? options?.schema?.names : Object.keys(data[0]);
+    const columnNames = ((opt.schema) != null) ? opt.schema.names : Object.keys(data[0]);
     for (const colName of columnNames) {
@@ -296,2 +300,5 @@ if (data.length !== 0 && !Object.prototype.hasOwnProperty.call(data[0], colName)) {
     }
+    if (schema !== undefined && schema !== null) {
+        schema = (0, sanitize_1.sanitizeSchema)(schema);
+    }
     // Convert from ArrowTable to Record<String, Vector>
@@ -397,2 +404,5 @@ const colEntries = [...Array(table.numCols).keys()].map((_, idx) => {
 async function fromRecordsToBuffer(data, embeddings, schema) {
+    if (schema !== undefined && schema !== null) {
+        schema = (0, sanitize_1.sanitizeSchema)(schema);
+    }
     const table = await convertToTable(data, embeddings, { schema });
@@ -411,2 +421,5 @@ const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(table);
 async function fromRecordsToStreamBuffer(data, embeddings, schema) {
+    if (schema !== null && schema !== undefined) {
+        schema = (0, sanitize_1.sanitizeSchema)(schema);
+    }
     const table = await convertToTable(data, embeddings, { schema });
@@ -426,2 +439,5 @@ const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(table);
 async function fromTableToBuffer(table, embeddings, schema) {
+    if (schema !== null && schema !== undefined) {
+        schema = (0, sanitize_1.sanitizeSchema)(schema);
+    }
     const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
@@ -441,2 +457,5 @@ const writer = apache_arrow_1.RecordBatchFileWriter.writeAll(tableWithEmbeddings);
 async function fromTableToStreamBuffer(table, embeddings, schema) {
+    if (schema !== null && schema !== undefined) {
+        schema = (0, sanitize_1.sanitizeSchema)(schema);
+    }
     const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema);
@@ -470,5 +489,5 @@ const writer = apache_arrow_1.RecordBatchStreamWriter.writeAll(tableWithEmbeddings);
 function createEmptyTable(schema) {
-    return new apache_arrow_1.Table(schema);
+    return new apache_arrow_1.Table((0, sanitize_1.sanitizeSchema)(schema));
 }
 exports.createEmptyTable = createEmptyTable;
 //# sourceMappingURL=arrow.js.map

dist/index.d.ts

@@ -182,2 +182,3 @@ import { type Schema, Table as ArrowTable } from 'apache-arrow';
      * @param replace If false, fail if an index already exists on the column
+     * it is always set to true for remote connections
      *
@@ -226,3 +227,3 @@ * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
      */
-    createScalarIndex: (column: string, replace: boolean) => Promise<void>;
+    createScalarIndex: (column: string, replace?: boolean) => Promise<void>;
     /**
@@ -338,3 +339,54 @@ * Returns the number of rows in this table.
     schema: Promise<Schema>;
+    /**
+     * Add new columns with defined values.
+     *
+     * @param newColumnTransforms pairs of column names and the SQL expression to use
+     *                            to calculate the value of the new column. These
+     *                            expressions will be evaluated for each row in the
+     *                            table, and can reference existing columns in the table.
+     */
+    addColumns(newColumnTransforms: Array<{
+        name: string;
+        valueSql: string;
+    }>): Promise<void>;
+    /**
+     * Alter the name or nullability of columns.
+     *
+     * @param columnAlterations One or more alterations to apply to columns.
+     */
+    alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
+    /**
+     * Drop one or more columns from the dataset
+     *
+     * This is a metadata-only operation and does not remove the data from the
+     * underlying storage. In order to remove the data, you must subsequently
+     * call ``compact_files`` to rewrite the data without the removed columns and
+     * then call ``cleanup_files`` to remove the old files.
+     *
+     * @param columnNames The names of the columns to drop. These can be nested
+     *                    column references (e.g. "a.b.c") or top-level column
+     *                    names (e.g. "a").
+     */
+    dropColumns(columnNames: string[]): Promise<void>;
 }
+/**
+ * A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+    /**
+     * The path to the column to alter. This is a dot-separated path to the column.
+     * If it is a top-level column then it is just the name of the column. If it is
+     * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+     * `c` nested inside a column `b` nested inside a column `a`.
+     */
+    path: string;
+    rename?: string;
+    /**
+     * Set the new nullability. Note that a nullable column cannot be made non-nullable.
+     */
+    nullable?: boolean;
+}
 export interface UpdateArgs {
@@ -493,3 +545,3 @@ /**
     createIndex(indexParams: VectorIndexParams): Promise<any>;
-    createScalarIndex(column: string, replace: boolean): Promise<void>;
+    createScalarIndex(column: string, replace?: boolean): Promise<void>;
     /**
@@ -547,2 +599,8 @@ * Returns the number of rows in this table.
     private checkElectron;
+    addColumns(newColumnTransforms: Array<{
+        name: string;
+        valueSql: string;
+    }>): Promise<void>;
+    alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
+    dropColumns(columnNames: string[]): Promise<void>;
 }
@@ -549,0 +607,0 @@ export interface CleanupStats {

dist/index.js

@@ -24,3 +24,3 @@ "use strict";
 const util_1 = require("./util");
-const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateScalarIndex, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableMergeInsert, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats, tableSchema
+const { databaseNew, databaseTableNames, databaseOpenTable, databaseDropTable, tableCreate, tableAdd, tableCreateScalarIndex, tableCreateVectorIndex, tableCountRows, tableDelete, tableUpdate, tableMergeInsert, tableCleanupOldVersions, tableCompactFiles, tableListIndices, tableIndexStats, tableSchema, tableAddColumns, tableAlterColumns, tableDropColumns
 // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -239,2 +239,5 @@ } = require('../native.js');
     async createScalarIndex(column, replace) {
+        if (replace === undefined) {
+            replace = true;
+        }
         return tableCreateScalarIndex.call(this._tbl, column, replace);
@@ -383,2 +386,11 @@ }
     }
+    async addColumns(newColumnTransforms) {
+        return tableAddColumns.call(this._tbl, newColumnTransforms);
+    }
+    async alterColumns(columnAlterations) {
+        return tableAlterColumns.call(this._tbl, columnAlterations);
+    }
+    async dropColumns(columnNames) {
+        return tableDropColumns.call(this._tbl, columnNames);
+    }
 }
@@ -385,0 +397,0 @@ exports.LocalTable = LocalTable;

dist/remote/index.d.ts

@@ -1,2 +0,2 @@
-import { type EmbeddingFunction, type Table, type VectorIndexParams, type Connection, type ConnectionOptions, type CreateTableOptions, type VectorIndex, type WriteOptions, type IndexStats, type UpdateArgs, type UpdateSqlArgs, type MergeInsertArgs } from '../index';
+import { type EmbeddingFunction, type Table, type VectorIndexParams, type Connection, type ConnectionOptions, type CreateTableOptions, type VectorIndex, type WriteOptions, type IndexStats, type UpdateArgs, type UpdateSqlArgs, type MergeInsertArgs, type ColumnAlteration } from '../index';
 import { Query } from '../query';
@@ -39,3 +39,3 @@ import { Table as ArrowTable } from 'apache-arrow';
     createIndex(indexParams: VectorIndexParams): Promise<void>;
-    createScalarIndex(column: string, replace: boolean): Promise<void>;
+    createScalarIndex(column: string): Promise<void>;
     countRows(): Promise<number>;
@@ -46,2 +46,8 @@ delete(filter: string): Promise<void>;
     indexStats(indexUuid: string): Promise<IndexStats>;
+    addColumns(newColumnTransforms: Array<{
+        name: string;
+        valueSql: string;
+    }>): Promise<void>;
+    alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>;
+    dropColumns(columnNames: string[]): Promise<void>;
 }

dist/remote/index.js

@@ -284,3 +284,3 @@ "use strict";
         const column = indexParams.column ?? 'vector';
-        const indexType = 'vector'; // only vector index is supported for remote connections
+        const indexType = 'vector';
         const metricType = indexParams.metric_type ?? 'L2';
@@ -301,4 +301,15 @@ const indexCacheSize = indexParams.index_cache_size ?? null;
     }
-    async createScalarIndex(column, replace) {
-        throw new Error('Not implemented');
+    async createScalarIndex(column) {
+        const indexType = 'scalar';
+        const data = {
+            column,
+            index_type: indexType,
+            replace: true
+        };
+        const res = await this._client.post(`/v1/table/${this._name}/create_scalar_index/`, data);
+        if (res.status !== 200) {
+            throw new Error(`Server Error, status: ${res.status}, ` +
+                // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+                `message: ${res.statusText}: ${res.data}`);
+        }
     }
@@ -348,4 +359,13 @@ async countRows() {
     }
+    async addColumns(newColumnTransforms) {
+        throw new Error('Add columns is not yet supported in LanceDB Cloud.');
+    }
+    async alterColumns(columnAlterations) {
+        throw new Error('Alter columns is not yet supported in LanceDB Cloud.');
+    }
+    async dropColumns(columnNames) {
+        throw new Error('Drop columns is not yet supported in LanceDB Cloud.');
+    }
 }
 exports.RemoteTable = RemoteTable;
 //# sourceMappingURL=index.js.map

dist/test/arrow.test.js

@@ -21,2 +21,3 @@ "use strict";
 const apache_arrow_1 = require("apache-arrow");
+const apache_arrow_old_1 = require("apache-arrow-old");
 (0, chai_1.use)(chaiAsPromised);
@@ -250,2 +251,25 @@ function sampleRecords() {
 });
+(0, mocha_1.describe)('when using two versions of arrow', function () {
+    it('can still import data', async function () {
+        const schema = new apache_arrow_old_1.Schema([
+            new apache_arrow_old_1.Field('id', new apache_arrow_old_1.Int32()),
+            new apache_arrow_old_1.Field('vector', new apache_arrow_old_1.FixedSizeList(1024, new apache_arrow_old_1.Field("item", new apache_arrow_old_1.Float32(), true))),
+            new apache_arrow_old_1.Field('struct', new apache_arrow_old_1.Struct([
+                new apache_arrow_old_1.Field('nested', new apache_arrow_old_1.Dictionary(new apache_arrow_old_1.Utf8(), new apache_arrow_old_1.Int32(), 1, true)),
+                new apache_arrow_old_1.Field('ts_with_tz', new apache_arrow_old_1.TimestampNanosecond("some_tz")),
+                new apache_arrow_old_1.Field('ts_no_tz', new apache_arrow_old_1.TimestampNanosecond(null))
+            ]))
+        ]);
+        // We use arrow version 13 to emulate a "foreign arrow" and this version doesn't have metadataVersion
+        // In theory, this wouldn't matter.  We don't rely on that property.  However, it causes deepEqual to
+        // fail so we patch it back in
+        schema.metadataVersion = apache_arrow_1.MetadataVersion.V5;
+        const table = (0, arrow_1.makeArrowTable)([], { schema });
+        const buf = await (0, arrow_1.fromTableToBuffer)(table);
+        chai_1.assert.isAbove(buf.byteLength, 0);
+        const actual = (0, apache_arrow_1.tableFromIPC)(buf);
+        const actualSchema = actual.schema;
+        chai_1.assert.deepEqual(actualSchema, schema);
+    });
+});
 //# sourceMappingURL=arrow.test.js.map

dist/test/test.js

@@ -160,3 +160,3 @@ "use strict";
                 .search([0.1, 0.1])
-                .select(['is_active'])
+                .select(['is_active', 'vector'])
                 .execute();
@@ -875,2 +875,54 @@ assert.equal(results.length, 2);
 });
+(0, mocha_1.describe)('schema evolution', function () {
+    // Create a new sample table
+    it('can add a new column to the schema', async function () {
+        const dir = await (0, temp_1.track)().mkdir('lancejs');
+        const con = await lancedb.connect(dir);
+        const table = await con.createTable('vectors', [
+            { id: 1n, vector: [0.1, 0.2] }
+        ]);
+        await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as float)' }]);
+        const expectedSchema = new apache_arrow_1.Schema([
+            new apache_arrow_1.Field('id', new apache_arrow_1.Int64()),
+            new apache_arrow_1.Field('vector', new apache_arrow_1.FixedSizeList(2, new apache_arrow_1.Field('item', new apache_arrow_1.Float32(), true))),
+            new apache_arrow_1.Field('price', new apache_arrow_1.Float32())
+        ]);
+        expect(await table.schema).to.deep.equal(expectedSchema);
+    });
+    it('can alter the columns in the schema', async function () {
+        const dir = await (0, temp_1.track)().mkdir('lancejs');
+        const con = await lancedb.connect(dir);
+        const schema = new apache_arrow_1.Schema([
+            new apache_arrow_1.Field('id', new apache_arrow_1.Int64(), false),
+            new apache_arrow_1.Field('vector', new apache_arrow_1.FixedSizeList(2, new apache_arrow_1.Field('item', new apache_arrow_1.Float32(), true))),
+            new apache_arrow_1.Field('price', new apache_arrow_1.Float64(), false)
+        ]);
+        const table = await con.createTable('vectors', [
+            { id: 1n, vector: [0.1, 0.2], price: 10.0 }
+        ]);
+        expect(await table.schema).to.deep.equal(schema);
+        await table.alterColumns([
+            { path: 'id', rename: 'new_id' },
+            { path: 'price', nullable: true }
+        ]);
+        const expectedSchema = new apache_arrow_1.Schema([
+            new apache_arrow_1.Field('new_id', new apache_arrow_1.Int64(), false),
+            new apache_arrow_1.Field('vector', new apache_arrow_1.FixedSizeList(2, new apache_arrow_1.Field('item', new apache_arrow_1.Float32(), true))),
+            new apache_arrow_1.Field('price', new apache_arrow_1.Float64(), true)
+        ]);
+        expect(await table.schema).to.deep.equal(expectedSchema);
+    });
+    it('can drop a column from the schema', async function () {
+        const dir = await (0, temp_1.track)().mkdir('lancejs');
+        const con = await lancedb.connect(dir);
+        const table = await con.createTable('vectors', [
+            { id: 1n, vector: [0.1, 0.2] }
+        ]);
+        await table.dropColumns(['vector']);
+        const expectedSchema = new apache_arrow_1.Schema([
+            new apache_arrow_1.Field('id', new apache_arrow_1.Int64(), false)
+        ]);
+        expect(await table.schema).to.deep.equal(expectedSchema);
+    });
+});
 //# sourceMappingURL=test.js.map

package.json

 {
   "name": "vectordb",
-  "version": "0.4.11",
+  "version": "0.4.12",
   "description": " Serverless, low-latency vector database for AI applications",
@@ -44,2 +44,3 @@ "main": "dist/index.js",
     "@typescript-eslint/eslint-plugin": "^5.59.1",
+    "apache-arrow-old": "npm:apache-arrow@13.0.0",
     "cargo-cp-artifact": "^0.1",
@@ -65,7 +66,9 @@ "chai": "^4.3.7",
   "dependencies": {
-    "@apache-arrow/ts": "^14.0.2",
     "@neon-rs/load": "^0.0.74",
-    "apache-arrow": "^14.0.2",
     "axios": "^1.4.0"
   },
+  "peerDependencies": {
+    "@apache-arrow/ts": "^14.0.2",
+    "apache-arrow": "^14.0.2"
+  },
   "os": [
@@ -90,8 +93,8 @@ "darwin",
   "optionalDependencies": {
-    "@lancedb/vectordb-darwin-arm64": "0.4.11",
-    "@lancedb/vectordb-darwin-x64": "0.4.11",
-    "@lancedb/vectordb-linux-arm64-gnu": "0.4.11",
-    "@lancedb/vectordb-linux-x64-gnu": "0.4.11",
-    "@lancedb/vectordb-win32-x64-msvc": "0.4.11"
+    "@lancedb/vectordb-darwin-arm64": "0.4.12",
+    "@lancedb/vectordb-darwin-x64": "0.4.12",
+    "@lancedb/vectordb-linux-arm64-gnu": "0.4.12",
+    "@lancedb/vectordb-linux-x64-gnu": "0.4.12",
+    "@lancedb/vectordb-win32-x64-msvc": "0.4.12"
   }
-}
+}

src/arrow.ts

@@ -23,3 +23,3 @@ // Copyright 2023 Lance Developers.
   vectorFromArray,
-  type Schema,
+  Schema,
   Table as ArrowTable,
@@ -31,3 +31,3 @@ RecordBatchStreamWriter,
   Struct,
-  type Float,
+  Float,
   DataType,
@@ -38,2 +38,3 @@ Binary,
 import { type EmbeddingFunction } from './index'
+import { sanitizeSchema } from './sanitize'
 
@@ -207,6 +208,9 @@ /*
   const opt = new MakeArrowTableOptions(options !== undefined ? options : {})
+  if (opt.schema !== undefined && opt.schema !== null) {
+    opt.schema = sanitizeSchema(opt.schema)
+  }
   const columns: Record<string, Vector> = {}
   // TODO: sample dataset to find missing columns
   // Prefer the field ordering of the schema, if present
-  const columnNames = ((options?.schema) != null) ? (options?.schema?.names as string[]) : Object.keys(data[0])
+  const columnNames = ((opt.schema) != null) ? (opt.schema.names as string[]) : Object.keys(data[0])
   for (const colName of columnNames) {
@@ -336,2 +340,5 @@ if (data.length !== 0 && !Object.prototype.hasOwnProperty.call(data[0], colName)) {
   }
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema)
+  }
 
@@ -447,2 +454,5 @@ // Convert from ArrowTable to Record<String, Vector>
 ): Promise<Buffer> {
+  if (schema !== undefined && schema !== null) {
+    schema = sanitizeSchema(schema)
+  }
   const table = await convertToTable(data, embeddings, { schema })
@@ -465,2 +475,5 @@ const writer = RecordBatchFileWriter.writeAll(table)
 ): Promise<Buffer> {
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
+  }
   const table = await convertToTable(data, embeddings, { schema })
@@ -484,2 +497,5 @@ const writer = RecordBatchStreamWriter.writeAll(table)
 ): Promise<Buffer> {
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
+  }
   const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema)
@@ -503,2 +519,5 @@ const writer = RecordBatchFileWriter.writeAll(tableWithEmbeddings)
 ): Promise<Buffer> {
+  if (schema !== null && schema !== undefined) {
+    schema = sanitizeSchema(schema)
+  }
   const tableWithEmbeddings = await applyEmbeddings(table, embeddings, schema)
@@ -540,3 +559,3 @@ const writer = RecordBatchStreamWriter.writeAll(tableWithEmbeddings)
 export function createEmptyTable (schema: Schema): ArrowTable {
-  return new ArrowTable(schema)
+  return new ArrowTable(sanitizeSchema(schema))
 }

src/index.ts

@@ -45,3 +45,6 @@ // Copyright 2023 Lance Developers.
   tableIndexStats,
-  tableSchema
+  tableSchema,
+  tableAddColumns,
+  tableAlterColumns,
+  tableDropColumns
   // eslint-disable-next-line @typescript-eslint/no-var-requires
@@ -342,2 +345,3 @@ } = require('../native.js')
    * @param replace If false, fail if an index already exists on the column
+   * it is always set to true for remote connections
    *
@@ -386,3 +390,3 @@ * Scalar indices, like vector indices, can be used to speed up scans.  A scalar
    */
-  createScalarIndex: (column: string, replace: boolean) => Promise<void>
+  createScalarIndex: (column: string, replace?: boolean) => Promise<void>
 
@@ -506,4 +510,57 @@ /**
   schema: Promise<Schema>
+
+  // TODO: Support BatchUDF
+  /**
+   * Add new columns with defined values.
+   *
+   * @param newColumnTransforms pairs of column names and the SQL expression to use
+   *                            to calculate the value of the new column. These
+   *                            expressions will be evaluated for each row in the
+   *                            table, and can reference existing columns in the table.
+   */
+  addColumns(newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void>
+
+  /**
+   * Alter the name or nullability of columns.
+   *
+   * @param columnAlterations One or more alterations to apply to columns.
+   */
+  alterColumns(columnAlterations: ColumnAlteration[]): Promise<void>
+
+  /**
+   * Drop one or more columns from the dataset
+   *
+   * This is a metadata-only operation and does not remove the data from the
+   * underlying storage. In order to remove the data, you must subsequently
+   * call ``compact_files`` to rewrite the data without the removed columns and
+   * then call ``cleanup_files`` to remove the old files.
+   *
+   * @param columnNames The names of the columns to drop. These can be nested
+   *                    column references (e.g. "a.b.c") or top-level column
+   *                    names (e.g. "a").
+   */
+  dropColumns(columnNames: string[]): Promise<void>
 }
 
+/**
+ * A definition of a column alteration. The alteration changes the column at
+ * `path` to have the new name `name`, to be nullable if `nullable` is true,
+ * and to have the data type `data_type`. At least one of `rename` or `nullable`
+ * must be provided.
+ */
+export interface ColumnAlteration {
+  /**
+   * The path to the column to alter. This is a dot-separated path to the column.
+   * If it is a top-level column then it is just the name of the column. If it is
+   * a nested column then it is the path to the column, e.g. "a.b.c" for a column
+   * `c` nested inside a column `b` nested inside a column `a`.
+   */
+  path: string
+  rename?: string
+  /**
+   * Set the new nullability. Note that a nullable column cannot be made non-nullable.
+   */
+  nullable?: boolean
+}
+
 export interface UpdateArgs {
@@ -865,3 +922,6 @@ /**
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
+  async createScalarIndex (column: string, replace?: boolean): Promise<void> {
+    if (replace === undefined) {
+      replace = true
+    }
     return tableCreateScalarIndex.call(this._tbl, column, replace)
@@ -1036,2 +1096,14 @@ }
   }
+
+  async addColumns (newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void> {
+    return tableAddColumns.call(this._tbl, newColumnTransforms)
+  }
+
+  async alterColumns (columnAlterations: ColumnAlteration[]): Promise<void> {
+    return tableAlterColumns.call(this._tbl, columnAlterations)
+  }
+
+  async dropColumns (columnNames: string[]): Promise<void> {
+    return tableDropColumns.call(this._tbl, columnNames)
+  }
 }
@@ -1038,0 +1110,0 @@

src/remote/index.ts

@@ -28,3 +28,4 @@ // Copyright 2023 LanceDB Developers.
   makeArrowTable,
-  type MergeInsertArgs
+  type MergeInsertArgs,
+  type ColumnAlteration
 } from '../index'
@@ -400,3 +401,3 @@ import { Query } from '../query'
     const column = indexParams.column ?? 'vector'
-    const indexType = 'vector' // only vector index is supported for remote connections
+    const indexType = 'vector'
     const metricType = indexParams.metric_type ?? 'L2'
@@ -424,4 +425,21 @@ const indexCacheSize = indexParams.index_cache_size ?? null
 
-  async createScalarIndex (column: string, replace: boolean): Promise<void> {
-    throw new Error('Not implemented')
+  async createScalarIndex (column: string): Promise<void> {
+    const indexType = 'scalar'
+
+    const data = {
+      column,
+      index_type: indexType,
+      replace: true
+    }
+    const res = await this._client.post(
+      `/v1/table/${this._name}/create_scalar_index/`,
+      data
+    )
+    if (res.status !== 200) {
+      throw new Error(
+        `Server Error, status: ${res.status}, ` +
+          // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
+          `message: ${res.statusText}: ${res.data}`
+      )
+    }
   }
@@ -480,2 +498,14 @@
   }
+
+  async addColumns (newColumnTransforms: Array<{ name: string, valueSql: string }>): Promise<void> {
+    throw new Error('Add columns is not yet supported in LanceDB Cloud.')
+  }
+
+  async alterColumns (columnAlterations: ColumnAlteration[]): Promise<void> {
+    throw new Error('Alter columns is not yet supported in LanceDB Cloud.')
+  }
+
+  async dropColumns (columnNames: string[]): Promise<void> {
+    throw new Error('Drop columns is not yet supported in LanceDB Cloud.')
+  }
 }

src/test/arrow.test.ts

@@ -37,4 +37,16 @@ // Copyright 2024 Lance Developers.
   Dictionary,
-  Int64
+  Int64,
+  MetadataVersion
 } from 'apache-arrow'
+import {
+  Dictionary as OldDictionary,
+  Field as OldField,
+  FixedSizeList as OldFixedSizeList,
+  Float32 as OldFloat32,
+  Int32 as OldInt32,
+  Struct as OldStruct,
+  Schema as OldSchema,
+  TimestampNanosecond as OldTimestampNanosecond,
+  Utf8 as OldUtf8
+} from 'apache-arrow-old'
 import { type EmbeddingFunction } from '../embedding/embedding_function'
@@ -322,1 +334,29 @@
 })
+
+describe('when using two versions of arrow', function () {
+  it('can still import data', async function() {
+    const schema = new OldSchema([
+      new OldField('id', new OldInt32()),
+      new OldField('vector', new OldFixedSizeList(1024, new OldField("item", new OldFloat32(), true))),
+      new OldField('struct', new OldStruct([
+        new OldField('nested', new OldDictionary(new OldUtf8(), new OldInt32(), 1, true)),
+        new OldField('ts_with_tz', new OldTimestampNanosecond("some_tz")),
+        new OldField('ts_no_tz', new OldTimestampNanosecond(null))
+      ]))
+    ]) as any
+    // We use arrow version 13 to emulate a "foreign arrow" and this version doesn't have metadataVersion
+    // In theory, this wouldn't matter.  We don't rely on that property.  However, it causes deepEqual to
+    // fail so we patch it back in
+    schema.metadataVersion = MetadataVersion.V5
+    const table = makeArrowTable(
+      [],
+      { schema }
+    )
+
+    const buf = await fromTableToBuffer(table)
+    assert.isAbove(buf.byteLength, 0)
+    const actual = tableFromIPC(buf)
+    const actualSchema = actual.schema
+    assert.deepEqual(actualSchema, schema)
+  })
+})

src/test/test.ts

@@ -40,4 +40,6 @@ // Copyright 2023 LanceDB Developers.
   vectorFromArray,
+  Float64,
   Float32,
-  Float16
+  Float16,
+  Int64
 } from 'apache-arrow'
@@ -200,3 +202,3 @@
         .search([0.1, 0.1])
-        .select(['is_active'])
+        .select(['is_active', 'vector'])
         .execute()
@@ -1062,1 +1064,61 @@ assert.equal(results.length, 2)
 })
+
+describe('schema evolution', function () {
+  // Create a new sample table
+  it('can add a new column to the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+
+    await table.addColumns([{ name: 'price', valueSql: 'cast(10.0 as float)' }])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64()),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float32())
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+
+  it('can alter the columns in the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const schema = new Schema([
+      new Field('id', new Int64(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float64(), false)
+    ])
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2], price: 10.0 }
+    ])
+    expect(await table.schema).to.deep.equal(schema)
+
+    await table.alterColumns([
+      { path: 'id', rename: 'new_id' },
+      { path: 'price', nullable: true }
+    ])
+
+    const expectedSchema = new Schema([
+      new Field('new_id', new Int64(), false),
+      new Field('vector', new FixedSizeList(2, new Field('item', new Float32(), true))),
+      new Field('price', new Float64(), true)
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+
+  it('can drop a column from the schema', async function () {
+    const dir = await track().mkdir('lancejs')
+    const con = await lancedb.connect(dir)
+    const table = await con.createTable('vectors', [
+      { id: 1n, vector: [0.1, 0.2] }
+    ])
+    await table.dropColumns(['vector'])
+
+    const expectedSchema = new Schema([
+      new Field('id', new Int64(), false)
+    ])
+    expect(await table.schema).to.deep.equal(expectedSchema)
+  })
+})

No alert changes

Improved metrics

Total package byte prevSize

491182

increased by15.47%

Number of package files

77

increased by5.48%

Lines of code

9581

increased by14.69%

Worsened metrics

Dev dependency count

28

increased by3.7%

Dependency changes

• + Added@lancedb/vectordb-darwin-arm64@0.4.12(transitive)

• + Added@lancedb/vectordb-darwin-x64@0.4.12(transitive)

• + Added@lancedb/vectordb-linux-arm64-gnu@0.4.12(transitive)

• + Added@lancedb/vectordb-linux-x64-gnu@0.4.12(transitive)

• + Added@lancedb/vectordb-win32-x64-msvc@0.4.12(transitive)

• - Removed@apache-arrow/ts@^14.0.2

• - Removedapache-arrow@^14.0.2

• - Removed@lancedb/vectordb-darwin-arm64@0.4.11(transitive)

• - Removed@lancedb/vectordb-darwin-x64@0.4.11(transitive)

• - Removed@lancedb/vectordb-linux-arm64-gnu@0.4.11(transitive)

• - Removed@lancedb/vectordb-linux-x64-gnu@0.4.11(transitive)

• - Removed@lancedb/vectordb-win32-x64-msvc@0.4.11(transitive)