@lancedb/lancedb - npm Package Compare versions

Comparing version 0.7.1 to 0.8.0

dist/arrow.d.ts

 /// <reference types="node" />
-import { Table as ArrowTable, Binary, BufferType, DataType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
+import { Table as ArrowTable, Binary, BufferType, Field, FixedSizeBinary, FixedSizeList, Float, Int, LargeBinary, List, Null, RecordBatch, Schema, Struct, Utf8 } from "apache-arrow";
 import { Buffers } from "apache-arrow/data";
@@ -40,4 +40,6 @@ import { type EmbeddingFunction } from "./embedding/embedding_function";
 export type IntoVector = Float32Array | Float64Array | number[] | Promise<Float32Array | Float64Array | number[]>;
+export type FloatLike = import("apache-arrow-13").Float | import("apache-arrow-14").Float | import("apache-arrow-15").Float | import("apache-arrow-16").Float | import("apache-arrow-17").Float;
+export type DataTypeLike = import("apache-arrow-13").DataType | import("apache-arrow-14").DataType | import("apache-arrow-15").DataType | import("apache-arrow-16").DataType | import("apache-arrow-17").DataType;
 export declare function isArrowTable(value: object): value is TableLike;
-export declare function isDataType(value: unknown): value is DataType;
+export declare function isDataType(value: unknown): value is DataTypeLike;
 export declare function isNull(value: unknown): value is Null;
@@ -204,3 +206,3 @@ export declare function isInt(value: unknown): value is Int;
 /** Creates the Arrow Type for a Vector column with dimension `dim` */
-export declare function newVectorType<T extends Float>(dim: number, innerType: T): FixedSizeList<T>;
+export declare function newVectorType<T extends Float>(dim: number, innerType: unknown): FixedSizeList<T>;
 /**
@@ -207,0 +209,0 @@ * Serialize an Array of records into a buffer using the Arrow IPC File serialization

dist/arrow.js

@@ -469,3 +469,3 @@ "use strict";
     const registry = (0, registry_1.getRegistry)();
-    const functions = registry.parseFunctions(schema.metadata);
+    const functions = await registry.parseFunctions(schema.metadata);
     const columns = Object.fromEntries(table.schema.fields.map((field) => [
@@ -472,0 +472,0 @@ field.name,

dist/embedding/embedding_function.d.ts

 import "reflect-metadata";
-import { DataType, Float, type IntoVector } from "../arrow";
+import { DataType, DataTypeLike, FloatLike, type IntoVector } from "../arrow";
 /**
@@ -47,2 +47,3 @@ * Options for a given embedding function
     abstract toJSON(): Partial<M>;
+    init?(): Promise<void>;
     /**
@@ -55,3 +56,3 @@ * sourceField is used in combination with `LanceSchema` to provide a declarative data model
      */
-    sourceField(optionsOrDatatype: Partial<FieldOptions> | DataType): [DataType, Map<string, EmbeddingFunction>];
+    sourceField(optionsOrDatatype: Partial<FieldOptions> | DataTypeLike): [DataTypeLike, Map<string, EmbeddingFunction>];
     /**
@@ -68,3 +69,3 @@ * vectorField is used in combination with `LanceSchema` to provide a declarative data model
     /** The datatype of the embeddings */
-    abstract embeddingDataType(): Float;
+    abstract embeddingDataType(): FloatLike;
     /**
@@ -71,0 +72,0 @@ * Creates a vector representation for the given values.

dist/embedding/index.d.ts

@@ -5,2 +5,3 @@ import { Schema } from "../arrow";
 export * from "./openai";
+export * from "./transformers";
 export * from "./registry";
@@ -7,0 +8,0 @@ /**

dist/embedding/index.js

@@ -39,2 +39,3 @@ "use strict";
 __exportStar(require("./openai"), exports);
+__exportStar(require("./transformers"), exports);
 __exportStar(require("./registry"), exports);
@@ -41,0 +42,0 @@ /**

dist/embedding/registry.d.ts

 import { type EmbeddingFunction, type EmbeddingFunctionConstructor } from "./embedding_function";
 import "reflect-metadata";
 import { OpenAIEmbeddingFunction } from "./openai";
+import { TransformersEmbeddingFunction } from "./transformers";
+type CreateReturnType<T> = T extends {
+    init: () => Promise<void>;
+} ? Promise<T> : T;
 interface EmbeddingFunctionCreate<T extends EmbeddingFunction> {
-    create(options?: T["TOptions"]): T;
+    create(options?: T["TOptions"]): CreateReturnType<T>;
 }
@@ -22,8 +26,6 @@ /**
     register<T extends EmbeddingFunctionConstructor = EmbeddingFunctionConstructor>(this: EmbeddingFunctionRegistry, alias?: string): (ctor: T) => any;
+    get(name: "openai"): EmbeddingFunctionCreate<OpenAIEmbeddingFunction>;
+    get(name: "huggingface"): EmbeddingFunctionCreate<TransformersEmbeddingFunction>;
+    get<T extends EmbeddingFunction<unknown>>(name: string): EmbeddingFunctionCreate<T> | undefined;
     /**
-     * Fetch an embedding function by name
-     * @param name The name of the function
-     */
-    get<T extends EmbeddingFunction<unknown>, Name extends string = "">(name: Name extends "openai" ? "openai" : string): Name extends "openai" ? EmbeddingFunctionCreate<OpenAIEmbeddingFunction> : EmbeddingFunctionCreate<T> | undefined;
-    /**
      * reset the registry to the initial state
@@ -35,3 +37,3 @@ */
      */
-    parseFunctions(this: EmbeddingFunctionRegistry, metadata: Map<string, string>): Map<string, EmbeddingFunctionConfig>;
+    parseFunctions(this: EmbeddingFunctionRegistry, metadata: Map<string, string>): Promise<Map<string, EmbeddingFunctionConfig>>;
     functionToMetadata(conf: EmbeddingFunctionConfig): Record<string, any>;
@@ -38,0 +40,0 @@ getTableMetadata(functions: EmbeddingFunctionConfig[]): Map<string, string>;

dist/embedding/registry.js

@@ -53,8 +53,24 @@ "use strict";
         if (!factory) {
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
             return undefined;
         }
+        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+        let create;
+        if (factory.prototype.init) {
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            create = async function (options) {
+                const instance = new factory(options);
+                await instance.init();
+                return instance;
+            };
+        }
+        else {
+            // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+            create = function (options) {
+                const instance = new factory(options);
+                return instance;
+            };
+        }
         return {
-            create: function (options) {
-                return new factory(options);
-            },
+            create,
         };
@@ -71,3 +87,3 @@ }
      */
-    parseFunctions(metadata) {
+    async parseFunctions(metadata) {
         if (!metadata.has("embedding_functions")) {
@@ -78,3 +94,3 @@ return new Map();
             const functions = (JSON.parse(metadata.get("embedding_functions")));
-            return new Map(functions.map((f) => {
+            const items = await Promise.all(functions.map(async (f) => {
                 const fn = this.get(f.name);
@@ -84,2 +100,3 @@ if (!fn) {
                 }
+                const func = await this.get(f.name).create(f.model);
                 return [
@@ -90,6 +107,7 @@ f.name,
                         vectorColumn: f.vectorColumn,
-                        function: this.get(f.name).create(f.model),
+                        function: func,
                     },
                 ];
             }));
+            return new Map(items);
         }
@@ -96,0 +114,0 @@ }

dist/query.js

@@ -132,18 +132,24 @@ "use strict";
     select(columns) {
-        let columnTuples;
+        const selectColumns = (columnArray) => {
+            this.doCall((inner) => {
+                inner.selectColumns(columnArray);
+            });
+        };
+        const selectMapping = (columnTuples) => {
+            this.doCall((inner) => {
+                inner.select(columnTuples);
+            });
+        };
         if (typeof columns === "string") {
-            columns = [columns];
+            selectColumns([columns]);
         }
-        if (Array.isArray(columns)) {
-            columnTuples = columns.map((c) => [c, c]);
+        else if (Array.isArray(columns)) {
+            selectColumns(columns);
         }
         else if (columns instanceof Map) {
-            columnTuples = Array.from(columns.entries());
+            selectMapping(Array.from(columns.entries()));
         }
         else {
-            columnTuples = Object.entries(columns);
+            selectMapping(Object.entries(columns));
         }
-        this.doCall((inner) => {
-            inner.select(columnTuples);
-        });
         return this;
@@ -150,0 +156,0 @@ }

dist/remote/client.d.ts

@@ -7,3 +7,3 @@ /// <reference types="node" />
     #private;
-    constructor(dbName: string, apiKey: string, region: string, hostOverride?: string, connectionTimeout?: number, readTimeout?: number);
+    constructor(dbName: string, apiKey: string, region: string, hostOverride?: string, timeout?: number);
     get session(): import("axios").AxiosInstance;
@@ -10,0 +10,0 @@ get url(): string;

dist/remote/client.js

@@ -25,6 +25,5 @@ "use strict";
     #closed = false;
-    #connectionTimeout = 12 * 1000; // 12 seconds;
-    #readTimeout = 30 * 1000; // 30 seconds;
+    #timeout = 12 * 1000; // 12 seconds;
     #session;
-    constructor(dbName, apiKey, region, hostOverride, connectionTimeout, readTimeout) {
+    constructor(dbName, apiKey, region, hostOverride, timeout) {
         this.#dbName = dbName;
@@ -34,4 +33,3 @@ this.#apiKey = apiKey;
         this.#hostOverride = hostOverride ?? this.#hostOverride;
-        this.#connectionTimeout = connectionTimeout ?? this.#connectionTimeout;
-        this.#readTimeout = readTimeout ?? this.#readTimeout;
+        this.#timeout = timeout ?? this.#timeout;
     }
@@ -51,3 +49,3 @@ // todo: cache the session.
                 transformResponse: decodeErrorData,
-                timeout: this.#connectionTimeout,
+                timeout: this.#timeout,
             });
@@ -97,3 +95,3 @@ }
         catch (e) {
-            if (e instanceof axios_1.AxiosError) {
+            if (e instanceof axios_1.AxiosError && e.response) {
                 response = e.response;
@@ -127,3 +125,3 @@ }
         catch (e) {
-            if (e instanceof axios_1.AxiosError) {
+            if (e instanceof axios_1.AxiosError && e.response) {
                 response = e.response;
@@ -130,0 +128,0 @@ }

dist/remote/connection.d.ts

@@ -8,8 +8,7 @@ import { Data, SchemaLike } from "../arrow";
     hostOverride?: string;
-    connectionTimeout?: number;
-    readTimeout?: number;
+    timeout?: number;
 }
 export declare class RemoteConnection extends Connection {
     #private;
-    constructor(url: string, { apiKey, region, hostOverride, connectionTimeout, readTimeout, }: RemoteConnectionOptions);
+    constructor(url: string, { apiKey, region, hostOverride, timeout }: RemoteConnectionOptions);
     isOpen(): boolean;
@@ -16,0 +15,0 @@ close(): void;

dist/remote/connection.js

@@ -16,3 +16,3 @@ "use strict";
     #tableCache = new util_1.TTLCache(300000);
-    constructor(url, { apiKey, region, hostOverride, connectionTimeout, readTimeout, }) {
+    constructor(url, { apiKey, region, hostOverride, timeout }) {
         super();
@@ -34,3 +34,3 @@ apiKey = apiKey ?? process.env.LANCEDB_API_KEY;
         this.#region = region;
-        this.#client = new client_1.RestfulLanceDBClient(this.#dbName, this.#apiKey, this.#region, hostOverride, connectionTimeout, readTimeout);
+        this.#client = new client_1.RestfulLanceDBClient(this.#dbName, this.#apiKey, this.#region, hostOverride, timeout);
     }
@@ -37,0 +37,0 @@ isOpen() {

dist/table.d.ts

@@ -224,2 +224,4 @@ /// <reference types="node" />
      * @note  If no embedding functions are defined in the table, this will error when collecting the results.
+     *
+     * This is just a convenience method for calling `.query().nearestTo(await myEmbeddingFunction(query))`
      */
@@ -231,2 +233,3 @@ abstract search(query: string): VectorQuery;
      * @param {IntoVector} query - the query vector
+     * This is just a convenience method for calling `.query().nearestTo(query)`
      */
@@ -233,0 +236,0 @@ abstract search(query: IntoVector): VectorQuery;

dist/table.js

@@ -96,3 +96,3 @@ "use strict";
         const registry = (0, registry_1.getRegistry)();
-        const functions = registry.parseFunctions(schema.metadata);
+        const functions = await registry.parseFunctions(schema.metadata);
         const buffer = await (0, arrow_1.fromDataToBuffer)(data, functions.values().next().value, schema);
@@ -99,0 +99,0 @@ await this.inner.add(buffer, mode);

package.json

@@ -13,3 +13,3 @@ {
   ],
-  "version": "0.7.1",
+  "version": "0.8.0",
   "main": "dist/index.js",
@@ -36,11 +36,16 @@ "exports": {
   "devDependencies": {
+    "@aws-sdk/client-dynamodb": "^3.33.0",
     "@aws-sdk/client-kms": "^3.33.0",
     "@aws-sdk/client-s3": "^3.33.0",
-    "@aws-sdk/client-dynamodb": "^3.33.0",
     "@biomejs/biome": "^1.7.3",
     "@jest/globals": "^29.7.0",
     "@napi-rs/cli": "^2.18.3",
+    "@types/axios": "^0.14.0",
     "@types/jest": "^29.1.2",
     "@types/tmp": "^0.2.6",
-    "apache-arrow-old": "npm:apache-arrow@13.0.0",
+    "apache-arrow-13": "npm:apache-arrow@13.0.0",
+    "apache-arrow-14": "npm:apache-arrow@14.0.0",
+    "apache-arrow-15": "npm:apache-arrow@15.0.0",
+    "apache-arrow-16": "npm:apache-arrow@16.0.0",
+    "apache-arrow-17": "npm:apache-arrow@17.0.0",
     "eslint": "^8.57.0",
@@ -51,7 +56,6 @@ "jest": "^29.7.0",
     "ts-jest": "^29.1.2",
-    "typedoc": "^0.25.7",
-    "typedoc-plugin-markdown": "^3.17.1",
+    "typedoc": "^0.26.4",
+    "typedoc-plugin-markdown": "^4.2.1",
     "typescript": "^5.3.3",
-    "typescript-eslint": "^7.1.0",
-    "@types/axios": "^0.14.0"
+    "typescript-eslint": "^7.1.0"
   },
@@ -83,3 +87,2 @@ "ava": {
     "lint-fix": "biome check --write . && biome format --write .",
-    "prepublishOnly": "napi prepublish -t npm",
     "test": "jest --verbose",
@@ -95,11 +98,11 @@ "integration": "S3_TEST=1 npm run test",
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-arm64": "0.7.1",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.7.1",
-    "@lancedb/lancedb-darwin-x64": "0.7.1",
-    "@lancedb/lancedb-linux-x64-gnu": "0.7.1",
-    "@lancedb/lancedb-win32-x64-msvc": "0.7.1"
+    "@lancedb/lancedb-darwin-arm64": "0.8.0",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.8.0",
+    "@lancedb/lancedb-darwin-x64": "0.8.0",
+    "@lancedb/lancedb-linux-x64-gnu": "0.8.0",
+    "@lancedb/lancedb-win32-x64-msvc": "0.8.0"
   },
   "peerDependencies": {
-    "apache-arrow": "^15.0.0"
+    "apache-arrow": ">=13.0.0 <=17.0.0"
   }
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Uses eval

Supply chain risk

Package uses dynamic code execution (e.g., eval()), which is a dangerous practice. This can prevent the code from running in certain environments and increases the risk that the code may contain exploits or malicious behavior.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Number of low supply chain risk alerts

5

decreased by-44.44%

Worsened metrics

Total package byte prevSize

230728

decreased by-63.72%

Dev dependency count

23

increased by21.05%

Number of package files

39

decreased by-63.21%

Lines of code

5446

decreased by-66.29%

Number of medium supply chain risk alerts

4

increased by100%

No dependency changes