@lancedb/lancedb - npm Package Compare versions

Comparing version 0.14.1-beta.3 to 0.14.1-beta.4

dist/indices.d.ts

@@ -317,2 +317,41 @@ /**
     withPosition?: boolean;
+    /**
+     * The tokenizer to use when building the index.
+     * The default is "simple".
+     *
+     * The following tokenizers are available:
+     *
+     * "simple" - Simple tokenizer. This tokenizer splits the text into tokens using whitespace and punctuation as a delimiter.
+     *
+     * "whitespace" - Whitespace tokenizer. This tokenizer splits the text into tokens using whitespace as a delimiter.
+     *
+     * "raw" - Raw tokenizer. This tokenizer does not split the text into tokens and indexes the entire text as a single token.
+     */
+    baseTokenizer?: "simple" | "whitespace" | "raw";
+    /**
+     * language for stemming and stop words
+     * this is only used when `stem` or `remove_stop_words` is true
+     */
+    language?: string;
+    /**
+     * maximum token length
+     * tokens longer than this length will be ignored
+     */
+    maxTokenLength?: number;
+    /**
+     * whether to lowercase tokens
+     */
+    lowercase?: boolean;
+    /**
+     * whether to stem tokens
+     */
+    stem?: boolean;
+    /**
+     * whether to remove stop words
+     */
+    removeStopWords?: boolean;
+    /**
+     * whether to remove punctuation
+     */
+    asciiFolding?: boolean;
 }
@@ -319,0 +358,0 @@ export declare class Index {

dist/indices.js

@@ -104,3 +104,3 @@ "use strict";
     static fts(options) {
-        return new Index(native_1.Index.fts(options?.withPosition));
+        return new Index(native_1.Index.fts(options?.withPosition, options?.baseTokenizer, options?.language, options?.maxTokenLength, options?.lowercase, options?.stem, options?.removeStopWords, options?.asciiFolding));
     }
@@ -107,0 +107,0 @@ /**

dist/native.d.ts

@@ -264,3 +264,3 @@ /* tslint:disable */
   static labelList(): Index
-  static fts(withPosition?: boolean | undefined | null): Index
+  static fts(withPosition?: boolean | undefined | null, baseTokenizer?: string | undefined | null, language?: string | undefined | null, maxTokenLength?: number | undefined | null, lowerCase?: boolean | undefined | null, stem?: boolean | undefined | null, removeStopWords?: boolean | undefined | null, asciiFolding?: boolean | undefined | null): Index
   static hnswPq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, numSubVectors?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index
@@ -267,0 +267,0 @@ static hnswSq(distanceType?: string | undefined | null, numPartitions?: number | undefined | null, maxIterations?: number | undefined | null, sampleRate?: number | undefined | null, m?: number | undefined | null, efConstruction?: number | undefined | null): Index

package.json

@@ -14,3 +14,3 @@ {
   "private": false,
-  "version": "0.14.1-beta.3",
+  "version": "0.14.1-beta.4",
   "main": "dist/index.js",
@@ -102,10 +102,10 @@ "exports": {
   "optionalDependencies": {
-    "@lancedb/lancedb-darwin-x64": "0.14.1-beta.3",
-    "@lancedb/lancedb-darwin-arm64": "0.14.1-beta.3",
-    "@lancedb/lancedb-linux-x64-gnu": "0.14.1-beta.3",
-    "@lancedb/lancedb-linux-arm64-gnu": "0.14.1-beta.3",
-    "@lancedb/lancedb-linux-x64-musl": "0.14.1-beta.3",
-    "@lancedb/lancedb-linux-arm64-musl": "0.14.1-beta.3",
-    "@lancedb/lancedb-win32-x64-msvc": "0.14.1-beta.3",
-    "@lancedb/lancedb-win32-arm64-msvc": "0.14.1-beta.3"
+    "@lancedb/lancedb-darwin-x64": "0.14.1-beta.4",
+    "@lancedb/lancedb-darwin-arm64": "0.14.1-beta.4",
+    "@lancedb/lancedb-linux-x64-gnu": "0.14.1-beta.4",
+    "@lancedb/lancedb-linux-arm64-gnu": "0.14.1-beta.4",
+    "@lancedb/lancedb-linux-x64-musl": "0.14.1-beta.4",
+    "@lancedb/lancedb-linux-arm64-musl": "0.14.1-beta.4",
+    "@lancedb/lancedb-win32-x64-msvc": "0.14.1-beta.4",
+    "@lancedb/lancedb-win32-arm64-msvc": "0.14.1-beta.4"
   },
@@ -112,0 +112,0 @@ "peerDependencies": {

No alert changes

Improved metrics

Total package byte prevSize

247167

increased by0.66%

Lines of code

5880

increased by0.68%

No dependency changes