@singlestore/ai - npm Package Compare versions

Comparing version 0.0.30 to 0.0.31

dist/index.d.ts

@@ -122,3 +122,11 @@ import { ChatCompletionMessageParam } from 'openai/resources/chat/completions';
 
-interface AIConfig<TChatCompletionTools extends AnyChatCompletionTool[] | undefined, TChatCompletionsManager extends ChatCompletionsManager<TChatCompletionTools>, TEmbeddingsManager extends EmbeddingsManager> {
+interface TextSplitterSplitOptions {
+    chunkSize?: number;
+    delimiter?: string;
+}
+declare class TextSplitter {
+    split(text: string, options?: TextSplitterSplitOptions): string[];
+}
+
+interface AIConfig<TChatCompletionTools extends AnyChatCompletionTool[] | undefined, TChatCompletionsManager extends ChatCompletionsManager<TChatCompletionTools>, TEmbeddingsManager extends EmbeddingsManager, TTextSplitter extends TextSplitter> {
     openAIApiKey?: string;
@@ -128,10 +136,12 @@ chatCompletionTools?: TChatCompletionTools;
     embeddingsManager?: TEmbeddingsManager;
+    textSplitter?: TTextSplitter;
 }
-type AnyAI = AI<AnyChatCompletionTool[] | undefined, ChatCompletionsManager<any>, EmbeddingsManager>;
-declare class AI<TChatCompletionTools extends AnyChatCompletionTool[] | undefined = undefined, TChatCompletions extends ChatCompletionsManager<any> = OpenAIChatCompletions<TChatCompletionTools>, TEmbeddingsManager extends EmbeddingsManager = OpenAIEmbeddingsManager> {
+type AnyAI = AI<AnyChatCompletionTool[] | undefined, ChatCompletionsManager<any>, EmbeddingsManager, TextSplitter>;
+declare class AI<TChatCompletionTools extends AnyChatCompletionTool[] | undefined = undefined, TChatCompletions extends ChatCompletionsManager<any> = OpenAIChatCompletions<TChatCompletionTools>, TEmbeddingsManager extends EmbeddingsManager = OpenAIEmbeddingsManager, TTextSplitter extends TextSplitter = TextSplitter> {
     embeddings: TEmbeddingsManager;
     chatCompletions: TChatCompletions;
-    constructor(config: AIConfig<TChatCompletionTools, TChatCompletions, TEmbeddingsManager>);
+    textSplitter: TTextSplitter;
+    constructor(config: AIConfig<TChatCompletionTools, TChatCompletions, TEmbeddingsManager, TTextSplitter>);
 }
 
-export { AI, type AIConfig, type AnyAI, type AnyChatCompletionTool, type ChatCompletion, type ChatCompletionMessage, type ChatCompletionStream, ChatCompletionTool, type ChatCompletionToolCall, ChatCompletionsManager, type CreateChatCompletionParams, type CreateChatCompletionResult, type CreateEmbeddingsParams, type Embedding, EmbeddingsManager, type MergeChatCompletionTools, MessageLengthExceededError, MessagesLengthExceededError, type OnChatCompletionChunk, type OpenAIChatCompletionModel, OpenAIChatCompletions, type OpenAICreateChatCompletionParams, type OpenAICreateEmbeddingsParams, type OpenAIEmbeddingModel, OpenAIEmbeddingsManager, parseLengthErrorMessage };
+export { AI, type AIConfig, type AnyAI, type AnyChatCompletionTool, type ChatCompletion, type ChatCompletionMessage, type ChatCompletionStream, ChatCompletionTool, type ChatCompletionToolCall, ChatCompletionsManager, type CreateChatCompletionParams, type CreateChatCompletionResult, type CreateEmbeddingsParams, type Embedding, EmbeddingsManager, type MergeChatCompletionTools, MessageLengthExceededError, MessagesLengthExceededError, type OnChatCompletionChunk, type OpenAIChatCompletionModel, OpenAIChatCompletions, type OpenAICreateChatCompletionParams, type OpenAICreateEmbeddingsParams, type OpenAIEmbeddingModel, OpenAIEmbeddingsManager, TextSplitter, type TextSplitterSplitOptions, parseLengthErrorMessage };

dist/index.js

@@ -39,2 +39,3 @@ "use strict";
   MessagesLengthExceededError: () => MessagesLengthExceededError,
+  TextSplitter: () => TextSplitter,
   parseLengthErrorMessage: () => parseLengthErrorMessage
@@ -299,2 +300,39 @@ });
 
+// src/text-splitter/text-splitter.ts
+var TextSplitter = class {
+  split(text, options = {}) {
+    const { chunkSize = 1024, delimiter = /(?<=[.!?])[\s\n]+/ } = options;
+    const chunks = [];
+    let accChunk = [];
+    let accChunkLength = 0;
+    for (const chunk of text.split(delimiter)) {
+      if (chunk.length > chunkSize) {
+        let subChunks = [];
+        if (delimiter === " ") {
+          subChunks.push(chunk);
+        } else {
+          subChunks = this.split(chunk, { chunkSize, delimiter: " " });
+        }
+        if (accChunkLength > 0) {
+          chunks.push(accChunk.join(" "));
+          accChunk = [];
+          accChunkLength = 0;
+        }
+        chunks.push(...subChunks);
+      } else if (accChunkLength + chunk.length <= chunkSize) {
+        accChunk.push(chunk);
+        accChunkLength += chunk.length;
+      } else {
+        chunks.push(accChunk.join(" "));
+        accChunk = [chunk];
+        accChunkLength = chunk.length;
+      }
+    }
+    if (accChunk.length > 0) {
+      chunks.push(accChunk.join(" "));
+    }
+    return chunks;
+  }
+};
+
 // src/ai.ts
@@ -305,2 +343,3 @@ var import_openai2 = __toESM(require("openai"));
   chatCompletions;
+  textSplitter;
   constructor(config) {
@@ -310,2 +349,3 @@ const openai = new import_openai2.default({ apiKey: config.openAIApiKey });
     this.embeddings = config.embeddingsManager ?? new OpenAIEmbeddingsManager(openai);
+    this.textSplitter = config.textSplitter ?? new TextSplitter();
     if (config.chatCompletionTools?.length) {
@@ -324,4 +364,5 @@ this.chatCompletions.initTools(config.chatCompletionTools);
   MessagesLengthExceededError,
+  TextSplitter,
   parseLengthErrorMessage
 });
 //# sourceMappingURL=index.js.map

package.json

 {
   "name": "@singlestore/ai",
-  "version": "0.0.30",
+  "version": "0.0.31",
   "license": "Apache-2.0",
@@ -5,0 +5,0 @@ "sideEffects": false,

README.md

@@ -19,2 +19,3 @@ # SingleStore AI
     - [With Custom Chat Completion Tools](#with-custom-chat-completion-tools)
+    - [With Custom Text Splitter](#with-custom-text-splitter)
     - [Additional Notes](#additional-notes)
@@ -33,2 +34,4 @@ - [Embeddings](#embeddings)
       - [Additional Notes](#additional-notes-1)
+  - [Text Splitter](#text-splitter)
+    - [Split Text](#split-text)
 
@@ -147,2 +150,18 @@ </details>
 
+#### With Custom Text Splitter
+
+You can define a custom text splitter by extending the `TextSplitter` class to handle how text is split.
+
+```ts
+import { AI, TextSplitter, TextSplitterSplitOptions } from "@singlestore/ai";
+
+class CustomTextSplitter extends TextSplitter {
+  split(text: string, options?: TextSplitterSplitOptions): string[] {
+    return [];
+  }
+}
+
+const ai = new AI({ textSplitter: new CustomTextSplitter() });
+```
+
 #### Additional Notes
@@ -240,1 +259,19 @@
 ---
+
+### Text Splitter
+
+#### Split Text
+
+Breaks a given text into smaller chunks, making it easier to handle for tasks like generating embeddings. By default, it splits text by sentences, but you can customize it to use a different delimiter or set the maximum chunk size.
+
+```ts
+const chunks = ai.textSplitter.split(
+  text,
+  {
+    chunkSize: 1024, // Optional; 1024 be deafult
+    delimiter: " ", // Optional; Sentence splitting by default
+  }, // Optional
+);
+```
+
+---

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

99833

increased by10.19%

Lines of code

794

increased by12.62%

Number of lines in readme file

274

increased by15.61%

No dependency changes