embeddings-splitter
Advanced tools
+1
-1
@@ -1,2 +0,2 @@ | ||
| export declare function split(prompt: string): string[]; | ||
| export declare function split(prompt: string, maxTokens?: number): string[]; | ||
| type Batches = { | ||
@@ -3,0 +3,0 @@ data: string; |
+3
-3
@@ -15,3 +15,3 @@ "use strict"; | ||
| const tokenizer = (0, tiktoken_1.get_encoding)('cl100k_base'); | ||
| function splitIntoMany(text, maxTokens = 500) { | ||
| function splitIntoMany(text, maxTokens) { | ||
| // Split the text into sentences | ||
@@ -47,3 +47,3 @@ const sentences = text.split('. '); | ||
| } | ||
| function split(prompt) { | ||
| function split(prompt, maxTokens = 500) { | ||
| // should split into tokens | ||
@@ -53,3 +53,3 @@ if (!prompt || prompt.length === 0) { | ||
| } | ||
| const chunks = splitIntoMany(prompt); | ||
| const chunks = splitIntoMany(prompt, maxTokens); | ||
| return chunks; | ||
@@ -56,0 +56,0 @@ } |
+2
-2
| { | ||
| "name": "embeddings-splitter", | ||
| "version": "0.0.4", | ||
| "description": "A typescript library to split your long texts into smaller chunks to send them to OpenAI Embeddings", | ||
| "version": "0.0.5", | ||
| "description": "A typescript library to split your long texts into smaller chunks to send them to OpenAI Embeddings API", | ||
| "main": "lib/index.js", | ||
@@ -6,0 +6,0 @@ "types": "lib/index.d.ts", |
+4
-2
| <p align="center">embeddings-splitter</p> | ||
| <p align="center">A typescript library to split texts into chunks so they can be embedded with OpenAI Embeddings</p> | ||
| <p align="center">A typescript library to split texts into chunks so they can be embedded with OpenAI Embeddings API</p> | ||
| <a href="https://github.com/hebertcisco/ts-npm-package-boilerplate/issues/new/choose">Request Feature</a> | ||
@@ -13,4 +13,6 @@ | ||
| ### Split files | ||
| ### Split long strings | ||
| `split` makes sure your string will are short enough to be embedded | ||
| ```js | ||
@@ -17,0 +19,0 @@ import { split } from 'embeddings-splitter'; |
Long strings
Supply chain riskContains long string literals, which may be a sign of obfuscated or packed code.
Found 1 instance in 1 package
Long strings
Supply chain riskContains long string literals, which may be a sign of obfuscated or packed code.
Found 1 instance in 1 package
193520
0.07%70
2.94%