embeddings-splitter - npm Package Compare versions

Comparing version 0.1.0 to 0.2.0

lib/helpers/getChunksByMaxToken.d.ts

 import { TiktokenEmbedding } from '@dqbd/tiktoken';
-export declare function getChunksByMaxToken(text: string, max_tokens?: number, encoding_name?: TiktokenEmbedding): Promise<any[]>;
+export declare function getChunksByMaxToken(text: string, callback: (chunk: string) => void, { maxTokens, encoding_name }: {
+    maxTokens?: number;
+    encoding_name?: TiktokenEmbedding;
+}): Promise<void>;

lib/helpers/getChunksByMaxToken.js

@@ -36,6 +36,5 @@ "use strict";
 }
-function getChunksByMaxToken(text, max_tokens = EMBEDDING_CTX_LENGTH, encoding_name = EMBEDDING_ENCODING) {
+function getChunksByMaxToken(text, callback, { maxTokens = EMBEDDING_CTX_LENGTH, encoding_name = EMBEDDING_ENCODING }) {
     return __awaiter(this, void 0, void 0, function* () {
-        const chunks = [];
-        for (const chunk of chunked_tokens(text, encoding_name, max_tokens)) {
+        for (const chunk of chunked_tokens(text, encoding_name, maxTokens)) {
             const enc = (0, tiktoken_1.get_encoding)(encoding_name);
@@ -45,3 +44,3 @@ // eslint-disable-next-line @typescript-eslint/ban-ts-comment
             const _chunk = new TextDecoder().decode(enc.decode(chunk));
-            chunks.push(_chunk);
+            callback(_chunk);
         }
@@ -56,5 +55,4 @@ // removing for now but would be cool to add it as a seperate function
         //   }
-        return chunks;
     });
 }
 exports.getChunksByMaxToken = getChunksByMaxToken;

package.json

 {
   "name": "embeddings-splitter",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "A typescript library to split your long texts into smaller chunks to send them to OpenAI Embeddings API",
@@ -5,0 +5,0 @@ "main": "lib/index.js",

New alerts

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Found 1 instance in 1 package

Fixed alerts

Network access

Supply chain risk

This module accesses the network.

Found 1 instance in 1 package

Long strings

Supply chain risk

Contains long string literals, which may be a sign of obfuscated or packed code.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

221906

0.02%

Lines of code

1942

0.05%

No dependency changes