@lenml/tokenizers - npm Package Compare versions

Comparing version 1.0.9 to 1.1.1

src/prepatch.ts

+import { AutoTokenizer, PreTrainedTokenizer } from "./tokenizers/tokenizers";
+
+// NOTE: Set this to turn off the warning of claude tokenizer
+// @ts-ignore
+AutoTokenizer.TOKENIZER_CLASS_MAPPING.ClaudeTokenizer = PreTrainedTokenizer;

package.json

 {
   "name": "@lenml/tokenizers",
-  "version": "1.0.9",
+  "version": "1.1.1",
   "description": "a lightweight no-dependency fork of transformers.js (only tokenizers)",
@@ -5,0 +5,0 @@ "source": "src/main.ts",

src/main.ts

@@ -0,3 +1,4 @@
+import "./prepatch";
 export * from "./TokenizerLoader";
 export * as tokenizers from "./tokenizers/tokenizers";
 export * from "./types";

src/tokenizers/utils/tensor.js

@@ -53,2 +53,6 @@ /**
   constructor(...args) {
+    throw new Error(
+      `You seem to be creating a tensor object. Unfortunately, the '@lenml/tokenizers' library does not support returning any ONNX-related object instances. Please try setting 'return_tensor=false' to avoid this error`
+    );
+
     if (args[0] instanceof ONNXTensor) {
@@ -55,0 +59,0 @@ // Create shallow copy

No alert changes

Improved metrics

Total package byte prevSize

1906909

0.78%

Number of package files

21

5%

Lines of code

9519

0.87%

No dependency changes