@dqbd/tiktoken - npm Package Compare versions

Comparing version 1.0.13 to 1.0.14

lite/tiktoken_bg.js

@@ -87,2 +87,3 @@ let wasm;
         offset += ret.written;
+        ptr = realloc(ptr, len, offset, 1) >>> 0;
     }
@@ -168,3 +169,5 @@
 
-const TiktokenFinalization = new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0));
+const TiktokenFinalization = (typeof FinalizationRegistry === 'undefined')
+    ? { register: () => {}, unregister: () => {} }
+    : new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0));
 /**
@@ -355,3 +358,3 @@ */
 
-export function __wbg_stringify_daa6661e90c04140() { return handleError(function (arg0) {
+export function __wbg_stringify_8887fe74e1c50d81() { return handleError(function (arg0) {
     const ret = JSON.stringify(getObject(arg0));
@@ -380,3 +383,3 @@ return addHeapObject(ret);
 
-export function __wbg_parse_06816e879d29d4df() { return handleError(function (arg0, arg1) {
+export function __wbg_parse_66d1801634e099ac() { return handleError(function (arg0, arg1) {
     const ret = JSON.parse(getStringFromWasm0(arg0, arg1));
@@ -383,0 +386,0 @@ return addHeapObject(ret);

model_to_encoding.json

@@ -50,2 +50,4 @@ {
     "gpt-4-32k-0613": "cl100k_base",
+    "gpt-4-turbo": "cl100k_base",
+    "gpt-4-turbo-2024-04-09": "cl100k_base",
     "gpt-4-turbo-preview": "cl100k_base",
@@ -52,0 +54,0 @@ "gpt-4-1106-preview": "cl100k_base",

package.json

 {
   "name": "@dqbd/tiktoken",
-  "version": "1.0.13",
+  "version": "1.0.14",
   "description": "JS/WASM bindings for tiktoken",
@@ -5,0 +5,0 @@ "license": "MIT",

README.md

@@ -315,4 +315,12 @@ # ⏳ tiktoken
 
+## Development
+
+To build the `tiktoken` library, make sure to have:
+- Rust and [`wasm-pack`](https://github.com/rustwasm/wasm-pack) installed.
+- Node.js 18+ is required to build the JS bindings and fetch the latest encoder ranks via `fetch`.
+
+Install all the dev-dependencies with `yarn install` and build both WASM binary and JS bindings with `yarn build`.
+
 ## Acknowledgements
 
 - https://github.com/zurawiki/tiktoken-rs

tiktoken_bg.js

@@ -13,16 +13,2 @@ let wasm;
 
-let heap_next = heap.length;
-
-function dropObject(idx) {
-    if (idx < 132) return;
-    heap[idx] = heap_next;
-    heap_next = idx;
-}
-
-function takeObject(idx) {
-    const ret = getObject(idx);
-    dropObject(idx);
-    return ret;
-}
-
 let WASM_VECTOR_LEN = 0;
@@ -88,2 +74,3 @@
         offset += ret.written;
+        ptr = realloc(ptr, len, offset, 1) >>> 0;
     }
@@ -108,2 +95,16 @@
 
+let heap_next = heap.length;
+
+function dropObject(idx) {
+    if (idx < 132) return;
+    heap[idx] = heap_next;
+    heap_next = idx;
+}
+
+function takeObject(idx) {
+    const ret = getObject(idx);
+    dropObject(idx);
+    return ret;
+}
+
 const lTextDecoder = typeof TextDecoder === 'undefined' ? (0, module.require)('util').TextDecoder : TextDecoder;
@@ -129,2 +130,10 @@
 
+function handleError(f, args) {
+    try {
+        return f.apply(this, args);
+    } catch (e) {
+        wasm.__wbindgen_export_2(addHeapObject(e));
+    }
+}
+
 let cachedUint32Memory0 = null;
@@ -210,11 +219,5 @@
 
-function handleError(f, args) {
-    try {
-        return f.apply(this, args);
-    } catch (e) {
-        wasm.__wbindgen_export_3(addHeapObject(e));
-    }
-}
-
-const TiktokenFinalization = new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0));
+const TiktokenFinalization = (typeof FinalizationRegistry === 'undefined')
+    ? { register: () => {}, unregister: () => {} }
+    : new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0));
 /**
@@ -271,3 +274,3 @@ */
                 v1 = getStringFromWasm0(r0, r1).slice();
-                wasm.__wbindgen_export_2(r0, r1 * 1, 1);
+                wasm.__wbindgen_export_3(r0, r1 * 1, 1);
             }
@@ -300,3 +303,3 @@ return v1;
             var v2 = getArrayU32FromWasm0(r0, r1).slice();
-            wasm.__wbindgen_export_2(r0, r1 * 4, 4);
+            wasm.__wbindgen_export_3(r0, r1 * 4, 4);
             return v2;
@@ -321,3 +324,3 @@ } finally {
             var v2 = getArrayU32FromWasm0(r0, r1).slice();
-            wasm.__wbindgen_export_2(r0, r1 * 4, 4);
+            wasm.__wbindgen_export_3(r0, r1 * 4, 4);
             return v2;
@@ -377,3 +380,3 @@ } finally {
             var v2 = getArrayU8FromWasm0(r0, r1).slice();
-            wasm.__wbindgen_export_2(r0, r1 * 1, 1);
+            wasm.__wbindgen_export_3(r0, r1 * 1, 1);
             return v2;
@@ -396,3 +399,3 @@ } finally {
             var v1 = getArrayU8FromWasm0(r0, r1).slice();
-            wasm.__wbindgen_export_2(r0, r1 * 1, 1);
+            wasm.__wbindgen_export_3(r0, r1 * 1, 1);
             return v1;
@@ -413,2 +416,12 @@ } finally {
 
+export function __wbindgen_string_get(arg0, arg1) {
+    if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
+    const obj = getObject(arg1);
+    const ret = typeof(obj) === 'string' ? obj : undefined;
+    var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1);
+    var len1 = WASM_VECTOR_LEN;
+    getInt32Memory0()[arg0 / 4 + 1] = len1;
+    getInt32Memory0()[arg0 / 4 + 0] = ptr1;
+};
+
 export function __wbindgen_object_drop_ref(arg0) {
@@ -423,3 +436,3 @@ takeObject(arg0);
 
-export function __wbg_stringify_daa6661e90c04140() { return handleError(function (arg0) {
+export function __wbg_stringify_8887fe74e1c50d81() { return handleError(function (arg0) {
     const ret = JSON.stringify(getObject(arg0));
@@ -429,12 +442,2 @@ return addHeapObject(ret);
 
-export function __wbindgen_string_get(arg0, arg1) {
-    if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
-    const obj = getObject(arg1);
-    const ret = typeof(obj) === 'string' ? obj : undefined;
-    var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1);
-    var len1 = WASM_VECTOR_LEN;
-    getInt32Memory0()[arg0 / 4 + 1] = len1;
-    getInt32Memory0()[arg0 / 4 + 0] = ptr1;
-};
-
 export function __wbindgen_error_new(arg0, arg1) {
@@ -445,3 +448,3 @@ const ret = new Error(getStringFromWasm0(arg0, arg1));
 
-export function __wbg_parse_06816e879d29d4df() { return handleError(function (arg0, arg1) {
+export function __wbg_parse_66d1801634e099ac() { return handleError(function (arg0, arg1) {
     const ret = JSON.parse(getStringFromWasm0(arg0, arg1));
@@ -448,0 +451,0 @@ return addHeapObject(ret);

tiktoken_bg.wasm.d.ts

@@ -18,4 +18,4 @@ /* tslint:disable */
 export function __wbindgen_export_1(a: number, b: number, c: number, d: number): number;
+export function __wbindgen_export_2(a: number): void;
 export function __wbindgen_add_to_stack_pointer(a: number): number;
-export function __wbindgen_export_2(a: number, b: number, c: number): void;
-export function __wbindgen_export_3(a: number): void;
+export function __wbindgen_export_3(a: number, b: number, c: number): void;

tiktoken.d.ts

@@ -64,2 +64,4 @@ /* tslint:disable */
     | "gpt-4-32k-0613"
+    | "gpt-4-turbo"
+    | "gpt-4-turbo-2024-04-09"
     | "gpt-4-turbo-preview"
@@ -66,0 +68,0 @@ | "gpt-4-1106-preview"

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Native code

Supply chain risk

Contains native code (e.g., compiled binaries or shared libraries). Including native code can obscure malicious behavior.

Found 2 instances in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

14323542

increased by0.03%

Lines of code

2702

increased by0.6%

Number of lines in readme file

326

increased by2.52%

Worsened metrics

Number of medium supply chain risk alerts

6

increased by50%

No dependency changes