tiktoken - npm Package Compare versions

Comparing version 1.0.15 to 1.0.16

lite/tiktoken_bg.js

@@ -29,9 +29,9 @@ let wasm;
 
-let cachedUint8Memory0 = null;
+let cachedUint8ArrayMemory0 = null;
 
-function getUint8Memory0() {
-    if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) {
-        cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer);
+function getUint8ArrayMemory0() {
+    if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
+        cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
     }
-    return cachedUint8Memory0;
+    return cachedUint8ArrayMemory0;
 }
@@ -61,3 +61,3 @@
         const ptr = malloc(buf.length, 1) >>> 0;
-        getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
+        getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
         WASM_VECTOR_LEN = buf.length;
@@ -70,3 +70,3 @@ return ptr;
 
-    const mem = getUint8Memory0();
+    const mem = getUint8ArrayMemory0();
 
@@ -86,3 +86,3 @@ let offset = 0;
         ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
-        const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
+        const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
         const ret = encodeString(arg, view);
@@ -102,9 +102,9 @@
 
-let cachedInt32Memory0 = null;
+let cachedDataViewMemory0 = null;
 
-function getInt32Memory0() {
-    if (cachedInt32Memory0 === null || cachedInt32Memory0.byteLength === 0) {
-        cachedInt32Memory0 = new Int32Array(wasm.memory.buffer);
+function getDataViewMemory0() {
+    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
+        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
     }
-    return cachedInt32Memory0;
+    return cachedDataViewMemory0;
 }
@@ -120,3 +120,3 @@
     ptr = ptr >>> 0;
-    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+    return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
 }
@@ -133,9 +133,9 @@
 
-let cachedUint32Memory0 = null;
+let cachedUint32ArrayMemory0 = null;
 
-function getUint32Memory0() {
-    if (cachedUint32Memory0 === null || cachedUint32Memory0.byteLength === 0) {
-        cachedUint32Memory0 = new Uint32Array(wasm.memory.buffer);
+function getUint32ArrayMemory0() {
+    if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
+        cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
     }
-    return cachedUint32Memory0;
+    return cachedUint32ArrayMemory0;
 }
@@ -145,3 +145,3 @@
     ptr = ptr >>> 0;
-    return getUint32Memory0().subarray(ptr / 4, ptr / 4 + len);
+    return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
 }
@@ -151,3 +151,3 @@
     const ptr = malloc(arg.length * 1, 1) >>> 0;
-    getUint8Memory0().set(arg, ptr / 1);
+    getUint8ArrayMemory0().set(arg, ptr / 1);
     WASM_VECTOR_LEN = arg.length;
@@ -159,3 +159,3 @@ return ptr;
     const ptr = malloc(arg.length * 4, 4) >>> 0;
-    getUint32Memory0().set(arg, ptr / 4);
+    getUint32ArrayMemory0().set(arg, ptr / 4);
     WASM_VECTOR_LEN = arg.length;
@@ -167,3 +167,3 @@ return ptr;
     ptr = ptr >>> 0;
-    return getUint8Memory0().subarray(ptr / 1, ptr / 1 + len);
+    return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
 }
@@ -181,3 +181,3 @@
     ? { register: () => {}, unregister: () => {} }
-    : new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0));
+    : new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0, 1));
 /**
@@ -197,3 +197,3 @@ */
         const ptr = this.__destroy_into_raw();
-        wasm.__wbg_tiktoken_free(ptr);
+        wasm.__wbg_tiktoken_free(ptr, 0);
     }
@@ -213,2 +213,3 @@ /**
         this.__wbg_ptr = ret >>> 0;
+        TiktokenFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -223,4 +224,4 @@ }
             wasm.tiktoken_name(retptr, this.__wbg_ptr);
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
             let v1;
@@ -249,6 +250,6 @@ if (r0 !== 0) {
             wasm.tiktoken_encode(retptr, this.__wbg_ptr, ptr0, len0, addHeapObject(allowed_special), addHeapObject(disallowed_special));
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
-            var r2 = getInt32Memory0()[retptr / 4 + 2];
-            var r3 = getInt32Memory0()[retptr / 4 + 3];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+            var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
+            var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
             if (r3) {
@@ -275,4 +276,4 @@ throw takeObject(r2);
             wasm.tiktoken_encode_ordinary(retptr, this.__wbg_ptr, ptr0, len0);
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
             var v2 = getArrayU32FromWasm0(r0, r1).slice();
@@ -298,5 +299,5 @@ wasm.__wbindgen_export_2(r0, r1 * 4, 4);
             wasm.tiktoken_encode_with_unstable(retptr, this.__wbg_ptr, ptr0, len0, addHeapObject(allowed_special), addHeapObject(disallowed_special));
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
-            var r2 = getInt32Memory0()[retptr / 4 + 2];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+            var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
             if (r2) {
@@ -332,4 +333,4 @@ throw takeObject(r1);
             wasm.tiktoken_decode(retptr, this.__wbg_ptr, ptr0, len0);
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
             var v2 = getArrayU8FromWasm0(r0, r1).slice();
@@ -351,4 +352,4 @@ wasm.__wbindgen_export_2(r0, r1 * 1, 1);
             wasm.tiktoken_decode_single_token_bytes(retptr, this.__wbg_ptr, token);
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
             var v1 = getArrayU8FromWasm0(r0, r1).slice();
@@ -376,7 +377,2 @@ wasm.__wbindgen_export_2(r0, r1 * 1, 1);
 
-export function __wbg_stringify_8887fe74e1c50d81() { return handleError(function (arg0) {
-    const ret = JSON.stringify(getObject(arg0));
-    return addHeapObject(ret);
-}, arguments) };
-
 export function __wbindgen_object_drop_ref(arg0) {
@@ -392,4 +388,4 @@ takeObject(arg0);
     var len1 = WASM_VECTOR_LEN;
-    getInt32Memory0()[arg0 / 4 + 1] = len1;
-    getInt32Memory0()[arg0 / 4 + 0] = ptr1;
+    getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+    getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
 };
@@ -402,3 +398,3 @@
 
-export function __wbg_parse_66d1801634e099ac() { return handleError(function (arg0, arg1) {
+export function __wbg_parse_52202f117ec9ecfa() { return handleError(function (arg0, arg1) {
     const ret = JSON.parse(getStringFromWasm0(arg0, arg1));
@@ -408,2 +404,7 @@ return addHeapObject(ret);
 
+export function __wbg_stringify_bbf45426c92a6bf5() { return handleError(function (arg0) {
+    const ret = JSON.stringify(getObject(arg0));
+    return addHeapObject(ret);
+}, arguments) };
+
 export function __wbindgen_throw(arg0, arg1) {
@@ -410,0 +411,0 @@ throw new Error(getStringFromWasm0(arg0, arg1));

lite/tiktoken_bg.wasm.d.ts

 /* tslint:disable */
 /* eslint-disable */
 export const memory: WebAssembly.Memory;
-export function __wbg_tiktoken_free(a: number): void;
+export function __wbg_tiktoken_free(a: number, b: number): void;
 export function tiktoken_new(a: number, b: number, c: number, d: number, e: number): number;
@@ -6,0 +6,0 @@ export function tiktoken_name(a: number, b: number): void;

lite/tiktoken.js

@@ -0,1 +1,2 @@
+
 import * as wasm from "./tiktoken_bg.wasm";
@@ -2,0 +3,0 @@ import { __wbg_set_wasm } from "./tiktoken_bg.js";

model_to_encoding.json

@@ -23,2 +23,4 @@ {
     "text-embedding-ada-002": "cl100k_base",
+    "text-embedding-3-small": "cl100k_base",
+    "text-embedding-3-large": "cl100k_base",
     "text-similarity-davinci-001": "r50k_base",
@@ -58,3 +60,6 @@ "text-similarity-curie-001": "r50k_base",
     "gpt-4o": "o200k_base",
-    "gpt-4o-2024-05-13": "o200k_base"
+    "gpt-4o-2024-05-13": "o200k_base",
+    "gpt-4o-2024-08-06":"o200k_base",
+    "gpt-4o-mini-2024-07-18": "o200k_base",
+    "gpt-4o-mini": "o200k_base"
 }

package.json

 {
   "name": "tiktoken",
-  "version": "1.0.15",
+  "version": "1.0.16",
   "description": "JS/WASM bindings for tiktoken",
@@ -5,0 +5,0 @@ "license": "MIT",

tiktoken_bg.js

@@ -13,11 +13,25 @@ let wasm;
 
+let heap_next = heap.length;
+
+function dropObject(idx) {
+    if (idx < 132) return;
+    heap[idx] = heap_next;
+    heap_next = idx;
+}
+
+function takeObject(idx) {
+    const ret = getObject(idx);
+    dropObject(idx);
+    return ret;
+}
+
 let WASM_VECTOR_LEN = 0;
 
-let cachedUint8Memory0 = null;
+let cachedUint8ArrayMemory0 = null;
 
-function getUint8Memory0() {
-    if (cachedUint8Memory0 === null || cachedUint8Memory0.byteLength === 0) {
-        cachedUint8Memory0 = new Uint8Array(wasm.memory.buffer);
+function getUint8ArrayMemory0() {
+    if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) {
+        cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer);
     }
-    return cachedUint8Memory0;
+    return cachedUint8ArrayMemory0;
 }
@@ -47,3 +61,3 @@
         const ptr = malloc(buf.length, 1) >>> 0;
-        getUint8Memory0().subarray(ptr, ptr + buf.length).set(buf);
+        getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf);
         WASM_VECTOR_LEN = buf.length;
@@ -56,3 +70,3 @@ return ptr;
 
-    const mem = getUint8Memory0();
+    const mem = getUint8ArrayMemory0();
 
@@ -72,3 +86,3 @@ let offset = 0;
         ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0;
-        const view = getUint8Memory0().subarray(ptr + offset, ptr + len);
+        const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len);
         const ret = encodeString(arg, view);
@@ -88,25 +102,11 @@
 
-let cachedInt32Memory0 = null;
+let cachedDataViewMemory0 = null;
 
-function getInt32Memory0() {
-    if (cachedInt32Memory0 === null || cachedInt32Memory0.byteLength === 0) {
-        cachedInt32Memory0 = new Int32Array(wasm.memory.buffer);
+function getDataViewMemory0() {
+    if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) {
+        cachedDataViewMemory0 = new DataView(wasm.memory.buffer);
     }
-    return cachedInt32Memory0;
+    return cachedDataViewMemory0;
 }
 
-let heap_next = heap.length;
-
-function dropObject(idx) {
-    if (idx < 132) return;
-    heap[idx] = heap_next;
-    heap_next = idx;
-}
-
-function takeObject(idx) {
-    const ret = getObject(idx);
-    dropObject(idx);
-    return ret;
-}
-
 const lTextDecoder = typeof TextDecoder === 'undefined' ? (0, module.require)('util').TextDecoder : TextDecoder;
@@ -120,3 +120,3 @@
     ptr = ptr >>> 0;
-    return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
+    return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len));
 }
@@ -133,17 +133,9 @@
 
-function handleError(f, args) {
-    try {
-        return f.apply(this, args);
-    } catch (e) {
-        wasm.__wbindgen_export_2(addHeapObject(e));
-    }
-}
+let cachedUint32ArrayMemory0 = null;
 
-let cachedUint32Memory0 = null;
-
-function getUint32Memory0() {
-    if (cachedUint32Memory0 === null || cachedUint32Memory0.byteLength === 0) {
-        cachedUint32Memory0 = new Uint32Array(wasm.memory.buffer);
+function getUint32ArrayMemory0() {
+    if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) {
+        cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer);
     }
-    return cachedUint32Memory0;
+    return cachedUint32ArrayMemory0;
 }
@@ -153,3 +145,3 @@
     ptr = ptr >>> 0;
-    return getUint32Memory0().subarray(ptr / 4, ptr / 4 + len);
+    return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len);
 }
@@ -159,3 +151,3 @@
     const ptr = malloc(arg.length * 1, 1) >>> 0;
-    getUint8Memory0().set(arg, ptr / 1);
+    getUint8ArrayMemory0().set(arg, ptr / 1);
     WASM_VECTOR_LEN = arg.length;
@@ -167,3 +159,3 @@ return ptr;
     const ptr = malloc(arg.length * 4, 4) >>> 0;
-    getUint32Memory0().set(arg, ptr / 4);
+    getUint32ArrayMemory0().set(arg, ptr / 4);
     WASM_VECTOR_LEN = arg.length;
@@ -175,3 +167,3 @@ return ptr;
     ptr = ptr >>> 0;
-    return getUint8Memory0().subarray(ptr / 1, ptr / 1 + len);
+    return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len);
 }
@@ -190,5 +182,5 @@ /**
         wasm.get_encoding(retptr, ptr0, len0, addHeapObject(extend_special_tokens));
-        var r0 = getInt32Memory0()[retptr / 4 + 0];
-        var r1 = getInt32Memory0()[retptr / 4 + 1];
-        var r2 = getInt32Memory0()[retptr / 4 + 2];
+        var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+        var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+        var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
         if (r2) {
@@ -215,5 +207,5 @@ throw takeObject(r1);
         wasm.encoding_for_model(retptr, ptr0, len0, addHeapObject(extend_special_tokens));
-        var r0 = getInt32Memory0()[retptr / 4 + 0];
-        var r1 = getInt32Memory0()[retptr / 4 + 1];
-        var r2 = getInt32Memory0()[retptr / 4 + 2];
+        var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+        var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+        var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
         if (r2) {
@@ -228,5 +220,13 @@ throw takeObject(r1);
 
+function handleError(f, args) {
+    try {
+        return f.apply(this, args);
+    } catch (e) {
+        wasm.__wbindgen_export_3(addHeapObject(e));
+    }
+}
+
 const TiktokenFinalization = (typeof FinalizationRegistry === 'undefined')
     ? { register: () => {}, unregister: () => {} }
-    : new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0));
+    : new FinalizationRegistry(ptr => wasm.__wbg_tiktoken_free(ptr >>> 0, 1));
 /**
@@ -254,3 +254,3 @@ */
         const ptr = this.__destroy_into_raw();
-        wasm.__wbg_tiktoken_free(ptr);
+        wasm.__wbg_tiktoken_free(ptr, 0);
     }
@@ -270,2 +270,3 @@ /**
         this.__wbg_ptr = ret >>> 0;
+        TiktokenFinalization.register(this, this.__wbg_ptr, this);
         return this;
@@ -280,8 +281,8 @@ }
             wasm.tiktoken_name(retptr, this.__wbg_ptr);
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
             let v1;
             if (r0 !== 0) {
                 v1 = getStringFromWasm0(r0, r1).slice();
-                wasm.__wbindgen_export_3(r0, r1 * 1, 1);
+                wasm.__wbindgen_export_2(r0, r1 * 1, 1);
             }
@@ -306,6 +307,6 @@ return v1;
             wasm.tiktoken_encode(retptr, this.__wbg_ptr, ptr0, len0, addHeapObject(allowed_special), addHeapObject(disallowed_special));
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
-            var r2 = getInt32Memory0()[retptr / 4 + 2];
-            var r3 = getInt32Memory0()[retptr / 4 + 3];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+            var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
+            var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true);
             if (r3) {
@@ -315,3 +316,3 @@ throw takeObject(r2);
             var v2 = getArrayU32FromWasm0(r0, r1).slice();
-            wasm.__wbindgen_export_3(r0, r1 * 4, 4);
+            wasm.__wbindgen_export_2(r0, r1 * 4, 4);
             return v2;
@@ -333,6 +334,6 @@ } finally {
             wasm.tiktoken_encode_ordinary(retptr, this.__wbg_ptr, ptr0, len0);
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
             var v2 = getArrayU32FromWasm0(r0, r1).slice();
-            wasm.__wbindgen_export_3(r0, r1 * 4, 4);
+            wasm.__wbindgen_export_2(r0, r1 * 4, 4);
             return v2;
@@ -356,5 +357,5 @@ } finally {
             wasm.tiktoken_encode_with_unstable(retptr, this.__wbg_ptr, ptr0, len0, addHeapObject(allowed_special), addHeapObject(disallowed_special));
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
-            var r2 = getInt32Memory0()[retptr / 4 + 2];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
+            var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true);
             if (r2) {
@@ -390,6 +391,6 @@ throw takeObject(r1);
             wasm.tiktoken_decode(retptr, this.__wbg_ptr, ptr0, len0);
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
             var v2 = getArrayU8FromWasm0(r0, r1).slice();
-            wasm.__wbindgen_export_3(r0, r1 * 1, 1);
+            wasm.__wbindgen_export_2(r0, r1 * 1, 1);
             return v2;
@@ -409,6 +410,6 @@ } finally {
             wasm.tiktoken_decode_single_token_bytes(retptr, this.__wbg_ptr, token);
-            var r0 = getInt32Memory0()[retptr / 4 + 0];
-            var r1 = getInt32Memory0()[retptr / 4 + 1];
+            var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true);
+            var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true);
             var v1 = getArrayU8FromWasm0(r0, r1).slice();
-            wasm.__wbindgen_export_3(r0, r1 * 1, 1);
+            wasm.__wbindgen_export_2(r0, r1 * 1, 1);
             return v1;
@@ -429,12 +430,2 @@ } finally {
 
-export function __wbindgen_string_get(arg0, arg1) {
-    if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
-    const obj = getObject(arg1);
-    const ret = typeof(obj) === 'string' ? obj : undefined;
-    var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1);
-    var len1 = WASM_VECTOR_LEN;
-    getInt32Memory0()[arg0 / 4 + 1] = len1;
-    getInt32Memory0()[arg0 / 4 + 0] = ptr1;
-};
-
 export function __wbindgen_object_drop_ref(arg0) {
@@ -449,6 +440,11 @@ takeObject(arg0);
 
-export function __wbg_stringify_8887fe74e1c50d81() { return handleError(function (arg0) {
-    const ret = JSON.stringify(getObject(arg0));
-    return addHeapObject(ret);
-}, arguments) };
+export function __wbindgen_string_get(arg0, arg1) {
+    if (wasm == null) throw new Error("tiktoken: WASM binary has not been propery initialized.");
+    const obj = getObject(arg1);
+    const ret = typeof(obj) === 'string' ? obj : undefined;
+    var ptr1 = isLikeNone(ret) ? 0 : passStringToWasm0(ret, wasm.__wbindgen_export_0, wasm.__wbindgen_export_1);
+    var len1 = WASM_VECTOR_LEN;
+    getDataViewMemory0().setInt32(arg0 + 4 * 1, len1, true);
+    getDataViewMemory0().setInt32(arg0 + 4 * 0, ptr1, true);
+};
 
@@ -460,3 +456,3 @@ export function __wbindgen_error_new(arg0, arg1) {
 
-export function __wbg_parse_66d1801634e099ac() { return handleError(function (arg0, arg1) {
+export function __wbg_parse_52202f117ec9ecfa() { return handleError(function (arg0, arg1) {
     const ret = JSON.parse(getStringFromWasm0(arg0, arg1));
@@ -466,2 +462,7 @@ return addHeapObject(ret);
 
+export function __wbg_stringify_bbf45426c92a6bf5() { return handleError(function (arg0) {
+    const ret = JSON.stringify(getObject(arg0));
+    return addHeapObject(ret);
+}, arguments) };
+
 export function __wbindgen_throw(arg0, arg1) {
@@ -468,0 +469,0 @@ throw new Error(getStringFromWasm0(arg0, arg1));

tiktoken_bg.wasm.d.ts

 /* tslint:disable */
 /* eslint-disable */
 export const memory: WebAssembly.Memory;
-export function __wbg_tiktoken_free(a: number): void;
+export function __wbg_tiktoken_free(a: number, b: number): void;
 export function tiktoken_new(a: number, b: number, c: number, d: number, e: number): number;
@@ -18,4 +18,4 @@ export function tiktoken_name(a: number, b: number): void;
 export function __wbindgen_export_1(a: number, b: number, c: number, d: number): number;
-export function __wbindgen_export_2(a: number): void;
 export function __wbindgen_add_to_stack_pointer(a: number): number;
-export function __wbindgen_export_3(a: number, b: number, c: number): void;
+export function __wbindgen_export_2(a: number, b: number, c: number): void;
+export function __wbindgen_export_3(a: number): void;

tiktoken.d.ts

@@ -72,2 +72,5 @@ /* tslint:disable */
     | "gpt-4o-2024-05-13"
+    | "gpt-4o-2024-08-06"
+    | "gpt-4o-mini-2024-07-18"
+    | "gpt-4o-mini"
 
@@ -74,0 +77,0 @@ /**

tiktoken.js

@@ -0,1 +1,2 @@
+
 import * as wasm from "./tiktoken_bg.wasm";
@@ -2,0 +3,0 @@ import { __wbg_set_wasm } from "./tiktoken_bg.js";

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Lines of code

2733

increased by0.44%

Worsened metrics

Total package byte prevSize

23493474

decreased by-0.57%

No dependency changes