tiktoken - npm Package Compare versions

Comparing version 1.0.14 to 1.0.15

model_to_encoding.json

@@ -55,3 +55,5 @@ {
     "gpt-4-0125-preview": "cl100k_base",
-    "gpt-4-vision-preview": "cl100k_base"
+    "gpt-4-vision-preview": "cl100k_base",
+    "gpt-4o": "o200k_base",
+    "gpt-4o-2024-05-13": "o200k_base"
 }

package.json

 {
   "name": "tiktoken",
-  "version": "1.0.14",
+  "version": "1.0.15",
   "description": "JS/WASM bindings for tiktoken",
@@ -109,4 +109,11 @@ "license": "MIT",
       "default": "./encoders/cl100k_base.js"
+    },
+    "./encoders/o200k_base.json": "./encoders/o200k_base.json",
+    "./encoders/o200k_base": {
+      "types": "./encoders/o200k_base.d.ts",
+      "edge-light": "./encoders/o200k_base.js",
+      "node": "./encoders/o200k_base.cjs",
+      "default": "./encoders/o200k_base.js"
     }
   }
 }

registry.json

@@ -49,3 +49,11 @@ {
         "pat_str": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+    },
+    "o200k_base": {
+      "load_tiktoken_bpe": "https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken",
+      "special_tokens": {
+        "<|endoftext|>": 199999,
+        "<|endofprompt|>": 200018
+      },
+      "pat_str": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
     }
-}
+}

tiktoken.d.ts

 /* tslint:disable */
 /* eslint-disable */
 
-export type TiktokenEncoding = "gpt2" | "r50k_base" | "p50k_base" | "p50k_edit" | "cl100k_base"; 
+export type TiktokenEncoding = "gpt2" | "r50k_base" | "p50k_base" | "p50k_edit" | "cl100k_base" | "o200k_base";
 
@@ -70,2 +70,4 @@ /**
     | "gpt-4-vision-preview"
+    | "gpt-4o"
+    | "gpt-4o-2024-05-13"
 
@@ -72,0 +74,0 @@ /**

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

23627026

increased by64.95%

Number of package files

57

increased by7.55%

Lines of code

2721

increased by0.7%

No dependency changes