js-tiktoken - npm Package Compare versions

Comparing version 1.0.3 to 1.0.4

dist/index.d.ts

@@ -11,3 +11,4 @@ type TiktokenEncoding = "gpt2" | "r50k_base" | "p50k_base" | "p50k_edit" | "cl100k_base";
     constructor(ranks: TiktokenBPE, extendedSpecialTokens?: Record<string, number>);
-    encode(text: string, allowedSpecial?: Set<string> | "all", disallowedSpecial?: Set<string> | "all"): number[];
+    private static specialTokenRegex;
+    encode(text: string, allowedSpecial?: Array<string> | "all", disallowedSpecial?: Array<string> | "all"): number[];
     decode(tokens: number[]): string;
@@ -14,0 +15,0 @@ }

package.json

 {
   "name": "js-tiktoken",
-  "version": "1.0.3",
+  "version": "1.0.4",
   "description": "JavaScript port of tiktoken",
@@ -21,2 +21,27 @@ "license": "MIT",
       "default": "./dist/index.js"
+    },
+    "./ranks/cl100k_base": {
+      "types": "./dist/ranks/cl100k_base.d.ts",
+      "node": "./dist/ranks/cl100k_base.cjs",
+      "default": "./dist/ranks/cl100k_base.js"
+    },
+    "./ranks/gpt2": {
+      "types": "./dist/ranks/gpt2.d.ts",
+      "node": "./dist/ranks/gpt2.cjs",
+      "default": "./dist/ranks/gpt2.js"
+    },
+    "./ranks/p50k_base": {
+      "types": "./dist/ranks/p50k_base.d.ts",
+      "node": "./dist/ranks/p50k_base.cjs",
+      "default": "./dist/ranks/p50k_base.js"
+    },
+    "./ranks/p50k_edit": {
+      "types": "./dist/ranks/p50k_edit.d.ts",
+      "node": "./dist/ranks/p50k_edit.cjs",
+      "default": "./dist/ranks/p50k_edit.js"
+    },
+    "./ranks/r50k_base": {
+      "types": "./dist/ranks/r50k_base.d.ts",
+      "node": "./dist/ranks/r50k_base.cjs",
+      "default": "./dist/ranks/r50k_base.js"
     }
@@ -39,3 +64,8 @@ },
     "entry": [
-      "src/index.ts"
+      "src/index.ts",
+      "src/ranks/cl100k_base.ts",
+      "src/ranks/gpt2.ts",
+      "src/ranks/p50k_base.ts",
+      "src/ranks/p50k_edit.ts",
+      "src/ranks/r50k_base.ts"
     ],
@@ -42,0 +72,0 @@ "format": [

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

9841696

increased by49.93%

Number of package files

26

increased by420%

Lines of code

35404

increased by49.47%

Worsened metrics

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes