@microsoft/tiktokenizer - npm Package Compare versions

Comparing version 1.0.4 to 1.0.5

dist/tikTokenizer.js

@@ -151,8 +151,5 @@ "use strict";
         let match;
-        const matches = Array.from(text.substring(start, end).matchAll(this.regex));
-        for (var i = 0; i < matches.length; i++) {
-            match = matches[i];
-            if (match === undefined) {
-                break;
-            }
+        const substring = text.substring(start, end);
+        this.regex.lastIndex = 0;
+        while ((match = this.regex.exec(substring))) {
             if (this.cache.has(match[0])) {
@@ -179,5 +176,5 @@ tokenIds.push(...this.cache.get(match[0]));
         let match;
-        const matches = Array.from(text.substring(start, end).matchAll(this.regex));
-        for (var i = 0; i < matches.length; i++) {
-            match = matches[i];
+        const substring = text.substring(start, end);
+        this.regex.lastIndex = 0;
+        while ((match = this.regex.exec(substring))) {
             const piece = match[0];
@@ -301,5 +298,5 @@ if (this.cache.has(piece)) {
                 let match;
-                const matches = Array.from(text.substring(start, end).matchAll(this.regex));
-                for (var i = 0; i < matches.length; i++) {
-                    match = matches[i];
+                const substring = text.substring(start, end);
+                this.regex.lastIndex = 0;
+                while ((match = this.regex.exec(substring))) {
                     const piece = match[0];
@@ -306,0 +303,0 @@ if (this.cache.has(piece)) {

package.json

@@ -5,3 +5,3 @@ {
   "description": "Tokenizer for OpenAI large language models.",
-  "version": "1.0.4",
+  "version": "1.0.5",
   "author": {
@@ -8,0 +8,0 @@ "name": "Microsoft Corporation"

No alert changes

Worsened metrics

Total package byte prevSize

59370

decreased by-0.64%

Lines of code

841

decreased by-0.36%

No dependency changes