gpt-tokenizer - npm Package Compare versions

Comparing version 1.0.2 to 1.0.3

cjs/encoder.d.ts

@@ -0,1 +1,2 @@
+export declare function encodeGenerator(text: string, cache?: Map<string, string>): Generator<number[], void, undefined>;
 /**
@@ -2,0 +3,0 @@ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens

cjs/encoder.js

@@ -8,3 +8,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.decode = exports.decodeAsyncGenerator = exports.decodeGenerator = exports.endsWithIncompleteUtfPairSurrogate = exports.decodeToken = exports.encode = exports.isWithinTokenLimit = void 0;
+exports.decode = exports.decodeAsyncGenerator = exports.decodeGenerator = exports.endsWithIncompleteUtfPairSurrogate = exports.decodeToken = exports.encode = exports.isWithinTokenLimit = exports.encodeGenerator = void 0;
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment, @typescript-eslint/prefer-ts-expect-error
@@ -110,2 +110,3 @@ // @ts-ignore
 }
+exports.encodeGenerator = encodeGenerator;
 /**
@@ -112,0 +113,0 @@ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens

esm/encoder.d.ts

@@ -0,1 +1,2 @@
+export declare function encodeGenerator(text: string, cache?: Map<string, string>): Generator<number[], void, undefined>;
 /**
@@ -2,0 +3,0 @@ * @returns {false | number} false if token limit is exceeded, otherwise the number of tokens

esm/encoder.js

@@ -92,3 +92,3 @@ /* eslint-disable unicorn/prefer-spread */
 }
-function* encodeGenerator(text, cache = new Map()) {
+export function* encodeGenerator(text, cache = new Map()) {
     for (let [token] of text.matchAll(pat)) {
@@ -95,0 +95,0 @@ token = [...textEncoder.encode(token)]

package.json

 {
   "name": "gpt-tokenizer",
-  "version": "1.0.2",
+  "version": "1.0.3",
   "description": "BPE Encoder Decoder for GPT-2 / GPT-3",
@@ -5,0 +5,0 @@ "keywords": [

src/encoder.ts

@@ -129,3 +129,3 @@ /* eslint-disable unicorn/prefer-spread */
 
-function* encodeGenerator(
+export function* encodeGenerator(
   text: string,
@@ -132,0 +132,0 @@ cache: Map<string, string> = new Map(),

No alert changes

Improved metrics

Total package byte prevSize

2423210

increased by0.01%

Lines of code

1068

increased by0.28%

No dependency changes