@lenml/llama2-tokenizer - npm Package Compare versions

+54

-2

dist/main.d.ts

		@@ -59,13 +59,65 @@ /**
		constructor();
		/**
		* Install the provided vocabulary into the class instance.
		*
		* @param {Record<string, number>} vocab - The vocabulary to be installed
		*/
		install_vocab(vocab: Record<string, number>): void;
		load_llama2_vocab(): void;
		/**
		* Get the size of the vocabulary, including special tokens.
		*
		* @return {number} the size of the vocabulary
		*/
		get vocab_size(): number;
		/**
		* Get the maximum id from the vocab_ids and special_tokens.
		*
		* @return {number} the maximum id
		*/
		get max_id(): number;
		/**
		* Adds a special token with an optional token ID.
		*
		* @param {string} token - the special token to be added
		* @param {number} [token_id] - the optional token ID
		* @return {void}
		*/
		add_special_token(token: string, token_id?: number): void;
		add_special_tokens(tokens: string[]): void;
		/**
		* Adds special tokens to the list of tokens.
		*
		* @param {Array} tokens - An array of tokens to add. Each token can be a string or an object with `token` and `token_id` properties.
		*/
		add_special_tokens(tokens: (string \| {
		token: string;
		token_id: number;
		})[]): void;
		/**
		* Convert an id to a token.
		*
		* @param {number} id - The id to be converted to a token.
		* @return {string} The corresponding token for the given id.
		*/
		ids_to_token(id: number): string;
		/**
		* token_to_id function takes a token as input and returns its corresponding id if found in the vocabulary, otherwise throws an error.
		*
		* @param {string} token - the input token
		* @return {number} the corresponding id of the input token
		*/
		token_to_id(token: string): number;
		/**
		* Retrieve the vocabulary.
		*
		* @return {Object} a shallow copy of the vocabulary
		*/
		get_vocab(): {
		[x: string]: number;
		};
		/**
		* Checks if the token is a valid token.
		*
		* @param {string} token - the token to be checked
		* @return {boolean} true if the token is valid, false otherwise
		*/
		valid_token(token: string): boolean;
		@@ -72,0 +124,0 @@ /**

+12

-3

package.json

		{
		"name": "@lenml/llama2-tokenizer",
		"version": "1.0.1",
		"version": "1.1.2",
		"description": "",
		@@ -50,3 +50,11 @@ "source": "src/main.ts",
		"tsup": "^8.0.1",
		"typescript": "^5.3.3"
		"typescript": "^5.3.3",
		"@lenml/llama2-tokenizer-vocab-baichuan2": "^1.1.0",
		"@lenml/llama2-tokenizer-vocab-internlm2": "^1.1.0",
		"@lenml/llama2-tokenizer-vocab-chatglm3": "^1.1.0",
		"@lenml/llama2-tokenizer-vocab-falcon": "^1.1.0",
		"@lenml/llama2-tokenizer-vocab-llama2": "^1.1.0",
		"@lenml/llama2-tokenizer-vocab-yi": "^1.1.0",
		"@lenml/llama2-tokenizer-vocab-neox": "^1.1.0",
		"@lenml/llama2-tokenizer-vocab-emoji": "^1.1.0"
		},
		@@ -58,4 +66,5 @@ "scripts": {
		"start": "pnpm build",
		"test": "jest"
		"test": "jest",
		"test:coverage": "jest --coverage"
		}
		}

+59

-4

README.md

		@@ -7,6 +7,10 @@ # 🦙Llama2 Tokenizer for JavaScript

		> online playground: https://lenml.github.io/llama-tokenizer-playground/
		>
		> (vocab: llama2)

		# Features

		- fast
		- support llama2
		- API like Llama2Tokenizer (python)
		- typescript
		@@ -19,2 +23,8 @@ - 95% test coverage
		- zephyr
		- vicuna
		- baichuan2
		- chatglm3
		- internlm2
		- yi
		- ...

		@@ -24,2 +34,18 @@ # Why llama2 ?

		# Packages

		\| Library Name \| Description \| Compatibility \|
		\|---------------------------------------\|-------------------------------------------\|-------------------------------------------------------\|
		\| @lenml/llama2-tokenizer \| Tokenizer library for text segmentation \| \|
		\| @lenml/llama2-tokenizer-vocab-llama2 \| Vocabulary for llama2 \| mistral, zephyr, vicuna, llama2 \|
		\| @lenml/llama2-tokenizer-vocab-baichuan2 \| Vocabulary for baichuan2 \| baichuan2 \|
		\| @lenml/llama2-tokenizer-vocab-chatglm3 \| Vocabulary for chatglm3 \| chatglm3 \|
		\| @lenml/llama2-tokenizer-vocab-internlm2 \| Vocabulary for internlm2 \| internlm2 \|
		\| @lenml/llama2-tokenizer-vocab-yi \| Vocabulary for yi \| yi \|
		\| @lenml/llama2-tokenizer-vocab-falcon \| Vocabulary for falcon (🚧WIP) \| falcon (🚧WIP) \|
		\| @lenml/llama2-tokenizer-vocab-neox \| Vocabulary for neox (🚧WIP) \| neox, RWKV (🚧WIP) \|
		\| @lenml/llama2-tokenizer-vocab-emoji \| a vocab demo (🚧WIP) \| 🚧WIP \|

		This table lists the name of each library, its description, and its compatibility.

		## Installation
		@@ -31,2 +57,12 @@

		### install vocab
		```bash
		npm install @lenml/llama2-tokenizer-vocab-llama2
		# npm install @lenml/llama2-tokenizer-vocab-baichuan2
		# npm install @lenml/llama2-tokenizer-vocab-chatglm3
		# npm install @lenml/llama2-tokenizer-vocab-falcon
		# npm install @lenml/llama2-tokenizer-vocab-internlm2
		# npm install @lenml/llama2-tokenizer-vocab-yi
		```

		## Usage
		@@ -38,2 +74,3 @@
		import { Llama2Tokenizer } from "@lenml/llama2-tokenizer";
		import { load_vocab } from "@lenml/llama2-tokenizer-vocab-llama2"
		```
		@@ -45,3 +82,4 @@
		const tokenizer = new Llama2Tokenizer();
		tokenizer.load_llama2_vocab();
		const vocab_model = load_vocab();
		tokenizer.install_vocab(vocab_model);
		```
		@@ -83,3 +121,3 @@

		> It is not recommended to use `[XX]` (like `[CLS]` or `[PAD]`) as a special token for this pattern, as it can easily lead to conflicts. Because `"_["` is also a usable token, it is difficult to be compatible with this bad case without adjusting the word list order.
		> It is not recommended to use ```[XX]``` (like ```[CLS]``` or ```[PAD]```) as a special token for this pattern, as it can easily lead to conflicts. Because ```"_["``` is also a usable token, it is difficult to be compatible with this bad case without adjusting the word list order.

		@@ -105,5 +143,9 @@ ### Getting Vocabulary
		```typescript
		import { Llama2Tokenizer } from "@lenml/llama2-tokenizer";
		import { load_vocab } from "@lenml/llama2-tokenizer-vocab-llama2"

		const main = async () => {
		const tokenizer = new Llama2Tokenizer();
		tokenizer.load_llama2_vocab();
		const vocab_model = load_vocab();
		tokenizer.install_vocab(vocab_model);
		console.log(tokenizer.tokenize("你好，世界！"));
		@@ -140,4 +182,17 @@ console.log(tokenizer.encode("你好，世界！"));

		# TODOs
		- [x] support llama2 vocab
		- [x] support chatglm vocab
		- [x] support baichuan vocab
		- [x] support yi vocab
		- [x] support internlm2 vocab
		- [ ] support RWKV(neox) vocab
		- [ ] support falcon
		- [ ] Chat Template

		# How to build
		read [this](./build-guide.md)

		## License

		This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

dist/main.global.js