gpt-3-encoder - npm Package Compare versions

Comparing version 1.0.5 to 1.1.0

Encoder.js

		@@ -21,4 +21,6 @@ // This file includes code which was modified from https://github.com/openai/gpt-2

		const textEncoder = new TextEncoder("utf-8")
		const encodeStr = str => {
		return str.split('').map(x => ord(x))
		//return str.split('').map(x => ord(x))
		return Array.from(textEncoder.encode(str)).map(x => x.toString())
		}
		@@ -92,2 +94,3 @@
		let word = token.split('')

		let pairs = get_pairs(word)
		@@ -105,2 +108,5 @@
		})



		const bigram = minPairs[Math.min(...Object.keys(minPairs).map(x => {
		@@ -156,3 +162,6 @@ return parseInt(x)
		for (let token of matches) {
		token = encodeStr(token).map(x => byte_encoder[x]).join('')
		token = encodeStr(token).map(x => {
		return byte_encoder[x]
		}).join('')

		const new_tokens = bpe(token).split(' ').map(x => encoder[x])
		@@ -170,5 +179,3 @@ bpe_tokens = bpe_tokens.concat(new_tokens)

		module.exports = {
		encode,
		decode
		}
		// const encoded = encode('hello 👋 world 🌍 This is a long string to test whether or not the emoji issue was fixed!')
		// console.log({encoded})

package.json

		{
		"name": "gpt-3-encoder",
		"version": "1.0.5",
		"version": "1.1.0",
		"description": "Javascript BPE Encoder Decoder for GPT-2 / GPT-3",
		"main": "encoder.js",
		"files": ["encoder.json", "vocab.bpe"],
		"files": [
		"encoder.json",
		"vocab.bpe"
		],
		"scripts": {
		@@ -8,0 +11,0 @@ "test": "echo \"Error: no test specified\" && exit 1"

README.md

		@@ -5,3 +5,3 @@ GPT-3-Encoder
		## About
		GPT-2 and GPT-3 use byte pair encoding to turn text into a serires of integers to feed into the model. This is a javascript implementation of OpenAI's original python encoder/decoder which can be found [here](https://github.com/openai/gpt-2)
		GPT-2 and GPT-3 use byte pair encoding to turn text into a series of integers to feed into the model. This is a javascript implementation of OpenAI's original python encoder/decoder which can be found [here](https://github.com/openai/gpt-2)

		@@ -8,0 +8,0 @@ ## Install with npm

New alerts