openai-api - npm Package Compare versions

Comparing version 1.0.18 to 1.1.0

index.js

@@ -5,2 +5,3 @@ "use strict";
     axios = require('axios');
+const { ByteLevelBPETokenizer } = require('tokenizers');
 
@@ -45,22 +46,31 @@ class OpenAI {
 
-complete(opts) {
-    return this._send_request(opts);
-}
+    complete(opts) {
+        return this._send_request(opts);
+    }
 
-search(opts) {
-    const url = config.searchURL(opts.engine);
-    const reqOpts = {
-        headers: {
-            'Authorization': `Bearer ${this._api_key}`,
-            'Content-Type': 'application/json'
-        }
-    };
-    const data = {
-        documents: opts.documents,
-        query: opts.query
-    };
-    return axios.post(url, data, reqOpts);
+    encode(str) {
+        return ByteLevelBPETokenizer.fromOptions({
+            vocabFile: './data/gpt2_vocab_file.json',
+            mergesFile: './data/gpt2_merges_file.txt',
+        }).then((tokenizer) => {
+            return tokenizer.encode(str)
+        })
+    }
+
+    search(opts) {
+        const url = config.searchURL(opts.engine);
+        const reqOpts = {
+            headers: {
+                'Authorization': `Bearer ${this._api_key}`,
+                'Content-Type': 'application/json'
+            }
+        };
+        const data = {
+            documents: opts.documents,
+            query: opts.query
+        };
+        return axios.post(url, data, reqOpts);
+    }
 }
-}
 
 module.exports = OpenAI;

package.json

 {
   "name": "openai-api",
-  "version": "1.0.18",
+  "version": "1.1.0",
   "description": "A tiny client module for the openAI API",
@@ -25,7 +25,9 @@ "main": "index.js",
   "dependencies": {
-    "axios": "^0.21.1"
+    "axios": "^0.21.1",
+    "tokenizers": "^0.7.0"
   },
   "devDependencies": {
-    "mocha": "^8.1.1"
+    "chai": "^4.3.4",
+    "mocha": "^8.3.2"
   }
 }

README.md

@@ -69,3 +69,14 @@ # openai-api
 
+### Get number of tokens for string
+
+The token limit is 2048 for completions using the OpenAI API. This method allows you to get the number of tokens in your prompt. This is done offline (no API call is made).
+
+```json
+openai.encode('This is an encoding test. Number of tokens is not necessarily the same as word count.').then((result) => {
+     console.log("Number of tokens for string:" + result.length);
+ });
+```
+
 ### Search API call
+
 ```js
@@ -81,4 +92,10 @@ (async () => {
 })();
+
+(async () => {
+  await const encoded = openai.encode(str);
+  // to get the number of tokens of this string
+  console.log(encoded.length);
+})();
 ```
 
 

No alert changes

Improved metrics

Total package byte prevSize

1506190

increased by24414.81%

Number of package files

7

increased by40%

Lines of code

130

increased by18.18%

Number of lines in readme file

100

increased by20.48%

Worsened metrics

Dependency count

2

increased by100%

Dev dependency count

2

increased by100%

Dependency changes

• + Addedtokenizers@^0.7.0

• + Added@types/node@13.13.52(transitive)

• + Addedabbrev@1.1.1(transitive)

• + Addedansi-regex@2.1.1(transitive)

• + Addedaproba@1.2.0(transitive)

• + Addedare-we-there-yet@1.1.7(transitive)

• + Addedbalanced-match@1.0.2(transitive)

• + Addedbrace-expansion@1.1.11(transitive)

• + Addedchownr@1.1.4(transitive)

• + Addedcode-point-at@1.1.0(transitive)

• + Addedconcat-map@0.0.1(transitive)

• + Addedconsole-control-strings@1.1.0(transitive)

• + Addedcore-util-is@1.0.3(transitive)

• + Addeddebug@3.2.7(transitive)

• + Addeddeep-extend@0.6.0(transitive)

• + Addeddelegates@1.0.0(transitive)

• + Addeddetect-libc@1.0.3(transitive)

• + Addedfs-minipass@1.2.7(transitive)

• + Addedfs.realpath@1.0.0(transitive)

• + Addedgauge@2.7.4(transitive)

• + Addedglob@7.2.3(transitive)

• + Addedhas-unicode@2.0.1(transitive)

• + Addediconv-lite@0.4.24(transitive)

• + Addedignore-walk@3.0.4(transitive)

• + Addedinflight@1.0.6(transitive)

• + Addedinherits@2.0.4(transitive)

• + Addedini@1.3.8(transitive)

• + Addedis-fullwidth-code-point@1.0.0(transitive)

• + Addedisarray@1.0.0(transitive)

• + Addedminimatch@3.1.2(transitive)

• + Addedminimist@1.2.8(transitive)

• + Addedminipass@2.9.0(transitive)

• + Addedminizlib@1.3.3(transitive)

• + Addedmkdirp@0.5.6(transitive)

• + Addedms@2.1.3(transitive)

• + Addedneedle@2.9.1(transitive)

• + Addednode-pre-gyp@0.14.0(transitive)

• + Addednopt@4.0.3(transitive)

• + Addednpm-bundled@1.1.2(transitive)

• + Addednpm-normalize-package-bin@1.0.1(transitive)

• + Addednpm-packlist@1.4.8(transitive)

• + Addednpmlog@4.1.2(transitive)

• + Addednumber-is-nan@1.0.1(transitive)

• + Addedobject-assign@4.1.1(transitive)

• + Addedonce@1.4.0(transitive)

• + Addedos-homedir@1.0.2(transitive)

• + Addedos-tmpdir@1.0.2(transitive)

• + Addedosenv@0.1.5(transitive)

• + Addedpath-is-absolute@1.0.1(transitive)

• + Addedprocess-nextick-args@2.0.1(transitive)

• + Addedrc@1.2.8(transitive)

• + Addedreadable-stream@2.3.8(transitive)

• + Addedrimraf@2.7.1(transitive)

• + Addedsafe-buffer@5.1.25.2.1(transitive)

• + Addedsafer-buffer@2.1.2(transitive)

• + Addedsax@1.4.1(transitive)

• + Addedsemver@5.7.2(transitive)

• + Addedset-blocking@2.0.0(transitive)

• + Addedsignal-exit@3.0.7(transitive)

• + Addedstring-width@1.0.2(transitive)

• + Addedstring_decoder@1.1.1(transitive)

• + Addedstrip-ansi@3.0.1(transitive)

• + Addedstrip-json-comments@2.0.1(transitive)

• + Addedtar@4.4.19(transitive)

• + Addedtokenizers@0.7.0(transitive)

• + Addedutil-deprecate@1.0.2(transitive)

• + Addedwide-align@1.1.5(transitive)

• + Addedwrappy@1.0.2(transitive)

• + Addedyallist@3.1.1(transitive)