@lenml/tokenizers - npm Package Compare versions

Comparing version 3.0.2 to 3.0.3

dist/main.d.ts

@@ -840,3 +840,3 @@ /**
      * @param {Object} config The configuration object for the normalizer.
-     * @returns {Normalizer | null} A Normalizer object.
+     * @returns {Normalizer|null} A Normalizer object.
      * @throws {Error} If an unknown Normalizer type is specified in the config.
@@ -843,0 +843,0 @@ */

package.json

 {
   "name": "@lenml/tokenizers",
-  "version": "3.0.2",
+  "version": "3.0.3",
   "description": "a lightweight no-dependency fork of transformers.js (only tokenizers)",
@@ -5,0 +5,0 @@ "source": "src/main.ts",

src/tokenizers/utils/constants.js

 export const GITHUB_ISSUE_URL =
-  "https://github.com/xenova/transformers.js/issues/new/choose";
+  "https://github.com/huggingface/transformers.js/issues/new/choose";

src/tokenizers/utils/data-structures.js

@@ -231,3 +231,3 @@ /**
   extend(texts) {
-    for (let text of texts) {
+    for (const text of texts) {
       this.push(text);
@@ -243,3 +243,3 @@ }
     let node = this.root;
-    for (let ch of text) {
+    for (const ch of text) {
       let child = node.children.get(ch);
@@ -262,8 +262,10 @@ if (child === undefined) {
     let node = this.root;
+    if (node === undefined) return;
+
     let prefix = "";
-    for (let i = 0; i < text.length && node !== undefined; ++i) {
-      const ch = text[i];
+    for (const ch of text) {
       prefix += ch;
       node = node.children.get(ch);
-      if (node !== undefined && node.isLeaf) {
+      if (node === undefined) return;
+      if (node.isLeaf) {
         yield prefix;
@@ -310,4 +312,4 @@ }
   constructor(sentence, bosTokenId, eosTokenId) {
-    this.sentence = sentence;
-    this.len = sentence.length;
+    this.chars = Array.from(sentence);
+    this.len = this.chars.length;
     this.bosTokenId = bosTokenId;
@@ -346,3 +348,3 @@ this.eosTokenId = eosTokenId;
    *
-   * @returns {TokenLatticeNode[]} The array of nodes representing the most likely sequence of tokens.
+   * @returns {TokenLatticeNode[]} The most likely sequence of tokens.
    */
@@ -401,7 +403,7 @@ viterbi() {
   piece(node) {
-    return this.sentence.slice(node.pos, node.pos + node.length);
+    return this.chars.slice(node.pos, node.pos + node.length).join("");
   }
 
   /**
-   * @returns {Array} The array of nodes representing the most likely sequence of tokens.
+   * @returns {string[]} The most likely sequence of tokens.
    */
@@ -414,3 +416,3 @@ tokens() {
   /**
-   * @returns {Array} The array of nodes representing the most likely sequence of tokens.
+   * @returns {number[]} The most likely sequence of token ids.
    */
@@ -417,0 +419,0 @@ tokenIds() {

src/tokenizers/utils/maths.js

@@ -163,8 +163,17 @@ /**
 export function log_softmax(arr) {
+  // Compute the maximum value in the array
+  const maxVal = max(arr)[0];
+
+  // Compute the sum of the exponentials
+  let sumExps = 0;
+  for (let i = 0; i < arr.length; ++i) {
+    sumExps += Math.exp(arr[i] - maxVal);
+  }
+
+  // Compute the log of the sum
+  const logSum = Math.log(sumExps);
+
   // Compute the softmax values
-  const softmaxArr = softmax(arr);
+  const logSoftmaxArr = arr.map((x) => x - maxVal - logSum);
 
-  // Apply log formula to each element
-  const logSoftmaxArr = softmaxArr.map((x) => Math.log(x));
-
   return /** @type {T} */ (logSoftmaxArr);
@@ -222,3 +231,3 @@ }
  * @param {number[]|TypedArray} arr array of numbers.
- * @returns {number[]} the value and index of the minimum element, of the form: [valueOfMin, indexOfMin]
+ * @returns {[number, number]} the value and index of the minimum element, of the form: [valueOfMin, indexOfMin]
  * @throws {Error} If array is empty.
@@ -225,0 +234,0 @@ */

New alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Found 1 instance in 1 package

Fixed alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

2080446

0.04%

Lines of code

8567

0.09%

No dependency changes