@nlpjs/utils - npm Package Compare versions

Comparing version 4.8.2 to 4.8.3

package.json

 {
   "name": "@nlpjs/utils",
-  "version": "4.8.2",
+  "version": "4.8.3",
   "description": "NLP utils",
@@ -30,3 +30,3 @@ "author": {
   },
-  "gitHead": "999815e4fe26238edf0de7ebc63bfed7aca6d574"
+  "gitHead": "bdb43424794d81aaf87b26a0a2491b286c2ea67a"
 }

src/base-fn.js

@@ -47,5 +47,17 @@ /*
 
+function wordPunctTokenize(text) {
+  return text
+    .split(/([A-zÀ-ÿ-]+|[0-9._]+|.|!|\?|'|"|:|;|,|-)/i)
+    .filter((x) => x.trim());
+}
+
+function lowerNormalize(text) {
+  return text.toLowerCase();
+}
+
 module.exports = {
   tokenize,
   normalize,
+  wordPunctTokenize,
+  lowerNormalize,
 };

src/index.js

@@ -26,2 +26,3 @@ /*
 const TfIdf = require('./tfidf');
+const MarkovChain = require('./markov');
 
@@ -31,2 +32,3 @@ module.exports = {
   TfIdf,
+  MarkovChain,
 };

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

19638

increased by49.67%

Number of package files

8

increased by14.29%

Lines of code

477

increased by72.83%

No dependency changes