wink-nlp - npm Package Compare versions

Comparing version 2.2.2 to 2.3.0

.nyc_output/processinfo/index.json

@@ -1,1 +0,1 @@
-{"processes":{"bf5b47d7-073c-423a-a200-3b5f909e29ee":{"parent":null,"children":[]}},"files":{"/Users/neilsbohr/dev/winkjs/wink-nlp/src/wink-nlp.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/dd-wrapper.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/constants.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/doc-v2.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/contained-entities.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/locate.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/get-parent-item.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/search.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-get-item.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-get-item.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-each.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-each.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-filter.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-filter.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/itm-token-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/its.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/sort4FT.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/sentence-wise-importance.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/reconstruct-spaces.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/allowed.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/as.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/contained-markings.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-tokens-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-tokens-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/itm-entity-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-entities-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-entities-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/itm-sentence-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-sentences-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/itm-document-out.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/print-tokens.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/cache.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/tokenizer.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/recursive-tokenizer.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/compile-trex.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/tokens-mappers.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/examples-compiler.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/automaton.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/compose-patterns.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/identify-marked-area.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/helper.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/utilities/similarity.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/utilities/bm25-vectorizer.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"],"/Users/neilsbohr/dev/winkjs/wink-nlp/utilities/allowed.js":["bf5b47d7-073c-423a-a200-3b5f909e29ee"]},"externalIds":{}}
+{"processes":{"c978e462-eaf6-4635-aa2f-228deed05de6":{"parent":null,"children":[]}},"files":{"/Users/neilsbohr/dev/winkjs/wink-nlp/src/wink-nlp.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/dd-wrapper.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/constants.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/doc-v2.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/contained-entities.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/locate.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/get-parent-item.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/search.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-get-item.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-get-item.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-each.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-each.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-filter.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-filter.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/itm-token-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/its.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/sort4FT.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/sentence-wise-importance.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/reconstruct-spaces.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/allowed.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/as.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/contained-markings.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-tokens-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-tokens-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/itm-entity-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-entities-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/sel-entities-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/itm-sentence-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/col-sentences-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/itm-document-out.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/api/print-tokens.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/cache.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/tokenizer.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/recursive-tokenizer.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/compile-trex.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/tokens-mappers.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/examples-compiler.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/automaton.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/compose-patterns.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/identify-marked-area.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/src/helper.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/utilities/similarity.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/utilities/bm25-vectorizer.js":["c978e462-eaf6-4635-aa2f-228deed05de6"],"/Users/neilsbohr/dev/winkjs/wink-nlp/utilities/allowed.js":["c978e462-eaf6-4635-aa2f-228deed05de6"]},"externalIds":{}}

CHANGELOG.md

@@ -0,1 +1,7 @@
+# [Enabled more special space characters handling](https://github.com/winkjs/wink-nlp/releases/tag/2.3.0)
+## Version 2.3.0 May 19, 2024
+
+### ✨ Features
+- Detokenization now restores em/en, third/quarter, thin/hair, medium math space characters & narrow non breaking space characters besides the regular nbsp. 👏 🙌 🛰️
+
 # [Improved error handling in contextual vectors](https://github.com/winkjs/wink-nlp/releases/tag/2.2.2)
@@ -2,0 +8,0 @@ ## Version 2.2.2 May 08, 2024

package.json

 {
   "name": "wink-nlp",
-  "version": "2.2.2",
+  "version": "2.3.0",
   "description": "Developer friendly Natural Language Processing ✨",
@@ -5,0 +5,0 @@ "keywords": [

src/tokenizer.js

@@ -200,3 +200,3 @@ //     wink-nlp
       // Non-empty token:
-      const hasNBSP = ( /\u00a0/ ).test( t );
+      const hasNBSP = ( /[\u00a0\u2002-\u2005\u2009\u200a\u202f\u205f]/ ).test( t );
       if ( t[ 0 ] === ' ' || hasNBSP ) {
@@ -203,0 +203,0 @@ // This indicates spaces: count them.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

652950

increased by0.06%

No dependency changes