@stdlib/nlp-tokenize
Advanced tools
Comparing version 0.0.10 to 0.0.11
@@ -34,4 +34,4 @@ /** | ||
var REGEXP_PREFIXES = /^([,([{*<"“'`‘.])/gi; | ||
var REGEXP_SUFFIXES = /([,.!?%*>:;"'”`)\]}])$/gi; | ||
var REGEXP_PREFIXES = /^([,([{*<"“'`‘]|\.{1,3})/gi; | ||
var REGEXP_SUFFIXES = /([,.!?%*>:;"'”`)\]}]|\.\.\.)$/gi; | ||
@@ -108,3 +108,11 @@ | ||
res = prefixes; | ||
res.push( substr ); | ||
if ( substr ) { | ||
res.push( substr ); | ||
} | ||
// If the last suffix is an ellipsis, move it to the front of the suffix array: | ||
if ( suffixes[ suffixes.length-1 ] === '...' ) { | ||
suffixes.pop(); | ||
suffixes.unshift( '...' ); | ||
} | ||
extend( res, suffixes ); | ||
@@ -111,0 +119,0 @@ return res; |
{ | ||
"name": "@stdlib/nlp-tokenize", | ||
"version": "0.0.10", | ||
"version": "0.0.11", | ||
"description": "Tokenize a string.", | ||
@@ -5,0 +5,0 @@ "license": "Apache-2.0", |
@@ -162,4 +162,4 @@ <!-- | ||
[test-image]: https://github.com/stdlib-js/nlp-tokenize/actions/workflows/test.yml/badge.svg?branch=v0.0.10 | ||
[test-url]: https://github.com/stdlib-js/nlp-tokenize/actions/workflows/test.yml?query=branch:v0.0.10 | ||
[test-image]: https://github.com/stdlib-js/nlp-tokenize/actions/workflows/test.yml/badge.svg?branch=v0.0.11 | ||
[test-url]: https://github.com/stdlib-js/nlp-tokenize/actions/workflows/test.yml?query=branch:v0.0.11 | ||
@@ -166,0 +166,0 @@ [coverage-image]: https://img.shields.io/codecov/c/github/stdlib-js/nlp-tokenize/main.svg |
33016
591