tokenizr - npm Package Compare versions

Comparing version 1.1.0 to 1.1.1

bower.json

 {
     "name":        "tokenizr",
-    "version":     "1.1.0",
+    "version":     "1.1.1",
     "description": "String Tokenization Library for JavaScript",
@@ -5,0 +5,0 @@ "main":        "lib/tokenizr.js",

lib/tokenizr.js

@@ -34,3 +34,3 @@ /*
 },{"./tokenizr-2-token":2}],5:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{default:t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},_createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2.default(this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+this._state[this._state.length-2]+">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._tag[t]===!0}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"before",value:function(t){return this._before=t,this}},{key:"after",value:function(t){return this._after=t,this}},{key:"finish",value:function(t){return this._finish=t,this}},{key:"rule",value:function(t,e,n){var i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:"unknown";if(2===arguments.length&&"function"==typeof e){var r=[t,e];e=r[0],n=r[1],t="*"}else if(3===arguments.length&&"function"==typeof e){var o=[t,e,n];e=o[0],n=o[1],i=o[2],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===("undefined"==typeof e?"undefined":_typeof(e))&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');if("string"!=typeof i)throw new Error('parameter "name" not a String');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),i=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:i}});var s="g";return e.multiline&&(s+="m"),e.ignoreCase&&(s+="i"),e=new RegExp(e.source,s),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n,name:i}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,i=this._column,r=this._input,o=t;o<e;o++){var s=r.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+i+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){var t=this;if(this._pos>=this._len)return void(this._eof||(null!==this._finish&&this._finish.call(this._ctx,this._ctx),this._eof=!0,this._pending.push(new _tokenizr2Token2.default("EOF","","",this._pos,this._line,this._column))));for(var e=!0;e;){if(e=!1,this._debug){var n=(0,_tokenizr1Excerpt2.default)(this._input,this._pos),i=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+i+">, text: "+(n.prologTrunc?"...":'"')+(n.prologText+"<"+n.tokenText+">"+n.epilogText)+(n.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var r=0;r<this._rules.length;r++){if(this._debug){var o=this._rules[r].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+o+">, pattern: "+this._rules[r].pattern.source)}var s=!1,a=this._rules[r].state.map(function(t){return t.state}),h=a.indexOf("*");if(h<0&&(h=a.indexOf(this._state[this._state.length-1])),h>=0){s=!0;var l=this._rules[r].state[h].tags;l=l.filter(function(e){return!t._tag[e]}),l.length>0&&(s=!1)}if(s){this._rules[r].pattern.lastIndex=this._pos;var u=this._rules[r].pattern.exec(this._input);if(this._rules[r].pattern.lastIndex=this._pos,null!==(u=this._rules[r].pattern.exec(this._input))&&u.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(u)),this._ctx._match=u,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,null!==this._before&&this._before.call(this._ctx,this._ctx,u,this._rules[r]),this._rules[r].action.call(this._ctx,this._ctx,u),null!==this._after&&this._after.call(this._ctx,this._ctx,u,this._rules[r]),this._ctx._reject)continue;if(this._ctx._repeat){e=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[r].pattern.lastIndex),this._pos=this._rules[r].pattern.lastIndex,this._pos>=this._len)return;e=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[r].pattern.lastIndex),void(this._pos=this._rules[r].pattern.lastIndex);throw new Error('action of pattern "'+this._rules[r].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function t(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: "+t.toString()),t}return null}},{key:"tokens",value:function(){for(var t=[],e=void 0;null!==(e=this.token());)t.push(e);return t}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();if(this._log("CONSUME: "+i.toString()),2===arguments.length){if(!i.isA(t,e))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+")>, "+("found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+_typeof(i.value)+")>"),i.pos,i.line,i.column,this._input)}else if(!i.isA(t))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+_typeof(i.value)+")>"),i.pos,i.line,i.column,this._input);return i}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,i=Array(n),r=0;r<n;r++)i[r]=arguments[r];for(var o=0;o<i.length;o++)try{this.begin(),t=i[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: "+t.toString()),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2.default(t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2.default,Tokenizr.ParsingError=_tokenizr3Error2.default,Tokenizr.ActionContext=_tokenizr4Context2.default,exports.default=Tokenizr;
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{default:t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},_createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2.default(this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+this._state[this._state.length-2]+">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._tag[t]===!0}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"before",value:function(t){return this._before=t,this}},{key:"after",value:function(t){return this._after=t,this}},{key:"finish",value:function(t){return this._finish=t,this}},{key:"rule",value:function(t,e,n){var i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:"unknown";if(2===arguments.length&&"function"==typeof e){var r=[t,e];e=r[0],n=r[1],t="*"}else if(3===arguments.length&&"function"==typeof e){var o=[t,e,n];e=o[0],n=o[1],i=o[2],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===("undefined"==typeof e?"undefined":_typeof(e))&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');if("string"!=typeof i)throw new Error('parameter "name" not a String');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),i=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:i}});var s="g";return e.multiline&&(s+="m"),e.ignoreCase&&(s+="i"),e=new RegExp(e.source,s),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n,name:i}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,i=this._column,r=this._input,o=t;o<e;o++){var s=r.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+i+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){var t=this,e=function(){t._eof||(null!==t._finish&&t._finish.call(t._ctx,t._ctx),t._eof=!0,t._pending.push(new _tokenizr2Token2.default("EOF","","",t._pos,t._line,t._column)))};if(this._pos>=this._len)return void e();for(var n=!0;n;){if(n=!1,this._debug){var i=(0,_tokenizr1Excerpt2.default)(this._input,this._pos),r=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+r+">, text: "+(i.prologTrunc?"...":'"')+(i.prologText+"<"+i.tokenText+">"+i.epilogText)+(i.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var o=0;o<this._rules.length;o++){if(this._debug){var s=this._rules[o].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+s+">, pattern: "+this._rules[o].pattern.source)}var a=!1,h=this._rules[o].state.map(function(t){return t.state}),l=h.indexOf("*");if(l<0&&(l=h.indexOf(this._state[this._state.length-1])),l>=0){a=!0;var u=this._rules[o].state[l].tags;u=u.filter(function(e){return!t._tag[e]}),u.length>0&&(a=!1)}if(a){this._rules[o].pattern.lastIndex=this._pos;var _=this._rules[o].pattern.exec(this._input);if(this._rules[o].pattern.lastIndex=this._pos,null!==(_=this._rules[o].pattern.exec(this._input))&&_.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(_)),this._ctx._match=_,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,null!==this._before&&this._before.call(this._ctx,this._ctx,_,this._rules[o]),this._rules[o].action.call(this._ctx,this._ctx,_),null!==this._after&&this._after.call(this._ctx,this._ctx,_,this._rules[o]),this._ctx._reject)continue;if(this._ctx._repeat){n=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,this._pos>=this._len)return void e();n=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,void(this._pos>=this._len&&e());throw new Error('action of pattern "'+this._rules[o].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function t(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: "+t.toString()),t}return null}},{key:"tokens",value:function(){for(var t=[],e=void 0;null!==(e=this.token());)t.push(e);return t}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();if(this._log("CONSUME: "+i.toString()),2===arguments.length){if(!i.isA(t,e))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+")>, "+("found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+_typeof(i.value)+")>"),i.pos,i.line,i.column,this._input)}else if(!i.isA(t))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+_typeof(i.value)+")>"),i.pos,i.line,i.column,this._input);return i}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,i=Array(n),r=0;r<n;r++)i[r]=arguments[r];for(var o=0;o<i.length;o++)try{this.begin(),t=i[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: "+t.toString()),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2.default(t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2.default,Tokenizr.ParsingError=_tokenizr3Error2.default,Tokenizr.ActionContext=_tokenizr4Context2.default,exports.default=Tokenizr;
 },{"./tokenizr-1-excerpt":1,"./tokenizr-2-token":2,"./tokenizr-3-error":3,"./tokenizr-4-context":4}],6:[function(_dereq_,module,exports){
@@ -37,0 +37,0 @@ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}var _tokenizr5Tokenizer=_dereq_("./tokenizr-5-tokenizer"),_tokenizr5Tokenizer2=_interopRequireDefault(_tokenizr5Tokenizer);module.exports=_tokenizr5Tokenizer2.default;

package.json

 {
     "name":        "tokenizr",
-    "version":     "1.1.0",
+    "version":     "1.1.1",
     "description": "String Tokenization Library for JavaScript",
@@ -5,0 +5,0 @@ "keywords":    [ "string", "token", "scanner", "lexer" ],

src/tokenizr-5-tokenizer.js

@@ -263,4 +263,4 @@ /*
     _tokenize () {
-        /*  tokenize only as long as there is input left  */
-        if (this._pos >= this._len) {
+        /*  helper function for finishing parsing  */
+        const finish = () => {
             if (!this._eof) {
@@ -272,2 +272,7 @@ if (this._finish !== null)
             }
+        }
+
+        /*  tokenize only as long as there is input left  */
+        if (this._pos >= this._len) {
+            finish()
             return
@@ -351,4 +356,6 @@ }
                         this._pos = this._rules[i].pattern.lastIndex
-                        if (this._pos >= this._len)
+                        if (this._pos >= this._len) {
+                            finish()
                             return
+                        }
                         continued = true
@@ -361,2 +368,4 @@ break
                         this._pos = this._rules[i].pattern.lastIndex
+                        if (this._pos >= this._len)
+                            finish()
                         return
@@ -363,0 +372,0 @@ }

tst/tokenizr.js

@@ -85,3 +85,3 @@ /*
         expect(tokens).to.be.a("array")
-        expect(tokens).to.have.length(4)
+        expect(tokens).to.have.length(5)
         expect(tokens[0]).to.be.a("object").and.to.be.deep
@@ -88,0 +88,0 @@ .equal({ type: "symbol", value: "foo", text: "foo", pos: 0, line: 1, column: 1 })

No alert changes

Improved metrics

Total package byte prevSize

82314

increased by0.36%

Lines of code

1131

increased by0.71%

No dependency changes