tokenizr - npm Package Compare versions

Comparing version 1.1.8 to 1.2.0

bower.json

 {
     "name":        "tokenizr",
-    "version":     "1.1.8",
+    "version":     "1.2.0",
     "description": "String Tokenization Library for JavaScript",
@@ -5,0 +5,0 @@ "main":        "lib/tokenizr.js",

lib/tokenizr.js

@@ -32,5 +32,5 @@ /*
 },{"./tokenizr-1-excerpt":1}],4:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{default:t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},_createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){return this._tokenizr.push.apply(this._tokenizr,arguments),this}},{key:"pop",value:function(){return this._tokenizr.pop.apply(this._tokenizr,arguments)}},{key:"state",value:function(){return arguments.length>0?(this._tokenizr.state.apply(this._tokenizr,arguments),this):this._tokenizr.state.apply(this._tokenizr,arguments)}},{key:"tag",value:function(){return this._tokenizr.tag.apply(this._tokenizr,arguments),this}},{key:"tagged",value:function(){return this._tokenizr.tagged.apply(this._tokenizr,arguments)}},{key:"untag",value:function(){return this._tokenizr.untag.apply(this._tokenizr,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+(void 0===e?"undefined":_typeof(e))+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2.default(t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports.default=ActionContext;
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{default:t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},_createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){return this._tokenizr.push.apply(this._tokenizr,arguments),this}},{key:"pop",value:function(){return this._tokenizr.pop.apply(this._tokenizr,arguments)}},{key:"state",value:function(){return arguments.length>0?(this._tokenizr.state.apply(this._tokenizr,arguments),this):this._tokenizr.state.apply(this._tokenizr,arguments)}},{key:"tag",value:function(){return this._tokenizr.tag.apply(this._tokenizr,arguments),this}},{key:"tagged",value:function(){return this._tokenizr.tagged.apply(this._tokenizr,arguments)}},{key:"untag",value:function(){return this._tokenizr.untag.apply(this._tokenizr,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+(void 0===e?"undefined":_typeof(e))+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2.default(t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}},{key:"stop",value:function(){return this._tokenizr._stopped=!0,this}}]),t}();exports.default=ActionContext;
 },{"./tokenizr-2-token":2}],5:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{default:t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},_createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2.default(this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): old: <"+this._state[this._state.length-1]+">, new: <"+t+">"),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): old: <"+this._state[this._state.length-1]+">, new: <"+this._state[this._state.length-2]+">"),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): old: <"+this._state[this._state.length-1]+">, new: <"+t+">"),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return!0===this._tag[t]}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"before",value:function(t){return this._before=t,this}},{key:"after",value:function(t){return this._after=t,this}},{key:"finish",value:function(t){return this._finish=t,this}},{key:"rule",value:function(t,e,n){var i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:"unknown";if(2===arguments.length&&"function"==typeof e){var r=[t,e];e=r[0],n=r[1],t="*"}else if(3===arguments.length&&"function"==typeof e){var o=[t,e,n];e=o[0],n=o[1],i=o[2],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===(void 0===e?"undefined":_typeof(e))&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');if("string"!=typeof i)throw new Error('parameter "name" not a String');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),i=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:i}});var s="g";return e.multiline&&(s+="m"),e.ignoreCase&&(s+="i"),e=new RegExp(e.source,s),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n,name:i}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,i=this._column,r=this._input,o=t;o<e;o++){var s=r.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", from: <line "+n+", column "+i+">, to: <line "+this._line+", column "+this._column+">")}},{key:"_tokenize",value:function(){var t=this,e=function(){t._eof||(null!==t._finish&&t._finish.call(t._ctx,t._ctx),t._eof=!0,t._pending.push(new _tokenizr2Token2.default("EOF","","",t._pos,t._line,t._column)))};if(this._pos>=this._len)return void e();for(var n=!0;n;){if(n=!1,this._debug){var i=(0,_tokenizr1Excerpt2.default)(this._input,this._pos),r=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+r+">, text: "+(i.prologTrunc?"...":'"')+i.prologText+"<"+i.tokenText+">"+i.epilogText+(i.epilogTrunc?"...":'"')+", at: <line "+this._line+", column "+this._column+">")}for(var o=0;o<this._rules.length;o++){if(this._debug){var s=this._rules[o].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+s+">, pattern: "+this._rules[o].pattern.source)}var a=!1,h=this._rules[o].state.map(function(t){return t.state}),l=h.indexOf("*");if(l<0&&(l=h.indexOf(this._state[this._state.length-1])),l>=0){a=!0;var u=this._rules[o].state[l].tags;u=u.filter(function(e){return!t._tag[e]}),u.length>0&&(a=!1)}if(a){this._rules[o].pattern.lastIndex=this._pos;var _=this._rules[o].pattern.exec(this._input);if(this._rules[o].pattern.lastIndex=this._pos,null!==(_=this._rules[o].pattern.exec(this._input))&&_.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(_)),this._ctx._match=_,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,null!==this._before&&this._before.call(this._ctx,this._ctx,_,this._rules[o]),this._rules[o].action.call(this._ctx,this._ctx,_),null!==this._after&&this._after.call(this._ctx,this._ctx,_,this._rules[o]),this._ctx._reject)continue;if(this._ctx._repeat){n=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,this._pos>=this._len)return void e();n=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,void(this._pos>=this._len&&e());throw new Error('action of pattern "'+this._rules[o].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: "+t.toString()),t}return null}},{key:"tokens",value:function(){for(var t=[],e=void 0;null!==(e=this.token());)t.push(e);return t}},{key:"peek",value:function(t){void 0===t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){void 0===t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();if(this._log("CONSUME: "+i.toString()),2===arguments.length){if(!i.isA(t,e))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+(void 0===e?"undefined":_typeof(e))+")>, found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+_typeof(i.value)+")>",i.pos,i.line,i.column,this._input)}else if(!i.isA(t))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: * (any)>, found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+_typeof(i.value)+")>",i.pos,i.line,i.column,this._input);return i}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,i=Array(n),r=0;r<n;r++)i[r]=arguments[r];for(var o=0;o<i.length;o++)try{this.begin(),t=i[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: "+t.toString()),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2.default(t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2.default,Tokenizr.ParsingError=_tokenizr3Error2.default,Tokenizr.ActionContext=_tokenizr4Context2.default,exports.default=Tokenizr;
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{default:t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},_createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._stopped=!1,this._ctx=new _tokenizr4Context2.default(this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): old: <"+this._state[this._state.length-1]+">, new: <"+t+">"),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): old: <"+this._state[this._state.length-1]+">, new: <"+this._state[this._state.length-2]+">"),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): old: <"+this._state[this._state.length-1]+">, new: <"+t+">"),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return!0===this._tag[t]}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"before",value:function(t){return this._before=t,this}},{key:"after",value:function(t){return this._after=t,this}},{key:"finish",value:function(t){return this._finish=t,this}},{key:"rule",value:function(t,e,n){var i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:"unknown";if(2===arguments.length&&"function"==typeof e){var r=[t,e];e=r[0],n=r[1],t="*"}else if(3===arguments.length&&"function"==typeof e){var o=[t,e,n];e=o[0],n=o[1],i=o[2],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===(void 0===e?"undefined":_typeof(e))&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');if("string"!=typeof i)throw new Error('parameter "name" not a String');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),i=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:i}});var s="g";return e.multiline&&(s+="m"),e.ignoreCase&&(s+="i"),e=new RegExp(e.source,s),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n,name:i}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,i=this._column,r=this._input,o=t;o<e;o++){var s=r.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", from: <line "+n+", column "+i+">, to: <line "+this._line+", column "+this._column+">")}},{key:"_tokenize",value:function(){var t=this,e=function(){t._eof||(null!==t._finish&&t._finish.call(t._ctx,t._ctx),t._eof=!0,t._pending.push(new _tokenizr2Token2.default("EOF","","",t._pos,t._line,t._column)))};if(this._stopped||this._pos>=this._len)return void e();for(var n=!0;n;){if(n=!1,this._debug){var i=(0,_tokenizr1Excerpt2.default)(this._input,this._pos),r=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+r+">, text: "+(i.prologTrunc?"...":'"')+i.prologText+"<"+i.tokenText+">"+i.epilogText+(i.epilogTrunc?"...":'"')+", at: <line "+this._line+", column "+this._column+">")}for(var o=0;o<this._rules.length;o++){if(this._debug){var s=this._rules[o].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+s+">, pattern: "+this._rules[o].pattern.source)}var a=!1,h=this._rules[o].state.map(function(t){return t.state}),l=h.indexOf("*");if(l<0&&(l=h.indexOf(this._state[this._state.length-1])),l>=0){a=!0;var u=this._rules[o].state[l].tags;u=u.filter(function(e){return!t._tag[e]}),u.length>0&&(a=!1)}if(a){this._rules[o].pattern.lastIndex=this._pos;var _=this._rules[o].pattern.exec(this._input);if(this._rules[o].pattern.lastIndex=this._pos,null!==(_=this._rules[o].pattern.exec(this._input))&&_.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(_)),this._ctx._match=_,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,null!==this._before&&this._before.call(this._ctx,this._ctx,_,this._rules[o]),this._rules[o].action.call(this._ctx,this._ctx,_),null!==this._after&&this._after.call(this._ctx,this._ctx,_,this._rules[o]),this._ctx._reject)continue;if(this._ctx._repeat){n=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,this._pos>=this._len)return void e();n=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,void(this._pos>=this._len&&e());throw new Error('action of pattern "'+this._rules[o].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: "+t.toString()),t}return null}},{key:"tokens",value:function(){for(var t=[],e=void 0;null!==(e=this.token());)t.push(e);return t}},{key:"peek",value:function(t){void 0===t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){void 0===t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();if(this._log("CONSUME: "+i.toString()),2===arguments.length){if(!i.isA(t,e))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+(void 0===e?"undefined":_typeof(e))+")>, found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+_typeof(i.value)+")>",i.pos,i.line,i.column,this._input)}else if(!i.isA(t))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: * (any)>, found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+_typeof(i.value)+")>",i.pos,i.line,i.column,this._input);return i}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,i=Array(n),r=0;r<n;r++)i[r]=arguments[r];for(var o=0;o<i.length;o++)try{this.begin(),t=i[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: "+t.toString()),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2.default(t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2.default,Tokenizr.ParsingError=_tokenizr3Error2.default,Tokenizr.ActionContext=_tokenizr4Context2.default,exports.default=Tokenizr;
 },{"./tokenizr-1-excerpt":1,"./tokenizr-2-token":2,"./tokenizr-3-error":3,"./tokenizr-4-context":4}],6:[function(_dereq_,module,exports){
@@ -37,0 +37,0 @@ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}var _tokenizr5Tokenizer=_dereq_("./tokenizr-5-tokenizer"),_tokenizr5Tokenizer2=_interopRequireDefault(_tokenizr5Tokenizer);module.exports=_tokenizr5Tokenizer2.default;

package.json

 {
     "name":        "tokenizr",
-    "version":     "1.1.8",
+    "version":     "1.2.0",
     "description": "String Tokenization Library for JavaScript",
@@ -5,0 +5,0 @@ "keywords":    [ "string", "token", "scanner", "lexer" ],

README.md

@@ -421,2 +421,6 @@
 
+- Method: `Tokenizr.ActionContext#stop(): Tokenizr.ActionContext`<br/>
+  Immediately stop entire tokenization. After this the
+  `Tokenizr#token()` method immediately starts to return `null`.
+
 Implementation Notice
@@ -423,0 +427,0 @@ ---------------------

src/tokenizr-4-context.js

@@ -106,3 +106,3 @@ /*
 
-    /*  accept current matching is a new token  */
+    /*  accept current matching as a new token  */
     accept (type, value) {
@@ -118,3 +118,9 @@ if (arguments.length < 2)
     }
+
+    /*  immediately stop tokenization  */
+    stop () {
+        this._tokenizr._stopped = true
+        return this
+    }
 }
 

src/tokenizr-5-tokenizer.js

@@ -54,2 +54,3 @@ /*
         this._pending     = []
+        this._stopped     = false
         this._ctx         = new ActionContext(this)
@@ -274,4 +275,4 @@ return this
 
-        /*  tokenize only as long as there is input left  */
-        if (this._pos >= this._len) {
+        /*  tokenize only as long as we were not stopped and there is input left  */
+        if (this._stopped || this._pos >= this._len) {
             finish()
@@ -278,0 +279,0 @@ return

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

82698

increased by0.6%

Lines of code

1138

increased by0.62%

Number of lines in readme file

462

increased by0.87%

No dependency changes