tokenizr - npm Package Compare versions

Comparing version 0.9.6 to 0.9.7

bower.json

 {
     "name":        "tokenizr",
-    "version":     "0.9.6",
+    "version":     "0.9.7",
     "description": "String Tokenization Library for JavaScript",
@@ -5,0 +5,0 @@ "main":        "lib/tokenizr.js",

lib/tokenizr.js

@@ -32,5 +32,5 @@ /*
 },{"./tokenizr-1-excerpt":1}],4:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"state",value:function(t){if("string"==typeof t)this._tokenizr._log("    STATE (PUSH): "+("old: "+this._tokenizr._state[this._tokenizr._state.length-1]+", ")+("new: "+t)),this._tokenizr._state.push(t);else{if(!(this._tokenizr._state.length>=2))throw new Error("internal error: no more states to pop");this._tokenizr._log("    STATE (POP): "+("old: "+this._tokenizr._state[this._tokenizr._state.length-1]+", ")+("new: "+this._tokenizr._state[this._tokenizr._state.length-2])),this._tokenizr._state.pop()}return this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+typeof e+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2["default"](t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports["default"]=ActionContext,module.exports=exports["default"];
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){return this._tokenizr.push.apply(this._tokenizr,arguments),this}},{key:"pop",value:function(){return this._tokenizr.pop.apply(this._tokenizr,arguments)}},{key:"state",value:function(){return arguments.length>0?(this._tokenizr.state.apply(this._tokenizr,arguments),this):this._tokenizr.state.apply(this._tokenizr,arguments)}},{key:"tag",value:function(){return this._tokenizr.tag.apply(this._tokenizr,arguments),this}},{key:"untag",value:function(){return this._tokenizr.untag.apply(this._tokenizr,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+typeof e+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2["default"](t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports["default"]=ActionContext,module.exports=exports["default"];
 },{"./tokenizr-2-token":2}],5:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2["default"](this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');this._state.push(t)}else{if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");this._state.pop()}return this}},{key:"rule",value:function(t,e,n){if(2===arguments.length){var i=[t,e];e=i[0],n=i[1],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"==typeof e&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');t=t.split(/\s*,\s*/g);var r="g";return e.multiline&&(r+="m"),e.ignoreCase&&(r+="i"),e=new RegExp(e.source,r),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,i=this._column,r=this._input,o=t;e>o;o++){var s=r.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"	"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+i+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){if(this._pos>=this._len)return void(this._eof||(this._eof=!0,this._pending.push(new _tokenizr2Token2["default"]("EOF","","",this._pos,this._line,this._column))));for(var t=!0;t;){if(t=!1,this._debug){var e=_tokenizr1Excerpt2["default"](this._input,this._pos);this._log("INPUT: state: "+this._state[this._state.length-1]+", text: "+(e.prologTrunc?"...":'"')+(e.prologText+"<"+e.tokenText+">"+e.epilogText)+(e.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var n=0;n<this._rules.length;n++)if(this._debug&&this._log("  RULE: state(s): "+this._rules[n].state.join(",")+", pattern: "+this._rules[n].pattern.source),1===this._rules[n].state.length&&"*"===this._rules[n].state[0]||this._rules[n].state.indexOf(this._state[this._state.length-1])>=0){this._rules[n].pattern.lastIndex=this._pos;var i=this._rules[n].pattern.exec(this._input);if(this._rules[n].pattern.lastIndex=this._pos,null!==(i=this._rules[n].pattern.exec(this._input))&&i.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(i)),this._ctx._match=i,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,this._rules[n].action.call(this._ctx,this._ctx,i),this._ctx._reject)continue;if(this._ctx._repeat){t=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[n].pattern.lastIndex),this._pos=this._rules[n].pattern.lastIndex,this._pos>=this._len)return;t=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[n].pattern.lastIndex),void(this._pos=this._rules[n].pattern.lastIndex);throw new Error('action of pattern "'+this._rules[n].pattern.source+'" neither rejected nor accepted any token(s)')}}}throw this.error("token not recognized")}},{key:"token",value:function e(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var e=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(e),this._log("TOKEN: "+e.toString()),e}return null}},{key:"tokens",value:function n(){for(var n=[],t=void 0;null!==(t=this.token());)n.push(t);return n}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-->0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();if(this._log("CONSUME: "+i.toString()),2===arguments.length){if(!i.isA(t,e))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+typeof e+")>, "+("found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+typeof i.value+")>"),i.pos,i.line,i.column,this._input)}else if(!i.isA(t))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+typeof i.value+")>"),i.pos,i.line,i.column,this._input);return i}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,i=Array(n),r=0;n>r;r++)i[r]=arguments[r];for(var o=0;o<i.length;o++)try{this.begin(),t=i[o](),this.commit();break}catch(s){this._log("EXCEPTION: "+s.toString()),e.push({ex:s,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2["default"](t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2["default"],Tokenizr.ParsingError=_tokenizr3Error2["default"],Tokenizr.ActionContext=_tokenizr4Context2["default"],exports["default"]=Tokenizr,module.exports=exports["default"];
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r)}}return function(e,n,r){return n&&t(e.prototype,n),r&&t(e,r),e}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2["default"](this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+this._state[this._state.length-2]+">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"rule",value:function(t,e,n){if(2===arguments.length){var r=[t,e];e=r[0],n=r[1],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"==typeof e&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var i="g";return e.multiline&&(i+="m"),e.ignoreCase&&(i+="i"),e=new RegExp(e.source,i),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;e>o;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"	"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+r+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){var t=this;if(this._pos>=this._len)return void(this._eof||(this._eof=!0,this._pending.push(new _tokenizr2Token2["default"]("EOF","","",this._pos,this._line,this._column))));for(var e=!0;e;){if(e=!1,this._debug){var n=_tokenizr1Excerpt2["default"](this._input,this._pos),r=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+r+">, text: "+(n.prologTrunc?"...":'"')+(n.prologText+"<"+n.tokenText+">"+n.epilogText)+(n.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var i=0;i<this._rules.length;i++){if(this._debug){var o=this._rules[i].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+o+">, pattern: "+this._rules[i].pattern.source)}var s=!1,a=this._rules[i].state.map(function(t){return t.state}),h=a.indexOf("*");if(0>h&&(h=a.indexOf(this._state[this._state.length-1])),h>=0){s=!0;var r=this._rules[i].state[h].tags;r=r.filter(function(e){return!t._tag[e]}),r.length>0&&(s=!1)}if(s){this._rules[i].pattern.lastIndex=this._pos;var u=this._rules[i].pattern.exec(this._input);if(this._rules[i].pattern.lastIndex=this._pos,null!==(u=this._rules[i].pattern.exec(this._input))&&u.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(u)),this._ctx._match=u,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,this._rules[i].action.call(this._ctx,this._ctx,u),this._ctx._reject)continue;if(this._ctx._repeat){e=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[i].pattern.lastIndex),this._pos=this._rules[i].pattern.lastIndex,this._pos>=this._len)return;e=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[i].pattern.lastIndex),void(this._pos=this._rules[i].pattern.lastIndex);throw new Error('action of pattern "'+this._rules[i].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function e(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var e=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(e),this._log("TOKEN: "+e.toString()),e}return null}},{key:"tokens",value:function n(){for(var n=[],t=void 0;null!==(t=this.token());)n.push(t);return n}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-->0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var r=this.token();if(this._log("CONSUME: "+r.toString()),2===arguments.length){if(!r.isA(t,e))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+typeof e+")>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+typeof r.value+")>"),r.pos,r.line,r.column,this._input)}else if(!r.isA(t))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+typeof r.value+")>"),r.pos,r.line,r.column,this._input);return r}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=Array(n),i=0;n>i;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o](),this.commit();break}catch(s){this._log("EXCEPTION: "+s.toString()),e.push({ex:s,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2["default"](t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2["default"],Tokenizr.ParsingError=_tokenizr3Error2["default"],Tokenizr.ActionContext=_tokenizr4Context2["default"],exports["default"]=Tokenizr,module.exports=exports["default"];
 },{"./tokenizr-1-excerpt":1,"./tokenizr-2-token":2,"./tokenizr-3-error":3,"./tokenizr-4-context":4}],6:[function(_dereq_,module,exports){
@@ -37,0 +37,0 @@ "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{"default":e}}Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr5Tokenizer=_dereq_("./tokenizr-5-tokenizer"),_tokenizr5Tokenizer2=_interopRequireDefault(_tokenizr5Tokenizer);exports["default"]=_tokenizr5Tokenizer2["default"],module.exports=exports["default"];

package.json

 {
     "name":        "tokenizr",
-    "version":     "0.9.6",
+    "version":     "0.9.7",
     "description": "String Tokenization Library for JavaScript",
@@ -29,3 +29,3 @@ "keywords":    [ "string", "token", "scanner", "lexer" ],
         "browserify":           "~11.0.1",
-        "babelify":             "~6.1.3",
+        "babelify":             "~6.2.0",
         "minifyify":            "~7.0.5",
@@ -32,0 +32,0 @@ "browserify-header":    "~0.9.2",

README.md

@@ -183,7 +183,21 @@
 
-- Method: `Tokenizr#state(state?: String): Tokenizr`<br/>
-  Push or pop a state onto the state stack. Use this to initialy start
-  tokenizing with a custom state. The initial and default state (which
-  cannot be pop'ed at all) is named `default`.
+- Method: `Tokenizr#push(state: String): Tokenizr`<br/>
+  Push a state onto the state stack.
 
+- Method: `Tokenizr#pop(): String`<br/>
+  Pop a state from the state stack.
+  The initial/first/lowest stack value cannot be popped.
+
+- Method: `Tokenizr#state(state: String): Tokenizr`<br/>
+  Method: `Tokenizr#state(): String`<br/>
+  Set or get the state on the top of the state stack. Use this to
+  initialy start tokenizing with a custom state. The initial state is
+  named `default`.
+
+- Method: `Tokenizr#tag(tag: String): Tokenizr`<br/>
+  Set a tag. The tag has to be matched by rules.
+
+- Method: `Tokenizr#untag(tag: String): Tokenizr`<br/>
+  Unset a tag. The tag no longer has to be matched by rules.
+
 - Method: `Tokenizr#rule(state?: String, pattern: RegExp, action: (ctx: TokenizerContext, match: Array[String]) => Void): Tokenizr`<br/>
@@ -320,4 +334,9 @@ Configure a token matching rule which executes its `action` in case the
 
-- Method: `Tokenizr.ActionContext#state(new?: String): String`<br/>
-  Push or pop a state to/from the current tokenization state stack.
+- Method: `Tokenizr.ActionContext#push(state: String): Tokenizr`<br/>
+  Method: `Tokenizr.ActionContext#pop(): String`<br/>
+  Method: `Tokenizr.ActionContext#state(state: String): Tokenizr.ActionContext`<br/>
+  Method: `Tokenizr.ActionContext#state(): String`<br/>
+  Method: `Tokenizr.ActionContext#tag(tag: String): Tokenizr.ActionContext`<br/>
+  Method: `Tokenizr.ActionContext#untag(tag: String): Tokenizr.ActionContext`<br/>
+  Methods just passed-through to the attached Tokenizr. See above for details.
 
@@ -324,0 +343,0 @@ - Method: `Tokenizr.ActionContext#repeat(): Tokenizr.ActionContext`<br/>

src/tokenizr-4-context.js

@@ -57,20 +57,26 @@ /*
 
-    /*  set a new state in the attached tokenizer  */
-    state (state) {
-        if (typeof state === "string") {
-            this._tokenizr._log(`    STATE (PUSH): ` +
-                `old: ${this._tokenizr._state[this._tokenizr._state.length- 1]}, ` +
-                `new: ${state}`)
-            this._tokenizr._state.push(state)
+    /*  pass-through functions to attached tokenizer  */
+    push () {
+        this._tokenizr.push.apply(this._tokenizr, arguments)
+        return this
+    }
+    pop () {
+        return this._tokenizr.pop.apply(this._tokenizr, arguments)
+    }
+    state () {
+        if (arguments.length > 0) {
+            this._tokenizr.state.apply(this._tokenizr, arguments)
+            return this
         }
-        else if (this._tokenizr._state.length >= 2) {
-            this._tokenizr._log(`    STATE (POP): ` +
-                `old: ${this._tokenizr._state[this._tokenizr._state.length - 1]}, ` +
-                `new: ${this._tokenizr._state[this._tokenizr._state.length - 2]}`)
-            this._tokenizr._state.pop()
-        }
         else
-            throw new Error("internal error: no more states to pop")
+            return this._tokenizr.state.apply(this._tokenizr, arguments)
+    }
+    tag () {
+        this._tokenizr.tag.apply(this._tokenizr, arguments)
         return this
     }
+    untag () {
+        this._tokenizr.untag.apply(this._tokenizr, arguments)
+        return this
+    }
 
@@ -77,0 +83,0 @@ /*  mark current matching to be repeated from scratch  */

src/tokenizr-5-tokenizer.js

@@ -48,2 +48,3 @@ /*
         this._state       = [ "default" ]
+        this._tag         = {}
         this._transaction = []
@@ -80,19 +81,81 @@ this._pending     = []
 
-    /*  change state  */
+    /*  push state  */
+    push (state) {
+        /*  sanity check arguments  */
+        if (arguments.length !== 1)
+            throw new Error("invalid number of arguments")
+        if (typeof state !== "string")
+            throw new Error("parameter \"state\" not a String")
+
+        /*  push new state  */
+        this._log(`    STATE (PUSH): ` +
+            `old: <${this._state[this._state.length - 1]}>, ` +
+            `new: <${state}>`)
+        this._state.push(state)
+        return this
+    }
+
+    /*  pop state  */
+    pop () {
+        /*  sanity check arguments  */
+        if (arguments.length !== 0)
+            throw new Error("invalid number of arguments")
+        if (this._state.length < 2)
+            throw new Error("no more custom states to pop")
+
+        /*  pop old state  */
+        this._log(`    STATE (POP): ` +
+            `old: <${this._state[this._state.length - 1]}>, ` +
+            `new: <${this._state[this._state.length - 2]}>`)
+        return this._state.pop()
+    }
+
+    /*  get/set state  */
     state (state) {
         if (arguments.length === 1) {
+            /*  sanity check arguments  */
             if (typeof state !== "string")
                 throw new Error("parameter \"state\" not a String")
-            this._state.push(state)
+
+            /*  change current state  */
+            this._log(`    STATE (SET): ` +
+                `old: <${this._state[this._state.length - 1]}>, ` +
+                `new: <${state}>`)
+            this._state[this._state.length - 1] = state
+            return this
         }
-        else if (arguments.length === 0) {
-            if (this._state.length < 2)
-                throw new Error("no more custom states to pop")
-            this._state.pop()
-        }
+        else if (arguments.length === 0)
+            return this._state[this._state.length - 1]
         else
             throw new Error("invalid number of arguments")
+    }
+
+    /*  set a tag  */
+    tag (tag) {
+        /*  sanity check arguments  */
+        if (arguments.length !== 1)
+            throw new Error("invalid number of arguments")
+        if (typeof tag !== "string")
+            throw new Error("parameter \"tag\" not a String")
+
+        /*  set tag  */
+        this._log(`    TAG (ADD): ${tag}`)
+        this._tag[tag] = true
         return this
     }
 
+    /*  unset a tag  */
+    untag (tag) {
+        /*  sanity check arguments  */
+        if (arguments.length !== 1)
+            throw new Error("invalid number of arguments")
+        if (typeof tag !== "string")
+            throw new Error("parameter \"tag\" not a String")
+
+        /*  delete tag  */
+        this._log(`    TAG (DEL): ${tag}`)
+        delete this._tag[tag]
+        return this
+    }
+
     /*  configure a tokenization rule  */
@@ -115,3 +178,11 @@ rule (state, pattern, action) {
         /*  post-process state  */
-        state = state.split(/\s*,\s*/g)
+        state = state.split(/\s*,\s*/g).map((entry) => {
+            let items  = entry.split(/\s+/g)
+            let states = items.filter((item) => item.match(/^#/) === null)
+            let tags   = items.filter((item) => item.match(/^#/) !== null)
+                .map((tag) => tag.replace(/^#/, ""))
+            if (states.length !== 1)
+                throw new Error("exactly one state required")
+            return { state: states[0], tags: tags }
+        })
 
@@ -173,3 +244,4 @@ /*  post-process pattern  */
                 let e = excerpt(this._input, this._pos)
-                this._log(`INPUT: state: ${this._state[this._state.length - 1]}, text: ` +
+                let tags = Object.keys(this._tag).map((tag) => `#${tag}`).join(" ")
+                this._log(`INPUT: state: <${this._state[this._state.length - 1]}>, tags: <${tags}>, text: ` +
                     (e.prologTrunc ? "..." : "\"") + `${e.prologText}<${e.tokenText}>${e.epilogText}` +
@@ -181,10 +253,27 @@ (e.epilogTrunc ? "..." : "\"") + `, at: <line ${this._line}, column ${this._column}>`)
             for (let i = 0; i < this._rules.length; i++) {
-                if (this._debug)
-                    this._log(`  RULE: state(s): ${this._rules[i].state.join(",")}, ` +
+                if (this._debug) {
+                    let state = this._rules[i].state.map((item) => {
+                        let output = item.state
+                        if (item.tags.length > 0)
+                            output += " " + item.tags.map((tag) => `#${tag}`).join(" ")
+                        return output
+                    }).join(", ")
+                    this._log(`  RULE: state(s): <${state}>, ` +
                         `pattern: ${this._rules[i].pattern.source}`)
+                }
 
-                /*  one of rule's states has to match  */
-                if (!(   (   this._rules[i].state.length === 1
-                          && this._rules[i].state[0] === "*"  )
-                      || this._rules[i].state.indexOf(this._state[this._state.length - 1]) >= 0))
+                /*  one of rule's states (and all of its tags) has to match  */
+                let matches = false
+                let states = this._rules[i].state.map((item) => item.state)
+                let idx = states.indexOf("*")
+                if (idx < 0)
+                    idx = states.indexOf(this._state[this._state.length - 1])
+                if (idx >= 0) {
+                    matches = true
+                    let tags = this._rules[i].state[idx].tags
+                    tags = tags.filter((tag) => !this._tag[tag])
+                    if (tags.length > 0)
+                        matches = false
+                }
+                if (!matches)
                     continue
@@ -191,0 +280,0 @@

tst/tokenizr.js

@@ -53,10 +53,15 @@ /*
         tokenizr.rule("default", /\/\*/, function (ctx /*, m */) {
-            ctx.state("comment")
+            ctx.push("comment")
+            ctx.tag("bar")
             ctx.ignore()
         })
-        tokenizr.rule("comment", /\*\//, function (ctx /*, m */) {
-            ctx.state()
+        tokenizr.rule("comment #foo #bar", /\*\//, function (/* ctx, m */) {
+            throw new Error("should never enter")
+        })
+        tokenizr.rule("comment #bar", /\*\//, function (ctx /*, m */) {
+            ctx.untag("bar")
+            ctx.pop()
             ctx.ignore()
         })
-        tokenizr.rule("comment", /./, function (ctx /*, m */) {
+        tokenizr.rule("comment #bar", /./, function (ctx /*, m */) {
             ctx.ignore()
@@ -63,0 +68,0 @@ })

No alert changes

Improved metrics

Total package byte prevSize

75085

increased by8.68%

Lines of code

1049

increased by10.19%

Number of lines in readme file

397

increased by5.03%

No dependency changes