tokenizr - npm Package Compare versions

Comparing version 1.0.1 to 1.0.2

bower.json

 {
     "name":        "tokenizr",
-    "version":     "1.0.1",
+    "version":     "1.0.2",
     "description": "String Tokenization Library for JavaScript",
@@ -5,0 +5,0 @@ "main":        "lib/tokenizr.js",

Gruntfile.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

lib/tokenizr.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -26,13 +26,13 @@ **  Permission is hereby granted, free of charge, to any person obtaining
 (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Tokenizr = f()}})(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);var f=new Error("Cannot find module '"+o+"'");throw f.code="MODULE_NOT_FOUND",f}var l=n[o]={exports:{}};t[o][0].call(l.exports,function(e){var n=t[o][1][e];return s(n?n:e)},l,l.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(_dereq_,module,exports){
-"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var excerpt=function(e,r){var t=e.length,n=r-20;0>n&&(n=0);var u=r+20;u>t&&(u=t);var c=function(e){return e.charCodeAt(0).toString(16).toUpperCase()},o=function(e,r,t){return e.substr(r,t).replace(/\\/g,"\\\\").replace(/\x08/g,"\\b").replace(/\t/g,"\\t").replace(/\n/g,"\\n").replace(/\f/g,"\\f").replace(/\r/g,"\\r").replace(/[\x00-\x07\x0B\x0E\x0F]/g,function(e){return"\\x0"+c(e)}).replace(/[\x10-\x1F\x80-\xFF]/g,function(e){return"\\x"+c(e)}).replace(/[\u0100-\u0FFF]/g,function(e){return"\\u0"+c(e)}).replace(/[\u1000-\uFFFF]/g,function(e){return"\\u"+c(e)})};return{prologTrunc:n>0,prologText:o(e,n,r-n),tokenText:o(e,r,1),epilogText:o(e,r+1,u-(r+1)),epilogTrunc:t>u}};exports["default"]=excerpt;
+"use strict";Object.defineProperty(exports,"__esModule",{value:!0});var excerpt=function(e,r){var t=e.length,n=r-20;n<0&&(n=0);var u=r+20;u>t&&(u=t);var c=function(e){return e.charCodeAt(0).toString(16).toUpperCase()},o=function(e,r,t){return e.substr(r,t).replace(/\\/g,"\\\\").replace(/\x08/g,"\\b").replace(/\t/g,"\\t").replace(/\n/g,"\\n").replace(/\f/g,"\\f").replace(/\r/g,"\\r").replace(/[\x00-\x07\x0B\x0E\x0F]/g,function(e){return"\\x0"+c(e)}).replace(/[\x10-\x1F\x80-\xFF]/g,function(e){return"\\x"+c(e)}).replace(/[\u0100-\u0FFF]/g,function(e){return"\\u0"+c(e)}).replace(/[\u1000-\uFFFF]/g,function(e){return"\\u"+c(e)})};return{prologTrunc:n>0,prologText:o(e,n,r-n),tokenText:o(e,r,1),epilogText:o(e,r+1,u-(r+1)),epilogTrunc:u<t}};exports.default=excerpt;
 },{}],2:[function(_dereq_,module,exports){
-"use strict";function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}var _createClass=function(){function e(e,t){for(var n=0;n<t.length;n++){var s=t[n];s.enumerable=s.enumerable||!1,s.configurable=!0,"value"in s&&(s.writable=!0),Object.defineProperty(e,s.key,s)}}return function(t,n,s){return n&&e(t.prototype,n),s&&e(t,s),t}}();Object.defineProperty(exports,"__esModule",{value:!0});var Token=function(){function e(t,n,s){var r=arguments.length<=3||void 0===arguments[3]?0:arguments[3],i=arguments.length<=4||void 0===arguments[4]?0:arguments[4],a=arguments.length<=5||void 0===arguments[5]?0:arguments[5];_classCallCheck(this,e),this.type=t,this.value=n,this.text=s,this.pos=r,this.line=i,this.column=a}return _createClass(e,[{key:"toString",value:function(){return"<type: "+this.type+", "+("value: "+JSON.stringify(this.value)+", ")+("text: "+JSON.stringify(this.text)+", ")+("pos: "+this.pos+", ")+("line: "+this.line+", ")+("column: "+this.column+">")}},{key:"isA",value:function(e,t){return e!==this.type?!1:2===arguments.length&&t!==this.value?!1:!0}}]),e}();exports["default"]=Token;
+"use strict";function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var n=0;n<t.length;n++){var i=t[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(e,i.key,i)}}return function(t,n,i){return n&&e(t.prototype,n),i&&e(t,i),t}}(),Token=function(){function e(t,n,i){var s=arguments.length>3&&void 0!==arguments[3]?arguments[3]:0,l=arguments.length>4&&void 0!==arguments[4]?arguments[4]:0,r=arguments.length>5&&void 0!==arguments[5]?arguments[5]:0;_classCallCheck(this,e),this.type=t,this.value=n,this.text=i,this.pos=s,this.line=l,this.column=r}return _createClass(e,[{key:"toString",value:function(){return"<type: "+this.type+", "+("value: "+JSON.stringify(this.value)+", ")+("text: "+JSON.stringify(this.text)+", ")+("pos: "+this.pos+", ")+("line: "+this.line+", ")+("column: "+this.column+">")}},{key:"isA",value:function(e,t){return e===this.type&&(2!==arguments.length||t===this.value)}}]),e}();exports.default=Token;
 },{}],3:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{"default":e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}();Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),ParsingError=function(e){function t(e,r,n,o,i){_classCallCheck(this,t);var u=_possibleConstructorReturn(this,Object.getPrototypeOf(t).call(this,e));return u.name="ParsingError",u.message=e,u.pos=r,u.line=n,u.column=o,u.input=i,u}return _inherits(t,e),_createClass(t,[{key:"toString",value:function(){for(var e=(0,_tokenizr1Excerpt2["default"])(this.input,this.pos),t="line "+this.line+" (column "+this.column+"): ",r="",n=0;n<t.length+e.prologText.length;n++)r+=" ";var o="Parsing Error: "+this.message+"\n"+t+e.prologText+e.tokenText+e.epilogText+"\n"+r+"^";return o}}]),t}(Error);exports["default"]=ParsingError;
+"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),ParsingError=function(e){function t(e,r,n,o,i){_classCallCheck(this,t);var u=_possibleConstructorReturn(this,(t.__proto__||Object.getPrototypeOf(t)).call(this,e));return u.name="ParsingError",u.message=e,u.pos=r,u.line=n,u.column=o,u.input=i,u}return _inherits(t,e),_createClass(t,[{key:"toString",value:function(){for(var e=(0,_tokenizr1Excerpt2.default)(this.input,this.pos),t="line "+this.line+" (column "+this.column+"): ",r="",n=0;n<t.length+e.prologText.length;n++)r+=" ";var o="Parsing Error: "+this.message+"\n"+t+e.prologText+e.tokenText+e.epilogText+"\n"+r+"^";return o}}]),t}(Error);exports.default=ParsingError;
 },{"./tokenizr-1-excerpt":1}],4:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _typeof(t){return t&&"undefined"!=typeof Symbol&&t.constructor===Symbol?"symbol":typeof t}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}();Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){return this._tokenizr.push.apply(this._tokenizr,arguments),this}},{key:"pop",value:function(){return this._tokenizr.pop.apply(this._tokenizr,arguments)}},{key:"state",value:function(){return arguments.length>0?(this._tokenizr.state.apply(this._tokenizr,arguments),this):this._tokenizr.state.apply(this._tokenizr,arguments)}},{key:"tag",value:function(){return this._tokenizr.tag.apply(this._tokenizr,arguments),this}},{key:"tagged",value:function(){return this._tokenizr.tagged.apply(this._tokenizr,arguments)}},{key:"untag",value:function(){return this._tokenizr.untag.apply(this._tokenizr,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2["default"](t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports["default"]=ActionContext;
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{default:t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},_createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){return this._tokenizr.push.apply(this._tokenizr,arguments),this}},{key:"pop",value:function(){return this._tokenizr.pop.apply(this._tokenizr,arguments)}},{key:"state",value:function(){return arguments.length>0?(this._tokenizr.state.apply(this._tokenizr,arguments),this):this._tokenizr.state.apply(this._tokenizr,arguments)}},{key:"tag",value:function(){return this._tokenizr.tag.apply(this._tokenizr,arguments),this}},{key:"tagged",value:function(){return this._tokenizr.tagged.apply(this._tokenizr,arguments)}},{key:"untag",value:function(){return this._tokenizr.untag.apply(this._tokenizr,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2.default(t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports.default=ActionContext;
 },{"./tokenizr-2-token":2}],5:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _typeof(t){return t&&"undefined"!=typeof Symbol&&t.constructor===Symbol?"symbol":typeof t}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r)}}return function(e,n,r){return n&&t(e.prototype,n),r&&t(e,r),e}}();Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2["default"](this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+this._state[this._state.length-2]+">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._tag[t]===!0}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"rule",value:function(t,e,n){if(2===arguments.length){var r=[t,e];e=r[0],n=r[1],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===("undefined"==typeof e?"undefined":_typeof(e))&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var i="g";return e.multiline&&(i+="m"),e.ignoreCase&&(i+="i"),e=new RegExp(e.source,i),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;e>o;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"	"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+r+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){var t=this;if(this._pos>=this._len)return void(this._eof||(this._eof=!0,this._pending.push(new _tokenizr2Token2["default"]("EOF","","",this._pos,this._line,this._column))));for(var e=!0;e;){if(e=!1,this._debug){var n=(0,_tokenizr1Excerpt2["default"])(this._input,this._pos),r=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+r+">, text: "+(n.prologTrunc?"...":'"')+(n.prologText+"<"+n.tokenText+">"+n.epilogText)+(n.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var i=0;i<this._rules.length;i++){if(this._debug){var o=this._rules[i].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+o+">, pattern: "+this._rules[i].pattern.source)}var s=!1,a=this._rules[i].state.map(function(t){return t.state}),h=a.indexOf("*");if(0>h&&(h=a.indexOf(this._state[this._state.length-1])),h>=0){s=!0;var r=this._rules[i].state[h].tags;r=r.filter(function(e){return!t._tag[e]}),r.length>0&&(s=!1)}if(s){this._rules[i].pattern.lastIndex=this._pos;var u=this._rules[i].pattern.exec(this._input);if(this._rules[i].pattern.lastIndex=this._pos,null!==(u=this._rules[i].pattern.exec(this._input))&&u.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(u)),this._ctx._match=u,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,this._rules[i].action.call(this._ctx,this._ctx,u),this._ctx._reject)continue;if(this._ctx._repeat){e=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[i].pattern.lastIndex),this._pos=this._rules[i].pattern.lastIndex,this._pos>=this._len)return;e=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[i].pattern.lastIndex),void(this._pos=this._rules[i].pattern.lastIndex);throw new Error('action of pattern "'+this._rules[i].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: "+t.toString()),t}return null}},{key:"tokens",value:function(){for(var t=[],e=void 0;null!==(e=this.token());)t.push(e);return t}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var r=this.token();if(this._log("CONSUME: "+r.toString()),2===arguments.length){if(!r.isA(t,e))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+")>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+_typeof(r.value)+")>"),r.pos,r.line,r.column,this._input)}else if(!r.isA(t))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+_typeof(r.value)+")>"),r.pos,r.line,r.column,this._input);return r}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=Array(n),i=0;n>i;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o](),this.commit();break}catch(s){this._log("EXCEPTION: "+s.toString()),e.push({ex:s,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2["default"](t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2["default"],Tokenizr.ParsingError=_tokenizr3Error2["default"],Tokenizr.ActionContext=_tokenizr4Context2["default"],exports["default"]=Tokenizr;
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{default:t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},_createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r)}}return function(e,n,r){return n&&t(e.prototype,n),r&&t(e,r),e}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2.default(this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+this._state[this._state.length-2]+">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._tag[t]===!0}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"rule",value:function(t,e,n){if(2===arguments.length){var r=[t,e];e=r[0],n=r[1],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===("undefined"==typeof e?"undefined":_typeof(e))&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var i="g";return e.multiline&&(i+="m"),e.ignoreCase&&(i+="i"),e=new RegExp(e.source,i),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;o<e;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+r+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){var t=this;if(this._pos>=this._len)return void(this._eof||(this._eof=!0,this._pending.push(new _tokenizr2Token2.default("EOF","","",this._pos,this._line,this._column))));for(var e=!0;e;){if(e=!1,this._debug){var n=(0,_tokenizr1Excerpt2.default)(this._input,this._pos),r=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+r+">, text: "+(n.prologTrunc?"...":'"')+(n.prologText+"<"+n.tokenText+">"+n.epilogText)+(n.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var i=0;i<this._rules.length;i++){if(this._debug){var o=this._rules[i].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+o+">, pattern: "+this._rules[i].pattern.source)}var s=!1,a=this._rules[i].state.map(function(t){return t.state}),h=a.indexOf("*");if(h<0&&(h=a.indexOf(this._state[this._state.length-1])),h>=0){s=!0;var u=this._rules[i].state[h].tags;u=u.filter(function(e){return!t._tag[e]}),u.length>0&&(s=!1)}if(s){this._rules[i].pattern.lastIndex=this._pos;var l=this._rules[i].pattern.exec(this._input);if(this._rules[i].pattern.lastIndex=this._pos,null!==(l=this._rules[i].pattern.exec(this._input))&&l.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(l)),this._ctx._match=l,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,this._rules[i].action.call(this._ctx,this._ctx,l),this._ctx._reject)continue;if(this._ctx._repeat){e=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[i].pattern.lastIndex),this._pos=this._rules[i].pattern.lastIndex,this._pos>=this._len)return;e=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[i].pattern.lastIndex),void(this._pos=this._rules[i].pattern.lastIndex);throw new Error('action of pattern "'+this._rules[i].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function t(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: "+t.toString()),t}return null}},{key:"tokens",value:function(){for(var t=[],e=void 0;null!==(e=this.token());)t.push(e);return t}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var r=this.token();if(this._log("CONSUME: "+r.toString()),2===arguments.length){if(!r.isA(t,e))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+")>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+_typeof(r.value)+")>"),r.pos,r.line,r.column,this._input)}else if(!r.isA(t))throw new _tokenizr3Error2.default("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+_typeof(r.value)+")>"),r.pos,r.line,r.column,this._input);return r}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=Array(n),i=0;i<n;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: "+t.toString()),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2.default(t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2.default,Tokenizr.ParsingError=_tokenizr3Error2.default,Tokenizr.ActionContext=_tokenizr4Context2.default,exports.default=Tokenizr;
 },{"./tokenizr-1-excerpt":1,"./tokenizr-2-token":2,"./tokenizr-3-error":3,"./tokenizr-4-context":4}],6:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{"default":e}}var _tokenizr5Tokenizer=_dereq_("./tokenizr-5-tokenizer"),_tokenizr5Tokenizer2=_interopRequireDefault(_tokenizr5Tokenizer);module.exports=_tokenizr5Tokenizer2["default"];
+"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{default:e}}var _tokenizr5Tokenizer=_dereq_("./tokenizr-5-tokenizer"),_tokenizr5Tokenizer2=_interopRequireDefault(_tokenizr5Tokenizer);module.exports=_tokenizr5Tokenizer2.default;
 },{"./tokenizr-5-tokenizer":5}]},{},[6])(6)
@@ -39,0 +39,0 @@ });

package.json

 {
     "name":        "tokenizr",
-    "version":     "1.0.1",
+    "version":     "1.0.2",
     "description": "String Tokenization Library for JavaScript",
@@ -20,17 +20,20 @@ "keywords":    [ "string", "token", "scanner", "lexer" ],
     "devDependencies": {
-        "grunt":                "~0.4.5",
-        "grunt-cli":            "~0.1.13",
-        "grunt-contrib-jshint": "~0.11.3",
-        "grunt-contrib-clean":  "~0.7.0",
-        "grunt-browserify":     "~4.0.1",
-        "grunt-mocha-test":     "~0.12.7",
-        "mocha":                "~2.3.4",
-        "chai":                 "~3.4.1",
-        "browserify":           "~12.0.1",
-        "babelify":             "~7.2.0",
-        "babel-preset-es2015":  "~6.3.13",
-        "minifyify":            "~7.1.0",
+        "grunt":                "~1.0.1",
+        "grunt-cli":            "~1.2.0",
+        "grunt-contrib-jshint": "~1.0.0",
+        "grunt-contrib-clean":  "~1.0.0",
+        "grunt-browserify":     "~5.0.0",
+        "grunt-mocha-test":     "~0.13.2",
+        "mocha":                "~3.1.2",
+        "chai":                 "~3.5.0",
+        "browserify":           "~13.1.0",
+        "babelify":             "~7.3.0",
+        "babel-preset-es2015":  "~6.16.0",
+        "minifyify":            "~7.3.3",
         "browserify-header":    "~0.9.2",
         "browserify-derequire": "~0.9.4"
+    },
+    "scripts": {
+        "sample": "babel-node --presets es2015 smp/sample.js"
     }
 }

README.md

@@ -5,3 +5,3 @@
 
-String Tokenization Library for JavaScript
+Flexible String Tokenization Library for JavaScript
 
@@ -17,6 +17,7 @@ <p/>
 
-Tokenizr is a small but flexible JavaScript library, providing string
-tokenization functionality. It is intended to be be used as the
-underlying "lexical scanner" in a Recursive Descent based "syntax
-parser". Its distinct features are:
+Tokenizr is a small JavaScript library, providing powerful and flexible
+string tokenization functionality. It is intended to be be used as
+the underlying "lexical scanner" in a Recursive Descent based "syntax
+parser", but can be used for other parsing purposes, too. Its distinct
+features are:
 
@@ -31,6 +32,6 @@ - **Efficient Iteration**:<br/>
 - **Regular Expression Matching**:<br/>
-  Its tokenization is based on Regular Expressions for matching the input string.
+  Its tokenization is based on powerful Regular Expressions for matching the input string.
 
 - **Match Repeating**:<br/>
-  Rule actions can (change the state and then) force the repeating of
+  Rule actions can change the state and then force the repeating of
   the matching process from scratch at the current input position.
@@ -68,4 +69,4 @@
 - **Token Look-Ahead**:<br/>
-  The forthcoming tokens can be inspected to support alternative decisions
-  from within the parser based on look-ahead tokens.
+  The forthcoming tokens can be inspected, to support alternative decisions
+  from within the parser, based on look-ahead tokens.
 
@@ -142,20 +143,20 @@ Installation
 ```
-<type: id, value: "foo", text: "foo", pos: 5, line: 2, column: 5>
-<type: char, value: "{", text: "{", pos: 9, line: 2, column: 9>
-<type: id, value: "baz", text: "baz", pos: 19, line: 3, column: 9>
-<type: char, value: "=", text: "=", pos: 23, line: 3, column: 13>
-<type: number, value: 1, text: "1", pos: 25, line: 3, column: 15>
-<type: id, value: "bar", text: "bar", pos: 53, line: 4, column: 9>
-<type: char, value: "{", text: "{", pos: 57, line: 4, column: 13>
-<type: id, value: "quux", text: "quux", pos: 71, line: 5, column: 13>
-<type: char, value: "=", text: "=", pos: 76, line: 5, column: 18>
-<type: number, value: 42, text: "42", pos: 78, line: 5, column: 20>
-<type: id, value: "hello", text: "hello", pos: 93, line: 6, column: 13>
-<type: char, value: "=", text: "=", pos: 99, line: 6, column: 19>
-<type: string, value: "hello \"world\"!", text: "hello "world"!"", pos: 101, line: 6, column: 21>
-<type: char, value: "}", text: "}", pos: 126, line: 7, column: 9>
-<type: id, value: "quux", text: "quux", pos: 136, line: 8, column: 9>
-<type: char, value: "=", text: "=", pos: 141, line: 8, column: 14>
-<type: number, value: 7, text: "7", pos: 143, line: 8, column: 16>
-<type: char, value: "}", text: "}", pos: 149, line: 9, column: 5>
+<type: id, value: "foo", text: "foo", pos: 0, line: 1, column: 1>
+<type: char, value: "{", text: "{", pos: 4, line: 1, column: 5>
+<type: id, value: "baz", text: "baz", pos: 10, line: 2, column: 5>
+<type: char, value: "=", text: "=", pos: 14, line: 2, column: 9>
+<type: number, value: 1, text: "1", pos: 16, line: 2, column: 11>
+<type: id, value: "bar", text: "bar", pos: 40, line: 3, column: 5>
+<type: char, value: "{", text: "{", pos: 44, line: 3, column: 9>
+<type: id, value: "quux", text: "quux", pos: 54, line: 4, column: 9>
+<type: char, value: "=", text: "=", pos: 59, line: 4, column: 14>
+<type: number, value: 42, text: "42", pos: 61, line: 4, column: 16>
+<type: id, value: "hello", text: "hello", pos: 72, line: 5, column: 9>
+<type: char, value: "=", text: "=", pos: 78, line: 5, column: 15>
+<type: string, value: "hello \"world\"!", text: "\"hello \\\"world\\\"!\"", pos: 80, line: 5, column: 17>
+<type: char, value: "}", text: "}", pos: 103, line: 6, column: 5>
+<type: id, value: "quux", text: "quux", pos: 109, line: 7, column: 5>
+<type: char, value: "=", text: "=", pos: 114, line: 7, column: 10>
+<type: number, value: 7, text: "7", pos: 116, line: 7, column: 12>
+<type: char, value: "}", text: "}", pos: 118, line: 8, column: 1>
 ```
@@ -166,3 +167,3 @@
 you can [write powerful Recursive Descent based parsers](https://github.com/rse/parsing-techniques/blob/master/cfg2kv-3-ls-rdp-ast/cfg2kv.js)
-which parse input into an AST and then process the AST.
+which parse such a token stream into an AST and then query and process the AST.
 
@@ -180,3 +181,4 @@ Application Programming Interface (API)
 - Method: `Tokenizr#reset(): Tokenizr`<br/>
-  Reset the tokenization instance to a fresh one.
+  Reset the tokenization instance to a fresh one by
+  discarding all internal state information.
 
@@ -195,3 +197,3 @@ - Method: `Tokenizr#debug(enable: Boolean): Tokenizr`<br/>
   Pop a state from the state stack.
-  The initial/first/lowest stack value cannot be popped.
+  The initial (aka first or lowest) stack value (`default`) cannot be popped.
 
@@ -208,14 +210,19 @@ - Method: `Tokenizr#state(state: String): Tokenizr`<br/>
 - Method: `Tokenizr#tagged(tag: String): Boolean`<br/>
-  Check whether a tag is set.
+  Check whether a particular tag is set.
 
 - Method: `Tokenizr#untag(tag: String): Tokenizr`<br/>
-  Unset a tag. The tag no longer has to be matched by rules.
+  Unset a particular tag. The tag no longer will be matched by rules.
 
 - Method: `Tokenizr#rule(state?: String, pattern: RegExp, action: (ctx: TokenizerContext, match: Array[String]) => Void): Tokenizr`<br/>
-  Configure a token matching rule which executes its `action` in case the
-  current tokenization state is one of the states in the comma-separated `state` (by default
-  the rule matches all states if `state` is not specified) and the
-  next input characters match against the `pattern`. The `ctx` argument provides
-  a context object for token repeating/rejecting/ignoring/accepting, the
-  `match` argument is the result of the underlying `RegExp#exec` call.
+  Configure a token matching rule which executes its `action` in case
+  the current tokenization state is one of the states (and all of the
+  currently set tags) in `state` (by default the rule matches all states
+  if `state` is not specified) and the next input characters match
+  against the `pattern`. The exact syntax of `state` is
+  `<state>[ #<tag> #<tag> ...][, <state>[ #<tag> #<tag> ...], ...]`, i.e.,
+  it is one or more comma-separated state matches (OR-combined) and each state
+  match has exactly one state and zero or more space-separated tags
+  (AND-combined). The `ctx` argument provides a context object for token
+  repeating/rejecting/ignoring/accepting, the `match` argument is the
+  result of the underlying `RegExp#exec` call.
 
@@ -230,5 +237,14 @@ - Method: `Tokenizr#token(): Tokenizr.Token`<br/>
   Tokenizes the entire input and returns all the corresponding tokens.
-  This is a convenience method only. Usually one takes single
-  tokens at a time.
+  This is a convenience method only. Usually one takes just single
+  tokens at a time with `Tokenizr#token()`.
 
+- Method: `Tokenizr#skip(next?: Number): Tokenizr`<br/>
+  Get and discard the `next` number of following tokens with `Tokenizr#token()`.
+
+- Method: `Tokenizr#consume(type: String, value?: String): Tokenizr`<br/>
+  Match (with `Tokenizr.Token#isA`) the next token. If it matches
+  `type` and optionally also `value`, consume it. If it does not match,
+  throw a `Tokenizr.ParsingError`. This is the primary function used in
+  Recursive Descent parsers.
+
 - Method: `Tokenizr#peek(offset?: Number): Tokenizr.Token`<br/>
@@ -239,15 +255,9 @@ Peek at the following token at the (0-based) offset without consuming
 
-- Method: `Tokenizr#skip(next?: Number): Tokenizr`<br/>
-  Consume the `next` number of following tokens.
-
-- Method: `Tokenizr#consume(type: String, value?: String): Tokenizr`<br/>
-  Match (with `Tokenizr.Token#isA`) the next token. If it matches,
-  consume it. If it does not match, throw a `Tokenizr.ParsingError`.
-  This is the primary function used in Recursive Descent parsers.
-
 - Method: `Tokenizr#begin(): Tokenizr`<br/>
-  Begin a transaction. Until `commit` or `rollback` are called, all
-  consumed tokens will be internally remembered and be either thrown
-  away (on `commit`) or pushed back (on `rollback`). This can be used
-  multiple times and supports nested transactions.
+  Begin a transaction. Until `Tokenizr#commit()` or
+  `Tokenizr#rollback()` are called, all consumed tokens will
+  be internally remembered and be either thrown away (on
+  `Tokenizr#commit()`) or pushed back (on `Tokenizr#rollback()`). This
+  can be used multiple times and this way supports nested transactions.
+  It is intended to be used for tokenizing alternatives.
 
@@ -258,6 +268,6 @@ - Method: `Tokenizr#depth(): Number`<br/>
   are parsed and in case all failed, to report the error for
-  the most specific, i.e., the one which consumed most tokens.
+  the most specific one, i.e., the one which consumed most tokens.
 
 - Method: `Tokenizr#commit(): Tokenizr`<br/>
-  End a transaction successfully. All consumed tokens are thrown away.
+  End a transaction successfully. All consumed tokens are finally gone.
 
@@ -275,7 +285,9 @@ - Method: `Tokenizr#rollback(): Tokenizr`<br/>
   most-specific alterative (the one with the largest transaction depth)
-  is re-thrown.
+  is re-thrown. The `this` in each callback function points to the
+  `Tokenizr` object on which `alternatives` was called.
 
 - Method: `Tokenizr#error(message: String): Tokenizr.ParsingError`<br/>
-  Returns a new instance of `Tokenizr.ParsingError`, based on the
-  current input character stream position.
+  Returns a new instance of `Tokenizr.ParsingError`, based
+  on the current input character stream position, and with
+  `Tokenizr.ParsingError#message` set to `message`.
 
@@ -287,7 +299,8 @@ ### Class `Tokenizr.Token`
 - Property: `Tokenizr.Token#type: String`<br/>
-  The type of the token.
+  The type of the token as specified on `Tokenizr.ActionContext#accept()`.
 
 - Property: `Tokenizr.Token#value: any`<br/>
   The value of the token. By default this is the same as
-  `Tokenizr.Token#text`, but can be any pre-processed value.
+  `Tokenizr.Token#text`, but can be any pre-processed value
+  as specified on `Tokenizr.ActionContext#accept()`.
 
@@ -307,7 +320,9 @@ - Property: `Tokenizr.Token#text: String`<br/>
 - Method: `Tokenizr.Token#toString(): String`<br/>
-  Returns a formatted representation of the token.
+  Returns a formatted representation of the token,
+  usually for debugging or tracing purposes only.
 
 - Method: `Tokenizr.Token#isA(type: String, value?: any): String`<br/>
-  Checks whether token matches against a particular type and optionally
-  a particular value.
+  Checks whether token matches against a particular `type` and optionally
+  a particular `value`. This is especially used internally by
+  `Tokenizr#consume()`.
 
@@ -335,6 +350,7 @@ ### Class `Tokenizr.ParsingError`
 - Property: `Tokenizr.ParsingError#input: String`<br/>
-  The input itself.
+  The total input itself.
 
 - Method: `Tokenizr.ParsingError#toString(): String`<br/>
-  Returns a formatted representation of the error.
+  Returns a formatted representation of the error,
+  usually for convenient error displaying purposes.
 
@@ -347,3 +363,3 @@ ### Class `Tokenizr.ActionContext`
   Store or retrieve any user data (indexed by `key`) to the action
-  context for sharing data between rules.
+  context for sharing data between two or more rules.
 
@@ -360,3 +376,3 @@ - Method: `Tokenizr.ActionContext#info(): { line: number, column: number, pos: number, len: number }`<br/>
   Method: `Tokenizr.ActionContext#untag(tag: String): Tokenizr.ActionContext`<br/>
-  Methods just passed-through to the attached Tokenizr. See above for details.
+  Methods just passed-through to the attached `Tokenizr` object. See above for details.
 
@@ -366,3 +382,3 @@ - Method: `Tokenizr.ActionContext#repeat(): Tokenizr.ActionContext`<br/>
   input position from scratch. You first have to switch to a different
-  state or this will lead to an endless loop, of course.
+  state with `Tokenizr.ActionContext#state()` or this will lead to an endless loop, of course!
 
@@ -381,3 +397,3 @@ - Method: `Tokenizr.ActionContext#reject(): Tokenizr.ActionContext`<br/>
   of the matched text). This function can be called multiple times to
-  produce one or more tokens.
+  produce one or more distinct tokens in sequence.
 
@@ -400,3 +416,3 @@ Implementation Notice
 
-Copyright (c) 2015 Ralf S. Engelschall (http://engelschall.com/)
+Copyright (c) 2015-2016 Ralf S. Engelschall (http://engelschall.com/)
 
@@ -403,0 +419,0 @@ Permission is hereby granted, free of charge, to any person obtaining

src/tokenizr-1-excerpt.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

src/tokenizr-2-token.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

src/tokenizr-3-error.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

src/tokenizr-4-context.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

src/tokenizr-5-tokenizer.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -454,3 +454,3 @@ **  Permission is hereby granted, free of charge, to any person obtaining
                 this.begin()
-                result = alternatives[i]()
+                result = alternatives[i].call(this)
                 this.commit()
@@ -457,0 +457,0 @@ break

src/tokenizr.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

tst/common.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

tst/tokenizr.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015 Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2016 Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

New alerts

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

78300

increased by3.2%

Number of package files

18

increased by12.5%

Lines of code

1090

increased by2.64%

Number of lines in readme file

417

increased by3.99%

Worsened metrics

Number of low supply chain risk alerts

3

increased by50%

No dependency changes