tokenizr - npm Package Compare versions

Comparing version 0.9.9 to 1.0.0

bower.json

 {
     "name":        "tokenizr",
-    "version":     "0.9.9",
+    "version":     "1.0.0",
     "description": "String Tokenization Library for JavaScript",
@@ -5,0 +5,0 @@ "main":        "lib/tokenizr.js",

lib/tokenizr.js

@@ -28,11 +28,11 @@ /*
 },{}],2:[function(_dereq_,module,exports){
-"use strict";function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}var _createClass=function(){function e(e,t){for(var n=0;n<t.length;n++){var r=t[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(e,r.key,r)}}return function(t,n,r){return n&&e(t.prototype,n),r&&e(t,r),t}}();Object.defineProperty(exports,"__esModule",{value:!0});var Token=function(){function e(t,n,r){var s=arguments.length<=3||void 0===arguments[3]?0:arguments[3],a=arguments.length<=4||void 0===arguments[4]?0:arguments[4];_classCallCheck(this,e);var i=arguments.length<=5||void 0===arguments[5]?0:arguments[5];this.type=t,this.value=n,this.text=r,this.pos=s,this.line=a,this.column=i}return _createClass(e,[{key:"toString",value:function(){return"<type: "+this.type+", "+("value: "+JSON.stringify(this.value)+", ")+("text: "+JSON.stringify(this.text)+", ")+("pos: "+this.pos+", ")+("line: "+this.line+", ")+("column: "+this.column+">")}},{key:"isA",value:function(e,t){return e!==this.type?!1:2===arguments.length&&t!==this.value?!1:!0}}]),e}();exports["default"]=Token;
+"use strict";function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}var _createClass=function(){function e(e,t){for(var n=0;n<t.length;n++){var s=t[n];s.enumerable=s.enumerable||!1,s.configurable=!0,"value"in s&&(s.writable=!0),Object.defineProperty(e,s.key,s)}}return function(t,n,s){return n&&e(t.prototype,n),s&&e(t,s),t}}();Object.defineProperty(exports,"__esModule",{value:!0});var Token=function(){function e(t,n,s){var r=arguments.length<=3||void 0===arguments[3]?0:arguments[3],i=arguments.length<=4||void 0===arguments[4]?0:arguments[4],a=arguments.length<=5||void 0===arguments[5]?0:arguments[5];_classCallCheck(this,e),this.type=t,this.value=n,this.text=s,this.pos=r,this.line=i,this.column=a}return _createClass(e,[{key:"toString",value:function(){return"<type: "+this.type+", "+("value: "+JSON.stringify(this.value)+", ")+("text: "+JSON.stringify(this.text)+", ")+("pos: "+this.pos+", ")+("line: "+this.line+", ")+("column: "+this.column+">")}},{key:"isA",value:function(e,t){return e!==this.type?!1:2===arguments.length&&t!==this.value?!1:!0}}]),e}();exports["default"]=Token;
 },{}],3:[function(_dereq_,module,exports){
 "use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{"default":e}}function _classCallCheck(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}function _possibleConstructorReturn(e,t){if(!e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return!t||"object"!=typeof t&&"function"!=typeof t?e:t}function _inherits(e,t){if("function"!=typeof t&&null!==t)throw new TypeError("Super expression must either be null or a function, not "+typeof t);e.prototype=Object.create(t&&t.prototype,{constructor:{value:e,enumerable:!1,writable:!0,configurable:!0}}),t&&(Object.setPrototypeOf?Object.setPrototypeOf(e,t):e.__proto__=t)}var _createClass=function(){function e(e,t){for(var r=0;r<t.length;r++){var n=t[r];n.enumerable=n.enumerable||!1,n.configurable=!0,"value"in n&&(n.writable=!0),Object.defineProperty(e,n.key,n)}}return function(t,r,n){return r&&e(t.prototype,r),n&&e(t,n),t}}();Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),ParsingError=function(e){function t(e,r,n,o,i){_classCallCheck(this,t);var u=_possibleConstructorReturn(this,Object.getPrototypeOf(t).call(this,e));return u.name="ParsingError",u.message=e,u.pos=r,u.line=n,u.column=o,u.input=i,u}return _inherits(t,e),_createClass(t,[{key:"toString",value:function(){for(var e=(0,_tokenizr1Excerpt2["default"])(this.input,this.pos),t="line "+this.line+" (column "+this.column+"): ",r="",n=0;n<t.length+e.prologText.length;n++)r+=" ";var o="Parsing Error: "+this.message+"\n"+t+e.prologText+e.tokenText+e.epilogText+"\n"+r+"^";return o}}]),t}(Error);exports["default"]=ParsingError;
 },{"./tokenizr-1-excerpt":1}],4:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function _typeof(t){return t&&"undefined"!=typeof Symbol&&t.constructor===Symbol?"symbol":typeof t}var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}();Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){return this._tokenizr.push.apply(this._tokenizr,arguments),this}},{key:"pop",value:function(){return this._tokenizr.pop.apply(this._tokenizr,arguments)}},{key:"state",value:function(){return arguments.length>0?(this._tokenizr.state.apply(this._tokenizr,arguments),this):this._tokenizr.state.apply(this._tokenizr,arguments)}},{key:"tag",value:function(){return this._tokenizr.tag.apply(this._tokenizr,arguments),this}},{key:"tagged",value:function(){return this._tokenizr.tagged.apply(this._tokenizr,arguments)}},{key:"untag",value:function(){return this._tokenizr.untag.apply(this._tokenizr,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2["default"](t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports["default"]=ActionContext;
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _typeof(t){return t&&"undefined"!=typeof Symbol&&t.constructor===Symbol?"symbol":typeof t}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}();Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){return this._tokenizr.push.apply(this._tokenizr,arguments),this}},{key:"pop",value:function(){return this._tokenizr.pop.apply(this._tokenizr,arguments)}},{key:"state",value:function(){return arguments.length>0?(this._tokenizr.state.apply(this._tokenizr,arguments),this):this._tokenizr.state.apply(this._tokenizr,arguments)}},{key:"tag",value:function(){return this._tokenizr.tag.apply(this._tokenizr,arguments),this}},{key:"tagged",value:function(){return this._tokenizr.tagged.apply(this._tokenizr,arguments)}},{key:"untag",value:function(){return this._tokenizr.untag.apply(this._tokenizr,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2["default"](t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports["default"]=ActionContext;
 },{"./tokenizr-2-token":2}],5:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _typeof(t){return t&&"undefined"!=typeof Symbol&&t.constructor===Symbol?"symbol":typeof t}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r)}}return function(e,n,r){return n&&t(e.prototype,n),r&&t(e,r),e}}();Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2["default"](this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+this._state[this._state.length-2]+">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._tag[t]===!0}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"rule",value:function(t,e,n){if(2===arguments.length){var r=[t,e];e=r[0],n=r[1],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===("undefined"==typeof e?"undefined":_typeof(e))&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var i="g";return e.multiline&&(i+="m"),e.ignoreCase&&(i+="i"),e=new RegExp(e.source,i),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;e>o;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"	"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+r+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){var t=this;if(this._pos>=this._len)return void(this._eof||(this._eof=!0,this._pending.push(new _tokenizr2Token2["default"]("EOF","","",this._pos,this._line,this._column))));for(var e=!0;e;){if(e=!1,this._debug){var n=(0,_tokenizr1Excerpt2["default"])(this._input,this._pos),r=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+r+">, text: "+(n.prologTrunc?"...":'"')+(n.prologText+"<"+n.tokenText+">"+n.epilogText)+(n.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var i=0;i<this._rules.length;i++){if(this._debug){var o=this._rules[i].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+o+">, pattern: "+this._rules[i].pattern.source)}var s=!1,a=this._rules[i].state.map(function(t){return t.state}),h=a.indexOf("*");if(0>h&&(h=a.indexOf(this._state[this._state.length-1])),h>=0){s=!0;var r=this._rules[i].state[h].tags;r=r.filter(function(e){return!t._tag[e]}),r.length>0&&(s=!1)}if(s){this._rules[i].pattern.lastIndex=this._pos;var u=this._rules[i].pattern.exec(this._input);if(this._rules[i].pattern.lastIndex=this._pos,null!==(u=this._rules[i].pattern.exec(this._input))&&u.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(u)),this._ctx._match=u,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,this._rules[i].action.call(this._ctx,this._ctx,u),this._ctx._reject)continue;if(this._ctx._repeat){e=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[i].pattern.lastIndex),this._pos=this._rules[i].pattern.lastIndex,this._pos>=this._len)return;e=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[i].pattern.lastIndex),void(this._pos=this._rules[i].pattern.lastIndex);throw new Error('action of pattern "'+this._rules[i].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: "+t.toString()),t}return null}},{key:"tokens",value:function(){for(var t=[],e=void 0;null!==(e=this.token());)t.push(e);return t}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-->0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var r=this.token();if(this._log("CONSUME: "+r.toString()),2===arguments.length){if(!r.isA(t,e))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+")>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+_typeof(r.value)+")>"),r.pos,r.line,r.column,this._input)}else if(!r.isA(t))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+_typeof(r.value)+")>"),r.pos,r.line,r.column,this._input);return r}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=Array(n),i=0;n>i;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o](),this.commit();break}catch(s){this._log("EXCEPTION: "+s.toString()),e.push({ex:s,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2["default"](t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2["default"],Tokenizr.ParsingError=_tokenizr3Error2["default"],Tokenizr.ActionContext=_tokenizr4Context2["default"],exports["default"]=Tokenizr;
+"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _typeof(t){return t&&"undefined"!=typeof Symbol&&t.constructor===Symbol?"symbol":typeof t}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r)}}return function(e,n,r){return n&&t(e.prototype,n),r&&t(e,r),e}}();Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2["default"](this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+this._state[this._state.length-2]+">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+("old: <"+this._state[this._state.length-1]+">, ")+("new: <"+t+">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): "+t),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._tag[t]===!0}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): "+t),delete this._tag[t],this}},{key:"rule",value:function(t,e,n){if(2===arguments.length){var r=[t,e];e=r[0],n=r[1],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===("undefined"==typeof e?"undefined":_typeof(e))&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var i="g";return e.multiline&&(i+="m"),e.ignoreCase&&(i+="i"),e=new RegExp(e.source,i),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;e>o;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"	"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+r+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){var t=this;if(this._pos>=this._len)return void(this._eof||(this._eof=!0,this._pending.push(new _tokenizr2Token2["default"]("EOF","","",this._pos,this._line,this._column))));for(var e=!0;e;){if(e=!1,this._debug){var n=(0,_tokenizr1Excerpt2["default"])(this._input,this._pos),r=Object.keys(this._tag).map(function(t){return"#"+t}).join(" ");this._log("INPUT: state: <"+this._state[this._state.length-1]+">, tags: <"+r+">, text: "+(n.prologTrunc?"...":'"')+(n.prologText+"<"+n.tokenText+">"+n.epilogText)+(n.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var i=0;i<this._rules.length;i++){if(this._debug){var o=this._rules[i].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#"+t}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <"+o+">, pattern: "+this._rules[i].pattern.source)}var s=!1,a=this._rules[i].state.map(function(t){return t.state}),h=a.indexOf("*");if(0>h&&(h=a.indexOf(this._state[this._state.length-1])),h>=0){s=!0;var r=this._rules[i].state[h].tags;r=r.filter(function(e){return!t._tag[e]}),r.length>0&&(s=!1)}if(s){this._rules[i].pattern.lastIndex=this._pos;var u=this._rules[i].pattern.exec(this._input);if(this._rules[i].pattern.lastIndex=this._pos,null!==(u=this._rules[i].pattern.exec(this._input))&&u.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(u)),this._ctx._match=u,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,this._rules[i].action.call(this._ctx,this._ctx,u),this._ctx._reject)continue;if(this._ctx._repeat){e=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[i].pattern.lastIndex),this._pos=this._rules[i].pattern.lastIndex,this._pos>=this._len)return;e=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[i].pattern.lastIndex),void(this._pos=this._rules[i].pattern.lastIndex);throw new Error('action of pattern "'+this._rules[i].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: "+t.toString()),t}return null}},{key:"tokens",value:function(){for(var t=[],e=void 0;null!==(e=this.token());)t.push(e);return t}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var r=this.token();if(this._log("CONSUME: "+r.toString()),2===arguments.length){if(!r.isA(t,e))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+("undefined"==typeof e?"undefined":_typeof(e))+")>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+_typeof(r.value)+")>"),r.pos,r.line,r.column,this._input)}else if(!r.isA(t))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+r.type+", value: "+JSON.stringify(r.value)+" ("+_typeof(r.value)+")>"),r.pos,r.line,r.column,this._input);return r}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=Array(n),i=0;n>i;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o](),this.commit();break}catch(s){this._log("EXCEPTION: "+s.toString()),e.push({ex:s,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2["default"](t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2["default"],Tokenizr.ParsingError=_tokenizr3Error2["default"],Tokenizr.ActionContext=_tokenizr4Context2["default"],exports["default"]=Tokenizr;
 },{"./tokenizr-1-excerpt":1,"./tokenizr-2-token":2,"./tokenizr-3-error":3,"./tokenizr-4-context":4}],6:[function(_dereq_,module,exports){
-"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{"default":e}}Object.defineProperty(exports,"__esModule",{value:!0});var _tokenizr5Tokenizer=_dereq_("./tokenizr-5-tokenizer"),_tokenizr5Tokenizer2=_interopRequireDefault(_tokenizr5Tokenizer);exports["default"]=_tokenizr5Tokenizer2["default"];
+"use strict";function _interopRequireDefault(e){return e&&e.__esModule?e:{"default":e}}var _tokenizr5Tokenizer=_dereq_("./tokenizr-5-tokenizer"),_tokenizr5Tokenizer2=_interopRequireDefault(_tokenizr5Tokenizer);module.exports=_tokenizr5Tokenizer2["default"];
 },{"./tokenizr-5-tokenizer":5}]},{},[6])(6)
@@ -39,0 +39,0 @@ });

package.json

 {
     "name":        "tokenizr",
-    "version":     "0.9.9",
+    "version":     "1.0.0",
     "description": "String Tokenization Library for JavaScript",
@@ -23,10 +23,10 @@ "keywords":    [ "string", "token", "scanner", "lexer" ],
         "grunt-contrib-jshint": "~0.11.3",
-        "grunt-contrib-clean":  "~0.6.0",
+        "grunt-contrib-clean":  "~0.7.0",
         "grunt-browserify":     "~4.0.1",
         "grunt-mocha-test":     "~0.12.7",
-        "mocha":                "~2.3.3",
+        "mocha":                "~2.3.4",
         "chai":                 "~3.4.1",
         "browserify":           "~12.0.1",
         "babelify":             "~7.2.0",
-        "babel-preset-es2015":  "~6.1.2",
+        "babel-preset-es2015":  "~6.3.13",
         "minifyify":            "~7.1.0",
@@ -33,0 +33,0 @@ "browserify-header":    "~0.9.2",

src/tokenizr.js

@@ -26,3 +26,4 @@ /*
 import Tokenizr from "./tokenizr-5-tokenizer"
-export default Tokenizr
 
+module.exports = Tokenizr
+

tst/tokenizr.js

@@ -31,3 +31,3 @@ /*
 
-var Tokenizr = require("../lib/tokenizr.js").default
+var Tokenizr = require("../lib/tokenizr.js")
 
@@ -34,0 +34,0 @@ describe("Tokenizr Library", function () {

Fixed alerts

No v1

Quality

Package is not semver >=1. This means it is not stable and does not support ^ ranges.

Found 1 instance in 1 package

Improved metrics

Number of low quality alerts

1

decreased by-50%

Worsened metrics

Total package byte prevSize

75882

decreased by-0.25%

Lines of code

1062

decreased by-0.09%

No dependency changes