Comparing version 1.6.7 to 1.6.8
(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Tokenizr = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(_dereq_,module,exports){ | ||
"use strict";function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}function _inherits(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function");t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,writable:!0,configurable:!0}}),Object.defineProperty(t,"prototype",{writable:!1}),e&&_setPrototypeOf(t,e)}function _createSuper(t){var e=_isNativeReflectConstruct();return function(){var n,r=_getPrototypeOf(t);if(e){var i=_getPrototypeOf(this).constructor;n=Reflect.construct(r,arguments,i)}else n=r.apply(this,arguments);return _possibleConstructorReturn(this,n)}}function _possibleConstructorReturn(t,e){if(e&&("object"===_typeof(e)||"function"==typeof e))return e;if(void 0!==e)throw new TypeError("Derived constructors may only return object or undefined");return _assertThisInitialized(t)}function _assertThisInitialized(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}function _wrapNativeSuper(t){var e="function"==typeof Map?new Map:void 0;return(_wrapNativeSuper=function(t){if(null===t||!_isNativeFunction(t))return t;if("function"!=typeof t)throw new TypeError("Super expression must either be null or a function");if(void 0!==e){if(e.has(t))return e.get(t);e.set(t,n)}function n(){return _construct(t,arguments,_getPrototypeOf(this).constructor)}return n.prototype=Object.create(t.prototype,{constructor:{value:n,enumerable:!1,writable:!0,configurable:!0}}),_setPrototypeOf(n,t)})(t)}function _construct(t,e,n){return(_construct=_isNativeReflectConstruct()?Reflect.construct.bind():function(t,e,n){var r=[null];r.push.apply(r,e);var i=new(Function.bind.apply(t,r));return n&&_setPrototypeOf(i,n.prototype),i}).apply(null,arguments)}function _isNativeReflectConstruct(){if("undefined"==typeof Reflect||!Reflect.construct)return!1;if(Reflect.construct.sham)return!1;if("function"==typeof Proxy)return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){})),!0}catch(t){return!1}}function _isNativeFunction(t){return-1!==Function.toString.call(t).indexOf("[native code]")}function _setPrototypeOf(t,e){return(_setPrototypeOf=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(t,e){return t.__proto__=e,t})(t,e)}function _getPrototypeOf(t){return(_getPrototypeOf=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(t){return t.__proto__||Object.getPrototypeOf(t)})(t)}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function _defineProperties(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,r.key,r)}}function _createClass(t,e,n){return e&&_defineProperties(t.prototype,e),n&&_defineProperties(t,n),Object.defineProperty(t,"prototype",{writable:!1}),t}var excerpt=function(t,e){var n=t.length,r=e-20;r<0&&(r=0);var i=e+20;i>n&&(i=n);var o=function(t){return t.charCodeAt(0).toString(16).toUpperCase()},s=function(t,e,n){return t.substr(e,n).replace(/\\/g,"\\\\").replace(/\x08/g,"\\b").replace(/\t/g,"\\t").replace(/\n/g,"\\n").replace(/\f/g,"\\f").replace(/\r/g,"\\r").replace(/[\x00-\x07\x0B\x0E\x0F]/g,function(t){return"\\x0"+o(t)}).replace(/[\x10-\x1F\x80-\xFF]/g,function(t){return"\\x"+o(t)}).replace(/[\u0100-\u0FFF]/g,function(t){return"\\u0"+o(t)}).replace(/[\u1000-\uFFFF]/g,function(t){return"\\u"+o(t)})};return{prologTrunc:r>0,prologText:s(t,r,e-r),tokenText:s(t,e,1),epilogText:s(t,e+1,i-(e+1)),epilogTrunc:i<n}},Token=function(){function t(e,n,r){var i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:0,o=arguments.length>4&&void 0!==arguments[4]?arguments[4]:0,s=arguments.length>5&&void 0!==arguments[5]?arguments[5]:0;_classCallCheck(this,t),this.type=e,this.value=n,this.text=r,this.pos=i,this.line=o,this.column=s}return _createClass(t,[{key:"toString",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:function(t,e){return e};return"".concat(t("type",this.type)," ")+"(value: ".concat(t("value",JSON.stringify(this.value)),", ")+"text: ".concat(t("text",JSON.stringify(this.text)),", ")+"pos: ".concat(t("pos",this.pos),", ")+"line: ".concat(t("line",this.line),", ")+"column: ".concat(t("column",this.column),")")}},{key:"isA",value:function(t,e){return t===this.type&&(2!==arguments.length||e===this.value)}}]),t}(),ParsingError=function(t){_inherits(n,_wrapNativeSuper(Error));var e=_createSuper(n);function n(t,r,i,o,s){var a;return _classCallCheck(this,n),(a=e.call(this,t)).name="ParsingError",a.message=t,a.pos=r,a.line=i,a.column=o,a.input=s,a}return _createClass(n,[{key:"toString",value:function(){for(var t=excerpt(this.input,this.pos),e="line ".concat(this.line," (column ").concat(this.column,"): "),n="",r=0;r<e.length+t.prologText.length;r++)n+=" ";return"Parsing Error: "+this.message+"\n"+e+t.prologText+t.tokenText+t.epilogText+"\n"+n+"^"}}]),n}(),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){var t;return(t=this._tokenizr).push.apply(t,arguments),this}},{key:"pop",value:function(){var t;return(t=this._tokenizr).pop.apply(t,arguments)}},{key:"state",value:function(){var t,e;return arguments.length>0?((e=this._tokenizr).state.apply(e,arguments),this):(t=this._tokenizr).state.apply(t,arguments)}},{key:"tag",value:function(){var t;return(t=this._tokenizr).tag.apply(t,arguments),this}},{key:"tagged",value:function(){var t;return(t=this._tokenizr).tagged.apply(t,arguments)}},{key:"untag",value:function(){var t;return(t=this._tokenizr).untag.apply(t,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log(" REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log(" REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log(" IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log(" ACCEPT: type: ".concat(t,", value: ")+"".concat(JSON.stringify(e)," (").concat(_typeof(e),'), text: "').concat(this._match[0],'"')),this._tokenizr._pending.push(new Token(t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}},{key:"stop",value:function(){return this._tokenizr._stopped=!0,this}}]),t}(),Tokenizr=function(){function t(){_classCallCheck(this,t),this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._stopped=!1,this._ctx=new ActionContext(this),this}},{key:"error",value:function(t){return new ParsingError(t,this._pos,this._line,this._column,this._input)}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: ".concat(t))}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log(" STATE (PUSH): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(t,">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log(" STATE (POP): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(this._state[this._state.length-2],">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log(" STATE (SET): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(t,">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log(" TAG (ADD): ".concat(t)),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return!0===this._tag[t]}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log(" TAG (DEL): ".concat(t)),delete this._tag[t],this}},{key:"before",value:function(t){return this._before=t,this}},{key:"after",value:function(t){return this._after=t,this}},{key:"finish",value:function(t){return this._finish=t,this}},{key:"rule",value:function(t,e,n){var r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:"unknown";if(2===arguments.length&&"function"==typeof e){var i=[t,e];e=i[0],n=i[1],t="*"}else if(3===arguments.length&&"function"==typeof e){var o=[t,e,n];e=o[0],n=o[1],r=o[2],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===_typeof(e)&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');if("string"!=typeof r)throw new Error('parameter "name" not a String');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var s="g";try{"boolean"==typeof new RegExp("","y").sticky&&(s="y")}catch(t){}return"boolean"==typeof e.multiline&&e.multiline&&(s+="m"),"boolean"==typeof e.dotAll&&e.dotAll&&(s+="s"),"boolean"==typeof e.ignoreCase&&e.ignoreCase&&(s+="i"),"boolean"==typeof e.unicode&&e.unicode&&(s+="u"),e=new RegExp(e.source,s),this._log("rule: configure rule (state: ".concat(t,", pattern: ").concat(e.source,")")),this._rules.push({state:t,pattern:e,action:n,name:r}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;o<e;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log(" PROGRESS: characters: ".concat(e-t,", ")+"from: <line ".concat(n,", column ").concat(r,">, ")+"to: <line ".concat(this._line,", column ").concat(this._column,">"))}},{key:"_tokenize",value:function(){var t=this,e=function(){t._eof||(null!==t._finish&&t._finish.call(t._ctx,t._ctx),t._eof=!0,t._pending.push(new Token("EOF","","",t._pos,t._line,t._column)))};if(!(this._stopped||this._pos>=this._len)){for(var n=!0;n;){if(n=!1,this._debug){var r=excerpt(this._input,this._pos),i=Object.keys(this._tag).map(function(t){return"#".concat(t)}).join(" ");this._log("INPUT: state: <".concat(this._state[this._state.length-1],">, tags: <").concat(i,">, text: ")+(r.prologTrunc?"...":'"')+"".concat(r.prologText,"<").concat(r.tokenText,">").concat(r.epilogText)+(r.epilogTrunc?"...":'"')+", at: <line ".concat(this._line,", column ").concat(this._column,">"))}for(var o=0;o<this._rules.length;o++){if(this._debug){var s=this._rules[o].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#".concat(t)}).join(" ")),e}).join(", ");this._log(" RULE: state(s): <".concat(s,">, ")+"pattern: ".concat(this._rules[o].pattern.source))}var a=!1,c=this._rules[o].state.map(function(t){return t.state}),u=c.indexOf("*");if(u<0&&(u=c.indexOf(this._state[this._state.length-1])),u>=0){a=!0;var h=this._rules[o].state[u].tags;(h=h.filter(function(e){return!t._tag[e]})).length>0&&(a=!1)}if(a){this._rules[o].pattern.lastIndex=this._pos;var l=this._rules[o].pattern.exec(this._input);if(this._rules[o].pattern.lastIndex=this._pos,null!==(l=this._rules[o].pattern.exec(this._input))&&l.index===this._pos){if(this._debug&&this._log(" MATCHED: "+JSON.stringify(l)),this._ctx._match=l,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,null!==this._before&&this._before.call(this._ctx,this._ctx,l,this._rules[o]),this._rules[o].action.call(this._ctx,this._ctx,l),null!==this._after&&this._after.call(this._ctx,this._ctx,l,this._rules[o]),this._ctx._reject)continue;if(this._ctx._repeat){n=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,this._pos>=this._len)return void e();n=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,void(this._pos>=this._len&&e());throw new Error('action of pattern "'+this._rules[o].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}e()}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: ".concat(t.toString())),t}return null}},{key:"tokens",value:function(){for(var t,e=[];null!==(t=this.token());)e.push(t);return e}},{key:"peek",value:function(t){void 0===t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: ".concat(this._pending[t].toString())),this._pending[t]}},{key:"skip",value:function(t){void 0===t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=this,r=0;r<this._pending.length+1;r++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();this._log("CONSUME: ".concat(i.toString()));var o=function(){throw new ParsingError("expected: <type: ".concat(t,", value: ").concat(JSON.stringify(e)," (").concat(_typeof(e),")>, ")+"found: <type: ".concat(i.type,", value: ").concat(JSON.stringify(i.value)," (").concat(_typeof(i.value),")>"),i.pos,i.line,i.column,n._input)};return 2!==arguments.length||i.isA(t,e)?i.isA(t)||o():o(JSON.stringify(e),_typeof(e)),i}},{key:"begin",value:function(){return this._log("BEGIN: level ".concat(this._transaction.length)),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");var t=this._transaction.shift();return this._transaction.length>0&&(this._transaction[0]=this._transaction[0].concat(t)),this._log("COMMIT: level ".concat(this._transaction.length)),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");var t=this._transaction.shift();return this._pending=t.concat(this._pending),this._log("ROLLBACK: level ".concat(this._transaction.length)),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=new Array(n),i=0;i<n;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: ".concat(t.toString())),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw(e=e.sort(function(t,e){return t.depth-e.depth}))[0].ex;return t}}]),t}();Tokenizr.Token=Token,Tokenizr.ParsingError=ParsingError,Tokenizr.ActionContext=ActionContext,module.exports=Tokenizr; | ||
"use strict";function _inherits(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function");t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,writable:!0,configurable:!0}}),Object.defineProperty(t,"prototype",{writable:!1}),e&&_setPrototypeOf(t,e)}function _createSuper(t){var e=_isNativeReflectConstruct();return function(){var n,r=_getPrototypeOf(t);if(e){var i=_getPrototypeOf(this).constructor;n=Reflect.construct(r,arguments,i)}else n=r.apply(this,arguments);return _possibleConstructorReturn(this,n)}}function _possibleConstructorReturn(t,e){if(e&&("object"===_typeof(e)||"function"==typeof e))return e;if(void 0!==e)throw new TypeError("Derived constructors may only return object or undefined");return _assertThisInitialized(t)}function _assertThisInitialized(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}function _wrapNativeSuper(t){var e="function"==typeof Map?new Map:void 0;return(_wrapNativeSuper=function(t){if(null===t||!_isNativeFunction(t))return t;if("function"!=typeof t)throw new TypeError("Super expression must either be null or a function");if(void 0!==e){if(e.has(t))return e.get(t);e.set(t,n)}function n(){return _construct(t,arguments,_getPrototypeOf(this).constructor)}return n.prototype=Object.create(t.prototype,{constructor:{value:n,enumerable:!1,writable:!0,configurable:!0}}),_setPrototypeOf(n,t)})(t)}function _construct(t,e,n){return(_construct=_isNativeReflectConstruct()?Reflect.construct.bind():function(t,e,n){var r=[null];r.push.apply(r,e);var i=new(Function.bind.apply(t,r));return n&&_setPrototypeOf(i,n.prototype),i}).apply(null,arguments)}function _isNativeReflectConstruct(){if("undefined"==typeof Reflect||!Reflect.construct)return!1;if(Reflect.construct.sham)return!1;if("function"==typeof Proxy)return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){})),!0}catch(t){return!1}}function _isNativeFunction(t){return-1!==Function.toString.call(t).indexOf("[native code]")}function _setPrototypeOf(t,e){return(_setPrototypeOf=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(t,e){return t.__proto__=e,t})(t,e)}function _getPrototypeOf(t){return(_getPrototypeOf=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(t){return t.__proto__||Object.getPrototypeOf(t)})(t)}function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function _defineProperties(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,_toPropertyKey(r.key),r)}}function _createClass(t,e,n){return e&&_defineProperties(t.prototype,e),n&&_defineProperties(t,n),Object.defineProperty(t,"prototype",{writable:!1}),t}function _toPropertyKey(t){var e=_toPrimitive(t,"string");return"symbol"===_typeof(e)?e:String(e)}function _toPrimitive(t,e){if("object"!==_typeof(t)||null===t)return t;var n=t[Symbol.toPrimitive];if(void 0!==n){var r=n.call(t,e||"default");if("object"!==_typeof(r))return r;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===e?String:Number)(t)}var excerpt=function(t,e){var n=t.length,r=e-20;r<0&&(r=0);var i=e+20;i>n&&(i=n);var o=function(t){return t.charCodeAt(0).toString(16).toUpperCase()},s=function(t,e,n){return t.substr(e,n).replace(/\\/g,"\\\\").replace(/\x08/g,"\\b").replace(/\t/g,"\\t").replace(/\n/g,"\\n").replace(/\f/g,"\\f").replace(/\r/g,"\\r").replace(/[\x00-\x07\x0B\x0E\x0F]/g,function(t){return"\\x0"+o(t)}).replace(/[\x10-\x1F\x80-\xFF]/g,function(t){return"\\x"+o(t)}).replace(/[\u0100-\u0FFF]/g,function(t){return"\\u0"+o(t)}).replace(/[\u1000-\uFFFF]/g,function(t){return"\\u"+o(t)})};return{prologTrunc:r>0,prologText:s(t,r,e-r),tokenText:s(t,e,1),epilogText:s(t,e+1,i-(e+1)),epilogTrunc:i<n}},Token=function(){function t(e,n,r){var i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:0,o=arguments.length>4&&void 0!==arguments[4]?arguments[4]:0,s=arguments.length>5&&void 0!==arguments[5]?arguments[5]:0;_classCallCheck(this,t),this.type=e,this.value=n,this.text=r,this.pos=i,this.line=o,this.column=s}return _createClass(t,[{key:"toString",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:function(t,e){return e};return"".concat(t("type",this.type)," ")+"(value: ".concat(t("value",JSON.stringify(this.value)),", ")+"text: ".concat(t("text",JSON.stringify(this.text)),", ")+"pos: ".concat(t("pos",this.pos),", ")+"line: ".concat(t("line",this.line),", ")+"column: ".concat(t("column",this.column),")")}},{key:"isA",value:function(t,e){return t===this.type&&(2!==arguments.length||e===this.value)}}]),t}(),ParsingError=function(t){_inherits(n,_wrapNativeSuper(Error));var e=_createSuper(n);function n(t,r,i,o,s){var a;return _classCallCheck(this,n),(a=e.call(this,t)).name="ParsingError",a.message=t,a.pos=r,a.line=i,a.column=o,a.input=s,a}return _createClass(n,[{key:"toString",value:function(){for(var t=excerpt(this.input,this.pos),e="line ".concat(this.line," (column ").concat(this.column,"): "),n="",r=0;r<e.length+t.prologText.length;r++)n+=" ";return"Parsing Error: "+this.message+"\n"+e+t.prologText+t.tokenText+t.epilogText+"\n"+n+"^"}}]),n}(),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){var t;return(t=this._tokenizr).push.apply(t,arguments),this}},{key:"pop",value:function(){var t;return(t=this._tokenizr).pop.apply(t,arguments)}},{key:"state",value:function(){var t,e;return arguments.length>0?((e=this._tokenizr).state.apply(e,arguments),this):(t=this._tokenizr).state.apply(t,arguments)}},{key:"tag",value:function(){var t;return(t=this._tokenizr).tag.apply(t,arguments),this}},{key:"tagged",value:function(){var t;return(t=this._tokenizr).tagged.apply(t,arguments)}},{key:"untag",value:function(){var t;return(t=this._tokenizr).untag.apply(t,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log(" REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log(" REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log(" IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log(" ACCEPT: type: ".concat(t,", value: ")+"".concat(JSON.stringify(e)," (").concat(_typeof(e),'), text: "').concat(this._match[0],'"')),this._tokenizr._pending.push(new Token(t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}},{key:"stop",value:function(){return this._tokenizr._stopped=!0,this}}]),t}(),Tokenizr=function(){function t(){_classCallCheck(this,t),this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._stopped=!1,this._ctx=new ActionContext(this),this}},{key:"error",value:function(t){return new ParsingError(t,this._pos,this._line,this._column,this._input)}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: ".concat(t))}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log(" STATE (PUSH): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(t,">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log(" STATE (POP): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(this._state[this._state.length-2],">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log(" STATE (SET): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(t,">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log(" TAG (ADD): ".concat(t)),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return!0===this._tag[t]}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log(" TAG (DEL): ".concat(t)),delete this._tag[t],this}},{key:"before",value:function(t){return this._before=t,this}},{key:"after",value:function(t){return this._after=t,this}},{key:"finish",value:function(t){return this._finish=t,this}},{key:"rule",value:function(t,e,n){var r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:"unknown";if(2===arguments.length&&"function"==typeof e){var i=[t,e];e=i[0],n=i[1],t="*"}else if(3===arguments.length&&"function"==typeof e){var o=[t,e,n];e=o[0],n=o[1],r=o[2],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===_typeof(e)&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');if("string"!=typeof r)throw new Error('parameter "name" not a String');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var s="g";try{"boolean"==typeof new RegExp("","y").sticky&&(s="y")}catch(t){}return"boolean"==typeof e.multiline&&e.multiline&&(s+="m"),"boolean"==typeof e.dotAll&&e.dotAll&&(s+="s"),"boolean"==typeof e.ignoreCase&&e.ignoreCase&&(s+="i"),"boolean"==typeof e.unicode&&e.unicode&&(s+="u"),e=new RegExp(e.source,s),this._log("rule: configure rule (state: ".concat(t,", pattern: ").concat(e.source,")")),this._rules.push({state:t,pattern:e,action:n,name:r}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;o<e;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log(" PROGRESS: characters: ".concat(e-t,", ")+"from: <line ".concat(n,", column ").concat(r,">, ")+"to: <line ".concat(this._line,", column ").concat(this._column,">"))}},{key:"_tokenize",value:function(){var t=this,e=function(){t._eof||(null!==t._finish&&t._finish.call(t._ctx,t._ctx),t._eof=!0,t._pending.push(new Token("EOF","","",t._pos,t._line,t._column)))};if(!(this._stopped||this._pos>=this._len)){for(var n=!0;n;){if(n=!1,this._debug){var r=excerpt(this._input,this._pos),i=Object.keys(this._tag).map(function(t){return"#".concat(t)}).join(" ");this._log("INPUT: state: <".concat(this._state[this._state.length-1],">, tags: <").concat(i,">, text: ")+(r.prologTrunc?"...":'"')+"".concat(r.prologText,"<").concat(r.tokenText,">").concat(r.epilogText)+(r.epilogTrunc?"...":'"')+", at: <line ".concat(this._line,", column ").concat(this._column,">"))}for(var o=0;o<this._rules.length;o++){if(this._debug){var s=this._rules[o].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#".concat(t)}).join(" ")),e}).join(", ");this._log(" RULE: state(s): <".concat(s,">, ")+"pattern: ".concat(this._rules[o].pattern.source))}var a=!1,c=this._rules[o].state.map(function(t){return t.state}),u=c.indexOf("*");if(u<0&&(u=c.indexOf(this._state[this._state.length-1])),u>=0){a=!0;var l=this._rules[o].state[u].tags;(l=l.filter(function(e){return!t._tag[e]})).length>0&&(a=!1)}if(a){this._rules[o].pattern.lastIndex=this._pos;var h=this._rules[o].pattern.exec(this._input);if(this._rules[o].pattern.lastIndex=this._pos,null!==(h=this._rules[o].pattern.exec(this._input))&&h.index===this._pos){if(this._debug&&this._log(" MATCHED: "+JSON.stringify(h)),this._ctx._match=h,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,null!==this._before&&this._before.call(this._ctx,this._ctx,h,this._rules[o]),this._rules[o].action.call(this._ctx,this._ctx,h),null!==this._after&&this._after.call(this._ctx,this._ctx,h,this._rules[o]),this._ctx._reject)continue;if(this._ctx._repeat){n=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,this._pos>=this._len)return void e();n=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,void(this._pos>=this._len&&e());throw new Error('action of pattern "'+this._rules[o].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}e()}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: ".concat(t.toString())),t}return null}},{key:"tokens",value:function(){for(var t,e=[];null!==(t=this.token());)e.push(t);return e}},{key:"peek",value:function(t){void 0===t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: ".concat(this._pending[t].toString())),this._pending[t]}},{key:"skip",value:function(t){void 0===t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=this,r=0;r<this._pending.length+1;r++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();this._log("CONSUME: ".concat(i.toString()));var o=function(){throw new ParsingError("expected: <type: ".concat(t,", value: ").concat(JSON.stringify(e)," (").concat(_typeof(e),")>, ")+"found: <type: ".concat(i.type,", value: ").concat(JSON.stringify(i.value)," (").concat(_typeof(i.value),")>"),i.pos,i.line,i.column,n._input)};return 2!==arguments.length||i.isA(t,e)?i.isA(t)||o():o(JSON.stringify(e),_typeof(e)),i}},{key:"begin",value:function(){return this._log("BEGIN: level ".concat(this._transaction.length)),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");var t=this._transaction.shift();return this._transaction.length>0&&(this._transaction[0]=this._transaction[0].concat(t)),this._log("COMMIT: level ".concat(this._transaction.length)),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");var t=this._transaction.shift();return this._pending=t.concat(this._pending),this._log("ROLLBACK: level ".concat(this._transaction.length)),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=new Array(n),i=0;i<n;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: ".concat(t.toString())),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw(e=e.sort(function(t,e){return t.depth-e.depth}))[0].ex;return t}}]),t}();Tokenizr.Token=Token,Tokenizr.ParsingError=ParsingError,Tokenizr.ActionContext=ActionContext,module.exports=Tokenizr; | ||
},{}]},{},[1])(1) | ||
}); |
{ | ||
"name": "tokenizr", | ||
"version": "1.6.7", | ||
"version": "1.6.8", | ||
"description": "String Tokenization Library for JavaScript", | ||
@@ -26,17 +26,17 @@ "keywords": [ "string", "token", "scanner", "lexer" ], | ||
"grunt-mocha-test": "0.13.3", | ||
"grunt-eslint": "24.0.0", | ||
"typescript": "4.7.4", | ||
"mocha": "10.0.0", | ||
"chai": "4.3.6", | ||
"grunt-eslint": "24.0.1", | ||
"typescript": "4.9.4", | ||
"mocha": "10.2.0", | ||
"chai": "4.3.7", | ||
"browserify": "17.0.0", | ||
"babelify": "10.0.0", | ||
"@babel/core": "7.18.6", | ||
"@babel/cli": "7.18.6", | ||
"@babel/node": "7.18.6", | ||
"@babel/preset-env": "7.18.6", | ||
"@babel/core": "7.20.7", | ||
"@babel/cli": "7.20.7", | ||
"@babel/node": "7.20.7", | ||
"@babel/preset-env": "7.20.2", | ||
"uglifyify": "5.0.2", | ||
"browserify-derequire": "1.1.1", | ||
"eslint": "8.18.0", | ||
"eslint": "8.30.0", | ||
"eslint-config-standard": "17.0.0", | ||
"eslint-plugin-promise": "6.0.0", | ||
"eslint-plugin-promise": "6.1.1", | ||
"eslint-plugin-import": "2.26.0", | ||
@@ -43,0 +43,0 @@ "eslint-plugin-node": "11.1.0" |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
82227
1065