tokenizr - npm Package Compare versions

Comparing version 1.6.10 to 1.7.0

Gruntfile.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015-2023 Dr. Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2024 Dr. Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

lib/tokenizr.js

 (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Tokenizr = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(_dereq_,module,exports){
-"use strict";function _inherits(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function");t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,writable:!0,configurable:!0}}),Object.defineProperty(t,"prototype",{writable:!1}),e&&_setPrototypeOf(t,e)}function _createSuper(t){var e=_isNativeReflectConstruct();return function(){var n,r=_getPrototypeOf(t);if(e){var i=_getPrototypeOf(this).constructor;n=Reflect.construct(r,arguments,i)}else n=r.apply(this,arguments);return _possibleConstructorReturn(this,n)}}function _possibleConstructorReturn(t,e){if(e&&("object"===_typeof(e)||"function"==typeof e))return e;if(void 0!==e)throw new TypeError("Derived constructors may only return object or undefined");return _assertThisInitialized(t)}function _assertThisInitialized(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}function _wrapNativeSuper(t){var e="function"==typeof Map?new Map:void 0;return(_wrapNativeSuper=function(t){if(null===t||!_isNativeFunction(t))return t;if("function"!=typeof t)throw new TypeError("Super expression must either be null or a function");if(void 0!==e){if(e.has(t))return e.get(t);e.set(t,n)}function n(){return _construct(t,arguments,_getPrototypeOf(this).constructor)}return n.prototype=Object.create(t.prototype,{constructor:{value:n,enumerable:!1,writable:!0,configurable:!0}}),_setPrototypeOf(n,t)})(t)}function _construct(t,e,n){return(_construct=_isNativeReflectConstruct()?Reflect.construct.bind():function(t,e,n){var r=[null];r.push.apply(r,e);var i=new(Function.bind.apply(t,r));return n&&_setPrototypeOf(i,n.prototype),i}).apply(null,arguments)}function _isNativeReflectConstruct(){if("undefined"==typeof Reflect||!Reflect.construct)return!1;if(Reflect.construct.sham)return!1;if("function"==typeof Proxy)return!0;try{return Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){})),!0}catch(t){return!1}}function _isNativeFunction(t){return-1!==Function.toString.call(t).indexOf("[native code]")}function _setPrototypeOf(t,e){return(_setPrototypeOf=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(t,e){return t.__proto__=e,t})(t,e)}function _getPrototypeOf(t){return(_getPrototypeOf=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(t){return t.__proto__||Object.getPrototypeOf(t)})(t)}function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function _defineProperties(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,_toPropertyKey(r.key),r)}}function _createClass(t,e,n){return e&&_defineProperties(t.prototype,e),n&&_defineProperties(t,n),Object.defineProperty(t,"prototype",{writable:!1}),t}function _toPropertyKey(t){var e=_toPrimitive(t,"string");return"symbol"===_typeof(e)?e:String(e)}function _toPrimitive(t,e){if("object"!==_typeof(t)||null===t)return t;var n=t[Symbol.toPrimitive];if(void 0!==n){var r=n.call(t,e||"default");if("object"!==_typeof(r))return r;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===e?String:Number)(t)}var excerpt=function(t,e){var n=t.length,r=e-20;r<0&&(r=0);var i=e+20;i>n&&(i=n);var o=function(t){return t.charCodeAt(0).toString(16).toUpperCase()},s=function(t,e,n){return t.substr(e,n).replace(/\\/g,"\\\\").replace(/\x08/g,"\\b").replace(/\t/g,"\\t").replace(/\n/g,"\\n").replace(/\f/g,"\\f").replace(/\r/g,"\\r").replace(/[\x00-\x07\x0B\x0E\x0F]/g,function(t){return"\\x0"+o(t)}).replace(/[\x10-\x1F\x80-\xFF]/g,function(t){return"\\x"+o(t)}).replace(/[\u0100-\u0FFF]/g,function(t){return"\\u0"+o(t)}).replace(/[\u1000-\uFFFF]/g,function(t){return"\\u"+o(t)})};return{prologTrunc:r>0,prologText:s(t,r,e-r),tokenText:s(t,e,1),epilogText:s(t,e+1,i-(e+1)),epilogTrunc:i<n}},Token=function(){function t(e,n,r){var i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:0,o=arguments.length>4&&void 0!==arguments[4]?arguments[4]:0,s=arguments.length>5&&void 0!==arguments[5]?arguments[5]:0;_classCallCheck(this,t),this.type=e,this.value=n,this.text=r,this.pos=i,this.line=o,this.column=s}return _createClass(t,[{key:"toString",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:function(t,e){return e};return"".concat(t("type",this.type)," ")+"(value: ".concat(t("value",JSON.stringify(this.value)),", ")+"text: ".concat(t("text",JSON.stringify(this.text)),", ")+"pos: ".concat(t("pos",this.pos),", ")+"line: ".concat(t("line",this.line),", ")+"column: ".concat(t("column",this.column),")")}},{key:"isA",value:function(t,e){return t===this.type&&(2!==arguments.length||e===this.value)}}]),t}(),ParsingError=function(t){_inherits(n,_wrapNativeSuper(Error));var e=_createSuper(n);function n(t,r,i,o,s){var a;return _classCallCheck(this,n),(a=e.call(this,t)).name="ParsingError",a.message=t,a.pos=r,a.line=i,a.column=o,a.input=s,a}return _createClass(n,[{key:"toString",value:function(){for(var t=excerpt(this.input,this.pos),e="line ".concat(this.line," (column ").concat(this.column,"): "),n="",r=0;r<e.length+t.prologText.length;r++)n+=" ";return"Parsing Error: "+this.message+"\n"+e+t.prologText+t.tokenText+t.epilogText+"\n"+n+"^"}}]),n}(),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){var t;return(t=this._tokenizr).push.apply(t,arguments),this}},{key:"pop",value:function(){var t;return(t=this._tokenizr).pop.apply(t,arguments)}},{key:"state",value:function(){var t,e;return arguments.length>0?((e=this._tokenizr).state.apply(e,arguments),this):(t=this._tokenizr).state.apply(t,arguments)}},{key:"tag",value:function(){var t;return(t=this._tokenizr).tag.apply(t,arguments),this}},{key:"tagged",value:function(){var t;return(t=this._tokenizr).tagged.apply(t,arguments)}},{key:"untag",value:function(){var t;return(t=this._tokenizr).untag.apply(t,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: ".concat(t,", value: ")+"".concat(JSON.stringify(e)," (").concat(_typeof(e),'), text: "').concat(this._match[0],'"')),this._tokenizr._pending.push(new Token(t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}},{key:"stop",value:function(){return this._tokenizr._stopped=!0,this}}]),t}(),Tokenizr=function(){function t(){_classCallCheck(this,t),this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._stopped=!1,this._ctx=new ActionContext(this),this}},{key:"error",value:function(t){return new ParsingError(t,this._pos,this._line,this._column,this._input)}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: ".concat(t))}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(t,">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(this._state[this._state.length-2],">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(t,">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): ".concat(t)),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return!0===this._tag[t]}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): ".concat(t)),delete this._tag[t],this}},{key:"before",value:function(t){return this._before=t,this}},{key:"after",value:function(t){return this._after=t,this}},{key:"finish",value:function(t){return this._finish=t,this}},{key:"rule",value:function(t,e,n){var r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:"unknown";if(2===arguments.length&&"function"==typeof e){var i=[t,e];e=i[0],n=i[1],t="*"}else if(3===arguments.length&&"function"==typeof e){var o=[t,e,n];e=o[0],n=o[1],r=o[2],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===_typeof(e)&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');if("string"!=typeof r)throw new Error('parameter "name" not a String');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var s="g";try{"boolean"==typeof new RegExp("","y").sticky&&(s="y")}catch(t){}return"boolean"==typeof e.multiline&&e.multiline&&(s+="m"),"boolean"==typeof e.dotAll&&e.dotAll&&(s+="s"),"boolean"==typeof e.ignoreCase&&e.ignoreCase&&(s+="i"),"boolean"==typeof e.unicode&&e.unicode&&(s+="u"),e=new RegExp(e.source,s),this._log("rule: configure rule (state: ".concat(t,", pattern: ").concat(e.source,")")),this._rules.push({state:t,pattern:e,action:n,name:r}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;o<e;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: ".concat(e-t,", ")+"from: <line ".concat(n,", column ").concat(r,">, ")+"to: <line ".concat(this._line,", column ").concat(this._column,">"))}},{key:"_tokenize",value:function(){var t=this,e=function(){t._eof||(null!==t._finish&&t._finish.call(t._ctx,t._ctx),t._eof=!0,t._pending.push(new Token("EOF","","",t._pos,t._line,t._column)))};if(!(this._stopped||this._pos>=this._len)){for(var n=!0;n;){if(n=!1,this._debug){var r=excerpt(this._input,this._pos),i=Object.keys(this._tag).map(function(t){return"#".concat(t)}).join(" ");this._log("INPUT: state: <".concat(this._state[this._state.length-1],">, tags: <").concat(i,">, text: ")+(r.prologTrunc?"...":'"')+"".concat(r.prologText,"<").concat(r.tokenText,">").concat(r.epilogText)+(r.epilogTrunc?"...":'"')+", at: <line ".concat(this._line,", column ").concat(this._column,">"))}for(var o=0;o<this._rules.length;o++){if(this._debug){var s=this._rules[o].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#".concat(t)}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <".concat(s,">, ")+"pattern: ".concat(this._rules[o].pattern.source))}var a=!1,c=this._rules[o].state.map(function(t){return t.state}),u=c.indexOf("*");if(u<0&&(u=c.indexOf(this._state[this._state.length-1])),u>=0){a=!0;var l=this._rules[o].state[u].tags;(l=l.filter(function(e){return!t._tag[e]})).length>0&&(a=!1)}if(a){this._rules[o].pattern.lastIndex=this._pos;var h=this._rules[o].pattern.exec(this._input);if(this._rules[o].pattern.lastIndex=this._pos,null!==(h=this._rules[o].pattern.exec(this._input))&&h.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(h)),this._ctx._match=h,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,null!==this._before&&this._before.call(this._ctx,this._ctx,h,this._rules[o]),this._rules[o].action.call(this._ctx,this._ctx,h),null!==this._after&&this._after.call(this._ctx,this._ctx,h,this._rules[o]),this._ctx._reject)continue;if(this._ctx._repeat){n=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,this._pos>=this._len)return void e();n=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,void(this._pos>=this._len&&e());throw new Error('action of pattern "'+this._rules[o].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}e()}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: ".concat(t.toString())),t}return null}},{key:"tokens",value:function(){for(var t,e=[];null!==(t=this.token());)e.push(t);return e}},{key:"peek",value:function(t){void 0===t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: ".concat(this._pending[t].toString())),this._pending[t]}},{key:"skip",value:function(t){void 0===t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=this,r=0;r<this._pending.length+1;r++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();this._log("CONSUME: ".concat(i.toString()));var o=function(){throw new ParsingError("expected: <type: ".concat(t,", value: ").concat(JSON.stringify(e)," (").concat(_typeof(e),")>, ")+"found: <type: ".concat(i.type,", value: ").concat(JSON.stringify(i.value)," (").concat(_typeof(i.value),")>"),i.pos,i.line,i.column,n._input)};return 2!==arguments.length||i.isA(t,e)?i.isA(t)||o():o(JSON.stringify(e),_typeof(e)),i}},{key:"begin",value:function(){return this._log("BEGIN: level ".concat(this._transaction.length)),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");var t=this._transaction.shift();return this._transaction.length>0&&(this._transaction[0]=this._transaction[0].concat(t)),this._log("COMMIT: level ".concat(this._transaction.length)),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");var t=this._transaction.shift();return this._pending=t.concat(this._pending),this._log("ROLLBACK: level ".concat(this._transaction.length)),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=new Array(n),i=0;i<n;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: ".concat(t.toString())),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw(e=e.sort(function(t,e){return t.depth-e.depth}))[0].ex;return t}}]),t}();Tokenizr.Token=Token,Tokenizr.ParsingError=ParsingError,Tokenizr.ActionContext=ActionContext,module.exports=Tokenizr;
+"use strict";function _callSuper(t,e,n){return e=_getPrototypeOf(e),_possibleConstructorReturn(t,_isNativeReflectConstruct()?Reflect.construct(e,n||[],_getPrototypeOf(t).constructor):e.apply(t,n))}function _possibleConstructorReturn(t,e){if(e&&("object"===_typeof(e)||"function"==typeof e))return e;if(void 0!==e)throw new TypeError("Derived constructors may only return object or undefined");return _assertThisInitialized(t)}function _assertThisInitialized(t){if(void 0===t)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return t}function _inherits(t,e){if("function"!=typeof e&&null!==e)throw new TypeError("Super expression must either be null or a function");t.prototype=Object.create(e&&e.prototype,{constructor:{value:t,writable:!0,configurable:!0}}),Object.defineProperty(t,"prototype",{writable:!1}),e&&_setPrototypeOf(t,e)}function _wrapNativeSuper(t){var e="function"==typeof Map?new Map:void 0;return(_wrapNativeSuper=function(t){if(null===t||!_isNativeFunction(t))return t;if("function"!=typeof t)throw new TypeError("Super expression must either be null or a function");if(void 0!==e){if(e.has(t))return e.get(t);e.set(t,n)}function n(){return _construct(t,arguments,_getPrototypeOf(this).constructor)}return n.prototype=Object.create(t.prototype,{constructor:{value:n,enumerable:!1,writable:!0,configurable:!0}}),_setPrototypeOf(n,t)})(t)}function _construct(t,e,n){if(_isNativeReflectConstruct())return Reflect.construct.apply(null,arguments);var r=[null];r.push.apply(r,e);var i=new(t.bind.apply(t,r));return n&&_setPrototypeOf(i,n.prototype),i}function _isNativeReflectConstruct(){try{var t=!Boolean.prototype.valueOf.call(Reflect.construct(Boolean,[],function(){}))}catch(t){}return(_isNativeReflectConstruct=function(){return!!t})()}function _isNativeFunction(t){try{return-1!==Function.toString.call(t).indexOf("[native code]")}catch(e){return"function"==typeof t}}function _setPrototypeOf(t,e){return(_setPrototypeOf=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(t,e){return t.__proto__=e,t})(t,e)}function _getPrototypeOf(t){return(_getPrototypeOf=Object.setPrototypeOf?Object.getPrototypeOf.bind():function(t){return t.__proto__||Object.getPrototypeOf(t)})(t)}function _typeof(t){return(_typeof="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t})(t)}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function _defineProperties(t,e){for(var n=0;n<e.length;n++){var r=e[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(t,_toPropertyKey(r.key),r)}}function _createClass(t,e,n){return e&&_defineProperties(t.prototype,e),n&&_defineProperties(t,n),Object.defineProperty(t,"prototype",{writable:!1}),t}function _toPropertyKey(t){var e=_toPrimitive(t,"string");return"symbol"==_typeof(e)?e:String(e)}function _toPrimitive(t,e){if("object"!=_typeof(t)||!t)return t;var n=t[Symbol.toPrimitive];if(void 0!==n){var r=n.call(t,e||"default");if("object"!=_typeof(r))return r;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===e?String:Number)(t)}var excerpt=function(t,e){var n=t.length,r=e-20;r<0&&(r=0);var i=e+20;i>n&&(i=n);var o=function(t){return t.charCodeAt(0).toString(16).toUpperCase()},s=function(t,e,n){return t.substr(e,n).replace(/\\/g,"\\\\").replace(/\x08/g,"\\b").replace(/\t/g,"\\t").replace(/\n/g,"\\n").replace(/\f/g,"\\f").replace(/\r/g,"\\r").replace(/[\x00-\x07\x0B\x0E\x0F]/g,function(t){return"\\x0"+o(t)}).replace(/[\x10-\x1F\x80-\xFF]/g,function(t){return"\\x"+o(t)}).replace(/[\u0100-\u0FFF]/g,function(t){return"\\u0"+o(t)}).replace(/[\u1000-\uFFFF]/g,function(t){return"\\u"+o(t)})};return{prologTrunc:r>0,prologText:s(t,r,e-r),tokenText:s(t,e,1),epilogText:s(t,e+1,i-(e+1)),epilogTrunc:i<n}},Token=function(){function t(e,n,r){var i=arguments.length>3&&void 0!==arguments[3]?arguments[3]:0,o=arguments.length>4&&void 0!==arguments[4]?arguments[4]:0,s=arguments.length>5&&void 0!==arguments[5]?arguments[5]:0;_classCallCheck(this,t),this.type=e,this.value=n,this.text=r,this.pos=i,this.line=o,this.column=s}return _createClass(t,[{key:"toString",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:function(t,e){return e};return"".concat(t("type",this.type)," ")+"(value: ".concat(t("value",JSON.stringify(this.value)),", ")+"text: ".concat(t("text",JSON.stringify(this.text)),", ")+"pos: ".concat(t("pos",this.pos),", ")+"line: ".concat(t("line",this.line),", ")+"column: ".concat(t("column",this.column),")")}},{key:"isA",value:function(t,e){return t===this.type&&(2!==arguments.length||e===this.value)}}]),t}(),ParsingError=function(t){function e(t,n,r,i,o){var s;return _classCallCheck(this,e),(s=_callSuper(this,e,[t])).name="ParsingError",s.message=t,s.pos=n,s.line=r,s.column=i,s.input=o,s}return _inherits(e,_wrapNativeSuper(Error)),_createClass(e,[{key:"toString",value:function(){for(var t=excerpt(this.input,this.pos),e="line ".concat(this.line," (column ").concat(this.column,"): "),n="",r=0;r<e.length+t.prologText.length;r++)n+=" ";return"Parsing Error: "+this.message+"\n"+e+t.prologText+t.tokenText+t.epilogText+"\n"+n+"^"}}]),e}(),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"push",value:function(){var t;return(t=this._tokenizr).push.apply(t,arguments),this}},{key:"pop",value:function(){var t;return(t=this._tokenizr).pop.apply(t,arguments)}},{key:"state",value:function(){var t,e;return arguments.length>0?((e=this._tokenizr).state.apply(e,arguments),this):(t=this._tokenizr).state.apply(t,arguments)}},{key:"tag",value:function(){var t;return(t=this._tokenizr).tag.apply(t,arguments),this}},{key:"tagged",value:function(){var t;return(t=this._tokenizr).tagged.apply(t,arguments)}},{key:"untag",value:function(){var t;return(t=this._tokenizr).untag.apply(t,arguments),this}},{key:"repeat",value:function(){return this._tokenizr._log("    REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log("    REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log("    IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log("    ACCEPT: type: ".concat(t,", value: ")+"".concat(JSON.stringify(e)," (").concat(_typeof(e),'), text: "').concat(this._match[0],'"')),this._tokenizr._pending.push(new Token(t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}},{key:"stop",value:function(){return this._tokenizr._stopped=!0,this}}]),t}(),Tokenizr=function(){function t(){_classCallCheck(this,t),this._before=null,this._after=null,this._finish=null,this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._tag={},this._transaction=[],this._pending=[],this._stopped=!1,this._ctx=new ActionContext(this),this}},{key:"error",value:function(t){return new ParsingError(t,this._pos,this._line,this._column,this._input)}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: ".concat(t))}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"push",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (PUSH): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(t,">")),this._state.push(t),this}},{key:"pop",value:function(){if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");return this._log("    STATE (POP): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(this._state[this._state.length-2],">")),this._state.pop()}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');return this._log("    STATE (SET): "+"old: <".concat(this._state[this._state.length-1],">, ")+"new: <".concat(t,">")),this._state[this._state.length-1]=t,this}if(0===arguments.length)return this._state[this._state.length-1];throw new Error("invalid number of arguments")}},{key:"tag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (ADD): ".concat(t)),this._tag[t]=!0,this}},{key:"tagged",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return!0===this._tag[t]}},{key:"untag",value:function(t){if(1!==arguments.length)throw new Error("invalid number of arguments");if("string"!=typeof t)throw new Error('parameter "tag" not a String');return this._log("    TAG (DEL): ".concat(t)),delete this._tag[t],this}},{key:"before",value:function(t){return this._before=t,this}},{key:"after",value:function(t){return this._after=t,this}},{key:"finish",value:function(t){return this._finish=t,this}},{key:"rule",value:function(t,e,n){var r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:"unknown";if(2===arguments.length&&"function"==typeof e){var i=[t,e];e=i[0],n=i[1],t="*"}else if(3===arguments.length&&"function"==typeof e){var o=[t,e,n];e=o[0],n=o[1],r=o[2],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"===_typeof(e)&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');if("string"!=typeof r)throw new Error('parameter "name" not a String');t=t.split(/\s*,\s*/g).map(function(t){var e=t.split(/\s+/g),n=e.filter(function(t){return null===t.match(/^#/)}),r=e.filter(function(t){return null!==t.match(/^#/)}).map(function(t){return t.replace(/^#/,"")});if(1!==n.length)throw new Error("exactly one state required");return{state:n[0],tags:r}});var s="g";try{"boolean"==typeof new RegExp("","y").sticky&&(s="y")}catch(t){}return"boolean"==typeof e.multiline&&e.multiline&&(s+="m"),"boolean"==typeof e.dotAll&&e.dotAll&&(s+="s"),"boolean"==typeof e.ignoreCase&&e.ignoreCase&&(s+="i"),"boolean"==typeof e.unicode&&e.unicode&&(s+="u"),e=new RegExp(e.source,s),this._log("rule: configure rule (state: ".concat(t,", pattern: ").concat(e.source,")")),this._rules.push({state:t,pattern:e,action:n,name:r}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,r=this._column,i=this._input,o=t;o<e;o++){var s=i.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):"\t"===s?this._column+=8-this._column%8:this._column++}this._log("    PROGRESS: characters: ".concat(e-t,", ")+"from: <line ".concat(n,", column ").concat(r,">, ")+"to: <line ".concat(this._line,", column ").concat(this._column,">"))}},{key:"_tokenize",value:function(){var t=this,e=function(){t._eof||(null!==t._finish&&t._finish.call(t._ctx,t._ctx),t._eof=!0,t._pending.push(new Token("EOF","","",t._pos,t._line,t._column)))};if(!(this._stopped||this._pos>=this._len)){for(var n=!0;n;){if(n=!1,this._debug){var r=excerpt(this._input,this._pos),i=Object.keys(this._tag).map(function(t){return"#".concat(t)}).join(" ");this._log("INPUT: state: <".concat(this._state[this._state.length-1],">, tags: <").concat(i,">, text: ")+(r.prologTrunc?"...":'"')+"".concat(r.prologText,"<").concat(r.tokenText,">").concat(r.epilogText)+(r.epilogTrunc?"...":'"')+", at: <line ".concat(this._line,", column ").concat(this._column,">"))}for(var o=0;o<this._rules.length;o++){if(this._debug){var s=this._rules[o].state.map(function(t){var e=t.state;return t.tags.length>0&&(e+=" "+t.tags.map(function(t){return"#".concat(t)}).join(" ")),e}).join(", ");this._log("  RULE: state(s): <".concat(s,">, ")+"pattern: ".concat(this._rules[o].pattern.source))}var a=!1,c=this._rules[o].state.map(function(t){return t.state}),u=c.indexOf("*");if(u<0&&(u=c.indexOf(this._state[this._state.length-1])),u>=0){a=!0;var l=this._rules[o].state[u].tags;(l=l.filter(function(e){return!t._tag[e]})).length>0&&(a=!1)}if(a){this._rules[o].pattern.lastIndex=this._pos;var h=this._rules[o].pattern.exec(this._input);if(this._rules[o].pattern.lastIndex=this._pos,null!==(h=this._rules[o].pattern.exec(this._input))&&h.index===this._pos){if(this._debug&&this._log("    MATCHED: "+JSON.stringify(h)),this._ctx._match=h,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,null!==this._before&&this._before.call(this._ctx,this._ctx,h,this._rules[o]),this._rules[o].action.call(this._ctx,this._ctx,h),null!==this._after&&this._after.call(this._ctx,this._ctx,h,this._rules[o]),this._ctx._reject)continue;if(this._ctx._repeat){n=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,this._pos>=this._len)return void e();n=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[o].pattern.lastIndex),this._pos=this._rules[o].pattern.lastIndex,void(this._pos>=this._len&&e());throw new Error('action of pattern "'+this._rules[o].pattern.source+'" neither rejected nor accepted any token(s)')}}}}throw this.error("token not recognized")}e()}},{key:"token",value:function(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var t=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(t),this._log("TOKEN: ".concat(t.toString())),t}return null}},{key:"tokens",value:function(){for(var t,e=[];null!==(t=this.token());)e.push(t);return e}},{key:"peek",value:function(t){if(void 0===t&&(t=0),t>=this._pending.length){0===this._pending.length&&this._tokenize();for(var e=0;e<t-this._pending.length;e++)this._tokenize()}if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: ".concat(this._pending[t].toString())),this._pending[t]}},{key:"skip",value:function(t){void 0===t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-- >0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=this,r=0;r<this._pending.length+1;r++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();this._log("CONSUME: ".concat(i.toString()));var o=function(){throw new ParsingError("expected: <type: ".concat(t,", value: ").concat(JSON.stringify(e)," (").concat(_typeof(e),")>, ")+"found: <type: ".concat(i.type,", value: ").concat(JSON.stringify(i.value)," (").concat(_typeof(i.value),")>"),i.pos,i.line,i.column,n._input)};return 2!==arguments.length||i.isA(t,e)?i.isA(t)||o():o(JSON.stringify(e),_typeof(e)),i}},{key:"begin",value:function(){return this._log("BEGIN: level ".concat(this._transaction.length)),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");var t=this._transaction.shift();return this._transaction.length>0&&(this._transaction[0]=this._transaction[0].concat(t)),this._log("COMMIT: level ".concat(this._transaction.length)),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");var t=this._transaction.shift();return this._pending=t.concat(this._pending),this._log("ROLLBACK: level ".concat(this._transaction.length)),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,r=new Array(n),i=0;i<n;i++)r[i]=arguments[i];for(var o=0;o<r.length;o++)try{this.begin(),t=r[o].call(this),this.commit();break}catch(t){this._log("EXCEPTION: ".concat(t.toString())),e.push({ex:t,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw(e=e.sort(function(t,e){return t.depth-e.depth}))[0].ex;return t}}]),t}();Tokenizr.Token=Token,Tokenizr.ParsingError=ParsingError,Tokenizr.ActionContext=ActionContext,module.exports=Tokenizr;
 },{}]},{},[1])(1)
 });

package.json

 {
     "name":        "tokenizr",
-    "version":     "1.6.10",
+    "version":     "1.7.0",
     "description": "String Tokenization Library for JavaScript",
@@ -26,18 +26,18 @@ "keywords":    [ "string", "token", "scanner", "lexer" ],
         "grunt-mocha-test":          "0.13.3",
-        "grunt-eslint":              "24.2.0",
-        "typescript":                "5.1.6",
-        "mocha":                     "10.2.0",
-        "chai":                      "4.3.7",
+        "grunt-eslint":              "24.3.0",
+        "typescript":                "5.4.2",
+        "mocha":                     "10.3.0",
+        "chai":                      "4.4.1",
         "browserify":                "17.0.0",
         "babelify":                  "10.0.0",
-        "@babel/core":               "7.22.5",
-        "@babel/cli":                "7.22.5",
-        "@babel/node":               "7.22.5",
-        "@babel/preset-env":         "7.22.5",
+        "@babel/core":               "7.24.0",
+        "@babel/cli":                "7.23.9",
+        "@babel/node":               "7.23.9",
+        "@babel/preset-env":         "7.24.0",
         "uglifyify":                 "5.0.2",
         "browserify-derequire":      "1.1.1",
-        "eslint":                    "8.43.0",
+        "eslint":                    "8.57.0",
         "eslint-config-standard":    "17.1.0",
         "eslint-plugin-promise":     "6.1.1",
-        "eslint-plugin-import":      "2.27.5",
+        "eslint-plugin-import":      "2.29.1",
         "eslint-plugin-node":        "11.1.0"
@@ -44,0 +44,0 @@ },

README.md

@@ -470,3 +470,3 @@
 
-Copyright (c) 2015-2023 Dr. Ralf S. Engelschall (http://engelschall.com/)
+Copyright (c) 2015-2024 Dr. Ralf S. Engelschall (http://engelschall.com/)
 
@@ -473,0 +473,0 @@ Permission is hereby granted, free of charge, to any person obtaining

src/tokenizr.d.ts

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015-2023 Dr. Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2024 Dr. Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

src/tokenizr.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015-2023 Dr. Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2024 Dr. Ralf S. Engelschall <[e-mail]>
 **
@@ -454,3 +454,3 @@ **  Permission is hereby granted, free of charge, to any person obtaining
 
-    /*  determine and return the next token  */
+    /*  determine and provide the next token  */
     _tokenize () {
@@ -606,4 +606,10 @@ /*  helper function for finishing parsing  */
             offset = 0
-        for (let i = 0; i < this._pending.length + offset; i++)
-            this._tokenize()
+
+        /*  if no more tokens are pending, try to determine new ones  */
+        if (offset >= this._pending.length) {
+            if (this._pending.length === 0)
+                this._tokenize()
+            for (let i = 0; i < offset - this._pending.length; i++)
+                this._tokenize()
+        }
         if (offset >= this._pending.length)
@@ -610,0 +616,0 @@ throw new Error("not enough tokens available for peek operation")

tst/common.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015-2023 Dr. Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2024 Dr. Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

tst/tokenizr.js

 /*
 **  Tokenizr -- String Tokenization Library
-**  Copyright (c) 2015-2023 Dr. Ralf S. Engelschall <[e-mail]>
+**  Copyright (c) 2015-2024 Dr. Ralf S. Engelschall <[e-mail]>
 **
@@ -5,0 +5,0 @@ **  Permission is hereby granted, free of charge, to any person obtaining

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

82365

increased by0.13%

Lines of code

1070

increased by0.47%

No dependency changes