Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

tokenizr

Package Overview
Dependencies
Maintainers
1
Versions
58
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

tokenizr - npm Package Compare versions

Comparing version 0.9.5 to 0.9.6

2

bower.json
{
"name": "tokenizr",
"version": "0.9.5",
"version": "0.9.6",
"description": "String Tokenization Library for JavaScript",

@@ -5,0 +5,0 @@ "main": "lib/tokenizr.js",

2

lib/tokenizr.js

@@ -32,3 +32,3 @@ /*

},{"./tokenizr-1-excerpt":1}],4:[function(_dereq_,module,exports){
"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"state",value:function(t){if("string"==typeof t)this._tokenizr._log(" STATE (PUSH): "+("old: "+this._tokenizr._state[this._tokenizr._state.length-1]+", ")+("new: "+t)),this._tokenizr._state.push(t);else{if(!(this._tokenizr._state.length>=2))throw new Error("internal error: no more states to pop");this._tokenizr._log(" STATE (POP): "+("old: "+this._tokenizr._state[this._tokenizr._state.length-1]+", ")+("new: "+this._tokenizr._state[this._tokenizr._state.length-2])),this._tokenizr._state.pop()}return this}},{key:"repeat",value:function(){return this._tokenizr._log(" REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log(" REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log(" IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log(" ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+typeof e+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2["default"](t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports["default"]=ActionContext,module.exports=exports["default"];
"use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),ActionContext=function(){function t(e){_classCallCheck(this,t),this._tokenizr=e,this._data={},this._repeat=!1,this._reject=!1,this._ignore=!1,this._match=null}return _createClass(t,[{key:"data",value:function(t,e){var n=this._data[t];return 2===arguments.length&&(this._data[t]=e),n}},{key:"info",value:function(){return{line:this._tokenizr._line,column:this._tokenizr._column,pos:this._tokenizr._pos,len:this._match[0].length}}},{key:"state",value:function(t){if("string"==typeof t)this._tokenizr._log(" STATE (PUSH): "+("old: "+this._tokenizr._state[this._tokenizr._state.length-1]+", ")+("new: "+t)),this._tokenizr._state.push(t);else{if(!(this._tokenizr._state.length>=2))throw new Error("internal error: no more states to pop");this._tokenizr._log(" STATE (POP): "+("old: "+this._tokenizr._state[this._tokenizr._state.length-1]+", ")+("new: "+this._tokenizr._state[this._tokenizr._state.length-2])),this._tokenizr._state.pop()}return this}},{key:"repeat",value:function(){return this._tokenizr._log(" REPEAT"),this._repeat=!0,this}},{key:"reject",value:function(){return this._tokenizr._log(" REJECT"),this._reject=!0,this}},{key:"ignore",value:function(){return this._tokenizr._log(" IGNORE"),this._ignore=!0,this}},{key:"accept",value:function(t,e){return arguments.length<2&&(e=this._match[0]),this._tokenizr._log(" ACCEPT: type: "+t+", value: "+JSON.stringify(e)+" ("+typeof e+'), text: "'+this._match[0]+'"'),this._tokenizr._pending.push(new _tokenizr2Token2["default"](t,e,this._match[0],this._tokenizr._pos,this._tokenizr._line,this._tokenizr._column)),this}}]),t}();exports["default"]=ActionContext,module.exports=exports["default"];
},{"./tokenizr-2-token":2}],5:[function(_dereq_,module,exports){

@@ -35,0 +35,0 @@ "use strict";function _interopRequireDefault(t){return t&&t.__esModule?t:{"default":t}}function _classCallCheck(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}Object.defineProperty(exports,"__esModule",{value:!0});var _createClass=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),_tokenizr1Excerpt=_dereq_("./tokenizr-1-excerpt"),_tokenizr1Excerpt2=_interopRequireDefault(_tokenizr1Excerpt),_tokenizr2Token=_dereq_("./tokenizr-2-token"),_tokenizr2Token2=_interopRequireDefault(_tokenizr2Token),_tokenizr3Error=_dereq_("./tokenizr-3-error"),_tokenizr3Error2=_interopRequireDefault(_tokenizr3Error),_tokenizr4Context=_dereq_("./tokenizr-4-context"),_tokenizr4Context2=_interopRequireDefault(_tokenizr4Context),Tokenizr=function(){function t(){_classCallCheck(this,t),this._rules=[],this._debug=!1,this.reset()}return _createClass(t,[{key:"reset",value:function(){return this._input="",this._len=0,this._eof=!1,this._pos=0,this._line=1,this._column=1,this._state=["default"],this._transaction=[],this._pending=[],this._ctx=new _tokenizr4Context2["default"](this),this}},{key:"debug",value:function(t){return this._debug=t,this}},{key:"_log",value:function(t){this._debug&&console.log("tokenizr: "+t)}},{key:"input",value:function(t){if("string"!=typeof t)throw new Error('parameter "input" not a String');return this.reset(),this._input=t,this._len=t.length,this}},{key:"state",value:function(t){if(1===arguments.length){if("string"!=typeof t)throw new Error('parameter "state" not a String');this._state.push(t)}else{if(0!==arguments.length)throw new Error("invalid number of arguments");if(this._state.length<2)throw new Error("no more custom states to pop");this._state.pop()}return this}},{key:"rule",value:function(t,e,n){if(2===arguments.length){var i=[t,e];e=i[0],n=i[1],t="*"}if("string"!=typeof t)throw new Error('parameter "state" not a String');if(!("object"==typeof e&&e instanceof RegExp))throw new Error('parameter "pattern" not a RegExp');if("function"!=typeof n)throw new Error('parameter "action" not a Function');t=t.split(/\s*,\s*/g);var r="g";return e.multiline&&(r+="m"),e.ignoreCase&&(r+="i"),e=new RegExp(e.source,r),this._log("rule: configure rule (state: "+t+", pattern: "+e.source+")"),this._rules.push({state:t,pattern:e,action:n}),this}},{key:"_progress",value:function(t,e){for(var n=this._line,i=this._column,r=this._input,o=t;e>o;o++){var s=r.charAt(o);"\r"===s?this._column=1:"\n"===s?(this._line++,this._column=1):" "===s?this._column+=8-this._column%8:this._column++}this._log(" PROGRESS: characters: "+(e-t)+", "+("from: <line "+n+", column "+i+">, ")+("to: <line "+this._line+", column "+this._column+">"))}},{key:"_tokenize",value:function(){if(this._pos>=this._len)return void(this._eof||(this._eof=!0,this._pending.push(new _tokenizr2Token2["default"]("EOF","","",this._pos,this._line,this._column))));for(var t=!0;t;){if(t=!1,this._debug){var e=_tokenizr1Excerpt2["default"](this._input,this._pos);this._log("INPUT: state: "+this._state[this._state.length-1]+", text: "+(e.prologTrunc?"...":'"')+(e.prologText+"<"+e.tokenText+">"+e.epilogText)+(e.epilogTrunc?"...":'"')+(", at: <line "+this._line+", column "+this._column+">"))}for(var n=0;n<this._rules.length;n++)if(this._debug&&this._log(" RULE: state(s): "+this._rules[n].state.join(",")+", pattern: "+this._rules[n].pattern.source),1===this._rules[n].state.length&&"*"===this._rules[n].state[0]||this._rules[n].state.indexOf(this._state[this._state.length-1])>=0){this._rules[n].pattern.lastIndex=this._pos;var i=this._rules[n].pattern.exec(this._input);if(this._rules[n].pattern.lastIndex=this._pos,null!==(i=this._rules[n].pattern.exec(this._input))&&i.index===this._pos){if(this._debug&&this._log(" MATCHED: "+JSON.stringify(i)),this._ctx._match=i,this._ctx._repeat=!1,this._ctx._reject=!1,this._ctx._ignore=!1,this._rules[n].action.call(this._ctx,this._ctx,i),this._ctx._reject)continue;if(this._ctx._repeat){t=!0;break}if(this._ctx._ignore){if(this._progress(this._pos,this._rules[n].pattern.lastIndex),this._pos=this._rules[n].pattern.lastIndex,this._pos>=this._len)return;t=!0;break}if(this._pending.length>0)return this._progress(this._pos,this._rules[n].pattern.lastIndex),void(this._pos=this._rules[n].pattern.lastIndex);throw new Error('action of pattern "'+this._rules[n].pattern.source+'" neither rejected nor accepted any token(s)')}}}throw this.error("token not recognized")}},{key:"token",value:function e(){if(0===this._pending.length&&this._tokenize(),this._pending.length>0){var e=this._pending.shift();return this._transaction.length>0&&this._transaction[0].push(e),this._log("TOKEN: "+e.toString()),e}return null}},{key:"tokens",value:function n(){for(var n=[],t=void 0;null!==(t=this.token());)n.push(t);return n}},{key:"peek",value:function(t){"undefined"==typeof t&&(t=0);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>=this._pending.length)throw new Error("not enough tokens available for peek operation");return this._log("PEEK: "+this._pending[t].toString()),this._pending[t]}},{key:"skip",value:function(t){"undefined"==typeof t&&(t=1);for(var e=0;e<this._pending.length+t;e++)this._tokenize();if(t>this._pending.length)throw new Error("not enough tokens available for skip operation");for(;t-->0;)this.token();return this}},{key:"consume",value:function(t,e){for(var n=0;n<this._pending.length+1;n++)this._tokenize();if(0===this._pending.length)throw new Error("not enough tokens available for consume operation");var i=this.token();if(this._log("CONSUME: "+i.toString()),2===arguments.length){if(!i.isA(t,e))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: "+JSON.stringify(e)+" ("+typeof e+")>, "+("found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+typeof i.value+")>"),i.pos,i.line,i.column,this._input)}else if(!i.isA(t))throw new _tokenizr3Error2["default"]("expected: <type: "+t+", value: * (any)>, "+("found: <type: "+i.type+", value: "+JSON.stringify(i.value)+" ("+typeof i.value+")>"),i.pos,i.line,i.column,this._input);return i}},{key:"begin",value:function(){return this._log("BEGIN: level "+this._transaction.length),this._transaction.unshift([]),this}},{key:"depth",value:function(){if(0===this._transaction.length)throw new Error("cannot determine depth -- no active transaction");return this._transaction[0].length}},{key:"commit",value:function(){if(0===this._transaction.length)throw new Error("cannot commit transaction -- no active transaction");return this._transaction.shift(),this._log("COMMIT: level "+this._transaction.length),this}},{key:"rollback",value:function(){if(0===this._transaction.length)throw new Error("cannot rollback transaction -- no active transaction");return this._pending=this._transaction[0].concat(this._pending),this._transaction.shift(),this._log("ROLLBACK: level "+this._transaction.length),this}},{key:"alternatives",value:function(){for(var t=null,e=[],n=arguments.length,i=Array(n),r=0;n>r;r++)i[r]=arguments[r];for(var o=0;o<i.length;o++)try{this.begin(),t=i[o](),this.commit();break}catch(s){this._log("EXCEPTION: "+s.toString()),e.push({ex:s,depth:this.depth()}),this.rollback();continue}if(null===t&&e.length>0)throw e=e.sort(function(t,e){return t.depth-e.depth}),e[0].ex;return t}},{key:"error",value:function(t){return new _tokenizr3Error2["default"](t,this._pos,this._line,this._column,this._input)}}]),t}();Tokenizr.Token=_tokenizr2Token2["default"],Tokenizr.ParsingError=_tokenizr3Error2["default"],Tokenizr.ActionContext=_tokenizr4Context2["default"],exports["default"]=Tokenizr,module.exports=exports["default"];

2

package.json
{
"name": "tokenizr",
"version": "0.9.5",
"version": "0.9.6",
"description": "String Tokenization Library for JavaScript",

@@ -5,0 +5,0 @@ "keywords": [ "string", "token", "scanner", "lexer" ],

3

README.md

@@ -316,2 +316,5 @@

- Method: `Tokenizr.ActionContext#info(): { line: number, column: number, pos: number, len: number }`<br/>
Retrieve information about the current matching.
- Method: `Tokenizr.ActionContext#state(new?: String): String`<br/>

@@ -318,0 +321,0 @@ Push or pop a state to/from the current tokenization state stack.

16

src/tokenizr-4-context.js

@@ -47,2 +47,12 @@ /*

/* retrieve information of current matching */
info () {
return {
line: this._tokenizr._line,
column: this._tokenizr._column,
pos: this._tokenizr._pos,
len: this._match[0].length
}
}
/* set a new state in the attached tokenizer */

@@ -74,3 +84,3 @@ state (state) {

/* mark current action to be rejected */
/* mark current matching to be rejected */
reject () {

@@ -82,3 +92,3 @@ this._tokenizr._log(` REJECT`)

/* mark current action to be ignored */
/* mark current matching to be ignored */
ignore () {

@@ -90,3 +100,3 @@ this._tokenizr._log(` IGNORE`)

/* accept a new token */
/* accept current matching is a new token */
accept (type, value) {

@@ -93,0 +103,0 @@ if (arguments.length < 2)

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc