tokenize-this - npm Package Compare versions

Comparing version 1.4.1 to 1.4.2

package.json

 {
   "name": "tokenize-this",
-  "version": "1.4.1",
+  "version": "1.4.2",
   "description": "Turns a string into tokens.",
@@ -5,0 +5,0 @@ "main": "TokenizeThis.js",

README.md

@@ -163,3 +163,3 @@ # TokenizeThis
 
-sends each token to the `forEachToken(token:String, surroundedBy:String)` callback.
+sends each token to the `forEachToken(token:String, surroundedBy:String, index:Integer)` callback.
 
@@ -169,8 +169,15 @@ ```js
 var str = 'Tokenize "this"!';
+
 var tokens = [];
-var forEachToken = function(token, surroundedBy) {
+var indices = [];
+var forEachToken = function(token, surroundedBy, index) {
+
     tokens.push(surroundedBy+token+surroundedBy);
+    indices.push(index);
 };
+
 tokenizer.tokenize(str, forEachToken);
+
 equals(tokens, ['Tokenize', '"this"', '!']);
+equals(indices, [8, 14, 15]);
 ```
@@ -177,0 +184,0 @@

test/TokenizeThis.js

@@ -30,3 +30,3 @@ "use strict";
             equals(tokens, ['Tokenize', 'this', '!']);
-            
+
         });
@@ -45,3 +45,3 @@
             });
-            
+
             equals(tokens, [5, '+', 6, '-', '(', 4, '/', 2, ')', '+', 'gcd', '(', 10, ',', 5, ')']);
@@ -58,3 +58,3 @@ });
             tokenizer.tokenize(str, function(token, surroundedBy) {
-                
+
                 if (surroundedBy) {
@@ -101,3 +101,3 @@
             var tokenizer = new TokenizeThis();
-            
+
             var str = 'Hi!, I want to add 5+6';
@@ -203,3 +203,3 @@
 
-                it('sends each token to the `forEachToken(token:String, surroundedBy:String)` callback', function() {
+                it('sends each token to the `forEachToken(token:String, surroundedBy:String, index:Integer)` callback', function() {
 
@@ -210,5 +210,7 @@ var tokenizer = new TokenizeThis();
                     var tokens = [];
-                    var forEachToken = function(token, surroundedBy) {
+                    var indices = [];
+                    var forEachToken = function(token, surroundedBy, index) {
 
                         tokens.push(surroundedBy+token+surroundedBy);
+                        indices.push(index);
                     };
@@ -219,2 +221,3 @@
                     equals(tokens, ['Tokenize', '"this"', '!']);
+                    equals(indices, [8, 14, 15]);
                 });
@@ -303,3 +306,3 @@
                     });
-                    
+
                     equals(tokens, ['These', 'are', '"quotes"']);
@@ -306,0 +309,0 @@ });

tokenize-this.min.js

@@ -1,1 +0,1 @@
-!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var t;t="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,t.TokenizeThis=e()}}(function(){return function e(t,n,r){function i(s,u){if(!n[s]){if(!t[s]){var h="function"==typeof require&&require;if(!u&&h)return h(s,!0);if(o)return o(s,!0);var a=new Error("Cannot find module '"+s+"'");throw a.code="MODULE_NOT_FOUND",a}var c=n[s]={exports:{}};t[s][0].call(c.exports,function(e){var n=t[s][1][e];return i(n?n:e)},c,c.exports,e,t,n,r)}return n[s].exports}for(var o="function"==typeof require&&require,s=0;s<r.length;s++)i(r[s]);return i}({1:[function(e,t,n){"use strict";function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}var i=function(){function e(e,t){for(var n=0;n<t.length;n++){var r=t[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(e,r.key,r)}}return function(t,n,r){return n&&e(t.prototype,n),r&&e(t,r),t}}(),o="modeNone",s="modeDefault",u="modeMatch",h=function(e,t){return e.length>t.length?-1:e.length<t.length?1:0},a=function(e,t){return e.indexOf(t,e.length-t.length)!==-1},c=function(){function e(t,n,i){r(this,e),this.factory=t,this.str=n,this.forEachToken=i,this.previousChr="",this.toMatch="",this.currentToken="",this.modeStack=[o]}return i(e,[{key:"getCurrentMode",value:function(){return this.modeStack[this.modeStack.length-1]}},{key:"setCurrentMode",value:function(e){return this.modeStack.push(e)}},{key:"completeCurrentMode",value:function(){var e=this.getCurrentMode();return e===s&&this.pushDefaultModeTokenizables(),(e===u&&""===this.currentToken||""!==this.currentToken)&&this.push(this.currentToken),this.currentToken="",this.modeStack.pop()}},{key:"push",value:function(e){var t="";if(this.factory.convertLiterals&&this.getCurrentMode()!==u)switch(e.toLowerCase()){case"null":e=null;break;case"true":e=!0;break;case"false":e=!1;break;default:isFinite(e)&&(e=Number(e))}else t=this.toMatch;this.forEachToken&&this.forEachToken(e,t)}},{key:"tokenize",value:function(){for(var e=0;e<this.str.length;)this.consume(this.str.charAt(e++));for(;this.getCurrentMode()!==o;)this.completeCurrentMode()}},{key:"consume",value:function(e){this[this.getCurrentMode()](e),this.previousChr=e}},{key:o,value:function(e){return this.factory.matchMap[e]?(this.setCurrentMode(u),void(this.toMatch=e)):(this.setCurrentMode(s),this.consume(e))}},{key:s,value:function(e){if(this.factory.delimiterMap[e])return this.completeCurrentMode();if(this.factory.matchMap[e])for(var t=0;t<this.factory.tokenizeList.length;)if(a(this.currentToken,this.factory.tokenizeList[t++]))return this.completeCurrentMode(),this.consume(e);return this.currentToken+=e,this.currentToken}},{key:"pushDefaultModeTokenizables",value:function(){for(var e=0,t=1/0,n=null;this.currentToken&&e<this.factory.tokenizeList.length;){var r=this.factory.tokenizeList[e++],i=this.currentToken.indexOf(r);i!==-1&&i<t&&(t=i,n=r)}if(n)return t>0&&this.push(this.currentToken.substring(0,t)),t!==-1?(this.push(n),this.currentToken=this.currentToken.substring(t+n.length),this.pushDefaultModeTokenizables()):void 0}},{key:u,value:function(e){if(e===this.toMatch){if(this.previousChr!==this.factory.escapeCharacter)return this.completeCurrentMode();this.currentToken=this.currentToken.substring(0,this.currentToken.length-1)}return this.currentToken+=e,this.currentToken}}]),e}(),f=function(){function e(t){var n=this;r(this,e),t||(t={}),t=Object.assign({},this.constructor.defaultConfig,t),this.convertLiterals=t.convertLiterals,this.escapeCharacter=t.escapeCharacter,this.tokenizeList=[],this.tokenizeMap={},this.matchList=[],this.matchMap={},this.delimiterList=[],this.delimiterMap={},t.shouldTokenize.sort(h).forEach(function(e){n.tokenizeMap[e]||(n.tokenizeList.push(e),n.tokenizeMap[e]=e)}),t.shouldMatch.forEach(function(e){n.matchMap[e]||(n.matchList.push(e),n.matchMap[e]=e)}),t.shouldDelimitBy.forEach(function(e){n.delimiterMap[e]||(n.delimiterList.push(e),n.delimiterMap[e]=e)})}return i(e,[{key:"tokenize",value:function(e,t){var n=new c(this,e,t);return n.tokenize()}}],[{key:"defaultConfig",get:function(){return{shouldTokenize:["(",")",",","*","/","%","+","-","=","!=","!","<",">","<=",">=","^"],shouldMatch:['"',"'","`"],shouldDelimitBy:[" ","\n","\r","\t"],convertLiterals:!0,escapeCharacter:"\\"}}}]),e}();t.exports=f},{}]},{},[1])(1)});
+!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var t;t="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,t.TokenizeThis=e()}}(function(){return function(){function e(t,n,r){function i(s,u){if(!n[s]){if(!t[s]){var h="function"==typeof require&&require;if(!u&&h)return h(s,!0);if(o)return o(s,!0);var c=new Error("Cannot find module '"+s+"'");throw c.code="MODULE_NOT_FOUND",c}var a=n[s]={exports:{}};t[s][0].call(a.exports,function(e){return i(t[s][1][e]||e)},a,a.exports,e,t,n,r)}return n[s].exports}for(var o="function"==typeof require&&require,s=0;s<r.length;s++)i(r[s]);return i}return e}()({1:[function(e,t,n){"use strict";function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}var i=function(){function e(e,t){for(var n=0;n<t.length;n++){var r=t[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(e,r.key,r)}}return function(t,n,r){return n&&e(t.prototype,n),r&&e(t,r),t}}(),o="modeNone",s=function(e,t){return e.length>t.length?-1:e.length<t.length?1:0},u=function(e,t){return-1!==e.indexOf(t,e.length-t.length)},h=function(){function e(t,n,i){r(this,e),this.factory=t,this.str=n,this.forEachToken=i,this.previousChr="",this.toMatch="",this.currentToken="",this.modeStack=[o],this.currentIndex=0}return i(e,[{key:"getCurrentMode",value:function(){return this.modeStack[this.modeStack.length-1]}},{key:"setCurrentMode",value:function(e){return this.modeStack.push(e)}},{key:"completeCurrentMode",value:function(){var e=this.getCurrentMode();return"modeDefault"===e&&this.pushDefaultModeTokenizables(),("modeMatch"===e&&""===this.currentToken||""!==this.currentToken)&&this.push(this.currentToken),this.currentToken="",this.modeStack.pop()}},{key:"push",value:function(e){var t="";if(this.factory.convertLiterals&&"modeMatch"!==this.getCurrentMode())switch(e.toLowerCase()){case"null":e=null;break;case"true":e=!0;break;case"false":e=!1;break;default:isFinite(e)&&(e=Number(e))}else t=this.toMatch;this.forEachToken&&this.forEachToken(e,t,this.currentIndex)}},{key:"tokenize",value:function(){for(var e=0;e<this.str.length;)this.currentIndex=e,this.consume(this.str.charAt(e++));for(;this.getCurrentMode()!==o;)this.completeCurrentMode()}},{key:"consume",value:function(e){this[this.getCurrentMode()](e),this.previousChr=e}},{key:o,value:function(e){if(!this.factory.matchMap[e])return this.setCurrentMode("modeDefault"),this.consume(e);this.setCurrentMode("modeMatch"),this.toMatch=e}},{key:"modeDefault",value:function(e){if(this.factory.delimiterMap[e])return this.completeCurrentMode();if(this.factory.matchMap[e])for(var t=0;t<this.factory.tokenizeList.length;)if(u(this.currentToken,this.factory.tokenizeList[t++]))return this.completeCurrentMode(),this.consume(e);return this.currentToken+=e,this.currentToken}},{key:"pushDefaultModeTokenizables",value:function(){for(var e=0,t=1/0,n=null;this.currentToken&&e<this.factory.tokenizeList.length;){var r=this.factory.tokenizeList[e++],i=this.currentToken.indexOf(r);-1!==i&&i<t&&(t=i,n=r)}if(n)return t>0&&this.push(this.currentToken.substring(0,t)),-1!==t?(this.push(n),this.currentToken=this.currentToken.substring(t+n.length),this.pushDefaultModeTokenizables()):void 0}},{key:"modeMatch",value:function(e){if(e===this.toMatch){if(this.previousChr!==this.factory.escapeCharacter)return this.completeCurrentMode();this.currentToken=this.currentToken.substring(0,this.currentToken.length-1)}return this.currentToken+=e,this.currentToken}}]),e}(),c=function(){function e(t){var n=this;r(this,e),t||(t={}),t=Object.assign({},this.constructor.defaultConfig,t),this.convertLiterals=t.convertLiterals,this.escapeCharacter=t.escapeCharacter,this.tokenizeList=[],this.tokenizeMap={},this.matchList=[],this.matchMap={},this.delimiterList=[],this.delimiterMap={},t.shouldTokenize.sort(s).forEach(function(e){n.tokenizeMap[e]||(n.tokenizeList.push(e),n.tokenizeMap[e]=e)}),t.shouldMatch.forEach(function(e){n.matchMap[e]||(n.matchList.push(e),n.matchMap[e]=e)}),t.shouldDelimitBy.forEach(function(e){n.delimiterMap[e]||(n.delimiterList.push(e),n.delimiterMap[e]=e)})}return i(e,[{key:"tokenize",value:function(e,t){return new h(this,e,t).tokenize()}}],[{key:"defaultConfig",get:function(){return{shouldTokenize:["(",")",",","*","/","%","+","-","=","!=","!","<",">","<=",">=","^"],shouldMatch:['"',"'","`"],shouldDelimitBy:[" ","\n","\r","\t"],convertLiterals:!0,escapeCharacter:"\\"}}}]),e}();t.exports=c},{}]},{},[1])(1)});

TokenizeThis.js

@@ -92,2 +92,4 @@ "use strict";
         this.modeStack = [MODE_NONE];
+
+        this.currentIndex = 0;
     }
@@ -110,3 +112,3 @@
     setCurrentMode(mode) {
-        
+
         return this.modeStack.push(mode);
@@ -178,3 +180,3 @@ }
         if (this.forEachToken) {
-            this.forEachToken(token, surroundedBy);
+            this.forEachToken(token, surroundedBy, this.currentIndex);
         }
@@ -189,2 +191,3 @@ }
 
+            this.currentIndex = index;
             this.consume(this.str.charAt(index++));
@@ -337,3 +340,3 @@ }
         this.currentToken+=chr;
-        
+
         return this.currentToken;
@@ -371,3 +374,3 @@ }
         /**
-         * 
+         *
          * @type {string}
@@ -481,2 +484,2 @@ */
  */
-module.exports = TokenizeThis;
+module.exports = TokenizeThis;

No alert changes

Improved metrics

Lines of code

636

0.95%

Number of lines in readme file

270

2.66%

Worsened metrics

Total package byte prevSize

39483

-51.13%

Number of package files

17

-5.56%

No dependency changes