tokenize-this - npm Package Compare versions

Comparing version 1.3.7 to 1.4.0

package.json

 {
   "name": "tokenize-this",
-  "version": "1.3.7",
+  "version": "1.4.0",
   "description": "Turns a string into tokens.",
-  "main": "tokenize-this.min.js",
+  "main": "TokenizeThis.js",
   "directories": {
@@ -7,0 +7,0 @@ "test": "test"

README.md

@@ -72,2 +72,7 @@ # TokenizeThis
 // var TokenizeThis = require('tokenize-this');
+// OR
+// var TokenizeThis = require('tokenize-this/tokenize-this.min.js'); // for node.js < 4.0
+// OR
+// <script src="tokenize-this/tokenize-this.min.js"></script> <!-- if in browser -->
+
 var tokenizer = new TokenizeThis();
@@ -74,0 +79,0 @@

test/TokenizeThis.js

@@ -92,2 +92,7 @@ "use strict";
             // var TokenizeThis = require('tokenize-this');
+            // OR
+            // var TokenizeThis = require('tokenize-this/tokenize-this.min.js'); // for node.js < 4.0
+            // OR
+            // <script src="tokenize-this/tokenize-this.min.js"></script> <!-- if in browser -->
+
             var tokenizer = new TokenizeThis();
@@ -119,3 +124,4 @@
                         shouldDelimitBy: [' ', "\n", "\r", "\t", ':', ','],
-                        convertLiterals: true
+                        convertLiterals: true,
+                        escapeCharacter: "\\"
                     };
@@ -165,2 +171,4 @@
                     });
+                    
+                    console.log(tokens);
 
@@ -167,0 +175,0 @@ equals(tokens,

tokenize-this.min.js

@@ -1,1 +0,1 @@
-!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var t;t="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,t.TokenizeThis=e()}}(function(){return function e(t,n,r){function i(s,u){if(!n[s]){if(!t[s]){var h="function"==typeof require&&require;if(!u&&h)return h(s,!0);if(o)return o(s,!0);var a=new Error("Cannot find module '"+s+"'");throw a.code="MODULE_NOT_FOUND",a}var c=n[s]={exports:{}};t[s][0].call(c.exports,function(e){var n=t[s][1][e];return i(n?n:e)},c,c.exports,e,t,n,r)}return n[s].exports}for(var o="function"==typeof require&&require,s=0;s<r.length;s++)i(r[s]);return i}({1:[function(e,t,n){"use strict";function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}var i=function(){function e(e,t){for(var n=0;n<t.length;n++){var r=t[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(e,r.key,r)}}return function(t,n,r){return n&&e(t.prototype,n),r&&e(t,r),t}}(),o="modeNone",s="modeDefault",u="modeMatch",h=function(e,t){return e.length>t.length?-1:e.length<t.length?1:0},a=function(){function e(t,n,i){r(this,e),this.factory=t,this.str=n,this.forEachToken=i,this.previousChr="",this.toMatch="",this.currentToken="",this.modeStack=[o]}return i(e,[{key:"getCurrentMode",value:function(){return this.modeStack[this.modeStack.length-1]}},{key:"setCurrentMode",value:function(e){return this.modeStack.push(e)}},{key:"completeCurrentMode",value:function(){var e=this.getCurrentMode();return e===s&&this.pushDefaultModeTokenizables(),(e===u&&""===this.currentToken||""!==this.currentToken)&&this.push(this.currentToken),this.currentToken="",this.modeStack.pop()}},{key:"push",value:function(e){var t="";if(this.factory.convertLiterals&&this.getCurrentMode()!==u)switch(e.toLowerCase()){case"null":e=null;break;case"true":e=!0;break;case"false":e=!1;break;default:isFinite(e)&&(e=Number(e))}else t=this.toMatch;this.forEachToken&&this.forEachToken(e,t)}},{key:"tokenize",value:function(){for(var e=0;e<this.str.length;)this.consume(this.str.charAt(e++));for(;this.getCurrentMode()!==o;)this.completeCurrentMode()}},{key:"consume",value:function(e){this[this.getCurrentMode()](e),this.previousChr=e}},{key:o,value:function(e){return this.factory.matchMap[e]?(this.setCurrentMode(u),void(this.toMatch=e)):(this.setCurrentMode(s),this.consume(e))}},{key:s,value:function(e){return this.factory.delimiterMap[e]?this.completeCurrentMode():this.factory.matchMap[e]?(this.completeCurrentMode(),this.consume(e)):(this.currentToken+=e,this.currentToken)}},{key:"pushDefaultModeTokenizables",value:function(){for(var e=0,t=1/0,n=null;this.currentToken&&e<this.factory.tokenizeList.length;){var r=this.factory.tokenizeList[e++],i=this.currentToken.indexOf(r);i!==-1&&i<t&&(t=i,n=r)}if(n)return t>0&&this.push(this.currentToken.substring(0,t)),t!==-1?(this.push(n),this.currentToken=this.currentToken.substring(t+n.length),this.pushDefaultModeTokenizables()):void 0}},{key:u,value:function(e){if(e===this.toMatch){if(this.previousChr!==this.factory.escapeCharacter)return this.completeCurrentMode();this.currentToken=this.currentToken.substring(0,this.currentToken.length-1)}return this.currentToken+=e,this.currentToken}}]),e}(),c=function(){function e(t){var n=this;r(this,e),t||(t={}),t=Object.assign({},this.constructor.defaultConfig,t),this.convertLiterals=t.convertLiterals,this.escapeCharacter=t.escapeCharacter,this.tokenizeList=[],this.tokenizeMap={},this.matchList=[],this.matchMap={},this.delimiterList=[],this.delimiterMap={},t.shouldTokenize.sort(h).forEach(function(e){n.tokenizeMap[e]||(n.tokenizeList.push(e),n.tokenizeMap[e]=e)}),t.shouldMatch.forEach(function(e){n.matchMap[e]||(n.matchList.push(e),n.matchMap[e]=e)}),t.shouldDelimitBy.forEach(function(e){n.delimiterMap[e]||(n.delimiterList.push(e),n.delimiterMap[e]=e)})}return i(e,[{key:"tokenize",value:function(e,t){var n=new a(this,e,t);return n.tokenize()}}],[{key:"defaultConfig",get:function(){return{shouldTokenize:["(",")",",","*","/","%","+","-","=","!=","!","<",">","<=",">=","^"],shouldMatch:['"',"'","`"],shouldDelimitBy:[" ","\n","\r","\t"],convertLiterals:!0,escapeCharacter:"\\"}}}]),e}();t.exports=c},{}]},{},[1])(1)});
+!function(e){if("object"==typeof exports&&"undefined"!=typeof module)module.exports=e();else if("function"==typeof define&&define.amd)define([],e);else{var t;t="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof self?self:this,t.TokenizeThis=e()}}(function(){return function e(t,n,r){function i(s,u){if(!n[s]){if(!t[s]){var h="function"==typeof require&&require;if(!u&&h)return h(s,!0);if(o)return o(s,!0);var a=new Error("Cannot find module '"+s+"'");throw a.code="MODULE_NOT_FOUND",a}var c=n[s]={exports:{}};t[s][0].call(c.exports,function(e){var n=t[s][1][e];return i(n?n:e)},c,c.exports,e,t,n,r)}return n[s].exports}for(var o="function"==typeof require&&require,s=0;s<r.length;s++)i(r[s]);return i}({1:[function(e,t,n){"use strict";function r(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}var i=function(){function e(e,t){for(var n=0;n<t.length;n++){var r=t[n];r.enumerable=r.enumerable||!1,r.configurable=!0,"value"in r&&(r.writable=!0),Object.defineProperty(e,r.key,r)}}return function(t,n,r){return n&&e(t.prototype,n),r&&e(t,r),t}}(),o="modeNone",s="modeDefault",u="modeMatch",h=function(e,t){return e.length>t.length?-1:e.length<t.length?1:0},a=function(e,t){return e.indexOf(t,e.length-t.length)!==-1},c=function(){function e(t,n,i){r(this,e),this.factory=t,this.str=n,this.forEachToken=i,this.previousChr="",this.toMatch="",this.currentToken="",this.modeStack=[o]}return i(e,[{key:"getCurrentMode",value:function(){return this.modeStack[this.modeStack.length-1]}},{key:"setCurrentMode",value:function(e){return this.modeStack.push(e)}},{key:"completeCurrentMode",value:function(){var e=this.getCurrentMode();return e===s&&this.pushDefaultModeTokenizables(),(e===u&&""===this.currentToken||""!==this.currentToken)&&this.push(this.currentToken),this.currentToken="",this.modeStack.pop()}},{key:"push",value:function(e){var t="";if(this.factory.convertLiterals&&this.getCurrentMode()!==u)switch(e.toLowerCase()){case"null":e=null;break;case"true":e=!0;break;case"false":e=!1;break;default:isFinite(e)&&(e=Number(e))}else t=this.toMatch;this.forEachToken&&this.forEachToken(e,t)}},{key:"tokenize",value:function(){for(var e=0;e<this.str.length;)this.consume(this.str.charAt(e++));for(;this.getCurrentMode()!==o;)this.completeCurrentMode()}},{key:"consume",value:function(e){this[this.getCurrentMode()](e),this.previousChr=e}},{key:o,value:function(e){return this.factory.matchMap[e]?(this.setCurrentMode(u),void(this.toMatch=e)):(this.setCurrentMode(s),this.consume(e))}},{key:s,value:function(e){if(this.factory.delimiterMap[e])return this.completeCurrentMode();if(this.factory.matchMap[e])for(var t=0;t<this.factory.tokenizeList.length;)if(a(this.currentToken,this.factory.tokenizeList[t++]))return this.completeCurrentMode(),this.consume(e);return this.currentToken+=e,this.currentToken}},{key:"pushDefaultModeTokenizables",value:function(){for(var e=0,t=1/0,n=null;this.currentToken&&e<this.factory.tokenizeList.length;){var r=this.factory.tokenizeList[e++],i=this.currentToken.indexOf(r);i!==-1&&i<t&&(t=i,n=r)}if(n)return t>0&&this.push(this.currentToken.substring(0,t)),t!==-1?(this.push(n),this.currentToken=this.currentToken.substring(t+n.length),this.pushDefaultModeTokenizables()):void 0}},{key:u,value:function(e){if(e===this.toMatch){if(this.previousChr!==this.factory.escapeCharacter)return this.completeCurrentMode();this.currentToken=this.currentToken.substring(0,this.currentToken.length-1)}return this.currentToken+=e,this.currentToken}}]),e}(),f=function(){function e(t){var n=this;r(this,e),t||(t={}),t=Object.assign({},this.constructor.defaultConfig,t),this.convertLiterals=t.convertLiterals,this.escapeCharacter=t.escapeCharacter,this.tokenizeList=[],this.tokenizeMap={},this.matchList=[],this.matchMap={},this.delimiterList=[],this.delimiterMap={},t.shouldTokenize.sort(h).forEach(function(e){n.tokenizeMap[e]||(n.tokenizeList.push(e),n.tokenizeMap[e]=e)}),t.shouldMatch.forEach(function(e){n.matchMap[e]||(n.matchList.push(e),n.matchMap[e]=e)}),t.shouldDelimitBy.forEach(function(e){n.delimiterMap[e]||(n.delimiterList.push(e),n.delimiterMap[e]=e)})}return i(e,[{key:"tokenize",value:function(e,t){var n=new c(this,e,t);return n.tokenize()}}],[{key:"defaultConfig",get:function(){return{shouldTokenize:["(",")",",","*","/","%","+","-","=","!=","!","<",">","<=",">=","^"],shouldMatch:['"',"'","`"],shouldDelimitBy:[" ","\n","\r","\t"],convertLiterals:!0,escapeCharacter:"\\"}}}]),e}();t.exports=f},{}]},{},[1])(1)});

TokenizeThis.js

@@ -25,2 +25,6 @@ "use strict";
 
+const endsWith = (str, suffix) => {
+    return str.indexOf(suffix, str.length - suffix.length) !== -1;
+};
+
 /**
@@ -226,2 +230,5 @@ * Create an instance of this class for each new string you wish to parse.
 
+        /**
+         * If we encounter a delimiter, its time to push out the current token.
+         */
         if (this.factory.delimiterMap[chr]) {
@@ -231,7 +238,17 @@
         }
-        
+
+        /**
+         * If we encounter a quote, only push out the current token if there's a sub-token directly before it.
+         */
         if (this.factory.matchMap[chr]) {
-            
-            this.completeCurrentMode();
-            return this.consume(chr);
+
+            let tokenizeIndex = 0;
+
+            while (tokenizeIndex < this.factory.tokenizeList.length) {
+
+                if (endsWith(this.currentToken, this.factory.tokenizeList[tokenizeIndex++])) {
+                    this.completeCurrentMode();
+                    return this.consume(chr);
+                }
+            }
         }
@@ -252,3 +269,3 @@
         // TODO: refactor this to be more performant.
-        
+
         let tokenizeIndex = 0;
@@ -255,0 +272,0 @@ let lowestIndexOfTokenize = Infinity;

No alert changes

Improved metrics

Total package byte prevSize

77157

2.59%

Lines of code

631

3.27%

Number of lines in readme file

263

1.94%

No dependency changes