@tsimons/shortcode-tokenizer - npm Package Compare versions

Comparing version 0.7.3 to 0.7.4

dist/shortcode-tokenizer.js

@@ -1,2 +0,2 @@
-module.exports=function(t){function e(i){if(n[i])return n[i].exports;var r=n[i]={i:i,l:!1,exports:{}};return t[i].call(r.exports,r,r.exports,e),r.l=!0,r.exports}var n={};return e.m=t,e.c=n,e.d=function(t,n,i){e.o(t,n)||Object.defineProperty(t,n,{configurable:!1,enumerable:!0,get:i})},e.n=function(t){var n=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(n,"a",n),n},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="/",e(e.s=0)}([function(t,e,n){t.exports=n(1)},function(t,e,n){"use strict";function i(t){if(Array.isArray(t)){for(var e=0,n=Array(t.length);e<t.length;e++)n[e]=t[e];return n}return Array.from(t)}function r(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t){return"/"===t[1]?l:"/"===t[t.length-2]?f:h}function o(t){return/^\d+$/.test(t)?Number(t):/^\d+.\d+$/.test(t)?Number(t):/^(true|false|yes|no)$/i.test(t)?"true"===(t=t.toLowerCase())||"yes"===t:t.replace(/(^['"]|['"]$)/g,"")}Object.defineProperty(e,"__esModule",{value:!0}),n.d(e,"Token",function(){return x});var u="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},a=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),h="OPEN",l="CLOSE",f="SELF_CLOSING",c="[a-zA-Z][a-zA-Z0-9_-]*",p="(?:(?:[a-zA-Z][a-zA-Z0-9_-]*=\\d+\\.\\d+|[a-zA-Z][a-zA-Z0-9_-]*=\\d+|[a-zA-Z][a-zA-Z0-9_-]*=(true|false|yes|no)|[a-zA-Z][a-zA-Z0-9_-]*=\"[^\\]\"]*(<.*\".*)?\"|[a-zA-Z][a-zA-Z0-9_-]*='[^\\]']*(<.*'.*)?'|[a-zA-Z][a-zA-Z0-9_-]*)(?:(?!\\s+/?\\])\\s|))+",y="\\[("+c+")(\\s"+p+")?\\]",d="\\[("+c+")(\\s"+p+")?\\s?\\/\\]",v=new RegExp(p.substring(0,p.length-1),"ig"),b=new RegExp("\\[\\/?([a-zA-Z][^\\]]+)\\]","i"),g=new RegExp("([a-zA-Z][a-zA-Z0-9_-]*)","i"),m=new RegExp(y,"i"),w=new RegExp("\\[\\/([a-zA-Z][a-zA-Z0-9_-]*)\\]","i"),E=new RegExp(d,"i"),x=function(){function t(e,n){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0,s=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];r(this,t),this.name=null,this.type=e,this.body=n,this.pos=i,this.strict=s,this.children=[],this.params={},this.isClosed=e===f,this.init()}return a(t,[{key:"init",value:function(){if("TEXT"!==this.type&&"ERROR"!==this.type){var t=this.matchBody();if(null===t){if(this.strict)throw new SyntaxError("Invalid "+this.type+" token: "+this.body)}else this.initName(t),t[2]&&this.initParams(t[2])}}},{key:"initName",value:function(t){this.name=t[1]}},{key:"initParams",value:function(t){var e=t.match(v);this.params=e.reduce(function(t,e){e=e.trim();var n=e.indexOf("=");return~n?t[e.substring(0,n)]=o(e.substring(n+1)):t[e]=!0,t},{})}},{key:"buildParams",value:function(){var t="";for(var e in this.params)if(this.params.hasOwnProperty(e)){var n=this.params[e],i=void 0===n?"undefined":u(n);if("string"===i)t=t+" "+e+'="'+n+'"';else if("boolean"===i){var r=n?"true":"false";t=t+" "+e+"="+r}else t=t+" "+e+"="+n}return t}},{key:"toString",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null;t instanceof Object&&(this.params=t);var e="string"==typeof t?" "+t.trim():this.buildParams();switch(this.type){case"TEXT":return this.body;case h:return"["+this.name+e+"]"+(this.children.length?"{slot}":"")+"[/"+this.name+"]";case f:return"["+this.name+e+"/]";default:return""}}},{key:"matchBody",value:function(){var t=void 0;if(this.type===l)t=w;else if(this.type===h)t=m;else{if(this.type!==f)throw new SyntaxError("Unknown token: "+this.type);t=E}return this.body.match(t)}},{key:"canClose",value:function(t){return this.name===t.name}}]),t}(),k=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null,n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{strict:!0,skipWhiteSpace:!1,whitelist:!1};r(this,t),"boolean"==typeof n&&(n={strict:n,skipWhiteSpace:!1,whitelist:!1}),this.options=Object.assign({strict:!0,skipWhiteSpace:!1,whitelist:!1},n),this.buf=null,this.originalBuf=null,this.pos=0,e&&this.input(e)}return a(t,[{key:"input",value:function(t){if("string"!=typeof t)throw new Error("Invalid input");return this.buf=this.originalBuf=t,this.pos=0,this}},{key:"reset",value:function(){return this.buf=this.originalBuf,this.pos=0,this}},{key:"tokens",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null;if(t&&this.input(t),"string"!=typeof this.buf&&this.options.strict)throw new Error("Invalid input");for(var e=[],n=[];null!==(e=this._next());)e=Array.isArray(e)?e:[e],n.push.apply(n,i(e));return n}},{key:"ast",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null,e=this.tokens(t),n=[],i=[],r=null,s=void 0,o=!0,u=!1,a=void 0;try{for(var c,p=e[Symbol.iterator]();!(o=(c=p.next()).done);o=!0)if(s=c.value,"TEXT"===s.type){if(this.options.skipWhiteSpace&&0===s.body.replace(/\s+/g,"").length)continue;r?r.children.push(s):i.push(s)}else if(s.type===h)r?(r.children.push(s),n.push(r),r=s):(r=s,i.push(r));else if(s.type===l)if(r&&s.canClose(r))r.isClosed=!0,r=n.pop();else{if(this.options.strict)throw new SyntaxError("Unmatched close token: "+s.body);var y=new x("ERROR",s.body,0,this.options.strict);r?r.children.push(y):i.push(y)}else if(s.type===f)r?r.children.push(s):i.push(s);else if(this.options.strict)throw new SyntaxError("Unknown token: "+s.type)}catch(y){u=!0,a=y}finally{try{!o&&p.return&&p.return()}finally{if(u)throw a}}if(r){if(this.options.strict)throw new SyntaxError("Unmatched open token: "+r.body);i.push(new x("ERROR",s.body,0,this.options.strict))}return i}},{key:"buildTemplate",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null;if(!t)return"";if(!(t instanceof x))throw new Error("Expected Token instance.");return function t(n){var i=n.children.map(function(e){return e.children.length?t(e):e.toString()});return n.toString(e).replace("{slot}",i.join(""))}(t)}},{key:"_next",value:function(){if(!this.buf)return null;var t=this.buf.match(b);if(t&&this.options.whitelist){var e=t[1].match(g)[1];-1===this.options.whitelist.indexOf(e)&&(t=null)}if(null===t){var n=new x("TEXT",this.buf,this.pos,this.options.strict);return this.pos+=this.buf.length,this.buf=null,n}var i=[];return 0!==t.index&&i.push(new x("TEXT",this.buf.substring(0,t.index),this.pos,this.options.strict)),i.push(new x(s(t[0]),t[0],this.pos+t.index,this.options.strict)),this.buf=this.buf.substring(t.index+t[0].length),this.pos+=t.index+t[0].length,0===this.buf.length&&(this.buf=null),i}},{key:"strict",get:function(){return console.warn("Deprecated: use options.strict instead"),this.options.strict},set:function(t){console.warn("Deprecated: use options.strict = "+t+" instead"),this.options.strict=t}}]),t}();e.default=k,Object.assign(k,{TEXT:"TEXT",ERROR:"ERROR",OPEN:h,CLOSE:l,SELF_CLOSING:f,rxParams:v,rxEnclosure:b,rxOpen:m,rxClose:w,rxSelfclosing:E})}]);
+module.exports=function(t){function e(i){if(n[i])return n[i].exports;var r=n[i]={i:i,l:!1,exports:{}};return t[i].call(r.exports,r,r.exports,e),r.l=!0,r.exports}var n={};return e.m=t,e.c=n,e.d=function(t,n,i){e.o(t,n)||Object.defineProperty(t,n,{configurable:!1,enumerable:!0,get:i})},e.n=function(t){var n=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(n,"a",n),n},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="/",e(e.s=0)}([function(t,e,n){t.exports=n(1)},function(t,e,n){"use strict";function i(t){if(Array.isArray(t)){for(var e=0,n=Array(t.length);e<t.length;e++)n[e]=t[e];return n}return Array.from(t)}function r(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t){return"/"===t[1]?c:"/"===t[t.length-2]?f:l}function o(t){return/^\d+$/.test(t)?Number(t):/^\d+.\d+$/.test(t)?Number(t):/^(true|false|yes|no)$/i.test(t)?"true"===(t=t.toLowerCase())||"yes"===t:t.replace(/(^['"]|['"]$)/g,"")}Object.defineProperty(e,"__esModule",{value:!0}),n.d(e,"Token",function(){return E});var u="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},a=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),h="TEXT",l="OPEN",c="CLOSE",f="SELF_CLOSING",p="[a-zA-Z][a-zA-Z0-9_-]*",y="(?:(?:[a-zA-Z][a-zA-Z0-9_-]*=\\d+\\.\\d+|[a-zA-Z][a-zA-Z0-9_-]*=\\d+|[a-zA-Z][a-zA-Z0-9_-]*=(true|false|yes|no)|[a-zA-Z][a-zA-Z0-9_-]*=\"[^\\]\"]*(<.*\".*)?\"|[a-zA-Z][a-zA-Z0-9_-]*='[^\\]']*(<.*'.*)?'|[a-zA-Z][a-zA-Z0-9_-]*)(?:(?!\\s+/?\\])\\s|))+",d="\\[("+p+")(\\s"+y+")?\\]",v="\\[("+p+")(\\s"+y+")?\\s?\\/\\]",b=new RegExp(y.substring(0,y.length-1),"ig"),g=new RegExp("\\[\\/?([a-zA-Z][^\\]]+)\\]","i"),m=new RegExp("([a-zA-Z][a-zA-Z0-9_-]*)","i"),w=new RegExp(d,"i"),x=new RegExp("\\[\\/([a-zA-Z][a-zA-Z0-9_-]*)\\]","i"),k=new RegExp(v,"i"),E=function(){function t(e,n){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0,s=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];r(this,t),this.name=null,this.type=e,this.body=n,this.pos=i,this.strict=s,this.children=[],this.params={},this.isClosed=e===f,this.init()}return a(t,[{key:"init",value:function(){if(this.type!==h&&"ERROR"!==this.type){var t=this.matchBody();if(null===t){if(this.strict)throw new SyntaxError("Invalid "+this.type+" token: "+this.body)}else this.initName(t),t[2]&&this.initParams(t[2])}}},{key:"initName",value:function(t){this.name=t[1]}},{key:"initParams",value:function(t){var e=t.match(b);this.params=e.reduce(function(t,e){e=e.trim();var n=e.indexOf("=");return~n?t[e.substring(0,n)]=o(e.substring(n+1)):t[e]=!0,t},{})}},{key:"buildParams",value:function(){var t="";for(var e in this.params)if(this.params.hasOwnProperty(e)){var n=this.params[e],i=void 0===n?"undefined":u(n);if("string"===i)t=t+" "+e+'="'+n+'"';else if("boolean"===i){var r=n?"true":"false";t=t+" "+e+"="+r}else t=t+" "+e+"="+n}return t}},{key:"toString",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null;t instanceof Object&&(this.params=t);var e="string"==typeof t?" "+t.trim():this.buildParams();switch(this.type){case h:return this.body;case l:return"["+this.name+e+"]"+(this.children.length?"{slot}":"")+"[/"+this.name+"]";case f:return"["+this.name+e+"/]";default:return""}}},{key:"matchBody",value:function(){var t=void 0;if(this.type===c)t=x;else if(this.type===l)t=w;else{if(this.type!==f)throw new SyntaxError("Unknown token: "+this.type);t=k}return this.body.match(t)}},{key:"canClose",value:function(t){return this.name===t.name}}]),t}(),A=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null,n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{strict:!0,skipWhiteSpace:!1,whitelist:!1};r(this,t),"boolean"==typeof n&&(n={strict:n,skipWhiteSpace:!1,whitelist:!1}),this.options=Object.assign({strict:!0,skipWhiteSpace:!1,whitelist:!1},n),this.buf=null,this.originalBuf=null,this.pos=0,e&&this.input(e)}return a(t,[{key:"input",value:function(t){if("string"!=typeof t)throw new Error("Invalid input");return this.buf=this.originalBuf=t,this.pos=0,this}},{key:"reset",value:function(){return this.buf=this.originalBuf,this.pos=0,this}},{key:"tokens",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null;if(t&&this.input(t),"string"!=typeof this.buf&&this.options.strict)throw new Error("Invalid input");for(var e=[],n=[];null!==(e=this._next());)e=Array.isArray(e)?e:[e],n.push.apply(n,i(e));return n}},{key:"ast",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null,e=this.tokens(t),n=[],i=[],r=null,s=void 0,o=!0,u=!1,a=void 0;try{for(var p,y=e[Symbol.iterator]();!(o=(p=y.next()).done);o=!0)if(s=p.value,s.type===h){if(this.options.skipWhiteSpace&&0===s.body.replace(/\s+/g,"").length)continue;r?r.children.push(s):i.push(s)}else if(s.type===l)r?(r.children.push(s),n.push(r),r=s):(r=s,i.push(r));else if(s.type===c)if(r&&s.canClose(r))r.isClosed=!0,r=n.pop();else{if(this.options.strict)throw new SyntaxError("Unmatched close token: "+s.body);var d=new E("ERROR",s.body,0,this.options.strict);r?r.children.push(d):i.push(d)}else if(s.type===f)r?r.children.push(s):i.push(s);else if(this.options.strict)throw new SyntaxError("Unknown token: "+s.type)}catch(d){u=!0,a=d}finally{try{!o&&y.return&&y.return()}finally{if(u)throw a}}if(r){if(this.options.strict)throw new SyntaxError("Unmatched open token: "+r.body);i.push(new E("ERROR",s.body,0,this.options.strict))}return i}},{key:"buildTemplate",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null;if(!t)return"";if(!(t instanceof E))throw new Error("Expected Token instance.");return function t(n){var i=n.children.map(function(e){return e.children.length?t(e):e.toString()});return n.toString(e).replace("{slot}",i.join(""))}(t)}},{key:"_next",value:function(){if(!this.buf)return null;var t=this.buf.match(g);if(null===t){var e=new E(h,this.buf,this.pos,this.options.strict);return this.pos+=this.buf.length,this.buf=null,e}var n=[];0!==t.index&&n.push(new E(h,this.buf.substring(0,t.index),this.pos,this.options.strict));var i=t[1].match(m)[1],r=null;return r=this.options.whitelist&&-1===this.options.whitelist.indexOf(i)?h:s(t[0]),n.push(new E(r,t[0],this.pos+t.index,this.options.strict)),this.buf=this.buf.substring(t.index+t[0].length),this.pos+=t.index+t[0].length,0===this.buf.length&&(this.buf=null),n}},{key:"strict",get:function(){return console.warn("Deprecated: use options.strict instead"),this.options.strict},set:function(t){console.warn("Deprecated: use options.strict = "+t+" instead"),this.options.strict=t}}]),t}();e.default=A,Object.assign(A,{TEXT:h,ERROR:"ERROR",OPEN:l,CLOSE:c,SELF_CLOSING:f,rxParams:b,rxEnclosure:g,rxOpen:w,rxClose:x,rxSelfclosing:k})}]);
 //# sourceMappingURL=shortcode-tokenizer.js.map

package.json

 {
   "name": "@tsimons/shortcode-tokenizer",
-  "version": "0.7.3",
+  "version": "0.7.4",
   "public": true,
@@ -8,4 +8,3 @@ "description": "",
   "scripts": {
-    "test":
-      "poi test && cat ./coverage/report-lcov/lcov.info | coveralls -v && rm -rf ./coverage ",
+    "test": "poi test && cat ./coverage/report-lcov/lcov.info | coveralls -v && rm -rf ./coverage ",
     "watch": "poi",
@@ -12,0 +11,0 @@ "build": "poi build --format cjs",

src/shortcode-tokenizer.js

@@ -433,11 +433,2 @@ /** @module ShortcodeTokenizer */
 
-    // Whitelist - Treat as text if not whitelisted
-    if (match && this.options.whitelist) {
-      let name = match[1].match(rxName)[1]
-
-      if (this.options.whitelist.indexOf(name) === -1) {
-        match = null
-      }
-    }
-
     // all text
@@ -463,5 +454,15 @@ if (match === null) {
 
-    // matching token
+    // set TokenType,
+    // non-whitelisted = TEXT
+    const name = match[1].match(rxName)[1];
+    let tokenType = null;
+    if (this.options.whitelist && this.options.whitelist.indexOf(name) === -1) {
+      tokenType = TEXT;
+    } else {
+      tokenType = getTokenType(match[0]);
+    }
+
+    // Regular TOKEN
     tokens.push(new Token(
-      getTokenType(match[0]),
+      tokenType,
       match[0],
@@ -468,0 +469,0 @@ this.pos + match.index,

test/unit/shortcode-tokenizer.test.js

@@ -179,9 +179,11 @@ import * as lib from '../../src/shortcode-tokenizer'
     it('should treat non-whitelisted tokens as TEXT', () => {
-      let input = '[thisone] [notthisone]'
+      let input = '[notthisone] [thisone]'
       tokenizer.options.whitelist = ['thisone'];
       const result = tokenizer.tokens(input);
-      expect(result[0].name).to.equal('thisone');
-      expect(result[0].type).to.equal('OPEN');
+      expect(result[0].name).to.equal(null);
+      expect(result[0].type).to.equal("TEXT");
       expect(result[1].name).to.equal(null);
-      expect(result[1].type).to.equal('TEXT');
+      expect(result[1].type).to.equal("TEXT");
+      expect(result[2].name).to.equal("thisone");
+      expect(result[2].type).to.equal("OPEN");
     })
@@ -188,0 +190,0 @@ })

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

395819

increased by0.06%

Lines of code

946

increased by0.42%

No dependency changes