@tsimons/shortcode-tokenizer - npm Package Compare versions

Comparing version 0.7.2 to 0.7.3

dist/shortcode-tokenizer.js

@@ -1,2 +0,2 @@
-module.exports=function(t){function e(i){if(n[i])return n[i].exports;var r=n[i]={i:i,l:!1,exports:{}};return t[i].call(r.exports,r,r.exports,e),r.l=!0,r.exports}var n={};return e.m=t,e.c=n,e.d=function(t,n,i){e.o(t,n)||Object.defineProperty(t,n,{configurable:!1,enumerable:!0,get:i})},e.n=function(t){var n=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(n,"a",n),n},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="/",e(e.s=0)}([function(t,e,n){t.exports=n(1)},function(t,e,n){"use strict";function i(t){if(Array.isArray(t)){for(var e=0,n=Array(t.length);e<t.length;e++)n[e]=t[e];return n}return Array.from(t)}function r(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t){return"/"===t[1]?h:"/"===t[t.length-2]?c:l}function o(t){return/^\d+$/.test(t)?Number(t):/^\d+.\d+$/.test(t)?Number(t):/^(true|false|yes|no)$/i.test(t)?"true"===(t=t.toLowerCase())||"yes"===t:t.replace(/(^['"]|['"]$)/g,"")}Object.defineProperty(e,"__esModule",{value:!0}),n.d(e,"Token",function(){return E});var u="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},a=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),l="OPEN",h="CLOSE",c="SELF_CLOSING",f="[a-zA-Z][a-zA-Z0-9_-]*",p="(?:(?:[a-zA-Z][a-zA-Z0-9_-]*=\\d+\\.\\d+|[a-zA-Z][a-zA-Z0-9_-]*=\\d+|[a-zA-Z][a-zA-Z0-9_-]*=(true|false|yes|no)|[a-zA-Z][a-zA-Z0-9_-]*=\"[^\\]\"]*(<.*\".*)?\"|[a-zA-Z][a-zA-Z0-9_-]*='[^\\]']*(<.*'.*)?'|[a-zA-Z][a-zA-Z0-9_-]*)(?:(?!\\s+/?\\])\\s|))+",y="\\[("+f+")(\\s"+p+")?\\]",d="\\[("+f+")(\\s"+p+")?\\s?\\/\\]",v=new RegExp(p.substring(0,p.length-1),"ig"),b=new RegExp("\\[\\/?[a-zA-Z][^\\]]+\\]","i"),g=new RegExp(y,"i"),m=new RegExp("\\[\\/([a-zA-Z][a-zA-Z0-9_-]*)\\]","i"),w=new RegExp(d,"i"),E=function(){function t(e,n){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0,s=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];r(this,t),this.name=null,this.type=e,this.body=n,this.pos=i,this.strict=s,this.children=[],this.params={},this.isClosed=e===c,this.init()}return a(t,[{key:"init",value:function(){if("TEXT"!==this.type&&"ERROR"!==this.type){var t=this.matchBody();if(null===t){if(this.strict)throw new SyntaxError("Invalid "+this.type+" token: "+this.body)}else this.initName(t),t[2]&&this.initParams(t[2])}}},{key:"initName",value:function(t){this.name=t[1]}},{key:"initParams",value:function(t){var e=t.match(v);this.params=e.reduce(function(t,e){e=e.trim();var n=e.indexOf("=");return~n?t[e.substring(0,n)]=o(e.substring(n+1)):t[e]=!0,t},{})}},{key:"buildParams",value:function(){var t="";for(var e in this.params)if(this.params.hasOwnProperty(e)){var n=this.params[e],i=void 0===n?"undefined":u(n);if("string"===i)t=t+" "+e+'="'+n+'"';else if("boolean"===i){var r=n?"true":"false";t=t+" "+e+"="+r}else t=t+" "+e+"="+n}return t}},{key:"toString",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null;t instanceof Object&&(this.params=t);var e="string"==typeof t?" "+t.trim():this.buildParams();switch(this.type){case"TEXT":return this.body;case l:return"["+this.name+e+"]"+(this.children.length?"{slot}":"")+"[/"+this.name+"]";case c:return"["+this.name+e+"/]";default:return""}}},{key:"matchBody",value:function(){var t=void 0;if(this.type===h)t=m;else if(this.type===l)t=g;else{if(this.type!==c)throw new SyntaxError("Unknown token: "+this.type);t=w}return this.body.match(t)}},{key:"canClose",value:function(t){return this.name===t.name}}]),t}(),x=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null,n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{strict:!0,skipWhiteSpace:!1};r(this,t),"boolean"==typeof n&&(n={strict:n,skipWhiteSpace:!1}),this.options=Object.assign({strict:!0,skipWhiteSpace:!1},n),this.buf=null,this.originalBuf=null,this.pos=0,e&&this.input(e)}return a(t,[{key:"input",value:function(t){if("string"!=typeof t)throw new Error("Invalid input");return this.buf=this.originalBuf=t,this.pos=0,this}},{key:"reset",value:function(){return this.buf=this.originalBuf,this.pos=0,this}},{key:"tokens",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null;if(t&&this.input(t),"string"!=typeof this.buf&&this.options.strict)throw new Error("Invalid input");for(var e=[],n=[];null!==(e=this._next());)e=Array.isArray(e)?e:[e],n.push.apply(n,i(e));return n}},{key:"ast",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null,e=this.tokens(t),n=[],i=[],r=null,s=void 0,o=!0,u=!1,a=void 0;try{for(var f,p=e[Symbol.iterator]();!(o=(f=p.next()).done);o=!0)if(s=f.value,"TEXT"===s.type){if(this.options.skipWhiteSpace&&0===s.body.replace(/\s+/g,"").length)continue;r?r.children.push(s):i.push(s)}else if(s.type===l)r?(r.children.push(s),n.push(r),r=s):(r=s,i.push(r));else if(s.type===h)if(r&&s.canClose(r))r.isClosed=!0,r=n.pop();else{if(this.options.strict)throw new SyntaxError("Unmatched close token: "+s.body);var y=new E("ERROR",s.body,0,this.options.strict);r?r.children.push(y):i.push(y)}else if(s.type===c)r?r.children.push(s):i.push(s);else if(this.options.strict)throw new SyntaxError("Unknown token: "+s.type)}catch(y){u=!0,a=y}finally{try{!o&&p.return&&p.return()}finally{if(u)throw a}}if(r){if(this.options.strict)throw new SyntaxError("Unmatched open token: "+r.body);i.push(new E("ERROR",s.body,0,this.options.strict))}return i}},{key:"buildTemplate",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null;if(!t)return"";if(!(t instanceof E))throw new Error("Expected Token instance.");return function t(n){var i=n.children.map(function(e){return e.children.length?t(e):e.toString()});return n.toString(e).replace("{slot}",i.join(""))}(t)}},{key:"_next",value:function(){if(!this.buf)return null;var t=this.buf.match(b);if(null===t){var e=new E("TEXT",this.buf,this.pos,this.options.strict);return this.pos+=this.buf.length,this.buf=null,e}var n=[];return 0!==t.index&&n.push(new E("TEXT",this.buf.substring(0,t.index),this.pos,this.options.strict)),n.push(new E(s(t[0]),t[0],this.pos+t.index,this.options.strict)),this.buf=this.buf.substring(t.index+t[0].length),this.pos+=t.index+t[0].length,0===this.buf.length&&(this.buf=null),n}},{key:"strict",get:function(){return console.warn("Deprecated: use options.strict instead"),this.options.strict},set:function(t){console.warn("Deprecated: use options.strict = "+t+" instead"),this.options.strict=t}}]),t}();e.default=x,Object.assign(x,{TEXT:"TEXT",ERROR:"ERROR",OPEN:l,CLOSE:h,SELF_CLOSING:c,rxParams:v,rxEnclosure:b,rxOpen:g,rxClose:m,rxSelfclosing:w})}]);
+module.exports=function(t){function e(i){if(n[i])return n[i].exports;var r=n[i]={i:i,l:!1,exports:{}};return t[i].call(r.exports,r,r.exports,e),r.l=!0,r.exports}var n={};return e.m=t,e.c=n,e.d=function(t,n,i){e.o(t,n)||Object.defineProperty(t,n,{configurable:!1,enumerable:!0,get:i})},e.n=function(t){var n=t&&t.__esModule?function(){return t.default}:function(){return t};return e.d(n,"a",n),n},e.o=function(t,e){return Object.prototype.hasOwnProperty.call(t,e)},e.p="/",e(e.s=0)}([function(t,e,n){t.exports=n(1)},function(t,e,n){"use strict";function i(t){if(Array.isArray(t)){for(var e=0,n=Array(t.length);e<t.length;e++)n[e]=t[e];return n}return Array.from(t)}function r(t,e){if(!(t instanceof e))throw new TypeError("Cannot call a class as a function")}function s(t){return"/"===t[1]?l:"/"===t[t.length-2]?f:h}function o(t){return/^\d+$/.test(t)?Number(t):/^\d+.\d+$/.test(t)?Number(t):/^(true|false|yes|no)$/i.test(t)?"true"===(t=t.toLowerCase())||"yes"===t:t.replace(/(^['"]|['"]$)/g,"")}Object.defineProperty(e,"__esModule",{value:!0}),n.d(e,"Token",function(){return x});var u="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(t){return typeof t}:function(t){return t&&"function"==typeof Symbol&&t.constructor===Symbol&&t!==Symbol.prototype?"symbol":typeof t},a=function(){function t(t,e){for(var n=0;n<e.length;n++){var i=e[n];i.enumerable=i.enumerable||!1,i.configurable=!0,"value"in i&&(i.writable=!0),Object.defineProperty(t,i.key,i)}}return function(e,n,i){return n&&t(e.prototype,n),i&&t(e,i),e}}(),h="OPEN",l="CLOSE",f="SELF_CLOSING",c="[a-zA-Z][a-zA-Z0-9_-]*",p="(?:(?:[a-zA-Z][a-zA-Z0-9_-]*=\\d+\\.\\d+|[a-zA-Z][a-zA-Z0-9_-]*=\\d+|[a-zA-Z][a-zA-Z0-9_-]*=(true|false|yes|no)|[a-zA-Z][a-zA-Z0-9_-]*=\"[^\\]\"]*(<.*\".*)?\"|[a-zA-Z][a-zA-Z0-9_-]*='[^\\]']*(<.*'.*)?'|[a-zA-Z][a-zA-Z0-9_-]*)(?:(?!\\s+/?\\])\\s|))+",y="\\[("+c+")(\\s"+p+")?\\]",d="\\[("+c+")(\\s"+p+")?\\s?\\/\\]",v=new RegExp(p.substring(0,p.length-1),"ig"),b=new RegExp("\\[\\/?([a-zA-Z][^\\]]+)\\]","i"),g=new RegExp("([a-zA-Z][a-zA-Z0-9_-]*)","i"),m=new RegExp(y,"i"),w=new RegExp("\\[\\/([a-zA-Z][a-zA-Z0-9_-]*)\\]","i"),E=new RegExp(d,"i"),x=function(){function t(e,n){var i=arguments.length>2&&void 0!==arguments[2]?arguments[2]:0,s=!(arguments.length>3&&void 0!==arguments[3])||arguments[3];r(this,t),this.name=null,this.type=e,this.body=n,this.pos=i,this.strict=s,this.children=[],this.params={},this.isClosed=e===f,this.init()}return a(t,[{key:"init",value:function(){if("TEXT"!==this.type&&"ERROR"!==this.type){var t=this.matchBody();if(null===t){if(this.strict)throw new SyntaxError("Invalid "+this.type+" token: "+this.body)}else this.initName(t),t[2]&&this.initParams(t[2])}}},{key:"initName",value:function(t){this.name=t[1]}},{key:"initParams",value:function(t){var e=t.match(v);this.params=e.reduce(function(t,e){e=e.trim();var n=e.indexOf("=");return~n?t[e.substring(0,n)]=o(e.substring(n+1)):t[e]=!0,t},{})}},{key:"buildParams",value:function(){var t="";for(var e in this.params)if(this.params.hasOwnProperty(e)){var n=this.params[e],i=void 0===n?"undefined":u(n);if("string"===i)t=t+" "+e+'="'+n+'"';else if("boolean"===i){var r=n?"true":"false";t=t+" "+e+"="+r}else t=t+" "+e+"="+n}return t}},{key:"toString",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null;t instanceof Object&&(this.params=t);var e="string"==typeof t?" "+t.trim():this.buildParams();switch(this.type){case"TEXT":return this.body;case h:return"["+this.name+e+"]"+(this.children.length?"{slot}":"")+"[/"+this.name+"]";case f:return"["+this.name+e+"/]";default:return""}}},{key:"matchBody",value:function(){var t=void 0;if(this.type===l)t=w;else if(this.type===h)t=m;else{if(this.type!==f)throw new SyntaxError("Unknown token: "+this.type);t=E}return this.body.match(t)}},{key:"canClose",value:function(t){return this.name===t.name}}]),t}(),k=function(){function t(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null,n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{strict:!0,skipWhiteSpace:!1,whitelist:!1};r(this,t),"boolean"==typeof n&&(n={strict:n,skipWhiteSpace:!1,whitelist:!1}),this.options=Object.assign({strict:!0,skipWhiteSpace:!1,whitelist:!1},n),this.buf=null,this.originalBuf=null,this.pos=0,e&&this.input(e)}return a(t,[{key:"input",value:function(t){if("string"!=typeof t)throw new Error("Invalid input");return this.buf=this.originalBuf=t,this.pos=0,this}},{key:"reset",value:function(){return this.buf=this.originalBuf,this.pos=0,this}},{key:"tokens",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null;if(t&&this.input(t),"string"!=typeof this.buf&&this.options.strict)throw new Error("Invalid input");for(var e=[],n=[];null!==(e=this._next());)e=Array.isArray(e)?e:[e],n.push.apply(n,i(e));return n}},{key:"ast",value:function(){var t=arguments.length>0&&void 0!==arguments[0]?arguments[0]:null,e=this.tokens(t),n=[],i=[],r=null,s=void 0,o=!0,u=!1,a=void 0;try{for(var c,p=e[Symbol.iterator]();!(o=(c=p.next()).done);o=!0)if(s=c.value,"TEXT"===s.type){if(this.options.skipWhiteSpace&&0===s.body.replace(/\s+/g,"").length)continue;r?r.children.push(s):i.push(s)}else if(s.type===h)r?(r.children.push(s),n.push(r),r=s):(r=s,i.push(r));else if(s.type===l)if(r&&s.canClose(r))r.isClosed=!0,r=n.pop();else{if(this.options.strict)throw new SyntaxError("Unmatched close token: "+s.body);var y=new x("ERROR",s.body,0,this.options.strict);r?r.children.push(y):i.push(y)}else if(s.type===f)r?r.children.push(s):i.push(s);else if(this.options.strict)throw new SyntaxError("Unknown token: "+s.type)}catch(y){u=!0,a=y}finally{try{!o&&p.return&&p.return()}finally{if(u)throw a}}if(r){if(this.options.strict)throw new SyntaxError("Unmatched open token: "+r.body);i.push(new x("ERROR",s.body,0,this.options.strict))}return i}},{key:"buildTemplate",value:function(t){var e=arguments.length>1&&void 0!==arguments[1]?arguments[1]:null;if(!t)return"";if(!(t instanceof x))throw new Error("Expected Token instance.");return function t(n){var i=n.children.map(function(e){return e.children.length?t(e):e.toString()});return n.toString(e).replace("{slot}",i.join(""))}(t)}},{key:"_next",value:function(){if(!this.buf)return null;var t=this.buf.match(b);if(t&&this.options.whitelist){var e=t[1].match(g)[1];-1===this.options.whitelist.indexOf(e)&&(t=null)}if(null===t){var n=new x("TEXT",this.buf,this.pos,this.options.strict);return this.pos+=this.buf.length,this.buf=null,n}var i=[];return 0!==t.index&&i.push(new x("TEXT",this.buf.substring(0,t.index),this.pos,this.options.strict)),i.push(new x(s(t[0]),t[0],this.pos+t.index,this.options.strict)),this.buf=this.buf.substring(t.index+t[0].length),this.pos+=t.index+t[0].length,0===this.buf.length&&(this.buf=null),i}},{key:"strict",get:function(){return console.warn("Deprecated: use options.strict instead"),this.options.strict},set:function(t){console.warn("Deprecated: use options.strict = "+t+" instead"),this.options.strict=t}}]),t}();e.default=k,Object.assign(k,{TEXT:"TEXT",ERROR:"ERROR",OPEN:h,CLOSE:l,SELF_CLOSING:f,rxParams:v,rxEnclosure:b,rxOpen:m,rxClose:w,rxSelfclosing:E})}]);
 //# sourceMappingURL=shortcode-tokenizer.js.map

package.json

 {
   "name": "@tsimons/shortcode-tokenizer",
-  "version": "0.7.2",
+  "version": "0.7.3",
   "public": true,
@@ -5,0 +5,0 @@ "description": "",

src/shortcode-tokenizer.js

@@ -26,3 +26,5 @@ /** @module ShortcodeTokenizer */
    finding potentia code tokens */
-const RX_ENCLOSURE   = '\\[\\/?[a-zA-Z][^\\]]+\\]'
+const RX_ENCLOSURE   = '\\[\\/?([a-zA-Z][^\\]]+)\\]'
+/* matches the name in the enclosure */
+const RX_NAME        = '(' + RX_KEY + ')'
 /* matches opening code tokens [row] */
@@ -38,2 +40,3 @@ const RX_OPEN        = '\\[(' + RX_KEY + ')(\\s' + RX_PARAMS + ')?\\]'
 const rxEnclosure   = new RegExp(RX_ENCLOSURE, 'i')
+const rxName        = new RegExp(RX_NAME, 'i')
 const rxOpen        = new RegExp(RX_OPEN, 'i')
@@ -236,7 +239,7 @@ const rxClose       = new RegExp(RX_CLOSE, 'i')
 
-  constructor(input = null, options = {strict: true, skipWhiteSpace: false}) {
+  constructor(input = null, options = {strict: true, skipWhiteSpace: false, whitelist: false }) {
     if (typeof options === 'boolean') {
-      options = {strict: options, skipWhiteSpace: false}
+      options = {strict: options, skipWhiteSpace: false, whitelist: false }
     }
-    this.options = Object.assign({strict: true, skipWhiteSpace: false}, options)
+    this.options = Object.assign({strict: true, skipWhiteSpace: false, whitelist: false }, options)
     this.buf = null
@@ -433,2 +436,11 @@ this.originalBuf = null
 
+    // Whitelist - Treat as text if not whitelisted
+    if (match && this.options.whitelist) {
+      let name = match[1].match(rxName)[1]
+
+      if (this.options.whitelist.indexOf(name) === -1) {
+        match = null
+      }
+    }
+
     // all text
@@ -435,0 +447,0 @@ if (match === null) {

test/unit/shortcode-tokenizer.test.js

@@ -177,2 +177,12 @@ import * as lib from '../../src/shortcode-tokenizer'
     })
+
+    it('should treat non-whitelisted tokens as TEXT', () => {
+      let input = '[thisone] [notthisone]'
+      tokenizer.options.whitelist = ['thisone'];
+      const result = tokenizer.tokens(input);
+      expect(result[0].name).to.equal('thisone');
+      expect(result[0].type).to.equal('OPEN');
+      expect(result[1].name).to.equal(null);
+      expect(result[1].type).to.equal('TEXT');
+    })
   })
@@ -179,0 +189,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

395598

increased by0.53%

Lines of code

942

increased by2.17%

No dependency changes