@adguard/css-tokenizer - npm Package Compare versions

Comparing version 0.0.1 to 1.0.0-alpha.0

dist/build.txt

@@ -1,1 +0,1 @@
-version=0.0.1
+version=1.0.0-alpha.0

dist/csstokenizer.d.ts

 /*
- * CSSTokenizer v0.0.1 (build date: Mon, 30 Oct 2023 16:57:22 GMT)
- * (c) 2023 AdGuard Software Ltd.
+ * CSSTokenizer v1.0.0-alpha.0 (build date: Mon, 29 Jul 2024 13:42:35 GMT)
+ * (c) 2024 Adguard Software Ltd.
  * Released under the MIT license
@@ -181,2 +181,6 @@ * https://github.com/AdguardTeam/tsurlfilter/tree/master/packages/css-tokenizer#readme
     /**
+     * Consumes a single whitespace code point, if the current code point is a whitespace
+     */
+    consumeSingleWhitespace(): void;
+    /**
      * Consumes everything until the end of the comment (or the end of the source)
@@ -214,5 +218,4 @@ */
  * not otherwise set
- * @todo Create a more specific type for `props` parameter, if needed & possible
  */
-type OnTokenCallback = (type: TokenType, start: number, end: number, props?: object) => void;
+type OnTokenCallback = (type: TokenType, start: number, end: number, props?: Record<string, unknown>) => void;
 /**
@@ -241,9 +244,12 @@ * Callback which is called when a parsing error is found. According to the spec, parsing errors are not fatal and
 /**
- * Pairs of token types and their base names
- */
-declare const TOKEN_NAMES: Record<TokenType, string>;
-/**
  * Get base token name by token type
  *
  * @param type Token type
+ *
+ * @example
+ * ```ts
+ * getBaseTokenName(TokenType.Ident); // 'ident'
+ * getBaseTokenName(-1); // 'unknown'
+ * ```
+ *
  * @returns Base token name or 'unknown' if token type is unknown
@@ -256,2 +262,9 @@ */
  * @param type Token type
+ *
+ * @example
+ * ```ts
+ * getFormattedTokenName(TokenType.Ident); // '<ident-token>'
+ * getFormattedTokenName(-1); // '<unknown-token>'
+ * ```
+ *
  * @returns Formatted token name or `'<unknown-token>'` if token type is unknown
@@ -304,6 +317,24 @@ */
 /**
+ * @file CSS identifier decoder.
+ */
+/**
+ * Decodes a CSS identifier according to the CSS Syntax Module Level 3 specification.
+ *
+ * @param ident CSS identifier to decode.
+ *
+ * @example
+ * ```ts
+ * decodeIdent(String.raw`\00075\00072\0006C`); // 'url'
+ * decodeIdent('url'); // 'url'
+ * ```
+ *
+ * @returns Decoded CSS identifier.
+ */
+declare const decodeIdent: (ident: string) => string;
+
+/**
  * @file Package version
  */
-declare const version: string;
+declare const CSS_TOKENIZER_VERSION: string;
 
-export { type OnErrorCallback, type OnTokenCallback, TOKEN_NAMES, TokenType, TokenizerContext, type TokenizerContextFunction, getBaseTokenName, getFormattedTokenName, tokenize, tokenizeExtended, version };
+export { CSS_TOKENIZER_VERSION, type OnErrorCallback, type OnTokenCallback, TokenType, TokenizerContext, type TokenizerContextFunction, decodeIdent, getBaseTokenName, getFormattedTokenName, tokenize, tokenizeExtended };

dist/csstokenizer.iife.min.js

 /*
- * CSSTokenizer v0.0.1 (build date: Mon, 30 Oct 2023 16:57:22 GMT)
- * (c) 2023 AdGuard Software Ltd.
+ * CSSTokenizer v1.0.0-alpha.0 (build date: Mon, 29 Jul 2024 13:42:35 GMT)
+ * (c) 2024 Adguard Software Ltd.
  * Released under the MIT license
  * https://github.com/AdguardTeam/tsurlfilter/tree/master/packages/css-tokenizer#readme
  */
-function _defineProperty(e,o,n){return(o=_toPropertyKey(o))in e?Object.defineProperty(e,o,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[o]=n,e}function _toPropertyKey(e){var o=_toPrimitive(e,"string");return"symbol"==typeof o?o:String(o)}function _toPrimitive(e,o){if("object"!=typeof e||null===e)return e;var n=e[Symbol.toPrimitive];if(void 0!==n){var t=n.call(e,o||"default");if("object"!=typeof t)return t;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===o?String:Number)(e)}var CSSTokenizer=function(e){"use strict";function o(e,o,n){return e>=o&&e<=n}function n(e,o){return e>=o}function t(e){return o(e,48,57)}function i(e){return t(e)||o(e,65,70)||o(e,97,102)}function r(e){return o(e,65,90)}function s(e){return o(e,97,122)}function c(e){return r(e)||s(e)}function d(e){return n(e,128)}function a(e){return c(e)||d(e)||95===e}function f(e){return a(e)||t(e)||45===e}function T(e){return 10===e||13===e||12===e}function u(e){return T(e)||9===e||32===e}const k=(e,o)=>92===e&&!T(o),l=(e,o,n)=>45===e?a(o)||45===o||k(o,n):!!a(e)||92===e&&k(e,o),p=(e,o,n)=>43===e||45===e?!!t(o)||46===o&&t(n):t(46===e?o:e);function m(e,o,n){let t=5381;for(let i=o;i<n;i+=1)t=33*t^(32|e[i]);return t>>>0}class C{constructor(e,o,n,t){var i;if(_defineProperty(this,"length",void 0),_defineProperty(this,"onToken",void 0),_defineProperty(this,"onError",void 0),_defineProperty(this,"codes",void 0),_defineProperty(this,"cursor",void 0),_defineProperty(this,"customFunctionHandlers",void 0),this.length=e.length,this.preprocess(e),this.cursor=65279===(i=this.codes[0])||65534===i?1:0,this.onToken=o,this.onError=n,t){this.customFunctionHandlers=new Map;for(const[e,o]of t)this.customFunctionHandlers.set(e,o)}}preprocess(e){const o=e.length;this.codes=new Int32Array(o+1);for(let n=0;n<o;n+=1)this.codes[n]=e.charCodeAt(n);this.codes[o]=-1}getFunctionHandler(e){var o;return null===(o=this.customFunctionHandlers)||void 0===o?void 0:o.get(e)}hasFunctionHandler(e){var o;return(null===(o=this.customFunctionHandlers)||void 0===o?void 0:o.has(e))??!1}get offset(){return this.cursor}get code(){return this.codes[this.offset]}get prevCode(){return this.codes[this.offset-1]}get nextCode(){return this.codes[this.offset+1]}getRelativeCode(e){return this.codes[this.offset+e]}isEof(){return this.offset>=this.length}isNextEof(){return this.cursor+1===this.length}isLessThanEqualToEof(){return this.offset<=this.length}consumeCodePoint(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;this.cursor+=e}getNextNonWsCode(){let e=this.cursor;for(;e<this.length&&u(this.codes[e]);)e+=1;return this.codes[e]}consumeWhitespace(){for(;this.code&&u(this.code);)this.consumeCodePoint()}consumeUntilCommentEnd(){for(;this.cursor<this.length;){if(42===this.code&&47===this.nextCode){this.cursor+=2;break}this.cursor+=1}}consumeTrivialToken(e){this.onToken(e,this.cursor,++this.cursor)}getHashFrom(e){return m(this.codes,e,this.cursor)}}var h;e.TokenType=void 0,(h=e.TokenType||(e.TokenType={}))[h.Eof=0]="Eof",h[h.Ident=1]="Ident",h[h.Function=2]="Function",h[h.AtKeyword=3]="AtKeyword",h[h.Hash=4]="Hash",h[h.String=5]="String",h[h.BadString=6]="BadString",h[h.Url=7]="Url",h[h.BadUrl=8]="BadUrl",h[h.Delim=9]="Delim",h[h.Number=10]="Number",h[h.Percentage=11]="Percentage",h[h.Dimension=12]="Dimension",h[h.Whitespace=13]="Whitespace",h[h.Cdo=14]="Cdo",h[h.Cdc=15]="Cdc",h[h.Colon=16]="Colon",h[h.Semicolon=17]="Semicolon",h[h.Comma=18]="Comma",h[h.OpenSquareBracket=19]="OpenSquareBracket",h[h.CloseSquareBracket=20]="CloseSquareBracket",h[h.OpenParenthesis=21]="OpenParenthesis",h[h.CloseParenthesis=22]="CloseParenthesis",h[h.OpenCurlyBracket=23]="OpenCurlyBracket",h[h.CloseCurlyBracket=24]="CloseCurlyBracket",h[h.Comment=25]="Comment";const v=Object.freeze({[e.TokenType.Eof]:"eof",[e.TokenType.Ident]:"ident",[e.TokenType.Function]:"function",[e.TokenType.AtKeyword]:"at-keyword",[e.TokenType.Hash]:"hash",[e.TokenType.String]:"string",[e.TokenType.BadString]:"bad-string",[e.TokenType.Url]:"url",[e.TokenType.BadUrl]:"bad-url",[e.TokenType.Delim]:"delim",[e.TokenType.Number]:"number",[e.TokenType.Percentage]:"percentage",[e.TokenType.Dimension]:"dimension",[e.TokenType.Whitespace]:"whitespace",[e.TokenType.Cdo]:"CDO",[e.TokenType.Cdc]:"CDC",[e.TokenType.Colon]:"colon",[e.TokenType.Semicolon]:"semicolon",[e.TokenType.Comma]:"comma",[e.TokenType.OpenSquareBracket]:"[",[e.TokenType.CloseSquareBracket]:"]",[e.TokenType.OpenParenthesis]:"(",[e.TokenType.CloseParenthesis]:")",[e.TokenType.OpenCurlyBracket]:"{",[e.TokenType.CloseCurlyBracket]:"}",[e.TokenType.Comment]:"comment"}),y=e=>v[e]??"unknown",g=e=>{if(e.consumeCodePoint(),i(e.code)){let o=0;for(;i(e.code)&&o<5;)e.consumeCodePoint(),o+=1}e.isEof()&&e.onError("Unexpected end of file while parsing escaped code point.",e.offset,e.offset)},P=e=>{for(;!e.isEof();)if(f(e.code))e.consumeCodePoint();else{if(!k(e.code,e.nextCode))return;e.consumeCodePoint(),g(e)}};function b(e){for(;!e.isEof();e.consumeCodePoint()){if(41===e.code)return void e.consumeCodePoint();k(e.getRelativeCode(1),e.getRelativeCode(2))&&(e.consumeCodePoint(),g(e))}}function E(o,n){b(o),o.onToken(e.TokenType.BadUrl,n,o.offset)}const R=(n,t)=>{for(;u(n.code);)n.consumeCodePoint();for(;n.offset<=n.length;){if(41===n.code)return n.consumeCodePoint(),void n.onToken(e.TokenType.Url,t,n.offset);if(n.isEof())return n.onToken(e.TokenType.Url,t,n.offset),void n.onError("Unexpected end of file while parsing URL.",t,n.offset);if(u(n.code)){for(;u(n.code);)n.consumeCodePoint();return 41===n.code||n.isEof()?(n.consumeCodePoint(),n.onToken(e.TokenType.Url,t,n.offset),void n.onError("Unexpected end of file while parsing URL.",t,n.offset)):(n.onError("Unexpected character in URL.",t,n.offset),void E(n,t))}if(34===n.code||39===n.code||40===n.code||(o(i=n.code,0,8)||11===i||o(i,14,31)||127===i))return n.onError("Unexpected character in URL.",t,n.offset),void E(n,t);if(92===n.code){if(k(n.code,n.nextCode)){n.consumeCodePoint(),g(n);continue}return n.onError("Unexpected character in URL.",t,n.offset),void E(n,t)}n.consumeCodePoint()}var i},S=o=>{const n=o.offset;if(P(o),40===o.code){const t=o.getHashFrom(n);if(o.consumeCodePoint(),193422222===t){const t=o.getNextNonWsCode();return 34===t||39===t?void o.onToken(e.TokenType.Function,n,o.offset):void R(o,n)}return o.hasFunctionHandler(t)?(o.onToken(e.TokenType.Function,n,o.offset),void o.getFunctionHandler(t)(o)):void o.onToken(e.TokenType.Function,n,o.offset)}o.onToken(e.TokenType.Ident,n,o.offset)},x=e=>{for(43!==e.code&&45!==e.code||e.consumeCodePoint();t(e.code);)e.consumeCodePoint();if(46===e.code&&t(e.nextCode))for(e.consumeCodePoint(2);t(e.code);)e.consumeCodePoint();if(69===e.code||101===e.code)if(45!==e.nextCode&&43!==e.nextCode||!t(e.getRelativeCode(2))){if(t(e.nextCode))for(e.consumeCodePoint(2);t(e.code);)e.consumeCodePoint()}else for(e.consumeCodePoint(3);t(e.code);)e.consumeCodePoint()},U=o=>{const n=o.offset;return x(o),l(o.code,o.nextCode,o.getRelativeCode(2))?(P(o),void o.onToken(e.TokenType.Dimension,n,o.offset)):37===o.code?(o.consumeCodePoint(),void o.onToken(e.TokenType.Percentage,n,o.offset)):void o.onToken(e.TokenType.Number,n,o.offset)},w=o=>{const n=o.code,t=o.offset;for(o.consumeCodePoint();o.isLessThanEqualToEof();)switch(o.code){case n:return o.consumeCodePoint(),void o.onToken(e.TokenType.String,t,o.offset);case-1:return o.onToken(e.TokenType.String,t,o.offset),void o.onError("Unexpected end of file while parsing string token.",t,o.offset);case 13:case 10:case 12:return 13===o.code&&10===o.nextCode&&o.consumeCodePoint(1),o.consumeCodePoint(1),o.onToken(e.TokenType.BadString,t,o.offset),void o.onError("Unexpected newline while parsing string token.",t,o.offset);case 92:if(o.isNextEof())return o.consumeCodePoint(),o.onToken(e.TokenType.String,t,o.offset),void o.onError("Unexpected end of file while parsing string token.",t,o.offset);if(T(o.nextCode)){o.consumeCodePoint(2);break}k(o.code,o.nextCode)&&(o.consumeCodePoint(),g(o));break;default:o.consumeCodePoint()}},B=o=>{const n=o.offset;o.consumeWhitespace(),o.onToken(e.TokenType.Whitespace,n,o.offset)},D=function(o,n){const t=new C(o,n,arguments.length>2&&void 0!==arguments[2]?arguments[2]:()=>{},arguments.length>3?arguments[3]:void 0);for(;!t.isEof();)switch(t.code){case 9:case 32:case 10:case 12:case 13:B(t);break;case 48:case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57:U(t);break;case 40:t.consumeTrivialToken(e.TokenType.OpenParenthesis);break;case 41:t.consumeTrivialToken(e.TokenType.CloseParenthesis);break;case 44:t.consumeTrivialToken(e.TokenType.Comma);break;case 58:t.consumeTrivialToken(e.TokenType.Colon);break;case 59:t.consumeTrivialToken(e.TokenType.Semicolon);break;case 91:t.consumeTrivialToken(e.TokenType.OpenSquareBracket);break;case 93:t.consumeTrivialToken(e.TokenType.CloseSquareBracket);break;case 123:t.consumeTrivialToken(e.TokenType.OpenCurlyBracket);break;case 125:t.consumeTrivialToken(e.TokenType.CloseCurlyBracket);break;case 39:case 34:w(t);break;case 35:if(f(t.getRelativeCode(1))||k(t.getRelativeCode(1),t.getRelativeCode(2))){const o=t.offset;t.consumeCodePoint(),P(t),t.onToken(e.TokenType.Hash,o,t.offset);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 43:case 46:if(p(t.code,t.getRelativeCode(1),t.getRelativeCode(2))){U(t);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 45:if(p(t.code,t.getRelativeCode(1),t.getRelativeCode(2))){U(t);break}if(45===t.getRelativeCode(1)&&62===t.getRelativeCode(2)){t.consumeCodePoint(3),t.onToken(e.TokenType.Cdc,t.offset-3,t.offset);break}if(l(t.code,t.getRelativeCode(1),t.getRelativeCode(2))){S(t);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 60:if(33===t.getRelativeCode(1)&&45===t.getRelativeCode(2)&&45===t.getRelativeCode(3)){t.consumeCodePoint(4),t.onToken(e.TokenType.Cdo,t.offset-4,t.offset);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 64:if(l(t.getRelativeCode(1),t.getRelativeCode(2),t.getRelativeCode(3))){const o=t.offset;t.consumeCodePoint(),P(t),t.onToken(e.TokenType.AtKeyword,o,t.offset);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 92:if(k(t.code,t.getRelativeCode(1))){S(t);break}t.consumeTrivialToken(e.TokenType.Delim),t.onError("Invalid escape sequence.",t.offset-1,t.offset);break;case 47:if(42===t.getRelativeCode(1)){const o=t.offset;t.consumeCodePoint(2),t.consumeUntilCommentEnd(),t.isEof()&&t.onError("Unterminated comment.",o,t.length-2),t.onToken(e.TokenType.Comment,o,t.offset);break}t.consumeTrivialToken(e.TokenType.Delim);break;default:if(a(t.code)){S(t);break}t.consumeTrivialToken(e.TokenType.Delim)}},F=o=>{const n=o.offset;if(o.consumeWhitespace(),39===o.code||34===o.code)return void(o.offset>n&&o.onToken(e.TokenType.Whitespace,n,o.offset));let t=1;for(let t=n;t<o.offset;t+=1)o.onToken(e.TokenType.Delim,t,t+1);for(;!o.isEof();){if(40===o.code&&92!==o.prevCode)t+=1;else if(41===o.code&&92!==o.prevCode&&(t-=1,0===t))break;o.consumeTrivialToken(e.TokenType.Delim)}};function O(e,o){const n=new Map;for(const[o,t]of e)n.set(o,t);for(const[e,t]of o)n.set(e,t);return n}const H=new Map([[1989084725,F],[2399470598,F],[1221663855,F],[102304302,F],[2923888231,F],[1739713050,F],[196571984,o=>{const n=o.offset;if(o.consumeWhitespace(),39===o.code||34===o.code)return void(o.offset>n&&o.onToken(e.TokenType.Whitespace,n,o.offset));let t=1;for(let t=n;t<o.offset;t+=1)o.onToken(e.TokenType.Delim,t,t+1);let i=!1;for(;!o.isEof();){if(34===o.code&&92!==o.prevCode&&(i=!i),!i)if(40===o.code&&92!==o.prevCode)t+=1;else if(41===o.code&&92!==o.prevCode&&(t-=1,0===t))break;o.consumeTrivialToken(e.TokenType.Delim)}}]]);return e.TOKEN_NAMES=v,e.TokenizerContext=C,e.getBaseTokenName=y,e.getFormattedTokenName=e=>`<${y(e)}-token>`,e.tokenize=D,e.tokenizeExtended=function(e,o){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:()=>{},t=arguments.length>3&&void 0!==arguments[3]?arguments[3]:new Map;D(e,o,n,t.size>0?O(H,t):H)},e.version="0.0.1",e}({});
+function _defineProperty(e,o,n){return(o=_toPropertyKey(o))in e?Object.defineProperty(e,o,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[o]=n,e}function _toPropertyKey(e){var o=_toPrimitive(e,"string");return"symbol"==typeof o?o:o+""}function _toPrimitive(e,o){if("object"!=typeof e||!e)return e;var n=e[Symbol.toPrimitive];if(void 0!==n){var t=n.call(e,o||"default");if("object"!=typeof t)return t;throw new TypeError("@@toPrimitive must return a primitive value.")}return("string"===o?String:Number)(e)}var CSSTokenizer=function(e){"use strict";function o(e,o,n){return e>=o&&e<=n}function n(e,o){return e>o}function t(e,o){return e>=o}function r(e){return o(e,48,57)}function i(e){return r(e)||o(e,65,70)||o(e,97,102)}function s(e){return o(e,65,90)}function c(e){return o(e,97,122)}function d(e){return s(e)||c(e)}function a(e){return t(e,128)}function f(e){return d(e)||a(e)||95===e}function T(e){return f(e)||r(e)||45===e}function u(e){return 10===e||13===e||12===e}function l(e){return u(e)||9===e||32===e}function k(e){return o(e,55296,56319)}function p(e){return o(e,56320,57343)}function m(e){return n(e,1114111)}const C=(e,o)=>92===e&&!u(o),h=(e,o,n)=>45===e?f(o)||45===o||C(o,n):!!f(e)||92===e&&C(e,o),v=(e,o,n)=>43===e||45===e?!!r(o)||46===o&&r(n):r(46===e?o:e);function y(e,o,n){let t=5381;for(let r=o;r<n;r+=1)t=33*t^(32|e[r]);return t>>>0}class g{constructor(e,o,n,t){var r;if(_defineProperty(this,"length",void 0),_defineProperty(this,"onToken",void 0),_defineProperty(this,"onError",void 0),_defineProperty(this,"codes",void 0),_defineProperty(this,"cursor",void 0),_defineProperty(this,"customFunctionHandlers",void 0),this.length=e.length,this.preprocess(e),this.cursor=65279===(r=this.codes[0])||65534===r?1:0,this.onToken=o,this.onError=n,t){this.customFunctionHandlers=new Map;for(const[e,o]of t)this.customFunctionHandlers.set(e,o)}}preprocess(e){const o=e.length;this.codes=new Int32Array(o+1);for(let n=0;n<o;n+=1)this.codes[n]=e.charCodeAt(n);this.codes[o]=-1}getFunctionHandler(e){var o;return null===(o=this.customFunctionHandlers)||void 0===o?void 0:o.get(e)}hasFunctionHandler(e){var o;return(null===(o=this.customFunctionHandlers)||void 0===o?void 0:o.has(e))??!1}get offset(){return this.cursor}get code(){return this.codes[this.offset]}get prevCode(){return this.codes[this.offset-1]}get nextCode(){return this.codes[this.offset+1]}getRelativeCode(e){return this.codes[this.offset+e]}isEof(){return this.offset>=this.length}isNextEof(){return this.cursor+1===this.length}isLessThanEqualToEof(){return this.offset<=this.length}consumeCodePoint(){let e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:1;this.cursor+=e}getNextNonWsCode(){let e=this.cursor;for(;e<this.length&&l(this.codes[e]);)e+=1;return this.codes[e]}consumeWhitespace(){for(;this.code&&l(this.code);)this.consumeCodePoint()}consumeSingleWhitespace(){l(this.code)&&(this.cursor+=13===this.code&&10===this.nextCode?2:1)}consumeUntilCommentEnd(){for(;this.cursor<this.length;){if(42===this.code&&47===this.nextCode){this.cursor+=2;break}this.cursor+=1}}consumeTrivialToken(e){this.onToken(e,this.cursor,++this.cursor)}getHashFrom(e){return y(this.codes,e,this.cursor)}}var P;e.TokenType=void 0,(P=e.TokenType||(e.TokenType={}))[P.Eof=0]="Eof",P[P.Ident=1]="Ident",P[P.Function=2]="Function",P[P.AtKeyword=3]="AtKeyword",P[P.Hash=4]="Hash",P[P.String=5]="String",P[P.BadString=6]="BadString",P[P.Url=7]="Url",P[P.BadUrl=8]="BadUrl",P[P.Delim=9]="Delim",P[P.Number=10]="Number",P[P.Percentage=11]="Percentage",P[P.Dimension=12]="Dimension",P[P.Whitespace=13]="Whitespace",P[P.Cdo=14]="Cdo",P[P.Cdc=15]="Cdc",P[P.Colon=16]="Colon",P[P.Semicolon=17]="Semicolon",P[P.Comma=18]="Comma",P[P.OpenSquareBracket=19]="OpenSquareBracket",P[P.CloseSquareBracket=20]="CloseSquareBracket",P[P.OpenParenthesis=21]="OpenParenthesis",P[P.CloseParenthesis=22]="CloseParenthesis",P[P.OpenCurlyBracket=23]="OpenCurlyBracket",P[P.CloseCurlyBracket=24]="CloseCurlyBracket",P[P.Comment=25]="Comment";const b=Object.freeze({[e.TokenType.Eof]:"eof",[e.TokenType.Ident]:"ident",[e.TokenType.Function]:"function",[e.TokenType.AtKeyword]:"at-keyword",[e.TokenType.Hash]:"hash",[e.TokenType.String]:"string",[e.TokenType.BadString]:"bad-string",[e.TokenType.Url]:"url",[e.TokenType.BadUrl]:"bad-url",[e.TokenType.Delim]:"delim",[e.TokenType.Number]:"number",[e.TokenType.Percentage]:"percentage",[e.TokenType.Dimension]:"dimension",[e.TokenType.Whitespace]:"whitespace",[e.TokenType.Cdo]:"CDO",[e.TokenType.Cdc]:"CDC",[e.TokenType.Colon]:"colon",[e.TokenType.Semicolon]:"semicolon",[e.TokenType.Comma]:"comma",[e.TokenType.OpenSquareBracket]:"[",[e.TokenType.CloseSquareBracket]:"]",[e.TokenType.OpenParenthesis]:"(",[e.TokenType.CloseParenthesis]:")",[e.TokenType.OpenCurlyBracket]:"{",[e.TokenType.CloseCurlyBracket]:"}",[e.TokenType.Comment]:"comment"}),E=e=>b[e]??"unknown",S=e=>{if(e.consumeCodePoint(),i(e.code)){let o=0;for(;i(e.code)&&o<=6;)e.consumeCodePoint(),o+=1;e.consumeSingleWhitespace()}e.isEof()&&e.onError("Unexpected end of file while parsing escaped code point.",e.offset,e.offset)},R=e=>{for(;!e.isEof();)if(T(e.code))e.consumeCodePoint();else{if(!C(e.code,e.nextCode))return;e.consumeCodePoint(),S(e)}};function x(e){for(;!e.isEof();e.consumeCodePoint()){if(41===e.code)return void e.consumeCodePoint();C(e.getRelativeCode(1),e.getRelativeCode(2))&&(e.consumeCodePoint(),S(e))}}function U(o,n){x(o),o.onToken(e.TokenType.BadUrl,n,o.offset)}const w=(n,t)=>{for(;l(n.code);)n.consumeCodePoint();for(;n.offset<=n.length;){if(41===n.code)return n.consumeCodePoint(),void n.onToken(e.TokenType.Url,t,n.offset);if(n.isEof())return n.onToken(e.TokenType.Url,t,n.offset),void n.onError("Unexpected end of file while parsing URL.",t,n.offset);if(l(n.code)){for(;l(n.code);)n.consumeCodePoint();return 41===n.code||n.isEof()?(n.consumeCodePoint(),n.onToken(e.TokenType.Url,t,n.offset),void n.onError("Unexpected end of file while parsing URL.",t,n.offset)):(n.onError("Unexpected character in URL.",t,n.offset),void U(n,t))}if(34===n.code||39===n.code||40===n.code||(o(r=n.code,0,8)||11===r||o(r,14,31)||127===r))return n.onError("Unexpected character in URL.",t,n.offset),void U(n,t);if(92===n.code){if(C(n.code,n.nextCode)){n.consumeCodePoint(),S(n);continue}return n.onError("Unexpected character in URL.",t,n.offset),void U(n,t)}n.consumeCodePoint()}var r},B=o=>{const n=o.offset;if(R(o),40===o.code){const t=o.getHashFrom(n);if(o.consumeCodePoint(),193422222===t){const t=o.getNextNonWsCode();return 34===t||39===t?void o.onToken(e.TokenType.Function,n,o.offset):void w(o,n)}return o.hasFunctionHandler(t)?(o.onToken(e.TokenType.Function,n,o.offset),void o.getFunctionHandler(t)(o)):void o.onToken(e.TokenType.Function,n,o.offset)}o.onToken(e.TokenType.Ident,n,o.offset)},D=e=>{for(43!==e.code&&45!==e.code||e.consumeCodePoint();r(e.code);)e.consumeCodePoint();if(46===e.code&&r(e.nextCode))for(e.consumeCodePoint(2);r(e.code);)e.consumeCodePoint();if(69===e.code||101===e.code)if(45!==e.nextCode&&43!==e.nextCode||!r(e.getRelativeCode(2))){if(r(e.nextCode))for(e.consumeCodePoint(2);r(e.code);)e.consumeCodePoint()}else for(e.consumeCodePoint(3);r(e.code);)e.consumeCodePoint()},F=o=>{const n=o.offset;return D(o),h(o.code,o.nextCode,o.getRelativeCode(2))?(R(o),void o.onToken(e.TokenType.Dimension,n,o.offset)):37===o.code?(o.consumeCodePoint(),void o.onToken(e.TokenType.Percentage,n,o.offset)):void o.onToken(e.TokenType.Number,n,o.offset)},O=o=>{const n=o.code,t=o.offset;for(o.consumeCodePoint();o.isLessThanEqualToEof();)switch(o.code){case n:return o.consumeCodePoint(),void o.onToken(e.TokenType.String,t,o.offset);case-1:return o.onToken(e.TokenType.String,t,o.offset),void o.onError("Unexpected end of file while parsing string token.",t,o.offset);case 13:case 10:case 12:return 13===o.code&&10===o.nextCode&&o.consumeCodePoint(1),o.consumeCodePoint(1),o.onToken(e.TokenType.BadString,t,o.offset),void o.onError("Unexpected newline while parsing string token.",t,o.offset);case 92:if(o.isNextEof())return o.consumeCodePoint(),o.onToken(e.TokenType.String,t,o.offset),void o.onError("Unexpected end of file while parsing string token.",t,o.offset);if(u(o.nextCode)){o.consumeCodePoint(2);break}C(o.code,o.nextCode)&&(o.consumeCodePoint(),S(o));break;default:o.consumeCodePoint()}},H=o=>{const n=o.offset;o.consumeWhitespace(),o.onToken(e.TokenType.Whitespace,n,o.offset)},N=function(o,n){const t=new g(o,n,arguments.length>2&&void 0!==arguments[2]?arguments[2]:()=>{},arguments.length>3?arguments[3]:void 0);for(;!t.isEof();)switch(t.code){case 9:case 32:case 10:case 12:case 13:H(t);break;case 48:case 49:case 50:case 51:case 52:case 53:case 54:case 55:case 56:case 57:F(t);break;case 40:t.consumeTrivialToken(e.TokenType.OpenParenthesis);break;case 41:t.consumeTrivialToken(e.TokenType.CloseParenthesis);break;case 44:t.consumeTrivialToken(e.TokenType.Comma);break;case 58:t.consumeTrivialToken(e.TokenType.Colon);break;case 59:t.consumeTrivialToken(e.TokenType.Semicolon);break;case 91:t.consumeTrivialToken(e.TokenType.OpenSquareBracket);break;case 93:t.consumeTrivialToken(e.TokenType.CloseSquareBracket);break;case 123:t.consumeTrivialToken(e.TokenType.OpenCurlyBracket);break;case 125:t.consumeTrivialToken(e.TokenType.CloseCurlyBracket);break;case 39:case 34:O(t);break;case 35:if(T(t.getRelativeCode(1))||C(t.getRelativeCode(1),t.getRelativeCode(2))){const o=t.offset;t.consumeCodePoint(),R(t),t.onToken(e.TokenType.Hash,o,t.offset);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 43:case 46:if(v(t.code,t.getRelativeCode(1),t.getRelativeCode(2))){F(t);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 45:if(v(t.code,t.getRelativeCode(1),t.getRelativeCode(2))){F(t);break}if(45===t.getRelativeCode(1)&&62===t.getRelativeCode(2)){t.consumeCodePoint(3),t.onToken(e.TokenType.Cdc,t.offset-3,t.offset);break}if(h(t.code,t.getRelativeCode(1),t.getRelativeCode(2))){B(t);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 60:if(33===t.getRelativeCode(1)&&45===t.getRelativeCode(2)&&45===t.getRelativeCode(3)){t.consumeCodePoint(4),t.onToken(e.TokenType.Cdo,t.offset-4,t.offset);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 64:if(h(t.getRelativeCode(1),t.getRelativeCode(2),t.getRelativeCode(3))){const o=t.offset;t.consumeCodePoint(),R(t),t.onToken(e.TokenType.AtKeyword,o,t.offset);break}t.consumeTrivialToken(e.TokenType.Delim);break;case 92:if(C(t.code,t.getRelativeCode(1))){B(t);break}t.consumeTrivialToken(e.TokenType.Delim),t.onError("Invalid escape sequence.",t.offset-1,t.offset);break;case 47:if(42===t.getRelativeCode(1)){const o=t.offset;t.consumeCodePoint(2),t.consumeUntilCommentEnd(),t.isEof()&&t.onError("Unterminated comment.",o,t.length-2),t.onToken(e.TokenType.Comment,o,t.offset);break}t.consumeTrivialToken(e.TokenType.Delim);break;default:if(f(t.code)){B(t);break}t.consumeTrivialToken(e.TokenType.Delim)}},W=o=>{const n=o.offset;if(o.consumeWhitespace(),39===o.code||34===o.code)return void(o.offset>n&&o.onToken(e.TokenType.Whitespace,n,o.offset));let t=1;for(let t=n;t<o.offset;t+=1)o.onToken(e.TokenType.Delim,t,t+1);for(;!o.isEof();){if(40===o.code&&92!==o.prevCode)t+=1;else if(41===o.code&&92!==o.prevCode&&(t-=1,0===t))break;o.consumeTrivialToken(e.TokenType.Delim)}};function _(e,o){const n=new Map;for(const[o,t]of e)n.set(o,t);for(const[e,t]of o)n.set(e,t);return n}const q=new Map([[1989084725,W],[2399470598,W],[1221663855,W],[102304302,W],[2923888231,W],[1739713050,W],[1860790666,W],[3376104318,W],[196571984,o=>{const n=o.offset;if(o.consumeWhitespace(),39===o.code||34===o.code)return void(o.offset>n&&o.onToken(e.TokenType.Whitespace,n,o.offset));let t=1;for(let t=n;t<o.offset;t+=1)o.onToken(e.TokenType.Delim,t,t+1);let r=!1;for(;!o.isEof();){if(34===o.code&&92!==o.prevCode&&(r=!r),!r)if(40===o.code&&92!==o.prevCode)t+=1;else if(41===o.code&&92!==o.prevCode&&(t-=1,0===t))break;o.consumeTrivialToken(e.TokenType.Delim)}}]]);return e.CSS_TOKENIZER_VERSION="1.0.0-alpha.0",e.TokenizerContext=g,e.decodeIdent=e=>{const o=[];for(let t=0;t<e.length;t+=1){if(92===e.charCodeAt(t)){if(i(e.charCodeAt(t+1))){let r=0,s=0;for(;s<6&&i(e.charCodeAt(t+s+1));)r=16*r+parseInt(e[t+s+1],16),s+=1;o.push(String.fromCodePoint(0===r||(k(n=r)||p(n))||m(r)?65533:r)),t+=s;const c=e.charCodeAt(t+1);l(c)&&(t+=1,13===c&&10===e.charCodeAt(t+1)&&(t+=1))}}else o.push(e[t])}var n;return o.join("")},e.getBaseTokenName=E,e.getFormattedTokenName=e=>`<${E(e)}-token>`,e.tokenize=N,e.tokenizeExtended=function(e,o){let n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:()=>{},t=arguments.length>3&&void 0!==arguments[3]?arguments[3]:new Map;N(e,o,n,t.size>0?_(q,t):q)},e}({});

package.json

 {
   "name": "@adguard/css-tokenizer",
-  "version": "0.0.1",
+  "version": "1.0.0-alpha.0",
   "description": "CSS / Extended CSS tokenizer",
@@ -9,3 +9,3 @@ "keywords": [
   ],
-  "author": "AdGuard Software Ltd.",
+  "author": "Adguard Software Ltd.",
   "license": "MIT",
@@ -22,22 +22,18 @@ "repository": {
   "main": "dist/csstokenizer.js",
-  "browser": "dist/csstokenizer.iife.min.js",
+  "module": "dist/csstokenizer.esm.mjs",
+  "browser": "dist/csstokenizer.umd.min.js",
   "types": "dist/csstokenizer.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/csstokenizer.d.ts",
+      "import": "./dist/csstokenizer.esm.mjs",
+      "require": "./dist/csstokenizer.js"
+    },
+    "./es": "./dist/csstokenizer.esm.mjs",
+    "./iife": "./dist/csstokenizer.iife.min.js",
+    "./umd": "./dist/csstokenizer.umd.min.js"
+  },
   "files": [
     "dist"
   ],
-  "scripts": {
-    "build": "yarn clean && yarn build-types && yarn build-txt && yarn rollup --config rollup.config.ts --configPlugin @rollup/plugin-json --configPlugin @rollup/plugin-typescript && yarn clean-types",
-    "build-txt": "yarn ts-node scripts/build-txt.ts",
-    "build-types": "tsc --declaration --emitDeclarationOnly --outdir dist/types",
-    "benchmark": "yarn build && node -r esbuild-register benchmark/index.ts",
-    "check-types": "tsc --noEmit",
-    "clean": "rimraf dist",
-    "clean-types": "rimraf dist/types",
-    "coverage": "jest --coverage",
-    "increment": "yarn version --patch --no-git-tag-version",
-    "lint": "yarn lint:ts && yarn lint:md",
-    "lint:md": "markdownlint .",
-    "lint:ts": "eslint . --cache --ext .ts",
-    "test": "jest --runInBand"
-  },
   "devDependencies": {
@@ -84,2 +80,3 @@ "@babel/core": "^7.22.8",
     "parse-css": "^0.1.0",
+    "rimraf": "^5.0.5",
     "rollup": "^3.29.4",
@@ -93,3 +90,17 @@ "rollup-plugin-dts": "^6.0.2",
   },
-  "dependencies": {}
-}
+  "scripts": {
+    "build": "pnpm clean && pnpm build-types && pnpm build-txt && pnpm rollup --config rollup.config.ts --configPlugin @rollup/plugin-json --configPlugin @rollup/plugin-typescript && pnpm clean-types",
+    "build-txt": "pnpm ts-node scripts/build-txt.ts",
+    "build-types": "tsc --declaration --emitDeclarationOnly --outdir dist/types",
+    "benchmark": "pnpm build && node -r esbuild-register benchmark/index.ts",
+    "check-types": "tsc --noEmit",
+    "clean": "rimraf dist",
+    "clean-types": "rimraf dist/types",
+    "coverage": "jest --coverage",
+    "increment": "pnpm version patch --no-git-tag-version",
+    "lint": "pnpm lint:ts && pnpm lint:md",
+    "lint:md": "markdownlint .",
+    "lint:ts": "eslint . --cache --ext .ts",
+    "test": "jest --runInBand"
+  }
+}

README.md

@@ -23,2 +23,13 @@ <!-- omit in toc -->
 - [API](#api)
+    - [Tokenizer functions](#tokenizer-functions)
+        - [`tokenize`](#tokenize)
+        - [`tokenizeExtended`](#tokenizeextended)
+    - [Utilities](#utilities)
+        - [`TokenizerContext`](#tokenizercontext)
+        - [`decodeIdent`](#decodeident)
+    - [`CSS_TOKENIZER_VERSION`](#css_tokenizer_version)
+    - [Token types](#token-types)
+        - [`TokenType`](#tokentype)
+        - [`getBaseTokenName`](#getbasetokenname)
+        - [`getFormattedTokenName`](#getformattedtokenname)
 - [Benchmark results](#benchmark-results)
@@ -48,5 +59,5 @@ - [Ideas \& Questions](#ideas--questions)
 <!--markdownlint-disable MD013-->
-- <img src="https://cdn.adguard.com/website/github.com/AGLint/adg_logo.svg" width="14px"> [AdGuard: *Extended CSS capabilities*][adg-ext-css]
-- <img src="https://cdn.adguard.com/website/github.com/AGLint/ubo_logo.svg" width="14px"> [uBlock Origin: *Procedural cosmetic filters*][ubo-procedural]
-- <img src="https://cdn.adguard.com/website/github.com/AGLint/abp_logo.svg" width="14px"> [Adblock Plus: *Extended CSS selectors*][abp-ext-css]
+- <img src="https://cdn.adguard.com/website/github.com/AGLint/adg_logo.svg" alt="AdGuard logo" width="14px"> [AdGuard: *Extended CSS capabilities*][adg-ext-css]
+- <img src="https://cdn.adguard.com/website/github.com/AGLint/ubo_logo.svg" alt="uBlock Origin logo" width="14px"> [uBlock Origin: *Procedural cosmetic filters*][ubo-procedural]
+- <img src="https://cdn.adguard.com/website/github.com/AGLint/abp_logo.svg" alt="Adblock Plus logo" width="14px"> [Adblock Plus: *Extended CSS selectors*][abp-ext-css]
 <!--markdownlint-enable MD013-->
@@ -139,21 +150,169 @@
 
-Tokenization is accomplished by calling the tokenize or tokenizeExtended function. Both functions accept the following
-arguments:
+### Tokenizer functions
 
-- `source`: The CSS source string to tokenize.
-- `onToken`: A callback function invoked for each token found in the source string, with the following arguments:
-    <!-- TODO: Add link -->
-    - `token`: The token type (you can see token types here).
-    - `start`: The starting index of the token in the source string.
-    - `end`: The ending index of the token in the source string.
-- `onError`: A callback function called when an error occurs during tokenization. Errors do not break the tokenization
-process, as the tokenizer is tolerant and attempts to recover from errors in line with the CSS Syntax Level 3
-specification. The callback function accepts the following arguments:
-    - `message`: The error message.
-    - `start`: The starting index of the error in the source string.
-    - `end`: The ending index of the error in the source string.
-- `functionHandlers`: This allows for the customized handling of functions. Map keys correspond to function names,
-while the values are void callback functions serving as "tokenizer context" functions. These functions can be used to
-manage pseudo-classes and have only one argument: the shared tokenizer context object.
+#### `tokenize`
 
+```ts
+/**
+ * CSS tokenizer function
+ *
+ * @param source Source code to tokenize
+ * @param onToken Tokenizer callback which is called for each token found in source code
+ * @param onError Error callback which is called when a parsing error is found (optional)
+ * @param functionHandlers Custom function handlers (optional)
+ */
+function tokenize(
+    source: string,
+    onToken: OnTokenCallback,
+    onError: OnErrorCallback = () => {},
+    functionHandlers?: Map<number, TokenizerContextFunction>,
+): void;
+```
+
+where
+
+```ts
+/**
+ * Callback which is called when a token is found
+ *
+ * @param type Token type
+ * @param start Token start offset
+ * @param end Token end offset
+ * @param props Other token properties (if any)
+ * @note Hash tokens have a type flag set to either "id" or "unrestricted". The type flag defaults to "unrestricted" if
+ * not otherwise set
+ */
+type OnTokenCallback = (type: TokenType, start: number, end: number, props?: Record<string, unknown>) => void;
+```
+
+```ts
+/**
+ * Callback which is called when a parsing error is found. According to the spec, parsing errors are not fatal and
+ * therefore the tokenizer is quite permissive, but if needed, the error callback can be used.
+ *
+ * @param message Error message
+ * @param start Error start offset
+ * @param end Error end offset
+ * @see {@link https://www.w3.org/TR/css-syntax-3/#error-handling}
+ */
+type OnErrorCallback = (message: string, start: number, end: number) => void;
+```
+
+```ts
+/**
+ * Function handler
+ *
+ * @param context Reference to the tokenizer context instance
+ * @param ...args Additional arguments (if any)
+ */
+type TokenizerContextFunction = (context: TokenizerContext, ...args: any[]) => void;
+```
+
+#### `tokenizeExtended`
+
+`tokenizeExtended` is an extended version of the `tokenize` function that supports custom function handlers. This
+function is designed to handle special pseudo-classes like `:contains()` and `:xpath()`.
+
+```ts
+/**
+ * Extended CSS tokenizer function
+ *
+ * @param source Source code to tokenize
+ * @param onToken Tokenizer callback which is called for each token found in source code
+ * @param onError Error callback which is called when a parsing error is found (optional)
+ * @param functionHandlers Custom function handlers (optional)
+ * @note If you specify custom function handlers, they will be merged with the default function handlers. If you
+ * duplicate a function handler, the custom one will be used instead of the default one, so you can override the default
+ * function handlers this way, if you want to.
+ */
+function tokenizeExtended(
+    source: string,
+    onToken: OnTokenCallback,
+    onError: OnErrorCallback = () => {},
+    functionHandlers: Map<number, TokenizerContextFunction> = new Map(),
+): void
+```
+
+### Utilities
+
+#### `TokenizerContext`
+
+A class that represents the tokenizer context. It is used to manage the tokenizer state and provides access to the
+source code, current position, and other relevant information.
+
+#### `decodeIdent`
+
+```ts
+/**
+ * Decodes a CSS identifier according to the CSS Syntax Module Level 3 specification.
+ *
+ * @param ident CSS identifier to decode.
+ *
+ * @example
+ * ```ts
+ * decodeIdent(String.raw`\00075\00072\0006C`); // 'url'
+ * decodeIdent('url'); // 'url'
+ * ```
+ *
+ * @returns Decoded CSS identifier.
+ */
+function decodeIdent(ident: string): string;
+```
+
+### `CSS_TOKENIZER_VERSION`
+
+```ts
+/**
+ * @adguard/css-tokenizer version
+ */
+const CSS_TOKENIZER_VERSION: string;
+```
+
+### Token types
+
+#### `TokenType`
+
+An enumeration of token types recognized by the tokenizer.
+They are strictly based on the CSS Syntax Level 3 specification.
+
+See https://www.w3.org/TR/css-syntax-3/#tokenization for more details.
+
+#### `getBaseTokenName`
+
+```ts
+/**
+ * Get base token name by token type
+ *
+ * @param type Token type
+ *
+ * @example
+ * ```ts
+ * getBaseTokenName(TokenType.Ident); // 'ident'
+ * getBaseTokenName(-1); // 'unknown'
+ * ```
+ *
+ * @returns Base token name or 'unknown' if token type is unknown
+ */
+function getBaseTokenName(type: TokenType): string;
+```
+
+#### `getFormattedTokenName`
+
+```ts
+/**
+ * Get formatted token name by token type
+ *
+ * @param type Token type
+ *
+ * @example
+ * ```ts
+ * getFormattedTokenName(TokenType.Ident); // '<ident-token>'
+ * getFormattedTokenName(-1); // '<unknown-token>'
+ * ```
+ *
+ * @returns Formatted token name or `'<unknown-token>'` if token type is unknown
+ */
+function getFormattedTokenName(type: TokenType): string;
+```
+
 > [!NOTE]
@@ -193,5 +352,5 @@ > Our API and token list is also compatible with the [CSSTree][css-tree-repo]'s tokenizer API, and in the long term, we
 [npm-url]: https://www.npmjs.com/package/@adguard/css-tokenizer
-[pnpm-pkg-manager-url]: https://pnpm.js.org/
+[pnpm-pkg-manager-url]: https://pnpm.io/
 [ubo-procedural]: https://github.com/gorhill/uBlock/wiki/Procedural-cosmetic-filters
 [xpath-mdn]: https://developer.mozilla.org/en-US/docs/Web/XPath
 [yarn-pkg-manager-url]: https://yarnpkg.com/en/docs/install

New alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

210289

increased by333.29%

Number of package files

9

increased by12.5%

Lines of code

3892

increased by885.32%

Number of lines in readme file

353

increased by81.96%

Worsened metrics

Dev dependency count

48

increased by2.13%

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes