edge-lexer
Advanced tools
Comparing version 1.0.8 to 2.0.0
@@ -1,8 +0,2 @@ | ||
declare enum NodeType { | ||
BLOCK = "block", | ||
RAW = "raw", | ||
NEWLINE = "newline", | ||
MUSTACHE = "mustache" | ||
} | ||
declare enum MustacheType { | ||
export declare enum MustacheTypes { | ||
SMUSTACHE = "s__mustache", | ||
@@ -13,49 +7,67 @@ ESMUSTACHE = "es__mustache", | ||
} | ||
declare enum WhiteSpaceModes { | ||
NONE = 0, | ||
ALL = 1 | ||
export declare enum TagTypes { | ||
TAG = "tag", | ||
ETAG = "e__tag" | ||
} | ||
interface IProp { | ||
export declare type ITagProp = { | ||
name: string; | ||
jsArg: string; | ||
raw: string; | ||
} | ||
interface IBlockProp extends IProp { | ||
selfclosed: boolean; | ||
} | ||
interface IMustacheProp { | ||
name: MustacheType; | ||
}; | ||
export declare type IMustacheProp = { | ||
jsArg: string; | ||
raw: string; | ||
textLeft: string; | ||
textRight: string; | ||
} | ||
interface INode { | ||
type: NodeType; | ||
value?: string; | ||
lineno: number; | ||
} | ||
interface IBlockNode extends INode { | ||
properties: IBlockProp; | ||
children: Array<INode | IBlockNode>; | ||
} | ||
interface IMustacheNode extends INode { | ||
properties: IProp; | ||
} | ||
interface ITagDefination { | ||
}; | ||
export declare type ILoc = { | ||
start: { | ||
line: number; | ||
col: number; | ||
}; | ||
end: { | ||
line: number; | ||
col: number; | ||
}; | ||
}; | ||
export declare type ITagDefination = { | ||
block: boolean; | ||
seekable: boolean; | ||
}; | ||
export declare type IRawToken = { | ||
type: 'raw'; | ||
value: string; | ||
line: number; | ||
}; | ||
export declare type INewLineToken = { | ||
type: 'newline'; | ||
line: number; | ||
}; | ||
export declare type IMustacheToken = { | ||
type: MustacheTypes; | ||
properties: IMustacheProp; | ||
loc: ILoc; | ||
}; | ||
export declare type ITagToken = { | ||
type: TagTypes; | ||
properties: ITagProp; | ||
loc: ILoc; | ||
children: Array<IRawToken | INewLineToken | ITagToken | IMustacheToken>; | ||
}; | ||
export declare type IRuntimeTag = { | ||
name: string; | ||
selfclosed: boolean; | ||
escaped?: boolean; | ||
col: number; | ||
line: number; | ||
block: boolean; | ||
seekable: boolean; | ||
new?(): any; | ||
} | ||
export { IProp as IProp }; | ||
export { INode as INode }; | ||
export { IBlockNode as IBlockNode }; | ||
export { IMustacheNode as IMustacheNode }; | ||
export { NodeType as NodeType }; | ||
export { IMustacheProp as IMustacheProp }; | ||
export { WhiteSpaceModes as WhiteSpaceModes }; | ||
export { MustacheType as MustacheType }; | ||
export { ITagDefination as ITagDefination }; | ||
export { IBlockProp as IBlockProp }; | ||
escaped: boolean; | ||
hasBrace: boolean; | ||
}; | ||
export declare type IRuntimeMustache = { | ||
escaped: boolean; | ||
safe: boolean; | ||
line: number; | ||
col: number; | ||
realCol: number; | ||
}; | ||
export declare type ITags = { | ||
[name: string]: ITagDefination; | ||
}; |
"use strict"; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
var NodeType; | ||
(function (NodeType) { | ||
NodeType["BLOCK"] = "block"; | ||
NodeType["RAW"] = "raw"; | ||
NodeType["NEWLINE"] = "newline"; | ||
NodeType["MUSTACHE"] = "mustache"; | ||
})(NodeType || (NodeType = {})); | ||
exports.NodeType = NodeType; | ||
var MustacheType; | ||
(function (MustacheType) { | ||
MustacheType["SMUSTACHE"] = "s__mustache"; | ||
MustacheType["ESMUSTACHE"] = "es__mustache"; | ||
MustacheType["MUSTACHE"] = "mustache"; | ||
MustacheType["EMUSTACHE"] = "e__mustache"; | ||
})(MustacheType || (MustacheType = {})); | ||
exports.MustacheType = MustacheType; | ||
var WhiteSpaceModes; | ||
(function (WhiteSpaceModes) { | ||
WhiteSpaceModes[WhiteSpaceModes["NONE"] = 0] = "NONE"; | ||
WhiteSpaceModes[WhiteSpaceModes["ALL"] = 1] = "ALL"; | ||
})(WhiteSpaceModes || (WhiteSpaceModes = {})); | ||
exports.WhiteSpaceModes = WhiteSpaceModes; | ||
var MustacheTypes; | ||
(function (MustacheTypes) { | ||
MustacheTypes["SMUSTACHE"] = "s__mustache"; | ||
MustacheTypes["ESMUSTACHE"] = "es__mustache"; | ||
MustacheTypes["MUSTACHE"] = "mustache"; | ||
MustacheTypes["EMUSTACHE"] = "e__mustache"; | ||
})(MustacheTypes = exports.MustacheTypes || (exports.MustacheTypes = {})); | ||
var TagTypes; | ||
(function (TagTypes) { | ||
TagTypes["TAG"] = "tag"; | ||
TagTypes["ETAG"] = "e__tag"; | ||
})(TagTypes = exports.TagTypes || (exports.TagTypes = {})); |
@@ -6,7 +6,7 @@ import { EdgeError } from 'edge-error'; | ||
}, filename: string): EdgeError; | ||
export declare function unwrappedJSExp(chars: string, pos: { | ||
export declare function unclosedParen(pos: { | ||
line: number; | ||
col: number; | ||
}, filename: string): EdgeError; | ||
export declare function unclosedParen(pos: { | ||
export declare function unopenedParen(pos: { | ||
line: number; | ||
@@ -13,0 +13,0 @@ col: number; |
@@ -12,4 +12,4 @@ "use strict"; | ||
exports.cannotSeekStatement = cannotSeekStatement; | ||
function unwrappedJSExp(chars, pos, filename) { | ||
return new edge_error_1.EdgeError(`Unexpected token "${chars}"`, 'E_UNWRAPPED_JS_EXPRESSION', { | ||
function unclosedParen(pos, filename) { | ||
return new edge_error_1.EdgeError(`Missing token ")"`, 'E_UNCLOSED_PAREN', { | ||
line: pos.line, | ||
@@ -20,5 +20,5 @@ col: pos.col, | ||
} | ||
exports.unwrappedJSExp = unwrappedJSExp; | ||
function unclosedParen(pos, filename) { | ||
return new edge_error_1.EdgeError(`Missing token ")"`, 'E_UNCLOSED_PAREN', { | ||
exports.unclosedParen = unclosedParen; | ||
function unopenedParen(pos, filename) { | ||
return new edge_error_1.EdgeError(`Missing token "("`, 'E_UNOPENED_PAREN', { | ||
line: pos.line, | ||
@@ -29,3 +29,3 @@ col: pos.col, | ||
} | ||
exports.unclosedParen = unclosedParen; | ||
exports.unopenedParen = unopenedParen; | ||
function unclosedCurlyBrace(pos, filename) { | ||
@@ -32,0 +32,0 @@ return new edge_error_1.EdgeError(`Missing token "}"`, 'E_UNCLOSED_CURLY_BRACE', { |
@@ -1,2 +0,2 @@ | ||
import { INode, IBlockNode, ITagDefination } from '../Contracts'; | ||
import { ITagToken, IMustacheToken, ITags, IRawToken, INewLineToken } from '../Contracts'; | ||
declare type tokenizerOptions = { | ||
@@ -9,3 +9,3 @@ filename: string; | ||
private options; | ||
tokens: Array<INode | IBlockNode>; | ||
tokens: Array<IRawToken | INewLineToken | ITagToken | IMustacheToken>; | ||
private blockStatement; | ||
@@ -15,19 +15,20 @@ private mustacheStatement; | ||
private openedTags; | ||
constructor(template: string, tagsDef: { | ||
[key: string]: ITagDefination; | ||
}, options: tokenizerOptions); | ||
constructor(template: string, tagsDef: ITags, options: tokenizerOptions); | ||
private _getRawNode; | ||
private _getNewLineNode; | ||
private _getTagNode; | ||
private _consumeTag; | ||
private _handleTagOpening; | ||
private _feedCharsToCurrentTag; | ||
private _getMustacheType; | ||
private _getMustacheNode; | ||
private _handleMustacheOpening; | ||
private _feedCharsToCurrentMustache; | ||
private _isClosingTag; | ||
private _consumeNode; | ||
private _pushNewLine; | ||
private _processText; | ||
private _checkForErrors; | ||
parse(): void; | ||
private getTag; | ||
private getTagNode; | ||
private getRawNode; | ||
private getBlankLineNode; | ||
private getMustacheNode; | ||
private isClosingTag; | ||
private isSeeking; | ||
private isSeeked; | ||
private consumeNode; | ||
private feedTextToBlockStatement; | ||
private feedTextToMustacheStatement; | ||
private processText; | ||
} | ||
export {}; |
"use strict"; | ||
Object.defineProperty(exports, "__esModule", { value: true }); | ||
const TagStatement_1 = require("../TagStatement"); | ||
const MustacheStatement_1 = require("../MustacheStatement"); | ||
const Detector_1 = require("../Detector"); | ||
const Scanner_1 = require("../Scanner"); | ||
const Exceptions_1 = require("../Exceptions"); | ||
const Contracts_1 = require("../Contracts"); | ||
const TAG_REGEX = /^(@{1,2})(!)?(\w+)/; | ||
const MUSTACHE_REGEX = /{{2}/; | ||
const ESCAPE_REGEX = /^(\s*)@/; | ||
const TRIM_TAG_REGEX = /^@/; | ||
class Tokenizer { | ||
@@ -22,72 +18,121 @@ constructor(template, tagsDef, options) { | ||
} | ||
parse() { | ||
const lines = this.template.split('\n'); | ||
while (lines.length) { | ||
this.line++; | ||
this.processText(lines.shift()); | ||
_getRawNode(text) { | ||
return { | ||
type: 'raw', | ||
value: text, | ||
line: this.line, | ||
}; | ||
} | ||
_getNewLineNode() { | ||
return { | ||
type: 'newline', | ||
line: this.line - 1, | ||
}; | ||
} | ||
_getTagNode(tag, jsArg, loc) { | ||
return { | ||
type: tag.escaped ? Contracts_1.TagTypes.ETAG : Contracts_1.TagTypes.TAG, | ||
properties: { | ||
name: tag.name, | ||
jsArg: jsArg, | ||
selfclosed: tag.selfclosed, | ||
}, | ||
loc: { | ||
start: { | ||
line: tag.line, | ||
col: tag.col, | ||
}, | ||
end: loc, | ||
}, | ||
children: [], | ||
}; | ||
} | ||
_consumeTag(tag, jsArg, loc) { | ||
if (tag.block && !tag.selfclosed) { | ||
this.openedTags.push(this._getTagNode(tag, jsArg, loc)); | ||
} | ||
if (this.blockStatement) { | ||
throw Exceptions_1.unclosedParen({ line: this.blockStatement.startPosition, col: 0 }, this.options.filename); | ||
else { | ||
this._consumeNode(this._getTagNode(tag, jsArg, loc)); | ||
} | ||
if (this.mustacheStatement) { | ||
throw Exceptions_1.unclosedCurlyBrace({ line: this.mustacheStatement.startPosition, col: 0 }, this.options.filename); | ||
} | ||
_handleTagOpening(line, tag) { | ||
if (tag.seekable && !tag.hasBrace) { | ||
throw Exceptions_1.unopenedParen({ line: tag.line, col: tag.col }, this.options.filename); | ||
} | ||
if (this.openedTags.length) { | ||
const openedTag = this.openedTags[this.openedTags.length - 1]; | ||
throw Exceptions_1.unclosedTag(openedTag.properties.name, { line: openedTag.lineno, col: 0 }, this.options.filename); | ||
if (!tag.seekable) { | ||
this._consumeTag(tag, '', { line: tag.line, col: tag.col }); | ||
return; | ||
} | ||
tag.col += 1; | ||
this.blockStatement = { | ||
tag: tag, | ||
scanner: new Scanner_1.Scanner(')', ['(', ')'], this.line, tag.col), | ||
}; | ||
this._feedCharsToCurrentTag(line.slice(tag.col)); | ||
} | ||
getTag(line) { | ||
const match = TAG_REGEX.exec(line.trim()); | ||
if (!match) { | ||
return null; | ||
_feedCharsToCurrentTag(content) { | ||
const { tag, scanner } = this.blockStatement; | ||
scanner.scan(content); | ||
if (!scanner.closed) { | ||
return; | ||
} | ||
const tagName = match[3]; | ||
if (!this.tagsDef[tagName]) { | ||
return null; | ||
this._consumeTag(tag, scanner.match, scanner.loc); | ||
if (scanner.leftOver.trim()) { | ||
throw Exceptions_1.cannotSeekStatement(scanner.leftOver, scanner.loc, this.options.filename); | ||
} | ||
if (match[1] === '@@') { | ||
return { | ||
escaped: true, | ||
block: false, | ||
selfclosed: false, | ||
seekable: false, | ||
}; | ||
this.blockStatement = null; | ||
} | ||
_getMustacheType(mustache) { | ||
if (mustache.safe) { | ||
return mustache.escaped ? Contracts_1.MustacheTypes.ESMUSTACHE : Contracts_1.MustacheTypes.SMUSTACHE; | ||
} | ||
const defination = this.tagsDef[tagName]; | ||
return Object.assign({ selfclosed: !!match[2] }, defination); | ||
return mustache.escaped ? Contracts_1.MustacheTypes.EMUSTACHE : Contracts_1.MustacheTypes.MUSTACHE; | ||
} | ||
getTagNode(properties, lineno) { | ||
_getMustacheNode(mustache, jsArg, loc) { | ||
return { | ||
type: Contracts_1.NodeType.BLOCK, | ||
properties, | ||
lineno, | ||
children: [], | ||
type: this._getMustacheType(mustache), | ||
properties: { | ||
jsArg: jsArg, | ||
}, | ||
loc: { | ||
start: { | ||
line: mustache.line, | ||
col: mustache.col, | ||
}, | ||
end: loc, | ||
}, | ||
}; | ||
} | ||
getRawNode(value) { | ||
return { | ||
type: Contracts_1.NodeType.RAW, | ||
value, | ||
lineno: this.line, | ||
_handleMustacheOpening(line, mustache) { | ||
const pattern = mustache.safe ? '}}}' : '}}'; | ||
const textLeftIndex = mustache.escaped ? mustache.realCol - 1 : mustache.realCol; | ||
if (textLeftIndex > 0) { | ||
this._consumeNode(this._getRawNode(line.slice(0, textLeftIndex))); | ||
} | ||
mustache.col += pattern.length; | ||
mustache.realCol += pattern.length; | ||
this.mustacheStatement = { | ||
mustache, | ||
scanner: new Scanner_1.Scanner(pattern, ['{', '}'], mustache.line, mustache.col), | ||
}; | ||
this._feedCharsToCurrentMustache(line.slice(mustache.realCol)); | ||
} | ||
getBlankLineNode() { | ||
return { | ||
type: Contracts_1.NodeType.NEWLINE, | ||
lineno: this.line, | ||
}; | ||
_feedCharsToCurrentMustache(content) { | ||
const { mustache, scanner } = this.mustacheStatement; | ||
scanner.scan(content); | ||
if (!scanner.closed) { | ||
return; | ||
} | ||
this._consumeNode(this._getMustacheNode(mustache, scanner.match, scanner.loc)); | ||
if (scanner.leftOver.trim()) { | ||
const anotherMustache = Detector_1.getMustache(scanner.leftOver, scanner.loc.line, scanner.loc.col); | ||
if (anotherMustache) { | ||
this._handleMustacheOpening(scanner.leftOver, anotherMustache); | ||
return; | ||
} | ||
this._consumeNode(this._getRawNode(scanner.leftOver)); | ||
} | ||
this.mustacheStatement = null; | ||
} | ||
getMustacheNode(properties, lineno) { | ||
return { | ||
type: Contracts_1.NodeType.MUSTACHE, | ||
lineno, | ||
properties: { | ||
name: properties.name, | ||
jsArg: properties.jsArg, | ||
raw: properties.raw, | ||
}, | ||
}; | ||
} | ||
isClosingTag(line) { | ||
_isClosingTag(line) { | ||
if (!this.openedTags.length) { | ||
@@ -99,9 +144,3 @@ return false; | ||
} | ||
isSeeking(statement) { | ||
return !!(statement && statement.seeking); | ||
} | ||
isSeeked(statement) { | ||
return statement && !statement.seeking; | ||
} | ||
consumeNode(tag) { | ||
_consumeNode(tag) { | ||
if (this.openedTags.length) { | ||
@@ -113,69 +152,61 @@ this.openedTags[this.openedTags.length - 1].children.push(tag); | ||
} | ||
feedTextToBlockStatement(text) { | ||
this.blockStatement.feed(text); | ||
if (!this.isSeeked(this.blockStatement)) { | ||
_pushNewLine() { | ||
if (this.line === 1) { | ||
return; | ||
} | ||
const { props, tagDef, startPosition } = this.blockStatement; | ||
if (tagDef.block && (!tagDef.selfclosed || !props.selfclosed)) { | ||
this.openedTags.push(this.getTagNode(props, startPosition)); | ||
} | ||
else { | ||
this.consumeNode(this.getTagNode(props, startPosition)); | ||
} | ||
this.blockStatement = null; | ||
this._consumeNode(this._getNewLineNode()); | ||
} | ||
feedTextToMustacheStatement(text) { | ||
this.mustacheStatement.feed(text); | ||
if (!this.isSeeked(this.mustacheStatement)) { | ||
_processText(line) { | ||
if (this.blockStatement) { | ||
this._feedCharsToCurrentTag('\n'); | ||
this._feedCharsToCurrentTag(line); | ||
return; | ||
} | ||
const { props, startPosition } = this.mustacheStatement; | ||
if (props.textLeft) { | ||
const textNode = this.getRawNode(props.textLeft); | ||
textNode.lineno = startPosition; | ||
this.consumeNode(textNode); | ||
} | ||
this.consumeNode(this.getMustacheNode(props, startPosition)); | ||
this.mustacheStatement = null; | ||
if (props.textRight) { | ||
this.processText(props.textRight); | ||
} | ||
else { | ||
this.consumeNode(this.getBlankLineNode()); | ||
} | ||
} | ||
processText(text) { | ||
if (this.isSeeking(this.blockStatement)) { | ||
this.feedTextToBlockStatement(text); | ||
if (this.mustacheStatement) { | ||
this._feedCharsToCurrentMustache('\n'); | ||
this._feedCharsToCurrentMustache(line); | ||
return; | ||
} | ||
if (this.isSeeking(this.mustacheStatement)) { | ||
this.feedTextToMustacheStatement(text); | ||
if (this._isClosingTag(line)) { | ||
this._consumeNode(this.openedTags.pop()); | ||
return; | ||
} | ||
const tag = this.getTag(text); | ||
if (tag && tag.escaped) { | ||
this.consumeNode(this.getRawNode(text.replace(ESCAPE_REGEX, '$1'))); | ||
this.consumeNode(this.getBlankLineNode()); | ||
return; | ||
} | ||
this._pushNewLine(); | ||
const tag = Detector_1.getTag(line, this.line, 0, this.tagsDef); | ||
if (tag) { | ||
this.blockStatement = new TagStatement_1.TagStatement(this.line, tag, this.options.filename); | ||
this.feedTextToBlockStatement(text.trim().replace(TRIM_TAG_REGEX, '')); | ||
this._handleTagOpening(line, tag); | ||
return; | ||
} | ||
if (this.isClosingTag(text)) { | ||
this.consumeNode(this.openedTags.pop()); | ||
const mustache = Detector_1.getMustache(line, this.line, 0); | ||
if (mustache) { | ||
this._handleMustacheOpening(line, mustache); | ||
return; | ||
} | ||
if (MUSTACHE_REGEX.test(text)) { | ||
this.mustacheStatement = new MustacheStatement_1.MustacheStatement(this.line); | ||
this.feedTextToMustacheStatement(text); | ||
return; | ||
this._consumeNode(this._getRawNode(line)); | ||
} | ||
_checkForErrors() { | ||
if (this.blockStatement) { | ||
const { tag } = this.blockStatement; | ||
throw Exceptions_1.unclosedParen({ line: tag.line, col: tag.col }, this.options.filename); | ||
} | ||
this.consumeNode(this.getRawNode(text)); | ||
this.consumeNode(this.getBlankLineNode()); | ||
if (this.mustacheStatement) { | ||
const { mustache } = this.mustacheStatement; | ||
throw Exceptions_1.unclosedCurlyBrace({ line: mustache.line, col: mustache.col }, this.options.filename); | ||
} | ||
if (this.openedTags.length) { | ||
const openedTag = this.openedTags[this.openedTags.length - 1]; | ||
throw Exceptions_1.unclosedTag(openedTag.properties.name, openedTag.loc.start, this.options.filename); | ||
} | ||
} | ||
parse() { | ||
const lines = this.template.split('\n'); | ||
const linesLength = lines.length; | ||
while (this.line < linesLength) { | ||
const line = lines[this.line]; | ||
this.line++; | ||
this._processText(line); | ||
} | ||
this._checkForErrors(); | ||
} | ||
} | ||
exports.Tokenizer = Tokenizer; |
@@ -0,1 +1,12 @@ | ||
<a name="2.0.0"></a> | ||
# [2.0.0](https://github.com/edge-js/edge-lexer/compare/1.0.8...2.0.0) (2018-11-08) | ||
### Features | ||
* rewrite for performance ([82e9752](https://github.com/edge-js/edge-lexer/commit/82e9752)) | ||
* track col numbers for better debugging experience ([5778f77](https://github.com/edge-js/edge-lexer/commit/5778f77)) | ||
<a name="1.0.8"></a> | ||
@@ -2,0 +13,0 @@ ## [1.0.8](https://github.com/poppinss/edge-lexer/compare/v1.0.7...v1.0.8) (2018-11-03) |
{ | ||
"name": "edge-lexer", | ||
"version": "1.0.8", | ||
"description": "Parses raw markup files to converts them to Tokens", | ||
"version": "2.0.0", | ||
"description": "Parses raw markup files to converts them to Edge tokens", | ||
"main": "build/src/Tokenizer/index.js", | ||
@@ -35,2 +35,3 @@ "typings": "build/src/Tokenizer/index.d.ts", | ||
"@types/node": "^10.12.2", | ||
"benchmark": "^2.1.4", | ||
"commitizen": "^3.0.4", | ||
@@ -53,4 +54,3 @@ "coveralls": "^3.0.2", | ||
"dependencies": { | ||
"edge-error": "^1.0.0", | ||
"is-whitespace-character": "^1.0.2" | ||
"edge-error": "^1.0.0" | ||
}, | ||
@@ -57,0 +57,0 @@ "config": { |
219
README.md
@@ -11,2 +11,9 @@ # Edge lexer | ||
- ✅ Zero dependencies (Actually one dependency that is also to standardize edge errors). | ||
- ✅ Just uses one regex statement. That also tested against [safe-regex](https://github.com/substack/safe-regex) for ReDOS | ||
- ✅ Allows multiline expressions | ||
- ✅ Collects line and columns for accurate stack traces. | ||
- ✅ Detects for unclosed tags. | ||
- ✅ Detects for unwrapped expressions and raises appropriate errors. | ||
Edge lexer produces a list of `tokens` by scanning for [Edge whitelisted syntax](https://github.com/edge-js/syntax). | ||
@@ -16,3 +23,3 @@ | ||
Instead, this module starts with some REGEX patterns to detect the [Edge whitelisted syntax](https://github.com/edge-js/syntax) and then starts the lexical analysis within the detected markup. | ||
Instead, this module starts by detecting for the [Edge whitelisted syntax](https://github.com/edge-js/syntax) and then starts the lexical analysis within the detected markup. | ||
@@ -23,3 +30,3 @@ --- | ||
Following measures are taken to keep the analysis performant | ||
Following measures are taken to keep the analysis performant. | ||
@@ -29,2 +36,3 @@ 1. Only analyse markup that is detected as Edge whitelisted syntax. | ||
3. Do not analyse Javascript expression and leave that for [edge-parser](https://github.com/edge-js/parser). | ||
4. Only uses one Regular expression. | ||
@@ -43,4 +51,3 @@ | ||
block: true, | ||
selfclosed: false, | ||
seekable: true | ||
seekable: true, | ||
} | ||
@@ -62,15 +69,13 @@ } | ||
## Features | ||
1. Allows multiline expressions. | ||
2. Whitespaces and newlines are retained. | ||
3. Detects for unclosed tags. | ||
4. Detects for unwrapped expressions and raises appropriate errors. | ||
## Terms used | ||
This guide makes use of the following terms to identify core pieces of the tokenizer. | ||
| Term | Node Type | Description | | ||
| Term | Token Type | Description | | ||
|------|-----------|------------ | | ||
| Tag | block | Tags are used to define logical blocks in the template engine. For example `if tag` or `include tag`. | | ||
| Mustache | mustache | Javascript expression wrapped in curly braces. `{{ }}` | | ||
| Tag | tag | Tags are used to define logical blocks in the template engine. For example `if tag` or `include tag`. | | ||
| Escaped Tag | e__tag | Escaped tag, Edge will not evaluate it at rutime. | | ||
| Mustache | mustache | Javascript expression wrapped in curly braces. `{{ }}`| | ||
| Safe Mustache | s__mustache | Safe mustache, that doesn't escape the output `{{{ }}}`| | ||
| Escaped Mustache | e__mustache | Mustache tag that is escaped | | ||
| Escaped Safe Mustache | es__mustache | Safe Mustache tag that is escaped | | ||
| Raw | raw | A raw string, which has no meaning for the template engine | | ||
@@ -82,11 +87,20 @@ | NewLine | newline | Newline | | ||
## Nodes | ||
## Tokens | ||
Following is the list of Nodes returned by the tokenizer. | ||
#### Block Node | ||
#### Tag Token | ||
```js | ||
{ | ||
type: 'block' | ||
lineno: number, | ||
type: 'tag' | ||
loc: { | ||
start: { | ||
line: 1, | ||
col: 4 | ||
}, | ||
end: { | ||
line: 1, | ||
col: 13 | ||
} | ||
}, | ||
properties: BlockProp, | ||
@@ -97,8 +111,29 @@ children: [] | ||
#### Raw Node | ||
#### Escaped Tag Token | ||
```diff | ||
{ | ||
- type: 'tag', | ||
+ type: 'e__tag', | ||
loc: { | ||
start: { | ||
line: 1, | ||
col: 4 | ||
}, | ||
end: { | ||
line: 1, | ||
col: 13 | ||
} | ||
}, | ||
properties: BlockProp, | ||
children: [] | ||
} | ||
``` | ||
#### Raw Token | ||
```js | ||
{ | ||
type: 'raw', | ||
lineno: number, | ||
line: number, | ||
value: string | ||
@@ -108,3 +143,3 @@ } | ||
#### Comment Node | ||
#### Comment Token | ||
@@ -114,3 +149,3 @@ ```js | ||
type: 'comment', | ||
lineno: number, | ||
line: number, | ||
value: string | ||
@@ -120,8 +155,26 @@ } | ||
#### Mustache Node | ||
#### NewLine Token | ||
```js | ||
{ | ||
type: 'newline', | ||
line: number | ||
} | ||
``` | ||
#### Mustache Token | ||
```js | ||
{ | ||
type: 'mustache', | ||
lineno: number, | ||
loc: { | ||
start: { | ||
line: 1, | ||
col: 4 | ||
}, | ||
end: { | ||
line: 1, | ||
col: 13 | ||
} | ||
}, | ||
properties: Prop | ||
@@ -131,19 +184,71 @@ } | ||
#### NewLine Node | ||
#### Safe Mustache Token | ||
```js | ||
```diff | ||
{ | ||
type: 'newline', | ||
lineno: number | ||
- type: 'mustache', | ||
+ type: 's__mustache', | ||
loc: { | ||
start: { | ||
line: 1, | ||
col: 4 | ||
}, | ||
end: { | ||
line: 1, | ||
col: 13 | ||
} | ||
}, | ||
properties: Prop | ||
} | ||
``` | ||
#### Escaped Mustache Token | ||
```diff | ||
{ | ||
- type: 'mustache', | ||
+ type: 'e__mustache', | ||
loc: { | ||
start: { | ||
line: 1, | ||
col: 4 | ||
}, | ||
end: { | ||
line: 1, | ||
col: 13 | ||
} | ||
}, | ||
properties: Prop | ||
} | ||
``` | ||
#### Escaped Safe Mustache Token | ||
```diff | ||
{ | ||
- type: 'mustache', | ||
+ type: 'es__mustache', | ||
loc: { | ||
start: { | ||
line: 1, | ||
col: 4 | ||
}, | ||
end: { | ||
line: 1, | ||
col: 13 | ||
} | ||
}, | ||
properties: Prop | ||
} | ||
``` | ||
| Key | Value | Description | | ||
|-----|------|-------------------| | ||
| type | string | The type of node determines the behavior of node | | ||
| lineno | number | The lineno in the source file | ||
| properties | Prop | Meta data for the node. See [Properties](#properties) to more info. | ||
| value | string | If node is a raw node, then value is the string in the source file | ||
| children | array | Array of recursive nodes. Only exists, when `type === 'block'`. | ||
| loc | object | `loc` is only present for tags and mustache tokens | | ||
| line | number | `line` is not present for tags and mustache tokens | | ||
| properties | Prop | Meta data for the node. See [Properties](#properties) to more info | | ||
| value | string | If token is a raw or comment token, then value is the string in the source file | | ||
| children | array | Array of recursive nodes. Only exists, when token is a tag | | ||
@@ -162,3 +267,2 @@ --- | ||
jsArg: string, | ||
raw: string, | ||
selfclosed: boolean | ||
@@ -172,5 +276,3 @@ } | ||
{ | ||
name: string | ||
jsArg: string, | ||
raw: string | ||
} | ||
@@ -181,5 +283,3 @@ ``` | ||
|-------|------------| | ||
| name | The name is the subtype for a given node. For example: `if` will be the name of the `@if` tag. | | ||
| jsArg | The `jsArg` is the Javascript expression to evaluate | | ||
| raw | The raw representation of a given expression. Used for debugging purposes. | | ||
| selfclosed | Whether or not the tag was `selfclosed` during usage. | | ||
@@ -249,26 +349,45 @@ | ||
```js | ||
```json | ||
[ | ||
{ | ||
"type": "block", | ||
"type": "tag", | ||
"properties": { | ||
"name": "if", | ||
"jsArg": "username", | ||
"raw": "if(username)", | ||
"selfclosed": false | ||
}, | ||
"lineno": 1, | ||
"loc": { | ||
"start": { | ||
"line": 1, | ||
"col": 4 | ||
}, | ||
"end": { | ||
"line": 1, | ||
"col": 13 | ||
} | ||
}, | ||
"children": [ | ||
{ | ||
"type": "newline", | ||
"line": 1 | ||
}, | ||
{ | ||
"type": "raw", | ||
"value": "<h2> Hello ", | ||
"lineno": 2 | ||
"line": 2 | ||
}, | ||
{ | ||
"type": "mustache", | ||
"lineno": 2, | ||
"properties": { | ||
"name": "mustache", | ||
"jsArg": " username ", | ||
"raw": "<h2> Hello {{ username }} </h2>" | ||
"jsArg": " username " | ||
}, | ||
"loc": { | ||
"start": { | ||
"line": 2, | ||
"col": 13 | ||
}, | ||
"end": { | ||
"line": 2, | ||
"col": 25 | ||
} | ||
} | ||
@@ -279,7 +398,3 @@ }, | ||
"value": " </h2>", | ||
"lineno": 2 | ||
}, | ||
{ | ||
"type": "newline", | ||
"lineno": 2 | ||
"line": 2 | ||
} | ||
@@ -317,3 +432,3 @@ ] | ||
[npm-image]: https://img.shields.io/npm/v/lexer.svg?style=flat-square&logo=npm | ||
[npm-url]: https://npmjs.org/package/lexer "npm" | ||
[npm-image]: https://img.shields.io/npm/v/edge-lexer.svg?style=flat-square&logo=npm | ||
[npm-url]: https://npmjs.org/package/edge-lexer "npm" |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
1
417
30125
18
14
530
1
- Removedis-whitespace-character@^1.0.2
- Removedis-whitespace-character@1.0.4(transitive)