Socket
Socket
Sign inDemoInstall

edge-lexer

Package Overview
Dependencies
Maintainers
1
Versions
62
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

edge-lexer - npm Package Compare versions

Comparing version 1.0.7 to 1.0.8

build/src/Exceptions/index.d.ts
build/src/Exceptions/index.js

39

build/src/CharBucket/index.d.ts

@@ -1,30 +0,2 @@

/**
* @module Lexer
*/
import { WhiteSpaceModes } from '../Contracts';
/**
* Char bucket is used to control the white space inside
* a string. You feed one character at a time to this
* class and it will make sure the whitespace is
* controlled as instructed.
*
* There are 3 modes in total
*
* 1. NONE - Zero whitespaces
* 2. ALL - All whitespaces
*
* ```
* const charBucket = new CharBucket(WhiteSpaceModes.NONE)
*
* charBucket.feed('h')
* charBucket.feed('i')
* charBucket.feed(' ')
* charBucket.feed(' ')
* charBucket.feed(' ')
* charBucket.feed('!')
*
* // Output
* charBucket.get() // hi!
* ```
*/
export declare class CharBucket {

@@ -35,16 +7,5 @@ private whitespace;

constructor(whitespace: WhiteSpaceModes);
/**
* Returns all chars recorded so far
*
* @returns string
*/
get(): string;
/**
* Feed a char to the bucket
*/
feed(char: string): void;
/**
* Remove last character from the string
*/
pop(): void;
}

47

build/src/CharBucket/index.js
"use strict";
/**
* @module Lexer
*/
Object.defineProperty(exports, "__esModule", { value: true });
/*
* edge-lexer
*
* (c) Harminder Virk <virk@adonisjs.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
const whitespace = require("is-whitespace-character");
const Contracts_1 = require("../Contracts");
/**
* Char bucket is used to control the white space inside
* a string. You feed one character at a time to this
* class and it will make sure the whitespace is
* controlled as instructed.
*
* There are 3 modes in total
*
* 1. NONE - Zero whitespaces
* 2. ALL - All whitespaces
*
* ```
* const charBucket = new CharBucket(WhiteSpaceModes.NONE)
*
* charBucket.feed('h')
* charBucket.feed('i')
* charBucket.feed(' ')
* charBucket.feed(' ')
* charBucket.feed(' ')
* charBucket.feed('!')
*
* // Output
* charBucket.get() // hi!
* ```
*/
class CharBucket {

@@ -47,13 +11,5 @@ constructor(whitespace) {

}
/**
* Returns all chars recorded so far
*
* @returns string
*/
get() {
return this.chars;
}
/**
* Feed a char to the bucket
*/
feed(char) {

@@ -70,5 +26,2 @@ this.lastChar = char;

}
/**
* Remove last character from the string
*/
pop() {

@@ -75,0 +28,0 @@ this.chars = this.chars.slice(0, -1);

3

build/src/Contracts/index.d.ts

@@ -1,4 +0,1 @@

/**
* @module Lexer
*/
declare enum NodeType {

@@ -5,0 +2,0 @@ BLOCK = "block",

3

build/src/Contracts/index.js
"use strict";
/**
* @module Lexer
*/
Object.defineProperty(exports, "__esModule", { value: true });

@@ -6,0 +3,0 @@ var NodeType;

69

build/src/MustacheStatement/index.d.ts

@@ -1,43 +0,6 @@

/**
* @module Lexer
*/
import { IMustacheProp } from '../Contracts';
/**
* The mustache statement parses the content inside the curly
* braces. Since the statement can be in multiple lines, this
* class seeks for more content unless closing braces are
* detected.
*
* ```
* const statement = new MustacheStatement(1)
* statement.feed('Hello {{ username }}!')
*
* console.log(statement.props)
* {
* name: 'mustache',
* jsArg: ' username ',
* raw: 'Hello {{ username }}!',
* textLeft: 'Hello ',
* textRight: '!'
* }
* ```
*/
export declare class MustacheStatement {
startPosition: number;
/**
* Whether or not the statement has been started. Statement
* is considered as started, when opening curly braces
* are detected.
*/
started: boolean;
/**
* Whether or not the statement has been ended. Once ended, you
* cannot feed more content.
*/
ended: boolean;
/**
* Statement meta data
*
* @type {IMustacheProp}
*/
props: IMustacheProp;

@@ -49,42 +12,10 @@ private firstCall;

constructor(startPosition: number);
/**
* Feed a new line to be parsed as mustache. For performance it is recommended
* to check that line contains alteast one `{{` statement and is not escaped
* before calling this method.
*/
feed(line: string): void;
/**
* Returns a boolean telling, if value is a safe mustache or
* escaped safe mustache type.
*/
private isSafeMustache;
/**
* Returns a boolean telling, if value is a mustache or
* escaped mustache type.
*/
private isMustache;
/**
* Returns the name of the type of the mustache tag. If char and
* surrounding chars, doesn't form an opening `{{` mustache
* pattern, then `null` will be returned
*/
private getName;
/**
* Returns a boolean telling whether the current char and surrounding
* chars form the closing of mustache.
*/
private isClosing;
/**
* Returns `true` when seeking for more content.
*/
readonly seeking: boolean;
/**
* Process one char at a time
*/
private processChar;
/**
* Sets the value from internal prop to the public prop
* as a string.
*/
private setProp;
}

119

build/src/MustacheStatement/index.js
"use strict";
/**
* @module Lexer
*/
Object.defineProperty(exports, "__esModule", { value: true });
/*
* edge-lexer
*
* (c) Harminder Virk <virk@adonisjs.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
const Contracts_1 = require("../Contracts");
const CharBucket_1 = require("../CharBucket");
/** @hidden */
const OPENING_BRACE = 123;
/** @hidden */
const CLOSING_BRACE = 125;
/**
* The mustache statement parses the content inside the curly
* braces. Since the statement can be in multiple lines, this
* class seeks for more content unless closing braces are
* detected.
*
* ```
* const statement = new MustacheStatement(1)
* statement.feed('Hello {{ username }}!')
*
* console.log(statement.props)
* {
* name: 'mustache',
* jsArg: ' username ',
* raw: 'Hello {{ username }}!',
* textLeft: 'Hello ',
* textRight: '!'
* }
* ```
*/
class MustacheStatement {
constructor(startPosition) {
this.startPosition = startPosition;
/**
* Whether or not the statement has been started. Statement
* is considered as started, when opening curly braces
* are detected.
*/
this.started = false;
/**
* Whether or not the statement has been ended. Once ended, you
* cannot feed more content.
*/
this.ended = false;

@@ -70,15 +28,3 @@ this.firstCall = true;

}
/**
* Feed a new line to be parsed as mustache. For performance it is recommended
* to check that line contains alteast one `{{` statement and is not escaped
* before calling this method.
*/
feed(line) {
if (this.ended) {
throw new Error(`Unexpected token {${line}}`);
}
/**
* If feed method is called consecutively, we need to append
* new lines
*/
if (!this.firstCall) {

@@ -92,5 +38,2 @@ this.props.raw += `\n${line}`;

}
/**
* Loop over all the characters and parse line
*/
const chars = line.split('');

@@ -101,5 +44,2 @@ while (chars.length) {

}
/**
* If not seeking, then set the last prop
*/
if (!this.seeking) {

@@ -110,21 +50,8 @@ this.setProp();

}
/**
* Returns a boolean telling, if value is a safe mustache or
* escaped safe mustache type.
*/
isSafeMustache(value) {
return [Contracts_1.MustacheType.SMUSTACHE, Contracts_1.MustacheType.ESMUSTACHE].indexOf(value) !== -1;
}
/**
* Returns a boolean telling, if value is a mustache or
* escaped mustache type.
*/
isMustache(value) {
return [Contracts_1.MustacheType.MUSTACHE, Contracts_1.MustacheType.EMUSTACHE].indexOf(value) !== -1;
}
/**
* Returns the name of the type of the mustache tag. If char and
* surrounding chars, doesn't form an opening `{{` mustache
* pattern, then `null` will be returned
*/
getName(chars, charCode) {

@@ -135,6 +62,2 @@ if (charCode !== OPENING_BRACE || !chars.length) {

let next = chars[0].charCodeAt(0);
/**
* Will be considered as mustache, when consecutive chars
* are {{
*/
const isMustache = next === OPENING_BRACE;

@@ -144,6 +67,2 @@ if (!isMustache) {

}
/**
* If mustache braces were escaped, then we need to ignore them
* and set the prop name properly
*/
const isEscaped = this.internalProps.textLeft.lastChar === '@';

@@ -157,6 +76,2 @@ if (isEscaped) {

}
/**
* Will be considered as `safe mustache`, when consecutive
* chars are {{{
*/
next = chars[0].charCodeAt(0);

@@ -170,6 +85,2 @@ const isEMustache = next === OPENING_BRACE;

}
/**
* Returns a boolean telling whether the current char and surrounding
* chars form the closing of mustache.
*/
isClosing(chars, charCode) {

@@ -179,6 +90,2 @@ if (charCode !== CLOSING_BRACE || this.internalBraces !== 0) {

}
/**
* If opening statement was detected as `s__mustache`, then expect
* 2 more consecutive chars as CLOSING_BRACE
*/
if (this.isSafeMustache(this.props.name) && chars.length >= 2) {

@@ -194,6 +101,2 @@ const next = chars[0].charCodeAt(0);

}
/**
* If opening statement was detected as `mustache`, then expect
* 1 more consecutive char as CLOSING_BRACE
*/
if (this.isMustache(this.props.name) && chars.length >= 1) {

@@ -209,25 +112,11 @@ const next = chars[0].charCodeAt(0);

}
/**
* Returns `true` when seeking for more content.
*/
get seeking() {
return this.started && !this.ended;
}
/**
* Process one char at a time
*/
processChar(chars, char) {
let name = null;
const charCode = char.charCodeAt(0);
/**
* Only process name, when are not in inside mustache
* statement.
*/
if (!this.started) {
name = this.getName(chars, charCode);
}
/**
* When a name is found, we consider it as a start
* of `mustache` statement
*/
if (name) {

@@ -240,6 +129,2 @@ this.props.name = name;

}
/**
* If statement was started and not ended and is a closing
* tag, then close mustache
*/
if (this.started && !this.ended && this.isClosing(chars, charCode)) {

@@ -259,6 +144,2 @@ this.setProp();

}
/**
* Sets the value from internal prop to the public prop
* as a string.
*/
setProp() {

@@ -265,0 +146,0 @@ this.props[this.currentProp] = this.internalProps[this.currentProp].get();

88

build/src/TagStatement/index.d.ts

@@ -1,39 +0,8 @@

/**
* @module Lexer
*/
import { IBlockProp, ITagDefination } from '../Contracts';
/**
* The tag statement parses multiline content inside
* an edge tag starting block.
*
* ```
* const statement = new TagStatement(1)
* statement.feed('@if(')
* statement.feed('username')
* statement.feed(')')
*
* console.log(statement.props)
* {
* name: 'if',
* jsArg: ' username ',
* raw: 'if(\nusername\n)'
* }
* ```
*/
export declare class TagStatement {
startPosition: number;
tagDef: ITagDefination;
/**
* Whether or not the statement has been started. This flag
* is set to true when we detect first `(`.
*/
private _fileName;
started: boolean;
/**
* Whether or not statement is ended. This flag is set when last closing
* `)` is detected.
*/
ended: boolean;
/**
* Prop defines the meta data for a statement
*/
props: IBlockProp;

@@ -44,67 +13,14 @@ private currentProp;

private firstCall;
constructor(startPosition: number, tagDef: ITagDefination);
/**
* Feed a new line to be tokenized into a statement.
* This method will collapse all whitespaces.
*
* ```js
* statement.feed('if(2 + 2 === 4)')
*
* statement.ended // true
* statement.props.name // if
* statement.props.jsArg // 2+2===4
* ```
*/
constructor(startPosition: number, tagDef: ITagDefination, _fileName: string);
feed(line: string): void;
/**
* Tells whether statement is seeking for more content
* or not. When seeking is false, it means the
* statement has been parsed successfully.
*/
readonly seeking: boolean;
/**
* Returns a boolean telling if charcode should be considered
* as the start of the statement.
*/
private isStartOfStatement;
/**
* Returns a boolean telling if charCode should be considered
* as the end of the statement
*/
private isEndOfStatement;
/**
* Starts the statement by switching the currentProp to
* `jsArg` and setting the started flag to true.
*/
private startStatement;
/**
* Ends the statement by switching the ended flag to true. Also
* if `started` flag was never switched on, then it will throw
* an exception.
*/
private endStatement;
/**
* Feeds character to the currentProp. Also this method will
* record the toll of `opening` and `closing` parenthesis.
*/
private feedChar;
/**
* Throws exception when end of the statement is reached, but there
* are more chars to be feeded. This can be because of unclosed
* statement or following code is not in a new line.
*/
private ensureNoMoreCharsToFeed;
/**
* Sets the prop value for the current Prop and set the
* corresponding ChatBucket to null.
*/
private setProp;
/**
* Feeds a non-seekable statement
*/
private feedNonSeekable;
/**
* Feeds a seekable statement
*/
private feedSeekable;
}

116

build/src/TagStatement/index.js
"use strict";
/**
* @module Lexer
*/
Object.defineProperty(exports, "__esModule", { value: true });
/*
* edge-lexer
*
* (c) Harminder Virk <virk@adonisjs.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
const Contracts_1 = require("../Contracts");
const CharBucket_1 = require("../CharBucket");
/** @hidden */
const Exceptions_1 = require("../Exceptions");
const OPENING_BRACE = 40;
/** @hidden */
const CLOSING_BRACE = 41;
/**
* The tag statement parses multiline content inside
* an edge tag starting block.
*
* ```
* const statement = new TagStatement(1)
* statement.feed('@if(')
* statement.feed('username')
* statement.feed(')')
*
* console.log(statement.props)
* {
* name: 'if',
* jsArg: ' username ',
* raw: 'if(\nusername\n)'
* }
* ```
*/
class TagStatement {
constructor(startPosition, tagDef) {
constructor(startPosition, tagDef, _fileName) {
this.startPosition = startPosition;
this.tagDef = tagDef;
/**
* Whether or not the statement has been started. This flag
* is set to true when we detect first `(`.
*/
this._fileName = _fileName;
this.started = false;
/**
* Whether or not statement is ended. This flag is set when last closing
* `)` is detected.
*/
this.ended = false;

@@ -66,22 +29,6 @@ this.currentProp = 'name';

}
/**
* Feed a new line to be tokenized into a statement.
* This method will collapse all whitespaces.
*
* ```js
* statement.feed('if(2 + 2 === 4)')
*
* statement.ended // true
* statement.props.name // if
* statement.props.jsArg // 2+2===4
* ```
*/
feed(line) {
if (this.ended) {
throw new Error(`Unexpected token {${line}}. Write in a new line`);
throw Exceptions_1.cannotSeekStatement(line, { line: this.startPosition, col: 0 }, this._fileName);
}
/**
* If feed is called consecutively, then we need to append
* a new line to the current prop and the raw prop
*/
if (!this.firstCall) {

@@ -95,6 +42,2 @@ this.props.raw += `\n${line}`;

}
/**
* If statement doesn't seek for args, then end it
* write their
*/
if (!this.tagDef.seekable) {

@@ -104,33 +47,13 @@ this.feedNonSeekable(line);

}
/**
* Feed a seekable string by tokenizing it
*/
this.feedSeekable(line);
}
/**
* Tells whether statement is seeking for more content
* or not. When seeking is false, it means the
* statement has been parsed successfully.
*/
get seeking() {
return !this.started || !this.ended;
}
/**
* Returns a boolean telling if charcode should be considered
* as the start of the statement.
*/
isStartOfStatement(charcode) {
return charcode === OPENING_BRACE && this.currentProp === 'name';
}
/**
* Returns a boolean telling if charCode should be considered
* as the end of the statement
*/
isEndOfStatement(charcode) {
return charcode === CLOSING_BRACE && this.internalParens === 0;
}
/**
* Starts the statement by switching the currentProp to
* `jsArg` and setting the started flag to true.
*/
startStatement() {

@@ -141,10 +64,5 @@ this.setProp();

}
/**
* Ends the statement by switching the ended flag to true. Also
* if `started` flag was never switched on, then it will throw
* an exception.
*/
endStatement(char) {
if (!this.started) {
throw new Error(`Unexpected token ${char}. Wrap statement inside ()`);
throw Exceptions_1.unwrappedJSExp(char, { line: this.startPosition, col: 0 }, this._fileName);
}

@@ -155,6 +73,2 @@ this.ended = true;

}
/**
* Feeds character to the currentProp. Also this method will
* record the toll of `opening` and `closing` parenthesis.
*/
feedChar(char, charCode) {

@@ -167,5 +81,2 @@ if (charCode === OPENING_BRACE) {

}
/**
* Ignore ! when tag is selfclosed and currentProp is name
*/
if (this.currentProp === 'name' && char === '!' && this.tagDef.selfclosed) {

@@ -177,22 +88,10 @@ this.props.selfclosed = true;

}
/**
* Throws exception when end of the statement is reached, but there
* are more chars to be feeded. This can be because of unclosed
* statement or following code is not in a new line.
*/
ensureNoMoreCharsToFeed(chars) {
if (chars.length) {
throw new Error(`Unexpected token {${chars.join('')}}. Write in a new line`);
throw Exceptions_1.cannotSeekStatement(chars.join(''), { line: this.startPosition, col: 0 }, this._fileName);
}
}
/**
* Sets the prop value for the current Prop and set the
* corresponding ChatBucket to null.
*/
setProp() {
this.props[this.currentProp] = this.internalProps[this.currentProp].get();
}
/**
* Feeds a non-seekable statement
*/
feedNonSeekable(line) {

@@ -204,5 +103,2 @@ this.props.name = line.trim();

}
/**
* Feeds a seekable statement
*/
feedSeekable(line) {

@@ -209,0 +105,0 @@ const chars = line.split('');

58

build/src/Tokenizer/index.d.ts

@@ -1,4 +0,1 @@

/**
* @module Lexer
*/
import { INode, IBlockNode, ITagDefination } from '../Contracts';

@@ -8,9 +5,2 @@ declare type tokenizerOptions = {

};
/**
* Tokenizer converts a bunch of text into an array of tokens. Later
* these tokens can be used to build the transformed text.
*
* Go through the README file to learn more about the syntax and
* the tokens output.
*/
export declare class Tokenizer {

@@ -28,64 +18,16 @@ private template;

}, options: tokenizerOptions);
/**
* Parses the AST
*/
parse(): void;
/**
* Returns the tag defination when line matches the regex
* of a tag.
*/
private getTag;
/**
* Returns the node for a tag
*/
private getTagNode;
/**
* Returns the node for a raw string
*/
private getRawNode;
/**
* Returns the node for a newline
*/
private getBlankLineNode;
/**
* Returns the mustache node
*/
private getMustacheNode;
/**
* Returns a boolean, when line content is a closing
* tag
*/
private isClosingTag;
/**
* Returns a boolean, telling if a given statement is seeking
* for more content or not
*/
private isSeeking;
/**
* Returns a boolean, telling if a given statement has ended or
* not.
*/
private isSeeked;
/**
* Here we add the node to tokens or as children for
* the recentOpenedTag (if one exists).
*/
private consumeNode;
/**
* Feeds the text to the currently opened block statement.
* Make sure that `seeking` is true on the block
* statement, before calling this method.
*/
private feedTextToBlockStatement;
/**
* Feeds text to the currently opened mustache statement. Make sure
* to check `seeking` is true, before calling this method.
*/
private feedTextToMustacheStatement;
/**
* Process a piece of text, by finding if text has reserved keywords,
* otherwise process it as a raw node.
*/
private processText;
}
export {};

141

build/src/Tokenizer/index.js
"use strict";
/**
* @module Lexer
*/
Object.defineProperty(exports, "__esModule", { value: true });
/*
* edge-lexer
*
* (c) Harminder Virk <virk@adonisjs.com>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
const edge_error_1 = require("edge-error");
const TagStatement_1 = require("../TagStatement");
const MustacheStatement_1 = require("../MustacheStatement");
const Exceptions_1 = require("../Exceptions");
const Contracts_1 = require("../Contracts");
/** @hidden */
const TAG_REGEX = /^(@{1,2})(!)?(\w+)/;
/** @hidden */
const MUSTACHE_REGEX = /{{2}/;
/** @hidden */
const ESCAPE_REGEX = /^(\s*)@/;
/** @hidden */
const TRIM_TAG_REGEX = /^@/;
/**
* Tokenizer converts a bunch of text into an array of tokens. Later
* these tokens can be used to build the transformed text.
*
* Go through the README file to learn more about the syntax and
* the tokens output.
*/
class Tokenizer {

@@ -44,5 +22,2 @@ constructor(template, tagsDef, options) {

}
/**
* Parses the AST
*/
parse() {

@@ -54,37 +29,13 @@ const lines = this.template.split('\n');

}
/**
* Process entire text, but there is an open statement, so we will
* process it as a raw node
*/
if (this.blockStatement) {
this.consumeNode(this.getRawNode(`@${this.blockStatement.props.raw}`));
this.blockStatement = null;
this.consumeNode(this.getBlankLineNode());
throw Exceptions_1.unclosedParen({ line: this.blockStatement.startPosition, col: 0 }, this.options.filename);
}
/**
* Process entire text, but there is an open statement, so we will
* process it as a raw node
*/
if (this.mustacheStatement) {
const { raw } = this.mustacheStatement.props;
this.mustacheStatement = null;
this.consumeNode(this.getRawNode(raw));
this.consumeNode(this.getBlankLineNode());
throw Exceptions_1.unclosedCurlyBrace({ line: this.mustacheStatement.startPosition, col: 0 }, this.options.filename);
}
/**
* Throw exception when there are opened tags
*/
if (this.openedTags.length) {
const openedTag = this.openedTags[this.openedTags.length - 1];
throw new edge_error_1.EdgeError(`Unclosed tag ${openedTag.properties.name}`, 'E_UNCLOSED_TAG', {
line: openedTag.lineno,
col: 0,
filename: this.options.filename,
});
throw Exceptions_1.unclosedTag(openedTag.properties.name, { line: openedTag.lineno, col: 0 }, this.options.filename);
}
}
/**
* Returns the tag defination when line matches the regex
* of a tag.
*/
getTag(line) {

@@ -96,11 +47,5 @@ const match = TAG_REGEX.exec(line.trim());

const tagName = match[3];
/**
* Makes sure the tag exists in the tags defination
*/
if (!this.tagsDef[tagName]) {
return null;
}
/**
* Tag is escaped
*/
if (match[1] === '@@') {

@@ -117,5 +62,2 @@ return {

}
/**
* Returns the node for a tag
*/
getTagNode(properties, lineno) {

@@ -129,5 +71,2 @@ return {

}
/**
* Returns the node for a raw string
*/
getRawNode(value) {

@@ -140,5 +79,2 @@ return {

}
/**
* Returns the node for a newline
*/
getBlankLineNode() {

@@ -150,5 +86,2 @@ return {

}
/**
* Returns the mustache node
*/
getMustacheNode(properties, lineno) {

@@ -165,6 +98,2 @@ return {

}
/**
* Returns a boolean, when line content is a closing
* tag
*/
isClosingTag(line) {

@@ -177,20 +106,8 @@ if (!this.openedTags.length) {

}
/**
* Returns a boolean, telling if a given statement is seeking
* for more content or not
*/
isSeeking(statement) {
return !!(statement && statement.seeking);
}
/**
* Returns a boolean, telling if a given statement has ended or
* not.
*/
isSeeked(statement) {
return statement && !statement.seeking;
}
/**
* Here we add the node to tokens or as children for
* the recentOpenedTag (if one exists).
*/
consumeNode(tag) {

@@ -203,7 +120,2 @@ if (this.openedTags.length) {

}
/**
* Feeds the text to the currently opened block statement.
* Make sure that `seeking` is true on the block
* statement, before calling this method.
*/
feedTextToBlockStatement(text) {

@@ -215,6 +127,2 @@ this.blockStatement.feed(text);

const { props, tagDef, startPosition } = this.blockStatement;
/**
* If tag is a block level, then we added it to the openedTags
* array, otherwise we add it to the tokens.
*/
if (tagDef.block && (!tagDef.selfclosed || !props.selfclosed)) {

@@ -228,6 +136,2 @@ this.openedTags.push(this.getTagNode(props, startPosition));

}
/**
* Feeds text to the currently opened mustache statement. Make sure
* to check `seeking` is true, before calling this method.
*/
feedTextToMustacheStatement(text) {

@@ -239,5 +143,2 @@ this.mustacheStatement.feed(text);

const { props, startPosition } = this.mustacheStatement;
/**
* Process text left when exists
*/
if (props.textLeft) {

@@ -248,11 +149,4 @@ const textNode = this.getRawNode(props.textLeft);

}
/**
* Then consume the actual mustache expression
*/
this.consumeNode(this.getMustacheNode(props, startPosition));
this.mustacheStatement = null;
/**
* Finally, there is no content to the right, then process
* it, otherwise add a new line token
*/
if (props.textRight) {

@@ -265,10 +159,3 @@ this.processText(props.textRight);

}
/**
* Process a piece of text, by finding if text has reserved keywords,
* otherwise process it as a raw node.
*/
processText(text) {
/**
* Block statement is seeking for more content
*/
if (this.isSeeking(this.blockStatement)) {

@@ -278,5 +165,2 @@ this.feedTextToBlockStatement(text);

}
/**
* Mustache statement is seeking for more content
*/
if (this.isSeeking(this.mustacheStatement)) {

@@ -287,5 +171,2 @@ this.feedTextToMustacheStatement(text);

const tag = this.getTag(text);
/**
* Text is a escaped tag
*/
if (tag && tag.escaped) {

@@ -296,13 +177,7 @@ this.consumeNode(this.getRawNode(text.replace(ESCAPE_REGEX, '$1')));

}
/**
* Text is a tag
*/
if (tag) {
this.blockStatement = new TagStatement_1.TagStatement(this.line, tag);
this.blockStatement = new TagStatement_1.TagStatement(this.line, tag, this.options.filename);
this.feedTextToBlockStatement(text.trim().replace(TRIM_TAG_REGEX, ''));
return;
}
/**
* Text is a closing block tag
*/
if (this.isClosingTag(text)) {

@@ -312,5 +187,2 @@ this.consumeNode(this.openedTags.pop());

}
/**
* Text contains mustache expressions
*/
if (MUSTACHE_REGEX.test(text)) {

@@ -321,5 +193,2 @@ this.mustacheStatement = new MustacheStatement_1.MustacheStatement(this.line);

}
/**
* A plain raw node
*/
this.consumeNode(this.getRawNode(text));

@@ -326,0 +195,0 @@ this.consumeNode(this.getBlankLineNode());

15

CHANGELOG.md

@@ -0,1 +1,16 @@

<a name="1.0.8"></a>
## [1.0.8](https://github.com/poppinss/edge-lexer/compare/v1.0.7...v1.0.8) (2018-11-03)
### Bug Fixes
* **tokenizer:** raise error when invalid block or mustache statements ([86718e9](https://github.com/poppinss/edge-lexer/commit/86718e9))
### Features
* **errors:** make errors consistent ([663a618](https://github.com/poppinss/edge-lexer/commit/663a618))
<a name="1.0.7"></a>

@@ -2,0 +17,0 @@ ## [1.0.7](https://github.com/poppinss/edge-lexer/compare/v1.0.6...v1.0.7) (2018-07-10)

34

package.json
{
"name": "edge-lexer",
"version": "1.0.7",
"version": "1.0.8",
"description": "Parses raw markup files to converts them to Tokens",

@@ -10,4 +10,3 @@ "main": "build/src/Tokenizer/index.js",

"pretest": "npm run lint",
"test": "nyc japa",
"posttest": "npm run coverage",
"test": "nyc node japaFile.js",
"build": "npm run compile",

@@ -35,5 +34,5 @@ "commit": "git-cz",

"devDependencies": {
"@adonisjs/mrm-preset": "^1.0.7",
"@types/node": "^10.5.2",
"commitizen": "^2.10.1",
"@adonisjs/mrm-preset": "^1.0.14",
"@types/node": "^10.12.2",
"commitizen": "^3.0.4",
"coveralls": "^3.0.2",

@@ -43,11 +42,12 @@ "cz-conventional-changelog": "^2.1.0",

"del-cli": "^1.1.0",
"japa": "^1.0.6",
"japa": "^2.0.6",
"japa-cli": "^1.0.1",
"mrm": "^1.2.0",
"nyc": "^12.0.2",
"pkg-ok": "^2.2.0",
"ts-node": "^7.0.0",
"tslint": "^5.10.0",
"tslint-eslint-rules": "^5.3.1",
"typescript": "^2.9.2"
"mrm": "^1.2.1",
"nyc": "^13.1.0",
"pkg-ok": "^2.3.1",
"ts-node": "^7.0.1",
"tslint": "^5.11.0",
"tslint-eslint-rules": "^5.4.0",
"typescript": "^3.1.6",
"yorkie": "^2.0.0"
},

@@ -68,6 +68,8 @@ "dependencies": {

"exclude": [
"test",
"japaFile.js"
"test"
]
},
"gitHooks": {
"commit-msg": "node ./node_modules/@adonisjs/mrm-preset/validateCommit/conventional/validate.js"
}
}

312

README.md
# Edge lexer
> Generating high level tokens from Edge whitelisted markup
[![travis-image]][travis-url]

@@ -9,10 +11,24 @@ [![appveyor-image]][appveyor-url]

Edge lexer detects tags from any markup language and converts them into tokens. Later these tokens can be used with a Javascript parser like `esprima` or `babylon` to complete a logical template engine ( this is what [edge-parser](https://github.com/poppinss/edge-parser) does).
Edge lexer produces a list of `tokens` by scanning for [Edge whitelisted syntax](https://github.com/edge-js/syntax).
This guide is an outline of the lexer.
This module is a blend of a `lexer` and an `AST generator`, since Edge doesn't need a pure [lexer](https://en.wikipedia.org/wiki/Lexical_analysis) that scans for each character. Edge markup is written within other markup languages like **HTML** or **Markdown** and walking over each character is waste of resources.
Instead, this module starts with some REGEX patterns to detect the [Edge whitelisted syntax](https://github.com/edge-js/syntax) and then starts the lexical analysis within the detected markup.
---
## Performance
Following measures are taken to keep the analysis performant
1. Only analyse markup that is detected as Edge whitelisted syntax.
2. Only analyse `tags`, that are passed to the tokenizer. Which means even if the syntax for tags is whitelisted, the tokeniser will analyse them if they are used by your app.
3. Do not analyse Javascript expression and leave that for [edge-parser](https://github.com/edge-js/parser).
---
## Usage
```
```js
import { Tokenizer } from 'edge-lexer'

@@ -29,13 +45,13 @@

const tokenizer = new Tokenizer(template, tags)
// Filename is required to add it to error messages
const options = {
filename: 'welcome.edge'
}
const tokenizer = new Tokenizer(template, tags, options)
tokenizer.parse()
console.log(tokenizer.tokens)
```
## Note
The code used in examples is only subject to work, when using Edge template engine. The lexer job is to tokenize the whitelisted syntax.
The real functionality is added by the template engine by using the tokens created from this package
---

@@ -46,4 +62,4 @@

2. Whitespaces and newlines are retained.
3. Works with any markup or plain text files.
4. Syntax is close to Javascript.
3. Detects for unclosed tags.
4. Detects for unwrapped expressions and raises appropriate errors.

@@ -56,6 +72,9 @@ ## Terms used

| Tag | block | Tags are used to define logical blocks in the template engine. For example `if tag` or `include tag`. |
| Mustache | mustache | Javascript expression wrapped around curly braces. `{{ }}` |
| Mustache | mustache | Javascript expression wrapped in curly braces. `{{ }}` |
| Raw | raw | A raw string, which has no meaning for the template engine |
| NewLine | newline | Newline |
| Comment | comment | Edge specific comment block. This will be ripped off in the output.
---
## Nodes

@@ -85,2 +104,12 @@ Following is the list of Nodes returned by the tokenizer.

#### Comment Node
```js
{
type: 'comment',
lineno: number,
value: string
}
```
#### Mustache Node

@@ -105,3 +134,2 @@

`Block Node` is the only node, which contains recursive child nodes.

@@ -114,9 +142,11 @@ | Key | Value | Description |

| value | string | If node is a raw node, then value is the string in the source file
| children | array | Array of recursive nodes.
| children | array | Array of recursive nodes. Only exists, when `type === 'block'`.
---
## Properties
The properties `Prop` is used to define meta data for a given Node. Nodes like `raw` and `newline`, doesn't need any metadata.
The properties `Prop` is used to define meta data for a given Node. Nodes like `raw`, `comment` and `newline`, doesn't need any metadata.
#### BlockProp
The block prop is by the `Tag` node. The only difference from the regular `Prop` is the additional of `selfclosed` attribute.
The block prop is used by the `Block` node. The only difference from the regular `Prop` is the addition of `selfclosed` attribute.

@@ -149,2 +179,7 @@ ```js

---
## Mustache expressions
For mustache nodes props, the `name` is the type of mustache expressions. The lexer supports 4 mustache expressions.

@@ -180,4 +215,20 @@

---
## Errors
Errors raised by the `lexer` are always an instance of [edge-error](https://github.com/edge-js/error) and will contain following properties.
```js
error.message
error.line
error.col
error.filename
error.code
```
---
## Example
Before reading more about the syntax and their output, let's check the following example.

@@ -232,210 +283,5 @@ ```html

## Supported Syntax
To make Edge an enjoyable template engine, we have kept the syntax very close the original Javascript syntax.
The following expressions are allowed.
## Tags (block)
1. Every block level tag starts with `@` symbol followed by the tagName.
2. It is important to close a block level using `@end<tagName>`
3. `@` and `tagName` cannot have spaces between them.
**VALID**
```
@if(username)
@endif
```
**VALID**
```
@if(
username
)
@endif
```
**VALID**
```
@if(
(
2 + 2
)
===
4
)
```
**VALID**
```
@if
(
username
)
```
**INVALID**
The opening of the tag must be in it's own line and so do the closing one
```
@if(username) Hello @endif
```
**INVALID**
```
@if(
username
) <p> Hello </p>
@endif
```
## Tags (inline)
The inline tags doesn't contain any childs and hence requires no `@end` statement.
**VALID**
```
@include('header')
```
**VALID**
```
@include(
'header'
)
```
**VALID**
```
@include
(
'header'
)
```
## Tags (block-self closed)
At times block level tags can work fine without any body inside them. To keep the syntax concise, you can **self-close** a block level tag.
**NORMAL**
```
@component('title')
<h1> Hello world </h1>
@endcomponent
```
**SELF CLOSED**
```
@!component('title', title = '<h1> Hello world </h1>')
```
## Mustache
The mustache braces `{{` are used to define inline Javascript expressions. The lexer allows
1. Multiline expressions.
2. A valid Javascript expression, that yields to a value.
3. The HTML output from mustache will be escaped, so make sure to use `{{{ '<p>' Hello world </p> }}}` for rendering HTML nodes.
**VALID**
```
{{ username }}
```
**VALID**
```
{{
username
}}
```
**VALID**
```
Your friends are {{
users.map((user) => {
return user.username
}).join(',')
}}
```
**VALID**
```
{{{ '<p>' Hello world </p> }}}
```
**INVALID**
The starting curly brace, must be in one line.
```
{
{
username
}
}
```
## Escaping
The backslash `\` has a special meaning in Javascript, that's why we make use of `@` to escape Edge statements.
> There is no need to escape the `@end` statements, since they are tightly mapped with the start statements. So if a start statement doesn't exists, the end statement will be considered as a raw node.
**ESCAPED**
```
@@if(username)
@endif
```
yields
```json
[
{
"type": "raw",
"value": "@if(username)",
"lineno": 1
},
{
"type": "newline",
"lineno": 1
},
{
"type": "raw",
"value": "@endif",
"lineno": 2
},
{
"type": "newline",
"lineno": 2
}
]
```
In the same fashion, the mustache braces can be escaped using `@`.
**ESCAPED**
```
Hello @{{ username }}
```
## Change log
The change log can be found in the [CHANGELOG.md](https://github.com/poppinss/edge-lexer/CHANGELOG.md) file.
The change log can be found in the [CHANGELOG.md](CHANGELOG.md) file.

@@ -451,16 +297,16 @@ ## Contributing

## Authors & License
[thetutlage](https://github.com/thetutlage) and [contributors](https://github.com/poppinss/edge-lexer/graphs/contributors).
[thetutlage](https://github.com/thetutlage) and [contributors](https://github.com/edge-js/lexer/graphs/contributors).
MIT License, see the included [MIT](LICENSE.md) file.
[travis-image]: https://img.shields.io/travis/poppinss/edge-lexer/master.svg?style=flat-square&logo=travis
[travis-url]: https://travis-ci.org/poppinss/edge-lexer "travis"
[travis-image]: https://img.shields.io/travis/edge-js/lexer/master.svg?style=flat-square&logo=travis
[travis-url]: https://travis-ci.org/edge-js/lexer "travis"
[appveyor-image]: https://img.shields.io/appveyor/ci/thetutlage/edge-lexer/master.svg?style=flat-square&logo=appveyor
[appveyor-url]: https://ci.appveyor.com/project/thetutlage/edge-lexer "appveyor"
[appveyor-image]: https://img.shields.io/appveyor/ci/thetutlage/lexer/master.svg?style=flat-square&logo=appveyor
[appveyor-url]: https://ci.appveyor.com/project/thetutlage/lexer "appveyor"
[coveralls-image]: https://img.shields.io/coveralls/poppinss/edge-lexer/master.svg?style=flat-square
[coveralls-url]: https://coveralls.io/github/poppinss/edge-lexer "coveralls"
[coveralls-image]: https://img.shields.io/coveralls/edge-js/lexer/master.svg?style=flat-square
[coveralls-url]: https://coveralls.io/github/edge-js/lexer "coveralls"
[npm-image]: https://img.shields.io/npm/v/edge-lexer.svg?style=flat-square&logo=npm
[npm-url]: https://npmjs.org/package/edge-lexer "npm"
[npm-image]: https://img.shields.io/npm/v/lexer.svg?style=flat-square&logo=npm
[npm-url]: https://npmjs.org/package/lexer "npm"
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc