🎩 You're Invited:Meet the Socket team at Black Hat in Las Vegas, August 3-6.RSVP →
Sign In

leac

Package Overview
Dependencies
Maintainers
1
Versions
5
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

leac - npm Package Compare versions

Comparing version
0.6.0
 to
0.7.0-preview.1
+172
lib/leac.d.cts
/** Options for a {@link Lexer} (not many so far). */
type Options = {
/**
* Enable line and column numbers computation.
*/
lineNumbers?: boolean;
};
/** Result returned by a {@link Lexer}. */
type LexerResult = {
/** Array of tokens. */
tokens: Token[];
/** Final offset. */
offset: number;
/**
* True if whole input string was processed.
*
* Check this to see whether some input left untokenized.
*/
complete: boolean;
};
/**
* Lexer function.
*
* @param str - A string to tokenize.
* @param offset - Initial offset. Used when composing lexers.
*/
type Lexer = (str: string, offset?: number) => LexerResult;
/** Token object, the result of matching an individual lexing rule. */
type Token = {
/** Name of the {@link Lexer} containing the rule produced this token. */
state: string;
/** Name of the {@link Rule} produced this token. */
name: string;
/** Text matched by the rule. _(Except for {@link ReplacementRule} that can alter this.)_ */
text: string;
/** Start index of the match in the input string. */
offset: number;
/**
* The length of the matched substring.
*
* _(Might be different from the text length in case of {@link ReplacementRule}.)_
*/
len: number;
/**
* Line number in the source string (1-based).
*
* _(Always zero if line numbers not enabled in {@link Options}.)_
*/
line: number;
/**
* Column number within the line in the source string (1-based).
*
* _(Always zero if line numbers not enabled in {@link Options}.)_
*/
column: number;
};
/**
* Lexing rule.
*
* Base rule looks for exact match by it's name.
*
* If the name and the lookup string have to be different
* then specify `str` property as defined in {@link StringRule}.
*/
interface Rule {
/** The name of the rule, also the name of tokens produced by this rule. */
name: string;
/**
* Matched token won't be added to the output array if this set to `true`.
*
* (_Think twice before using this._)
* */
discard?: boolean | undefined;
/**
* Switch to another lexer function after this match,
* concatenate it's results and continue from where it stopped.
*/
push?: Lexer | undefined;
/**
* Stop after this match and return.
*
* If there is a parent lexer - it will continue from this point.
*/
pop?: boolean | undefined;
}
/**
* String rule - looks for exact string match that
* can be different from the name of the rule.
*/
interface StringRule extends Rule {
/**
* Specify the exact string to match
* if it is different from the name of the rule.
*/
str: string;
}
/**
* Regex rule - looks for a regular expression match.
*/
interface RegexRule extends Rule {
/**
* Regular expression to match.
*
* - Can't have the global flag.
*
* - All regular expressions are used as sticky,
* you don't have to specify the sticky flag.
*
* - Empty matches are considered as non-matches -
* no token will be emitted in that case.
*/
regex: RegExp;
}
/**
* Replacement rule - {@link RegexRule} that also applies a replacement in the output {@link Token} text.
*/
interface ReplacementRule extends RegexRule {
/**
* Replacement string can include patterns,
* the same as [String.prototype.replace()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_a_parameter).
*
* This will only affect the text property of the output {@link Token}, not it's offset or length.
*
* Note: the regex has to be able to match the matched substring when taken out of context
* in order for replace to work - boundary/neighborhood conditions may prevent this.
*/
replace: string;
}
/**
* Non-empty array of {@link Rule}s.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name. For example, you can have
* separate rules for various keywords and use the same name "keyword".
*/
type Rules = [
(Rule | StringRule | RegexRule | ReplacementRule),
...(Rule | StringRule | RegexRule | ReplacementRule)[]
];
/**
* Create a lexer function.
*
* @param rules - Non-empty array of lexing rules.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name - you can have separate rules
* for keywords and use the same name "keyword" for example.
*
* @param state - The name of this lexer. Use when composing lexers.
* Empty string by default.
*
* @param options - Lexer options object.
*/
declare function createLexer(rules: Rules, state?: string, options?: Options): Lexer;
/**
* Create a lexer function.
*
* @param rules - Non-empty array of lexing rules.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name - you can have separate rules
* for keywords and use the same name "keyword" for example.
*
* @param options - Lexer options object.
*/
declare function createLexer(rules: Rules, options?: Options): Lexer;
export { type Lexer, type LexerResult, type Options, type RegexRule, type ReplacementRule, type Rule, type Rules, type StringRule, type Token, createLexer };
+172
lib/leac.d.mts
/** Options for a {@link Lexer} (not many so far). */
type Options = {
/**
* Enable line and column numbers computation.
*/
lineNumbers?: boolean;
};
/** Result returned by a {@link Lexer}. */
type LexerResult = {
/** Array of tokens. */
tokens: Token[];
/** Final offset. */
offset: number;
/**
* True if whole input string was processed.
*
* Check this to see whether some input left untokenized.
*/
complete: boolean;
};
/**
* Lexer function.
*
* @param str - A string to tokenize.
* @param offset - Initial offset. Used when composing lexers.
*/
type Lexer = (str: string, offset?: number) => LexerResult;
/** Token object, the result of matching an individual lexing rule. */
type Token = {
/** Name of the {@link Lexer} containing the rule produced this token. */
state: string;
/** Name of the {@link Rule} produced this token. */
name: string;
/** Text matched by the rule. _(Except for {@link ReplacementRule} that can alter this.)_ */
text: string;
/** Start index of the match in the input string. */
offset: number;
/**
* The length of the matched substring.
*
* _(Might be different from the text length in case of {@link ReplacementRule}.)_
*/
len: number;
/**
* Line number in the source string (1-based).
*
* _(Always zero if line numbers not enabled in {@link Options}.)_
*/
line: number;
/**
* Column number within the line in the source string (1-based).
*
* _(Always zero if line numbers not enabled in {@link Options}.)_
*/
column: number;
};
/**
* Lexing rule.
*
* Base rule looks for exact match by it's name.
*
* If the name and the lookup string have to be different
* then specify `str` property as defined in {@link StringRule}.
*/
interface Rule {
/** The name of the rule, also the name of tokens produced by this rule. */
name: string;
/**
* Matched token won't be added to the output array if this set to `true`.
*
* (_Think twice before using this._)
* */
discard?: boolean | undefined;
/**
* Switch to another lexer function after this match,
* concatenate it's results and continue from where it stopped.
*/
push?: Lexer | undefined;
/**
* Stop after this match and return.
*
* If there is a parent lexer - it will continue from this point.
*/
pop?: boolean | undefined;
}
/**
* String rule - looks for exact string match that
* can be different from the name of the rule.
*/
interface StringRule extends Rule {
/**
* Specify the exact string to match
* if it is different from the name of the rule.
*/
str: string;
}
/**
* Regex rule - looks for a regular expression match.
*/
interface RegexRule extends Rule {
/**
* Regular expression to match.
*
* - Can't have the global flag.
*
* - All regular expressions are used as sticky,
* you don't have to specify the sticky flag.
*
* - Empty matches are considered as non-matches -
* no token will be emitted in that case.
*/
regex: RegExp;
}
/**
* Replacement rule - {@link RegexRule} that also applies a replacement in the output {@link Token} text.
*/
interface ReplacementRule extends RegexRule {
/**
* Replacement string can include patterns,
* the same as [String.prototype.replace()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_a_parameter).
*
* This will only affect the text property of the output {@link Token}, not it's offset or length.
*
* Note: the regex has to be able to match the matched substring when taken out of context
* in order for replace to work - boundary/neighborhood conditions may prevent this.
*/
replace: string;
}
/**
* Non-empty array of {@link Rule}s.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name. For example, you can have
* separate rules for various keywords and use the same name "keyword".
*/
type Rules = [
(Rule | StringRule | RegexRule | ReplacementRule),
...(Rule | StringRule | RegexRule | ReplacementRule)[]
];
/**
* Create a lexer function.
*
* @param rules - Non-empty array of lexing rules.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name - you can have separate rules
* for keywords and use the same name "keyword" for example.
*
* @param state - The name of this lexer. Use when composing lexers.
* Empty string by default.
*
* @param options - Lexer options object.
*/
declare function createLexer(rules: Rules, state?: string, options?: Options): Lexer;
/**
* Create a lexer function.
*
* @param rules - Non-empty array of lexing rules.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name - you can have separate rules
* for keywords and use the same name "keyword" for example.
*
* @param options - Lexer options object.
*/
declare function createLexer(rules: Rules, options?: Options): Lexer;
export { type Lexer, type LexerResult, type Options, type RegexRule, type ReplacementRule, type Rule, type Rules, type StringRule, type Token, createLexer };
+133
-1
lib/leac.cjs

@@ -1,1 +0,133 @@

"use strict";Object.defineProperty(exports,"__esModule",{value:!0});const e=/\n/g;function t(t){const o=[...t.matchAll(e)].map((e=>e.index||0));o.unshift(-1);const s=n(o,0,o.length);return e=>r(s,e)}function n(e,t,r){if(r-t==1)return{offset:e[t],index:t+1};const o=Math.ceil((t+r)/2),s=n(e,t,o),l=n(e,o,r);return{offset:s.offset,low:s,high:l}}function r(e,t){return function(e){return Object.prototype.hasOwnProperty.call(e,"index")}(e)?{line:e.index,column:t-e.offset}:r(e.high.offset<t?e.high:e.low,t)}function o(e,t){return{...e,regex:s(e,t)}}function s(e,t){if(0===e.name.length)throw new Error(`Rule #${t} has empty name, which is not allowed.`);if(function(e){return Object.prototype.hasOwnProperty.call(e,"regex")}(e))return function(e){if(e.global)throw new Error(`Regular expression /${e.source}/${e.flags} contains the global flag, which is not allowed.`);return e.sticky?e:new RegExp(e.source,e.flags+"y")}(e.regex);if(function(e){return Object.prototype.hasOwnProperty.call(e,"str")}(e)){if(0===e.str.length)throw new Error(`Rule #${t} ("${e.name}") has empty "str" property, which is not allowed.`);return new RegExp(l(e.str),"y")}return new RegExp(l(e.name),"y")}function l(e){return e.replace(/[-[\]{}()*+!<=:?./\\^$|#\s,]/g,"\\$&")}exports.createLexer=function(e,n="",r={}){const s="string"!=typeof n?n:r,l="string"==typeof n?n:"",c=e.map(o),i=!!s.lineNumbers;return function(e,n=0){const r=i?t(e):()=>({line:0,column:0});let o=n;const s=[];e:for(;o<e.length;){let t=!1;for(const n of c){n.regex.lastIndex=o;const c=n.regex.exec(e);if(c&&c[0].length>0){if(!n.discard){const e=r(o),t="string"==typeof n.replace?c[0].replace(new RegExp(n.regex.source,n.regex.flags),n.replace):c[0];s.push({state:l,name:n.name,text:t,offset:o,len:c[0].length,line:e.line,column:e.column})}if(o=n.regex.lastIndex,t=!0,n.push){const t=n.push(e,o);s.push(...t.tokens),o=t.offset}if(n.pop)break e;break}}if(!t)break}return{tokens:s,offset:o,complete:e.length<=o}}};
'use strict';
const linebreaksRe = /\n/g;
function createPositionQuery(str) {
const offsets = [...str.matchAll(linebreaksRe)].map(m => m.index || 0);
offsets.unshift(-1);
let lineIndex = 1;
return (offset) => {
while (lineIndex > 1 && offset < offsets[lineIndex - 1]) {
lineIndex--;
}
while (lineIndex < offsets.length && offset > offsets[lineIndex]) {
lineIndex++;
}
return { line: lineIndex, column: offset - offsets[lineIndex - 1] };
};
}
function toUnifiedRule(r, i) {
return {
name: r.name,
discard: r.discard,
push: r.push,
pop: r.pop,
regex: toRegExp(r, i),
replacer: isReplacementRule(r)
? toReplacer(r.regex, r.replace)
: undefined,
};
}
function isStringRule(r) {
return Object.prototype.hasOwnProperty.call(r, 'str');
}
function isRegexRule(r) {
return Object.prototype.hasOwnProperty.call(r, 'regex');
}
function isReplacementRule(r) {
return Object.prototype.hasOwnProperty.call(r, 'replace');
}
function toReplacer(re, replace) {
const replaceSearch = toNonSticky(re);
return (match) => match.replace(replaceSearch, replace);
}
function toRegExp(r, i) {
if (r.name.length === 0) {
throw new Error(`Rule #${i} has empty name, which is not allowed.`);
}
if (isRegexRule(r)) {
return toSticky(r.regex);
}
if (isStringRule(r)) {
if (r.str.length === 0) {
throw new Error(`Rule #${i} ("${r.name}") has empty "str" property, which is not allowed.`);
}
return new RegExp(escapeRegExp(r.str), 'y');
}
return new RegExp(escapeRegExp(r.name), 'y');
}
function escapeRegExp(str) {
return str.replace(/[-[\]{}()*+!<=:?./\\^$|#\s,]/g, '\\$&');
}
function toSticky(re) {
if (re.global) {
throw new Error(`Regular expression /${re.source}/${re.flags} contains the global flag, which is not allowed.`);
}
return (re.sticky)
? re
: new RegExp(re.source, re.flags + 'y');
}
function toNonSticky(re) {
return (re.sticky)
? new RegExp(re.source, re.flags.replace('y', ''))
: re;
}
function createLexer(rules, state = '', options = {}) {
const options1 = (typeof state !== 'string') ? state : options;
const state1 = (typeof state === 'string') ? state : '';
const unifiedRules = rules.map(toUnifiedRule);
const isLineNumbers = !!options1.lineNumbers;
return function (str, offset = 0) {
const positionQuery = (isLineNumbers)
? createPositionQuery(str)
: undefined;
let position = { line: 0, column: 0 };
let currentIndex = offset;
const tokens = [];
loopStr: while (currentIndex < str.length) {
let anyMatch = false;
for (const rule of unifiedRules) {
rule.regex.lastIndex = currentIndex;
const match = rule.regex.exec(str);
if (match && match[0].length > 0) {
if (!rule.discard) {
if (positionQuery) {
position = positionQuery(currentIndex);
}
tokens.push({
state: state1,
name: rule.name,
text: (rule.replacer) ? rule.replacer(match[0]) : match[0],
offset: currentIndex,
len: match[0].length,
line: position.line,
column: position.column,
});
}
currentIndex = rule.regex.lastIndex;
anyMatch = true;
if (rule.push) {
const r = rule.push(str, currentIndex);
tokens.push(...r.tokens);
currentIndex = r.offset;
}
if (rule.pop) {
break loopStr;
}
break;
}
}
if (!anyMatch) {
break;
}
}
return {
tokens: tokens,
offset: currentIndex,
complete: str.length <= currentIndex,
};
};
}
exports.createLexer = createLexer;
+131
-1
lib/leac.mjs

@@ -1,1 +0,131 @@

const e=/\n/g;function n(n){const o=[...n.matchAll(e)].map((e=>e.index||0));o.unshift(-1);const s=t(o,0,o.length);return e=>r(s,e)}function t(e,n,r){if(r-n==1)return{offset:e[n],index:n+1};const o=Math.ceil((n+r)/2),s=t(e,n,o),l=t(e,o,r);return{offset:s.offset,low:s,high:l}}function r(e,n){return function(e){return Object.prototype.hasOwnProperty.call(e,"index")}(e)?{line:e.index,column:n-e.offset}:r(e.high.offset<n?e.high:e.low,n)}function o(e,t="",r={}){const o="string"!=typeof t?t:r,l="string"==typeof t?t:"",c=e.map(s),f=!!o.lineNumbers;return function(e,t=0){const r=f?n(e):()=>({line:0,column:0});let o=t;const s=[];e:for(;o<e.length;){let n=!1;for(const t of c){t.regex.lastIndex=o;const c=t.regex.exec(e);if(c&&c[0].length>0){if(!t.discard){const e=r(o),n="string"==typeof t.replace?c[0].replace(new RegExp(t.regex.source,t.regex.flags),t.replace):c[0];s.push({state:l,name:t.name,text:n,offset:o,len:c[0].length,line:e.line,column:e.column})}if(o=t.regex.lastIndex,n=!0,t.push){const n=t.push(e,o);s.push(...n.tokens),o=n.offset}if(t.pop)break e;break}}if(!n)break}return{tokens:s,offset:o,complete:e.length<=o}}}function s(e,n){return{...e,regex:l(e,n)}}function l(e,n){if(0===e.name.length)throw new Error(`Rule #${n} has empty name, which is not allowed.`);if(function(e){return Object.prototype.hasOwnProperty.call(e,"regex")}(e))return function(e){if(e.global)throw new Error(`Regular expression /${e.source}/${e.flags} contains the global flag, which is not allowed.`);return e.sticky?e:new RegExp(e.source,e.flags+"y")}(e.regex);if(function(e){return Object.prototype.hasOwnProperty.call(e,"str")}(e)){if(0===e.str.length)throw new Error(`Rule #${n} ("${e.name}") has empty "str" property, which is not allowed.`);return new RegExp(c(e.str),"y")}return new RegExp(c(e.name),"y")}function c(e){return e.replace(/[-[\]{}()*+!<=:?./\\^$|#\s,]/g,"\\$&")}export{o as createLexer};
const linebreaksRe = /\n/g;
function createPositionQuery(str) {
const offsets = [...str.matchAll(linebreaksRe)].map(m => m.index || 0);
offsets.unshift(-1);
let lineIndex = 1;
return (offset) => {
while (lineIndex > 1 && offset < offsets[lineIndex - 1]) {
lineIndex--;
}
while (lineIndex < offsets.length && offset > offsets[lineIndex]) {
lineIndex++;
}
return { line: lineIndex, column: offset - offsets[lineIndex - 1] };
};
}
function toUnifiedRule(r, i) {
return {
name: r.name,
discard: r.discard,
push: r.push,
pop: r.pop,
regex: toRegExp(r, i),
replacer: isReplacementRule(r)
? toReplacer(r.regex, r.replace)
: undefined,
};
}
function isStringRule(r) {
return Object.prototype.hasOwnProperty.call(r, 'str');
}
function isRegexRule(r) {
return Object.prototype.hasOwnProperty.call(r, 'regex');
}
function isReplacementRule(r) {
return Object.prototype.hasOwnProperty.call(r, 'replace');
}
function toReplacer(re, replace) {
const replaceSearch = toNonSticky(re);
return (match) => match.replace(replaceSearch, replace);
}
function toRegExp(r, i) {
if (r.name.length === 0) {
throw new Error(`Rule #${i} has empty name, which is not allowed.`);
}
if (isRegexRule(r)) {
return toSticky(r.regex);
}
if (isStringRule(r)) {
if (r.str.length === 0) {
throw new Error(`Rule #${i} ("${r.name}") has empty "str" property, which is not allowed.`);
}
return new RegExp(escapeRegExp(r.str), 'y');
}
return new RegExp(escapeRegExp(r.name), 'y');
}
function escapeRegExp(str) {
return str.replace(/[-[\]{}()*+!<=:?./\\^$|#\s,]/g, '\\$&');
}
function toSticky(re) {
if (re.global) {
throw new Error(`Regular expression /${re.source}/${re.flags} contains the global flag, which is not allowed.`);
}
return (re.sticky)
? re
: new RegExp(re.source, re.flags + 'y');
}
function toNonSticky(re) {
return (re.sticky)
? new RegExp(re.source, re.flags.replace('y', ''))
: re;
}
function createLexer(rules, state = '', options = {}) {
const options1 = (typeof state !== 'string') ? state : options;
const state1 = (typeof state === 'string') ? state : '';
const unifiedRules = rules.map(toUnifiedRule);
const isLineNumbers = !!options1.lineNumbers;
return function (str, offset = 0) {
const positionQuery = (isLineNumbers)
? createPositionQuery(str)
: undefined;
let position = { line: 0, column: 0 };
let currentIndex = offset;
const tokens = [];
loopStr: while (currentIndex < str.length) {
let anyMatch = false;
for (const rule of unifiedRules) {
rule.regex.lastIndex = currentIndex;
const match = rule.regex.exec(str);
if (match && match[0].length > 0) {
if (!rule.discard) {
if (positionQuery) {
position = positionQuery(currentIndex);
}
tokens.push({
state: state1,
name: rule.name,
text: (rule.replacer) ? rule.replacer(match[0]) : match[0],
offset: currentIndex,
len: match[0].length,
line: position.line,
column: position.column,
});
}
currentIndex = rule.regex.lastIndex;
anyMatch = true;
if (rule.push) {
const r = rule.push(str, currentIndex);
tokens.push(...r.tokens);
currentIndex = r.offset;
}
if (rule.pop) {
break loopStr;
}
break;
}
}
if (!anyMatch) {
break;
}
}
return {
tokens: tokens,
offset: currentIndex,
complete: str.length <= currentIndex,
};
};
}
export { createLexer };
+1
-1
LICENSE
MIT License
Copyright (c) 2021-2022 KillyMXI <killy@mxii.eu.org>
Copyright (c) 2021-2025 KillyMXI <killy@mxii.eu.org>

@@ -5,0 +5,0 @@ Permission is hereby granted, free of charge, to any person obtaining a copy

+53
-38
package.json
{
"name": "leac",
"version": "0.6.0",
"version": "0.7.0-preview.1",
"description": "Lexer / tokenizer",

@@ -20,7 +20,16 @@ "keywords": [

"author": "KillyMXI",
"funding": "https://ko-fi.com/killymxi",
"funding": "https://github.com/sponsors/KillyMXI",
"license": "MIT",
"exports": {
"import": "./lib/leac.mjs",
"require": "./lib/leac.cjs"
".": {
"import": {
"types": "./lib/leac.d.mts",
"default": "./lib/leac.mjs"
},
"require": {
"types": "./lib/leac.d.cts",
"default": "./lib/leac.cjs"
}
},
"./package.json": "./package.json"
},

@@ -30,3 +39,3 @@ "type": "module",

"module": "./lib/leac.mjs",
"types": "./lib/leac.d.ts",
"types": "./lib/leac.d.cts",
"files": [

@@ -36,40 +45,47 @@ "lib"

"scripts": {
"benchmark:js": "node ./benchmarks/benchmark.js",
"benchmark:ts": "tsimp ./benchmarks/benchmark.ts",
"benchmark": "npm run benchmark:ts && npm run benchmark:js",
"build:docs": "typedoc",
"build:deno": "denoify",
"build:deno": "denoify && replace-in-file \"/\\.ts\\/index.ts/g\" \".ts\" deno/**/*.ts",
"build:rollup": "rollup -c",
"build:types": "tsc --declaration --emitDeclarationOnly && rimraf lib/!(leac).d.ts",
"build": "npm run clean && concurrently npm:build:*",
"checkAll": "npm run lint && npm test",
"build": "npm run clean && npm run build:rollup && npm run build:docs && npm run build:deno",
"check:all": "npm run check:tsc && npm run lint && npm test",
"check:tsc": "tsc",
"clean": "rimraf lib && rimraf docs && rimraf deno",
"example:calc": "npm run ts -- ./examples/calc.ts",
"example:json": "npm run ts -- ./examples/json.ts",
"example:calc": "tsimp ./examples/calc.ts",
"example:json": "tsimp ./examples/json.ts",
"lint:eslint": "eslint .",
"lint:md": "markdownlint-cli2",
"lint": "concurrently npm:lint:*",
"prepublishOnly": "npm run build && npm run checkAll",
"test": "ava",
"ts": "node --experimental-specifier-resolution=node --loader ts-node/esm"
"lint": "npm run lint:eslint && npm run lint:md",
"prepublishOnly": "npm run build && npm run check:all",
"test": "npm run test:ava-c8",
"test:ava": "ava",
"test:ava-c8": "c8 ava"
},
"dependencies": {},
"devDependencies": {
"@rollup/plugin-typescript": "^8.3.4",
"@tsconfig/node14": "^1.0.3",
"@types/node": "14.18.23",
"@typescript-eslint/eslint-plugin": "^5.33.1",
"@typescript-eslint/parser": "^5.33.1",
"ava": "^4.3.1",
"concurrently": "^7.3.0",
"denoify": "^1.0.0",
"eslint": "^8.22.0",
"eslint-plugin-jsonc": "^2.4.0",
"eslint-plugin-tsdoc": "^0.2.16",
"markdownlint-cli2": "^0.5.1",
"rimraf": "^3.0.2",
"rollup": "^2.78.0",
"rollup-plugin-terser": "^7.0.2",
"ts-node": "^10.9.1",
"tslib": "^2.4.0",
"typedoc": "~0.22.18",
"typedoc-plugin-markdown": "~3.12.1",
"typescript": "~4.7.4"
"@rollup/plugin-typescript": "^12.1.2",
"@stylistic/eslint-plugin": "^4.0.1",
"@tsconfig/node18": "^18.2.4",
"@types/node": "18.19.76",
"@typescript-eslint/eslint-plugin": "^8.25.0",
"ava": "^6.2.0",
"c8": "^10.1.3",
"denoify": "^1.6.16",
"eslint": "^9.21.0",
"eslint-plugin-jsonc": "^2.19.1",
"eslint-plugin-tsdoc": "^0.4.0",
"iso-bench": "^2.4.7",
"markdownlint-cli2": "^0.17.2",
"replace-in-file": "^8.3.0",
"rimraf": "^6.0.1",
"rollup": "^4.34.8",
"rollup-plugin-delete": "^3.0.0",
"rollup-plugin-dts": "^6.1.1",
"ts-blank-space": "^0.6.0",
"tsimp": "^2.0.12",
"typedoc": "~0.27.9",
"typedoc-plugin-markdown": "~4.4.2",
"typescript": "~5.7.3",
"typescript-eslint": "^8.25.0"
},

@@ -84,4 +100,3 @@ "ava": {

"nodeArguments": [
"--loader=ts-node/esm",
"--experimental-specifier-resolution=node"
"--import=ts-blank-space/register"
],

@@ -88,0 +103,0 @@ "verbose": true

+89
-10
README.md

@@ -5,4 +5,6 @@ # leac

![test status badge](https://github.com/mxxii/leac/workflows/test/badge.svg)
![test coverage badge](https://img.shields.io/nycrc/mxxii/leac?config=.c8rc.json)
[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](https://github.com/mxxii/leac/blob/main/LICENSE)
[![npm](https://img.shields.io/npm/v/leac?logo=npm)](https://www.npmjs.com/package/leac)
[![npm](https://img.shields.io/npm/dw/leac?color=informational&logo=npm)](https://www.npmjs.com/package/leac)
[![deno](https://img.shields.io/badge/deno.land%2Fx%2F-leac-informational?logo=deno)](https://deno.land/x/leac)

@@ -27,7 +29,14 @@

- **No streaming** - accepts a string at a time.
- **No streaming** - accepts a string at a time (more on this below).
- **Only text tokens, no arbitrary values**. It seems to be a good habit to have tokens that are *trivially* serializable back into a valid input string. Don't do the parser's job. There are a couple of convenience features such as the ability to discard matches or string replacements for regular expression rules but that has to be used mindfully (more on this below).
- Pairs well with [peberminta](https://github.com/mxxii/peberminta) – parser combinators toolkit.
## Changelog
Available here: [CHANGELOG.md](https://github.com/mxxii/leac/blob/main/CHANGELOG.md)
## Install

@@ -55,5 +64,2 @@

- [JSON](https://github.com/mxxii/leac/blob/main/examples/json.ts) ([output snapshot](https://github.com/mxxii/leac/blob/main/test/snapshots/examples.ts.md#json));
- [Calc](https://github.com/mxxii/leac/blob/main/examples/calc.ts) ([output snapshot](https://github.com/mxxii/leac/blob/main/test/snapshots/examples.ts.md#calc)).
```typescript

@@ -70,12 +76,38 @@ const lex = createLexer([

- [JSON](https://github.com/mxxii/leac/blob/main/examples/json.ts) ([output snapshot](https://github.com/mxxii/leac/blob/main/test/snapshots/examples.ts.md#json));
- [Calc](https://github.com/mxxii/leac/blob/main/examples/calc.ts) ([output snapshot](https://github.com/mxxii/leac/blob/main/test/snapshots/examples.ts.md#calc));
- [User-provided examples](https://github.com/mxxii/leac/issues?q=label%3Aexample);
- *feel free to post or PR interesting compact grammar examples.*
### Published packages using `leac`
- [parseley](https://github.com/mxxii/parseley) - CSS selectors parser
## API
- [docs/index.md](https://github.com/mxxii/leac/blob/main/docs/index.md)
- [v0.7.0-preview.1](https://github.com/mxxii/leac/blob/main/docs/index.md)
- [v0.6.0](https://github.com/mxxii/leac/blob/v0.6.0/docs/index.md)
## A word of caution
## Lexer design notes
It is often really tempting to rewrite token on the go. But it can be dangerous unless you are absolutely mindful of all edge cases.
### Tokens are atoms
Don't try to pack as much as possible into a token. If you can identify actually atomic, indivisible parts of the grammar - it will be easier to work with them in the parser.
If you're
- defining a very long regular expression in a token,
- defining multiple tokens that have a notable common part,
- breaking down a token into parts in the parser
then your tokens might be too big.
### On rewriting tokens
It is often really tempting to rewrite a token on the go. But it can be dangerous unless you are absolutely mindful of all edge cases.
For example, who needs to carry string quotes around, right? Parser will only need the string content...

@@ -105,3 +137,3 @@

- Match the whole string (content and quotes) with a single regular expression, use capture groups and [replace](https://github.com/mxxii/leac/blob/main/docs/interfaces/RegexRule.md#replace) property. This can produce a non-zero length token with empty text.
- Match the whole string (content and quotes) with a single regular expression, use capture groups and [replace](https://github.com/mxxii/leac/blob/main/docs/interfaces/ReplacementRule.md#properties) property. This can produce a non-zero length token with empty text.

@@ -111,9 +143,56 @@ Another note about quotes: If the grammar allows for different quotes and you're still willing to get rid of them early - think how you're going to unescape the string later. Make sure you carry the information about the exact string kind in the token name at least - you will need it later.

## Performance
Internal benchmarks are [available](https://github.com/mxxii/leac/tree/main/benchmarks).
```
> tsimp ./benchmarks/benchmark.ts
[ISOBENCH ENDED] IsoBench
[GROUP ENDED] effect of stacking
without stacking - 432 177 op/s. 50 samples in 5343 ms. 1.000x (WORST)
with stacking - 517 294 op/s. 50 samples in 5294 ms. 1.197x (BEST)
[GROUP ENDED] effect of replacement
without replacement - 652 823 op/s. 50 samples in 5366 ms. 1.993x (BEST)
with replacement - 327 504 op/s. 50 samples in 5311 ms. 1.000x (WORST)
[GROUP ENDED] effect of line numbers
without line numbers - 687 916 op/s. 50 samples in 5299 ms. 1.885x (BEST)
with line numbers - 364 998 op/s. 50 samples in 5343 ms. 1.000x (WORST)
[GROUP ENDED] effect of rule type
by names - 610 921 op/s. 50 samples in 5304 ms. 1.047x
by strings - 595 716 op/s. 50 samples in 5431 ms. 1.021x
by non-sticky regexes - 583 369 op/s. 50 samples in 5344 ms. 1.000x (WORST)
by sticky regexes - 601 227 op/s. 50 samples in 5321 ms. 1.031x
by combined regex - 719 475 op/s. 50 samples in 5348 ms. 1.233x (BEST)
[GROUP ENDED] constructing, not running
from name - 697 850 op/s. 50 samples in 5297 ms. 1.022x
from string - 683 054 op/s. 50 samples in 5276 ms. 1.000x (WORST)
from non-sticky regex - 1 390 579 op/s. 50 samples in 5252 ms. 2.036x
from sticky regex - 5 697 033 op/s. 50 samples in 5265 ms. 8.341x (BEST)
from non-sticky regex with replace - 1 311 587 op/s. 50 samples in 5350 ms. 1.920x
from sticky regex with replace - 926 344 op/s. 50 samples in 5261 ms. 1.356x
```
Trivial observations:
- stacking lexers reduces the number of rules in each lexer and speeds things up a bit;
- replacement and line numbers have runtime costs;
- all rule types perform the same on the same trivial matches (converted to the same representation internally);
- combining same name rules in one regular expression rule can give slight performance boost by the cost of readability;
- constructing from sticky regular expressions (with `y` flag) is the cheapest, as long as you don't need replacement feature;
- lexers aren't something you'd be constructing in volumes that make this matter, so this is a curiosity, it shouldn't be prioritized over code readability;
I have no benchmark comparing with other lexers/tokenizers. I'd be grateful to anyone who can provide a good benchmark project to compare different lexers on similar tasks.
Shoutout to <https://github.com/Llorx/iso-bench>.
## What about ...?
- performance - The code is very simple but I won't put any unverified assumptions here. I'd be grateful to anyone who can provide a good benchmark project to compare different lexers.
- stable release - Current release is well thought out and tested. I leave a chance that some changes might be needed based on feedback. Before version 1.0.0 this will be done without a deprecation cycle.
- streaming - I have no use case for it - majority of practical scenarios have reasonable input size and there is no need to pay with the complexity. I may think about it again once I see a good use case.
## Some other lexer / tokenizer packages

@@ -120,0 +199,0 @@ 

-15
CHANGELOG.md
# Changelog
## Version 0.6.0
- Targeting Node.js version 14 and ES2020;
- Now should be discoverable with [denoify](https://github.com/garronej/denoify).
## Version 0.5.1
- Documentation updates.
## Version 0.5.0
- Initial release;
- Aiming at Node.js version 12 and up.
-165
lib/leac.d.ts
/** Lexer options (not many so far). */
export declare type Options = {
/**
* Enable line and column numbers computation.
*/
lineNumbers?: boolean;
};
/** Result returned by a lexer function. */
export declare type LexerResult = {
/** Array of tokens. */
tokens: Token[];
/** Final offset. */
offset: number;
/**
* True if whole input string was processed.
*
* Check this to see whether some input left untokenized.
*/
complete: boolean;
};
/**
* Lexer function.
*
* @param str - A string to tokenize.
* @param offset - Initial offset. Used when composing lexers.
*/
export declare type Lexer = (str: string, offset?: number) => LexerResult;
/** Token object, a result of matching an individual lexing rule. */
export declare type Token = {
/** Name of the lexer containing the rule produced this token. */
state: string;
/** Name of the rule produced this token. */
name: string;
/** Text matched by the rule. _(Unless a replace value was used by a RegexRule.)_ */
text: string;
/** Start index of the match in the input string. */
offset: number;
/**
* The length of the matched substring.
*
* _(Might be different from the text length in case replace value
* was used in a RegexRule.)_
*/
len: number;
/**
* Line number in the source string (1-based).
*
* _(Always zero if not enabled in the lexer options.)_
*/
line: number;
/**
* Column number within the line in the source string (1-based).
*
* _(Always zero if line numbers not enabled in the lexer options.)_
*/
column: number;
};
/**
* Lexing rule.
*
* Base rule looks for exact match by it's name.
*
* If the name and the lookup string have to be different
* then specify `str` property as defined in {@link StringRule}.
*/
export interface Rule {
/** The name of the rule, also the name of tokens produced by this rule. */
name: string;
/**
* Matched token won't be added to the output array if this set to `true`.
*
* (_Think twice before using this._)
* */
discard?: boolean;
/**
* Switch to another lexer function after this match,
* concatenate it's results and continue from where it stopped.
*/
push?: Lexer;
/**
* Stop after this match and return.
*
* If there is a parent parser - it will continue from this point.
*/
pop?: boolean;
}
/**
* String rule - looks for exact string match that
* can be different from the name of the rule.
*/
export interface StringRule extends Rule {
/**
* Specify the exact string to match
* if it is different from the name of the rule.
*/
str: string;
}
/**
* Regex rule - looks for a regular expression match.
*/
export interface RegexRule extends Rule {
/**
* Regular expression to match.
*
* - Can't have the global flag.
*
* - All regular expressions are used as sticky,
* you don't have to specify the sticky flag.
*
* - Empty matches are considered as non-matches -
* no token will be emitted in that case.
*/
regex: RegExp;
/**
* Replacement string can include patterns,
* the same as [String.prototype.replace()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/replace#specifying_a_string_as_a_parameter).
*
* This will only affect the text property of an output token, not it's offset or length.
*
* Note: the regex has to be able to match the matched substring when taken out of context
* in order for replace to work - boundary/neighborhood conditions may prevent this.
*/
replace?: string;
}
/**
* Non-empty array of rules.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name. For example, you can have
* separate rules for various keywords and use the same name "keyword".
*/
export declare type Rules = [
(Rule | StringRule | RegexRule),
...(Rule | StringRule | RegexRule)[]
];
/**
* Create a lexer function.
*
* @param rules - Non-empty array of lexing rules.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name - you can have separate rules
* for keywords and use the same name "keyword" for example.
*
* @param state - The name of this lexer. Use when composing lexers.
* Empty string by default.
*
* @param options - Lexer options object.
*/
export declare function createLexer(rules: Rules, state?: string, options?: Options): Lexer;
/**
* Create a lexer function.
*
* @param rules - Non-empty array of lexing rules.
*
* Rules are processed in provided order, first match is taken.
*
* Rules can have the same name - you can have separate rules
* for keywords and use the same name "keyword" for example.
*
* @param options - Lexer options object.
*/
export declare function createLexer(rules: Rules, options?: Options): Lexer;