Socket
Socket
Sign inDemoInstall

snapdragon-lexer

Package Overview
Dependencies
Maintainers
1
Versions
5
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

snapdragon-lexer - npm Package Compare versions

Comparing version 3.1.0 to 4.0.0

lib/location.js
lib/state.js
lib/token.js

580

index.js
'use strict';
/**
* Module dependencies
*/
const fs = require('fs');
const Events = require('events');
const assert = require('assert');
const typeOf = require('kind-of');
const Handlers = require('snapdragon-handlers');
let Stack = require('snapdragon-stack');
let Token = require('snapdragon-token');
const State = require('./lib/state');
const Token = require('./lib/token');
const Location = require('./lib/location');
const { Position } = Location;

@@ -22,52 +18,127 @@ /**

* @name Lexer
* @param {String|Object} `input` (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
* @param {Object} `options`
* @param {string|Object} `input` (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
* @param {object} `options`
* @api public
*/
class Lexer extends Handlers {
constructor(input, options) {
class Lexer extends Events {
constructor(input, options = {}) {
super();
if (typeof input !== 'string') {
options = input;
options = input || {};
input = '';
}
super(options);
if (Lexer.isLexer(options)) {
return this.create(options.options, options);
this.options = { ...options };
this.handlers = new Map();
this.types = new Set();
this.state = new State(input);
if (options.handlers) {
for (const [type, handler] of options.handlers) {
this.handler(type, handler);
}
}
}
define(this, 'isLexer', true);
this.string = '';
this.input = '';
this.init(input);
/**
* Returns true if we are still at the beginning-of-string, and
* no part of the string has been consumed.
*
* @name .bos
* @return {boolean}
* @api public
*/
bos() {
return !this.state.consumed;
}
/**
* Initialize lexer state properties
* Returns true if `lexer.string` and `lexer.queue` are empty.
*
* @name .eos
* @return {boolean}
* @api public
*/
init(input) {
if (input) this.input = this.string = input.toString();
if (!input && isString(this.options.source)) {
return this.init(fs.readFileSync(this.options.source));
}
eos() {
return this.state.string === '' && this.state.queue.length === 0;
}
this.consumed = '';
this.tokens = new this.Stack();
this.stack = new this.Stack();
this.stash = new this.Stack('');
/**
* Register a handler function.
*
* ```js
* lexer.set('star', function() {
* // do parser, lexer, or compiler stuff
* });
* ```
* @name .set
* @param {string} `type`
* @param {function} `fn` The handler function to register.
* @api public
*/
define(this.stash, 'append', function(val) {
this[this.length - 1] += val;
set(type, handler = tok => tok) {
this.types.add(type);
const lexer = this;
// can't do fat arrow here, we need to ensure that the handler
// context is always correct whether handlers are called directly
// or re-registered on a new instance, etc.
this.handlers.set(type, function(...args) {
let ctx = this || lexer;
let loc = ctx.location();
let tok = handler.call(ctx, ...args);
if (tok && isObject(tok) && !Token.isToken(tok)) {
tok = ctx.token(tok);
}
if (Token.isToken(tok) && !tok.type) {
tok.type = type;
}
return Token.isToken(tok) ? loc(tok) : tok;
});
return this;
}
this.queue = [];
this.loc = {
index: 0,
column: 0,
line: 1
};
/**
* Get a registered handler function.
*
* ```js
* lexer.set('star', function() {
* // do lexer stuff
* });
* const star = lexer.get('star');
* ```
* @name .get
* @param {string} `type`
* @param {function} `fn` The handler function to register.
* @api public
*/
get(type) {
let handler = this.handlers.get(type) || this.handlers.get('unknown');
assert(handler, `expected handler "${type}" to be a function`);
return handler;
}
/**
* Returns true if the lexer has a registered handler of the given `type`.
*
* ```js
* lexer.set('star', function() {});
* console.log(lexer.has('star')); // true
* ```
* @name .has
* @param {string} type
* @return {boolean}
* @api public
*/
has(type) {
return this.handlers.has(type);
}
/**
* Create a new [Token][snapdragon-token] with the given `type` and `value`.

@@ -82,5 +153,5 @@ *

* @emits token
* @param {String|Object} `type` (required) The type of token to create
* @param {String} `value` (optional) The captured string
* @param {Array} `match` (optional) Match arguments returned from `String.match` or `RegExp.exec`
* @param {string|Object} `type` (required) The type of token to create
* @param {string} `value` (optional) The captured string
* @param {array} `match` (optional) Match results from `String.match()` or `RegExp.exec()`
* @return {Object} Returns an instance of [snapdragon-token][]

@@ -91,3 +162,3 @@ * @api public

token(type, value, match) {
const token = new this.Token(type, value, match);
let token = new Token(type, value, match);
this.emit('token', token);

@@ -106,4 +177,4 @@ return token;

* @name .isToken
* @param {Object} `token`
* @return {Boolean}
* @param {object} `token`
* @return {boolean}
* @api public

@@ -113,3 +184,3 @@ */

isToken(token) {
return this.Token.isToken(token);
return Token.isToken(token);
}

@@ -119,3 +190,3 @@

* Consume the given length from `lexer.string`. The consumed value is used
* to update `lexer.consumed`, as well as the current position.
* to update `lexer.state.consumed`, as well as the current position.
*

@@ -127,4 +198,4 @@ * ```js

* @name .consume
* @param {Number} `len`
* @param {String} `value` Optionally pass the value being consumed.
* @param {number} `len`
* @param {string} `value` Optionally pass the value being consumed.
* @return {String} Returns the consumed value

@@ -134,6 +205,5 @@ * @api public

consume(len, value) {
if (!value) value = this.string.slice(0, len);
this.consumed += value;
this.string = this.string.slice(len);
consume(len, value = this.state.string.slice(0, len)) {
this.state.consumed += value;
this.state.string = this.state.string.slice(len);
this.updateLocation(value, len);

@@ -145,15 +215,33 @@ return value;

* Update column and line number based on `value`.
* @param {String} `value`
*
* @param {string} `value`
* @return {undefined}
*/
updateLocation(value, len) {
if (typeof len !== 'number') len = value.length;
const i = value.lastIndexOf('\n');
this.loc.column = ~i ? len - i : this.loc.column + len;
this.loc.line += Math.max(0, value.split('\n').length - 1);
this.loc.index += len;
updateLocation(value, len = value.length) {
let i = value.lastIndexOf('\n');
this.state.loc.column = ~i ? len - i : this.state.loc.column + len;
this.state.loc.line += Math.max(0, value.split('\n').length - 1);
this.state.loc.index += len;
}
/**
* Returns a function for updating a token with lexer
* location information.
*
* @return {function}
* @api public
*/
location() {
let start = new Position(this);
return token => {
let end = new Position(this);
define(token, 'loc', new Location(start, end, this));
return token;
};
}
/**
* Use the given `regex` to match a substring from `lexer.string`. Also validates

@@ -171,3 +259,3 @@ * the regex to ensure that it starts with `^` since matching should always be

* @name .match
* @param {RegExp} `regex` (required)
* @param {regExp} `regex` (required)
* @return {Array|null} Returns the match array from `RegExp.exec` or null.

@@ -178,10 +266,11 @@ * @api public

match(regex) {
assert.equal(typeOf(regex), 'regexp', 'expected a regular expression');
assert(regex instanceof RegExp, 'expected a regular expression');
if (regex.validated !== true) {
assert(/^\^/.test(regex.source), 'expected regex to start with "^"');
assert(regex.source[0] === '^', 'expected regex to start with "^"');
regex.validated = true;
}
const consumed = this.consumed;
const match = regex.exec(this.string);
let consumed = this.state.consumed;
let match = regex.exec(this.state.string);
if (!match) return null;

@@ -193,2 +282,3 @@

this.emit('match', match);
define(match, 'consumed', consumed);

@@ -217,4 +307,4 @@ this.consume(match[0].length, match[0]);

* @emits scan
* @param {String} `type`
* @param {RegExp} `regex`
* @param {string} `type`
* @param {regExp} `regex`
* @return {Object} Returns a token if a match is found, otherwise undefined.

@@ -226,11 +316,12 @@ * @api public

try {
const match = this.match(regex);
if (!match) return;
const token = this.token(type, match);
this.emit('scan', token);
return token;
let match = this.match(regex);
if (match) {
let tok = this.token(type, match[0], match);
this.emit('scan', tok);
return tok;
}
} catch (err) {
err.regex = regex;
err.type = type;
this.error(err);
throw err;
}

@@ -254,5 +345,5 @@ }

* @name .capture
* @param {String} `type` (required) The type of token being captured.
* @param {RegExp} `regex` (required) The regex for matching substrings.
* @param {Function} `fn` (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
* @param {string} `type` (required) The type of token being captured.
* @param {regExp} `regex` (required) The regex for matching substrings.
* @param {function} `fn` (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
* @return {Object}

@@ -263,3 +354,3 @@ * @api public

capture(type, regex, fn) {
this.set(type, function() {
let handler = function() {
let token = this.scan(regex, type);

@@ -269,3 +360,4 @@ if (token) {

}
});
};
this.set(type, handler);
return this;

@@ -290,3 +382,3 @@ }

* @emits handle
* @param {String} `type` The handler type to call on `lexer.string`
* @param {string} `type` The handler type to call on `lexer.string`
* @return {Object} Returns a token of the given `type` or undefined.

@@ -297,3 +389,3 @@ * @api public

handle(type) {
const token = this.get(type).call(this);
let token = this.get(type).call(this);
if (token) {

@@ -321,12 +413,12 @@ this.current = token;

advance() {
if (!this.string) return;
if (this.eos()) return;
if (this.options.mode === 'character') {
return (this.current = this.consume(1));
}
for (const type of this.types) {
const token = this.handle(type);
if (token) return token;
for (let type of this.types) {
let token = this.handle(type);
if (token) {
return token;
}
}
this.fail();

@@ -339,5 +431,6 @@ }

* ```js
* let lexer = new Lexer({ handlers: otherLexer.handlers })
* lexer.capture('slash', /^\//);
* lexer.capture('text', /^\w+/);
* const tokens = lexer.tokenize('a/b/c');
* const tokens = lexer.lex('a/b/c');
* console.log(tokens);

@@ -351,4 +444,4 @@ * // Results in:

* ```
* @name .tokenize
* @param {String} `input` The string to tokenize.
* @name .lex
* @param {string} `input` The string to lex.
* @return {Array} Returns an array of tokens.

@@ -358,8 +451,13 @@ * @api public

tokenize(input) {
if (input) this.init(input);
lex(input, options) {
if (options) this.options = { ...options };
if (input) this.state = new State(input);
while (this.push(this.next()));
return this.tokens;
return this.state.tokens;
}
tokenize(...args) {
return this.lex(...args);
}
/**

@@ -374,3 +472,3 @@ * Push a token onto the `lexer.queue` array.

* @name .enqueue
* @param {Object} `token`
* @param {object} `token`
* @return {Object} Returns the given token with updated `token.index`.

@@ -381,4 +479,3 @@ * @api public

enqueue(token) {
if (!token) return;
this.queue.push(token);
token && this.state.queue.push(token);
return token;

@@ -401,3 +498,3 @@ }

dequeue() {
return this.queue.length && this.queue.shift();
return this.state.queue.length && this.state.queue.shift();
}

@@ -412,3 +509,3 @@

* @name .lookbehind
* @param {Number} `n`
* @param {number} `n`
* @return {Object}

@@ -419,7 +516,8 @@ * @api public

lookbehind(n) {
return this.tokens.lookbehind(n);
assert(Number.isInteger(n), 'expected a positive integer');
return this.state.tokens[this.state.tokens.length - n];
}
/**
* Get the previous token.
* Get the previously lexed token.
*

@@ -430,3 +528,3 @@ * ```js

* @name .prev
* @returns {Object} Returns a token.
* @returns {Object|undefined} Returns a token or undefined.
* @api public

@@ -448,3 +546,3 @@ */

* @name .lookahead
* @param {Number} `n`
* @param {number} `n`
* @return {Object}

@@ -455,6 +553,6 @@ * @api public

lookahead(n) {
assert.equal(typeof n, 'number', 'expected a number');
let fetch = n - this.queue.length;
assert(Number.isInteger(n), 'expected a positive integer');
let fetch = n - this.state.queue.length;
while (fetch-- > 0 && this.enqueue(this.advance()));
return this.queue[--n];
return this.state.queue[--n];
}

@@ -469,3 +567,3 @@

* @name .peek
* @return {Object} `token`
* @return {Object} Returns a token.
* @api public

@@ -500,4 +598,4 @@ */

* @name .skip
* @param {Number} `n`
* @returns {Object} returns the very last lexed/skipped token.
* @param {number} `n`
* @returns {Object} returns an array of skipped tokens.
* @api public

@@ -512,3 +610,3 @@ */

/**
* Skip the given token `types`.
* Skip tokens while the given `fn` returns true.
*

@@ -518,4 +616,4 @@ * ```js

* ```
* @name .skipType
* @param {String|Array} `types` One or more token types to skip.
* @name .skipWhile
* @param {function} `fn` Return true if a token should be skipped.
* @returns {Array} Returns an array if skipped tokens.

@@ -525,3 +623,3 @@ * @api public

skipWhile(fn) {
skipWhile(fn = !this.eos()) {
const skipped = [];

@@ -539,3 +637,3 @@ while (fn.call(this, this.peek())) skipped.push(this.next());

* @name .skipType
* @param {String|Array} `types` One or more token types to skip.
* @param {string|Array} `types` One or more token types to skip.
* @returns {Array} Returns an array if skipped tokens.

@@ -546,3 +644,3 @@ * @api public

skipTo(type) {
return this.skipWhile(tok => tok.type !== type).concat(this.next());
return this.skipWhile(tok => tok && tok.type !== type).concat(this.next());
}

@@ -558,3 +656,3 @@

* @name .skipType
* @param {String|Array} `types` One or more token types to skip.
* @param {string|Array} `types` One or more token types to skip.
* @returns {Array} Returns an array if skipped tokens

@@ -565,36 +663,6 @@ * @api public

skipType(types) {
return this.skipWhile(tok => ~arrayify(types).indexOf(tok.type));
return this.skipWhile(tok => [].concat(types).includes(tok.type));
}
/**
* Pushes the given `value` onto `lexer.stash`.
*
* ```js
* lexer.append('abc');
* lexer.append('/');
* lexer.append('*');
* lexer.append('.');
* lexer.append('js');
* console.log(lexer.stash);
* //=> ['abc', '/', '*', '.', 'js']
* ```
* @name .append
* @emits append
* @param {any} `value`
* @return {Object} Returns the Lexer instance.
* @api public
*/
append(value, enqueue) {
if (!value) return;
if (this.stash.last() === '') {
this.stash.append(value);
} else {
this.stash.push(value);
}
this.emit('append', value);
return this;
}
/**
* Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push

@@ -612,3 +680,3 @@ * `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting

* @emits push
* @param {Object|String} `token`
* @param {object|String} `token`
* @return {Object} Returns the given `token`.

@@ -619,3 +687,3 @@ * @api public

push(token) {
if (!token) return;
if (!token && token !== '') return;
if (this.options.mode !== 'character') {

@@ -626,7 +694,13 @@ assert(this.isToken(token), 'expected token to be an instance of Token');

this.emit('push', token);
this.tokens.push(token);
this.state.tokens.push(token);
if (this.options.append !== false && token.append !== false) {
this.append(this.value(token));
if (this.options.stash === false || token.stash === false) {
return token;
}
if (this.options.mode === 'character') {
this.append(token);
} else {
this.append(token.value);
}
return token;

@@ -636,63 +710,52 @@ }

/**
* Returns true if a token with the given `type` is on the stack.
* Append a string to the last element on `lexer.stash`, or push the
* string onto the stash if no elements exist.
*
* ```js
* if (lexer.isInside('bracket') || lexer.isInside('brace')) {
* // do stuff
* }
* const stack = new Stack();
* stack.push('a');
* stack.push('b');
* stack.push('c');
* stack.append('_foo');
* stack.append('_bar');
* console.log(stack);
* //=> Stack ['a', 'b', 'c_foo_bar']
* ```
* @name .isInside
* @param {String} `type` The type to check for.
* @return {Boolean}
* @name .append
* @param {String} `value`
* @return {String} Returns the last value in the array.
* @api public
*/
isInside(type) {
const last = this.stack.last();
return this.isToken(last) && last.type === type;
append(value) {
if (typeof value !== 'string') return;
let n = this.state.stash.length - 1;
if (this.state.stash[n] === '') {
this.state.stash[n] += value;
} else {
this.state.stash.push(value);
}
this.emit('append', value);
return this;
}
/**
* Returns the value of a token using the property defined on `lexer.options.value`
* or `token.value`.
* Returns true if a token with the given `type` is on the stack.
*
* @name .value
* @return {String|undefined}
* ```js
* if (lexer.isInside('bracket') || lexer.isInside('brace')) {
* // do stuff
* }
* ```
* @name .isInside
* @param {string} `type` The type to check for.
* @return {boolean}
* @api public
*/
value(token) {
return token[this.options.value || 'value'];
isInside(type) {
return this.state.stack.some(tok => tok.type === type);
}
/**
* Returns true if `lexer.string` and `lexer.queue` are empty.
*
* @name .eos
* @return {Boolean}
* @api public
*/
eos() {
return this.string.length === 0 && this.queue.length === 0;
}
/**
* Creates a new Lexer instance with the given options, and copy
* the handlers from the current instance to the new instance.
*
* @param {Object} `options`
* @param {Object} `parent` Optionally pass a different lexer instance to copy handlers from.
* @return {Object} Returns a new Lexer instance
* @api public
*/
create(options, parent = this) {
const lexer = new this.constructor(options);
lexer.handlers = parent.handlers;
lexer.types = parent.types;
return lexer;
}
/**
* Throw a formatted error message with details including the cursor position.

@@ -703,3 +766,3 @@ *

* if (tok.value !== 'foo') {
* throw this.error('expected token.value to be "foo"', tok);
* throw this.state.error('expected token.value to be "foo"', tok);
* }

@@ -709,4 +772,4 @@ * });

* @name .error
* @param {String} `msg` Message to use in the Error.
* @param {Object} `node`
* @param {string} `msg` Message to use in the Error.
* @param {object} `node`
* @return {undefined}

@@ -718,3 +781,3 @@ * @api public

if (typeof err === 'string') err = new Error(err);
if (this.hasListeners('error')) {
if (this.listenerCount('error') > 0) {
this.emit('error', err);

@@ -733,9 +796,10 @@ } else {

fail() {
if (this.stack.length) {
const last = this.stack.last();
const val = last.match ? last.match[0] : last[this.options.value || 'value'];
this.error(new Error(`unclosed: "${val}"`));
let token = this.state.stack.pop();
if (token) {
const match = token && token.match;
const value = match ? match[0] : token[this.options.value || 'value'];
throw new Error(`unclosed: "${value}"`);
}
if (this.string) {
this.error(new Error(`unmatched input: "${this.string.slice(0, 10)}"`));
if (this.state.string) {
throw new Error(`unmatched input: "${this.state.string.slice(0, 10)}"`);
}

@@ -745,22 +809,21 @@ }

/**
* Get or set the `Stack` constructor to use for initializing Lexer stacks.
* @name .Stack
* @api private
* Call a plugin function on the lexer instance.
*
* ```js
* lexer.use(function(lexer) {
* // do stuff to lexer
* });
* ```
* @name .use
* @param {function} `fn`
* @return {object} Returns the lexer instance.
* @api public
*/
get Stack() {
return this.options.Stack || Lexer.Stack;
use(fn) {
fn.call(this, this);
return this;
}
/**
* Get or set the `Token` constructor to use when calling `lexer.token()`.
* @name .Token
* @api private
*/
get Token() {
return this.options.Token || Lexer.Token;
}
/**
* Static method that returns true if the given value is an

@@ -776,3 +839,3 @@ * instance of `snapdragon-lexer`.

* @name Lexer#isLexer
* @param {Object} `lexer`
* @param {object} `lexer`
* @returns {Boolean}

@@ -784,9 +847,18 @@ * @api public

static isLexer(lexer) {
return lexer && lexer.isLexer === true;
return lexer instanceof Lexer;
}
/**
* Static method for getting or setting the `Stack` constructor.
* Static method that returns true if the given value is an
* instance of `snapdragon-token`. This is a proxy to `Token#isToken`.
*
* @name Lexer#Stack
* ```js
* const Token = require('snapdragon-token');
* const Lexer = require('snapdragon-lexer');
* console.log(Lexer.isToken(new Token({type: 'foo'}))); //=> true
* console.log(Lexer.isToken({})); //=> false
* ```
* @name Lexer#isToken
* @param {object} `lexer`
* @returns {Boolean}
* @api public

@@ -796,14 +868,9 @@ * @static

static set Stack(Ctor) {
Stack = Ctor;
static isToken(token) {
return token instanceof Token;
}
static get Stack() {
return Stack;
}
/**
* Static method for getting or setting the `Token` constructor, used
* by `lexer.token()` to create a new token.
*
* @name Lexer#Token
* The State class, exposed as a static property.
* @name Lexer#State
* @api public

@@ -813,22 +880,9 @@ * @static

static set Token(Ctor) {
Token = Ctor;
static get State() {
return State;
}
static get Token() {
return Token;
}
/**
* Static method that returns true if the given value is an
* instance of `snapdragon-token`. This is a proxy to `Token#isToken`.
*
* ```js
* const Token = require('snapdragon-token');
* const Lexer = require('snapdragon-lexer');
* console.log(Lexer.isToken(new Token({type: 'foo'}))); //=> true
* console.log(Lexer.isToken({})); //=> false
* ```
* @name Lexer#isToken
* @param {Object} `lexer`
* @returns {Boolean}
* The Token class, exposed as a static property.
* @name Lexer#Token
* @api public

@@ -838,24 +892,28 @@ * @static

static isToken(token) {
return this.Token.isToken(token);
static get Token() {
return Token;
}
};
}
function arrayify(value) {
return Array.isArray(value) ? value : [value];
/**
* Returns true if value is an object
*/
function isObject(val) {
return val && typeof val === 'object' && !Array.isArray(val);
}
/**
* Define a non-enumerable property on `obj`
*/
function define(obj, key, value) {
Reflect.defineProperty(obj, key, { configurable: false, writable: false, value: value });
Reflect.defineProperty(obj, key, { value });
}
function isString(input) {
return input && typeof input === 'string';
}
/**
* Expose `Lexer`
* @type {Function}
* @type {Class}
*/
module.exports = Lexer;

37

package.json
{
"name": "snapdragon-lexer",
"description": "Converts a string into an array of tokens, with useful methods for looking ahead and behind, capturing, matching, et cetera.",
"version": "3.1.0",
"version": "4.0.0",
"homepage": "https://github.com/here-be/snapdragon-lexer",

@@ -13,23 +13,18 @@ "author": "Jon Schlinkert (https://github.com/jonschlinkert)",

"files": [
"index.js"
"index.js",
"lib"
],
"main": "index.js",
"engines": {
"node": ">=6"
"node": ">=8"
},
"scripts": {
"test": "nyc mocha"
"test": "mocha",
"cover": "nyc --reporter=text --reporter=html mocha"
},
"dependencies": {
"kind-of": "^6.0.2",
"snapdragon-handlers": "^1.0.0",
"snapdragon-stack": "^2.1.0",
"snapdragon-token": "^3.0.1"
},
"devDependencies": {
"define-property": "^2.0.2",
"gulp-format-md": "^1.0.0",
"mocha": "^3.5.3",
"nyc": "^11.4.1",
"snapdragon-position": "^2.0.2"
"gulp-format-md": "^2.0.0",
"mocha": "^5.2.0",
"nyc": "^13.1.0"
},

@@ -44,13 +39,10 @@ "keywords": [

"render",
"scan",
"scanner",
"snapdragon",
"token",
"tokenize",
"tokenizer",
"transform"
],
"nyc": {
"reporter": [
"lcov",
"text-summary"
]
},
"verb": {

@@ -70,5 +62,4 @@ "toc": "collapsible",

"list": [
"snapdragon-node",
"snapdragon-position",
"snapdragon-token"
"snapdragon-parser",
"snapdragon-scanner"
]

@@ -75,0 +66,0 @@ },

307

README.md

@@ -66,3 +66,3 @@ # snapdragon-lexer [![NPM version](https://img.shields.io/npm/v/snapdragon-lexer.svg?style=flat)](https://www.npmjs.com/package/snapdragon-lexer) [![NPM monthly downloads](https://img.shields.io/npm/dm/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![NPM total downloads](https://img.shields.io/npm/dt/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![Linux Build Status](https://img.shields.io/travis/here-be/snapdragon-lexer.svg?style=flat&label=Travis)](https://travis-ci.org/here-be/snapdragon-lexer)

### [Lexer](index.js#L27)
### [Lexer](index.js#L23)

@@ -73,4 +73,4 @@ Create a new `Lexer` with the given `options`.

* `input` **{String|Object}**: (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
* `options` **{Object}**
* `input` **{string|Object}**: (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
* `options` **{object}**

@@ -84,4 +84,68 @@ **Example**

### [.token](index.js#L88)
### [.bos](index.js#L53)
Returns true if we are still at the beginning-of-string, and
no part of the string has been consumed.
* `returns` **{boolean}**
### [.eos](index.js#L65)
Returns true if `lexer.string` and `lexer.queue` are empty.
* `returns` **{boolean}**
### [.set](index.js#L83)
Register a handler function.
**Params**
* `type` **{string}**
* `fn` **{function}**: The handler function to register.
**Example**
```js
lexer.set('star', function() {
// do parser, lexer, or compiler stuff
});
```
### [.get](index.js#L119)
Get a registered handler function.
**Params**
* `type` **{string}**
* `fn` **{function}**: The handler function to register.
**Example**
```js
lexer.set('star', function() {
// do lexer stuff
});
const star = lexer.get('star');
```
### [.has](index.js#L138)
Returns true if the lexer has a registered handler of the given `type`.
**Params**
* **{string}**: type
* `returns` **{boolean}**
**Example**
```js
lexer.set('star', function() {});
console.log(lexer.has('star')); // true
```
### [.token](index.js#L159)
Create a new [Token](https://github.com/here-be/snapdragon-token) with the given `type` and `value`.

@@ -91,5 +155,5 @@

* `type` **{String|Object}**: (required) The type of token to create
* `value` **{String}**: (optional) The captured string
* `match` **{Array}**: (optional) Match arguments returned from `String.match` or `RegExp.exec`
* `type` **{string|Object}**: (required) The type of token to create
* `value` **{string}**: (optional) The captured string
* `match` **{array}**: (optional) Match results from `String.match()` or `RegExp.exec()`
* `returns` **{Object}**: Returns an instance of [snapdragon-token](https://github.com/here-be/snapdragon-token)

@@ -109,3 +173,3 @@

### [.isToken](index.js#L108)
### [.isToken](index.js#L179)

@@ -116,4 +180,4 @@ Returns true if the given value is a [snapdragon-token](https://github.com/here-be/snapdragon-token) instance.

* `token` **{Object}**
* `returns` **{Boolean}**
* `token` **{object}**
* `returns` **{boolean}**

@@ -128,10 +192,10 @@ **Example**

### [.consume](index.js#L127)
### [.consume](index.js#L198)
Consume the given length from `lexer.string`. The consumed value is used to update `lexer.consumed`, as well as the current position.
Consume the given length from `lexer.string`. The consumed value is used to update `lexer.state.consumed`, as well as the current position.
**Params**
* `len` **{Number}**
* `value` **{String}**: Optionally pass the value being consumed.
* `len` **{number}**
* `value` **{string}**: Optionally pass the value being consumed.
* `returns` **{String}**: Returns the consumed value

@@ -146,4 +210,9 @@

### [.match](index.js#L167)
Returns a function for updating a token with lexer
location information.
* `returns` **{function}**
### [.match](index.js#L255)
Use the given `regex` to match a substring from `lexer.string`. Also validates the regex to ensure that it starts with `^` since matching should always be against the beginning of the string, and throws if the regex matches an empty string, which can cause catastrophic backtracking.

@@ -153,3 +222,3 @@

* `regex` **{RegExp}**: (required)
* `regex` **{regExp}**: (required)
* `returns` **{Array|null}**: Returns the match array from `RegExp.exec` or null.

@@ -166,3 +235,3 @@

### [.scan](index.js#L211)
### [.scan](index.js#L301)

@@ -173,4 +242,4 @@ Scan for a matching substring by calling [.match()](#match) with the given `regex`. If a match is found, 1) a token of the specified `type` is created, 2) `match[0]` is used as `token.value`, and 3) the length of `match[0]` is sliced from `lexer.string` (by calling [.consume()](#consume)).

* `type` **{String}**
* `regex` **{RegExp}**
* `type` **{string}**
* `regex` **{regExp}**
* `returns` **{Object}**: Returns a token if a match is found, otherwise undefined.

@@ -194,3 +263,3 @@

### [.capture](index.js#L247)
### [.capture](index.js#L338)

@@ -201,5 +270,5 @@ Capture a token of the specified `type` using the provide `regex` for scanning and matching substrings. Automatically registers a handler when a function is passed as the last argument.

* `type` **{String}**: (required) The type of token being captured.
* `regex` **{RegExp}**: (required) The regex for matching substrings.
* `fn` **{Function}**: (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
* `type` **{string}**: (required) The type of token being captured.
* `regex` **{regExp}**: (required) The regex for matching substrings.
* `fn` **{function}**: (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
* `returns` **{Object}**

@@ -219,3 +288,3 @@

### [.handle](index.js#L278)
### [.handle](index.js#L370)

@@ -226,3 +295,3 @@ Calls handler `type` on `lexer.string`.

* `type` **{String}**: The handler type to call on `lexer.string`
* `type` **{string}**: The handler type to call on `lexer.string`
* `returns` **{Object}**: Returns a token of the given `type` or undefined.

@@ -248,3 +317,3 @@

### [.advance](index.js#L301)
### [.advance](index.js#L393)

@@ -261,3 +330,3 @@ Get the next token by iterating over `lexer.handlers` and calling each handler on `lexer.string` until a handler returns a token. If no handlers return a token, an error is thrown with the substring that couldn't be lexed.

### [.tokenize](index.js#L336)
### [.lex](index.js#L429)

@@ -268,3 +337,3 @@ Tokenizes a string and returns an array of tokens.

* `input` **{String}**: The string to tokenize.
* `input` **{string}**: The string to lex.
* `returns` **{Array}**: Returns an array of tokens.

@@ -275,5 +344,6 @@

```js
let lexer = new Lexer({ handlers: otherLexer.handlers })
lexer.capture('slash', /^\//);
lexer.capture('text', /^\w+/);
const tokens = lexer.tokenize('a/b/c');
const tokens = lexer.lex('a/b/c');
console.log(tokens);

@@ -288,3 +358,3 @@ // Results in:

### [.enqueue](index.js#L356)
### [.enqueue](index.js#L454)

@@ -295,3 +365,3 @@ Push a token onto the `lexer.queue` array.

* `token` **{Object}**
* `token` **{object}**
* `returns` **{Object}**: Returns the given token with updated `token.index`.

@@ -307,3 +377,3 @@

### [.dequeue](index.js#L375)
### [.dequeue](index.js#L472)

@@ -322,3 +392,3 @@ Shift a token from `lexer.queue`.

### [.lookbehind](index.js#L391)
### [.lookbehind](index.js#L488)

@@ -329,3 +399,3 @@ Lookbehind `n` tokens.

* `n` **{Number}**
* `n` **{number}**
* `returns` **{Object}**

@@ -339,7 +409,7 @@

### [.prev](index.js#L406)
### [.prev](index.js#L504)
Get the previous token.
Get the previously lexed token.
* `returns` **{Object}**: Returns a token.
* `returns` **{Object|undefined}**: Returns a token or undefined.

@@ -352,3 +422,3 @@ **Example**

### [.lookahead](index.js#L424)
### [.lookahead](index.js#L522)

@@ -359,3 +429,3 @@ Lookahead `n` tokens and return the last token. Pushes any intermediate tokens onto `lexer.tokens.` To lookahead a single token, use [.peek()](#peek).

* `n` **{Number}**
* `n` **{number}**
* `returns` **{Object}**

@@ -369,7 +439,7 @@

### [.peek](index.js#L442)
### [.peek](index.js#L540)
Lookahead a single token.
* `returns` **{Object}** `token`
* `returns` **{Object}**: Returns a token.

@@ -382,3 +452,3 @@ **Example**

### [.next](index.js#L457)
### [.next](index.js#L555)

@@ -395,3 +465,3 @@ Get the next token, either from the `queue` or by [advancing](#advance).

### [.skip](index.js#L473)
### [.skip](index.js#L571)

@@ -402,4 +472,4 @@ Skip `n` tokens or characters in the string. Skipped values are not enqueued.

* `n` **{Number}**
* `returns` **{Object}**: returns the very last lexed/skipped token.
* `n` **{number}**
* `returns` **{Object}**: returns an array of skipped tokens.

@@ -412,9 +482,9 @@ **Example**

### [.skipType](index.js#L490)
### [.skipWhile](index.js#L588)
Skip the given token `types`.
Skip tokens while the given `fn` returns true.
**Params**
* `types` **{String|Array}**: One or more token types to skip.
* `fn` **{function}**: Return true if a token should be skipped.
* `returns` **{Array}**: Returns an array if skipped tokens.

@@ -428,3 +498,3 @@

### [.skipType](index.js#L508)
### [.skipType](index.js#L606)

@@ -435,3 +505,3 @@ Skip the given token `types`.

* `types` **{String|Array}**: One or more token types to skip.
* `types` **{string|Array}**: One or more token types to skip.
* `returns` **{Array}**: Returns an array if skipped tokens.

@@ -445,3 +515,3 @@

### [.skipType](index.js#L525)
### [.skipType](index.js#L623)

@@ -452,3 +522,3 @@ Skip the given token `types`.

* `types` **{String|Array}**: One or more token types to skip.
* `types` **{string|Array}**: One or more token types to skip.
* `returns` **{Array}**: Returns an array if skipped tokens

@@ -463,14 +533,14 @@

### [.append](index.js#L548)
### [.push](index.js#L645)
Pushes the given `value` onto `lexer.stash`.
Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting `lexer.options.append` or `token.append` to `false`.
**Params**
* `value` **{any}**
* `returns` **{Object}**: Returns the Lexer instance.
* `token` **{object|String}**
* `returns` **{Object}**: Returns the given `token`.
**Events**
* `emits`: append
* `emits`: push

@@ -480,34 +550,31 @@ **Example**

```js
lexer.append('abc');
lexer.append('/');
lexer.append('*');
lexer.append('.');
lexer.append('js');
console.log(lexer.stash);
//=> ['abc', '/', '*', '.', 'js']
console.log(lexer.tokens.length); // 0
lexer.push(new Token('star', '*'));
console.log(lexer.tokens.length); // 1
console.log(lexer.stash) // ['*']
```
### [.push](index.js#L577)
### [.append](index.js#L686)
Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting `lexer.options.append` or `token.append` to `false`.
Append a string to the last element on `lexer.stash`, or push the string onto the stash if no elements exist.
**Params**
* `token` **{Object|String}**
* `returns` **{Object}**: Returns the given `token`.
* `value` **{String}**
* `returns` **{String}**: Returns the last value in the array.
**Events**
* `emits`: push
**Example**
```js
console.log(lexer.tokens.length); // 0
lexer.push(new Token('star', '*'));
console.log(lexer.tokens.length); // 1
console.log(lexer.stash) // ['*']
const stack = new Stack();
stack.push('a');
stack.push('b');
stack.push('c');
stack.append('_foo');
stack.append('_bar');
console.log(stack);
//=> Stack ['a', 'b', 'c_foo_bar']
```
### [.isInside](index.js#L606)
### [.isInside](index.js#L712)

@@ -518,4 +585,4 @@ Returns true if a token with the given `type` is on the stack.

* `type` **{String}**: The type to check for.
* `returns` **{Boolean}**
* `type` **{string}**: The type to check for.
* `returns` **{boolean}**

@@ -530,33 +597,30 @@ **Example**

### [.value](index.js#L620)
### [.error](index.js#L733)
Returns the value of a token using the property defined on `lexer.options.value`
or `token.value`.
Throw a formatted error message with details including the cursor position.
* `returns` **{String|undefined}**
**Params**
### [.eos](index.js#L632)
* `msg` **{string}**: Message to use in the Error.
* `node` **{object}**
* `returns` **{undefined}**
Returns true if `lexer.string` and `lexer.queue` are empty.
**Example**
* `returns` **{Boolean}**
```js
lexer.set('foo', function(tok) {
if (tok.value !== 'foo') {
throw this.state.error('expected token.value to be "foo"', tok);
}
});
```
Creates a new Lexer instance with the given options, and copy
the handlers from the current instance to the new instance.
### [.use](index.js#L774)
**Params**
Call a plugin function on the lexer instance.
* `options` **{Object}**
* `parent` **{Object}**: Optionally pass a different lexer instance to copy handlers from.
* `returns` **{Object}**: Returns a new Lexer instance
### [.error](index.js#L670)
Throw a formatted error message with details including the cursor position.
**Params**
* `msg` **{String}**: Message to use in the Error.
* `node` **{Object}**
* `returns` **{undefined}**
* `fn` **{function}**
* `returns` **{object}**: Returns the lexer instance.

@@ -566,10 +630,8 @@ **Example**

```js
lexer.set('foo', function(tok) {
if (tok.value !== 'foo') {
throw this.error('expected token.value to be "foo"', tok);
}
lexer.use(function(lexer) {
// do stuff to lexer
});
```
### [Lexer#isLexer](index.js#L733)
### [Lexer#isLexer](index.js#L796)

@@ -580,3 +642,3 @@ Static method that returns true if the given value is an instance of `snapdragon-lexer`.

* `lexer` **{Object}**
* `lexer` **{object}**
* `returns` **{Boolean}**

@@ -593,13 +655,4 @@

### [Lexer#Stack](index.js#L745)
### [Lexer#isToken](index.js#L817)
Static method for getting or setting the `Stack` constructor.
### [Lexer#Token](index.js#L761)
Static method for getting or setting the `Token` constructor, used
by `lexer.token()` to create a new token.
### [Lexer#isToken](index.js#L785)
Static method that returns true if the given value is an instance of `snapdragon-token`. This is a proxy to `Token#isToken`.

@@ -609,3 +662,3 @@

* `lexer` **{Object}**
* `lexer` **{object}**
* `returns` **{Boolean}**

@@ -622,2 +675,10 @@

### [Lexer#State](index.js#L828)
The State class, exposed as a static property.
### [Lexer#Token](index.js#L839)
The Token class, exposed as a static property.
### .set

@@ -844,5 +905,3 @@

* [snapdragon-node](https://www.npmjs.com/package/snapdragon-node): Snapdragon utility for creating a new AST node in custom code, such as plugins. | [homepage](https://github.com/jonschlinkert/snapdragon-node "Snapdragon utility for creating a new AST node in custom code, such as plugins.")
* [snapdragon-position](https://www.npmjs.com/package/snapdragon-position): Snapdragon util and plugin for patching the position on an AST node. | [homepage](https://github.com/here-be/snapdragon-position "Snapdragon util and plugin for patching the position on an AST node.")
* [snapdragon-token](https://www.npmjs.com/package/snapdragon-token): Create a snapdragon token. Used by the snapdragon lexer, but can also be used by… [more](https://github.com/here-be/snapdragon-token) | [homepage](https://github.com/here-be/snapdragon-token "Create a snapdragon token. Used by the snapdragon lexer, but can also be used by plugins.")
* [snapdragon-scanner](https://www.npmjs.com/package/snapdragon-scanner): Easily scan a string with an object of regex patterns to produce an array of… [more](https://github.com/here-be/snapdragon-scanner) | [homepage](https://github.com/here-be/snapdragon-scanner "Easily scan a string with an object of regex patterns to produce an array of tokens. ~100 sloc.")

@@ -853,5 +912,5 @@ ### Author

* [linkedin/in/jonschlinkert](https://linkedin.com/in/jonschlinkert)
* [github/jonschlinkert](https://github.com/jonschlinkert)
* [twitter/jonschlinkert](https://twitter.com/jonschlinkert)
* [GitHub Profile](https://github.com/jonschlinkert)
* [Twitter Profile](https://twitter.com/jonschlinkert)
* [LinkedIn Profile](https://linkedin.com/in/jonschlinkert)

@@ -865,2 +924,2 @@ ### License

_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.6.0, on February 16, 2018._
_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.8.0, on November 19, 2018._
LICENSE

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc