Socket
Socket
Sign inDemoInstall

snapdragon-lexer

Package Overview
Dependencies
Maintainers
1
Versions
5
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

snapdragon-lexer - npm Package Compare versions

Comparing version 1.0.0 to 2.0.0

CHANGELOG.md

488

index.js

@@ -7,7 +7,8 @@ 'use strict';

const fs = require('fs');
const assert = require('assert');
const Token = require('snapdragon-token');
const Emitter = require('@sellside/emitter');
const union = require('union-value');
const use = require('use');
const typeOf = require('kind-of');
const Handlers = require('snapdragon-handlers');
let Stack = require('snapdragon-stack');
let Token = require('snapdragon-token');

@@ -27,17 +28,17 @@ /**

class Lexer extends Emitter {
class Lexer extends Handlers {
constructor(input, options) {
super();
if (typeof input !== 'string') {
return new Lexer('', input);
options = input;
input = '';
}
super(options);
if (Lexer.isLexer(options)) {
return this.create(options.options, options);
}
Reflect.defineProperty(this, 'isLexer', { value: true });
this.options = Object.assign({type: 'root', Token}, options);
this.Token = this.options.Token;
this.handlers = {};
this.types = [];
define(this, 'isLexer', true);
this.string = '';
this.input = '';
this.init(input);
use(this);
}

@@ -50,14 +51,23 @@

init(input) {
if (input) this.input = this.string = input;
this.loc = { index: 0, column: 1, line: 1 };
if (input) this.input = this.string = input.toString();
if (!input && isString(this.options.source)) {
return this.init(fs.readFileSync(this.options.source));
}
this.consumed = '';
this.tokens = [];
this.tokens = new this.Stack();
this.stash = new this.Stack('');
this.stack = new this.Stack();
this.queue = [];
this.loc = {
index: 0,
column: 0,
line: 1
};
}
/**
* Create a new [Token][snapdragon-token] with the given `type` and `val`.
* Create a new [Token][snapdragon-token] with the given `type` and `value`.
*
* ```js
* console.log(lexer.token({type: 'star', val: '*'}));
* console.log(lexer.token({type: 'star', value: '*'}));
* console.log(lexer.token('star', '*'));

@@ -69,3 +79,3 @@ * console.log(lexer.token('star'));

* @param {String|Object} `type` (required) The type of token to create
* @param {String} `val` (optional) The captured string
* @param {String} `value` (optional) The captured string
* @param {Array} `match` (optional) Match arguments returned from `String.match` or `RegExp.exec`

@@ -76,4 +86,4 @@ * @return {Object} Returns an instance of [snapdragon-token][]

token(type, val, match) {
const token = new this.Token(type, val, match);
token(type, value, match) {
const token = new this.Token(type, value, match);
this.emit('token', token);

@@ -89,3 +99,3 @@ return token;

* lexer.isToken({}); // false
* lexer.isToken(new Token({type: 'star', val: '*'})); // true
* lexer.isToken(new Token({type: 'star', value: '*'})); // true
* ```

@@ -103,87 +113,35 @@ * @name .isToken

/**
* Register a lexer handler function for creating tokens by
* matching substrings of the given `type.`
* Consume the given length from `lexer.string`. The consumed value is used
* to update `lexer.consumed`, as well as the current position.
*
* ```js
* lexer.set('star', function() {
* const match = this.match(regex, type);
* if (match) {
* return this.token({val: match[0]}, match);
* }
* });
* lexer.consume(1);
* lexer.consume(1, '*');
* ```
* @name .set
* @param {String} `type`
* @param {Function} `fn` The handler function to register.
* @api public
*/
set(type, handler) {
// preserve order of registered handlers
union(this, 'types', type);
// late binding (doesn't work with fat arrows)
const lexer = this;
this.handlers[type] = function() {
return handler.call(lexer);
};
return this;
}
/**
* Get the registered lexer handler function of the given `type`.
* If a handler is not found, an error is thrown.
*
* ```js
* const handler = lexer.get('text');
* ```
* @name .get
* @param {String} `type`
* @param {Function} `fn` The handler function to register.
* @api public
*/
get(type) {
const handler = this.handlers[type];
assert.equal(typeof handler, 'function', 'expected a function');
return handler;
}
/**
* Removes the given `string` from the beginning of `lexer.string` and
* adds it to the end of `lexer.consumed`, then updates `lexer.line`
* and `lexer.column` with the current cursor position.
*
* ```js
* lexer.consume('*');
* ```
* @name .consume
* @param {String} `string`
* @return {Object} Returns the instance for chaining.
* @param {Number} `len`
* @param {String} `value` Optionally pass the value being consumed.
* @return {String} Returns the consumed value
* @api public
*/
consume(val, len = val.length) {
assert(typeof val === 'string', 'expected a string');
this.updateLocation(val, len);
consume(len, value) {
if (!value) value = this.string.slice(0, len);
this.consumed += value;
this.string = this.string.slice(len);
this.consumed += val;
return this;
this.updateLocation(value, len);
return value;
}
/**
* Update column and line number based on `val`.
* @param {String} `val`
* Update column and line number based on `value`.
* @param {String} `value`
* @return {Object} returns the instance for chaining.
*/
updateLocation(val, len = val.length) {
assert(typeof val === 'string', 'expected a string');
const lines = val.match(/\n/g);
if (lines) this.loc.line += lines.length;
const index = val.lastIndexOf('\n');
this.loc.column = ~index ? len - index : this.loc.column + len;
updateLocation(value, len) {
const i = value.lastIndexOf('\n');
this.loc.column = ~i ? len - i : this.loc.column + len;
this.loc.line += Math.max(0, value.split('\n').length - 1);
this.loc.index += len;
return this;
}

@@ -211,2 +169,3 @@

match(regex) {
assert.equal(typeOf(regex), 'regexp', 'expected a regular expression');
if (regex.validated !== true) {

@@ -217,2 +176,3 @@ assert(/^\^/.test(regex.source), 'expected regex to start with "^"');

const consumed = this.consumed;
const match = regex.exec(this.string);

@@ -224,3 +184,4 @@ if (!match) return;

this.consume(match[0]);
match.consumed = consumed;
this.consume(match[0].length, match[0]);
return match;

@@ -239,7 +200,7 @@ }

* console.log(lexer.scan(/^\//, 'slash'));
* //=> Token { type: 'slash', val: '/' }
* //=> Token { type: 'slash', value: '/' }
* console.log(lexer.scan(/^\w+/, 'text'));
* //=> Token { type: 'text', val: 'foo' }
* //=> Token { type: 'text', value: 'foo' }
* console.log(lexer.scan(/^\//, 'slash'));
* //=> Token { type: 'slash', val: '/' }
* //=> Token { type: 'slash', value: '/' }
* ```

@@ -255,6 +216,5 @@ * @name .scan

scan(regex, type) {
const match = this.match(regex);
if (!match) return;
try {
const match = this.match(regex);
if (!match) return;
const token = this.token(type, match);

@@ -272,12 +232,12 @@ this.emit('scan', token);

* Capture a token of the specified `type` using the provide `regex`
* for scanning and matching substrings. When [.tokenize](#tokenize) is
* use, captured tokens are pushed onto the `lexer.tokens` array.
* for scanning and matching substrings. Automatically registers a handler
* when a function is passed as the last argument.
*
* ```js
* lexer.capture('text', /^\w+/);
* lexer.capture('text', /^\w+/, tok => {
* if (tok.match[1] === 'foo') {
* lexer.capture('text', /^\w+/, token => {
* if (token.value === 'foo') {
* // do stuff
* }
* return tok;
* return token;
* });

@@ -310,11 +270,11 @@ * ```

* lexer.capture('text', /^\w+/);
* console.log(lexer.lex('text'));
* console.log(lexer.handle('text'));
* //=> undefined
* console.log(lexer.lex('slash'));
* //=> { type: 'slash', val: '/' }
* console.log(lexer.lex('text'));
* //=> { type: 'text', val: 'a' }
* console.log(lexer.handle('slash'));
* //=> { type: 'slash', value: '/' }
* console.log(lexer.handle('text'));
* //=> { type: 'text', value: 'a' }
* ```
* @name .lex
* @emits lex
* @name .handle
* @emits handle
* @param {String} `type` The handler type to call on `lexer.string`

@@ -325,6 +285,7 @@ * @return {Object} Returns a token of the given `type` or undefined.

lex(type) {
handle(type) {
const token = this.get(type).call(this);
if (token) {
this.emit('lex', token);
this.current = token;
this.emit('handle', token);
return token;

@@ -344,3 +305,3 @@ }

* @name .advance
* @return {Object} Returns the first token returned by a handler.
* @return {Object} Returns the first token returned by a handler, or the first character in the remaining string if `options.mode` is set to `character`.
* @api public

@@ -351,8 +312,9 @@ */

if (!this.string) return;
for (var type of this.types) {
var tok = this.lex(type);
if (tok) {
return tok;
}
if (this.options.mode === 'character') {
return (this.current = this.consume(1));
}
for (const type of this.types) {
const token = this.handle(type);
if (token) return token;
}
this.fail();

@@ -370,7 +332,7 @@ }

* // Results in:
* // [ Token { type: 'text', val: 'a' },
* // Token { type: 'slash', val: '/' },
* // Token { type: 'text', val: 'b' },
* // Token { type: 'slash', val: '/' },
* // Token { type: 'text', val: 'c' } ]
* // [ Token { type: 'text', value: 'a' },
* // Token { type: 'slash', value: '/' },
* // Token { type: 'text', value: 'b' },
* // Token { type: 'slash', value: '/' },
* // Token { type: 'text', value: 'c' } ]
* ```

@@ -384,3 +346,3 @@ * @name .tokenize

tokenize(input) {
this.init(input);
if (input) this.init(input);
while (this.push(this.next()));

@@ -423,3 +385,3 @@ return this.tokens;

dequeue() {
return this.queue.shift();
return this.queue.length && this.queue.shift();
}

@@ -445,17 +407,2 @@

/**
* Get the current token.
*
* ```js
* const token = lexer.current();
* ```
* @name .current
* @returns {Object} Returns a token.
* @api public
*/
current() {
return this.lookbehind(1);
}
/**
* Get the previous token.

@@ -472,3 +419,3 @@ *

prev() {
return this.lookbehind(2);
return this.lookbehind(1);
}

@@ -519,3 +466,3 @@

* @name .next
* @returns {Object} Returns a token.
* @returns {Object|String} Returns a token, or (when `options.mode` is set to `character`) either gets the next character from `lexer.queue`, or consumes the next charcter in the string.
* @api public

@@ -529,3 +476,3 @@ */

/**
* Skip `n` tokens. Skipped tokens are not enqueued.
* Skip `n` tokens or characters in the string. Skipped values are not enqueued.
*

@@ -543,4 +490,20 @@ * ```js

assert.equal(typeof n, 'number', 'expected a number');
let skipped = [], tok;
while (n-- > 0 && (tok = this.next())) skipped.push(tok);
return this.skipWhile(() => n-- > 0);
}
/**
* Skip the given token `types`.
*
* ```js
* lexer.skipWhile(tok => tok.type !== 'space');
* ```
* @name .skipType
* @param {String|Array} `types` One or more token types to skip.
* @returns {Array} Returns an array if skipped tokens.
* @api public
*/
skipWhile(fn) {
const skipped = [];
while (fn.call(this, this.peek())) skipped.push(this.dequeue());
return skipped;

@@ -563,6 +526,19 @@ }

skipType(types) {
let skipped = [];
while (~arrayify(types).indexOf(this.peek().type)) {
skipped.push(this.dequeue());
}
return this.skipWhile(tok => ~arrayify(types).indexOf(tok.type));
}
/**
* Consume spaces.
*
* ```js
* lexer.skipSpaces();
* ```
* @name .skipSpaces
* @returns {String} Returned the skipped string.
* @api public
*/
skipSpaces() {
let skipped = '';
while (this.string[0] === ' ') skipped += this.consume(1);
return skipped;

@@ -572,13 +548,46 @@ }

/**
* Push a token onto `lexer.tokens`.
* Pushes the given `value` onto `lexer.stash`.
*
* ```js
* lexer.append('abc');
* lexer.append('/');
* lexer.append('*');
* lexer.append('.');
* lexer.append('js');
* console.log(lexer.stash);
* //=> ['abc', '/', '*', '.', 'js']
* ```
* @name .append
* @emits append
* @param {any} `value`
* @return {Object} Returns the Lexer instance.
* @api public
*/
append(value, enqueue) {
if (!value) return;
if (this.stash.last === '') {
this.stash[this.stash.length - 1] += value;
} else {
this.stash.push(value);
}
this.emit('append', value);
return this;
}
/**
* Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push
* `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting
* `lexer.options.append` or `token.append` to `false`.
*
* ```js
* console.log(lexer.tokens.length); // 0
* lexer.push(new Token('star', '*'));
* console.log(lexer.tokens.length); // 1
* console.log(lexer.stash) // ['*']
* ```
* @name .push
* @emits push
* @param {Object} `token`
* @return {Object} Returns the given token with updated `token.index`.
* @param {Object|String} `token`
* @return {Object} Returns the given `token`.
* @api public

@@ -589,5 +598,12 @@ */

if (!token) return;
assert(this.isToken(token), 'expected token to be an instance of Token');
if (this.options.mode !== 'character') {
assert(this.isToken(token), 'expected token to be an instance of Token');
}
this.emit('push', token);
this.tokens.push(token);
if (this.options.append !== false && token.append !== false) {
this.append(this.value(token));
}
return token;

@@ -597,5 +613,51 @@ }

/**
* Returns true when the end-of-string has been reached, and
* `lexer.queue` is empty.
* Get the last value in the given array.
*
* ```js
* console.log(lexer.last(lexer.tokens));
* ```
* @name .last
* @param {Array} `array`
* @returns {any}
* @api public
*/
last(array) {
return Array.isArray(array) ? array[array.length - 1] : null;
}
/**
* Returns true if a token with the given `type` is on the stack.
*
* ```js
* if (lexer.isInside('bracket') || lexer.isInside('brace')) {
* // do stuff
* }
* ```
* @name .isInside
* @param {String} `type` The type to check for.
* @return {Boolean}
* @api public
*/
isInside(type) {
return this.isToken(this.stack.last) && this.stack.last.type === type;
}
/**
* Returns the value of a token using the property defined on `lexer.options.value`
* or `token.value`.
*
* @name .value
* @return {String|undefined}
* @api public
*/
value(token) {
return token[this.options.value || 'value'];
}
/**
* Returns true if `lexer.string` and `lexer.queue` are empty.
*
* @name .eos

@@ -611,12 +673,16 @@ * @return {Boolean}

/**
* Throws an error when the remaining string cannot be lexed
* by the currently registered handlers.
* @param {String} `type`
* @api private
* Creates a new Lexer instance with the given options, and copy
* the handlers from the current instance to the new instance.
*
* @param {Object} `options`
* @param {Object} `parent` Optionally pass a different lexer instance to copy handlers from.
* @return {Object} Returns a new Lexer instance
* @api public
*/
fail() {
if (this.string) {
this.error(new Error('unmatched input: ' + this.string));
}
create(options, parent = this) {
const lexer = new this.constructor(options);
lexer.handlers = parent.handlers;
lexer.types = parent.types;
return lexer;
}

@@ -628,5 +694,5 @@

* ```js
* parser.set('foo', function(tok) {
* if (tok.val !== 'foo') {
* throw this.error('expected token.val to be "foo"', tok);
* lexer.set('foo', function(tok) {
* if (tok.value !== 'foo') {
* throw this.error('expected token.value to be "foo"', tok);
* }

@@ -643,6 +709,4 @@ * });

error(err) {
if (typeof err === 'string') {
err = new Error(err);
}
if (this.emit && this.hasListeners('error')) {
if (typeof err === 'string') err = new Error(err);
if (this.hasListeners('error')) {
this.emit('error', err);

@@ -655,2 +719,39 @@ } else {

/**
* Throw an error if `lexer.stack` is not empty, or when the remaining string
* cannot be lexed by the currently registered handlers.
* @api private
*/
fail() {
if (this.stack.length) {
const last = this.stack.last;
const val = last.match ? last.match[0] : last[this.options.value || 'value'];
this.error(new Error(`unclosed: "${val}"`));
}
if (this.string) {
this.error(new Error(`unmatched input: "${this.string[0]}"`));
}
}
/**
* Get or set the `Stack` constructor to use for initializing Lexer stacks.
* @name .Stack
* @api private
*/
get Stack() {
return this.options.Stack || Lexer.Stack;
}
/**
* Get or set the `Token` constructor to use when calling `lexer.token()`.
* @name .Token
* @api private
*/
get Token() {
return this.options.Token || Lexer.Token;
}
/**
* Static method that returns true if the given value is an

@@ -677,2 +778,33 @@ * instance of `snapdragon-lexer`.

/**
* Static method for getting or setting the `Stack` constructor.
*
* @name Lexer#Stack
* @api public
* @static
*/
static set Stack(Ctor) {
Stack = Ctor;
}
static get Stack() {
return Stack;
}
/**
* Static method for getting or setting the `Token` constructor, used
* by `lexer.token()` to create a new token.
*
* @name Lexer#Token
* @api public
* @static
*/
static set Token(Ctor) {
Token = Ctor;
}
static get Token() {
return Token;
}
/**
* Static method that returns true if the given value is an

@@ -699,6 +831,14 @@ * instance of `snapdragon-token`. This is a proxy to `Token#isToken`.

function arrayify(val) {
return val != null ? (Array.isArray(val) ? val : [val]) : [];
function arrayify(value) {
return Array.isArray(value) ? value : [value];
}
function define(obj, key, value) {
Reflect.defineProperty(obj, key, {configurable: false, writable: false, value: value});
}
function isString(input) {
return input && typeof input === 'string';
}
/**

@@ -705,0 +845,0 @@ * Expose `Lexer`

41

package.json
{
"name": "snapdragon-lexer",
"description": "Converts a string into an array of tokens, with useful methods for looking ahead and behind, capturing, matching, et cetera.",
"version": "1.0.0",
"homepage": "https://github.com/here-be-snapdragons/snapdragon-lexer",
"version": "2.0.0",
"homepage": "https://github.com/here-be/snapdragon-lexer",
"author": "Jon Schlinkert (https://github.com/jonschlinkert)",
"repository": "here-be-snapdragons/snapdragon-lexer",
"repository": "here-be/snapdragon-lexer",
"bugs": {
"url": "https://github.com/here-be-snapdragons/snapdragon-lexer/issues"
"url": "https://github.com/here-be/snapdragon-lexer/issues"
},

@@ -17,18 +17,19 @@ "license": "MIT",

"engines": {
"node": ">=4"
"node": ">=6"
},
"scripts": {
"test": "mocha"
"test": "nyc mocha"
},
"dependencies": {
"@sellside/emitter": "^1.0.5",
"snapdragon-token": "^2.0.0",
"union-value": "^1.0.0",
"use": "^3.0.0"
"kind-of": "^6.0.2",
"snapdragon-handlers": "^1.0.0",
"snapdragon-stack": "^1.0.1",
"snapdragon-token": "^3.0.1"
},
"devDependencies": {
"snapdragon-position": "^1.0.0",
"mocha": "^4.0.1",
"define-property": "^2.0.0",
"gulp-format-md": "^1.0.0"
"gulp-format-md": "^1.0.0",
"mocha": "^3.5.3",
"nyc": "^11.4.1",
"snapdragon-position": "^2.0.2"
},

@@ -44,6 +45,14 @@ "keywords": [

"snapdragon",
"token",
"tokenizer",
"transform"
],
"nyc": {
"reporter": [
"lcov",
"text-summary"
]
},
"verb": {
"toc": false,
"toc": "collapsible",
"layout": "default",

@@ -68,5 +77,7 @@ "tasks": [

"snapdragon-position",
"snapdragon-token"
"snapdragon-token",
"snapdragon-location",
"snapdragon-parser"
]
}
}

538

README.md

@@ -1,2 +0,2 @@

# snapdragon-lexer [![NPM version](https://img.shields.io/npm/v/snapdragon-lexer.svg?style=flat)](https://www.npmjs.com/package/snapdragon-lexer) [![NPM monthly downloads](https://img.shields.io/npm/dm/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![NPM total downloads](https://img.shields.io/npm/dt/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![Linux Build Status](https://img.shields.io/travis/here-be-snapdragons/snapdragon-lexer.svg?style=flat&label=Travis)](https://travis-ci.org/here-be-snapdragons/snapdragon-lexer)
# snapdragon-lexer [![NPM version](https://img.shields.io/npm/v/snapdragon-lexer.svg?style=flat)](https://www.npmjs.com/package/snapdragon-lexer) [![NPM monthly downloads](https://img.shields.io/npm/dm/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![NPM total downloads](https://img.shields.io/npm/dt/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![Linux Build Status](https://img.shields.io/travis/here-be/snapdragon-lexer.svg?style=flat&label=Travis)](https://travis-ci.org/here-be/snapdragon-lexer)

@@ -7,2 +7,34 @@ > Converts a string into an array of tokens, with useful methods for looking ahead and behind, capturing, matching, et cetera.

## Table of Contents
<details>
<summary><strong>Details</strong></summary>
- [Install](#install)
- [Breaking changes in v2.0!](#breaking-changes-in-v20)
- [Usage](#usage)
- [API](#api)
* [.set](#set)
* [.get](#get)
- [Properties](#properties)
* [lexer.isLexer](#lexerislexer)
* [lexer.input](#lexerinput)
* [lexer.string](#lexerstring)
* [lexer.consumed](#lexerconsumed)
* [lexer.tokens](#lexertokens)
* [lexer.stash](#lexerstash)
* [lexer.stack](#lexerstack)
* [lexer.queue](#lexerqueue)
* [lexer.loc](#lexerloc)
- [Options](#options)
* [options.source](#optionssource)
* [options.mode](#optionsmode)
* [options.value](#optionsvalue)
- [Tokens](#tokens)
- [Plugins](#plugins)
* [Plugin Conventions](#plugin-conventions)
- [About](#about)
</details>
## Install

@@ -16,2 +48,6 @@

## Breaking changes in v2.0!
Please see the [changelog](CHANGELOG.md) for details!
## Usage

@@ -32,3 +68,3 @@

### [Lexer](index.js#L26)
### [Lexer](index.js#L27)

@@ -49,5 +85,5 @@ Create a new `Lexer` with the given `options`.

### [.token](index.js#L72)
### [.token](index.js#L82)
Create a new [Token](https://github.com/here-be-snapdragons/snapdragon-token) with the given `type` and `val`.
Create a new [Token](https://github.com/here-be/snapdragon-token) with the given `type` and `value`.

@@ -57,5 +93,5 @@ **Params**

* `type` **{String|Object}**: (required) The type of token to create
* `val` **{String}**: (optional) The captured string
* `value` **{String}**: (optional) The captured string
* `match` **{Array}**: (optional) Match arguments returned from `String.match` or `RegExp.exec`
* `returns` **{Object}**: Returns an instance of [snapdragon-token](https://github.com/here-be-snapdragons/snapdragon-token)
* `returns` **{Object}**: Returns an instance of [snapdragon-token](https://github.com/here-be/snapdragon-token)

@@ -69,3 +105,3 @@ **Events**

```js
console.log(lexer.token({type: 'star', val: '*'}));
console.log(lexer.token({type: 'star', value: '*'}));
console.log(lexer.token('star', '*'));

@@ -75,5 +111,5 @@ console.log(lexer.token('star'));

### [.isToken](index.js#L92)
### [.isToken](index.js#L102)
Returns true if the given value is a [snapdragon-token](https://github.com/here-be-snapdragons/snapdragon-token) instance.
Returns true if the given value is a [snapdragon-token](https://github.com/here-be/snapdragon-token) instance.

@@ -90,13 +126,14 @@ **Params**

lexer.isToken({}); // false
lexer.isToken(new Token({type: 'star', val: '*'})); // true
lexer.isToken(new Token({type: 'star', value: '*'})); // true
```
### [.set](index.js#L114)
### [.consume](index.js#L121)
Register a lexer handler function for creating tokens by matching substrings of the given `type.`
Consume the given length from `lexer.string`. The consumed value is used to update `lexer.consumed`, as well as the current position.
**Params**
* `type` **{String}**
* `fn` **{Function}**: The handler function to register.
* `len` **{Number}**
* `value` **{String}**: Optionally pass the value being consumed.
* `returns` **{String}**: Returns the consumed value

@@ -106,42 +143,8 @@ **Example**

```js
lexer.set('star', function() {
const match = this.match(regex, type);
if (match) {
return this.token({val: match[0]}, match);
}
});
lexer.consume(1);
lexer.consume(1, '*');
```
### [.get](index.js#L139)
### [.match](index.js#L161)
Get the registered lexer handler function of the given `type`. If a handler is not found, an error is thrown.
**Params**
* `type` **{String}**
* `fn` **{Function}**: The handler function to register.
**Example**
```js
const handler = lexer.get('text');
```
### [.consume](index.js#L159)
Removes the given `string` from the beginning of `lexer.string` and adds it to the end of `lexer.consumed`, then updates `lexer.line` and `lexer.column` with the current cursor position.
**Params**
* `string` **{String}**
* `returns` **{Object}**: Returns the instance for chaining.
**Example**
```js
lexer.consume('*');
```
### [.match](index.js#L203)
Capture a substring from `lexer.string` with the given `regex`. Also validates the regex to ensure that it starts with `^` since matching should always be against the beginning of the string, and throws if the regex matches an empty string, which can cause catastrophic backtracking in some cases.

@@ -163,3 +166,3 @@

### [.scan](index.js#L243)
### [.scan](index.js#L204)

@@ -183,12 +186,12 @@ Scan for a matching substring by calling [.match()](#match) with the given `regex`. If a match is found, 1) a token of the specified `type` is created, 2) `match[0]` is used as `token.value`, and 3) the length of `match[0]` is sliced from `lexer.string` (by calling [.consume()](#consume)).

console.log(lexer.scan(/^\//, 'slash'));
//=> Token { type: 'slash', val: '/' }
//=> Token { type: 'slash', value: '/' }
console.log(lexer.scan(/^\w+/, 'text'));
//=> Token { type: 'text', val: 'foo' }
//=> Token { type: 'text', value: 'foo' }
console.log(lexer.scan(/^\//, 'slash'));
//=> Token { type: 'slash', val: '/' }
//=> Token { type: 'slash', value: '/' }
```
### [.capture](index.js#L280)
### [.capture](index.js#L240)
Capture a token of the specified `type` using the provide `regex` for scanning and matching substrings. When [.tokenize](#tokenize) is use, captured tokens are pushed onto the `lexer.tokens` array.
Capture a token of the specified `type` using the provide `regex` for scanning and matching substrings. Automatically registers a handler when a function is passed as the last argument.

@@ -206,11 +209,11 @@ **Params**

lexer.capture('text', /^\w+/);
lexer.capture('text', /^\w+/, tok => {
if (tok.match[1] === 'foo') {
lexer.capture('text', /^\w+/, token => {
if (token.value === 'foo') {
// do stuff
}
return tok;
return token;
});
```
### [.lex](index.js#L311)
### [.handle](index.js#L271)

@@ -226,3 +229,3 @@ Calls handler `type` on `lexer.string`.

* `emits`: lex
* `emits`: handle

@@ -235,15 +238,15 @@ **Example**

lexer.capture('text', /^\w+/);
console.log(lexer.lex('text'));
console.log(lexer.handle('text'));
//=> undefined
console.log(lexer.lex('slash'));
//=> { type: 'slash', val: '/' }
console.log(lexer.lex('text'));
//=> { type: 'text', val: 'a' }
console.log(lexer.handle('slash'));
//=> { type: 'slash', value: '/' }
console.log(lexer.handle('text'));
//=> { type: 'text', value: 'a' }
```
### [.advance](index.js#L333)
### [.advance](index.js#L294)
Get the next token by iterating over `lexer.handlers` and calling each handler on `lexer.string` until a handler returns a token. If no handlers return a token, an error is thrown with the substring that couldn't be lexed.
* `returns` **{Object}**: Returns the first token returned by a handler.
* `returns` **{Object}**: Returns the first token returned by a handler, or the first character in the remaining string if `options.mode` is set to `character`.

@@ -256,3 +259,3 @@ **Example**

### [.tokenize](index.js#L365)
### [.tokenize](index.js#L327)

@@ -274,10 +277,10 @@ Tokenizes a string and returns an array of tokens.

// Results in:
// [ Token { type: 'text', val: 'a' },
// Token { type: 'slash', val: '/' },
// Token { type: 'text', val: 'b' },
// Token { type: 'slash', val: '/' },
// Token { type: 'text', val: 'c' } ]
// [ Token { type: 'text', value: 'a' },
// Token { type: 'slash', value: '/' },
// Token { type: 'text', value: 'b' },
// Token { type: 'slash', value: '/' },
// Token { type: 'text', value: 'c' } ]
```
### [.enqueue](index.js#L385)
### [.enqueue](index.js#L347)

@@ -299,3 +302,3 @@ Push a token onto the `lexer.queue` array.

### [.dequeue](index.js#L403)
### [.dequeue](index.js#L365)

@@ -314,3 +317,3 @@ Shift a token from `lexer.queue`.

### [.lookbehind](index.js#L419)
### [.lookbehind](index.js#L381)

@@ -330,16 +333,4 @@ Lookbehind `n` tokens.

### [.current](index.js#L435)
### [.prev](index.js#L397)
Get the current token.
* `returns` **{Object}**: Returns a token.
**Example**
```js
const token = lexer.current();
```
### [.prev](index.js#L450)
Get the previous token.

@@ -355,3 +346,3 @@

### [.lookahead](index.js#L468)
### [.lookahead](index.js#L415)

@@ -371,3 +362,3 @@ Lookahead `n` tokens and return the last token. Pushes any intermediate tokens onto `lexer.tokens.` To lookahead a single token, use [.peek()](#peek).

### [.peek](index.js#L486)
### [.peek](index.js#L433)

@@ -384,7 +375,7 @@ Lookahead a single token.

### [.next](index.js#L501)
### [.next](index.js#L448)
Get the next token, either from the `queue` or by [advancing](#advance).
* `returns` **{Object}**: Returns a token.
* `returns` **{Object|String}**: Returns a token, or (when `options.mode` is set to `character`) either gets the next character from `lexer.queue`, or consumes the next charcter in the string.

@@ -397,5 +388,5 @@ **Example**

### [.skip](index.js#L517)
### [.skip](index.js#L464)
Skip `n` tokens. Skipped tokens are not enqueued.
Skip `n` tokens or characters in the string. Skipped values are not enqueued.

@@ -413,3 +404,3 @@ **Params**

### [.skipType](index.js#L537)
### [.skipType](index.js#L481)

@@ -421,2 +412,17 @@ Skip the given token `types`.

* `types` **{String|Array}**: One or more token types to skip.
* `returns` **{Array}**: Returns an array if skipped tokens.
**Example**
```js
lexer.skipWhile(tok => tok.type !== 'space');
```
### [.skipType](index.js#L500)
Skip the given token `types`.
**Params**
* `types` **{String|Array}**: One or more token types to skip.
* `returns` **{Array}**: Returns an array if skipped tokens

@@ -431,13 +437,50 @@

### [.push](index.js#L560)
### [.skipSpaces](index.js#L515)
Push a token onto `lexer.tokens`.
Consume spaces.
* `returns` **{String}**: Returned the skipped string.
**Example**
```js
lexer.skipSpaces();
```
### [.append](index.js#L540)
Pushes the given `value` onto `lexer.stash`.
**Params**
* `token` **{Object}**
* `returns` **{Object}**: Returns the given token with updated `token.index`.
* `value` **{any}**
* `returns` **{Object}**: Returns the Lexer instance.
**Events**
* `emits`: append
**Example**
```js
lexer.append('abc');
lexer.append('/');
lexer.append('*');
lexer.append('.');
lexer.append('js');
console.log(lexer.stash);
//=> ['abc', '/', '*', '.', 'js']
```
### [.push](index.js#L569)
Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting `lexer.options.append` or `token.append` to `false`.
**Params**
* `token` **{Object|String}**
* `returns` **{Object}**: Returns the given `token`.
**Events**
* `emits`: push

@@ -451,13 +494,61 @@

console.log(lexer.tokens.length); // 1
console.log(lexer.stash) // ['*']
```
### [.eos](index.js#L577)
### [.last](index.js#L596)
Returns true when the end-of-string has been reached, and
`lexer.queue` is empty.
Get the last value in the given array.
**Params**
* `array` **{Array}**
* `returns` **{any}**
**Example**
```js
console.log(lexer.last(lexer.tokens));
```
### [.isInside](index.js#L614)
Returns true if a token with the given `type` is on the stack.
**Params**
* `type` **{String}**: The type to check for.
* `returns` **{Boolean}**
### [.error](index.js#L611)
**Example**
```js
if (lexer.isInside('bracket') || lexer.isInside('brace')) {
// do stuff
}
```
### [.value](index.js#L627)
Returns the value of a token using the property defined on `lexer.options.value`
or `token.value`.
* `returns` **{String|undefined}**
### [.eos](index.js#L639)
Returns true if `lexer.string` and `lexer.queue` are empty.
* `returns` **{Boolean}**
Creates a new Lexer instance with the given options, and copy
the handlers from the current instance to the new instance.
**Params**
* `options` **{Object}**
* `parent` **{Object}**: Optionally pass a different lexer instance to copy handlers from.
* `returns` **{Object}**: Returns a new Lexer instance
### [.error](index.js#L677)
Throw a formatted error message with details including the cursor position.

@@ -474,5 +565,5 @@

```js
parser.set('foo', function(tok) {
if (tok.val !== 'foo') {
throw this.error('expected token.val to be "foo"', tok);
lexer.set('foo', function(tok) {
if (tok.value !== 'foo') {
throw this.error('expected token.value to be "foo"', tok);
}

@@ -482,3 +573,3 @@ });

### [Lexer#isLexer](index.js#L639)
### [Lexer#isLexer](index.js#L740)

@@ -501,4 +592,13 @@ Static method that returns true if the given value is an instance of `snapdragon-lexer`.

### [Lexer#isToken](index.js#L660)
### [Lexer#Stack](index.js#L752)
Static method for getting or setting the `Stack` constructor.
### [Lexer#Token](index.js#L768)
Static method for getting or setting the `Token` constructor, used
by `lexer.token()` to create a new token.
### [Lexer#isToken](index.js#L792)
Static method that returns true if the given value is an instance of `snapdragon-token`. This is a proxy to `Token#isToken`.

@@ -520,77 +620,182 @@

## Plugins
### .set
Pass plugins to the `lexer.use()` method.
Register a handler function.
**Params**
* `type` **{String}**
* `fn` **{Function}**: The handler function to register.
**Example**
The [snapdragon-position](https://github.com/here-be-snapdragons/snapdragon-position) plugin adds a `position` property with line and column to tokens as they're created:
```js
const position = require('snapdragon-position');
const Lexer = require('snapdragon-lexer');
const lexer = new Lexer();
lexer.set('star', function(token) {
// do parser, lexer, or compiler stuff
});
```
lexer.use(position());
lexer.capture('slash', /^\//);
lexer.capture('text', /^\w+/);
lexer.capture('star', /^\*/);
As an alternative to `.set`, the [.capture](#capture) method will automatically register a handler when a function is passed as the last argument.
// "advance" captures a single token
console.log(lexer.advance());
console.log(lexer.advance());
console.log(lexer.advance());
```
### .get
Results in:
Get a registered handler function.
**Params**
* `type` **{String}**
* `fn` **{Function}**: The handler function to register.
**Example**
```js
Token {
type: 'text',
val: 'foo',
match: [ 'foo', index: 0, input: 'foo/*' ],
position: {
start: { index: 0, column: 1, line: 1 },
end: { index: 3, column: 4, line: 1 }
}
}
lexer.set('star', function() {
// do parser, lexer, or compiler stuff
});
const star = handlers.get('star');
```
Token {
type: 'slash',
val: '/',
match: [ '/', index: 0, input: '/*' ],
position: {
start: { index: 3, column: 4, line: 1 },
end: { index: 4, column: 5, line: 1 }
}
}
## Properties
Token {
type: 'star',
val: '*',
match: [ '*', index: 0, input: '*' ],
position: {
start: { index: 4, column: 5, line: 1 },
end: { index: 5, column: 6, line: 1 }
}
}
### lexer.isLexer
Type: **{boolean}**
Default: `true` (contant)
This property is defined as a convenience, to make it easy for plugins to check for an instance of Lexer.
### lexer.input
Type: **{string}**
Default: `''`
The unmodified source string provided by the user.
### lexer.string
Type: **{string}**
Default: `''`
The source string minus the part of the string that has already been [consumed](#consume).
### lexer.consumed
Type: **{string}**
Default: `''`
The part of the source string that has been consumed.
### lexer.tokens
Type: **{array}**
Default: `[]
Array of lexed tokens.
### lexer.stash
Type: **{array}**
Default: `['']`
Array of captured strings. Similar to the [lexer.tokens](#lexertokens) array, but stores strings instead of token objects.
### lexer.stack
Type: **{array}**
Default: `[]
LIFO (last in, first out) array. A token is pushed onto the stack when an "opening" character or character sequence needs to be tracked. When the (matching) "closing" character or character sequence is encountered, the (opening) token is popped off of the stack.
The stack is not used by any lexer methods, it's reserved for the user. Stacks are necessary for creating Abstract Syntax Trees (ASTs), but if you require this functionality it would be better to use a parser such as [snapdragon-parser][snapdragon-parser], with methods and other conveniences for creating an AST.
### lexer.queue
Type: **{array}**
Default: `[]
FIFO (first in, first out) array, for temporarily storing tokens that are created when [.lookahead()](#lookahead) is called (or a method that calls `.lookhead()`, such as [.peek()](#peek)).
Tokens are [dequeued](#dequeue) when [.next()](#next) is called.
### lexer.loc
Type: **{Object}**
Default: `{ index: 0, column: 0, line: 1 }`
The updated source string location with the following properties.
* `index` - 0-index
* `column` - 0-index
* `line` - 1-index
The following plugins are available for automatically updating tokens with the location:
* [snapdragon-location](https://github.com/here-be/snapdragon-location)
* [snapdragon-position](https://github.com/here-be/snapdragon-position)
## Options
### options.source
Type: **{string}**
Default: `undefined`
The source of the input string. This is typically a filename or file path, but can also be `'string'` if a string or buffer is provided directly.
If `lexer.input` is undefined, and `options.source` is a string, the lexer will attempt to set `lexer.input` by calling `fs.readFileSync()` on the value provided on `options.source`.
### options.mode
Type: **{string}**
Default: `undefined`
If `options.mode` is `character`, instead of calling handlers (which match using regex) the [.advance()](advance) method will [consume](#consume) and return one character at a time.
### options.value
Type: **{string}**
Default: `undefined`
Specify the token property to use when the [.push](#push) method pushes a value onto [lexer.stash](#lexerstash). The logic works something like this:
```js
lexer.append(token[lexer.options.value || 'value']);
```
## Tokens
See the [snapdragon-token](https://github.com/here-be/snapdragon-token) documentation for more details.
## Plugins
Plugins are registered with the `lexer.use()` method and use the following conventions.
### Plugin Conventions
Plugins are just functions that take an instance of snapdragon-lexer. However, it's recommended that you wrap your plugin function in a function that takes an options object, to allow users to pass options when using the plugin. _Even if your plugin doesn't take options, it's a best practice for users to always be able to use the same signature_.
Plugins are functions that take an instance of snapdragon-lexer.
However, it's recommended that you always wrap your plugin function in another function that takes an options object. This allow users to pass options when using the plugin. _Even if your plugin doesn't take options, it's a best practice for users to always be able to use the same signature_.
**Example**
```js
const Lexer = require('snapdragon-lexer');
const lexer = new Lexer();
function yourPlugin(options) {
function plugin(options) {
return function(lexer) {
// do stuff to lexer
// do stuff
};
}
lexer.use(yourPlugin());
lexer.use(plugin());
```

@@ -619,3 +824,2 @@

</details>
<details>

@@ -639,4 +843,4 @@ <summary><strong>Building docs</strong></summary>

* [snapdragon-node](https://www.npmjs.com/package/snapdragon-node): Snapdragon utility for creating a new AST node in custom code, such as plugins. | [homepage](https://github.com/jonschlinkert/snapdragon-node "Snapdragon utility for creating a new AST node in custom code, such as plugins.")
* [snapdragon-position](https://www.npmjs.com/package/snapdragon-position): Snapdragon util and plugin for patching the position on an AST node. | [homepage](https://github.com/here-be-snapdragons/snapdragon-position "Snapdragon util and plugin for patching the position on an AST node.")
* [snapdragon-token](https://www.npmjs.com/package/snapdragon-token): Create a snapdragon token. Used by the snapdragon lexer, but can also be used by… [more](https://github.com/here-be-snapdragons/snapdragon-token) | [homepage](https://github.com/here-be-snapdragons/snapdragon-token "Create a snapdragon token. Used by the snapdragon lexer, but can also be used by plugins.")
* [snapdragon-position](https://www.npmjs.com/package/snapdragon-position): Snapdragon util and plugin for patching the position on an AST node. | [homepage](https://github.com/here-be/snapdragon-position "Snapdragon util and plugin for patching the position on an AST node.")
* [snapdragon-token](https://www.npmjs.com/package/snapdragon-token): Create a snapdragon token. Used by the snapdragon lexer, but can also be used by… [more](https://github.com/here-be/snapdragon-token) | [homepage](https://github.com/here-be/snapdragon-token "Create a snapdragon token. Used by the snapdragon lexer, but can also be used by plugins.")

@@ -653,3 +857,3 @@ ### Author

Copyright © 2017, [Jon Schlinkert](https://github.com/jonschlinkert).
Copyright © 2018, [Jon Schlinkert](https://github.com/jonschlinkert).
Released under the [MIT License](LICENSE).

@@ -659,2 +863,2 @@

_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.6.0, on November 30, 2017._
_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.6.0, on January 08, 2018._
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc