snapdragon-lexer - npm Package Compare versions

Comparing version 3.1.0 to 4.0.0

index.js

 'use strict';
 
-/**
- * Module dependencies
- */
-
-const fs = require('fs');
+const Events = require('events');
 const assert = require('assert');
-const typeOf = require('kind-of');
-const Handlers = require('snapdragon-handlers');
-let Stack = require('snapdragon-stack');
-let Token = require('snapdragon-token');
+const State = require('./lib/state');
+const Token = require('./lib/token');
+const Location = require('./lib/location');
+const { Position } = Location;
 
@@ -22,52 +18,127 @@ /**
  * @name Lexer
- * @param {String|Object} `input` (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
- * @param {Object} `options`
+ * @param {string|Object} `input` (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
+ * @param {object} `options`
  * @api public
  */
 
-class Lexer extends Handlers {
-  constructor(input, options) {
+class Lexer extends Events {
+  constructor(input, options = {}) {
+    super();
+
     if (typeof input !== 'string') {
-      options = input;
+      options = input || {};
       input = '';
     }
-    super(options);
-    if (Lexer.isLexer(options)) {
-      return this.create(options.options, options);
+
+    this.options = { ...options };
+    this.handlers = new Map();
+    this.types = new Set();
+    this.state = new State(input);
+
+    if (options.handlers) {
+      for (const [type, handler] of options.handlers) {
+        this.handler(type, handler);
+      }
     }
+  }
 
-    define(this, 'isLexer', true);
-    this.string = '';
-    this.input = '';
-    this.init(input);
+  /**
+   * Returns true if we are still at the beginning-of-string, and
+   * no part of the string has been consumed.
+   *
+   * @name .bos
+   * @return {boolean}
+   * @api public
+   */
+
+  bos() {
+    return !this.state.consumed;
   }
 
   /**
-   * Initialize lexer state properties
+   * Returns true if `lexer.string` and `lexer.queue` are empty.
+   *
+   * @name .eos
+   * @return {boolean}
+   * @api public
    */
 
-  init(input) {
-    if (input) this.input = this.string = input.toString();
-    if (!input && isString(this.options.source)) {
-      return this.init(fs.readFileSync(this.options.source));
-    }
+  eos() {
+    return this.state.string === '' && this.state.queue.length === 0;
+  }
 
-    this.consumed = '';
-    this.tokens = new this.Stack();
-    this.stack = new this.Stack();
-    this.stash = new this.Stack('');
+  /**
+   * Register a handler function.
+   *
+   * ```js
+   * lexer.set('star', function() {
+   *   // do parser, lexer, or compiler stuff
+   * });
+   * ```
+   * @name .set
+   * @param {string} `type`
+   * @param {function} `fn` The handler function to register.
+   * @api public
+   */
 
-    define(this.stash, 'append', function(val) {
-      this[this.length - 1] += val;
+  set(type, handler = tok => tok) {
+    this.types.add(type);
+    const lexer = this;
+    // can't do fat arrow here, we need to ensure that the handler
+    // context is always correct whether handlers are called directly
+    // or re-registered on a new instance, etc.
+    this.handlers.set(type, function(...args) {
+      let ctx = this || lexer;
+      let loc = ctx.location();
+      let tok = handler.call(ctx, ...args);
+      if (tok && isObject(tok) && !Token.isToken(tok)) {
+        tok = ctx.token(tok);
+      }
+      if (Token.isToken(tok) && !tok.type) {
+        tok.type = type;
+      }
+      return Token.isToken(tok) ? loc(tok) : tok;
     });
+    return this;
+  }
 
-    this.queue = [];
-    this.loc = {
-      index: 0,
-      column: 0,
-      line: 1
-    };
+  /**
+   * Get a registered handler function.
+   *
+   * ```js
+   * lexer.set('star', function() {
+   *   // do lexer stuff
+   * });
+   * const star = lexer.get('star');
+   * ```
+   * @name .get
+   * @param {string} `type`
+   * @param {function} `fn` The handler function to register.
+   * @api public
+   */
+
+  get(type) {
+    let handler = this.handlers.get(type) || this.handlers.get('unknown');
+    assert(handler, `expected handler "${type}" to be a function`);
+    return handler;
   }
 
   /**
+   * Returns true if the lexer has a registered handler of the given `type`.
+   *
+   * ```js
+   * lexer.set('star', function() {});
+   * console.log(lexer.has('star')); // true
+   * ```
+   * @name .has
+   * @param {string} type
+   * @return {boolean}
+   * @api public
+   */
+
+  has(type) {
+    return this.handlers.has(type);
+  }
+
+  /**
    * Create a new [Token][snapdragon-token] with the given `type` and `value`.
@@ -82,5 +153,5 @@ *
    * @emits token
-   * @param {String|Object} `type` (required) The type of token to create
-   * @param {String} `value` (optional) The captured string
-   * @param {Array} `match` (optional) Match arguments returned from `String.match` or `RegExp.exec`
+   * @param {string|Object} `type` (required) The type of token to create
+   * @param {string} `value` (optional) The captured string
+   * @param {array} `match` (optional) Match results from `String.match()` or `RegExp.exec()`
    * @return {Object} Returns an instance of [snapdragon-token][]
@@ -91,3 +162,3 @@ * @api public
   token(type, value, match) {
-    const token = new this.Token(type, value, match);
+    let token = new Token(type, value, match);
     this.emit('token', token);
@@ -106,4 +177,4 @@ return token;
    * @name .isToken
-   * @param {Object} `token`
-   * @return {Boolean}
+   * @param {object} `token`
+   * @return {boolean}
    * @api public
@@ -113,3 +184,3 @@ */
   isToken(token) {
-    return this.Token.isToken(token);
+    return Token.isToken(token);
   }
@@ -119,3 +190,3 @@
    * Consume the given length from `lexer.string`. The consumed value is used
-   * to update `lexer.consumed`, as well as the current position.
+   * to update `lexer.state.consumed`, as well as the current position.
    *
@@ -127,4 +198,4 @@ * ```js
    * @name .consume
-   * @param {Number} `len`
-   * @param {String} `value` Optionally pass the value being consumed.
+   * @param {number} `len`
+   * @param {string} `value` Optionally pass the value being consumed.
    * @return {String} Returns the consumed value
@@ -134,6 +205,5 @@ * @api public
 
-  consume(len, value) {
-    if (!value) value = this.string.slice(0, len);
-    this.consumed += value;
-    this.string = this.string.slice(len);
+  consume(len, value = this.state.string.slice(0, len)) {
+    this.state.consumed += value;
+    this.state.string = this.state.string.slice(len);
     this.updateLocation(value, len);
@@ -145,15 +215,33 @@ return value;
    * Update column and line number based on `value`.
-   * @param {String} `value`
+   *
+   * @param {string} `value`
    * @return {undefined}
    */
 
-  updateLocation(value, len) {
-    if (typeof len !== 'number') len = value.length;
-    const i = value.lastIndexOf('\n');
-    this.loc.column = ~i ? len - i : this.loc.column + len;
-    this.loc.line += Math.max(0, value.split('\n').length - 1);
-    this.loc.index += len;
+  updateLocation(value, len = value.length) {
+    let i = value.lastIndexOf('\n');
+    this.state.loc.column = ~i ? len - i : this.state.loc.column + len;
+    this.state.loc.line += Math.max(0, value.split('\n').length - 1);
+    this.state.loc.index += len;
   }
 
   /**
+   * Returns a function for updating a token with lexer
+   * location information.
+   *
+   * @return {function}
+   * @api public
+   */
+
+  location() {
+    let start = new Position(this);
+
+    return token => {
+      let end = new Position(this);
+      define(token, 'loc', new Location(start, end, this));
+      return token;
+    };
+  }
+
+  /**
    * Use the given `regex` to match a substring from `lexer.string`. Also validates
@@ -171,3 +259,3 @@ * the regex to ensure that it starts with `^` since matching should always be
    * @name .match
-   * @param {RegExp} `regex` (required)
+   * @param {regExp} `regex` (required)
    * @return {Array|null} Returns the match array from `RegExp.exec` or null.
@@ -178,10 +266,11 @@ * @api public
   match(regex) {
-    assert.equal(typeOf(regex), 'regexp', 'expected a regular expression');
+    assert(regex instanceof RegExp, 'expected a regular expression');
+
     if (regex.validated !== true) {
-      assert(/^\^/.test(regex.source), 'expected regex to start with "^"');
+      assert(regex.source[0] === '^', 'expected regex to start with "^"');
       regex.validated = true;
     }
 
-    const consumed = this.consumed;
-    const match = regex.exec(this.string);
+    let consumed = this.state.consumed;
+    let match = regex.exec(this.state.string);
     if (!match) return null;
@@ -193,2 +282,3 @@
 
+    this.emit('match', match);
     define(match, 'consumed', consumed);
@@ -217,4 +307,4 @@ this.consume(match[0].length, match[0]);
    * @emits scan
-   * @param {String} `type`
-   * @param {RegExp} `regex`
+   * @param {string} `type`
+   * @param {regExp} `regex`
    * @return {Object} Returns a token if a match is found, otherwise undefined.
@@ -226,11 +316,12 @@ * @api public
     try {
-      const match = this.match(regex);
-      if (!match) return;
-      const token = this.token(type, match);
-      this.emit('scan', token);
-      return token;
+      let match = this.match(regex);
+      if (match) {
+        let tok = this.token(type, match[0], match);
+        this.emit('scan', tok);
+        return tok;
+      }
     } catch (err) {
       err.regex = regex;
       err.type = type;
-      this.error(err);
+      throw err;
     }
@@ -254,5 +345,5 @@ }
    * @name .capture
-   * @param {String} `type` (required) The type of token being captured.
-   * @param {RegExp} `regex` (required) The regex for matching substrings.
-   * @param {Function} `fn` (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
+   * @param {string} `type` (required) The type of token being captured.
+   * @param {regExp} `regex` (required) The regex for matching substrings.
+   * @param {function} `fn` (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
    * @return {Object}
@@ -263,3 +354,3 @@ * @api public
   capture(type, regex, fn) {
-    this.set(type, function() {
+    let handler = function() {
       let token = this.scan(regex, type);
@@ -269,3 +360,4 @@ if (token) {
       }
-    });
+    };
+    this.set(type, handler);
     return this;
@@ -290,3 +382,3 @@ }
    * @emits handle
-   * @param {String} `type` The handler type to call on `lexer.string`
+   * @param {string} `type` The handler type to call on `lexer.string`
    * @return {Object} Returns a token of the given `type` or undefined.
@@ -297,3 +389,3 @@ * @api public
   handle(type) {
-    const token = this.get(type).call(this);
+    let token = this.get(type).call(this);
     if (token) {
@@ -321,12 +413,12 @@ this.current = token;
   advance() {
-    if (!this.string) return;
+    if (this.eos()) return;
     if (this.options.mode === 'character') {
       return (this.current = this.consume(1));
     }
-
-    for (const type of this.types) {
-      const token = this.handle(type);
-      if (token) return token;
+    for (let type of this.types) {
+      let token = this.handle(type);
+      if (token) {
+        return token;
+      }
     }
-
     this.fail();
@@ -339,5 +431,6 @@ }
    * ```js
+   * let lexer = new Lexer({ handlers: otherLexer.handlers })
    * lexer.capture('slash', /^\//);
    * lexer.capture('text', /^\w+/);
-   * const tokens = lexer.tokenize('a/b/c');
+   * const tokens = lexer.lex('a/b/c');
    * console.log(tokens);
@@ -351,4 +444,4 @@ * // Results in:
    * ```
-   * @name .tokenize
-   * @param {String} `input` The string to tokenize.
+   * @name .lex
+   * @param {string} `input` The string to lex.
    * @return {Array} Returns an array of tokens.
@@ -358,8 +451,13 @@ * @api public
 
-  tokenize(input) {
-    if (input) this.init(input);
+  lex(input, options) {
+    if (options) this.options = { ...options };
+    if (input) this.state = new State(input);
     while (this.push(this.next()));
-    return this.tokens;
+    return this.state.tokens;
   }
 
+  tokenize(...args) {
+    return this.lex(...args);
+  }
+
   /**
@@ -374,3 +472,3 @@ * Push a token onto the `lexer.queue` array.
    * @name .enqueue
-   * @param {Object} `token`
+   * @param {object} `token`
    * @return {Object} Returns the given token with updated `token.index`.
@@ -381,4 +479,3 @@ * @api public
   enqueue(token) {
-    if (!token) return;
-    this.queue.push(token);
+    token && this.state.queue.push(token);
     return token;
@@ -401,3 +498,3 @@ }
   dequeue() {
-    return this.queue.length && this.queue.shift();
+    return this.state.queue.length && this.state.queue.shift();
   }
@@ -412,3 +509,3 @@
    * @name .lookbehind
-   * @param {Number} `n`
+   * @param {number} `n`
    * @return {Object}
@@ -419,7 +516,8 @@ * @api public
   lookbehind(n) {
-    return this.tokens.lookbehind(n);
+    assert(Number.isInteger(n), 'expected a positive integer');
+    return this.state.tokens[this.state.tokens.length - n];
   }
 
   /**
-   * Get the previous token.
+   * Get the previously lexed token.
    *
@@ -430,3 +528,3 @@ * ```js
    * @name .prev
-   * @returns {Object} Returns a token.
+   * @returns {Object|undefined} Returns a token or undefined.
    * @api public
@@ -448,3 +546,3 @@ */
    * @name .lookahead
-   * @param {Number} `n`
+   * @param {number} `n`
    * @return {Object}
@@ -455,6 +553,6 @@ * @api public
   lookahead(n) {
-    assert.equal(typeof n, 'number', 'expected a number');
-    let fetch = n - this.queue.length;
+    assert(Number.isInteger(n), 'expected a positive integer');
+    let fetch = n - this.state.queue.length;
     while (fetch-- > 0 && this.enqueue(this.advance()));
-    return this.queue[--n];
+    return this.state.queue[--n];
   }
@@ -469,3 +567,3 @@
    * @name .peek
-   * @return {Object} `token`
+   * @return {Object} Returns a token.
    * @api public
@@ -500,4 +598,4 @@ */
    * @name .skip
-   * @param {Number} `n`
-   * @returns {Object} returns the very last lexed/skipped token.
+   * @param {number} `n`
+   * @returns {Object} returns an array of skipped tokens.
    * @api public
@@ -512,3 +610,3 @@ */
   /**
-   * Skip the given token `types`.
+   * Skip tokens while the given `fn` returns true.
    *
@@ -518,4 +616,4 @@ * ```js
    * ```
-   * @name .skipType
-   * @param {String|Array} `types` One or more token types to skip.
+   * @name .skipWhile
+   * @param {function} `fn` Return true if a token should be skipped.
    * @returns {Array} Returns an array if skipped tokens.
@@ -525,3 +623,3 @@ * @api public
 
-  skipWhile(fn) {
+  skipWhile(fn = !this.eos()) {
     const skipped = [];
@@ -539,3 +637,3 @@ while (fn.call(this, this.peek())) skipped.push(this.next());
    * @name .skipType
-   * @param {String|Array} `types` One or more token types to skip.
+   * @param {string|Array} `types` One or more token types to skip.
    * @returns {Array} Returns an array if skipped tokens.
@@ -546,3 +644,3 @@ * @api public
   skipTo(type) {
-    return this.skipWhile(tok => tok.type !== type).concat(this.next());
+    return this.skipWhile(tok => tok && tok.type !== type).concat(this.next());
   }
@@ -558,3 +656,3 @@
    * @name .skipType
-   * @param {String|Array} `types` One or more token types to skip.
+   * @param {string|Array} `types` One or more token types to skip.
    * @returns {Array} Returns an array if skipped tokens
@@ -565,36 +663,6 @@ * @api public
   skipType(types) {
-    return this.skipWhile(tok => ~arrayify(types).indexOf(tok.type));
+    return this.skipWhile(tok => [].concat(types).includes(tok.type));
   }
 
   /**
-   * Pushes the given `value` onto `lexer.stash`.
-   *
-   * ```js
-   * lexer.append('abc');
-   * lexer.append('/');
-   * lexer.append('*');
-   * lexer.append('.');
-   * lexer.append('js');
-   * console.log(lexer.stash);
-   * //=> ['abc', '/', '*', '.', 'js']
-   * ```
-   * @name .append
-   * @emits append
-   * @param {any} `value`
-   * @return {Object} Returns the Lexer instance.
-   * @api public
-   */
-
-  append(value, enqueue) {
-    if (!value) return;
-    if (this.stash.last() === '') {
-      this.stash.append(value);
-    } else {
-      this.stash.push(value);
-    }
-    this.emit('append', value);
-    return this;
-  }
-
-  /**
    * Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push
@@ -612,3 +680,3 @@ * `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting
    * @emits push
-   * @param {Object|String} `token`
+   * @param {object|String} `token`
    * @return {Object} Returns the given `token`.
@@ -619,3 +687,3 @@ * @api public
   push(token) {
-    if (!token) return;
+    if (!token && token !== '') return;
     if (this.options.mode !== 'character') {
@@ -626,7 +694,13 @@ assert(this.isToken(token), 'expected token to be an instance of Token');
     this.emit('push', token);
-    this.tokens.push(token);
+    this.state.tokens.push(token);
 
-    if (this.options.append !== false && token.append !== false) {
-      this.append(this.value(token));
+    if (this.options.stash === false || token.stash === false) {
+      return token;
     }
+
+    if (this.options.mode === 'character') {
+      this.append(token);
+    } else {
+      this.append(token.value);
+    }
     return token;
@@ -636,63 +710,52 @@ }
   /**
-   * Returns true if a token with the given `type` is on the stack.
+   * Append a string to the last element on `lexer.stash`, or push the
+   * string onto the stash if no elements exist.
    *
    * ```js
-   * if (lexer.isInside('bracket') || lexer.isInside('brace')) {
-   *   // do stuff
-   * }
+   * const stack = new Stack();
+   * stack.push('a');
+   * stack.push('b');
+   * stack.push('c');
+   * stack.append('_foo');
+   * stack.append('_bar');
+   * console.log(stack);
+   * //=> Stack ['a', 'b', 'c_foo_bar']
    * ```
-   * @name .isInside
-   * @param {String} `type` The type to check for.
-   * @return {Boolean}
+   * @name .append
+   * @param {String} `value`
+   * @return {String} Returns the last value in the array.
    * @api public
    */
 
-  isInside(type) {
-    const last = this.stack.last();
-    return this.isToken(last) && last.type === type;
+  append(value) {
+    if (typeof value !== 'string') return;
+    let n = this.state.stash.length - 1;
+    if (this.state.stash[n] === '') {
+      this.state.stash[n] += value;
+    } else {
+      this.state.stash.push(value);
+    }
+    this.emit('append', value);
+    return this;
   }
 
   /**
-   * Returns the value of a token using the property defined on `lexer.options.value`
-   * or `token.value`.
+   * Returns true if a token with the given `type` is on the stack.
    *
-   * @name .value
-   * @return {String|undefined}
+   * ```js
+   * if (lexer.isInside('bracket') || lexer.isInside('brace')) {
+   *   // do stuff
+   * }
+   * ```
+   * @name .isInside
+   * @param {string} `type` The type to check for.
+   * @return {boolean}
    * @api public
    */
 
-  value(token) {
-    return token[this.options.value || 'value'];
+  isInside(type) {
+    return this.state.stack.some(tok => tok.type === type);
   }
 
   /**
-   * Returns true if `lexer.string` and `lexer.queue` are empty.
-   *
-   * @name .eos
-   * @return {Boolean}
-   * @api public
-   */
-
-  eos() {
-    return this.string.length === 0 && this.queue.length === 0;
-  }
-
-  /**
-   * Creates a new Lexer instance with the given options, and copy
-   * the handlers from the current instance to the new instance.
-   *
-   * @param {Object} `options`
-   * @param {Object} `parent` Optionally pass a different lexer instance to copy handlers from.
-   * @return {Object} Returns a new Lexer instance
-   * @api public
-   */
-
-  create(options, parent = this) {
-    const lexer = new this.constructor(options);
-    lexer.handlers = parent.handlers;
-    lexer.types = parent.types;
-    return lexer;
-  }
-
-  /**
    * Throw a formatted error message with details including the cursor position.
@@ -703,3 +766,3 @@ *
    *   if (tok.value !== 'foo') {
-   *     throw this.error('expected token.value to be "foo"', tok);
+   *     throw this.state.error('expected token.value to be "foo"', tok);
    *   }
@@ -709,4 +772,4 @@ * });
    * @name .error
-   * @param {String} `msg` Message to use in the Error.
-   * @param {Object} `node`
+   * @param {string} `msg` Message to use in the Error.
+   * @param {object} `node`
    * @return {undefined}
@@ -718,3 +781,3 @@ * @api public
     if (typeof err === 'string') err = new Error(err);
-    if (this.hasListeners('error')) {
+    if (this.listenerCount('error') > 0) {
       this.emit('error', err);
@@ -733,9 +796,10 @@ } else {
   fail() {
-    if (this.stack.length) {
-      const last = this.stack.last();
-      const val = last.match ? last.match[0] : last[this.options.value || 'value'];
-      this.error(new Error(`unclosed: "${val}"`));
+    let token = this.state.stack.pop();
+    if (token) {
+      const match = token && token.match;
+      const value = match ? match[0] : token[this.options.value || 'value'];
+      throw new Error(`unclosed: "${value}"`);
     }
-    if (this.string) {
-      this.error(new Error(`unmatched input: "${this.string.slice(0, 10)}"`));
+    if (this.state.string) {
+      throw new Error(`unmatched input: "${this.state.string.slice(0, 10)}"`);
     }
@@ -745,22 +809,21 @@ }
   /**
-   * Get or set the `Stack` constructor to use for initializing Lexer stacks.
-   * @name .Stack
-   * @api private
+   * Call a plugin function on the lexer instance.
+   *
+   * ```js
+   * lexer.use(function(lexer) {
+   *   // do stuff to lexer
+   * });
+   * ```
+   * @name .use
+   * @param {function} `fn`
+   * @return {object} Returns the lexer instance.
+   * @api public
    */
 
-  get Stack() {
-    return this.options.Stack || Lexer.Stack;
+  use(fn) {
+    fn.call(this, this);
+    return this;
   }
 
   /**
-   * Get or set the `Token` constructor to use when calling `lexer.token()`.
-   * @name .Token
-   * @api private
-   */
-
-  get Token() {
-    return this.options.Token || Lexer.Token;
-  }
-
-  /**
    * Static method that returns true if the given value is an
@@ -776,3 +839,3 @@ * instance of `snapdragon-lexer`.
    * @name Lexer#isLexer
-   * @param {Object} `lexer`
+   * @param {object} `lexer`
    * @returns {Boolean}
@@ -784,9 +847,18 @@ * @api public
   static isLexer(lexer) {
-    return lexer && lexer.isLexer === true;
+    return lexer instanceof Lexer;
   }
 
   /**
-   * Static method for getting or setting the `Stack` constructor.
+   * Static method that returns true if the given value is an
+   * instance of `snapdragon-token`. This is a proxy to `Token#isToken`.
    *
-   * @name Lexer#Stack
+   * ```js
+   * const Token = require('snapdragon-token');
+   * const Lexer = require('snapdragon-lexer');
+   * console.log(Lexer.isToken(new Token({type: 'foo'}))); //=> true
+   * console.log(Lexer.isToken({})); //=> false
+   * ```
+   * @name Lexer#isToken
+   * @param {object} `lexer`
+   * @returns {Boolean}
    * @api public
@@ -796,14 +868,9 @@ * @static
 
-  static set Stack(Ctor) {
-    Stack = Ctor;
+  static isToken(token) {
+    return token instanceof Token;
   }
-  static get Stack() {
-    return Stack;
-  }
 
   /**
-   * Static method for getting or setting the `Token` constructor, used
-   * by `lexer.token()` to create a new token.
-   *
-   * @name Lexer#Token
+   * The State class, exposed as a static property.
+   * @name Lexer#State
    * @api public
@@ -813,22 +880,9 @@ * @static
 
-  static set Token(Ctor) {
-    Token = Ctor;
+  static get State() {
+    return State;
   }
-  static get Token() {
-    return Token;
-  }
 
   /**
-   * Static method that returns true if the given value is an
-   * instance of `snapdragon-token`. This is a proxy to `Token#isToken`.
-   *
-   * ```js
-   * const Token = require('snapdragon-token');
-   * const Lexer = require('snapdragon-lexer');
-   * console.log(Lexer.isToken(new Token({type: 'foo'}))); //=> true
-   * console.log(Lexer.isToken({})); //=> false
-   * ```
-   * @name Lexer#isToken
-   * @param {Object} `lexer`
-   * @returns {Boolean}
+   * The Token class, exposed as a static property.
+   * @name Lexer#Token
    * @api public
@@ -838,24 +892,28 @@ * @static
 
-  static isToken(token) {
-    return this.Token.isToken(token);
+  static get Token() {
+    return Token;
   }
-};
+}
 
-function arrayify(value) {
-  return Array.isArray(value) ? value : [value];
+/**
+ * Returns true if value is an object
+ */
+
+function isObject(val) {
+  return val && typeof val === 'object' && !Array.isArray(val);
 }
 
+/**
+ * Define a non-enumerable property on `obj`
+ */
+
 function define(obj, key, value) {
-  Reflect.defineProperty(obj, key, { configurable: false, writable: false, value: value });
+  Reflect.defineProperty(obj, key, { value });
 }
 
-function isString(input) {
-  return input && typeof input === 'string';
-}
-
 /**
  * Expose `Lexer`
- * @type {Function}
+ * @type {Class}
  */
 
 module.exports = Lexer;

package.json

 {
   "name": "snapdragon-lexer",
   "description": "Converts a string into an array of tokens, with useful methods for looking ahead and behind, capturing, matching, et cetera.",
-  "version": "3.1.0",
+  "version": "4.0.0",
   "homepage": "https://github.com/here-be/snapdragon-lexer",
@@ -13,23 +13,18 @@ "author": "Jon Schlinkert (https://github.com/jonschlinkert)",
   "files": [
-    "index.js"
+    "index.js",
+    "lib"
   ],
   "main": "index.js",
   "engines": {
-    "node": ">=6"
+    "node": ">=8"
   },
   "scripts": {
-    "test": "nyc mocha"
+    "test": "mocha",
+    "cover": "nyc --reporter=text --reporter=html mocha"
   },
-  "dependencies": {
-    "kind-of": "^6.0.2",
-    "snapdragon-handlers": "^1.0.0",
-    "snapdragon-stack": "^2.1.0",
-    "snapdragon-token": "^3.0.1"
-  },
   "devDependencies": {
     "define-property": "^2.0.2",
-    "gulp-format-md": "^1.0.0",
-    "mocha": "^3.5.3",
-    "nyc": "^11.4.1",
-    "snapdragon-position": "^2.0.2"
+    "gulp-format-md": "^2.0.0",
+    "mocha": "^5.2.0",
+    "nyc": "^13.1.0"
   },
@@ -44,13 +39,10 @@ "keywords": [
     "render",
+    "scan",
+    "scanner",
     "snapdragon",
     "token",
+    "tokenize",
     "tokenizer",
     "transform"
   ],
-  "nyc": {
-    "reporter": [
-      "lcov",
-      "text-summary"
-    ]
-  },
   "verb": {
@@ -70,5 +62,4 @@ "toc": "collapsible",
       "list": [
-        "snapdragon-node",
-        "snapdragon-position",
-        "snapdragon-token"
+        "snapdragon-parser",
+        "snapdragon-scanner"
       ]
@@ -75,0 +66,0 @@ },

README.md

@@ -66,3 +66,3 @@ # snapdragon-lexer [![NPM version](https://img.shields.io/npm/v/snapdragon-lexer.svg?style=flat)](https://www.npmjs.com/package/snapdragon-lexer) [![NPM monthly downloads](https://img.shields.io/npm/dm/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![NPM total downloads](https://img.shields.io/npm/dt/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![Linux Build Status](https://img.shields.io/travis/here-be/snapdragon-lexer.svg?style=flat&label=Travis)](https://travis-ci.org/here-be/snapdragon-lexer)
 
-### [Lexer](index.js#L27)
+### [Lexer](index.js#L23)
 
@@ -73,4 +73,4 @@ Create a new `Lexer` with the given `options`.
 
-* `input` **{String|Object}**: (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
-* `options` **{Object}**
+* `input` **{string|Object}**: (optional) Input string or options. You can also set input directly on `lexer.input` after initializing.
+* `options` **{object}**
 
@@ -84,4 +84,68 @@ **Example**
 
-### [.token](index.js#L88)
+### [.bos](index.js#L53)
 
+Returns true if we are still at the beginning-of-string, and
+no part of the string has been consumed.
+
+* `returns` **{boolean}**
+
+### [.eos](index.js#L65)
+
+Returns true if `lexer.string` and `lexer.queue` are empty.
+
+* `returns` **{boolean}**
+
+### [.set](index.js#L83)
+
+Register a handler function.
+
+**Params**
+
+* `type` **{string}**
+* `fn` **{function}**: The handler function to register.
+
+**Example**
+
+```js
+lexer.set('star', function() {
+  // do parser, lexer, or compiler stuff
+});
+```
+
+### [.get](index.js#L119)
+
+Get a registered handler function.
+
+**Params**
+
+* `type` **{string}**
+* `fn` **{function}**: The handler function to register.
+
+**Example**
+
+```js
+lexer.set('star', function() {
+  // do lexer stuff
+});
+const star = lexer.get('star');
+```
+
+### [.has](index.js#L138)
+
+Returns true if the lexer has a registered handler of the given `type`.
+
+**Params**
+
+* **{string}**: type
+* `returns` **{boolean}**
+
+**Example**
+
+```js
+lexer.set('star', function() {});
+console.log(lexer.has('star')); // true
+```
+
+### [.token](index.js#L159)
+
 Create a new [Token](https://github.com/here-be/snapdragon-token) with the given `type` and `value`.
@@ -91,5 +155,5 @@
 
-* `type` **{String|Object}**: (required) The type of token to create
-* `value` **{String}**: (optional) The captured string
-* `match` **{Array}**: (optional) Match arguments returned from `String.match` or `RegExp.exec`
+* `type` **{string|Object}**: (required) The type of token to create
+* `value` **{string}**: (optional) The captured string
+* `match` **{array}**: (optional) Match results from `String.match()` or `RegExp.exec()`
 * `returns` **{Object}**: Returns an instance of [snapdragon-token](https://github.com/here-be/snapdragon-token)
@@ -109,3 +173,3 @@
 
-### [.isToken](index.js#L108)
+### [.isToken](index.js#L179)
 
@@ -116,4 +180,4 @@ Returns true if the given value is a [snapdragon-token](https://github.com/here-be/snapdragon-token) instance.
 
-* `token` **{Object}**
-* `returns` **{Boolean}**
+* `token` **{object}**
+* `returns` **{boolean}**
 
@@ -128,10 +192,10 @@ **Example**
 
-### [.consume](index.js#L127)
+### [.consume](index.js#L198)
 
-Consume the given length from `lexer.string`. The consumed value is used to update `lexer.consumed`, as well as the current position.
+Consume the given length from `lexer.string`. The consumed value is used to update `lexer.state.consumed`, as well as the current position.
 
 **Params**
 
-* `len` **{Number}**
-* `value` **{String}**: Optionally pass the value being consumed.
+* `len` **{number}**
+* `value` **{string}**: Optionally pass the value being consumed.
 * `returns` **{String}**: Returns the consumed value
@@ -146,4 +210,9 @@
 
-### [.match](index.js#L167)
+Returns a function for updating a token with lexer
+location information.
 
+* `returns` **{function}**
+
+### [.match](index.js#L255)
+
 Use the given `regex` to match a substring from `lexer.string`. Also validates the regex to ensure that it starts with `^` since matching should always be against the beginning of the string, and throws if the regex matches an empty string, which can cause catastrophic backtracking.
@@ -153,3 +222,3 @@
 
-* `regex` **{RegExp}**: (required)
+* `regex` **{regExp}**: (required)
 * `returns` **{Array|null}**: Returns the match array from `RegExp.exec` or null.
@@ -166,3 +235,3 @@
 
-### [.scan](index.js#L211)
+### [.scan](index.js#L301)
 
@@ -173,4 +242,4 @@ Scan for a matching substring by calling [.match()](#match) with the given `regex`. If a match is found, 1) a token of the specified `type` is created, 2) `match[0]` is used as `token.value`, and 3) the length of `match[0]` is sliced from `lexer.string` (by calling [.consume()](#consume)).
 
-* `type` **{String}**
-* `regex` **{RegExp}**
+* `type` **{string}**
+* `regex` **{regExp}**
 * `returns` **{Object}**: Returns a token if a match is found, otherwise undefined.
@@ -194,3 +263,3 @@
 
-### [.capture](index.js#L247)
+### [.capture](index.js#L338)
 
@@ -201,5 +270,5 @@ Capture a token of the specified `type` using the provide `regex` for scanning and matching substrings. Automatically registers a handler when a function is passed as the last argument.
 
-* `type` **{String}**: (required) The type of token being captured.
-* `regex` **{RegExp}**: (required) The regex for matching substrings.
-* `fn` **{Function}**: (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
+* `type` **{string}**: (required) The type of token being captured.
+* `regex` **{regExp}**: (required) The regex for matching substrings.
+* `fn` **{function}**: (optional) If supplied, the function will be called on the token before pushing it onto `lexer.tokens`.
 * `returns` **{Object}**
@@ -219,3 +288,3 @@
 
-### [.handle](index.js#L278)
+### [.handle](index.js#L370)
 
@@ -226,3 +295,3 @@ Calls handler `type` on `lexer.string`.
 
-* `type` **{String}**: The handler type to call on `lexer.string`
+* `type` **{string}**: The handler type to call on `lexer.string`
 * `returns` **{Object}**: Returns a token of the given `type` or undefined.
@@ -248,3 +317,3 @@
 
-### [.advance](index.js#L301)
+### [.advance](index.js#L393)
 
@@ -261,3 +330,3 @@ Get the next token by iterating over `lexer.handlers` and calling each handler on `lexer.string` until a handler returns a token. If no handlers return a token, an error is thrown with the substring that couldn't be lexed.
 
-### [.tokenize](index.js#L336)
+### [.lex](index.js#L429)
 
@@ -268,3 +337,3 @@ Tokenizes a string and returns an array of tokens.
 
-* `input` **{String}**: The string to tokenize.
+* `input` **{string}**: The string to lex.
 * `returns` **{Array}**: Returns an array of tokens.
@@ -275,5 +344,6 @@
 ```js
+let lexer = new Lexer({ handlers: otherLexer.handlers })
 lexer.capture('slash', /^\//);
 lexer.capture('text', /^\w+/);
-const tokens = lexer.tokenize('a/b/c');
+const tokens = lexer.lex('a/b/c');
 console.log(tokens);
@@ -288,3 +358,3 @@ // Results in:
 
-### [.enqueue](index.js#L356)
+### [.enqueue](index.js#L454)
 
@@ -295,3 +365,3 @@ Push a token onto the `lexer.queue` array.
 
-* `token` **{Object}**
+* `token` **{object}**
 * `returns` **{Object}**: Returns the given token with updated `token.index`.
@@ -307,3 +377,3 @@
 
-### [.dequeue](index.js#L375)
+### [.dequeue](index.js#L472)
 
@@ -322,3 +392,3 @@ Shift a token from `lexer.queue`.
 
-### [.lookbehind](index.js#L391)
+### [.lookbehind](index.js#L488)
 
@@ -329,3 +399,3 @@ Lookbehind `n` tokens.
 
-* `n` **{Number}**
+* `n` **{number}**
 * `returns` **{Object}**
@@ -339,7 +409,7 @@
 
-### [.prev](index.js#L406)
+### [.prev](index.js#L504)
 
-Get the previous token.
+Get the previously lexed token.
 
-* `returns` **{Object}**: Returns a token.
+* `returns` **{Object|undefined}**: Returns a token or undefined.
 
@@ -352,3 +422,3 @@ **Example**
 
-### [.lookahead](index.js#L424)
+### [.lookahead](index.js#L522)
 
@@ -359,3 +429,3 @@ Lookahead `n` tokens and return the last token. Pushes any intermediate tokens onto `lexer.tokens.` To lookahead a single token, use [.peek()](#peek).
 
-* `n` **{Number}**
+* `n` **{number}**
 * `returns` **{Object}**
@@ -369,7 +439,7 @@
 
-### [.peek](index.js#L442)
+### [.peek](index.js#L540)
 
 Lookahead a single token.
 
-* `returns` **{Object}** `token`
+* `returns` **{Object}**: Returns a token.
 
@@ -382,3 +452,3 @@ **Example**
 
-### [.next](index.js#L457)
+### [.next](index.js#L555)
 
@@ -395,3 +465,3 @@ Get the next token, either from the `queue` or by [advancing](#advance).
 
-### [.skip](index.js#L473)
+### [.skip](index.js#L571)
 
@@ -402,4 +472,4 @@ Skip `n` tokens or characters in the string. Skipped values are not enqueued.
 
-* `n` **{Number}**
-* `returns` **{Object}**: returns the very last lexed/skipped token.
+* `n` **{number}**
+* `returns` **{Object}**: returns an array of skipped tokens.
 
@@ -412,9 +482,9 @@ **Example**
 
-### [.skipType](index.js#L490)
+### [.skipWhile](index.js#L588)
 
-Skip the given token `types`.
+Skip tokens while the given `fn` returns true.
 
 **Params**
 
-* `types` **{String|Array}**: One or more token types to skip.
+* `fn` **{function}**: Return true if a token should be skipped.
 * `returns` **{Array}**: Returns an array if skipped tokens.
@@ -428,3 +498,3 @@
 
-### [.skipType](index.js#L508)
+### [.skipType](index.js#L606)
 
@@ -435,3 +505,3 @@ Skip the given token `types`.
 
-* `types` **{String|Array}**: One or more token types to skip.
+* `types` **{string|Array}**: One or more token types to skip.
 * `returns` **{Array}**: Returns an array if skipped tokens.
@@ -445,3 +515,3 @@
 
-### [.skipType](index.js#L525)
+### [.skipType](index.js#L623)
 
@@ -452,3 +522,3 @@ Skip the given token `types`.
 
-* `types` **{String|Array}**: One or more token types to skip.
+* `types` **{string|Array}**: One or more token types to skip.
 * `returns` **{Array}**: Returns an array if skipped tokens
@@ -463,14 +533,14 @@
 
-### [.append](index.js#L548)
+### [.push](index.js#L645)
 
-Pushes the given `value` onto `lexer.stash`.
+Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting `lexer.options.append` or `token.append` to `false`.
 
 **Params**
 
-* `value` **{any}**
-* `returns` **{Object}**: Returns the Lexer instance.
+* `token` **{object|String}**
+* `returns` **{Object}**: Returns the given `token`.
 
 **Events**
 
-* `emits`: append
+* `emits`: push
 
@@ -480,34 +550,31 @@ **Example**
 ```js
-lexer.append('abc');
-lexer.append('/');
-lexer.append('*');
-lexer.append('.');
-lexer.append('js');
-console.log(lexer.stash);
-//=> ['abc', '/', '*', '.', 'js']
+console.log(lexer.tokens.length); // 0
+lexer.push(new Token('star', '*'));
+console.log(lexer.tokens.length); // 1
+console.log(lexer.stash) // ['*']
 ```
 
-### [.push](index.js#L577)
+### [.append](index.js#L686)
 
-Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting `lexer.options.append` or `token.append` to `false`.
+Append a string to the last element on `lexer.stash`, or push the string onto the stash if no elements exist.
 
 **Params**
 
-* `token` **{Object|String}**
-* `returns` **{Object}**: Returns the given `token`.
+* `value` **{String}**
+* `returns` **{String}**: Returns the last value in the array.
 
-**Events**
-
-* `emits`: push
-
 **Example**
 
 ```js
-console.log(lexer.tokens.length); // 0
-lexer.push(new Token('star', '*'));
-console.log(lexer.tokens.length); // 1
-console.log(lexer.stash) // ['*']
+const stack = new Stack();
+stack.push('a');
+stack.push('b');
+stack.push('c');
+stack.append('_foo');
+stack.append('_bar');
+console.log(stack);
+//=> Stack ['a', 'b', 'c_foo_bar']
 ```
 
-### [.isInside](index.js#L606)
+### [.isInside](index.js#L712)
 
@@ -518,4 +585,4 @@ Returns true if a token with the given `type` is on the stack.
 
-* `type` **{String}**: The type to check for.
-* `returns` **{Boolean}**
+* `type` **{string}**: The type to check for.
+* `returns` **{boolean}**
 
@@ -530,33 +597,30 @@ **Example**
 
-### [.value](index.js#L620)
+### [.error](index.js#L733)
 
-Returns the value of a token using the property defined on `lexer.options.value`
-or `token.value`.
+Throw a formatted error message with details including the cursor position.
 
-* `returns` **{String|undefined}**
+**Params**
 
-### [.eos](index.js#L632)
+* `msg` **{string}**: Message to use in the Error.
+* `node` **{object}**
+* `returns` **{undefined}**
 
-Returns true if `lexer.string` and `lexer.queue` are empty.
+**Example**
 
-* `returns` **{Boolean}**
+```js
+lexer.set('foo', function(tok) {
+  if (tok.value !== 'foo') {
+    throw this.state.error('expected token.value to be "foo"', tok);
+  }
+});
+```
 
-Creates a new Lexer instance with the given options, and copy
-the handlers from the current instance to the new instance.
+### [.use](index.js#L774)
 
-**Params**
+Call a plugin function on the lexer instance.
 
-* `options` **{Object}**
-* `parent` **{Object}**: Optionally pass a different lexer instance to copy handlers from.
-* `returns` **{Object}**: Returns a new Lexer instance
-
-### [.error](index.js#L670)
-
-Throw a formatted error message with details including the cursor position.
-
 **Params**
 
-* `msg` **{String}**: Message to use in the Error.
-* `node` **{Object}**
-* `returns` **{undefined}**
+* `fn` **{function}**
+* `returns` **{object}**: Returns the lexer instance.
 
@@ -566,10 +630,8 @@ **Example**
 ```js
-lexer.set('foo', function(tok) {
-  if (tok.value !== 'foo') {
-    throw this.error('expected token.value to be "foo"', tok);
-  }
+lexer.use(function(lexer) {
+  // do stuff to lexer
 });
 ```
 
-### [Lexer#isLexer](index.js#L733)
+### [Lexer#isLexer](index.js#L796)
 
@@ -580,3 +642,3 @@ Static method that returns true if the given value is an instance of `snapdragon-lexer`.
 
-* `lexer` **{Object}**
+* `lexer` **{object}**
 * `returns` **{Boolean}**
@@ -593,13 +655,4 @@
 
-### [Lexer#Stack](index.js#L745)
+### [Lexer#isToken](index.js#L817)
 
-Static method for getting or setting the `Stack` constructor.
-
-### [Lexer#Token](index.js#L761)
-
-Static method for getting or setting the `Token` constructor, used
-by `lexer.token()` to create a new token.
-
-### [Lexer#isToken](index.js#L785)
-
 Static method that returns true if the given value is an instance of `snapdragon-token`. This is a proxy to `Token#isToken`.
@@ -609,3 +662,3 @@
 
-* `lexer` **{Object}**
+* `lexer` **{object}**
 * `returns` **{Boolean}**
@@ -622,2 +675,10 @@
 
+### [Lexer#State](index.js#L828)
+
+The State class, exposed as a static property.
+
+### [Lexer#Token](index.js#L839)
+
+The Token class, exposed as a static property.
+
 ### .set
@@ -844,5 +905,3 @@
 
-* [snapdragon-node](https://www.npmjs.com/package/snapdragon-node): Snapdragon utility for creating a new AST node in custom code, such as plugins. | [homepage](https://github.com/jonschlinkert/snapdragon-node "Snapdragon utility for creating a new AST node in custom code, such as plugins.")
-* [snapdragon-position](https://www.npmjs.com/package/snapdragon-position): Snapdragon util and plugin for patching the position on an AST node. | [homepage](https://github.com/here-be/snapdragon-position "Snapdragon util and plugin for patching the position on an AST node.")
-* [snapdragon-token](https://www.npmjs.com/package/snapdragon-token): Create a snapdragon token. Used by the snapdragon lexer, but can also be used by… [more](https://github.com/here-be/snapdragon-token) | [homepage](https://github.com/here-be/snapdragon-token "Create a snapdragon token. Used by the snapdragon lexer, but can also be used by plugins.")
+* [snapdragon-scanner](https://www.npmjs.com/package/snapdragon-scanner): Easily scan a string with an object of regex patterns to produce an array of… [more](https://github.com/here-be/snapdragon-scanner) | [homepage](https://github.com/here-be/snapdragon-scanner "Easily scan a string with an object of regex patterns to produce an array of tokens. ~100 sloc.")
 
@@ -853,5 +912,5 @@ ### Author
 
-* [linkedin/in/jonschlinkert](https://linkedin.com/in/jonschlinkert)
-* [github/jonschlinkert](https://github.com/jonschlinkert)
-* [twitter/jonschlinkert](https://twitter.com/jonschlinkert)
+* [GitHub Profile](https://github.com/jonschlinkert)
+* [Twitter Profile](https://twitter.com/jonschlinkert)
+* [LinkedIn Profile](https://linkedin.com/in/jonschlinkert)
 
@@ -865,2 +924,2 @@ ### License
 
-_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.6.0, on February 16, 2018._
+_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.8.0, on November 19, 2018._

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

49085

increased by6.69%

Dependency count

0

decreased by-100%

Dev dependency count

4

decreased by-20%

Number of package files

8

increased by60%

Lines of code

834

increased by16.64%

Number of lines in readme file

877

increased by7.21%

Number of low supply chain risk alerts

1

decreased by-50%

Dependency changes

• - Removedkind-of@^6.0.2

• - Removedsnapdragon-handlers@^1.0.0

• - Removedsnapdragon-stack@^2.1.0

• - Removedsnapdragon-token@^3.0.1

• - Removed@sellside/emitter@1.2.1(transitive)

• - Removedkind-of@6.0.3(transitive)

• - Removedsnapdragon-handlers@1.0.0(transitive)

• - Removedsnapdragon-stack@2.1.0(transitive)

• - Removedsnapdragon-token@3.0.1(transitive)

• - Removeduse@3.1.1(transitive)