Socket
Socket
Sign inDemoInstall

snapdragon-lexer

Package Overview
Dependencies
Maintainers
1
Versions
5
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

snapdragon-lexer - npm Package Compare versions

Comparing version 3.0.0 to 3.1.0

65

index.js

@@ -53,6 +53,12 @@ 'use strict';

}
this.consumed = '';
this.tokens = new this.Stack();
this.stack = new this.Stack();
this.stash = new this.Stack('');
this.stack = new this.Stack();
define(this.stash, 'append', function(val) {
this[this.length - 1] += val;
});
this.queue = [];

@@ -133,6 +139,7 @@ this.loc = {

* @param {String} `value`
* @return {Object} returns the instance for chaining.
* @return {undefined}
*/
updateLocation(value, len) {
if (typeof len !== 'number') len = value.length;
const i = value.lastIndexOf('\n');

@@ -145,7 +152,6 @@ this.loc.column = ~i ? len - i : this.loc.column + len;

/**
* Capture a substring from `lexer.string` with the given `regex`. Also
* validates the regex to ensure that it starts with `^` since matching
* should always be against the beginning of the string, and throws if
* the regex matches an empty string, which can cause catastrophic
* backtracking in some cases.
* Use the given `regex` to match a substring from `lexer.string`. Also validates
* the regex to ensure that it starts with `^` since matching should always be
* against the beginning of the string, and throws if the regex matches an empty
* string, which can cause catastrophic backtracking.
*

@@ -160,3 +166,3 @@ * ```js

* @param {RegExp} `regex` (required)
* @return {Array} Returns the match arguments from `RegExp.exec` or null.
* @return {Array|null} Returns the match array from `RegExp.exec` or null.
* @api public

@@ -174,3 +180,4 @@ */

const match = regex.exec(this.string);
if (!match) return;
if (!match) return null;
if (match[0] === '') {

@@ -180,3 +187,3 @@ throw new SyntaxError('regex should not match an empty string');

match.consumed = consumed;
define(match, 'consumed', consumed);
this.consume(match[0].length, match[0]);

@@ -305,2 +312,3 @@ return match;

}
for (const type of this.types) {

@@ -310,2 +318,3 @@ const token = this.handle(type);

}
this.fail();

@@ -391,4 +400,3 @@ }

lookbehind(n) {
assert.equal(typeof n, 'number', 'expected a number');
return this.tokens[this.tokens.length - n];
return this.tokens.lookbehind(n);
}

@@ -493,3 +501,3 @@

const skipped = [];
while (fn.call(this, this.peek())) skipped.push(this.dequeue());
while (fn.call(this, this.peek())) skipped.push(this.next());
return skipped;

@@ -502,30 +510,29 @@ }

* ```js
* lexer.skipType('space');
* lexer.skipType(['newline', 'space']);
* lexer.skipWhile(tok => tok.type !== 'space');
* ```
* @name .skipType
* @param {String|Array} `types` One or more token types to skip.
* @returns {Array} Returns an array if skipped tokens
* @returns {Array} Returns an array if skipped tokens.
* @api public
*/
skipType(types) {
return this.skipWhile(tok => ~arrayify(types).indexOf(tok.type));
skipTo(type) {
return this.skipWhile(tok => tok.type !== type).concat(this.next());
}
/**
* Consume spaces.
* Skip the given token `types`.
*
* ```js
* lexer.skipSpaces();
* lexer.skipType('space');
* lexer.skipType(['newline', 'space']);
* ```
* @name .skipSpaces
* @returns {String} Returned the skipped string.
* @name .skipType
* @param {String|Array} `types` One or more token types to skip.
* @returns {Array} Returns an array if skipped tokens
* @api public
*/
skipSpaces() {
let skipped = '';
while (this.string[0] === ' ') skipped += this.consume(1);
return skipped;
skipType(types) {
return this.skipWhile(tok => ~arrayify(types).indexOf(tok.type));
}

@@ -555,3 +562,3 @@

if (this.stash.last() === '') {
this.stash[this.stash.length - 1] += value;
this.stash.append(value);
} else {

@@ -697,3 +704,3 @@ this.stash.push(value);

if (this.string) {
this.error(new Error(`unmatched input: "${this.string[0]}"`));
this.error(new Error(`unmatched input: "${this.string.slice(0, 10)}"`));
}

@@ -801,3 +808,3 @@ }

function define(obj, key, value) {
Reflect.defineProperty(obj, key, {configurable: false, writable: false, value: value});
Reflect.defineProperty(obj, key, { configurable: false, writable: false, value: value });
}

@@ -804,0 +811,0 @@

6

package.json
{
"name": "snapdragon-lexer",
"description": "Converts a string into an array of tokens, with useful methods for looking ahead and behind, capturing, matching, et cetera.",
"version": "3.0.0",
"version": "3.1.0",
"homepage": "https://github.com/here-be/snapdragon-lexer",

@@ -25,7 +25,7 @@ "author": "Jon Schlinkert (https://github.com/jonschlinkert)",

"snapdragon-handlers": "^1.0.0",
"snapdragon-stack": "^2.0.0",
"snapdragon-stack": "^2.1.0",
"snapdragon-token": "^3.0.1"
},
"devDependencies": {
"define-property": "^2.0.0",
"define-property": "^2.0.2",
"gulp-format-md": "^1.0.0",

@@ -32,0 +32,0 @@ "mocha": "^3.5.3",

82

README.md

@@ -82,3 +82,3 @@ # snapdragon-lexer [![NPM version](https://img.shields.io/npm/v/snapdragon-lexer.svg?style=flat)](https://www.npmjs.com/package/snapdragon-lexer) [![NPM monthly downloads](https://img.shields.io/npm/dm/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![NPM total downloads](https://img.shields.io/npm/dt/snapdragon-lexer.svg?style=flat)](https://npmjs.org/package/snapdragon-lexer) [![Linux Build Status](https://img.shields.io/travis/here-be/snapdragon-lexer.svg?style=flat&label=Travis)](https://travis-ci.org/here-be/snapdragon-lexer)

### [.token](index.js#L82)
### [.token](index.js#L88)

@@ -106,3 +106,3 @@ Create a new [Token](https://github.com/here-be/snapdragon-token) with the given `type` and `value`.

### [.isToken](index.js#L102)
### [.isToken](index.js#L108)

@@ -124,3 +124,3 @@ Returns true if the given value is a [snapdragon-token](https://github.com/here-be/snapdragon-token) instance.

### [.consume](index.js#L121)
### [.consume](index.js#L127)

@@ -142,5 +142,5 @@ Consume the given length from `lexer.string`. The consumed value is used to update `lexer.consumed`, as well as the current position.

### [.match](index.js#L161)
### [.match](index.js#L167)
Capture a substring from `lexer.string` with the given `regex`. Also validates the regex to ensure that it starts with `^` since matching should always be against the beginning of the string, and throws if the regex matches an empty string, which can cause catastrophic backtracking in some cases.
Use the given `regex` to match a substring from `lexer.string`. Also validates the regex to ensure that it starts with `^` since matching should always be against the beginning of the string, and throws if the regex matches an empty string, which can cause catastrophic backtracking.

@@ -150,3 +150,3 @@ **Params**

* `regex` **{RegExp}**: (required)
* `returns` **{Array}**: Returns the match arguments from `RegExp.exec` or null.
* `returns` **{Array|null}**: Returns the match array from `RegExp.exec` or null.

@@ -162,3 +162,3 @@ **Example**

### [.scan](index.js#L204)
### [.scan](index.js#L211)

@@ -189,3 +189,3 @@ Scan for a matching substring by calling [.match()](#match) with the given `regex`. If a match is found, 1) a token of the specified `type` is created, 2) `match[0]` is used as `token.value`, and 3) the length of `match[0]` is sliced from `lexer.string` (by calling [.consume()](#consume)).

### [.capture](index.js#L240)
### [.capture](index.js#L247)

@@ -213,3 +213,3 @@ Capture a token of the specified `type` using the provide `regex` for scanning and matching substrings. Automatically registers a handler when a function is passed as the last argument.

### [.handle](index.js#L271)
### [.handle](index.js#L278)

@@ -241,3 +241,3 @@ Calls handler `type` on `lexer.string`.

### [.advance](index.js#L294)
### [.advance](index.js#L301)

@@ -254,3 +254,3 @@ Get the next token by iterating over `lexer.handlers` and calling each handler on `lexer.string` until a handler returns a token. If no handlers return a token, an error is thrown with the substring that couldn't be lexed.

### [.tokenize](index.js#L327)
### [.tokenize](index.js#L336)

@@ -279,3 +279,3 @@ Tokenizes a string and returns an array of tokens.

### [.enqueue](index.js#L347)
### [.enqueue](index.js#L356)

@@ -297,3 +297,3 @@ Push a token onto the `lexer.queue` array.

### [.dequeue](index.js#L366)
### [.dequeue](index.js#L375)

@@ -312,3 +312,3 @@ Shift a token from `lexer.queue`.

### [.lookbehind](index.js#L382)
### [.lookbehind](index.js#L391)

@@ -328,3 +328,3 @@ Lookbehind `n` tokens.

### [.prev](index.js#L398)
### [.prev](index.js#L406)

@@ -341,3 +341,3 @@ Get the previous token.

### [.lookahead](index.js#L416)
### [.lookahead](index.js#L424)

@@ -357,3 +357,3 @@ Lookahead `n` tokens and return the last token. Pushes any intermediate tokens onto `lexer.tokens.` To lookahead a single token, use [.peek()](#peek).

### [.peek](index.js#L434)
### [.peek](index.js#L442)

@@ -370,3 +370,3 @@ Lookahead a single token.

### [.next](index.js#L449)
### [.next](index.js#L457)

@@ -383,3 +383,3 @@ Get the next token, either from the `queue` or by [advancing](#advance).

### [.skip](index.js#L465)
### [.skip](index.js#L473)

@@ -399,3 +399,3 @@ Skip `n` tokens or characters in the string. Skipped values are not enqueued.

### [.skipType](index.js#L482)
### [.skipType](index.js#L490)

@@ -415,3 +415,3 @@ Skip the given token `types`.

### [.skipType](index.js#L501)
### [.skipType](index.js#L508)

@@ -423,3 +423,3 @@ Skip the given token `types`.

* `types` **{String|Array}**: One or more token types to skip.
* `returns` **{Array}**: Returns an array if skipped tokens
* `returns` **{Array}**: Returns an array if skipped tokens.

@@ -429,19 +429,22 @@ **Example**

```js
lexer.skipType('space');
lexer.skipType(['newline', 'space']);
lexer.skipWhile(tok => tok.type !== 'space');
```
### [.skipSpaces](index.js#L516)
### [.skipType](index.js#L525)
Consume spaces.
Skip the given token `types`.
* `returns` **{String}**: Returned the skipped string.
**Params**
* `types` **{String|Array}**: One or more token types to skip.
* `returns` **{Array}**: Returns an array if skipped tokens
**Example**
```js
lexer.skipSpaces();
lexer.skipType('space');
lexer.skipType(['newline', 'space']);
```
### [.append](index.js#L541)
### [.append](index.js#L548)

@@ -471,3 +474,3 @@ Pushes the given `value` onto `lexer.stash`.

### [.push](index.js#L570)
### [.push](index.js#L577)

@@ -494,3 +497,3 @@ Pushes the given `token` onto `lexer.tokens` and calls [.append()](#append) to push `token.value` onto `lexer.stash`. Disable pushing onto the stash by setting `lexer.options.append` or `token.append` to `false`.

### [.isInside](index.js#L599)
### [.isInside](index.js#L606)

@@ -512,3 +515,3 @@ Returns true if a token with the given `type` is on the stack.

### [.value](index.js#L613)
### [.value](index.js#L620)

@@ -520,3 +523,3 @@ Returns the value of a token using the property defined on `lexer.options.value`

### [.eos](index.js#L625)
### [.eos](index.js#L632)

@@ -536,3 +539,3 @@ Returns true if `lexer.string` and `lexer.queue` are empty.

### [.error](index.js#L663)
### [.error](index.js#L670)

@@ -557,3 +560,3 @@ Throw a formatted error message with details including the cursor position.

### [Lexer#isLexer](index.js#L726)
### [Lexer#isLexer](index.js#L733)

@@ -576,7 +579,7 @@ Static method that returns true if the given value is an instance of `snapdragon-lexer`.

### [Lexer#Stack](index.js#L738)
### [Lexer#Stack](index.js#L745)
Static method for getting or setting the `Stack` constructor.
### [Lexer#Token](index.js#L754)
### [Lexer#Token](index.js#L761)

@@ -586,3 +589,3 @@ Static method for getting or setting the `Token` constructor, used

### [Lexer#isToken](index.js#L778)
### [Lexer#isToken](index.js#L785)

@@ -808,2 +811,3 @@ Static method that returns true if the given value is an instance of `snapdragon-token`. This is a proxy to `Token#isToken`.

</details>
<details>

@@ -845,2 +849,2 @@ <summary><strong>Building docs</strong></summary>

_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.6.0, on January 11, 2018._
_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.6.0, on February 16, 2018._
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc