lezer - npm Package Compare versions

Comparing version 0.1.1 to 0.2.0

CHANGELOG.md

@@ -0,1 +1,21 @@
+## 0.2.0 (2019-08-02)
+
+### Bug fixes
+
+Don't include lezer-tree inline in `dist/index.js`.
+
+### New features
+
+The output tree now uses tags, rather than term names, to identify its nodes.
+
+Export `Tag` data structure from lezer-tree.
+
+Support per-grammar global tag suffixes in `Parser.deserialize`.
+
+### Breaking changes
+
+Grammars no longer have ids.
+
+Removes export of `allocateGrammarID` and `TagMap`.
+
 ## 0.1.1 (2019-07-09)
@@ -2,0 +22,0 @@

dist/index.d.ts

 export { Parser, ParseOptions, ParseContext, NestedGrammar, NestedGrammarSpec } from "./parse";
 export { InputStream, Token, TokenGroup, ExternalTokenizer } from "./token";
 export { Stack } from "./stack";
-export { Tree, Subtree, TagMap, allocateGrammarID } from "lezer-tree";
+export { Tree, Subtree, Tag } from "lezer-tree";

dist/parse.d.ts

 import { Stack } from "./stack";
 import { InputStream, Token, Tokenizer, TokenGroup } from "./token";
-import { Tree, TagMap } from "lezer-tree";
+import { Tree, Tag } from "lezer-tree";
 export declare type NestedGrammar = null | Parser | ((input: InputStream, stack: Stack) => NestedGrammarSpec);
@@ -61,7 +61,6 @@ export interface NestedGrammarSpec {
 export declare class Parser {
-    readonly id: number;
     readonly states: Readonly<Uint32Array>;
     readonly data: Readonly<Uint16Array>;
     readonly goto: Readonly<Uint16Array>;
-    readonly tags: TagMap<string>;
+    readonly tags: readonly Tag[];
     readonly tokenizers: readonly Tokenizer[];
@@ -84,3 +83,3 @@ readonly nested: readonly {
     };
-    constructor(id: number, states: Readonly<Uint32Array>, data: Readonly<Uint16Array>, goto: Readonly<Uint16Array>, tags: TagMap<string>, tokenizers: readonly Tokenizer[], nested: readonly {
+    constructor(states: Readonly<Uint32Array>, data: Readonly<Uint16Array>, goto: Readonly<Uint16Array>, tags: readonly Tag[], tokenizers: readonly Tokenizer[], nested: readonly {
         name: string;
@@ -107,5 +106,2 @@ grammar: NestedGrammar;
     overrides(token: number, prev: number): boolean;
-    tagMap<T>(values: {
-        [name: string]: T;
-    }): TagMap<T>;
     withNested(spec: {
@@ -115,3 +111,3 @@ [name: string]: NestedGrammar | null;
     getName(term: number): string;
-    static deserialize(states: string, stateData: string, goto: string, tags: readonly string[], tokenData: string, tokenizers: (Tokenizer | number)[], nested: [string, null | NestedGrammar, string, number, number][], specializeTable: number, specializations: readonly {
+    static deserialize(states: string, stateData: string, goto: string, tags: readonly string[], globalTag: string | null, tokenData: string, tokenizers: (Tokenizer | number)[], nested: [string, null | NestedGrammar, string, number, number][], specializeTable: number, specializations: readonly {
         [term: string]: number;
@@ -118,0 +114,0 @@ }[], tokenPrec: number, skippedNodes: number, termNames?: {

package.json

 {
   "name": "lezer",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Incremental parser",
@@ -17,3 +17,3 @@ "main": "dist/index.js",
   "dependencies": {
-    "lezer-tree": "^0.1.0"
+    "lezer-tree": "^0.2.0"
   },
@@ -20,0 +20,0 @@ "scripts": {

rollup.config.js

@@ -7,2 +7,3 @@ import typescript from "rollup-plugin-typescript2"
   input: "./src/index.ts",
+  external: ["lezer-tree"],
   output: {
@@ -9,0 +10,0 @@ format: "cjs",

src/index.ts

 export {Parser, ParseOptions, ParseContext, NestedGrammar, NestedGrammarSpec} from "./parse"
 export {InputStream, Token, TokenGroup, ExternalTokenizer} from "./token"
 export {Stack} from "./stack"
-export {Tree, Subtree, TagMap, allocateGrammarID} from "lezer-tree"
+export {Tree, Subtree, Tag} from "lezer-tree"

src/parse.ts

 import {Stack, Badness} from "./stack"
 import {Action, Specialize, Term, Seq, StateFlag, ParseState} from "./constants"
 import {InputStream, Token, StringStream, Tokenizer, TokenGroup} from "./token"
-import {DefaultBufferLength, grammarID, termID, Tree, TreeBuffer, TagMap, allocateGrammarID} from "lezer-tree"
+import {DefaultBufferLength, Tree, TreeBuffer, Tag} from "lezer-tree"
 import {decodeArray} from "./decode"
@@ -275,4 +275,4 @@
       for (let cached = this.cache.nodeAt(start); cached;) {
-        if (grammarID(cached.type) != parser.id) continue
-        let match = parser.getGoto(stack.state, termID(cached.type))
+        if (!cached.isPartOf(parser.tags)) continue
+        let match = parser.getGoto(stack.state, cached.type)
         if (match > -1 && !isFragile(cached)) {
@@ -306,4 +306,5 @@ stack.useNode(cached, match)
         let node = parseNode ? parseNode(clippedInput) : Tree.empty
+        let keepType = (node.type & Term.Tagged) || type < 0
         stack.useNode(new Tree(node.children, node.positions, end - stack.pos,
-                               node.type & Term.Tagged || type < 0 ? node.type : type | parser.id),
+                               keepType ? node.tags : parser.tags, keepType ? node.type : type),
                       parser.getGoto(stack.state, placeholder, true))
@@ -349,4 +350,5 @@ this.putStack(stack)
         let parentParser = parent.cx.parser, info = parentParser.nested[parentParser.startNested(parent.state)]
+        let keepType = info.type < 0
         let node = new Tree(tree.children, tree.positions.map(p => p - parent!.pos), stack.pos - parent.pos,
-                            info.type >= 0 ? info.type | parentParser.id : tree.type)
+                            keepType ? tree.tags : parentParser.tags, keepType ? tree.type : info.type)
         parent.useNode(node, parentParser.getGoto(parent.state, info.placeholder, true))
@@ -414,4 +416,2 @@ if (verbose) console.log(parent + ` (via unnest ${parentParser.getName(info.type)})`)
   constructor(
-    /// The grammar's ID. Used to create globally unique tree node types.
-    readonly id: number,
     /// The parse states for this grammar @internal
@@ -427,3 +427,3 @@ readonly states: Readonly<Uint32Array>,
     /// names.
-    readonly tags: TagMap<string>,
+    readonly tags: readonly Tag[],
     /// The tokenizer objects used by the grammar @internal
@@ -556,18 +556,2 @@ readonly tokenizers: readonly Tokenizer[],
 
-  /// Build up a tag map for this grammar. The values object should map
-  /// from tag names to associated values.
-  tagMap<T>(values: {[name: string]: T}): TagMap<T> {
-    let content: (T | null)[] = []
-    let tagArray = this.tags.grammars[this.id >> 16] || []
-    for (let i = 0; i < tagArray.length; i++) {
-      let tag = tagArray[i]!
-      content.push(
-        Object.prototype.hasOwnProperty.call(values, tag) ? values[tag] :
-        tag[0] == '"' && Object.prototype.hasOwnProperty.call(values, JSON.parse(tag)) ? values[JSON.parse(tag)] : null)
-    }
-    let grammars = []
-    grammars[this.id >> 16] = content
-    return new TagMap<T>(grammars)
-  }
-
   /// Create a new `Parser` instance with different values for (some
@@ -578,3 +562,3 @@ /// of) the nested grammars. This can be used to, for example, swap
   withNested(spec: {[name: string]: NestedGrammar | null}) {
-    return new Parser(this.id, this.states, this.data, this.goto, this.tags, this.tokenizers,
+    return new Parser(this.states, this.data, this.goto, this.tags, this.tokenizers,
                       this.nested.map(obj => {
@@ -592,7 +576,11 @@ if (!Object.prototype.hasOwnProperty.call(spec, obj.name)) return obj
   getName(term: number): string {
-    return this.termNames ? this.termNames[term] : this.tags.get(term) || String(term)
+    return this.termNames ? this.termNames[term] : (term & Term.Tagged) && this.tags[term >> 1].tag || String(term)
   }
 
   /// (Used by the output of the parser generator) @internal
-  static deserialize(states: string, stateData: string, goto: string, tags: readonly string[],
+  static deserialize(states: string,
+                     stateData: string,
+                     goto: string,
+                     tags: readonly string[],
+                     globalTag: string | null,
                      tokenData: string, tokenizers: (Tokenizer | number)[],
@@ -604,5 +592,5 @@ nested: [string, null | NestedGrammar, string, number, number][],
                      termNames?: {[id: number]: string}) {
-    let tokenArray = decodeArray(tokenData), id = allocateGrammarID()
-    return new Parser(id, decodeArray(states, Uint32Array), decodeArray(stateData),
-                      decodeArray(goto), TagMap.single(id, tags),
+    let tokenArray = decodeArray(tokenData)
+    return new Parser(decodeArray(states, Uint32Array), decodeArray(stateData),
+                      decodeArray(goto), tags.map(tag => new Tag(tag + (globalTag ? "." + globalTag : ""))),
                       tokenizers.map(value => typeof value == "number" ? new TokenGroup(tokenArray, value) : value),
@@ -635,9 +623,9 @@ nested.map(([name, grammar, endToken, type, placeholder]) =>
   let doneStart = false, doneEnd = false, fragile = node.type == Term.Err
-  if (!fragile) node.iterate(0, node.length, type => {
-    return doneStart || (type == Term.Err ? fragile = doneStart = true : undefined)
+  if (!fragile) node.iterate(0, node.length, tag => {
+    return doneStart || (tag.tag == "⚠" ? fragile = doneStart = true : undefined)
   }, type => {
     doneStart = true
   })
-  if (!fragile) node.iterate(node.length, 0, type => {
-    return doneEnd || (type == Term.Err ? fragile = doneEnd = true : undefined)
+  if (!fragile) node.iterate(node.length, 0, tag => {
+    return doneEnd || (tag.tag == "⚠" ? fragile = doneEnd = true : undefined)
   }, type => {
@@ -644,0 +632,0 @@ doneEnd = true

src/README.md

@@ -32,4 +32,2 @@ ### Parsing
 
-@TagMap
-
-@allocateGrammarID
+@Tag

src/stack.ts

@@ -333,3 +333,3 @@ import {Action, Term, StateFlag, ParseState} from "./constants"
   toTree(): Tree {
-    return Tree.build(StackBufferCursor.create(this), this.cx.parser.id, this.cx.maxBufferLength, this.cx.reused)
+    return Tree.build(StackBufferCursor.create(this), this.cx.parser.tags, this.cx.maxBufferLength, this.cx.reused)
   }
@@ -336,0 +336,0 @@ }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

1525715

increased by15.15%

Number of package files

191

increased by33.57%

Worsened metrics

Lines of code

2770

decreased by-23.31%

Number of low supply chain risk alerts

2

increased by100%

Dependency changes

• + Addedlezer-tree@0.2.0(transitive)

• - Removedlezer-tree@0.1.1(transitive)

• Updatedlezer-tree@^0.2.0