lezer-tree - npm Package Compare versions

Comparing version 0.2.0 to 0.3.0

CHANGELOG.md

@@ -0,1 +1,13 @@
+## 0.3.0 (2019-08-22)
+
+### New features
+
+Introduces node props.
+
+Node types are now objects holding a name, id, and set of props.
+
+### Breaking changes
+
+Tags are gone again, nodes have plain string names.
+
 ## 0.2.0 (2019-08-02)
@@ -2,0 +14,0 @@

dist/tree.d.ts

@@ -8,4 +8,4 @@ export declare const DefaultBufferLength = 1024;
 }
-export declare type EnterFunc<T> = (tag: Tag, start: number, end: number) => T | false | undefined;
-export declare type LeaveFunc = (tag: Tag, start: number, end: number) => void;
+export declare type EnterFunc<T> = (type: NodeType, start: number, end: number) => T | false | undefined;
+export declare type LeaveFunc = (type: NodeType, start: number, end: number) => void;
 declare class Iteration<T> {
@@ -17,7 +17,53 @@ readonly enter: EnterFunc<T>;
     readonly done: boolean;
-    doEnter(tag: Tag, start: number, end: number): boolean;
+    doEnter(type: NodeType, start: number, end: number): boolean;
 }
+export declare class NodeProp<T> {
+    id: number;
+    deserialize: (str: string) => T;
+    constructor({ deserialize }?: {
+        deserialize?: (str: string) => T;
+    });
+    static string(): NodeProp<string>;
+    static flag(): NodeProp<boolean>;
+    set(propObj: {
+        [prop: number]: any;
+    }, value: T): {
+        [prop: number]: any;
+    };
+    add(f: (type: NodeType) => T | undefined): NodePropSource;
+    static error: NodeProp<boolean>;
+    static skipped: NodeProp<boolean>;
+    static delim: NodeProp<string>;
+    static lang: NodeProp<string>;
+    static repeated: NodeProp<boolean>;
+}
+export declare class NodePropSource {
+    readonly prop: NodeProp<any>;
+    readonly f: (type: NodeType) => any;
+    constructor(prop: NodeProp<any>, f: (type: NodeType) => any);
+}
+export declare class NodeType {
+    readonly name: string;
+    readonly props: {
+        readonly [prop: number]: any;
+    };
+    readonly id: number;
+    constructor(name: string, props: {
+        readonly [prop: number]: any;
+    }, id: number);
+    prop<T>(prop: NodeProp<T>): T | undefined;
+    static none: NodeType;
+    static match<T>(map: {
+        [selector: string]: T;
+    }): (node: NodeType) => T | undefined;
+}
+export declare class NodeGroup {
+    readonly types: readonly NodeType[];
+    constructor(types: readonly NodeType[]);
+    extend(...props: NodePropSource[]): NodeGroup;
+}
 export declare abstract class Subtree {
     abstract parent: Subtree | null;
-    abstract tag: Tag;
+    abstract type: NodeType;
+    readonly name: string;
     abstract start: number;
@@ -37,13 +83,10 @@ abstract end: number;
 export declare class Tree extends Subtree {
-    readonly children: (Tree | TreeBuffer)[];
-    readonly positions: number[];
+    readonly type: NodeType;
+    readonly children: readonly (Tree | TreeBuffer)[];
+    readonly positions: readonly number[];
     readonly length: number;
-    readonly tags: readonly Tag[];
-    readonly type: number;
     parent: null;
-    constructor(children: (Tree | TreeBuffer)[], positions: number[], length: number, tags: readonly Tag[], type?: number);
+    constructor(type: NodeType, children: readonly (Tree | TreeBuffer)[], positions: readonly number[], length: number);
     readonly start: number;
     readonly end: number;
-    readonly tag: Tag;
-    isPartOf(tags: readonly Tag[]): boolean;
     toString(): string;
@@ -63,3 +106,3 @@ private partial;
     balance(maxBufferLength?: number): Tree;
-    static build(buffer: BufferCursor | readonly number[], tags: readonly Tag[], maxBufferLength?: number, reused?: Tree[]): Tree;
+    static build(buffer: BufferCursor | readonly number[], group: NodeGroup, topID?: number, maxBufferLength?: number, reused?: Tree[]): Tree;
 }
@@ -69,4 +112,4 @@ export declare class TreeBuffer {
     readonly length: number;
-    readonly tags: readonly Tag[];
-    constructor(buffer: Uint16Array, length: number, tags: readonly Tag[]);
+    readonly group: NodeGroup;
+    constructor(buffer: Uint16Array, length: number, group: NodeGroup);
     toString(): string;
@@ -81,5 +124,5 @@ childToString(index: number, parts: string[]): number;
 }
-export interface BufferCursor {
+interface BufferCursor {
     pos: number;
-    type: number;
+    id: number;
     start: number;
@@ -91,11 +134,2 @@ end: number;
 }
-export declare class Tag {
-    readonly tag: string;
-    private parts;
-    constructor(tag: string);
-    private find;
-    has(name: string, value?: string): boolean;
-    matches(tag: Tag): number;
-    static empty: Tag;
-}
 export {};

dist/tree.js

@@ -18,4 +18,2 @@ "use strict";
 exports.DefaultBufferLength = 1024;
-// Checks whether the given node type is a tagged node.
-function isTagged(type) { return (type & 1) > 0; }
 var Iteration = /** @class */ (function () {
@@ -32,4 +30,4 @@ function Iteration(enter, leave) {
     });
-    Iteration.prototype.doEnter = function (tag, start, end) {
-        var value = this.enter(tag, start, end);
+    Iteration.prototype.doEnter = function (type, start, end) {
+        var value = this.enter(type, start, end);
         if (value === undefined)
@@ -43,2 +41,165 @@ return true;
 }());
+var nextPropID = 0;
+/// Each [node type](#tree.NodeType) can have metadata associated with
+/// it in props. Instances of this class represent prop names.
+var NodeProp = /** @class */ (function () {
+    /// Create a new node prop type. You can optionally pass a
+    /// `deserialize` function.
+    function NodeProp(_a) {
+        var deserialize = (_a === void 0 ? {} : _a).deserialize;
+        this.id = nextPropID++;
+        this.deserialize = deserialize || (function () {
+            throw new Error("This node type doesn't define a deserialize function");
+        });
+    }
+    /// Create a string-valued node prop whose deserialize function is
+    /// the identity function.
+    NodeProp.string = function () { return new NodeProp({ deserialize: function (str) { return str; } }); };
+    /// Creates a boolean-valued node prop whose deserialize function
+    /// returns true for any input.
+    NodeProp.flag = function () { return new NodeProp({ deserialize: function () { return true; } }); };
+    /// Store a value for this prop in the given object. This can be
+    /// useful when building up a prop object to pass to the
+    /// [`NodeType`](#tree.NodeType) constructor. Returns its first
+    /// argument.
+    NodeProp.prototype.set = function (propObj, value) {
+        propObj[this.id] = value;
+        return propObj;
+    };
+    /// This is meant to be used with
+    /// [`NodeGroup.extend`](#tree.NodeGroup.extend) or
+    /// [`Parser.withProps`](#lezer.Parser.withProps) to compute prop
+    /// values for each node type in the group. Takes a function that
+    /// returns undefined if the node type doesn't get this prop, or the
+    /// prop's value if it does.
+    NodeProp.prototype.add = function (f) { return new NodePropSource(this, f); };
+    /// The special node type that the parser uses to represent parse
+    /// errors has this flag set. (You shouldn't use it for custom nodes
+    /// that represent erroneous content.)
+    NodeProp.error = NodeProp.flag();
+    /// Nodes that were produced by skipped expressions (such as
+    /// comments) have this prop set to true.
+    NodeProp.skipped = NodeProp.flag();
+    /// Prop that is used to describe a rule's delimiters. For example,
+    /// a parenthesized expression node would set this to the string `"(
+    /// )"` (the open and close strings separated by a space). This is
+    /// added by the parser generator's `@detectDelim` feature, but you
+    /// can also manually add them.
+    NodeProp.delim = NodeProp.string();
+    /// The top node for a grammar usually has a `lang` prop set to a
+    /// string identifying the grammar, to provide context for the nodes
+    /// inside of it.
+    NodeProp.lang = NodeProp.string();
+    /// A prop that indicates whether a node represents a repeated
+    /// expression. Abstractions like [`Subtree`](#tree.Subtree) hide
+    /// such nodes, so you usually won't see them, but if you directly
+    /// rummage through a tree's children, you'll find repeat nodes that
+    /// wrap repeated content into balanced trees.
+    NodeProp.repeated = NodeProp.flag();
+    return NodeProp;
+}());
+exports.NodeProp = NodeProp;
+/// Type returned by [`NodeProp.add`](#tree.NodeProp.add). Describes
+/// the way a prop should be added to each node type in a node group.
+var NodePropSource = /** @class */ (function () {
+    /// @internal
+    function NodePropSource(
+    /// @internal
+    prop, 
+    /// @internal
+    f) {
+        this.prop = prop;
+        this.f = f;
+    }
+    return NodePropSource;
+}());
+exports.NodePropSource = NodePropSource;
+/// Each node in a syntax tree has a node type associated with it.
+var NodeType = /** @class */ (function () {
+    /// @internal
+    function NodeType(
+    /// The name of the node type. Not necessarily unique, but if the
+    /// grammar was written properly, different node types with the
+    /// same name within a node group should play the same semantic
+    /// role.
+    name, 
+    /// @internal
+    props, 
+    /// The id of this node in its group. Corresponds to the term ids
+    /// used in the parser.
+    id) {
+        this.name = name;
+        this.props = props;
+        this.id = id;
+    }
+    /// Retrieves a node prop for this type. Will return `undefined` if
+    /// the prop isn't present on this node.
+    NodeType.prototype.prop = function (prop) { return this.props[prop.id]; };
+    /// Create a function from node types to arbitrary values by
+    /// specifying an object whose property names are node names. Often
+    /// useful with [`NodeProp.add`](#tree.NodeProp.add). You can put
+    /// multiple node names, separated by spaces, in a single property
+    /// name to map multiple node names to a single value.
+    NodeType.match = function (map) {
+        var direct = Object.create(null);
+        for (var prop in map)
+            for (var _i = 0, _a = prop.split(" "); _i < _a.length; _i++) {
+                var name = _a[_i];
+                direct[name] = map[prop];
+            }
+        return function (node) { return direct[node.name]; };
+    };
+    /// An empty dummy node type to use when no actual type is available.
+    NodeType.none = new NodeType("", Object.create(null), 0);
+    return NodeType;
+}());
+exports.NodeType = NodeType;
+/// A node group holds a collection of node types. It is used to
+/// compactly represent trees by storing their type ids, rather than a
+/// full pointer to the type object, in a number array. Each parser
+/// [has](#lezer.Parser.group) a node group, and [tree
+/// buffers](#tree.TreeBuffer) can only store collections of nodes
+/// from the same group. A group can have a maximum of 2**16 (65536)
+/// node types in it, so that the ids fit into 16-bit typed array
+/// slots.
+var NodeGroup = /** @class */ (function () {
+    /// Create a group with the given types. The `id` property of each
+    /// type should correspond to its position within the array.
+    function NodeGroup(types) {
+        this.types = types;
+        for (var i = 0; i < types.length; i++)
+            if (types[i].id != i)
+                throw new RangeError("Node type ids should correspond to array positions when creating a node group");
+    }
+    /// Create a copy of this group with some node properties added. The
+    /// arguments to this method should be created with
+    /// [`NodeProp.add`](#tree.NodeProp.add).
+    NodeGroup.prototype.extend = function () {
+        var props = [];
+        for (var _i = 0; _i < arguments.length; _i++) {
+            props[_i] = arguments[_i];
+        }
+        var newTypes = [];
+        for (var _a = 0, _b = this.types; _a < _b.length; _a++) {
+            var type = _b[_a];
+            var newProps = null;
+            for (var _c = 0, props_1 = props; _c < props_1.length; _c++) {
+                var source = props_1[_c];
+                var value = source.f(type);
+                if (value !== undefined) {
+                    if (!newProps) {
+                        newProps = Object.create(null);
+                        for (var prop in type.props)
+                            newProps[prop] = type.props[prop];
+                    }
+                    newProps[source.prop.id] = value;
+                }
+            }
+            newTypes.push(newProps ? new NodeType(type.name, newProps, type.id) : type);
+        }
+        return new NodeGroup(newTypes);
+    };
+    return NodeGroup;
+}());
+exports.NodeGroup = NodeGroup;
 /// A subtree is a representation of part of the syntax tree. It may
@@ -49,2 +210,8 @@ /// either be the tree root, or a tagged node.
     }
+    Object.defineProperty(Subtree.prototype, "name", {
+        // Shorthand for `.type.name`.
+        get: function () { return this.type.name; },
+        enumerable: true,
+        configurable: true
+    });
     Object.defineProperty(Subtree.prototype, "depth", {
@@ -125,2 +292,4 @@ /// The depth (number of parent nodes) of this subtree
     function Tree(
+    /// @internal
+    type, 
     /// The tree's child nodes. Children small enough to fit in a
@@ -133,15 +302,9 @@ /// `TreeBuffer` will be represented as such, other children can be
     positions, 
-    /// The total length of this tree.
-    length, 
-    /// Mapping from node types to tags for this grammar @internal
-    tags, 
-    /// This tree's node type. The root node has type 0. @internal
-    type) {
-        if (type === void 0) { type = 0; }
+    /// The total length of this tree @internal
+    length) {
         var _this = _super.call(this) || this;
+        _this.type = type;
         _this.children = children;
         _this.positions = positions;
         _this.length = length;
-        _this.tags = tags;
-        _this.type = type;
         return _this;
@@ -161,17 +324,6 @@ }
     });
-    Object.defineProperty(Tree.prototype, "tag", {
-        /// @internal
-        get: function () { return isTagged(this.type) ? this.tags[this.type >> 1] : Tag.empty; },
-        enumerable: true,
-        configurable: true
-    });
-    /// Check whether this tree's tag belongs to a given set of tags.
-    /// Can be used to determine that a node belongs to the grammar
-    /// defined by a specific parser.
-    Tree.prototype.isPartOf = function (tags) { return this.tags == tags; };
     /// @internal
     Tree.prototype.toString = function () {
-        var name = !isTagged(this.type) ? null : this.tag.tag;
         var children = this.children.map(function (c) { return c.toString(); }).join();
-        return !name ? children : name + (children.length ? "(" + children + ")" : "");
+        return !this.name ? children : (/\W/.test(this.name) ? JSON.stringify(this.name) : this.name) + (children.length ? "(" + children + ")" : "");
     };
@@ -228,3 +380,3 @@ Tree.prototype.partial = function (start, end, offset, children, positions) {
         }
-        return new Tree(children, positions, this.length + off, this.tags);
+        return new Tree(NodeType.none, children, positions, this.length + off);
     };
@@ -244,3 +396,3 @@ /// Take the part of the tree up to the given position.
         }
-        return new Tree(children, positions, at, this.tags, this.type);
+        return new Tree(this.type, children, positions, at);
     };
@@ -255,3 +407,3 @@ /// @internal
     Tree.prototype.iterInner = function (from, to, offset, iter) {
-        if (isTagged(this.type) && !iter.doEnter(this.tag, offset, offset + this.length))
+        if (this.type.name && !iter.doEnter(this.type, offset, offset + this.length))
             return;
@@ -278,4 +430,4 @@ if (from <= to) {
         }
-        if (iter.leave && isTagged(this.type))
-            iter.leave(this.tag, offset, offset + this.length);
+        if (iter.leave && this.type.name)
+            iter.leave(this.type, offset, offset + this.length);
         return;
@@ -315,3 +467,3 @@ };
                 if (child instanceof Tree) {
-                    if (isTagged(child.type))
+                    if (child.type.name)
                         return new NodeSubtree(child, childStart_1, parent);
@@ -339,3 +491,3 @@ return child.findChild(pos, side, childStart_1, parent);
             throw new Error("Can't append overlapping trees");
-        return new Tree(this.children.concat(other.children), this.positions.concat(other.positions), other.length, this.tags, this.type);
+        return new Tree(this.type, this.children.concat(other.children), this.positions.concat(other.positions), other.length);
     };
@@ -347,13 +499,14 @@ /// Balance the direct children of this tree. Should only be used on
         return this.children.length <= BalanceBranchFactor ? this :
-            balanceRange(this.type, this.tags, this.children, this.positions, 0, this.children.length, 0, maxBufferLength);
+            balanceRange(this.type, this.children, this.positions, 0, this.children.length, 0, maxBufferLength);
     };
     /// Build a tree from a postfix-ordered buffer of node information,
     /// or a cursor over such a buffer.
-    Tree.build = function (buffer, tags, maxBufferLength, reused) {
+    Tree.build = function (buffer, group, topID, maxBufferLength, reused) {
+        if (topID === void 0) { topID = 0; }
         if (maxBufferLength === void 0) { maxBufferLength = exports.DefaultBufferLength; }
         if (reused === void 0) { reused = []; }
-        return buildTree(Array.isArray(buffer) ? new FlatBufferCursor(buffer, buffer.length) : buffer, tags, maxBufferLength, reused);
+        return buildTree(Array.isArray(buffer) ? new FlatBufferCursor(buffer, buffer.length) : buffer, group, topID, maxBufferLength, reused);
     };
     /// The empty tree
-    Tree.empty = new Tree([], [], 0, []);
+    Tree.empty = new Tree(NodeType.none, [], [], 0);
     return Tree;
@@ -368,8 +521,7 @@ }(Subtree));
 var TreeBuffer = /** @class */ (function () {
-    /// Create a tree buffer
-    // FIXME store a type in here to efficiently represent nodes whose children all fit in a buffer (esp repeat nodes)?
-    function TreeBuffer(buffer, length, tags) {
+    /// Create a tree buffer @internal
+    function TreeBuffer(buffer, length, group) {
         this.buffer = buffer;
         this.length = length;
-        this.tags = tags;
+        this.group = group;
     }
@@ -385,4 +537,6 @@ /// @internal
     TreeBuffer.prototype.childToString = function (index, parts) {
-        var type = this.buffer[index], endIndex = this.buffer[index + 3];
-        var result = this.tags[type >> 1].tag;
+        var id = this.buffer[index], endIndex = this.buffer[index + 3];
+        var result = this.group.types[id].name;
+        if (/\W/.test(result))
+            result = JSON.stringify(result);
         index += 4;
@@ -410,3 +564,3 @@ if (endIndex > index) {
         }
-        return new TreeBuffer(newBuffer, Math.min(at, this.length), this.tags);
+        return new TreeBuffer(newBuffer, Math.min(at, this.length), this.group);
     };
@@ -425,10 +579,10 @@ /// @internal
     TreeBuffer.prototype.iterChild = function (from, to, offset, index, iter) {
-        var tag = this.tags[this.buffer[index++] >> 1], start = this.buffer[index++] + offset, end = this.buffer[index++] + offset, endIndex = this.buffer[index++];
+        var type = this.group.types[this.buffer[index++]], start = this.buffer[index++] + offset, end = this.buffer[index++] + offset, endIndex = this.buffer[index++];
         if (start > to)
             return this.buffer.length;
-        if (end >= from && iter.doEnter(tag, start, end)) {
+        if (end >= from && iter.doEnter(type, start, end)) {
             while (index < endIndex && !iter.done)
                 index = this.iterChild(from, to, offset, index, iter);
             if (iter.leave)
-                iter.leave(tag, start, end);
+                iter.leave(type, start, end);
         }
@@ -475,6 +629,6 @@ return endIndex;
                 var base = nextStart;
-                var type_1 = this.buffer[base], start_1 = this.buffer[base + 1] + offset, end_1 = this.buffer[base + 2] + offset;
+                var id_1 = this.buffer[base], start_1 = this.buffer[base + 1] + offset, end_1 = this.buffer[base + 2] + offset;
                 takeNext();
                 if (start_1 <= from && end_1 >= to) {
-                    if (!iter.doEnter(this.tags[type_1 >> 1], start_1, end_1)) {
+                    if (!iter.doEnter(this.group.types[id_1], start_1, end_1)) {
                         // Skip the entire node
@@ -488,7 +642,7 @@ index = base;
             }
-            var endIndex_1 = this.buffer[--index], end = this.buffer[--index] + offset, start = this.buffer[--index] + offset, type = this.buffer[--index];
+            var endIndex_1 = this.buffer[--index], end = this.buffer[--index] + offset, start = this.buffer[--index] + offset, id = this.buffer[--index];
             if (start > from || end < to)
                 continue;
-            if ((endIndex_1 != index + 4 || iter.doEnter(this.tags[type >> 1], start, end)) && iter.leave)
-                iter.leave(this.tags[type >> 1], start, end);
+            if ((endIndex_1 != index + 4 || iter.doEnter(this.group.types[id], start, end)) && iter.leave)
+                iter.leave(this.group.types[id], start, end);
         }
@@ -527,4 +681,4 @@ };
     }
-    Object.defineProperty(NodeSubtree.prototype, "tag", {
-        get: function () { return this.node.tag; },
+    Object.defineProperty(NodeSubtree.prototype, "type", {
+        get: function () { return this.node.type; },
         enumerable: true,
@@ -567,4 +721,4 @@ configurable: true
     }
-    Object.defineProperty(BufferSubtree.prototype, "tag", {
-        get: function () { return this.buffer.tags[this.buffer.buffer[this.index] >> 1]; },
+    Object.defineProperty(BufferSubtree.prototype, "type", {
+        get: function () { return this.buffer.group.types[this.buffer.buffer[this.index]]; },
         enumerable: true,
@@ -622,3 +776,3 @@ configurable: true
     }
-    Object.defineProperty(FlatBufferCursor.prototype, "type", {
+    Object.defineProperty(FlatBufferCursor.prototype, "id", {
         get: function () { return this.buffer[this.index - 4]; },
@@ -653,12 +807,17 @@ enumerable: true,
 var BalanceBranchFactor = 8;
-function buildTree(cursor, tags, maxBufferLength, reused) {
+var repeat = NodeProp.repeated; // Need this one a lot later on
+function buildTree(cursor, group, topID, maxBufferLength, reused) {
+    var types = group.types;
     function takeNode(parentStart, minPos, children, positions) {
-        var type = cursor.type, start = cursor.start, end = cursor.end, size = cursor.size, buffer;
-        var node, startPos = start - parentStart;
-        if (size < 0) {
+        var id = cursor.id, start = cursor.start, end = cursor.end, size = cursor.size, buffer;
+        var startPos = start - parentStart;
+        if (size < 0) { // Reused node
+            children.push(reused[id]);
+            positions.push(startPos);
             cursor.next();
-            node = reused[type];
+            return;
         }
-        else if (end - start <= maxBufferLength &&
-            (buffer = findBufferSize(cursor.pos - minPos, isTagged(type) ? -1 : type))) {
+        var type = types[id], node;
+        if (end - start <= maxBufferLength &&
+            (buffer = findBufferSize(cursor.pos - minPos, type.prop(repeat) ? id : -1))) {
             // Small enough for a buffer, and no reused nodes inside
@@ -669,6 +828,6 @@ var data = new Uint16Array(buffer.size - buffer.skip);
                 index = copyToBuffer(buffer.start, data, index);
-            node = new TreeBuffer(data, end - buffer.start, tags);
+            node = new TreeBuffer(data, end - buffer.start, group);
             // Wrap if this is a repeat node
-            if (!isTagged(type))
-                node = new Tree([node], [0], end - buffer.start, tags, type);
+            if (type.prop(repeat))
+                node = new Tree(type, [node], [0], end - buffer.start);
             startPos = buffer.start - parentStart;
@@ -684,3 +843,3 @@ }
             localPositions.reverse();
-            node = new Tree(localChildren, localPositions, end - start, tags, type);
+            node = new Tree(type, localChildren, localPositions, end - start);
         }
@@ -690,3 +849,3 @@ children.push(node);
         // End of a (possible) sequence of repeating nodes—might need to balance
-        if (!isTagged(type) && (cursor.pos == 0 || cursor.type != type))
+        if (type.prop(repeat) && (cursor.pos == 0 || cursor.id != id))
             maybeBalanceSiblings(children, positions, type);
@@ -704,3 +863,3 @@ }
         var start = positions[to - 1];
-        var wrapped = balanceRange(type, tags, children.slice(from, to).reverse(), positions.slice(from, to).reverse(), 0, to - from, start, maxBufferLength);
+        var wrapped = balanceRange(type, children.slice(from, to).reverse(), positions.slice(from, to).reverse(), 0, to - from, start, maxBufferLength);
         children.length = positions.length = from + 1;
@@ -710,3 +869,3 @@ children[from] = wrapped;
     }
-    function findBufferSize(maxSize, type) {
+    function findBufferSize(maxSize, id) {
         // Scan through the buffer to find previous siblings that fit
@@ -721,5 +880,5 @@ // together in a TreeBuffer, and don't contain any reused nodes
             var nodeSize = fork.size, startPos = fork.pos - nodeSize;
-            var localSkipped = isTagged(fork.type) ? 0 : 4;
-            if (nodeSize < 0 || startPos < minPos || fork.start < minStart || type > -1 && fork.type != type)
+            if (nodeSize < 0 || startPos < minPos || fork.start < minStart || id > -1 && fork.id != id)
                 break;
+            var localSkipped = types[fork.id].prop(repeat) ? 4 : 0;
             var nodeStart = fork.start;
@@ -730,3 +889,3 @@ fork.next();
                     break scan;
-                if (!isTagged(fork.type))
+                if (types[fork.id].prop(repeat))
                     localSkipped += 4;
@@ -742,3 +901,3 @@ fork.next();
     function copyToBuffer(bufferStart, buffer, index) {
-        var type = cursor.type, start = cursor.start, end = cursor.end, size = cursor.size;
+        var id = cursor.id, start = cursor.start, end = cursor.end, size = cursor.size;
         cursor.next();
@@ -751,7 +910,7 @@ var startIndex = index;
         }
-        if (isTagged(type)) { // Don't copy repeat nodes into buffers
+        if (!types[id].prop(repeat)) { // Don't copy repeat nodes into buffers
             buffer[--index] = startIndex;
             buffer[--index] = end - bufferStart;
             buffer[--index] = start - bufferStart;
-            buffer[--index] = type;
+            buffer[--index] = id;
         }
@@ -764,5 +923,5 @@ return index;
     var length = children.length ? positions[0] + children[0].length : 0;
-    return new Tree(children.reverse(), positions.reverse(), length, tags);
+    return new Tree(group.types[topID], children.reverse(), positions.reverse(), length);
 }
-function balanceRange(type, tags, children, positions, from, to, start, maxBufferLength) {
+function balanceRange(type, children, positions, from, to, start, maxBufferLength) {
     var length = (positions[to - 1] + children[to - 1].length) - start;
@@ -805,3 +964,3 @@ if (from == to - 1 && start == 0) {
                     // Wrap with our type to make reuse possible
-                    only = new Tree([only], [0], only.length, tags, type);
+                    only = new Tree(type, [only], [0], only.length);
                 }
@@ -811,3 +970,3 @@ localChildren.push(only);
             else {
-                localChildren.push(balanceRange(type, tags, children, positions, groupFrom, i, groupStart, maxBufferLength));
+                localChildren.push(balanceRange(type, children, positions, groupFrom, i, groupStart, maxBufferLength));
             }
@@ -817,103 +976,4 @@ localPositions.push(groupStart - start);
     }
-    return new Tree(localChildren, localPositions, length, tags, type);
+    return new Tree(type, localChildren, localPositions, length);
 }
-function badTag(tag) {
-    throw new SyntaxError("Invalid tag " + JSON.stringify(tag));
-}
-/// Tags represent information about nodes. They are an ordered
-/// collection of parts (more specific ones first) written in the form
-/// `boolean.literal.expression` (where `boolean` further specifies
-/// `literal`, which in turn further specifies `expression`).
-///
-/// A part may also have a value, written after an `=` sign, as in
-/// `tag.selector.lang=css`. Part names and values may be double
-/// quoted (using JSON string notation) when they contain non-word
-/// characters.
-///
-/// This wrapper object pre-parses the tag for easy querying.
-var Tag = /** @class */ (function () {
-    /// Create a tag object from a string.
-    function Tag(
-    /// The string that the tag is based on.
-    tag) {
-        this.tag = tag;
-        var parts = [];
-        if (tag.length)
-            for (var pos = 0;;) {
-                var start = pos;
-                while (pos < tag.length) {
-                    var next = tag.charCodeAt(pos);
-                    if (next == 61 || next == 46)
-                        break; // "=" or "."
-                    pos++;
-                }
-                if (pos == start)
-                    badTag(tag);
-                var name = tag.slice(start, pos), value = "";
-                if (tag.charCodeAt(pos) == 61) {
-                    var valStart = ++pos;
-                    if (tag.charCodeAt(pos) == 34 /* '"' */) {
-                        for (pos++;;) {
-                            if (pos >= tag.length)
-                                badTag(tag);
-                            var next = tag.charCodeAt(pos++);
-                            if (next == 92)
-                                pos++;
-                            else if (next == 34)
-                                break;
-                        }
-                        value = JSON.parse(tag.slice(valStart, pos));
-                    }
-                    else {
-                        while (pos < tag.length && tag.charCodeAt(pos) != 46)
-                            pos++;
-                        value = tag.slice(valStart, pos);
-                    }
-                }
-                parts.push(name, value);
-                if (pos == tag.length)
-                    break;
-                if (tag.charCodeAt(pos) != 46)
-                    badTag(tag);
-                pos++;
-            }
-        this.parts = parts;
-    }
-    Tag.prototype.find = function (name, value) {
-        for (var i = 0; i < this.parts.length; i += 2) {
-            if (this.parts[i] == name && (value == null || this.parts[i + 1] == value))
-                return i;
-        }
-        return -1;
-    };
-    /// Check whether this tag has a part by the given name. If `value`
-    /// is given, this will only return true when that part also has
-    /// that specific value.
-    Tag.prototype.has = function (name, value) {
-        return this.find(name, value) > -1;
-    };
-    /// See whether this tag contains all the parts present in the
-    /// argument tag, and, if the part has a value in the query tag, the
-    /// same value in this tag. Returns a specificity score—0 means
-    /// there was no match, a higher score means the query matched more
-    /// specific parts of the tag.
-    Tag.prototype.matches = function (tag) {
-        var score = 0;
-        if (tag.parts.length == 0)
-            return 1;
-        for (var i = 0; i < tag.parts.length; i += 2) {
-            var val = tag.parts[i + 1];
-            var found = this.find(tag.parts[i], val || undefined);
-            if (found < 0)
-                return 0;
-            score += Math.pow(2, ((tag.parts.length - i) >> 1)) + (val ? 1 : 0);
-        }
-        return score;
-    };
-    /// The empty tag, returned for nodes that don't have a meaningful
-    /// tag.
-    Tag.empty = new Tag("");
-    return Tag;
-}());
-exports.Tag = Tag;
 //# sourceMappingURL=tree.js.map

package.json

 {
   "name": "lezer-tree",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Syntax tree data structure for the lezer parser",
@@ -10,2 +10,4 @@ "main": "dist/tree.js",
   "devDependencies": {
+    "ist": "^1.1.1",
+    "mocha": "^6.2.0",
     "typescript": "^3.4.3"
@@ -15,4 +17,5 @@ },
     "watch": "tsc --outDir dist -w -d",
-    "prepare": "tsc --outDir dist -d"
+    "prepare": "tsc --outDir dist -d",
+    "test": "mocha test/test-*.js"
   }
 }

src/README.md

@@ -13,6 +13,12 @@ ### Trees
 
-### Node tags
+### Node types
 
-@Tag
+@NodeType
 
+@NodeGroup
+
+@NodeProp
+
+@NodePropSource
+
 ### Buffers
@@ -23,3 +29,1 @@
 @DefaultBufferLength
-
-@BufferCursor

src/tree.ts

 /// The default maximum length of a `TreeBuffer` node.
 export const DefaultBufferLength = 1024
 
-// Checks whether the given node type is a tagged node.
-function isTagged(type: number) { return (type & 1) > 0 }
-
 /// The `unchanged` method expects changed ranges in this format.
@@ -30,6 +27,6 @@ export interface ChangedRange {
 ///    value from the `iterate` method.
-export type EnterFunc<T> = (tag: Tag, start: number, end: number) => T | false | undefined
+export type EnterFunc<T> = (type: NodeType, start: number, end: number) => T | false | undefined
 
 /// Signature of the `leave` function passed to `Subtree.iterate`.
-export type LeaveFunc = (tag: Tag, start: number, end: number) => void
+export type LeaveFunc = (type: NodeType, start: number, end: number) => void
 
@@ -44,4 +41,4 @@ class Iteration<T> {
 
-  doEnter(tag: Tag, start: number, end: number) {
-    let value = this.enter(tag, start, end)
+  doEnter(type: NodeType, start: number, end: number) {
+    let value = this.enter(type, start, end)
     if (value === undefined) return true
@@ -53,2 +50,163 @@ if (value !== false) this.result = value
 
+let nextPropID = 0
+
+/// Each [node type](#tree.NodeType) can have metadata associated with
+/// it in props. Instances of this class represent prop names.
+export class NodeProp<T> {
+  /// @internal
+  id: number
+
+  /// A method that deserializes a value of this prop from a string.
+  /// Can be used to allow a prop to be directly written in a grammar
+  /// file. Defaults to raising an error.
+  deserialize: (str: string) => T
+
+  /// Create a new node prop type. You can optionally pass a
+  /// `deserialize` function.
+  constructor({deserialize}: {deserialize?: (str: string) => T} = {}) {
+    this.id = nextPropID++
+    this.deserialize = deserialize || (() => {
+      throw new Error("This node type doesn't define a deserialize function")
+    })
+  }
+
+  /// Create a string-valued node prop whose deserialize function is
+  /// the identity function.
+  static string() { return new NodeProp<string>({deserialize: str => str}) }
+
+  /// Creates a boolean-valued node prop whose deserialize function
+  /// returns true for any input.
+  static flag() { return new NodeProp<boolean>({deserialize: () => true}) }
+
+  /// Store a value for this prop in the given object. This can be
+  /// useful when building up a prop object to pass to the
+  /// [`NodeType`](#tree.NodeType) constructor. Returns its first
+  /// argument.
+  set(propObj: {[prop: number]: any}, value: T) {
+    propObj[this.id] = value
+    return propObj
+  }
+
+  /// This is meant to be used with
+  /// [`NodeGroup.extend`](#tree.NodeGroup.extend) or
+  /// [`Parser.withProps`](#lezer.Parser.withProps) to compute prop
+  /// values for each node type in the group. Takes a function that
+  /// returns undefined if the node type doesn't get this prop, or the
+  /// prop's value if it does.
+  add(f: (type: NodeType) => T | undefined): NodePropSource { return new NodePropSource(this, f) }
+
+  /// The special node type that the parser uses to represent parse
+  /// errors has this flag set. (You shouldn't use it for custom nodes
+  /// that represent erroneous content.)
+  static error = NodeProp.flag()
+
+  /// Nodes that were produced by skipped expressions (such as
+  /// comments) have this prop set to true.
+  static skipped = NodeProp.flag()
+
+  /// Prop that is used to describe a rule's delimiters. For example,
+  /// a parenthesized expression node would set this to the string `"(
+  /// )"` (the open and close strings separated by a space). This is
+  /// added by the parser generator's `@detectDelim` feature, but you
+  /// can also manually add them.
+  static delim = NodeProp.string()
+
+  /// The top node for a grammar usually has a `lang` prop set to a
+  /// string identifying the grammar, to provide context for the nodes
+  /// inside of it.
+  static lang = NodeProp.string()
+
+  /// A prop that indicates whether a node represents a repeated
+  /// expression. Abstractions like [`Subtree`](#tree.Subtree) hide
+  /// such nodes, so you usually won't see them, but if you directly
+  /// rummage through a tree's children, you'll find repeat nodes that
+  /// wrap repeated content into balanced trees.
+  static repeated = NodeProp.flag()
+}
+
+/// Type returned by [`NodeProp.add`](#tree.NodeProp.add). Describes
+/// the way a prop should be added to each node type in a node group.
+export class NodePropSource {
+  /// @internal
+  constructor(
+    /// @internal
+    readonly prop: NodeProp<any>,
+    /// @internal
+    readonly f: (type: NodeType) => any) {}
+}
+
+/// Each node in a syntax tree has a node type associated with it.
+export class NodeType {
+  /// @internal
+  constructor(
+    /// The name of the node type. Not necessarily unique, but if the
+    /// grammar was written properly, different node types with the
+    /// same name within a node group should play the same semantic
+    /// role.
+    readonly name: string,
+    /// @internal
+    readonly props: {readonly [prop: number]: any},
+    /// The id of this node in its group. Corresponds to the term ids
+    /// used in the parser.
+    readonly id: number) {}
+
+  /// Retrieves a node prop for this type. Will return `undefined` if
+  /// the prop isn't present on this node.
+  prop<T>(prop: NodeProp<T>): T | undefined { return this.props[prop.id] }
+
+  /// An empty dummy node type to use when no actual type is available.
+  static none: NodeType = new NodeType("", Object.create(null), 0)
+
+  /// Create a function from node types to arbitrary values by
+  /// specifying an object whose property names are node names. Often
+  /// useful with [`NodeProp.add`](#tree.NodeProp.add). You can put
+  /// multiple node names, separated by spaces, in a single property
+  /// name to map multiple node names to a single value.
+  static match<T>(map: {[selector: string]: T}): (node: NodeType) => T | undefined {
+    let direct = Object.create(null)
+    for (let prop in map)
+      for (let name of prop.split(" ")) direct[name] = map[prop]
+    return (node: NodeType) => direct[node.name]
+  }
+}
+
+/// A node group holds a collection of node types. It is used to
+/// compactly represent trees by storing their type ids, rather than a
+/// full pointer to the type object, in a number array. Each parser
+/// [has](#lezer.Parser.group) a node group, and [tree
+/// buffers](#tree.TreeBuffer) can only store collections of nodes
+/// from the same group. A group can have a maximum of 2**16 (65536)
+/// node types in it, so that the ids fit into 16-bit typed array
+/// slots.
+export class NodeGroup {
+  /// Create a group with the given types. The `id` property of each
+  /// type should correspond to its position within the array.
+  constructor(readonly types: readonly NodeType[]) {
+    for (let i = 0; i < types.length; i++) if (types[i].id != i)
+      throw new RangeError("Node type ids should correspond to array positions when creating a node group")
+  }
+
+  /// Create a copy of this group with some node properties added. The
+  /// arguments to this method should be created with
+  /// [`NodeProp.add`](#tree.NodeProp.add).
+  extend(...props: NodePropSource[]): NodeGroup {
+    let newTypes: NodeType[] = []
+    for (let type of this.types) {
+      let newProps = null
+      for (let source of props) {
+        let value = source.f(type)
+        if (value !== undefined) {
+          if (!newProps) {
+            newProps = Object.create(null)
+            for (let prop in type.props) newProps[prop] = type.props[prop]
+          }
+          newProps[source.prop.id] = value
+        }
+      }
+      newTypes.push(newProps ? new NodeType(type.name, newProps, type.id) : type)
+    }
+    return new NodeGroup(newTypes)
+  }
+}
+
 /// A subtree is a representation of part of the syntax tree. It may
@@ -60,4 +218,6 @@ /// either be the tree root, or a tagged node.
 
-  /// The node's tag. Will be `Tag.empty` for the root
-  abstract tag: Tag
+  /// The node's type
+  abstract type: NodeType
+  // Shorthand for `.type.name`.
+  get name() { return this.type.name }
   /// The start source offset of this subtree
@@ -142,15 +302,13 @@ abstract start: number
   constructor(
+    /// @internal
+    readonly type: NodeType,
     /// The tree's child nodes. Children small enough to fit in a
     /// `TreeBuffer` will be represented as such, other children can be
     /// further `Tree` instances with their own internal structure.
-    readonly children: (Tree | TreeBuffer)[],
+    readonly children: readonly (Tree | TreeBuffer)[],
     /// The positions (offsets relative to the start of this tree) of
     /// the children.
-    readonly positions: number[],
-    /// The total length of this tree.
-    readonly length: number,
-    /// Mapping from node types to tags for this grammar @internal
-    readonly tags: readonly Tag[],
-    /// This tree's node type. The root node has type 0. @internal
-    readonly type: number = 0
+    readonly positions: readonly number[],
+    /// The total length of this tree @internal
+    readonly length: number
   ) {
@@ -167,14 +325,5 @@ super()
   /// @internal
-  get tag() { return isTagged(this.type) ? this.tags[this.type >> 1] : Tag.empty }
-
-  /// Check whether this tree's tag belongs to a given set of tags.
-  /// Can be used to determine that a node belongs to the grammar
-  /// defined by a specific parser.
-  isPartOf(tags: readonly Tag[]) { return this.tags == tags }
-
-  /// @internal
   toString(): string {
-    let name = !isTagged(this.type) ? null : this.tag.tag
     let children = this.children.map(c => c.toString()).join()
-    return !name ? children : name + (children.length ? "(" + children + ")" : "")
+    return !this.name ? children : (/\W/.test(this.name) ? JSON.stringify(this.name) : this.name) + (children.length ? "(" + children + ")" : "")
   }
@@ -227,3 +376,3 @@
     }
-    return new Tree(children, positions, this.length + off, this.tags)
+    return new Tree(NodeType.none, children, positions, this.length + off)
   }
@@ -242,7 +391,7 @@
     }
-    return new Tree(children, positions, at, this.tags, this.type)
+    return new Tree(this.type, children, positions, at)
   }
 
   /// The empty tree
-  static empty = new Tree([], [], 0, [])
+  static empty = new Tree(NodeType.none, [], [], 0)
 
@@ -258,3 +407,3 @@ /// @internal
   iterInner<T>(from: number, to: number, offset: number, iter: Iteration<T>) {
-    if (isTagged(this.type) && !iter.doEnter(this.tag, offset, offset + this.length))
+    if (this.type.name && !iter.doEnter(this.type, offset, offset + this.length))
       return
@@ -277,3 +426,3 @@
     }      
-    if (iter.leave && isTagged(this.type)) iter.leave(this.tag, offset, offset + this.length)
+    if (iter.leave && this.type.name) iter.leave(this.type, offset, offset + this.length)
     return
@@ -312,3 +461,3 @@ }
         if (child instanceof Tree) {
-          if (isTagged(child.type)) return new NodeSubtree(child, childStart, parent)
+          if (child.type.name) return new NodeSubtree(child, childStart, parent)
           return child.findChild(pos, side, childStart, parent)
@@ -334,3 +483,3 @@ } else {
     if (other.children.length && other.positions[0] < this.length) throw new Error("Can't append overlapping trees")
-    return new Tree(this.children.concat(other.children), this.positions.concat(other.positions), other.length, this.tags, this.type)
+    return new Tree(this.type, this.children.concat(other.children), this.positions.concat(other.positions), other.length)
   }
@@ -342,3 +491,3 @@
     return this.children.length <= BalanceBranchFactor ? this :
-      balanceRange(this.type, this.tags, this.children, this.positions, 0, this.children.length, 0, maxBufferLength)
+      balanceRange(this.type, this.children, this.positions, 0, this.children.length, 0, maxBufferLength)
   }
@@ -348,8 +497,7 @@
   /// or a cursor over such a buffer.
-  static build(buffer: BufferCursor | readonly number[],
-               tags: readonly Tag[],
+  static build(buffer: BufferCursor | readonly number[], group: NodeGroup, topID: number = 0,
                maxBufferLength: number = DefaultBufferLength,
                reused: Tree[] = []) {
     return buildTree(Array.isArray(buffer) ? new FlatBufferCursor(buffer, buffer.length) : buffer as BufferCursor,
-                     tags, maxBufferLength, reused)
+                     group, topID, maxBufferLength, reused)
   }
@@ -365,5 +513,4 @@ }
 export class TreeBuffer {
-  /// Create a tree buffer
-  // FIXME store a type in here to efficiently represent nodes whose children all fit in a buffer (esp repeat nodes)?
-  constructor(readonly buffer: Uint16Array, readonly length: number, readonly tags: readonly Tag[]) {}
+  /// Create a tree buffer @internal
+  constructor(readonly buffer: Uint16Array, readonly length: number, readonly group: NodeGroup) {}
 
@@ -380,4 +527,5 @@ /// @internal
   childToString(index: number, parts: string[]): number {
-    let type = this.buffer[index], endIndex = this.buffer[index + 3]
-    let result = this.tags[type >> 1].tag
+    let id = this.buffer[index], endIndex = this.buffer[index + 3]
+    let result = this.group.types[id].name
+    if (/\W/.test(result)) result = JSON.stringify(result)
     index += 4
@@ -404,3 +552,3 @@ if (endIndex > index) {
     }
-    return new TreeBuffer(newBuffer, Math.min(at, this.length), this.tags)
+    return new TreeBuffer(newBuffer, Math.min(at, this.length), this.group)
   }
@@ -420,8 +568,8 @@
   iterChild<T>(from: number, to: number, offset: number, index: number, iter: Iteration<T>) {
-    let tag = this.tags[this.buffer[index++] >> 1], start = this.buffer[index++] + offset,
+    let type = this.group.types[this.buffer[index++]], start = this.buffer[index++] + offset,
         end = this.buffer[index++] + offset, endIndex = this.buffer[index++]
     if (start > to) return this.buffer.length
-    if (end >= from && iter.doEnter(tag, start, end)) {
+    if (end >= from && iter.doEnter(type, start, end)) {
       while (index < endIndex && !iter.done) index = this.iterChild(from, to, offset, index, iter)
-      if (iter.leave) iter.leave(tag, start, end)
+      if (iter.leave) iter.leave(type, start, end)
     }
@@ -465,6 +613,6 @@ return endIndex
         let base = nextStart
-        let type = this.buffer[base], start = this.buffer[base + 1] + offset, end = this.buffer[base + 2] + offset
+        let id = this.buffer[base], start = this.buffer[base + 1] + offset, end = this.buffer[base + 2] + offset
         takeNext()
         if (start <= from && end >= to) {
-          if (!iter.doEnter(this.tags[type >> 1], start, end)) {
+          if (!iter.doEnter(this.group.types[id], start, end)) {
             // Skip the entire node
@@ -478,6 +626,6 @@ index = base
       let endIndex = this.buffer[--index], end = this.buffer[--index] + offset,
-        start = this.buffer[--index] + offset, type = this.buffer[--index]
+        start = this.buffer[--index] + offset, id = this.buffer[--index]
       if (start > from || end < to) continue
-      if ((endIndex != index + 4 || iter.doEnter(this.tags[type >> 1], start, end)) && iter.leave)
-        iter.leave(this.tags[type >> 1], start, end)
+      if ((endIndex != index + 4 || iter.doEnter(this.group.types[id], start, end)) && iter.leave)
+        iter.leave(this.group.types[id], start, end)
     }
@@ -511,3 +659,3 @@ }
 
-  get tag() { return this.node.tag }
+  get type() { return this.node.type }
 
@@ -547,3 +695,3 @@ get end() { return this.start + this.node.length }
 
-  get tag() { return this.buffer.tags[this.buffer.buffer[this.index] >> 1] }
+  get type() { return this.buffer.group.types[this.buffer.buffer[this.index]] }
   get start() { return this.buffer.buffer[this.index + 1] + this.bufferStart }
@@ -586,7 +734,7 @@ get end() { return this.buffer.buffer[this.index + 2] + this.bufferStart }
 
-/// This is used by `Tree.build` as an abstraction for iterating over
-/// a tree buffer. You probably won't need it.
-export interface BufferCursor {
+// This is used by `Tree.build` as an abstraction for iterating over
+// a tree buffer.
+interface BufferCursor {
   pos: number
-  type: number
+  id: number
   start: number
@@ -602,3 +750,3 @@ end: number
 
-  get type() { return this.buffer[this.index - 4] }
+  get id() { return this.buffer[this.index - 4] }
   get start() { return this.buffer[this.index - 3] }
@@ -617,11 +765,19 @@ get end() { return this.buffer[this.index - 2] }
 
-function buildTree(cursor: BufferCursor, tags: readonly Tag[], maxBufferLength: number, reused: Tree[]): Tree {
+const repeat = NodeProp.repeated // Need this one a lot later on
+
+function buildTree(cursor: BufferCursor, group: NodeGroup, topID: number, maxBufferLength: number, reused: Tree[]): Tree {
+  let types = group.types
   function takeNode(parentStart: number, minPos: number, children: (Tree | TreeBuffer)[], positions: number[]) {
-    let {type, start, end, size} = cursor, buffer!: {size: number, start: number, skip: number} | null
-    let node, startPos = start - parentStart
-    if (size < 0) {
+    let {id, start, end, size} = cursor, buffer!: {size: number, start: number, skip: number} | null
+    let startPos = start - parentStart
+    if (size < 0) { // Reused node
+      children.push(reused[id])
+      positions.push(startPos)
       cursor.next()
-      node = reused[type]
-    } else if (end - start <= maxBufferLength &&
-               (buffer = findBufferSize(cursor.pos - minPos, isTagged(type) ? -1 : type))) {
+      return
+    }
+
+    let type = types[id], node
+    if (end - start <= maxBufferLength &&
+        (buffer = findBufferSize(cursor.pos - minPos, type.prop(repeat) ? id : -1))) {
       // Small enough for a buffer, and no reused nodes inside
@@ -632,5 +788,5 @@ let data = new Uint16Array(buffer.size - buffer.skip)
         index = copyToBuffer(buffer.start, data, index)
-      node = new TreeBuffer(data, end - buffer.start, tags)
+      node = new TreeBuffer(data, end - buffer.start, group)
       // Wrap if this is a repeat node
-      if (!isTagged(type)) node = new Tree([node], [0], end - buffer.start, tags, type)
+      if (type.prop(repeat)) node = new Tree(type, [node], [0], end - buffer.start)
       startPos = buffer.start - parentStart
@@ -644,3 +800,3 @@ } else { // Make it a node
       localChildren.reverse(); localPositions.reverse()
-      node = new Tree(localChildren, localPositions, end - start, tags, type)
+      node = new Tree(type, localChildren, localPositions, end - start)
     }
@@ -651,7 +807,7 @@
     // End of a (possible) sequence of repeating nodes—might need to balance
-    if (!isTagged(type) && (cursor.pos == 0 || cursor.type != type))
+    if (type.prop(repeat) && (cursor.pos == 0 || cursor.id != id))
       maybeBalanceSiblings(children, positions, type)
   }
 
-  function maybeBalanceSiblings(children: (Tree | TreeBuffer)[], positions: number[], type: number) {
+  function maybeBalanceSiblings(children: (Tree | TreeBuffer)[], positions: number[], type: NodeType) {
     let to = children.length, from = to - 1
@@ -664,3 +820,3 @@ for (; from > 0; from--) {
     let start = positions[to - 1]
-    let wrapped = balanceRange(type, tags, children.slice(from, to).reverse(), positions.slice(from, to).reverse(),
+    let wrapped = balanceRange(type, children.slice(from, to).reverse(), positions.slice(from, to).reverse(),
                                0, to - from, start, maxBufferLength)
@@ -672,3 +828,3 @@ children.length = positions.length = from + 1
 
-  function findBufferSize(maxSize: number, type: number) {
+  function findBufferSize(maxSize: number, id: number) {
     // Scan through the buffer to find previous siblings that fit
@@ -683,4 +839,4 @@ // together in a TreeBuffer, and don't contain any reused nodes
       let nodeSize = fork.size, startPos = fork.pos - nodeSize
-      let localSkipped = isTagged(fork.type) ? 0 : 4
-      if (nodeSize < 0 || startPos < minPos || fork.start < minStart || type > -1 && fork.type != type) break
+      if (nodeSize < 0 || startPos < minPos || fork.start < minStart || id > -1 && fork.id != id) break
+      let localSkipped = types[fork.id].prop(repeat) ? 4 : 0
       let nodeStart = fork.start
@@ -690,3 +846,3 @@ fork.next()
         if (fork.size < 0) break scan
-        if (!isTagged(fork.type)) localSkipped += 4
+        if (types[fork.id].prop(repeat)) localSkipped += 4
         fork.next()
@@ -702,3 +858,3 @@ }
   function copyToBuffer(bufferStart: number, buffer: Uint16Array, index: number): number {
-    let {type, start, end, size} = cursor
+    let {id, start, end, size} = cursor
     cursor.next()
@@ -711,7 +867,7 @@ let startIndex = index
     }
-    if (isTagged(type)) { // Don't copy repeat nodes into buffers
+    if (!types[id].prop(repeat)) { // Don't copy repeat nodes into buffers
       buffer[--index] = startIndex
       buffer[--index] = end - bufferStart
       buffer[--index] = start - bufferStart
-      buffer[--index] = type
+      buffer[--index] = id
     }
@@ -724,7 +880,6 @@ return index
   let length = children.length ? positions[0] + children[0].length : 0
-  return new Tree(children.reverse(), positions.reverse(), length, tags)
+  return new Tree(group.types[topID], children.reverse(), positions.reverse(), length)
 }
 
-function balanceRange(type: number,
-                      tags: readonly Tag[],
+function balanceRange(type: NodeType,
                       children: readonly (Tree | TreeBuffer)[], positions: readonly number[],
@@ -766,7 +921,7 @@ from: number, to: number,
           // Wrap with our type to make reuse possible
-          only = new Tree([only], [0], only.length, tags, type)
+          only = new Tree(type, [only], [0], only.length)
         }
         localChildren.push(only)
       } else {
-        localChildren.push(balanceRange(type, tags, children, positions, groupFrom, i, groupStart, maxBufferLength))
+        localChildren.push(balanceRange(type, children, positions, groupFrom, i, groupStart, maxBufferLength))
       }
@@ -776,95 +931,3 @@ localPositions.push(groupStart - start)
   }
-  return new Tree(localChildren, localPositions, length, tags, type)
+  return new Tree(type, localChildren, localPositions, length)
 }
-
-function badTag(tag: string): never {
-  throw new SyntaxError("Invalid tag " + JSON.stringify(tag))
-}
-
-/// Tags represent information about nodes. They are an ordered
-/// collection of parts (more specific ones first) written in the form
-/// `boolean.literal.expression` (where `boolean` further specifies
-/// `literal`, which in turn further specifies `expression`).
-///
-/// A part may also have a value, written after an `=` sign, as in
-/// `tag.selector.lang=css`. Part names and values may be double
-/// quoted (using JSON string notation) when they contain non-word
-/// characters.
-///
-/// This wrapper object pre-parses the tag for easy querying.
-export class Tag {
-  private parts: readonly string[]
-
-  /// Create a tag object from a string.
-  constructor(
-    /// The string that the tag is based on.
-    readonly tag: string
-  ) {
-    let parts = []
-    if (tag.length) for (let pos = 0;;) {
-      let start = pos
-      while (pos < tag.length) {
-        let next = tag.charCodeAt(pos)
-        if (next == 61 || next == 46) break // "=" or "."
-        pos++
-      }
-      if (pos == start) badTag(tag)
-      let name = tag.slice(start, pos), value = ""
-      if (tag.charCodeAt(pos) == 61) {
-        let valStart = ++pos
-        if (tag.charCodeAt(pos) == 34 /* '"' */) {
-          for (pos++;;) {
-            if (pos >= tag.length) badTag(tag)
-            let next = tag.charCodeAt(pos++)
-            if (next == 92) pos++
-            else if (next == 34) break
-          }
-          value = JSON.parse(tag.slice(valStart, pos))
-        } else {
-          while (pos < tag.length && tag.charCodeAt(pos) != 46) pos++
-          value = tag.slice(valStart, pos)
-        }
-      }
-      parts.push(name, value)
-      if (pos == tag.length) break
-      if (tag.charCodeAt(pos) != 46) badTag(tag)
-      pos++
-    }
-    this.parts = parts
-  }
-
-  private find(name: string, value?: string) {
-    for (let i = 0; i < this.parts.length; i += 2) {
-      if (this.parts[i] == name && (value == null || this.parts[i + 1] == value)) return i
-    }
-    return -1
-  }
-
-  /// Check whether this tag has a part by the given name. If `value`
-  /// is given, this will only return true when that part also has
-  /// that specific value.
-  has(name: string, value?: string) {
-    return this.find(name, value) > -1
-  }
-
-  /// See whether this tag contains all the parts present in the
-  /// argument tag, and, if the part has a value in the query tag, the
-  /// same value in this tag. Returns a specificity score—0 means
-  /// there was no match, a higher score means the query matched more
-  /// specific parts of the tag.
-  matches(tag: Tag) {
-    let score = 0
-    if (tag.parts.length == 0) return 1
-    for (let i = 0; i < tag.parts.length; i += 2) {
-      let val = tag.parts[i + 1]
-      let found = this.find(tag.parts[i], val || undefined)
-      if (found < 0) return 0
-      score += 2 ** ((tag.parts.length - i) >> 1) + (val ? 1 : 0)
-    }
-    return score
-  }
-
-  /// The empty tag, returned for nodes that don't have a meaningful
-  /// tag.
-  static empty = new Tag("")
-}

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

114253

increased by7.79%

Lines of code

1875

increased by8.38%

Worsened metrics

Dev dependency count

3

increased by200%

Number of package files

10

decreased by-9.09%

No dependency changes