antlr4-c3 - npm Package Compare versions

Comparing version 1.0.11 to 1.0.12

out/src/CodeCompletionCore.d.ts

 import { Parser, ParserRuleContext } from 'antlr4ts';
+export declare type TokenList = number[];
+export declare type RuleList = number[];
 export declare class CandidatesCollection {
-    tokens: Map<number, number[]>;
-    rules: Map<number, number[]>;
+    tokens: Map<number, TokenList>;
+    rules: Set<number>;
 }
@@ -20,2 +22,3 @@ export declare class CodeCompletionCore {
     private statesProcessed;
+    private shortcutMap;
     private candidates;
@@ -31,2 +34,3 @@ private static followSetsByATN;
     private processRule(startState, tokenIndex, callStack, indentation);
+    private atnStateTypeMap;
     private generateBaseDescription(state);
@@ -33,0 +37,0 @@ private printDescription(currentIndent, state, baseDescription, tokenIndex);

out/src/CodeCompletionCore.js

@@ -9,3 +9,3 @@ 'use strict';
         this.tokens = new Map();
-        this.rules = new Map();
+        this.rules = new Set();
     }
@@ -36,3 +36,19 @@ }
         this.statesProcessed = 0;
+        this.shortcutMap = new Map();
         this.candidates = new CandidatesCollection();
+        this.atnStateTypeMap = [
+            "invalid",
+            "basic",
+            "rule start",
+            "block start",
+            "plus block start",
+            "star block start",
+            "token start",
+            "rule stop",
+            "block end",
+            "star loop back",
+            "star loop entry",
+            "plus loop back",
+            "loop end"
+        ];
         this.parser = parser;
@@ -46,2 +62,3 @@ this.atn = parser.atn;
     collectCandidates(caretTokenIndex, context) {
+        this.shortcutMap.clear();
         this.candidates.rules.clear();
@@ -70,9 +87,5 @@ this.candidates.tokens.clear();
             console.log("\n\nCollected rules:\n");
-            for (let rule of this.candidates.rules) {
-                let path = "";
-                for (let token of rule[1]) {
-                    path += this.ruleNames[token] + " ";
-                }
-                console.log(this.ruleNames[rule[0]] + ", path: ", path);
-            }
+            this.candidates.rules.forEach(rule => {
+                console.log(this.ruleNames[rule]);
+            });
             let sortedTokens = new Set();
@@ -85,3 +98,3 @@ for (let token of this.candidates.tokens) {
             }
-            console.log("\n\nCollected tokens:");
+            console.log("\n\nCollected tokens:\n");
             for (let symbol of sortedTokens) {
@@ -102,17 +115,5 @@ console.log(symbol);
             if (this.preferredRules.has(ruleStack[i])) {
-                let path = ruleStack.slice(0, i);
-                let addNew = true;
-                for (let rule of this.candidates.rules) {
-                    if (rule[0] != ruleStack[i] || rule[1].length != path.length)
-                        continue;
-                    if (path.every((v, j) => v === rule[1][j])) {
-                        addNew = false;
-                        break;
-                    }
-                }
-                if (addNew) {
-                    this.candidates.rules.set(ruleStack[i], path);
-                    if (this.showDebugOutput)
-                        console.log("=====> collected: ", this.ruleNames[i]);
-                }
+                this.candidates.rules.add(ruleStack[i]);
+                if (this.showDebugOutput)
+                    console.log("=====> collected: ", this.ruleNames[i]);
                 return true;
@@ -202,3 +203,16 @@ }
     processRule(startState, tokenIndex, callStack, indentation) {
-        let result = [];
+        let result = new Set();
+        let positionMap = this.shortcutMap.get(startState.ruleIndex);
+        if (!positionMap) {
+            positionMap = new Map();
+            this.shortcutMap.set(startState.ruleIndex, positionMap);
+        }
+        else {
+            if (positionMap.has(tokenIndex)) {
+                if (this.showDebugOutput) {
+                    console.log("=====> shortcut");
+                }
+                return positionMap.get(tokenIndex);
+            }
+        }
         let setsPerState = CodeCompletionCore.followSetsByATN.get(this.parser.constructor.name);
@@ -233,4 +247,5 @@ if (!setsPerState) {
                             if (!this.ignoredTokens.has(symbol)) {
-                                if (this.showDebugOutput)
+                                if (this.showDebugOutput) {
                                     console.log("=====> collected: ", this.vocabulary.getDisplayName(symbol));
+                                }
                                 if (!this.candidates.tokens.has(symbol))
@@ -255,4 +270,2 @@ this.candidates.tokens.set(symbol, set.following);
         }
-        let isLeftRecursive = startState.leftFactored;
-        let forceLoopEnd = false;
         let statePipeline = [];
@@ -275,34 +288,6 @@ let currentEntry;
                     break;
-                case 7:
-                    {
-                        let returnIndex = callStack[callStack.length - 2];
-                        let transitions = currentEntry.state.getTransitions();
-                        const transitionCount = transitions.length;
-                        for (let i = transitionCount - 1; i >= 0; --i) {
-                            let transition = transitions[i];
-                            if (transition.target.ruleIndex == returnIndex) {
-                                let canAdd = true;
-                                for (let state of result) {
-                                    if (state.state == transition.target && state.tokenIndex == currentEntry.tokenIndex) {
-                                        canAdd = false;
-                                        break;
-                                    }
-                                }
-                                if (canAdd)
-                                    result.push({ state: transition.target, tokenIndex: currentEntry.tokenIndex });
-                            }
-                        }
-                        continue;
-                    }
-                case 10:
-                    if (forceLoopEnd) {
-                        for (let transition of currentEntry.state.getTransitions()) {
-                            if (transition.target.stateType == 12) {
-                                statePipeline.push({ state: transition.target, tokenIndex: currentEntry.tokenIndex });
-                                break;
-                            }
-                        }
-                        continue;
-                    }
-                    break;
+                case 7: {
+                    result.add(currentEntry.tokenIndex);
+                    continue;
+                }
                 default:
@@ -316,5 +301,5 @@ break;
                     let endStatus = this.processRule(transition.target, currentEntry.tokenIndex, callStack, indentation);
-                    statePipeline.push(...endStatus);
-                    if (isLeftRecursive && transition.target.ruleIndex == callStack[callStack.length - 1])
-                        forceLoopEnd = true;
+                    for (let position of endStatus) {
+                        statePipeline.push({ state: transition.followState, tokenIndex: position });
+                    }
                 }
@@ -373,2 +358,3 @@ else if (transition.serializationType == 4) {
         callStack.pop();
+        positionMap.set(tokenIndex, result);
         return result;
@@ -378,3 +364,3 @@ }
         let stateValue = state.stateNumber == atn_1.ATNState.INVALID_STATE_NUMBER ? "Invalid" : state.stateNumber;
-        return "[" + stateValue + " " + state.stateType + "] in " + this.ruleNames[state.ruleIndex];
+        return "[" + stateValue + " " + this.atnStateTypeMap[state.stateType] + "] in " + this.ruleNames[state.ruleIndex];
     }
@@ -401,3 +387,3 @@ printDescription(currentIndent, state, baseDescription, tokenIndex) {
                 transitionDescription += "\n" + currentIndent + "\t(" + labels + ") " + "[" + transition.target.stateNumber + " " +
-                    transition.target.stateType + "] in " + this.ruleNames[transition.target.ruleIndex];
+                    this.atnStateTypeMap[transition.target.stateType] + "] in " + this.ruleNames[transition.target.ruleIndex];
             }
@@ -404,0 +390,0 @@ }

package.json

 {
   "name": "antlr4-c3",
-  "version": "1.0.11",
+  "version": "1.0.12",
   "description": "A code completion core implmentation for ANTLR4 based parsers",
@@ -8,6 +8,5 @@ "main": "out/index.js",
   "scripts": {
-    "prepublish": "tsc",
-    "antlr4ts": "antlr4ts test/Expr.g4 test/CPP14.g4 -no-visitor -no-listener",
-    "compile": "tsc --outDir ./out/src",
-    "test": "mocha --compilers ts:ts-node/register,tsx:ts-node/register"
+    "prepare": "tsc",
+    "prepublish": "check-node-version --npm \">=4\" || npm run prepare",
+    "test": "tsc && mocha out/test"
   },
@@ -31,2 +30,3 @@ "repository": {
     "@types/node": "^6.0.40",
+    "@types/fs-extra": "^3.0.3",
     "antlr4ts-cli": "^0.4.0-alpha.4",
@@ -36,3 +36,6 @@ "chai": "^3.5.0",
     "typescript": "^2.0.3",
-    "ts-node": "^1.7.3"
+    "ts-node": "^1.7.3",
+    "fs-extra": "^3.0.1",
+    "path": "^0.12.7",
+    "check-node-version": "^1.1.2"
   },
@@ -39,0 +42,0 @@ "author": "Mike Lischke",

readme.md

@@ -21,3 +21,3 @@ [![NPM](https://nodei.co/npm/antlr4-c3.png?downloads=true&downloadRank=true)](https://nodei.co/npm/antlr4-c3/)
 
-While the symbol table provides symbols of a given type, we need to find out which type is actually required. This is the task of the c3 engine. In its simplest setup it will return only keywords that are allowed by the grammar for a given position (which is of course the same position used to find the context for a symbol lookup in your symbol table). Keywords are a fixed set of words (or word sequences) that usually don't live in a symbol table. You can get the actual text strings directly from the parser's vocabulary. The c3 engine only returns the lexer tokens for them.
+While the symbol table provides symbols of a given type, we need to find out which type is actually required. This is the task of the c3 engine. In its simplest setup it will return only keywords (and other lexer symbols) that are allowed by the grammar for a given position (which is of course the same position used to find the context for a symbol lookup in your symbol table). Keywords are a fixed set of words (or word sequences) that usually don't live in a symbol table. You can get the actual text strings directly from the parser's vocabulary. The c3 engine only returns the lexer tokens for them.
 
@@ -97,16 +97,16 @@ In order to also get other types like variables or class names you have to do 2 steps:
 
-All these could be passed in individually, but since your parser contains all of that anyway, the API has been designed to take a parser instead. In real world applications you will have a parser anyway (e.g. for error checking), which is perfect as ATN and input provider for the code completion core. But keep in mind: whatever parser you pass in it must have a fully set up token stream.
+All these could be passed in individually, but since your parser contains all of that anyway and we need a parser for predicate execution, the API has been designed to take a parser instead. In real world applications you will have a parser anyway (e.g. for error checking), which is perfect as ATN and input provider for the code completion core. But keep in mind: whatever parser you pass in it must have a fully set up token stream. It's not required that it parsed anything before calling the code completion engine and the current stream positions don't matter either.
 
 The returned candidate collection contains fields for lexer tokens (mostly keywords, but also other tokens if they are not on the ignore list) and parser rule indexes. This collection is defined as:
 
-    export class CandidatesCollection {
-        public tokens: Map<number, number[]> = new Map();
-        public rules: Map<number, number[]> = new Map();
+   class CandidatesCollection {
+        public tokens: Map<number, TokenList>;
+        public rules: RuleList;
     };
 
-where the map keys are the lexer tokens and the rule indexes, respectively. Both can come with additional numbers, which you may or may not use for your implementation. For parser rules the array represents a stack of rule indexes at which the given rule was found during evaluation. That's probably something only rarely used (mostly for debugging), however for the lexer tokens the array consists of further token ids which directly follow the given token in the grammar (if any). That's quite a neat additional feature which allows you to show token sequences to the user if they are always used together. For example consider this SQL rule:
+For the lexer tokens there can be a list of extra tokens which directly follow the given token in the grammar (if any). That's quite a neat additional feature which allows you to show token sequences to the user if they are always used together. For example consider this SQL rule:
 
     createTable: CREATE TABLE (IF NOT EXISTS)? ...;
 
-Here, if a possible candidate is the `IF` keyword, you can also show the entire `IF NOT EXISTS` sequence to the user (and let him complete all 3 words in one go in his/her source code). The engine will return a candidate entry for `IF` with an array containing `NOT` and `EXISTS`. This list will of course update properly when the user comes to `NOT`. Then you will get a candidate entry for `NOT` and an additional list of just `EXISTS`.
+Here, if a possible candidate is the `IF` keyword, you can also show the entire `IF NOT EXISTS` sequence to the user (and let him complete all 3 words in one go in the source code). The engine will return a candidate entry for `IF` with a token list containing `NOT` and `EXISTS`. This list will of course update properly when the user comes to `NOT`. Then you will get a candidate entry for `NOT` and an additional list of just `EXISTS`.
 
@@ -113,0 +113,0 @@ Essential for getting any rule index, which you can use to query your symbol table, is that you specify those you want in the `CodeCompletionCore.preferredRules` field before running `CodeCompletionCore.collectCandidates()`.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Worsened metrics

Total package byte prevSize

160681

decreased by-3.1%

Dev dependency count

12

increased by50%

Lines of code

1220

decreased by-0.81%

No dependency changes