@bfwk/expression-completion - npm Package Compare versions

Comparing version 0.6.7003 to 0.7.0

CHANGELOG.md

@@ -6,13 +6,10 @@ # Change Log
 
-## 0.6.7003 (2021-04-21)
+# [0.7.0](https://github.com/salesforce/builder-framework/compare/v0.6.49...v0.7.0) (2021-07-28)
 
+**Note:** Version bump only for package @bfwk/expression-completion
 
-### Features
 
-* refactor from lbf to bfwk @W-9091273 ([#1](https://github.com/salesforce/builder-framework/issues/1)) ([5300a61](https://github.com/salesforce/builder-framework/commit/5300a6117f7f1c763442d7e50d157443e22c1e36))
 
 
 
-
-
 ## [0.6.7](https://git.soma.salesforce.com/BuilderFramework/builder-framework/compare/v0.6.6...v0.6.7) (2021-04-05)
@@ -19,0 +16,0 @@

dist/types/customized-lexer.d.ts

 import antlr4 from 'antlr4';
-import ExpressionLexer from './gen/ExpressionLexer';
+import UnifiedLexer from './gen/UnifiedLexer';
 /**
@@ -10,6 +10,9 @@ * @author drobertson
  */
-export declare class CustomizedLexer extends ExpressionLexer {
+export declare class CustomizedLexer extends UnifiedLexer {
     static BeginTag: string;
     static EndTag: string;
+    private _fullFormulaGrammar;
     constructor(input: antlr4.InputStream);
+    get fullFormulaGrammar(): boolean;
+    set fullFormulaGrammar(ffg: boolean);
     emit(): antlr4.CommonToken;
@@ -16,0 +19,0 @@ /**

dist/types/expression-completion.d.ts

@@ -1,2 +0,2 @@
-import { Json } from '@lcem/declarative-type-validator';
+import { Json } from '@lcem/meta-schemas';
 /**
@@ -27,2 +27,7 @@ * The location of the insertion point within the provided expression text.
     /**
+     * Use full formula parser grammar (as opposed to fieldReference only).
+     * Defaults to false (for 234, anyway).
+     */
+    fullFormulaGrammar?: boolean;
+    /**
      * Enables some console debugging.
@@ -81,2 +86,3 @@ * Defaults to false.
     private get requireDelimiters();
+    private get fullFormulaGrammar();
     private get debug();
@@ -98,4 +104,5 @@ private get requiresReadaheadHack();
     private suggestCompletions;
+    private getTokenText;
     private buildContextMap;
     private addObject;
 }

dist/types/symbol-table-visitor.d.ts

 import antlr4 from 'antlr4';
-import ExpressionVisitor from './gen/ExpressionVisitor';
-import ExpressionParser from './gen/ExpressionParser';
+import UnifiedExpressionVisitor from './gen/UnifiedExpressionVisitor';
+import UnifiedExpression from './gen/UnifiedExpression';
 import { ContextMap } from './context-map';
@@ -24,3 +24,3 @@ import { ParseError } from './expression-completion';
  */
-export declare class SymbolTableVisitor extends ExpressionVisitor {
+export declare class SymbolTableVisitor extends UnifiedExpressionVisitor {
     private readonly contexts;
@@ -34,9 +34,11 @@ private scope?;
     private getNamespaceForContext2;
-    visitBegin(ctx: ExpressionParser.BeginContext): any;
-    visitEnd(ctx: ExpressionParser.EndContext): any;
-    visitPropertyContext(ctx: ExpressionParser.PropertyContextContext): any;
-    visitProperty(ctx: ExpressionParser.PropertyContext): any;
-    visitDotSeparator(ctx: ExpressionParser.DotSeparatorContext): any;
-    getText(ctx: antlr4.ParserRuleContext, caret?: number): string;
+    visitBegin(ctx: UnifiedExpression.BeginContext): any;
+    visitEnd(ctx: UnifiedExpression.EndContext): any;
+    visitFormula(ctx: UnifiedExpression.FormulaContext): void;
+    visitFieldReferenceLiteral(ctx: UnifiedExpression.FieldReferenceLiteralContext): any;
+    visitFieldReferenceLiteral2(ctx: UnifiedExpression.FieldReferenceLiteral2Context): any;
+    visitDotSeparator(ctx: UnifiedExpression.DotSeparatorContext): any;
+    visitFieldSelector(ctx: UnifiedExpression.FieldSelectorContext): any;
+    getText(ctx: antlr4.ParserRuleContext, caret?: number, index?: number): string;
     private getFullyQualifiedName;
 }

package.json

 {
   "name": "@bfwk/expression-completion",
-  "version": "0.6.7003",
+  "version": "0.7.0",
   "description": "LBF Expression Completion Engine",
@@ -9,3 +9,4 @@ "type": "module",
   "files": [
-    "dist/"
+    "dist/",
+    "index.xml"
   ],
@@ -22,7 +23,7 @@ "publishConfig": {
   "scripts": {
-    "clean": "rm -rf dist/ build/ src/gen/",
+    "clean": "rm -rf dist/ build/ src/gen/ src/Formula.g4 src/LexerRules.g4",
     "build:ts": "tsc",
     "build:target:npm": "rollup -c ./rollup-npm.config.js",
     "build:target:lwc": "rollup -c ./rollup-esNext.config.js",
-    "build": "yarn antlr4 && yarn build:ts && yarn cpgen && yarn build:target:lwc && yarn build:target:npm",
+    "build": "yarn cp:grammars && yarn antlr4 && yarn build:ts && yarn cp:gen && yarn build:target:lwc && yarn build:target:npm",
     "watch": "watch -p 'src/**/*.ts' -c 'yarn build'",
@@ -32,6 +33,8 @@ "lint": "eslint 'src/**/*.{js,ts}'",
     "test:coverage": "jest --coverage",
-    "antlr4": "java -jar bin/antlr-4.9.1-complete.jar -Dlanguage=JavaScript -visitor -Xexact-output-dir -o src/gen src/Expression.g4 src/ExpressionLexer.g4",
-    "cpgen": "cp -r src/gen/ build/gen/"
+    "antlr4": "java -jar bin/antlr-4.9.1-complete.jar -Dlanguage=JavaScript -visitor -Xexact-output-dir -o src/gen src/UnifiedExpression.g4 src/UnifiedLexer.g4",
+    "cp:grammars": "scripts/cpgrammars.sh",
+    "cp:gen": "cp -r src/gen/ build/gen/"
   },
   "dependencies": {
+    "@lcem/meta-schemas": "0.7.0",
     "antlr4": "^4.9.1"
@@ -38,0 +41,0 @@ },

README.md

@@ -1,9 +0,9 @@
-# Expression Editor
+# Expression Completion Engine
 
-The Expression Editor is a general purpose editor/validator for expressions defined by an
+The Expression Completion Engine is a general purpose editor/validator for expressions defined by an
 [ANTLR](https://www.antlr.org/) grammar and with input contexts described by JSON Schema.
 
-The grammar (in 232) is a small initial subset of the existing expression grammar (Formula.g4). In order to use a
+The grammar is a superset of the legacy expression grammar (Formula.g4). In order to use a
 grammar for code completion it is useful to inject various rules to facilitate the discovery of a token's scope wrt
-completion.
+completion, so there is some rule redefinition.
 
@@ -14,10 +14,17 @@ This uses the [Javascript ANTLR4 runtime](https://www.npmjs.com/package/antlr4).
 
-The build:dev target includes a code generation step, based upon the grammar
-(Expression.g4), to create various grammar specific source files in a non-repository gen/ directory. The code generator
-is Java-based and saved in the repository in the bin/ directory. Directions for running the code generator are
-available on the [ANTLR](https://www.antlr.org/) site.
+The build target includes a code generation step, based upon the grammar (UnifiedExpression.g4), to create various
+grammar specific source files in a non-repository gen/ directory. The code generator is Java-based and saved in the
+repository in the bin/ directory. Directions for running the code generator are available on the
+[ANTLR](https://www.antlr.org/) site.
 
-Do NOT checkin changes to the grammar without rebuilding and testing - the generated sources might be altered, and
-might become incompatible with the other source files.
+Do NOT checkin changes to the grammar without rebuilding and testing first - the generated sources will be altered and,
+while not saved in the repo, might become incompatible with the other source files.
 
+Note that we use an unchanged reference copy of the legacy parser and lexer grammars which we then munge slightly
+into a local copy for import into the unified grammars. The cpgrammars.sh script is the _only_ place that is aware of
+these reference grammars. The rest of the build process uses locally generated versions in place. For now the legacy
+grammars are copied into our repo, but they could be dynamically recovered from their
+[home repo](https://git.soma.salesforce.com/sfdc-commons/formula-engine) at some point. The script might need changes
+to keep the locally generated grammars consistent with our local expectations (documented within the script).
+
 ## Future Directions
@@ -29,1 +36,103 @@
 the branch https://git.soma.salesforce.com/BuilderFramework/builder-framework/tree/drobertson/DO-NOT-DELETE-typescript-completion-engine
+
+## How it Works
+
+The completion engine works generally as follows:
+
+1. Parse the candidate expression (often incomplete or otherwise non-grammar compliant) to get a token stream
+2. Locate the token most closely "associated" with the input caret position
+3. Locate the completion candidates associated with that token
+4. Optionally apply a candidate to the expression to create a new expression with a new caret position and a "smart"
+   substitution
+
+### Step 2. Finding the token (token-finder.ts)
+
+We need to navigate the parse tree to find the terminal node at the caret position. Note that this does not have to be
+at the end of the expression - the user might have clicked into the input field or left-arrowed to a position within the
+current expression.
+
+### Step 3. Locating the related completion candidates (symbol-table-visitor.ts)
+
+This is the bulk of the logic. This is where semantic knowledge is introduced.
+
+In order to capture the semantic state of the expression it is necessary to "massage" the base grammar
+so that the parser listener is triggered at semantically useful moments. This is easiest done by introducing parser
+rules at appropriate points within the grammar. This allows such things as sensible completion suggestions after a dot
+and scoping of candidates within nested field references.
+
+The symbol table that is built by the listener during the parse process is keyed on token.
+That is how the completion candidates are associated with the input token found during Step 2. The symbol table
+represents the aggregate semantic knowledge of the expression (wrt completion anyway) at any token position.
+
+The current implementation strives to base itself off the legacy formula
+grammar without requiring changes to it that would be visible to other consumers. It does this by manipulating the
+parser and lexer grammars "in-place" through a number of ANTLR tricks.
+
+### Entry points
+
+The current completion engine allows and can be configured for three parser entry points.
+
+1. Expressions with string interpolation (eg. 'abc{!exp1}def{!exp2}ghi')
+2. Expressions with delimiters (eg. '{!exp}') - the legacy formula grammar
+3. Raw, non-delimited expressions (eg. 'exp')
+
+These are defined in terms of each other wrt parser and lexer rules so as much as is possible the grammar and supporting
+code is reused.
+
+## Grammar hacks/issues
+
+### Essential ANTLR Knowledge
+
+There are three things you really have to understand about ANTLR to make sense of the grammar manipulations that are
+employed:
+
+1. An ANTLR import is an append operation. Everything imported is added to the end of the grammar containing the import.
+   The order of multiple imports is important too. The appends happen in the order that the imports appear.
+2. ANTLR always honors the first definition it finds.
+3. Order of rules is very important, especially for lexer rules. If you want don't want 'true' to be treated as an
+   identifier then you had better define TRUE before IDENT.
+
+### ANTLR tricks for local grammar manipulation
+
+Local manipulation of the legacy grammars comes in two forms:
+
+1. Direct edits of the grammar files
+2. Rules overrides and new rules within the unified grammars
+
+Direct edits are applied to the reference copies of the legacy grammars (in the reference/ directory) by shell scripts
+(in the scripts/ directory) to produce temporary modified legacy grammar files (in the src/ directory).
+These form the starting point for subsequent manipulation by the override tactics. An example of a direct edit is the
+removal of Java grammar embeds. These scripts are sensitive and will have to be maintained as changes to the legacy
+formula grammar are made. They strive not to change the underlying grammar syntax or semantic at all but rather simply
+aim to make the legacy grammar files usable within the unified grammar structure (pure vs combined grammars etc).
+
+We create new rules within the unified parser and lexer grammars and override legacy rules for a number of reasons:
+
+1. Adding entry points and removing existing entry points (managing EOF).
+2. Overriding the behavior of the troublesome IDENT token.
+3. Adding parser rules to facilitate the construction of the symbol table in the parser listener.
+4. Adding support for string interpolation.
+
+### Modal lexer
+
+String interpolation requires the use of a modal lexer. This has downstream consequences that affect the relationships
+between parser and lexer grammars. In particular, you cannot import a modal lexer into a parser grammar in ANTLR to form
+a combined grammar - you have to reference the tokens from a pure lexer grammar.
+
+### IDENT lexer token
+
+The formula grammar contains a definition of the IDENT token that is inappropriate for our needs. This is an intractable
+problem if we were to use the legacy lexer grammar as is. This is quite a severe problem. The legacy IDENT rule consumes
+all parts of a qualified name (ie. 'a.b.c') including the dot separators. It also allows for otherwise questionable
+identifiers like 'a.1'.
+
+For our purposes we needed to break up this lexer token and redefine the uses of IDENT to use the newly defined rules
+(see UnifiedLexer2.g4). This works fine, tortuous though it is, but it has the consequence that identifiers within the
+expression completion engine are more restrictive than those within the legacy formula grammar. _This is the only
+grammatical inconsistency as far as I know._
+
+### Code hack for readahead (expression-completion.ts, customized-lexer.ts)
+
+In order to get the string interpolation modal grammar to work we need a readahead mechanism.
+This works fine within the grammar for parsing, but we need compensation within the code to account for tokens that were
+read but not consumed. This is undoubtedly a hack, but at least it's isolated.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

746712

increased by37.61%

Number of package files

13

increased by8.33%

Lines of code

18930

increased by30.37%

Number of lines in readme file

137

increased by389.29%

Worsened metrics

Dependency count

2

increased by100%

Dependency changes

• + Added@lcem/meta-schemas@0.7.0

• + Added@lcem/meta-schemas@0.7.0(transitive)