@helios-lang/compiler-utils - npm Package Compare versions

Comparing version 0.1.54 to 0.1.55

package.json

 {
-    "name": "@helios-lang/compiler-utils",
-    "version": "0.1.54",
-    "description": "Helios language compiler library",
-    "main": "src/index.js",
-    "types": "types/index.d.ts",
-    "type": "module",
-    "author": "Christian Schmitz",
-    "license": "BSD-3-Clause",
-    "scripts": {
-        "build": "npm run prettify && npm run build:types && npm run test:suite",
-        "build:types": "npx tsc -p jsconfig.json --noEmit false --emitDeclarationOnly",
-        "prettify": "npx prettier . --write",
-        "test": "npm run test:pretty && npm run test:types && npm run test:suite",
-        "test:pretty": "npx prettier . --check",
-        "test:suite": "node --test  --experimental-test-coverage",
-        "test:types": "npx tsc -p jsconfig.json --noEmit"
-    },
-    "devDependencies": {
-        "@types/node": "^20.11.24",
-        "prettier": "^3.1.0",
-        "typescript": "^5.3.2"
-    },
-    "dependencies": {
-        "@helios-lang/codec-utils": "^0.1.30",
-        "@helios-lang/type-utils": "^0.1.18"
-    },
-    "prettier": {
-        "trailingComma": "none",
-        "tabWidth": 4,
-        "semi": false,
-        "singleQuote": false
-    }
-}
+  "name": "@helios-lang/compiler-utils",
+  "version": "0.1.55",
+  "description": "Helios language compiler library",
+  "main": "src/index.js",
+  "types": "types/index.d.ts",
+  "type": "module",
+  "author": "Christian Schmitz",
+  "license": "BSD-3-Clause",
+  "devDependencies": {
+    "@types/node": "^20.11.24",
+    "prettier": "^3.1.0",
+    "typescript": "^5.3.2"
+  },
+  "dependencies": {
+    "@helios-lang/codec-utils": "^0.1.35",
+    "@helios-lang/type-utils": "^0.1.18"
+  },
+  "prettier": {
+    "trailingComma": "none",
+    "tabWidth": 4,
+    "semi": false,
+    "singleQuote": false
+  },
+  "scripts": {
+    "build": "pnpm run prettify && pnpm run build:types && pnpm run test:suite",
+    "build:types": "tsc -p jsconfig.json --noEmit false --emitDeclarationOnly",
+    "prettify": "prettier . --write",
+    "test": "pnpm run test:pretty && pnpm run test:types && pnpm run test:suite",
+    "test:pretty": "prettier . --check",
+    "test:suite": "node --test  --experimental-test-coverage",
+    "test:types": "tsc -p jsconfig.json --noEmit"
+  }
+}

src/tokens/Source.js

@@ -7,2 +7,4 @@ /**
 
+import { None } from "@helios-lang/type-utils"
+import { segmentArray } from "@helios-lang/codec-utils"
 /**
@@ -20,3 +22,24 @@ * A Source instance wraps a string so we can use it cheaply as a reference inside a Site.
     /**
+     * Number of characters in source content
+     * @type {number}
      * @readonly
+     */
+    length
+
+    /**
+     * Number of characters in each chunk
+     * @type {number}
+     * @readonly
+     * */
+    chunkSize
+
+    /**
+     * Segemented zones of the source content for more efficient access
+     * @readonly
+     * @type {string[][]}
+     */
+    contentChunks
+
+    /**
+     * @readonly
      * @type {string}
@@ -27,2 +50,9 @@ */
     /**
+     * cache of line lengths in input source.  See lineLengths getter.
+     * @type{Option<number[]>}
+     * @private
+     */
+    _lineEndLocations
+
+    /**
      * @param {string} content
@@ -33,15 +63,17 @@ * @param {SourceOptions} options
         this.content = content
+        // one-step split to utf-8 runes in the content
+        const asCodePoints = [...content]
+        // heuristic for chunk size
+        this.chunkSize = Math.max(
+            100,
+            Math.floor(Math.sqrt(asCodePoints.length))
+        )
+        this.contentChunks = segmentArray(asCodePoints, this.chunkSize)
+        this.length = asCodePoints.length
         this.name = options.name ?? "unknown"
+        this._lineEndLocations = None
     }
 
     /**
-     * Number of characters in source content
-     * @type {number}
-     */
-    get length() {
-        return this.content.length
-    }
-
-    /**
-     * Get character from the underlying string.
+     * Get character from the underlying string index
      * Should work fine with utf-8 runes
@@ -52,3 +84,12 @@ * @param {number} i
     getChar(i) {
-        return this.content[i]
+        const targetChunk =
+            i == this.length
+                ? []
+                : this.contentChunks[Math.floor(i / this.chunkSize)]
+
+        if (!targetChunk) {
+            throw new Error(`invalid position in Source ${this.name}`)
+        }
+        const offset = i % this.chunkSize
+        return targetChunk[offset]
     }
@@ -84,3 +125,3 @@
 
-        let c = this.content[i]
+        let c = this.getChar(i)
 
@@ -90,3 +131,3 @@ while (isWordChar(c)) {
             i += 1
-            c = this.content[i]
+            c = this.getChar(i)
         }
@@ -97,2 +138,18 @@
 
+    /*
+     * Returns the location of each line-ending for fast line/column number lookup
+     * @returns {number[]}
+     */
+    get lineEndLocations() {
+        if (this._lineEndLocations) return this._lineEndLocations
+
+        let lastOffset = 0
+        return (this._lineEndLocations = this.content
+            .split("\n")
+            .map((line) => {
+                const len = [...line].length //utf-8 rune count
+                return (lastOffset += len + 1)
+            }))
+    }
+
     /**
@@ -104,13 +161,8 @@ * Calculates the line and column number where the given character is located
     getPosition(i) {
-        let col = 0
-        let line = 0
-
-        for (let j = 0; j < i; j++) {
-            if (this.content[j] == "\n") {
-                col = 0
-                line += 1
-            } else {
-                col += 1
-            }
+        const lineEndings = this.lineEndLocations
+        if (i < 0 || i > this.length) {
+            throw new Error("invalid position in Source")
         }
+        const line = lineEndings.findIndex((endOffset) => i < endOffset)
+        const col = i - (line > 0 ? lineEndings[line - 1] : 0)
 
@@ -117,0 +169,0 @@ return [line, col]

types/tokens/Comment.d.ts

@@ -20,3 +20,3 @@ /**
     value: string;
-    site: import("./Token.js").Site;
+    site: import("../errors/Site.js").Site;
     /**
@@ -23,0 +23,0 @@ * @param {Token} other

types/tokens/Group.d.ts

@@ -10,3 +10,3 @@ /**
  */
-export class Group<F extends import("./Token.js").Token[] | {
+export class Group<F extends Token[] | {
     tokens: Token[];
@@ -13,0 +13,0 @@ } = import("./Token.js").Token[]> implements Token {

types/tokens/index.d.ts

@@ -15,5 +15,5 @@ export { BoolLiteral } from "./BoolLiteral.js";
 export type Token = import("./Token.js").Token;
-export type TokenMatcher<T extends import("./Token.js").Token = import("./Token.js").Token> = import("./TokenMatcher.js").TokenMatcher<T>;
+export type TokenMatcher<T extends Token = import("./Token.js").Token> = import("./TokenMatcher.js").TokenMatcher<T>;
 export { REAL_PRECISION, RealLiteral } from "./RealLiteral.js";
 export { anySymbol, anyWord, boollit, byteslit, group, intlit, oneOf, reallit, strlit, symbol, wildcard, word } from "./TokenMatcher.js";
 //# sourceMappingURL=index.d.ts.map

types/tokens/Source.d.ts

 /**
- * @typedef {{
- *   name?: string
- * }} SourceOptions
- */
-/**
  * A Source instance wraps a string so we can use it cheaply as a reference inside a Site.
@@ -22,3 +17,21 @@ * Also used by VSCode plugin
     /**
+     * Number of characters in source content
+     * @type {number}
      * @readonly
+     */
+    readonly length: number;
+    /**
+     * Number of characters in each chunk
+     * @type {number}
+     * @readonly
+     * */
+    readonly chunkSize: number;
+    /**
+     * Segemented zones of the source content for more efficient access
+     * @readonly
+     * @type {string[][]}
+     */
+    readonly contentChunks: string[][];
+    /**
+     * @readonly
      * @type {string}
@@ -28,8 +41,9 @@ */
     /**
-     * Number of characters in source content
-     * @type {number}
+     * cache of line lengths in input source.  See lineLengths getter.
+     * @type{Option<number[]>}
+     * @private
      */
-    get length(): number;
+    private _lineEndLocations;
     /**
-     * Get character from the underlying string.
+     * Get character from the underlying string index
      * Should work fine with utf-8 runes
@@ -46,2 +60,3 @@ * @param {number} i
     getWord(i: number): string;
+    get lineEndLocations(): number[];
     /**
@@ -48,0 +63,0 @@ * Calculates the line and column number where the given character is located

types/tokens/SourceIndex.d.ts

@@ -10,3 +10,3 @@ /**
      */
-    constructor(source: Source, sourceMap: Option<import("./SourceMap.js").SourceMap>);
+    constructor(source: Source, sourceMap: Option<SourceMap>);
     /**
@@ -21,3 +21,3 @@ * @readonly
      */
-    readonly sourceMap: Option<import("./SourceMap.js").SourceMap>;
+    readonly sourceMap: Option<SourceMap>;
     /**
@@ -38,3 +38,3 @@ * @type {number}
      */
-    get site(): import("./TokenSite.js").Site;
+    get site(): import("../errors/Site.js").Site;
     /**
@@ -41,0 +41,0 @@ * @private

types/tokens/StringLiteral.d.ts

@@ -26,3 +26,3 @@ /**
     readonly value: string;
-    site: import("./TokenSite.js").Site;
+    site: import("../errors/Site.js").Site;
     /**
@@ -29,0 +29,0 @@ * @param {Token} other

types/tokens/Tokenizer.d.ts

@@ -68,3 +68,3 @@ export class Tokenizer {
      */
-    get currentSite(): import("./BoolLiteral.js").Site;
+    get currentSite(): import("../errors/Site.js").Site;
     /**
@@ -214,3 +214,3 @@ * @param {Site} start
 export type TokenizerOptions = {
-    sourceMap?: Map<number, import("./BoolLiteral.js").Site>;
+    sourceMap?: SourceMap;
     extraValidFirstLetters?: string;
@@ -217,0 +217,0 @@ realPrecision?: number;

types/tokens/TokenMatcher.d.ts

@@ -36,3 +36,3 @@ /**
  */
-export function oneOf<Matchers extends TokenMatcher<import("./Token.js").Token>[]>(matchers: [...Matchers]): TokenMatcher<Matchers extends TokenMatcher<infer T extends import("./Token.js").Token>[] ? T : never>;
+export function oneOf<Matchers extends TokenMatcher[]>(matchers: [...Matchers]): TokenMatcher<Matchers extends Array<TokenMatcher<infer T>> ? T : never>;
 /**
@@ -89,3 +89,3 @@ * @param {Option<string>} value
  */
-export type TokenMatcher<T extends import("./Token.js").Token = import("./Token.js").Token> = {
+export type TokenMatcher<T extends Token = import("./Token.js").Token> = {
     matches: (t: Token) => Option<T>;
@@ -92,0 +92,0 @@ toString: () => string;

types/tokens/TokenReader.d.ts

@@ -54,3 +54,3 @@ /**
      */
-    assert<Matchers extends TokenMatcher<import("./Token.js").Token>[]>(...matchers_0: Matchers): TokenReader;
+    assert<Matchers extends TokenMatcher[]>(...matchers_0: Matchers): TokenReader;
     end(): void;
@@ -64,3 +64,3 @@ /**
      */
-    findNext<Matchers_1 extends TokenMatcher<import("./Token.js").Token>[]>(...matchers_0: Matchers_1): Option<[TokenReader, ...MatcherTokens<Matchers_1>]>;
+    findNext<Matchers extends TokenMatcher[]>(...matchers_0: Matchers): Option<[TokenReader, ...MatcherTokens<Matchers>]>;
     /**
@@ -73,3 +73,3 @@ * Looks for the last token that matches the `matcher`
      */
-    findNextMatch<Matchers_2 extends TokenMatcher<import("./Token.js").Token>[]>(...matchers_0: Matchers_2): Option<[TokenReader, ...MatcherTokens<Matchers_2>]>;
+    findNextMatch<Matchers extends TokenMatcher[]>(...matchers_0: Matchers): Option<[TokenReader, ...MatcherTokens<Matchers>]>;
     /**
@@ -89,3 +89,3 @@ * @private
      */
-    findLast<Matchers_3 extends TokenMatcher<import("./Token.js").Token>[]>(...matchers_0: Matchers_3): Option<[TokenReader, ...MatcherTokens<Matchers_3>]>;
+    findLast<Matchers extends TokenMatcher[]>(...matchers_0: Matchers): Option<[TokenReader, ...MatcherTokens<Matchers>]>;
     /**
@@ -98,3 +98,3 @@ * Looks for the last token that matches the `matcher`
      */
-    findLastMatch<Matchers_4 extends TokenMatcher<import("./Token.js").Token>[]>(...matchers_0: Matchers_4): Option<[TokenReader, ...MatcherTokens<Matchers_4>]>;
+    findLastMatch<Matchers extends TokenMatcher[]>(...matchers_0: Matchers): Option<[TokenReader, ...MatcherTokens<Matchers>]>;
     /**
@@ -115,3 +115,3 @@ * @private
      */
-    readUntil<Matchers_5 extends TokenMatcher<import("./Token.js").Token>[]>(...matchers_0: Matchers_5): TokenReader;
+    readUntil<Matchers extends TokenMatcher[]>(...matchers_0: Matchers): TokenReader;
     /**
@@ -126,3 +126,3 @@ * @returns {boolean}
      */
-    matches<Matchers_6 extends TokenMatcher<import("./Token.js").Token>[]>(...matchers_0: Matchers_6): Option<UnwrapSingleton<MatcherTokens<Matchers_6>>>;
+    matches<Matchers extends TokenMatcher[]>(...matchers_0: Matchers): Option<UnwrapSingleton<MatcherTokens<Matchers>>>;
     /**
@@ -172,6 +172,6 @@ * @param {boolean | string} throwFail - defaults to true. `throwFail` as a string specifies a custom error message if all matches failed
 export type Token = import("./Token.js").Token;
-export type TokenMatcher<T extends import("./Token.js").Token = import("./Token.js").Token> = import("./TokenMatcher.js").TokenMatcher<T>;
-export type AugmentGroup<T extends import("./Token.js").Token> = T extends Group ? Group<TokenReader> : T;
-export type MatcherTokens<Matchers extends TokenMatcher<import("./Token.js").Token>[]> = { [M in keyof Matchers]: Matchers[M] extends TokenMatcher<infer T extends import("./Token.js").Token> ? AugmentGroup<T> : never; };
-export type UnwrapSingleton<Tokens extends (import("./Token.js").Token | Group<TokenReader>)[]> = Tokens extends [infer T] ? T : Tokens;
+export type TokenMatcher<T extends Token = import("./Token.js").Token> = import("./TokenMatcher.js").TokenMatcher<T>;
+export type AugmentGroup<T extends Token> = T extends Group ? Group<TokenReader> : T;
+export type MatcherTokens<Matchers extends TokenMatcher[]> = { [M in keyof Matchers]: Matchers[M] extends TokenMatcher<infer T> ? AugmentGroup<T> : never; };
+export type UnwrapSingleton<Tokens extends (Group<TokenReader> | Token)[]> = Tokens extends [infer T] ? T : Tokens;
 import { ErrorCollector } from "../errors/ErrorCollector.js";
@@ -178,0 +178,0 @@ import { Group } from "./Group.js";

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

136434

increased by2.56%

Lines of code

4161

increased by1.51%

Dependency changes

• Updated@helios-lang/codec-utils@^0.1.35