@helios-lang/compiler-utils - npm Package Compare versions

Comparing version 0.5.4 to 0.5.5

package.json

 {
-    "name": "@helios-lang/compiler-utils",
-    "version": "0.5.4",
-    "description": "Helios language compiler library",
-    "main": "src/index.js",
-    "types": "types/index.d.ts",
-    "type": "module",
-    "author": "Christian Schmitz",
-    "license": "BSD-3-Clause",
-    "scripts": {
-        "build": "pnpm run prettify && pnpm run lint && pnpm run build:types && pnpm run test:suite && pnpm run test:integ",
-        "build:docs": "typedoc --tsconfig ./jsconfig.json",
-        "build:types": "tsc -p jsconfig.json --noEmit false --emitDeclarationOnly",
-        "lint": "eslint .",
-        "prettify": "prettier . --write",
-        "test": "pnpm run test:pretty && pnpm run lint && pnpm run test:types && pnpm run test:suite && pnpm run test:integ",
-        "test:integ": "pnpm --prefix ./test/all-pure run test",
-        "test:pretty": "prettier . --check",
-        "test:suite": "node --test  --experimental-test-coverage",
-        "test:types": "tsc -p jsconfig.json --noEmit"
-    },
-    "devDependencies": {
-        "@types/node": "^20.11.24",
-        "@types/punycode": "^2.1.4",
-        "@typescript-eslint/eslint-plugin": "^8.8.1",
-        "eslint": "^9.12.0",
-        "eslint-plugin-jsdoc": "^50.3.1",
-        "prettier": "^3.3.3",
-        "typedoc": "^0.26.9",
-        "typedoc-plugin-mdn-links": "^3.3.3",
-        "typescript": "^5.6.2"
-    },
-    "dependencies": {
-        "@helios-lang/codec-utils": "^0.3.4",
-        "@helios-lang/type-utils": "^0.2.9"
-    },
-    "prettier": {
-        "trailingComma": "none",
-        "tabWidth": 4,
-        "semi": false,
-        "singleQuote": false
-    }
-}
+  "name": "@helios-lang/compiler-utils",
+  "version": "0.5.5",
+  "description": "Helios language compiler library",
+  "main": "src/index.js",
+  "types": "types/index.d.ts",
+  "type": "module",
+  "author": "Christian Schmitz",
+  "license": "BSD-3-Clause",
+  "devDependencies": {
+    "@types/node": "^20.11.24",
+    "@types/punycode": "^2.1.4",
+    "@typescript-eslint/eslint-plugin": "^8.8.1",
+    "eslint": "^9.12.0",
+    "eslint-plugin-jsdoc": "^50.3.1",
+    "prettier": "^3.3.3",
+    "typedoc": "^0.26.9",
+    "typedoc-plugin-mdn-links": "^3.3.3",
+    "typescript": "^5.6.2"
+  },
+  "dependencies": {
+    "@helios-lang/codec-utils": "^0.3.4",
+    "@helios-lang/type-utils": "^0.2.9"
+  },
+  "prettier": {
+    "trailingComma": "none",
+    "tabWidth": 4,
+    "semi": false,
+    "singleQuote": false
+  },
+  "scripts": {
+    "build": "pnpm run prettify && pnpm run lint && pnpm run build:types && pnpm run test:suite && pnpm run test:integ",
+    "build:docs": "typedoc --tsconfig ./jsconfig.json",
+    "build:types": "tsc -p jsconfig.json --noEmit false --emitDeclarationOnly",
+    "lint": "eslint .",
+    "prettify": "prettier . --write",
+    "test": "pnpm run test:pretty && pnpm run lint && pnpm run test:types && pnpm run test:suite && pnpm run test:integ",
+    "test:integ": "pnpm --prefix ./test/all-pure run test",
+    "test:pretty": "prettier . --check",
+    "test:suite": "node --test  --experimental-test-coverage",
+    "test:types": "tsc -p jsconfig.json --noEmit"
+  }
+}

src/index.js

@@ -24,2 +24,3 @@ export {
     makeIntLiteral,
+    makeNL,
     makeRealLiteral,
@@ -213,2 +214,10 @@ makeSource,
 /**
+ * @typedef {object} NL
+ * @prop {Site} site
+ * @prop {"newline"} kind
+ * @prop {(other: Token) => boolean} isEqual
+ * @prop {(preserveWhitespace?: boolean) => string} toString
+ */
+
+/**
  * @typedef {object} RealLiteral
@@ -260,2 +269,3 @@ * @prop {Site} site
  *   | IntLiteral
+ *   | NL
  *   | RealLiteral
@@ -262,0 +272,0 @@ *   | StringLiteral

src/tokens/index.js

@@ -10,2 +10,3 @@ export { makeBoolLiteral } from "./BoolLiteral.js"
 export { makeIntLiteral } from "./IntLiteral.js"
+export { makeNL } from "./NL.js"
 export { REAL_PRECISION, makeRealLiteral } from "./RealLiteral.js"
@@ -12,0 +13,0 @@ export { makeSource } from "./Source.js"

src/tokens/Tokenizer.js

@@ -14,2 +14,3 @@ import { encodeUtf8, hexToBytes } from "@helios-lang/codec-utils"
 import { makeIntLiteral } from "./IntLiteral.js"
+import { makeNL } from "./NL.js"
 import { REAL_PRECISION, makeRealLiteral } from "./RealLiteral.js"
@@ -35,2 +36,3 @@ import { makeSourceIndex } from "./SourceIndex.js"
  *   preserveComments?: boolean
+ *   preserveNewlines?: boolean
  *   allowLeadingZeroes?: boolean
@@ -53,2 +55,3 @@ *   errorCollector?: ErrorCollector
  * @param {boolean} [options.preserveComments]
+ * @param {boolean} [options.preserveNewlines]
  * @param {boolean} [options.allowLeadingZeroes]
@@ -92,2 +95,9 @@ * @param {ErrorCollector} [options.errorCollector]
      */
+    _preserveNewlines
+
+    /**
+     * @private
+     * @readonly
+     * @type {boolean}
+     */
     _allowLeadingZeroes
@@ -131,2 +141,3 @@
         this._preserveComments = options?.preserveComments ?? false
+        this._preserveNewlines = options?.preserveNewlines ?? false
         this._allowLeadingZeroes = options?.allowLeadingZeroes ?? false
@@ -309,2 +320,4 @@ this.errors = options.errorCollector ?? makeErrorCollector()
             this.readSymbol(site, c)
+        } else if (this._preserveNewlines && c == "\n") {
+            this.pushToken(makeNL(site))
         } else if (!(c == " " || c == "\n" || c == "\t" || c == "\r")) {
@@ -311,0 +324,0 @@ this.addSyntaxError(

src/tokens/TokenReader.js

@@ -15,2 +15,3 @@ import { isUndefined } from "@helios-lang/type-utils"
  *   errors?: ErrorCollector
+ *   ignoreNewlines?: boolean
  * }} args
@@ -20,3 +21,7 @@ * @returns {TokenReader}
 export function makeTokenReader(args) {
-    return new TokenReaderImpl(args.tokens, args.errors ?? makeErrorCollector())
+    return new TokenReaderImpl(
+        args.tokens,
+        args.errors ?? makeErrorCollector(),
+        args.ignoreNewlines ?? false
+    )
 }
@@ -29,5 +34,15 @@
     /**
+     * Tokens including newlines
+     * Can be used for semicolon injection
      * @readonly
      * @type {Token[]}
      */
+    originalTokens
+
+    /**
+     * Tokens excluding newlines
+     * (Newlines are ignored by the matchers)
+     * @readonly
+     * @type {Token[]}
+     */
     tokens
@@ -43,2 +58,9 @@
      * @private
+     * @readonly
+     * @type {boolean}
+     */
+    _ignoreNewlines
+
+    /**
+     * @private
      * @type {number}
@@ -57,6 +79,11 @@ */
      * @param {ErrorCollector} errors
+     * @param {boolean} ignoreNewlines
      */
-    constructor(tokens, errors) {
-        this.tokens = tokens
+    constructor(tokens, errors, ignoreNewlines) {
+        this.originalTokens = tokens
+        this.tokens = ignoreNewlines
+            ? tokens.filter((t) => t.kind != "newline")
+            : tokens
         this.errors = errors
+        this.originalTokens
         this._i = 0
@@ -67,2 +94,3 @@ this._failedMatches = []
     /**
+     * Excludes newlines
      * @type {Token[]}
@@ -178,5 +206,3 @@ */
                             .slice(i, i + n)
-                            .map((t) =>
-                                isGroup(t) ? augmentGroup(this, t) : t
-                            )
+                            .map((t) => (isGroup(t) ? this.augmentGroup(t) : t))
                     )
@@ -190,3 +216,4 @@
                             this.tokens.slice(i0, i),
-                            this.errors
+                            this.errors,
+                            this._ignoreNewlines
                         ),
@@ -261,5 +288,3 @@ ...matched
                             .slice(i, i + n)
-                            .map((t) =>
-                                isGroup(t) ? augmentGroup(this, t) : t
-                            )
+                            .map((t) => (isGroup(t) ? this.augmentGroup(t) : t))
                     )
@@ -273,3 +298,4 @@
                             this.tokens.slice(i0, i),
-                            this.errors
+                            this.errors,
+                            this._ignoreNewlines
                         ),
@@ -305,3 +331,7 @@ ...matched
         } else {
-            let reader = new TokenReaderImpl(this.tokens, this.errors)
+            let reader = new TokenReaderImpl(
+                this.tokens,
+                this.errors,
+                this._ignoreNewlines
+            )
 
@@ -339,3 +369,3 @@ reader._i = this._i
                         .slice(this._i, this._i + n)
-                        .map((t) => (isGroup(t) ? augmentGroup(this, t) : t))
+                        .map((t) => (isGroup(t) ? this.augmentGroup(t) : t))
                 )
@@ -394,16 +424,18 @@ this._failedMatches = []
     }
-}
 
-/**
- * @param {TokenReader} r
- * @param {GenericGroup<Token[]>} t
- * @returns {GenericGroup<TokenReader>}
- */
-function augmentGroup(r, t) {
-    return makeGroup({
-        kind: t.kind,
-        fields: t.fields.map((f) => new TokenReaderImpl(f, r.errors)),
-        separators: t.separators,
-        site: t.site
-    })
+    /**
+     * @private
+     * @param {GenericGroup<Token[]>} t
+     * @returns {GenericGroup<TokenReader>}
+     */
+    augmentGroup(t) {
+        return makeGroup({
+            kind: t.kind,
+            fields: t.fields.map(
+                (f) => new TokenReaderImpl(f, this.errors, this._ignoreNewlines)
+            ),
+            separators: t.separators,
+            site: t.site
+        })
+    }
 }

types/index.d.ts

@@ -172,2 +172,8 @@ export { translateImportPaths } from "./formatters/index.js";
 };
+export type NL = {
+    site: Site;
+    kind: "newline";
+    isEqual: (other: Token) => boolean;
+    toString: (preserveWhitespace?: boolean) => string;
+};
 export type RealLiteral = {
@@ -205,3 +211,3 @@ site: Site;
 };
-export type Token = (BoolLiteral | ByteArrayLiteral | Comment | IntLiteral | RealLiteral | StringLiteral | SymbolToken | TokenGroup | Word);
+export type Token = (BoolLiteral | ByteArrayLiteral | Comment | IntLiteral | NL | RealLiteral | StringLiteral | SymbolToken | TokenGroup | Word);
 export type Tokenizer = {
@@ -208,0 +214,0 @@ errors: ErrorCollector;

types/tokens/index.d.ts

@@ -5,2 +5,3 @@ export { makeBoolLiteral } from "./BoolLiteral.js";
 export { makeIntLiteral } from "./IntLiteral.js";
+export { makeNL } from "./NL.js";
 export { makeSource } from "./Source.js";
@@ -7,0 +8,0 @@ export { makeStringLiteral } from "./StringLiteral.js";

types/tokens/Tokenizer.d.ts

@@ -10,2 +10,3 @@ /**
  * @param {boolean} [options.preserveComments]
+ * @param {boolean} [options.preserveNewlines]
  * @param {boolean} [options.allowLeadingZeroes]
@@ -21,2 +22,3 @@ * @param {ErrorCollector} [options.errorCollector]
     preserveComments?: boolean | undefined;
+    preserveNewlines?: boolean | undefined;
     allowLeadingZeroes?: boolean | undefined;
@@ -32,2 +34,3 @@ errorCollector?: ErrorCollector | undefined;
     preserveComments?: boolean;
+    preserveNewlines?: boolean;
     allowLeadingZeroes?: boolean;
@@ -34,0 +37,0 @@ errorCollector?: ErrorCollector;

types/tokens/TokenReader.d.ts

@@ -9,2 +9,3 @@ /**
  *   errors?: ErrorCollector
+ *   ignoreNewlines?: boolean
  * }} args
@@ -16,2 +17,3 @@ * @returns {TokenReader}
     errors?: ErrorCollector;
+    ignoreNewlines?: boolean;
 }): TokenReader;
@@ -18,0 +20,0 @@ import type { Token } from "../index.js";

No alert changes

Improved metrics

Total package byte prevSize

155942

1.94%

Number of package files

98

3.16%

Lines of code

4707

2.59%

No dependency changes