@tokey/core - npm Package Compare versions

Comparing version 1.2.1 to 1.3.0

dist/core.d.ts

@@ -1,4 +0,4 @@
-import type { Token } from "./types";
+import type { Token } from './types';
 export interface TokyOptions<T extends Token<unknown>> {
-    shouldAddToken(type: T["type"], value: string): boolean;
+    shouldAddToken(type: T['type'], value: string): boolean;
     isStringDelimiter(char: string, previousChar: string): boolean;
@@ -10,3 +10,3 @@ isDelimiter(char: string, previousChar: string): boolean;
     getUnclosedComment(inComment: string): string;
-    createToken(value: string, type: T["type"], start: number, end: number): T;
+    createToken(value: string, type: T['type'], start: number, end: number): T;
     offset?: number;
@@ -13,0 +13,0 @@ }

dist/core.js

@@ -6,6 +6,6 @@ "use strict";
     const tokens = [];
-    let previousChar = "";
-    let buffer = "";
-    let inComment = "";
-    let inString = "";
+    let previousChar = '';
+    let buffer = '';
+    let inComment = '';
+    let inString = '';
     let start = offset;
@@ -17,5 +17,5 @@ let nextCharIndex = 0;
             buffer += ch;
-            if (ch === inString && previousChar !== "\\") {
-                pushBuffer("string");
-                inString = "";
+            if (ch === inString && previousChar !== '\\') {
+                pushBuffer('string');
+                inString = '';
             }
@@ -27,3 +27,3 @@ }
                 pushBuffer(inComment);
-                inComment = "";
+                inComment = '';
             }
@@ -63,3 +63,3 @@ }
         else if (inString) {
-            pushBuffer("unclosed-string");
+            pushBuffer('unclosed-string');
         }
@@ -75,3 +75,3 @@ else {
         const end = start + buffer.length;
-        type = type !== null && type !== void 0 ? type : (buffer.trim().length ? "text" : "space");
+        type = type !== null && type !== void 0 ? type : (buffer.trim().length ? 'text' : 'space');
         if (shouldAddToken(type, buffer)) {
@@ -81,3 +81,3 @@ tokens[tokens.length] = createToken(buffer, type, start, end);
         start = end;
-        buffer = "";
+        buffer = '';
     }
@@ -84,0 +84,0 @@ return tokens;

dist/helpers.d.ts

@@ -1,2 +0,2 @@
-import type { Token } from "./types";
+import type { Token } from './types';
 /**
@@ -30,7 +30,7 @@ * Checks if a token type is comment
  */
-export declare function getJSCommentStartType(ch: string, source: string, nextCharIndex: number): "" | "line-comment" | "multi-comment";
+export declare function getJSCommentStartType(ch: string, source: string, nextCharIndex: number): '' | 'line-comment' | 'multi-comment';
 /**
  * Get CSS type of comments for a specific set of start chars when no comment is detected empty string is used
  */
-export declare function getMultilineCommentStartType(ch: string, source: string, nextCharIndex: number): "" | "multi-comment";
+export declare function getMultilineCommentStartType(ch: string, source: string, nextCharIndex: number): '' | 'multi-comment';
 /**
@@ -43,3 +43,3 @@ * Given a JS comment type determine if this is the end of the comment
  */
-export declare function getUnclosedComment(commentType: string): "line-comment" | "unclosed-comment";
+export declare function getUnclosedComment(commentType: string): 'line-comment' | 'unclosed-comment';
 /**
@@ -56,3 +56,3 @@ * Get the text between two token indexes
  */
-export declare function groupTokens<Tokens extends Token<any>[], T extends string = "tokens">(tokens: Tokens, type?: T, source?: string): {
+export declare function groupTokens<Tokens extends Token<any>[], T extends string = 'tokens'>(tokens: Tokens, type?: T, source?: string): {
     type: T;
@@ -71,3 +71,3 @@ start: number;
  */
-export declare function last<T>(arr: T[]): T;
+export declare function last<T>(arr: T[]): T | undefined;
 //# sourceMappingURL=helpers.d.ts.map

dist/helpers.js

@@ -8,5 +8,3 @@ "use strict";
 function isComment(type) {
-    return (type === "line-comment" ||
-        type === "multi-comment" ||
-        type === "unclosed-comment");
+    return type === 'line-comment' || type === 'multi-comment' || type === 'unclosed-comment';
 }
@@ -18,3 +16,3 @@ exports.isComment = isComment;
 function isString(type) {
-    return type === "string" || type === "unclosed-string";
+    return type === 'string' || type === 'unclosed-string';
 }
@@ -25,3 +23,3 @@ exports.isString = isString;
  */
-const isStringDelimiter = (char) => char === `'` || char === `"` || char === "`";
+const isStringDelimiter = (char) => char === `'` || char === `"` || char === '`';
 exports.isStringDelimiter = isStringDelimiter;
@@ -31,3 +29,3 @@ /**
  */
-const isWhitespace = (char) => char === " " || char === `\t` || char === `\r` || char === "\n";
+const isWhitespace = (char) => char === ' ' || char === `\t` || char === `\r` || char === '\n';
 exports.isWhitespace = isWhitespace;
@@ -50,7 +48,7 @@ /**
 function getJSCommentStartType(ch, source, nextCharIndex) {
-    if (ch === "/" && source[nextCharIndex] === "/") {
-        return "line-comment";
+    if (ch === '/' && source[nextCharIndex] === '/') {
+        return 'line-comment';
     }
     else {
-        return ch === "/" && source[nextCharIndex] === "*" ? "multi-comment" : "";
+        return ch === '/' && source[nextCharIndex] === '*' ? 'multi-comment' : '';
     }
@@ -63,3 +61,3 @@ }
 function getMultilineCommentStartType(ch, source, nextCharIndex) {
-    return ch === "/" && source[nextCharIndex] === "*" ? "multi-comment" : "";
+    return ch === '/' && source[nextCharIndex] === '*' ? 'multi-comment' : '';
 }
@@ -71,8 +69,6 @@ exports.getMultilineCommentStartType = getMultilineCommentStartType;
 function isCommentEnd(commentType, ch, _source, _nextCharIndex, previousChar) {
-    if (commentType === "line-comment" && ch === "\n") {
+    if (commentType === 'line-comment' && ch === '\n') {
         return true;
     }
-    else if (commentType === "multi-comment" &&
-        ch === "/" &&
-        previousChar === "*") {
+    else if (commentType === 'multi-comment' && ch === '/' && previousChar === '*') {
         return true;
@@ -87,7 +83,7 @@ }
 function getUnclosedComment(commentType) {
-    if (commentType === "line-comment") {
+    if (commentType === 'line-comment') {
         return commentType;
     }
     else {
-        return "unclosed-comment";
+        return 'unclosed-comment';
     }
@@ -103,3 +99,3 @@ }
     if (tokens.length === 0) {
-        return "";
+        return '';
     }
@@ -113,3 +109,3 @@ if (upToIndex === -1) {
     else {
-        let res = "";
+        let res = '';
         for (let i = startIndex; i < upToIndex; i++) {
@@ -127,3 +123,3 @@ res += tokens[i].value;
  */
-function groupTokens(tokens, type = "tokens", source) {
+function groupTokens(tokens, type = 'tokens', source) {
     return {
@@ -130,0 +126,0 @@ type,

dist/index.d.ts

@@ -1,5 +0,5 @@
-export * from "./types";
-export * from "./core";
-export * from "./helpers";
-export * from "./seeker";
+export * from './types';
+export * from './core';
+export * from './helpers';
+export * from './seeker';
 //# sourceMappingURL=index.d.ts.map

dist/index.js

 "use strict";
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
-    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
 }) : (function(o, m, k, k2) {
@@ -6,0 +10,0 @@ if (k2 === undefined) k2 = k;

dist/seeker.d.ts

@@ -1,2 +0,2 @@
-import type { Token } from "./types";
+import type { Token } from './types';
 /**
@@ -13,4 +13,5 @@ * Minimal token traverse helper used to create structure from tokens
     peek(num?: number): T;
-    take(type: T["type"]): T | undefined;
-    takeMany(type: T["type"]): T[];
+    take(type: T['type']): T | undefined;
+    eat(type: T['type']): this;
+    takeMany(type: T['type']): T[];
     flatBlock(start: string, end: string, isEndError?: (token: Token<any>) => boolean): T[] | undefined;
@@ -17,0 +18,0 @@ done(): boolean;

dist/seeker.js

@@ -14,3 +14,3 @@ "use strict";
         this.index++;
-        return this.tokens[this.index] || { type: "" };
+        return this.tokens[this.index] || { type: '' };
     }
@@ -21,6 +21,6 @@ back() {
     peekBack() {
-        return this.tokens[this.index - 1] || { type: "" };
+        return this.tokens[this.index - 1] || { type: '' };
     }
     peek(num = 1) {
-        return this.tokens[this.index + num] || { type: "" };
+        return this.tokens[this.index + num] || { type: '' };
     }
@@ -33,2 +33,8 @@ take(type) {
     }
+    eat(type) {
+        while (this.peek().type === type) {
+            this.index++;
+        }
+        return this;
+    }
     takeMany(type) {
@@ -35,0 +41,0 @@ const tokens = [];

dist/types.d.ts

@@ -1,2 +0,2 @@
-export declare type Descriptors = "string" | "text" | "line-comment" | "multi-comment" | "unclosed-string" | "unclosed-comment" | "space";
+export declare type Descriptors = 'string' | 'text' | 'line-comment' | 'multi-comment' | 'unclosed-string' | 'unclosed-comment' | 'space';
 export interface Token<Type = Descriptors> {
@@ -3,0 +3,0 @@ type: Type;

package.json

 {
   "name": "@tokey/core",
   "description": "simple code like tokenizer",
-  "version": "1.2.1",
+  "version": "1.3.0",
   "main": "dist/index.js",
@@ -29,4 +29,3 @@ "types": "dist/index.d.ts",
   "repository": "https://github.com/wixplosives/tokey/packages/core",
-  "prettier": {},
   "sideEffects": false
 }

src/core.ts

@@ -1,114 +0,100 @@
-import type { Token } from "./types";
+import type { Token } from './types';
 
 export interface TokyOptions<T extends Token<unknown>> {
-  shouldAddToken(type: T["type"], value: string): boolean;
-  isStringDelimiter(char: string, previousChar: string): boolean;
-  isDelimiter(char: string, previousChar: string): boolean;
-  isWhitespace(char: string): boolean;
-  getCommentStartType(
-    ch: string,
-    source: string,
-    nextCharIndex: number
-  ): string;
-  isCommentEnd(
-    inComment: string,
-    ch: string,
-    source: string,
-    nextCharIndex: number,
-    previousChar: string
-  ): boolean;
-  getUnclosedComment(inComment: string): string;
-  createToken(value: string, type: T["type"], start: number, end: number): T;
-  offset?: number;
+    shouldAddToken(type: T['type'], value: string): boolean;
+    isStringDelimiter(char: string, previousChar: string): boolean;
+    isDelimiter(char: string, previousChar: string): boolean;
+    isWhitespace(char: string): boolean;
+    getCommentStartType(ch: string, source: string, nextCharIndex: number): string;
+    isCommentEnd(
+        inComment: string,
+        ch: string,
+        source: string,
+        nextCharIndex: number,
+        previousChar: string
+    ): boolean;
+    getUnclosedComment(inComment: string): string;
+    createToken(value: string, type: T['type'], start: number, end: number): T;
+    offset?: number;
 }
 
 export function tokenize<T extends Token<unknown>>(
-  source: string,
-  {
-    isDelimiter,
-    isStringDelimiter,
-    isWhitespace,
-    shouldAddToken,
-    createToken,
-    getCommentStartType,
-    isCommentEnd,
-    getUnclosedComment,
-    offset = 0,
-  }: TokyOptions<T>
+    source: string,
+    {
+        isDelimiter,
+        isStringDelimiter,
+        isWhitespace,
+        shouldAddToken,
+        createToken,
+        getCommentStartType,
+        isCommentEnd,
+        getUnclosedComment,
+        offset = 0,
+    }: TokyOptions<T>
 ): T[] {
-  const tokens: T[] = [];
-  let previousChar = "";
-  let buffer = "";
-  let inComment = "";
-  let inString = "";
-  let start = offset;
-  let nextCharIndex = 0;
-  for (const ch of source) {
-    nextCharIndex += ch.length;
-    if (inString) {
-      buffer += ch;
-      if (ch === inString && previousChar !== "\\") {
-        pushBuffer("string");
-        inString = "";
-      }
-    } else if (inComment) {
-      buffer += ch;
-      if (
-        isCommentEnd(
-          inComment,
-          ch,
-          source,
-          nextCharIndex,
-          previousChar
-        )
-      ) {
-        pushBuffer(inComment);
-        inComment = "";
-      }
-    } else if (
-      (inComment = getCommentStartType(ch, source, nextCharIndex))
-    ) {
-      pushBuffer();
-      buffer += ch;
-    } else if (isStringDelimiter(ch, previousChar)) {
-      pushBuffer();
-      buffer += ch;
-      inString = ch;
-    } else if (isDelimiter(ch, previousChar)) {
-      pushBuffer();
-      buffer += ch;
-      pushBuffer(ch);
-    } else if (isWhitespace(ch) && !isWhitespace(previousChar)) {
-      pushBuffer();
-      buffer += ch;
-    } else if (!isWhitespace(ch) && isWhitespace(previousChar)) {
-      pushBuffer();
-      buffer += ch;
-    } else {
-      buffer += ch;
+    const tokens: T[] = [];
+    let previousChar = '';
+    let buffer = '';
+    let inComment = '';
+    let inString = '';
+    let start = offset;
+    let nextCharIndex = 0;
+    for (const ch of source) {
+        nextCharIndex += ch.length;
+        if (inString) {
+            buffer += ch;
+            if (ch === inString && previousChar !== '\\') {
+                pushBuffer('string');
+                inString = '';
+            }
+        } else if (inComment) {
+            buffer += ch;
+            if (isCommentEnd(inComment, ch, source, nextCharIndex, previousChar)) {
+                pushBuffer(inComment);
+                inComment = '';
+            }
+        } else if ((inComment = getCommentStartType(ch, source, nextCharIndex))) {
+            pushBuffer();
+            buffer += ch;
+        } else if (isStringDelimiter(ch, previousChar)) {
+            pushBuffer();
+            buffer += ch;
+            inString = ch;
+        } else if (isDelimiter(ch, previousChar)) {
+            pushBuffer();
+            buffer += ch;
+            pushBuffer(ch);
+        } else if (isWhitespace(ch) && !isWhitespace(previousChar)) {
+            pushBuffer();
+            buffer += ch;
+        } else if (!isWhitespace(ch) && isWhitespace(previousChar)) {
+            pushBuffer();
+            buffer += ch;
+        } else {
+            buffer += ch;
+        }
+        previousChar = ch;
     }
-    previousChar = ch;
-  }
-  if (buffer.length) {
-    if (inComment) {
-      pushBuffer(getUnclosedComment(inComment));
-    } else if (inString) {
-      pushBuffer("unclosed-string");
-    } else {
-      pushBuffer();
+    if (buffer.length) {
+        if (inComment) {
+            pushBuffer(getUnclosedComment(inComment));
+        } else if (inString) {
+            pushBuffer('unclosed-string');
+        } else {
+            pushBuffer();
+        }
     }
-  }
-  function pushBuffer(type?: T["type"]) {
-    if (buffer.length === 0) {
-      return;
+    function pushBuffer(type?: T['type']) {
+        if (buffer.length === 0) {
+            return;
+        }
+        const end = start + buffer.length;
+        type = type ?? (buffer.trim().length ? 'text' : 'space');
+        if (shouldAddToken(type, buffer)) {
+            tokens[tokens.length] = createToken(buffer, type, start, end);
+        }
+        start = end;
+        buffer = '';
     }
-    const end = start + buffer.length;
-    type = type ?? (buffer.trim().length ? "text" : "space");
-    if (shouldAddToken(type, buffer)) {
-      tokens[tokens.length] = createToken(buffer, type, start, end);
-    }
-    start = end;
-    buffer = "";
-  }
-  return tokens;
+    return tokens;
 }

src/helpers.ts

@@ -1,2 +0,2 @@
-import type { Token } from "./types";
+import type { Token } from './types';
 
@@ -7,7 +7,3 @@ /**
 export function isComment(type: string) {
-  return (
-    type === "line-comment" ||
-    type === "multi-comment" ||
-    type === "unclosed-comment"
-  );
+    return type === 'line-comment' || type === 'multi-comment' || type === 'unclosed-comment';
 }
@@ -19,3 +15,3 @@
 export function isString(type: string) {
-  return type === "string" || type === "unclosed-string";
+    return type === 'string' || type === 'unclosed-string';
 }
@@ -26,4 +22,3 @@
  */
-export const isStringDelimiter = (char: string) =>
-  char === `'` || char === `"` || char === "`";
+export const isStringDelimiter = (char: string) => char === `'` || char === `"` || char === '`';
 
@@ -34,3 +29,3 @@ /**
 export const isWhitespace = (char: string) =>
-  char === " " || char === `\t` || char === `\r` || char === "\n";
+    char === ' ' || char === `\t` || char === `\r` || char === '\n';
 
@@ -41,13 +36,13 @@ /**
 export const createToken = <Type extends string>(
-  value: string,
-  type: Type,
-  start: number,
-  end: number
+    value: string,
+    type: Type,
+    start: number,
+    end: number
 ) => {
-  return {
-    value,
-    type,
-    start,
-    end,
-  };
+    return {
+        value,
+        type,
+        start,
+        end,
+    };
 };
@@ -59,11 +54,11 @@
 export function getJSCommentStartType(
-  ch: string,
-  source: string,
-  nextCharIndex: number
-): "" | "line-comment" | "multi-comment" {
-  if (ch === "/" && source[nextCharIndex] === "/") {
-    return "line-comment";
-  } else {
-    return ch === "/" && source[nextCharIndex] === "*" ? "multi-comment" : "";
-  }
+    ch: string,
+    source: string,
+    nextCharIndex: number
+): '' | 'line-comment' | 'multi-comment' {
+    if (ch === '/' && source[nextCharIndex] === '/') {
+        return 'line-comment';
+    } else {
+        return ch === '/' && source[nextCharIndex] === '*' ? 'multi-comment' : '';
+    }
 }
@@ -75,7 +70,7 @@
 export function getMultilineCommentStartType(
-  ch: string,
-  source: string,
-  nextCharIndex: number
-): "" | "multi-comment" {
-  return ch === "/" && source[nextCharIndex] === "*" ? "multi-comment" : "";
+    ch: string,
+    source: string,
+    nextCharIndex: number
+): '' | 'multi-comment' {
+    return ch === '/' && source[nextCharIndex] === '*' ? 'multi-comment' : '';
 }
@@ -87,18 +82,14 @@
 export function isCommentEnd(
-  commentType: string,
-  ch: string,
-  _source: string,
-  _nextCharIndex: number,
-  previousChar: string
+    commentType: string,
+    ch: string,
+    _source: string,
+    _nextCharIndex: number,
+    previousChar: string
 ): boolean {
-  if (commentType === "line-comment" && ch === "\n") {
-    return true;
-  } else if (
-    commentType === "multi-comment" &&
-    ch === "/" &&
-    previousChar === "*"
-  ) {
-    return true;
-  }
-  return false;
+    if (commentType === 'line-comment' && ch === '\n') {
+        return true;
+    } else if (commentType === 'multi-comment' && ch === '/' && previousChar === '*') {
+        return true;
+    }
+    return false;
 }
@@ -109,10 +100,8 @@
  */
-export function getUnclosedComment(
-  commentType: string
-): "line-comment" | "unclosed-comment" {
-  if (commentType === "line-comment") {
-    return commentType;
-  } else {
-    return "unclosed-comment";
-  }
+export function getUnclosedComment(commentType: string): 'line-comment' | 'unclosed-comment' {
+    if (commentType === 'line-comment') {
+        return commentType;
+    } else {
+        return 'unclosed-comment';
+    }
 }
@@ -124,24 +113,19 @@ /**
  */
-export function getText(
-  tokens: Token<any>[],
-  startIndex = 0,
-  upToIndex = -1,
-  source?: string
-) {
-  if (tokens.length === 0) {
-    return "";
-  }
-  if (upToIndex === -1) {
-    upToIndex = tokens.length;
-  }
+export function getText(tokens: Token<any>[], startIndex = 0, upToIndex = -1, source?: string) {
+    if (tokens.length === 0) {
+        return '';
+    }
+    if (upToIndex === -1) {
+        upToIndex = tokens.length;
+    }
 
-  if (source) {
-    return source.slice(tokens[startIndex].start, tokens[upToIndex - 1].end);
-  } else {
-    let res = "";
-    for (let i = startIndex; i < upToIndex; i++) {
-      res += tokens[i].value;
+    if (source) {
+        return source.slice(tokens[startIndex].start, tokens[upToIndex - 1].end);
+    } else {
+        let res = '';
+        for (let i = startIndex; i < upToIndex; i++) {
+            res += tokens[i].value;
+        }
+        return res;
     }
-    return res;
-  }
 }
@@ -154,13 +138,14 @@
  */
-export function groupTokens<
-  Tokens extends Token<any>[],
-  T extends string = "tokens"
->(tokens: Tokens, type: T = "tokens" as T, source?: string) {
-  return {
-    type,
-    start: tokens[0].start,
-    end: tokens[tokens.length - 1].end,
-    value: getText(tokens, undefined, undefined, source),
-    tokens,
-  };
+export function groupTokens<Tokens extends Token<any>[], T extends string = 'tokens'>(
+    tokens: Tokens,
+    type: T = 'tokens' as T,
+    source?: string
+) {
+    return {
+        type,
+        start: tokens[0].start,
+        end: tokens[tokens.length - 1].end,
+        value: getText(tokens, undefined, undefined, source),
+        tokens,
+    };
 }
@@ -172,22 +157,22 @@
 export function trimTokens<Tokens extends Token<any>[]>(
-  tokens: Tokens,
-  shouldTrimToken: (token: Token<any>) => boolean
+    tokens: Tokens,
+    shouldTrimToken: (token: Token<any>) => boolean
 ) {
-  let start = 0;
-  let end = tokens.length;
-  for (let i = 0; i < tokens.length; i++) {
-    if (shouldTrimToken(tokens[i])) {
-      start = i + 1;
-    } else {
-      break;
+    let start = 0;
+    let end = tokens.length;
+    for (let i = 0; i < tokens.length; i++) {
+        if (shouldTrimToken(tokens[i])) {
+            start = i + 1;
+        } else {
+            break;
+        }
     }
-  }
-  for (let i = tokens.length - 1; i > start; i--) {
-    if (shouldTrimToken(tokens[i])) {
-      end = i;
-    } else {
-      break;
+    for (let i = tokens.length - 1; i > start; i--) {
+        if (shouldTrimToken(tokens[i])) {
+            end = i;
+        } else {
+            break;
+        }
     }
-  }
-  return tokens.slice(start, end);
+    return tokens.slice(start, end);
 }
@@ -198,4 +183,4 @@
  */
-export function last<T>(arr: T[]): T {
-  return arr[arr.length - 1];
+export function last<T>(arr: T[]): T | undefined {
+    return arr[arr.length - 1];
 }

src/index.ts

@@ -1,4 +0,4 @@
-export * from "./types";
-export * from "./core";
-export * from "./helpers";
-export * from "./seeker";
+export * from './types';
+export * from './core';
+export * from './helpers';
+export * from './seeker';

src/seeker.ts

@@ -1,2 +0,2 @@
-import type { Token } from "./types";
+import type { Token } from './types';
 
@@ -7,80 +7,77 @@ /**
 export class Seeker<T extends Token<unknown>> {
-  index = -1;
-  constructor(public tokens: T[]) {}
-  next() {
-    this.index++;
-    return this.tokens[this.index] || { type: "" };
-  }
-  back() {
-    this.index--;
-  }
-  peekBack() {
-    return this.tokens[this.index - 1] || { type: "" };
-  }
-  peek(num = 1) {
-    return this.tokens[this.index + num] || { type: "" };
-  }
-  take(type: T["type"]) {
-    if (this.peek().type === type) {
-      return this.next();
+    index = -1;
+    constructor(public tokens: T[]) {}
+    next() {
+        this.index++;
+        return this.tokens[this.index] || { type: '' };
     }
-    return undefined;
-  }
-  takeMany(type: T["type"]) {
-    const tokens = [];
-    while (this.peek().type === type) {
-      tokens.push(this.next());
+    back() {
+        this.index--;
     }
-    return tokens;
-  }
-  flatBlock(
-    start: string,
-    end: string,
-    isEndError?: (token: Token<any>) => boolean
-  ) {
-    let token = this.next();
-    if (token.type !== start) {
-      return [];
+    peekBack() {
+        return this.tokens[this.index - 1] || { type: '' };
     }
-    const block = [];
-    let endIndex;
-    while ((token = this.next())) {
-      if (!token.type) {
-        if (endIndex !== undefined) {
-          this.index = endIndex - 1;
+    peek(num = 1) {
+        return this.tokens[this.index + num] || { type: '' };
+    }
+    take(type: T['type']) {
+        if (this.peek().type === type) {
+            return this.next();
         }
-        return;
-      }
-      if (isEndError && isEndError(token)) {
-        endIndex = this.index;
-      }
-      if (token.type === end) {
-        return block;
-      } else {
-        block.push(token);
-      }
+        return undefined;
     }
-    return [];
-  }
-  done() {
-    return this.index >= this.tokens.length - 1;
-  }
-  run<A>(
-    handleToken: (
-      token: T,
-      ast: A,
-      source: string,
-      seeker: this
-    ) => void | boolean,
-    ast: A,
-    source: string
-  ) {
-    let token;
-    while ((token = this.next()) && token.type) {
-      if (handleToken(token, ast, source, this) === false) {
-        break;
-      }
+    eat(type: T['type']) {
+        while (this.peek().type === type) {
+            this.index++;
+        }
+        return this;
     }
-    return ast;
-  }
+    takeMany(type: T['type']) {
+        const tokens = [];
+        while (this.peek().type === type) {
+            tokens.push(this.next());
+        }
+        return tokens;
+    }
+    flatBlock(start: string, end: string, isEndError?: (token: Token<any>) => boolean) {
+        let token = this.next();
+        if (token.type !== start) {
+            return [];
+        }
+        const block = [];
+        let endIndex;
+        while ((token = this.next())) {
+            if (!token.type) {
+                if (endIndex !== undefined) {
+                    this.index = endIndex - 1;
+                }
+                return;
+            }
+            if (isEndError && isEndError(token)) {
+                endIndex = this.index;
+            }
+            if (token.type === end) {
+                return block;
+            } else {
+                block.push(token);
+            }
+        }
+        return [];
+    }
+    done() {
+        return this.index >= this.tokens.length - 1;
+    }
+    run<A>(
+        handleToken: (token: T, ast: A, source: string, seeker: this) => void | boolean,
+        ast: A,
+        source: string
+    ) {
+        let token;
+        while ((token = this.next()) && token.type) {
+            if (handleToken(token, ast, source, this) === false) {
+                break;
+            }
+        }
+        return ast;
+    }
 }

src/types.ts

 export type Descriptors =
-  | "string"
-  | "text"
-  | "line-comment"
-  | "multi-comment"
-  | "unclosed-string"
-  | "unclosed-comment"
-  | "space";
+    | 'string'
+    | 'text'
+    | 'line-comment'
+    | 'multi-comment'
+    | 'unclosed-string'
+    | 'unclosed-comment'
+    | 'space';
 
 export interface Token<Type = Descriptors> {
-  type: Type;
-  start: number;
-  end: number;
-  value: string;
+    type: Type;
+    start: number;
+    end: number;
+    value: string;
 }
 
 export interface TokenGroup<GroupType, Type = Descriptors> {
-  type: GroupType;
-  start: number;
-  end: number;
-  value: string;
-  tokens: Token<Type>[];
+    type: GroupType;
+    start: number;
+    end: number;
+    value: string;
+    tokens: Token<Type>[];
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

39579

increased by3.59%

Worsened metrics

Lines of code

822

decreased by-2.95%

No dependency changes