strtok3 - npm Package Compare versions

Comparing version 9.0.1 to 9.1.0

lib/AbstractTokenizer.d.ts

@@ -13,5 +13,10 @@ import type { ITokenizer, IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
 export declare abstract class AbstractTokenizer implements ITokenizer {
-    fileInfo: IFileInfo;
     private onClose?;
+    private numBuffer;
+    abstract fileInfo: IFileInfo;
     /**
+     * Tokenizer-stream position
+     */
+    position: number;
+    /**
      * Constructor
@@ -22,8 +27,4 @@ * @param options Tokenizer options
     protected constructor(options?: ITokenizerOptions);
+    abstract supportsRandomAccess(): boolean;
     /**
-     * Tokenizer-stream position
-     */
-    position: number;
-    private numBuffer;
-    /**
      * Read buffer from tokenizer
@@ -30,0 +31,0 @@ * @param buffer - Target buffer to fill with data read from the tokenizer-stream

lib/AbstractTokenizer.js

@@ -12,2 +12,3 @@ import { EndOfStreamError } from 'peek-readable';
     constructor(options) {
+        this.numBuffer = new Uint8Array(8);
         /**
@@ -17,4 +18,2 @@ * Tokenizer-stream position
         this.position = 0;
-        this.numBuffer = new Uint8Array(8);
-        this.fileInfo = options?.fileInfo ?? {};
         this.onClose = options?.onClose;
@@ -21,0 +20,0 @@ if (options?.abortSignal) {

lib/BufferTokenizer.d.ts

@@ -1,5 +0,6 @@
-import type { ITokenizerOptions, IReadChunkOptions } from './types.js';
+import type { ITokenizerOptions, IReadChunkOptions, IRandomAccessFileInfo, IRandomAccessTokenizer } from './types.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
-export declare class BufferTokenizer extends AbstractTokenizer {
+export declare class BufferTokenizer extends AbstractTokenizer implements IRandomAccessTokenizer {
     private uint8Array;
+    fileInfo: IRandomAccessFileInfo;
     /**
@@ -26,2 +27,4 @@ * Construct BufferTokenizer
     close(): Promise<void>;
+    supportsRandomAccess(): boolean;
+    setPosition(position: number): void;
 }

lib/BufferTokenizer.js

@@ -12,3 +12,3 @@ import { EndOfStreamError } from 'peek-readable';
         this.uint8Array = uint8Array;
-        this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : uint8Array.length;
+        this.fileInfo = { ...options?.fileInfo ?? {}, ...{ size: uint8Array.length } };
     }
@@ -50,2 +50,8 @@ /**
     }
+    supportsRandomAccess() {
+        return true;
+    }
+    setPosition(position) {
+        this.position = position;
+    }
 }

lib/core.d.ts

@@ -7,3 +7,3 @@ import type { Readable } from 'node:stream';
 export { EndOfStreamError, type AnyWebByteStream } from 'peek-readable';
-export type { ITokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './types.js';
+export type { ITokenizer, IRandomAccessTokenizer, IFileInfo, ITokenizerOptions, IReadChunkOptions, OnClose } from './types.js';
 export type { IToken, IGetToken } from '@tokenizer/token';
@@ -10,0 +10,0 @@ export { AbstractTokenizer } from './AbstractTokenizer.js';

lib/FileTokenizer.d.ts

 import { AbstractTokenizer } from './AbstractTokenizer.js';
-import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
+import type { IRandomAccessTokenizer, IRandomAccessFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
 import { type FileHandle } from 'node:fs/promises';
-export declare class FileTokenizer extends AbstractTokenizer {
+interface IFileTokenizerOptions extends ITokenizerOptions {
+    /**
+     * Pass additional file information to the tokenizer
+     */
+    fileInfo: IRandomAccessFileInfo;
+}
+export declare class FileTokenizer extends AbstractTokenizer implements IRandomAccessTokenizer {
     private fileHandle;
-    constructor(fileHandle: FileHandle, options: ITokenizerOptions);
+    fileInfo: IRandomAccessFileInfo;
     /**
+     * Create tokenizer from provided file path
+     * @param sourceFilePath File path
+     */
+    static fromFile(sourceFilePath: string): Promise<FileTokenizer>;
+    protected constructor(fileHandle: FileHandle, options: IFileTokenizerOptions);
+    /**
      * Read buffer from file
@@ -22,3 +34,5 @@ * @param uint8Array - Uint8Array to write result to
     close(): Promise<void>;
+    setPosition(position: number): void;
+    supportsRandomAccess(): boolean;
 }
-export declare function fromFile(sourceFilePath: string): Promise<FileTokenizer>;
+export {};

lib/FileTokenizer.js

@@ -5,5 +5,15 @@ import { AbstractTokenizer } from './AbstractTokenizer.js';
 export class FileTokenizer extends AbstractTokenizer {
+    /**
+     * Create tokenizer from provided file path
+     * @param sourceFilePath File path
+     */
+    static async fromFile(sourceFilePath) {
+        const fileHandle = await fsOpen(sourceFilePath, 'r');
+        const stat = await fileHandle.stat();
+        return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
+    }
     constructor(fileHandle, options) {
         super(options);
         this.fileHandle = fileHandle;
+        this.fileInfo = options.fileInfo;
     }
@@ -46,7 +56,8 @@ /**
     }
+    setPosition(position) {
+        this.position = position;
+    }
+    supportsRandomAccess() {
+        return true;
+    }
 }
-export async function fromFile(sourceFilePath) {
-    const fileHandle = await fsOpen(sourceFilePath, 'r');
-    const stat = await fileHandle.stat();
-    return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
-}

lib/index.d.ts

 import type { Readable } from 'node:stream';
 import type { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import { type ITokenizerOptions } from './core.js';
-export { fromFile } from './FileTokenizer.js';
+import { FileTokenizer } from "./FileTokenizer.js";
+export { FileTokenizer } from './FileTokenizer.js';
 export * from './core.js';
@@ -15,1 +16,2 @@ export type { IToken, IGetToken } from '@tokenizer/token';
 export declare function fromStream(stream: Readable, options?: ITokenizerOptions): Promise<ReadStreamTokenizer>;
+export declare const fromFile: typeof FileTokenizer.fromFile;

lib/index.js

 import { stat as fsStat } from 'node:fs/promises';
 import { fromStream as coreFromStream } from './core.js';
-export { fromFile } from './FileTokenizer.js';
+import { FileTokenizer } from "./FileTokenizer.js";
+export { FileTokenizer } from './FileTokenizer.js';
 export * from './core.js';
@@ -13,10 +14,10 @@ /**
 export async function fromStream(stream, options) {
-    const augmentedOptions = options ?? {};
-    augmentedOptions.fileInfo = augmentedOptions.fileInfo ?? {};
+    const rst = coreFromStream(stream, options);
     if (stream.path) {
         const stat = await fsStat(stream.path);
-        augmentedOptions.fileInfo.path = stream.path;
-        augmentedOptions.fileInfo.size = stat.size;
+        rst.fileInfo.path = stream.path;
+        rst.fileInfo.size = stat.size;
     }
-    return coreFromStream(stream, augmentedOptions);
+    return rst;
 }
+export const fromFile = FileTokenizer.fromFile;

lib/ReadStreamTokenizer.d.ts

 import { AbstractTokenizer } from './AbstractTokenizer.js';
 import { type IStreamReader } from 'peek-readable';
-import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
+import type { IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
 export declare class ReadStreamTokenizer extends AbstractTokenizer {
     private streamReader;
+    fileInfo: IFileInfo;
     /**
@@ -28,2 +29,3 @@ * Constructor
     abort(): Promise<void>;
+    supportsRandomAccess(): boolean;
 }

lib/ReadStreamTokenizer.js

@@ -13,2 +13,3 @@ import { AbstractTokenizer } from './AbstractTokenizer.js';
         this.streamReader = streamReader;
+        this.fileInfo = options?.fileInfo ?? {};
     }
@@ -96,2 +97,5 @@ /**
     }
+    supportsRandomAccess() {
+        return false;
+    }
 }

lib/types.d.ts

@@ -20,2 +20,8 @@ import type { IGetToken } from '@tokenizer/token';
 }
+export interface IRandomAccessFileInfo extends IFileInfo {
+    /**
+     * File size in bytes
+     */
+    size: number;
+}
 export interface IReadChunkOptions {
@@ -33,10 +39,21 @@ /**
      * Default it is `tokenizer.position`.
-     * Position may not be less then `tokenizer.position`.
+     * Position may not be less than `tokenizer.position`, unless `supportsRandomAccess()` returns `true`.
      */
     position?: number;
     /**
-     * If set, will not throw an EOF error if not all of the requested data could be read
+     * If set, will not throw an EOF error if not all off the requested data could be read
      */
     mayBeLess?: boolean;
 }
+export interface IRandomAccessTokenizer extends ITokenizer {
+    /**
+     * Provide access to information of the underlying information stream or file.
+     */
+    fileInfo: IRandomAccessFileInfo;
+    /**
+     * Change the position (offset) of the tokenizer
+     * @param position New position
+     */
+    setPosition(position: number): void;
+}
 /**
@@ -50,7 +67,7 @@ * The tokenizer allows us to read or peek from the tokenizer-stream.
      */
-    fileInfo: IFileInfo;
+    readonly fileInfo: IFileInfo;
     /**
      * Offset in bytes (= number of bytes read) since beginning of file or stream
      */
-    position: number;
+    readonly position: number;
     /**
@@ -109,2 +126,6 @@ * Peek (read ahead) buffer from tokenizer
     abort(): Promise<void>;
+    /**
+     * Returns true when the underlying file supports random access
+     */
+    supportsRandomAccess(): boolean;
 }
@@ -111,0 +132,0 @@ export type OnClose = () => Promise<void>;

package.json

 {
   "name": "strtok3",
-  "version": "9.0.1",
+  "version": "9.1.0",
   "description": "A promise based streaming tokenizer",
@@ -5,0 +5,0 @@ "author": {

README.md

@@ -0,0 +0,0 @@ [![Node.js CI](https://github.com/Borewit/strtok3/actions/workflows/nodejs-ci.yml/badge.svg?branch=master)](https://github.com/Borewit/strtok3/actions/workflows/nodejs-ci.yml?query=branch%3Amaster)

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

52592

increased by5.06%

Lines of code

750

increased by9.33%

No dependency changes