strtok3 - npm Package Compare versions

Comparing version 7.1.1 to 8.0.0

lib/AbstractTokenizer.d.ts

@@ -1,3 +0,3 @@
-import { ITokenizer, IFileInfo, IReadChunkOptions } from './types.js';
-import { IGetToken, IToken } from '@tokenizer/token';
+import type { ITokenizer, IFileInfo, IReadChunkOptions, ITokenizerOptions } from './types.js';
+import type { IGetToken, IToken } from '@tokenizer/token';
 interface INormalizedReadChunkOptions extends IReadChunkOptions {
@@ -14,4 +14,10 @@ offset: number;
     fileInfo: IFileInfo;
-    protected constructor(fileInfo?: IFileInfo);
+    private onClose?;
     /**
+     * Constructor
+     * @param options Tokenizer options
+     * @protected
+     */
+    protected constructor(options?: ITokenizerOptions);
+    /**
      * Tokenizer-stream position
@@ -30,3 +36,3 @@ */
      * Peek (read ahead) buffer from tokenizer
-     * @param uint8Array- Target buffer to fill with data peek from the tokenizer-stream
+     * @param uint8Array - Target buffer to fill with data peeked from the tokenizer-stream
      * @param options - Peek behaviour options
@@ -33,0 +39,0 @@ * @returns Promise with number of bytes read

lib/AbstractTokenizer.js

@@ -6,3 +6,9 @@ import { EndOfStreamError } from 'peek-readable';
 export class AbstractTokenizer {
-    constructor(fileInfo) {
+    /**
+     * Constructor
+     * @param options Tokenizer options
+     * @protected
+     */
+    constructor(options) {
+        var _a;
         /**
@@ -13,3 +19,4 @@ * Tokenizer-stream position
         this.numBuffer = new Uint8Array(8);
-        this.fileInfo = fileInfo ? fileInfo : {};
+        this.fileInfo = (_a = options === null || options === void 0 ? void 0 : options.fileInfo) !== null && _a !== void 0 ? _a : {};
+        this.onClose = options === null || options === void 0 ? void 0 : options.onClose;
     }
@@ -81,3 +88,4 @@ /**
     async close() {
-        // empty
+        var _a;
+        await ((_a = this.onClose) === null || _a === void 0 ? void 0 : _a.call(this));
     }
@@ -84,0 +92,0 @@ normalizeOptions(uint8Array, options) {

lib/BufferTokenizer.d.ts

@@ -1,2 +0,2 @@
-import { IFileInfo, IReadChunkOptions } from './types.js';
+import { ITokenizerOptions, IReadChunkOptions } from './types.js';
 import { AbstractTokenizer } from './AbstractTokenizer.js';
@@ -8,5 +8,5 @@ export declare class BufferTokenizer extends AbstractTokenizer {
      * @param uint8Array - Uint8Array to tokenize
-     * @param fileInfo - Pass additional file information to the tokenizer
+     * @param options Tokenizer options
      */
-    constructor(uint8Array: Uint8Array, fileInfo?: IFileInfo);
+    constructor(uint8Array: Uint8Array, options?: ITokenizerOptions);
     /**
@@ -13,0 +13,0 @@ * Read buffer from tokenizer

lib/BufferTokenizer.js

@@ -7,6 +7,6 @@ import { EndOfStreamError } from 'peek-readable';
      * @param uint8Array - Uint8Array to tokenize
-     * @param fileInfo - Pass additional file information to the tokenizer
+     * @param options Tokenizer options
      */
-    constructor(uint8Array, fileInfo) {
-        super(fileInfo);
+    constructor(uint8Array, options) {
+        super(options);
         this.uint8Array = uint8Array;
@@ -49,5 +49,5 @@ this.fileInfo.size = this.fileInfo.size ? this.fileInfo.size : uint8Array.length;
     }
-    async close() {
-        // empty
+    close() {
+        return super.close();
     }
 }

lib/core.d.ts

 import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
 import { BufferTokenizer } from './BufferTokenizer.js';
-import { IFileInfo } from './types.js';
+import type { ITokenizerOptions } from './types.js';
 import type { ReadableStream } from 'node:stream/web';
 import type { Readable } from 'node:stream';
 export { EndOfStreamError } from 'peek-readable';
-export { ITokenizer, IFileInfo } from './types.js';
-export { IToken, IGetToken } from '@tokenizer/token';
+export type { ITokenizer, IFileInfo, ITokenizerOptions, OnClose } from './types.js';
+export type { IToken, IGetToken } from '@tokenizer/token';
 /**
@@ -13,6 +13,6 @@ * Construct ReadStreamTokenizer from given Stream.
  * @param stream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @param options - Tokenizer options
  * @returns ReadStreamTokenizer
  */
-export declare function fromStream(stream: Readable, fileInfo?: IFileInfo): ReadStreamTokenizer;
+export declare function fromStream(stream: Readable, options?: ITokenizerOptions): ReadStreamTokenizer;
 /**
@@ -22,12 +22,12 @@ * Construct ReadStreamTokenizer from given ReadableStream (WebStream API).
  * @param webStream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @param options - Tokenizer options
  * @returns ReadStreamTokenizer
  */
-export declare function fromWebStream(webStream: ReadableStream<Uint8Array>, fileInfo?: IFileInfo): ReadStreamTokenizer;
+export declare function fromWebStream(webStream: ReadableStream<Uint8Array>, options?: ITokenizerOptions): ReadStreamTokenizer;
 /**
  * Construct ReadStreamTokenizer from given Buffer.
  * @param uint8Array - Uint8Array to tokenize
- * @param fileInfo - Pass additional file information to the tokenizer
+ * @param options - Tokenizer options
  * @returns BufferTokenizer
  */
-export declare function fromBuffer(uint8Array: Uint8Array, fileInfo?: IFileInfo): BufferTokenizer;
+export declare function fromBuffer(uint8Array: Uint8Array, options?: ITokenizerOptions): BufferTokenizer;

lib/core.js

@@ -9,8 +9,7 @@ import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
  * @param stream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @param options - Tokenizer options
  * @returns ReadStreamTokenizer
  */
-export function fromStream(stream, fileInfo) {
-    fileInfo = fileInfo ? fileInfo : {};
-    return new ReadStreamTokenizer(new StreamReader(stream), fileInfo);
+export function fromStream(stream, options) {
+    return new ReadStreamTokenizer(new StreamReader(stream), options);
 }
@@ -21,8 +20,7 @@ /**
  * @param webStream - Read from Node.js Stream.Readable
- * @param fileInfo - Pass the file information, like size and MIME-type of the corresponding stream.
+ * @param options - Tokenizer options
  * @returns ReadStreamTokenizer
  */
-export function fromWebStream(webStream, fileInfo) {
-    fileInfo = fileInfo ? fileInfo : {};
-    return new ReadStreamTokenizer(new WebStreamReader(webStream), fileInfo);
+export function fromWebStream(webStream, options) {
+    return new ReadStreamTokenizer(new WebStreamReader(webStream), options);
 }
@@ -32,7 +30,7 @@ /**
  * @param uint8Array - Uint8Array to tokenize
- * @param fileInfo - Pass additional file information to the tokenizer
+ * @param options - Tokenizer options
  * @returns BufferTokenizer
  */
-export function fromBuffer(uint8Array, fileInfo) {
-    return new BufferTokenizer(uint8Array, fileInfo);
+export function fromBuffer(uint8Array, options) {
+    return new BufferTokenizer(uint8Array, options);
 }

lib/FileTokenizer.d.ts

 import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { IFileInfo, IReadChunkOptions } from './types.js';
+import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
 import { FileHandle } from 'node:fs/promises';
 export declare class FileTokenizer extends AbstractTokenizer {
     private fileHandle;
-    constructor(fileHandle: FileHandle, fileInfo: IFileInfo);
+    constructor(fileHandle: FileHandle, options: ITokenizerOptions);
     /**
@@ -8,0 +8,0 @@ * Read buffer from file

lib/FileTokenizer.js

@@ -5,4 +5,4 @@ import { AbstractTokenizer } from './AbstractTokenizer.js';
 export class FileTokenizer extends AbstractTokenizer {
-    constructor(fileHandle, fileInfo) {
-        super(fileInfo);
+    constructor(fileHandle, options) {
+        super(options);
         this.fileHandle = fileHandle;
@@ -41,3 +41,4 @@ }
     async close() {
-        return this.fileHandle.close();
+        await this.fileHandle.close();
+        return super.close();
     }
@@ -48,3 +49,3 @@ }
     const stat = await fileHandle.stat();
-    return new FileTokenizer(fileHandle, { path: sourceFilePath, size: stat.size });
+    return new FileTokenizer(fileHandle, { fileInfo: { path: sourceFilePath, size: stat.size } });
 }

lib/index.d.ts

 import { Readable } from 'node:stream';
 import { ReadStreamTokenizer } from './ReadStreamTokenizer.js';
-import * as core from './core.js';
+import { type ITokenizerOptions } from './core.js';
 export { fromFile } from './FileTokenizer.js';
-export { ITokenizer, EndOfStreamError, fromBuffer, fromWebStream, IFileInfo } from './core.js';
+export { ITokenizer, EndOfStreamError, fromBuffer, fromWebStream, IFileInfo, ITokenizerOptions, OnClose } from './core.js';
 export { IToken, IGetToken } from '@tokenizer/token';
@@ -11,5 +11,5 @@ /**
  * @param stream - Node.js Stream.Readable
- * @param fileInfo - Pass additional file information to the tokenizer
+ * @param options - Pass additional file information to the tokenizer
  * @returns Tokenizer
  */
-export declare function fromStream(stream: Readable, fileInfo?: core.IFileInfo): Promise<ReadStreamTokenizer>;
+export declare function fromStream(stream: Readable, options?: ITokenizerOptions): Promise<ReadStreamTokenizer>;

lib/index.js

 import { stat as fsStat } from 'node:fs/promises';
-import * as core from './core.js';
+import { fromStream as coreFromStream } from './core.js';
 export { fromFile } from './FileTokenizer.js';
@@ -9,13 +9,15 @@ export { EndOfStreamError, fromBuffer, fromWebStream } from './core.js';
  * @param stream - Node.js Stream.Readable
- * @param fileInfo - Pass additional file information to the tokenizer
+ * @param options - Pass additional file information to the tokenizer
  * @returns Tokenizer
  */
-export async function fromStream(stream, fileInfo) {
-    fileInfo = fileInfo ? fileInfo : {};
+export async function fromStream(stream, options) {
+    var _a;
+    const augmentedOptions = options !== null && options !== void 0 ? options : {};
+    augmentedOptions.fileInfo = (_a = augmentedOptions.fileInfo) !== null && _a !== void 0 ? _a : {};
     if (stream.path) {
         const stat = await fsStat(stream.path);
-        fileInfo.path = stream.path;
-        fileInfo.size = stat.size;
+        augmentedOptions.fileInfo.path = stream.path;
+        augmentedOptions.fileInfo.size = stat.size;
     }
-    return core.fromStream(stream, fileInfo);
+    return coreFromStream(stream, augmentedOptions);
 }

lib/ReadStreamTokenizer.d.ts

 import { AbstractTokenizer } from './AbstractTokenizer.js';
-import { IStreamReader } from 'peek-readable';
-import { IFileInfo, IReadChunkOptions } from './types.js';
+import { type IStreamReader } from 'peek-readable';
+import type { IReadChunkOptions, ITokenizerOptions } from './types.js';
 export declare class ReadStreamTokenizer extends AbstractTokenizer {
     private streamReader;
-    constructor(streamReader: IStreamReader, fileInfo?: IFileInfo);
     /**
-     * Get file information, an HTTP-client may implement this doing a HEAD request
-     * @return Promise with file information
+     * Constructor
+     * @param streamReader stream-reader to read from
+     * @param options Tokenizer options
      */
-    getFileInfo(): Promise<IFileInfo>;
+    constructor(streamReader: IStreamReader, options?: ITokenizerOptions);
     /**
@@ -13,0 +13,0 @@ * Read buffer from tokenizer

lib/ReadStreamTokenizer.js

@@ -5,12 +5,10 @@ import { AbstractTokenizer } from './AbstractTokenizer.js';
 export class ReadStreamTokenizer extends AbstractTokenizer {
-    constructor(streamReader, fileInfo) {
-        super(fileInfo);
-        this.streamReader = streamReader;
-    }
     /**
-     * Get file information, an HTTP-client may implement this doing a HEAD request
-     * @return Promise with file information
+     * Constructor
+     * @param streamReader stream-reader to read from
+     * @param options Tokenizer options
      */
-    async getFileInfo() {
-        return this.fileInfo;
+    constructor(streamReader, options) {
+        super(options);
+        this.streamReader = streamReader;
     }
@@ -17,0 +15,0 @@ /**

lib/types.d.ts

@@ -103,1 +103,12 @@ import { IGetToken } from '@tokenizer/token';
 }
+export type OnClose = () => Promise<void>;
+export interface ITokenizerOptions {
+    /**
+     * Pass additional file information to the tokenizer
+     */
+    fileInfo?: IFileInfo;
+    /**
+     * On tokenizer close handler
+     */
+    onClose?: OnClose;
+}

package.json

 {
   "name": "strtok3",
-  "version": "7.1.1",
+  "version": "8.0.0",
   "description": "A promise based streaming tokenizer",
@@ -14,3 +14,3 @@ "author": {
   "scripts": {
-    "clean": "del-cli lib/**/*.js lib/**/*.js.map lib/**/*.d.ts test/**/*.js test/**/*.js.map",
+    "clean": "del-cli 'lib/**/*.js' 'lib/**/*.js.map' 'lib/**/*.d.ts' 'test/**/*.js' 'test/**/*.js.map'",
     "compile-src": "tsc -p lib",
@@ -17,0 +17,0 @@ "compile-test": "tsc -p test",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

49059

increased by1.28%

Lines of code

664

increased by3.75%

No dependency changes