lexing - npm Package Compare versions

Comparing version 0.5.1 to 0.5.2

index.d.ts

@@ -137,3 +137,3 @@ /**
 /**
-Provide iterative access to a file.
+Provide buffered access to a stream of bytes, e.g., a file.
 
@@ -155,9 +155,7 @@ It is buffered, which means you can call `peek(same_number)` repeatedly without
     /**
-    Return the position in the file that would be read from if we called
-    read(...). This is different from the internally-held position, which
-    points to the end of the currently held buffer.
+    @returns {number} The position (byte offset) in the file that would be read from if we called read(...). This is different from the internally-held position, which points to the end of the currently held buffer.
     */
     position: number;
     /**
-    Return the total size (in bytes) of the underlying file.
+    @returns {number} The total size (in bytes) of the underlying file.
     */
@@ -168,4 +166,4 @@ size: number;
   
-    Returns false if the read operation reads fewer than the requested bytes,
-    usually signifying that EOF has been reached.
+    @param {number} length The number of bytes to consume and append to the buffer.
+    @returns {boolean} If the read operation reads fewer than the requested bytes, returns false, usually signifying that EOF has been reached. Returns true if it seems that there is more available data.
     */
@@ -179,17 +177,30 @@ private _fillBuffer(length);
   
-    This may return without the condition being met, if the end of the underlying
-    file has been reached.
+    @param {Function} predicate A function that takes a Buffer and returns true if it's long enough.
+    @returns {void} This may return without the condition being met, if the end of the underlying file has been reached.
     */
     protected _readWhile(predicate: (buffer: Buffer) => boolean): void;
+    /**
+    Read from the underlying source until we have at least `length` bytes in the buffer.
+  
+    @param {number} length The number of bytes to return without consuming. This is an upper bound, since the underlying source may contain fewer than the desired bytes.
+    */
+    protected _peekBytes(length: number): Buffer;
+    /**
+    Like _peekBytes(length), but consumes the bytes, so that subsequent calls return subsequent chunks.
+  
+    @param {number} length The number of bytes to return (upper bound).
+    */
+    protected _nextBytes(length: number): Buffer;
+    /**
+    Like _nextBytes(length), but doesn't ever slice off a buffer to hold the skipped bytes.
+  
+    @param {number} length The number of bytes that were skipped (consumed without returning), which may be fewer than `length` iff EOF has been reached.
+    */
+    protected _skipBytes(length: number): number;
 }
 export declare class SourceBufferIterator extends BufferedSourceReader implements BufferIterable {
     constructor(source: Source, position?: number, block_size?: number);
-    private _ensureLength(length);
-    next(length: number): Buffer;
-    peek(length: number): Buffer;
-    /**
-    Skip over the next `length` bytes, returning the number of skipped
-    bytes (which may be < `length` iff EOF has been reached).
-    */
-    skip(length: number): number;
+    next: (length: number) => Buffer;
+    peek: (length: number) => Buffer;
+    skip: (length: number) => number;
 }
@@ -199,5 +210,4 @@ export declare class SourceStringIterator extends BufferedSourceReader implements StringIterable {
     constructor(source: Source, _encoding: string, position?: number, block_size?: number);
-    private _ensureLength(length);
+    peek(length: number): string;
     next(length: number): string;
-    peek(length: number): string;
     /**
@@ -208,8 +218,5 @@ Skip over the next `length` characters, returning the number of skipped
     skip(length: number): number;
-    /**
-    Provide raw Buffer-level access, too.
-    */
-    nextBytes(length: number): Buffer;
-    peekBytes(length: number): Buffer;
-    skipBytes(length: number): number;
+    nextBytes: (length: number) => Buffer;
+    peekBytes: (length: number) => Buffer;
+    skipBytes: (length: number) => number;
 }
@@ -216,0 +223,0 @@ /**

index.js

@@ -148,6 +148,4 @@ var __extends = (this && this.__extends) || function (d, b) {
 exports.ArrayIterator = ArrayIterator;
-// #############################################################################
-//                          SYNCHRONOUS FILE READER
 /**
-Provide iterative access to a file.
+Provide buffered access to a stream of bytes, e.g., a file.
 
@@ -174,5 +172,3 @@ It is buffered, which means you can call `peek(same_number)` repeatedly without
         /**
-        Return the position in the file that would be read from if we called
-        read(...). This is different from the internally-held position, which
-        points to the end of the currently held buffer.
+        @returns {number} The position (byte offset) in the file that would be read from if we called read(...). This is different from the internally-held position, which points to the end of the currently held buffer.
         */
@@ -187,3 +183,3 @@ get: function () {
         /**
-        Return the total size (in bytes) of the underlying file.
+        @returns {number} The total size (in bytes) of the underlying file.
         */
@@ -199,4 +195,4 @@ get: function () {
   
-    Returns false if the read operation reads fewer than the requested bytes,
-    usually signifying that EOF has been reached.
+    @param {number} length The number of bytes to consume and append to the buffer.
+    @returns {boolean} If the read operation reads fewer than the requested bytes, returns false, usually signifying that EOF has been reached. Returns true if it seems that there is more available data.
     */
@@ -219,4 +215,4 @@ BufferedSourceReader.prototype._fillBuffer = function (length) {
   
-    This may return without the condition being met, if the end of the underlying
-    file has been reached.
+    @param {Function} predicate A function that takes a Buffer and returns true if it's long enough.
+    @returns {void} This may return without the condition being met, if the end of the underlying file has been reached.
     */
@@ -232,33 +228,30 @@ BufferedSourceReader.prototype._readWhile = function (predicate) {
     };
-    return BufferedSourceReader;
-})();
-exports.BufferedSourceReader = BufferedSourceReader;
-var SourceBufferIterator = (function (_super) {
-    __extends(SourceBufferIterator, _super);
-    function SourceBufferIterator(source, position, block_size) {
-        if (position === void 0) { position = 0; }
-        if (block_size === void 0) { block_size = 1024; }
-        _super.call(this, source, position, block_size);
-    }
-    SourceBufferIterator.prototype._ensureLength = function (length) {
+    /**
+    Read from the underlying source until we have at least `length` bytes in the buffer.
+  
+    @param {number} length The number of bytes to return without consuming. This is an upper bound, since the underlying source may contain fewer than the desired bytes.
+    */
+    BufferedSourceReader.prototype._peekBytes = function (length) {
         var _this = this;
-        // all the action happens only if we need more bytes than are in the buffer
         this._readWhile(function () { return length > _this._buffer.length; });
+        return this._buffer.slice(0, length);
     };
-    SourceBufferIterator.prototype.next = function (length) {
-        this._ensureLength(length);
-        var buffer = this._buffer.slice(0, length);
+    /**
+    Like _peekBytes(length), but consumes the bytes, so that subsequent calls return subsequent chunks.
+  
+    @param {number} length The number of bytes to return (upper bound).
+    */
+    BufferedSourceReader.prototype._nextBytes = function (length) {
+        var buffer = this._peekBytes(length);
         this._buffer = this._buffer.slice(length);
         return buffer;
     };
-    SourceBufferIterator.prototype.peek = function (length) {
-        this._ensureLength(length);
-        return this._buffer.slice(0, length);
-    };
     /**
-    Skip over the next `length` bytes, returning the number of skipped
-    bytes (which may be < `length` iff EOF has been reached).
+    Like _nextBytes(length), but doesn't ever slice off a buffer to hold the skipped bytes.
+  
+    @param {number} length The number of bytes that were skipped (consumed without returning), which may be fewer than `length` iff EOF has been reached.
     */
-    SourceBufferIterator.prototype.skip = function (length) {
-        this._ensureLength(length);
+    BufferedSourceReader.prototype._skipBytes = function (length) {
+        var _this = this;
+        this._readWhile(function () { return length > _this._buffer.length; });
         // we cannot skip more than `this._buffer.length` bytes
@@ -269,2 +262,15 @@ var bytesSkipped = Math.min(length, this._buffer.length);
     };
+    return BufferedSourceReader;
+})();
+exports.BufferedSourceReader = BufferedSourceReader;
+var SourceBufferIterator = (function (_super) {
+    __extends(SourceBufferIterator, _super);
+    function SourceBufferIterator(source, position, block_size) {
+        if (position === void 0) { position = 0; }
+        if (block_size === void 0) { block_size = 1024; }
+        _super.call(this, source, position, block_size);
+        this.next = this._nextBytes;
+        this.peek = this._peekBytes;
+        this.skip = this._skipBytes;
+    }
     return SourceBufferIterator;
@@ -280,21 +286,27 @@ })(BufferedSourceReader);
         this._encoding = _encoding;
+        // Provide raw Buffer-level access, too, beyond/outside the StringIterable interface.
+        this.nextBytes = this._nextBytes;
+        this.peekBytes = this._peekBytes;
+        this.skipBytes = this._skipBytes;
     }
-    SourceStringIterator.prototype._ensureLength = function (length) {
+    SourceStringIterator.prototype.peek = function (length) {
         var _this = this;
-        // TODO: count characters without reencoding
+        // TODO: don't re-encode the whole string and then only use a tiny bit of it
+        // ensure that our subsequent call to toString() will
+        // return a string that is at least `length` long.
         this._readWhile(function () { return length > _this._buffer.toString(_this._encoding).length; });
+        return this._buffer.toString(this._encoding).slice(0, length);
     };
     SourceStringIterator.prototype.next = function (length) {
-        // TODO: don't re-encode the whole string and then only use a tiny bit of it
-        this._ensureLength(length);
-        var str = this._buffer.toString(this._encoding).slice(0, length);
+        // TODO (see TODO in peek())
+        var str = this.peek(length);
+        // even though we know we consumed (at least) `length` number of characters,
+        // we also need to know exactly how long that string is in bytes, in order
+        // to advance the underlying buffer appropriately
         var byteLength = Buffer.byteLength(str, this._encoding);
+        // we cannot skip more than `this._buffer.length` bytes
+        // var bytesSkipped = Math.min(byteLength, this._buffer.length); // is this necessary?
         this._buffer = this._buffer.slice(byteLength);
         return str;
     };
-    SourceStringIterator.prototype.peek = function (length) {
-        // TODO (see TODO in next())
-        this._ensureLength(length);
-        return this._buffer.toString(this._encoding).slice(0, length);
-    };
     /**
@@ -306,34 +318,5 @@ Skip over the next `length` characters, returning the number of skipped
         // TODO (see TODO in next())
-        // _ensureLength(length) ensures that our subsequent call to toString() will
-        // return a string that is at least `length` long.
-        this._ensureLength(length);
-        var consumed_string = this._buffer.toString(this._encoding).slice(0, length);
-        // even though we know we consumed (at least) `length` number of characters,
-        // we also need to know exactly how long that string is in bytes, in order
-        // to advance the underlying buffer appropriately
-        var byteLength = Buffer.byteLength(consumed_string, this._encoding);
-        // we cannot skip more than `this._buffer.length` bytes
-        // var bytesSkipped = Math.min(byteLength, this._buffer.length); // is this necessary?
-        this._buffer = this._buffer.slice(byteLength);
+        var consumed_string = this.next(length);
         return consumed_string.length;
     };
-    /**
-    Provide raw Buffer-level access, too.
-    */
-    SourceStringIterator.prototype.nextBytes = function (length) {
-        this._ensureLength(length);
-        var buffer = this._buffer.slice(0, length);
-        this._buffer = this._buffer.slice(length);
-        return buffer;
-    };
-    SourceStringIterator.prototype.peekBytes = function (length) {
-        this._ensureLength(length);
-        return this._buffer.slice(0, length);
-    };
-    SourceStringIterator.prototype.skipBytes = function (length) {
-        this._ensureLength(length);
-        var bytesSkipped = Math.min(length, this._buffer.length);
-        this._buffer = this._buffer.slice(length);
-        return bytesSkipped;
-    };
     return SourceStringIterator;
@@ -340,0 +323,0 @@ })(BufferedSourceReader);

package.json

 {
   "name": "lexing",
-  "version": "0.5.1",
+  "version": "0.5.2",
   "description": "Regex-based lexer",
@@ -5,0 +5,0 @@ "keywords": [

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

43366

increased by1.73%

Worsened metrics

Lines of code

931

decreased by-1.69%

No dependency changes