lexing - npm Package Compare versions

Comparing version 0.2.0 to 0.3.0

index.js

@@ -0,5 +1,9 @@
+var __extends = this.__extends || function (d, b) {
+    for (var p in b) if (b.hasOwnProperty(p)) d[p] = b[p];
+    function __() { this.constructor = d; }
+    __.prototype = b.prototype;
+    d.prototype = new __();
+};
 /// <reference path="type_declarations/DefinitelyTyped/node/node.d.ts" />
 var fs = require('fs');
-// #############################################################################
-//                           BASIC BUFFER READER
 /**
@@ -9,6 +13,6 @@ Wraps a Buffer as a stateful iterable.
 var BufferIterator = (function () {
-    function BufferIterator(_buffer, _position) {
-        if (_position === void 0) { _position = 0; }
+    function BufferIterator(_buffer, position) {
+        if (position === void 0) { position = 0; }
         this._buffer = _buffer;
-        this._position = _position;
+        this.position = position;
     }
@@ -19,12 +23,2 @@ BufferIterator.fromString = function (str, encoding) {
     };
-    Object.defineProperty(BufferIterator.prototype, "position", {
-        /**
-        Return the current position within the underlying Buffer.
-        */
-        get: function () {
-            return this._position;
-        },
-        enumerable: true,
-        configurable: true
-    });
     Object.defineProperty(BufferIterator.prototype, "size", {
@@ -45,3 +39,3 @@ /**
     BufferIterator.prototype.peek = function (length) {
-        return this._buffer.slice(this._position, this._position + length);
+        return this._buffer.slice(this.position, this.position + length);
     };
@@ -57,4 +51,4 @@ /**
     BufferIterator.prototype.next = function (length) {
-        var buffer = this._buffer.slice(this._position, this._position + length);
-        this._position += buffer.length;
+        var buffer = this._buffer.slice(this.position, this.position + length);
+        this.position += buffer.length;
         return buffer;
@@ -69,4 +63,4 @@ };
     BufferIterator.prototype.skip = function (length) {
-        var bytesSkipped = Math.min(length, this._buffer.length - this._position);
-        this._position += bytesSkipped;
+        var bytesSkipped = Math.min(length, this._buffer.length - this.position);
+        this.position += bytesSkipped;
         return bytesSkipped;
@@ -78,2 +72,55 @@ };
 /**
+Wraps a string as a stateful iterable.
+*/
+var StringIterator = (function () {
+    function StringIterator(_string, position) {
+        if (position === void 0) { position = 0; }
+        this._string = _string;
+        this.position = position;
+    }
+    StringIterator.fromBuffer = function (buffer, encoding) {
+        var str = buffer.toString(encoding);
+        return new StringIterator(str);
+    };
+    Object.defineProperty(StringIterator.prototype, "size", {
+        /**
+        Return the total length of the underlying Buffer.
+        */
+        get: function () {
+            return this._string.length;
+        },
+        enumerable: true,
+        configurable: true
+    });
+    /**
+    Read the next `length` characters from the underlying string, or fewer iff
+    we reach EOF, without advancing our position within the string.
+    */
+    StringIterator.prototype.peek = function (length) {
+        return this._string.slice(this.position, this.position + length);
+    };
+    /**
+    Read the next `length` characters from the underlying string, or fewer iff
+    we reach EOF, and advance our position within the string.
+    */
+    StringIterator.prototype.next = function (length) {
+        var chunk = this._string.slice(this.position, this.position + length);
+        this.position += chunk.length;
+        return chunk;
+    };
+    /**
+    Skip over the next `length` characters, returning the number of skipped
+    characters (which may be < `length` iff EOF has been reached).
+  
+    We do not allow skipping beyond the end of the string.
+    */
+    StringIterator.prototype.skip = function (length) {
+        var charsSkipped = Math.min(length, this._string.length - this.position);
+        this.position += charsSkipped;
+        return charsSkipped;
+    };
+    return StringIterator;
+})();
+exports.StringIterator = StringIterator;
+/**
 Wrap an Array as an iterable.
@@ -123,5 +170,5 @@ */
 */
-var FileIterator = (function () {
+var BufferedFileReader = (function () {
     // when reading more data, pull in chunks of `_block_size` bytes.
-    function FileIterator(_fd, _position, _block_size) {
+    function BufferedFileReader(_fd, _position, _block_size) {
         if (_position === void 0) { _position = 0; }
@@ -134,10 +181,3 @@ if (_block_size === void 0) { _block_size = 1024; }
     }
-    FileIterator.open = function (filepath) {
-        var fd = fs.openSync(filepath, 'r');
-        return new FileIterator(fd);
-    };
-    FileIterator.prototype.close = function () {
-        fs.closeSync(this._fd);
-    };
-    Object.defineProperty(FileIterator.prototype, "position", {
+    Object.defineProperty(BufferedFileReader.prototype, "position", {
         /**
@@ -154,3 +194,3 @@ Return the position in the file that would be read from if we called
     });
-    Object.defineProperty(FileIterator.prototype, "size", {
+    Object.defineProperty(BufferedFileReader.prototype, "size", {
         /**
@@ -166,23 +206,8 @@ Return the total size (in bytes) of the underlying file.
     /**
-    Ensure that the available buffer is at least `length` bytes long.
+    Read data from the underlying file and append it to the buffer.
   
-    This may return without the condition being met (this.buffer.length >= length),
-    if the end of the underlying file has been reached.
-  
-    TODO: pull _fillBuffer into the loop, with the Buffer declaration outside.
+    Returns false if the read operation reads fewer than the requested bytes,
+    usually signifying that EOF has been reached.
     */
-    FileIterator.prototype._ensureLength = function (length) {
-        while (length > this._buffer.length) {
-            // all the action happens only if we need more bytes than are in the buffer
-            var EOF = this._fillBuffer(this._block_size);
-            if (EOF) {
-                break;
-            }
-        }
-    };
-    /**
-    Read data from the underlying file and append it to the buffer.
-  
-    Returns false iff EOF has been reached, otherwise returns true. */
-    FileIterator.prototype._fillBuffer = function (length) {
+    BufferedFileReader.prototype._fillBuffer = function (length) {
         var buffer = new Buffer(length);
@@ -197,3 +222,35 @@ // always read from the current position
     };
-    FileIterator.prototype.next = function (length) {
+    /**
+    Read from the underlying file, appending to the currently held buffer,
+    until the given predicate function returns false. That function will be called
+    repeatedly with no arguments. If it returns false the first time it is called,
+    nothing will be read.
+  
+    This may return without the condition being met, if the end of the underlying
+    file has been reached.
+    */
+    BufferedFileReader.prototype._readWhile = function (predicate) {
+        while (predicate(this._buffer)) {
+            var EOF = this._fillBuffer(this._block_size);
+            if (EOF) {
+                break;
+            }
+        }
+    };
+    return BufferedFileReader;
+})();
+exports.BufferedFileReader = BufferedFileReader;
+var FileBufferIterator = (function (_super) {
+    __extends(FileBufferIterator, _super);
+    function FileBufferIterator(_fd, _position, _block_size) {
+        if (_position === void 0) { _position = 0; }
+        if (_block_size === void 0) { _block_size = 1024; }
+        _super.call(this, _fd, _position, _block_size);
+    }
+    FileBufferIterator.prototype._ensureLength = function (length) {
+        var _this = this;
+        // all the action happens only if we need more bytes than are in the buffer
+        this._readWhile(function () { return length > _this._buffer.length; });
+    };
+    FileBufferIterator.prototype.next = function (length) {
         this._ensureLength(length);
@@ -204,3 +261,3 @@ var buffer = this._buffer.slice(0, length);
     };
-    FileIterator.prototype.peek = function (length) {
+    FileBufferIterator.prototype.peek = function (length) {
         this._ensureLength(length);
@@ -210,8 +267,70 @@ return this._buffer.slice(0, length);
     /**
+    Skip over the next `length` bytes, returning the number of skipped
+    bytes (which may be < `length` iff EOF has been reached).
+    */
+    FileBufferIterator.prototype.skip = function (length) {
+        this._ensureLength(length);
+        // we cannot skip more than `this.buffer.length` bytes
+        var bytesSkipped = Math.min(length, this._buffer.length);
+        this._buffer = this._buffer.slice(length);
+        return bytesSkipped;
+    };
+    return FileBufferIterator;
+})(BufferedFileReader);
+exports.FileBufferIterator = FileBufferIterator;
+var FileStringIterator = (function (_super) {
+    __extends(FileStringIterator, _super);
+    function FileStringIterator(_fd, _encoding, _position, _block_size) {
+        if (_position === void 0) { _position = 0; }
+        if (_block_size === void 0) { _block_size = 1024; }
+        _super.call(this, _fd, _position, _block_size);
+        this._encoding = _encoding;
+    }
+    FileStringIterator.prototype._ensureLength = function (length) {
+        var _this = this;
+        // TODO: count characters without reencoding
+        this._readWhile(function () { return length > _this._buffer.toString(_this._encoding).length; });
+    };
+    FileStringIterator.prototype.next = function (length) {
+        // TODO: don't re-encode the whole string and then only use a tiny bit of it
+        this._ensureLength(length);
+        var str = this._buffer.toString(this._encoding).slice(0, length);
+        var byteLength = Buffer.byteLength(str, this._encoding);
+        this._buffer = this._buffer.slice(byteLength);
+        return str;
+    };
+    FileStringIterator.prototype.peek = function (length) {
+        // TODO (see TODO in next())
+        this._ensureLength(length);
+        return this._buffer.toString(this._encoding).slice(0, length);
+    };
+    /**
     Skip over the next `length` characters, returning the number of skipped
     characters (which may be < `length` iff EOF has been reached).
     */
-    FileIterator.prototype.skip = function (length) {
+    FileStringIterator.prototype.skip = function (length) {
+        // TODO (see TODO in next())
         this._ensureLength(length);
-        // we cannot skip more than `this.buffer.length` bytes
+        var consumed_string = this._buffer.toString(this._encoding).slice(0, length);
+        var byteLength = Buffer.byteLength(consumed_string, this._encoding);
+        // we cannot skip more than `this._buffer.length` bytes
+        var bytesSkipped = Math.min(byteLength, this._buffer.length);
+        this._buffer = this._buffer.slice(byteLength);
+        return consumed_string.length;
+    };
+    /**
+    Provide raw Buffer-level access, too.
+    */
+    FileStringIterator.prototype.nextBytes = function (length) {
+        this._ensureLength(length);
+        var buffer = this._buffer.slice(0, length);
+        this._buffer = this._buffer.slice(length);
+        return buffer;
+    };
+    FileStringIterator.prototype.peekBytes = function (length) {
+        this._ensureLength(length);
+        return this._buffer.slice(0, length);
+    };
+    FileStringIterator.prototype.skipBytes = function (length) {
+        this._ensureLength(length);
         var bytesSkipped = Math.min(length, this._buffer.length);
@@ -221,5 +340,5 @@ this._buffer = this._buffer.slice(length);
     };
-    return FileIterator;
-})();
-exports.FileIterator = FileIterator;
+    return FileStringIterator;
+})(BufferedFileReader);
+exports.FileStringIterator = FileStringIterator;
 function Token(name, value) {
@@ -233,4 +352,2 @@ if (value === void 0) { value = null; }
 always a string).
-
-BufferIterable
 */
@@ -274,8 +391,7 @@ var Tokenizer = (function () {
         var rules = this.tokenizer.getRules(state);
-        var input = this.iterable.peek(256).toString('utf8');
+        var input = this.iterable.peek(256);
         for (var i = 0, rule; (rule = rules[i]); i++) {
             var match = input.match(rule[0]);
             if (match) {
-                var byteLength = Buffer.byteLength(match[0], 'utf8');
-                this.iterable.skip(byteLength);
+                this.iterable.skip(match[0].length);
                 return rule[1].call(this, match);
@@ -282,0 +398,0 @@ }

index.ts

@@ -42,6 +42,11 @@ /// <reference path="type_declarations/DefinitelyTyped/node/node.d.ts" />
 /**
+Commonly used special case.
+*/
+export interface BufferIterable extends StatefulChunkedIterable<Buffer> { }
+
+/**
 Wraps a Buffer as a stateful iterable.
 */
-export class BufferIterator implements StatefulChunkedIterable<Buffer> {
-  constructor(private _buffer: Buffer, private _position = 0) { }
+export class BufferIterator implements BufferIterable {
+  constructor(private _buffer: Buffer, public position = 0) { }
 
@@ -54,9 +59,2 @@ static fromString(str: string, encoding?: string): BufferIterator {
   /**
-  Return the current position within the underlying Buffer.
-  */
-  get position(): number {
-    return this._position;
-  }
-
-  /**
   Return the total length of the underlying Buffer.
@@ -73,3 +71,3 @@ */
   peek(length: number): Buffer {
-    return this._buffer.slice(this._position, this._position + length);
+    return this._buffer.slice(this.position, this.position + length);
   }
@@ -86,4 +84,4 @@
   next(length: number): Buffer {
-    var buffer = this._buffer.slice(this._position, this._position + length);
-    this._position += buffer.length;
+    var buffer = this._buffer.slice(this.position, this.position + length);
+    this.position += buffer.length;
     return buffer;
@@ -99,4 +97,4 @@ }
   skip(length: number): number {
-    var bytesSkipped = Math.min(length, this._buffer.length - this._position);
-    this._position += bytesSkipped;
+    var bytesSkipped = Math.min(length, this._buffer.length - this.position);
+    this.position += bytesSkipped;
     return bytesSkipped;
@@ -106,3 +104,54 @@ }
 
+export interface StringIterable extends StatefulChunkedIterable<string> { }
+
 /**
+Wraps a string as a stateful iterable.
+*/
+export class StringIterator implements StringIterable {
+  constructor(private _string: string, public position = 0) { }
+
+  static fromBuffer(buffer: Buffer, encoding?: string): StringIterator {
+    var str = buffer.toString(encoding);
+    return new StringIterator(str);
+  }
+
+  /**
+  Return the total length of the underlying Buffer.
+  */
+  get size(): number {
+    return this._string.length;
+  }
+
+  /**
+  Read the next `length` characters from the underlying string, or fewer iff
+  we reach EOF, without advancing our position within the string.
+  */
+  peek(length: number): string {
+    return this._string.slice(this.position, this.position + length);
+  }
+
+  /**
+  Read the next `length` characters from the underlying string, or fewer iff
+  we reach EOF, and advance our position within the string.
+  */
+  next(length: number): string {
+    var chunk = this._string.slice(this.position, this.position + length);
+    this.position += chunk.length;
+    return chunk;
+  }
+
+  /**
+  Skip over the next `length` characters, returning the number of skipped
+  characters (which may be < `length` iff EOF has been reached).
+
+  We do not allow skipping beyond the end of the string.
+  */
+  skip(length: number): number {
+    var charsSkipped = Math.min(length, this._string.length - this.position);
+    this.position += charsSkipped;
+    return charsSkipped;
+  }
+}
+
+/**
 Wrap an Array as an iterable.
@@ -148,4 +197,4 @@ */
 */
-export class FileIterator implements StatefulChunkedIterable<Buffer> {
-  private _buffer: Buffer = new Buffer(0);
+export class BufferedFileReader {
+  protected _buffer: Buffer = new Buffer(0);
 
@@ -155,11 +204,2 @@ // when reading more data, pull in chunks of `_block_size` bytes.
 
-  static open(filepath: string): FileIterator {
-    var fd = fs.openSync(filepath, 'r');
-    return new FileIterator(fd);
-  }
-
-  close(): void {
-    fs.closeSync(this._fd);
-  }
-
   /**
@@ -182,12 +222,29 @@ Return the position in the file that would be read from if we called
   /**
-  Ensure that the available buffer is at least `length` bytes long.
+  Read data from the underlying file and append it to the buffer.
 
-  This may return without the condition being met (this.buffer.length >= length),
-  if the end of the underlying file has been reached.
+  Returns false if the read operation reads fewer than the requested bytes,
+  usually signifying that EOF has been reached.
+  */
+  private _fillBuffer(length: number): boolean {
+    var buffer = new Buffer(length);
+    // always read from the current position
+    var bytesRead = fs.readSync(this._fd, buffer, 0, length, this._position);
+    // and update it accordingly
+    this._position += bytesRead;
+    // use the Buffer.concat totalLength argument to slice the fresh buffer if needed
+    this._buffer = Buffer.concat([this._buffer, buffer], this._buffer.length + bytesRead);
+    return bytesRead < length;
+  }
 
-  TODO: pull _fillBuffer into the loop, with the Buffer declaration outside.
+  /**
+  Read from the underlying file, appending to the currently held buffer,
+  until the given predicate function returns false. That function will be called
+  repeatedly with no arguments. If it returns false the first time it is called,
+  nothing will be read.
+
+  This may return without the condition being met, if the end of the underlying
+  file has been reached.
   */
-  private _ensureLength(length: number): void {
-    while (length > this._buffer.length) {
-      // all the action happens only if we need more bytes than are in the buffer
+  protected _readWhile(predicate: (buffer: Buffer) => boolean): void {
+    while (predicate(this._buffer)) {
       var EOF = this._fillBuffer(this._block_size);
@@ -200,16 +257,12 @@ if (EOF) {
   }
+}
 
-  /**
-  Read data from the underlying file and append it to the buffer.
+export class FileBufferIterator extends BufferedFileReader implements BufferIterable {
+  constructor(_fd: number, _position = 0, _block_size = 1024) {
+    super(_fd, _position, _block_size);
+  }
 
-  Returns false iff EOF has been reached, otherwise returns true. */
-  private _fillBuffer(length: number): boolean {
-    var buffer = new Buffer(length);
-    // always read from the current position
-    var bytesRead = fs.readSync(this._fd, buffer, 0, length, this._position);
-    // and update it accordingly
-    this._position += bytesRead;
-    // use the Buffer.concat totalLength argument to slice the fresh buffer if needed
-    this._buffer = Buffer.concat([this._buffer, buffer], this._buffer.length + bytesRead);
-    return bytesRead < length;
+  private _ensureLength(length: number): void {
+    // all the action happens only if we need more bytes than are in the buffer
+    this._readWhile(() => length > this._buffer.length);
   }
@@ -230,2 +283,40 @@
   /**
+  Skip over the next `length` bytes, returning the number of skipped
+  bytes (which may be < `length` iff EOF has been reached).
+  */
+  skip(length: number): number {
+    this._ensureLength(length);
+    // we cannot skip more than `this.buffer.length` bytes
+    var bytesSkipped = Math.min(length, this._buffer.length);
+    this._buffer = this._buffer.slice(length);
+    return bytesSkipped;
+  }
+}
+
+export class FileStringIterator extends BufferedFileReader implements StringIterable {
+  constructor(_fd: number, private _encoding: string, _position = 0, _block_size = 1024) {
+    super(_fd, _position, _block_size);
+  }
+
+  private _ensureLength(length: number): void {
+    // TODO: count characters without reencoding
+    this._readWhile(() => length > this._buffer.toString(this._encoding).length);
+  }
+
+  next(length: number): string {
+    // TODO: don't re-encode the whole string and then only use a tiny bit of it
+    this._ensureLength(length);
+    var str = this._buffer.toString(this._encoding).slice(0, length);
+    var byteLength = Buffer.byteLength(str, this._encoding);
+    this._buffer = this._buffer.slice(byteLength);
+    return str;
+  }
+
+  peek(length: number): string {
+    // TODO (see TODO in next())
+    this._ensureLength(length);
+    return this._buffer.toString(this._encoding).slice(0, length);
+  }
+
+  /**
   Skip over the next `length` characters, returning the number of skipped
@@ -235,4 +326,27 @@ characters (which may be < `length` iff EOF has been reached).
   skip(length: number): number {
+    // TODO (see TODO in next())
     this._ensureLength(length);
-    // we cannot skip more than `this.buffer.length` bytes
+    var consumed_string = this._buffer.toString(this._encoding).slice(0, length);
+    var byteLength = Buffer.byteLength(consumed_string, this._encoding);
+    // we cannot skip more than `this._buffer.length` bytes
+    var bytesSkipped = Math.min(byteLength, this._buffer.length);
+    this._buffer = this._buffer.slice(byteLength);
+    return consumed_string.length;
+  }
+
+  /**
+  Provide raw Buffer-level access, too.
+  */
+  nextBytes(length: number): Buffer {
+    this._ensureLength(length);
+    var buffer = this._buffer.slice(0, length);
+    this._buffer = this._buffer.slice(length);
+    return buffer;
+  }
+  peekBytes(length: number): Buffer {
+    this._ensureLength(length);
+    return this._buffer.slice(0, length);
+  }
+  skipBytes(length: number): number {
+    this._ensureLength(length);
     var bytesSkipped = Math.min(length, this._buffer.length);
@@ -250,7 +364,2 @@ this._buffer = this._buffer.slice(length);
 /**
-Commonly used special case.
-*/
-export interface BufferIterable extends StatefulChunkedIterable<Buffer> { }
-
-/**
 Tokenizer#map() and Combiner#map() both return Token iterators.
@@ -286,4 +395,2 @@
 always a string).
-
-BufferIterable
 */
@@ -304,3 +411,3 @@ export class Tokenizer<T> {
   */
-  map(iterable: BufferIterable, states: string[] = []): TokenIterable<T> {
+  map(iterable: StringIterable, states: string[] = []): TokenIterable<T> {
     return new TokenizerIterator(this, iterable, states);
@@ -312,3 +419,3 @@ }
   constructor(private tokenizer: Tokenizer<T>,
-              public iterable: BufferIterable,
+              public iterable: StringIterable,
               public states: string[]) { }
@@ -325,8 +432,7 @@
     var rules = this.tokenizer.getRules(state);
-    var input = this.iterable.peek(256).toString('utf8');
+    var input = this.iterable.peek(256);
     for (var i = 0, rule; (rule = rules[i]); i++) {
       var match = input.match(rule[0]);
       if (match) {
-        var byteLength = Buffer.byteLength(match[0], 'utf8');
-        this.iterable.skip(byteLength);
+        this.iterable.skip(match[0].length);
         return rule[1].call(this, match);
@@ -333,0 +439,0 @@ }

lexing.d.ts

@@ -31,14 +31,15 @@ /// <reference path="../../type_declarations/DefinitelyTyped/node/node.d.ts" />
     /**
+    Commonly used special case.
+    */
+    interface BufferIterable extends StatefulChunkedIterable<Buffer> {
+    }
+    /**
     Wraps a Buffer as a stateful iterable.
     */
-    class BufferIterator implements StatefulChunkedIterable<Buffer> {
+    class BufferIterator implements BufferIterable {
         private _buffer;
-        private _position;
-        constructor(_buffer: Buffer, _position?: number);
+        position: number;
+        constructor(_buffer: Buffer, position?: number);
         static fromString(str: string, encoding?: string): BufferIterator;
         /**
-        Return the current position within the underlying Buffer.
-        */
-        position: number;
-        /**
         Return the total length of the underlying Buffer.
@@ -69,3 +70,35 @@ */
     }
+    interface StringIterable extends StatefulChunkedIterable<string> {
+    }
     /**
+    Wraps a string as a stateful iterable.
+    */
+    class StringIterator implements StringIterable {
+        private _string;
+        position: number;
+        constructor(_string: string, position?: number);
+        static fromBuffer(buffer: Buffer, encoding?: string): StringIterator;
+        /**
+        Return the total length of the underlying Buffer.
+        */
+        size: number;
+        /**
+        Read the next `length` characters from the underlying string, or fewer iff
+        we reach EOF, without advancing our position within the string.
+        */
+        peek(length: number): string;
+        /**
+        Read the next `length` characters from the underlying string, or fewer iff
+        we reach EOF, and advance our position within the string.
+        */
+        next(length: number): string;
+        /**
+        Skip over the next `length` characters, returning the number of skipped
+        characters (which may be < `length` iff EOF has been reached).
+      
+        We do not allow skipping beyond the end of the string.
+        */
+        skip(length: number): number;
+    }
+    /**
     Wrap an Array as an iterable.
@@ -93,10 +126,8 @@ */
     */
-    class FileIterator implements StatefulChunkedIterable<Buffer> {
+    class BufferedFileReader {
         private _fd;
         private _position;
         private _block_size;
-        private _buffer;
+        protected _buffer: Buffer;
         constructor(_fd: number, _position?: number, _block_size?: number);
-        static open(filepath: string): FileIterator;
-        close(): void;
         /**
@@ -113,18 +144,37 @@ Return the position in the file that would be read from if we called
         /**
-        Ensure that the available buffer is at least `length` bytes long.
+        Read data from the underlying file and append it to the buffer.
       
-        This may return without the condition being met (this.buffer.length >= length),
-        if the end of the underlying file has been reached.
+        Returns false if the read operation reads fewer than the requested bytes,
+        usually signifying that EOF has been reached.
+        */
+        private _fillBuffer(length);
+        /**
+        Read from the underlying file, appending to the currently held buffer,
+        until the given predicate function returns false. That function will be called
+        repeatedly with no arguments. If it returns false the first time it is called,
+        nothing will be read.
       
-        TODO: pull _fillBuffer into the loop, with the Buffer declaration outside.
+        This may return without the condition being met, if the end of the underlying
+        file has been reached.
         */
+        protected _readWhile(predicate: (buffer: Buffer) => boolean): void;
+    }
+    class FileBufferIterator extends BufferedFileReader implements BufferIterable {
+        constructor(_fd: number, _position?: number, _block_size?: number);
         private _ensureLength(length);
-        /**
-        Read data from the underlying file and append it to the buffer.
-      
-        Returns false iff EOF has been reached, otherwise returns true. */
-        private _fillBuffer(length);
         next(length: number): Buffer;
         peek(length: number): Buffer;
         /**
+        Skip over the next `length` bytes, returning the number of skipped
+        bytes (which may be < `length` iff EOF has been reached).
+        */
+        skip(length: number): number;
+    }
+    class FileStringIterator extends BufferedFileReader implements StringIterable {
+        private _encoding;
+        constructor(_fd: number, _encoding: string, _position?: number, _block_size?: number);
+        private _ensureLength(length);
+        next(length: number): string;
+        peek(length: number): string;
+        /**
         Skip over the next `length` characters, returning the number of skipped
@@ -134,9 +184,10 @@ characters (which may be < `length` iff EOF has been reached).
         skip(length: number): number;
+        /**
+        Provide raw Buffer-level access, too.
+        */
+        nextBytes(length: number): Buffer;
+        peekBytes(length: number): Buffer;
+        skipBytes(length: number): number;
     }
     /**
-    Commonly used special case.
-    */
-    interface BufferIterable extends StatefulChunkedIterable<Buffer> {
-    }
-    /**
     Tokenizer#map() and Combiner#map() both return Token iterators.
@@ -169,4 +220,2 @@
     always a string).
-    
-    BufferIterable
     */
@@ -186,3 +235,3 @@ class Tokenizer<T> {
         */
-        map(iterable: BufferIterable, states?: string[]): TokenIterable<T>;
+        map(iterable: StringIterable, states?: string[]): TokenIterable<T>;
     }
@@ -189,0 +238,0 @@ interface CombinerAction<T, U> {

package.json

 {
   "name": "lexing",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Regex-based lexer",
@@ -5,0 +5,0 @@ "keywords": [

README.md

@@ -31,18 +31,23 @@ # lexing
 
-The tokenizer has one main function, `tokenizer.map(buffer_iterable)`, which returns a `TokenIterable`. `buffer_iterable` should implement the `BufferIterable` interface, i.e.:
+The tokenizer has one main function, `tokenizer.map(string_iterable)`, which returns a `TokenIterable`. `string_iterable` should implement the `StringIterable` interface, i.e.:
 
-    interface BufferIterable {
+    interface StringIterable {
       position: number;
       size: number;
-      next(length: number): Buffer;
-      peek(length: number): Buffer;
+      next(length: number): string;
+      peek(length: number): string;
       skip(length: number): number;
     }
 
-The following readers defined in `lexing` all return instances implementing the `BufferIterable` interface:
+The following readers defined in `lexing` all return instances implementing the `StringIterable` interface:
 
+* `new lexing.StringIterable(str)`
+* `lexing.StringIterable.fromBuffer(buffer, encoding)`
+* `new lexing.FileStringIterator(file_descriptor)`
+
+There are other Buffer-based readers as well:
+
 * `new lexing.BufferIterator(buffer)`
 * `lexing.BufferIterator.fromString(str, encoding)`
-* `new lexing.FileIterator(file_descriptor)`
-* `lexing.FileIterator.open(file_path)`
+* `new lexing.FileBufferIterator(file_descriptor)`
 
@@ -89,3 +94,3 @@ The `TokenIterable` instance returned by `tokenizer.map(...)` has one method: `next()`, which returns a non-null `Token`.
     var tokenizer = new lexing.Tokenizer(rules);
-    var input = lexing.BufferIterator.fromString("'It wasn't at all my fault', I cried.");
+    var input = new lexing.StringIterator("'It wasn't at all my fault', I cried.");
     var output = tokenizer.map(input);
@@ -92,0 +97,0 @@

test/simple.js

@@ -5,2 +5,3 @@ /// <reference path="../type_declarations/DefinitelyTyped/node/node.d.ts" />
 var lexing = require('../index');
+var Token = lexing.Token;
 function readToEOF(iterable) {
@@ -17,6 +18,2 @@ var items = [];
 }
-function Token(name, value) {
-    if (value === void 0) { value = null; }
-    return { name: name, value: value };
-}
 describe('tokenizer', function () {
@@ -29,3 +26,3 @@ var tokenizer = new lexing.Tokenizer([
     it('should lex simple string', function () {
-        var input_iterable = lexing.BufferIterator.fromString('This is a simple sentence');
+        var input_iterable = new lexing.StringIterator('This is a simple sentence');
         var output_iterable = tokenizer.map(input_iterable);
@@ -32,0 +29,0 @@ var expected_tokens = [

test/simple.ts

@@ -5,2 +5,3 @@ /// <reference path="../type_declarations/DefinitelyTyped/node/node.d.ts" />
 import lexing = require('../index');
+var Token = lexing.Token;
 
@@ -19,6 +20,2 @@ function readToEOF<T>(iterable: lexing.Iterable<lexing.Token<T>>): lexing.Token<T>[] {
 
-function Token<T>(name: string, value: T = null): lexing.Token<T> {
-  return {name: name, value: value};
-}
-
 describe('tokenizer', function() {
@@ -32,3 +29,3 @@ var tokenizer = new lexing.Tokenizer<any>([
   it('should lex simple string', function() {
-    var input_iterable = lexing.BufferIterator.fromString('This is a simple sentence');
+    var input_iterable = new lexing.StringIterator('This is a simple sentence');
     var output_iterable = tokenizer.map(input_iterable);
@@ -35,0 +32,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

122968

increased by9.32%

Lines of code

2743

increased by10.34%

Number of lines in readme file

135

increased by3.85%

No dependency changes