stream-json - npm Package Compare versions

Comparing version 1.6.1 to 1.7.0

jsonl/Parser.js

 'use strict';
 
-const {Transform} = require('stream');
+const Utf8Stream = require('../utils/Utf8Stream');
 
-class JsonlParser extends Transform {
+class JsonlParser extends Utf8Stream {
   static make(options) {
@@ -11,4 +11,4 @@ return new JsonlParser(options);
   constructor(options) {
-    super(Object.assign({}, options, {writableObjectMode: false, readableObjectMode: true}));
-    this._buffer = '';
+    super(Object.assign({}, options, {readableObjectMode: true}));
+    this._rest = '';
     this._counter = 0;
@@ -18,8 +18,8 @@ this._reviver = options && options.reviver;
 
-  _transform(chunk, _, callback) {
-    const lines = chunk.toString().split('\n');
-    this._buffer += lines[0];
+  _processBuffer(callback) {
+    const lines = this._buffer.split('\n');
+    this._rest += lines[0];
     if (lines.length > 1) {
-      this._buffer && this.push({key: this._counter++, value: JSON.parse(this._buffer, this._reviver)});
-      this._buffer = lines.pop();
+      this._rest && this.push({key: this._counter++, value: JSON.parse(this._rest, this._reviver)});
+      this._rest = lines.pop();
       for (let i = 1; i < lines.length; ++i) {
@@ -29,2 +29,3 @@ lines[i] && this.push({key: this._counter++, value: JSON.parse(lines[i], this._reviver)});
     }
+    this._buffer = '';
     callback(null);
@@ -34,7 +35,10 @@ }
   _flush(callback) {
-    if (this._buffer) {
-      this.push({key: this._counter++, value: JSON.parse(this._buffer, this._reviver)});
-      this._buffer = '';
-    }
-    callback(null);
+    super._flush(error => {
+      if (error) return callback(error);
+      if (this._rest) {
+        this.push({key: this._counter++, value: JSON.parse(this._rest, this._reviver)});
+        this._rest = '';
+      }
+      callback(null);
+    });
   }
@@ -41,0 +45,0 @@ }

package.json

 {
   "name": "stream-json",
-  "version": "1.6.1",
+  "version": "1.7.0",
   "description": "stream-json is the micro-library of Node.js stream components for creating custom JSON processing pipelines with a minimal memory footprint. It can parse JSON files far exceeding available memory streaming individual primitives using a SAX-inspired API. Includes utilities to stream JSON database dumps.",
@@ -5,0 +5,0 @@ "homepage": "http://github.com/uhop/stream-json",

Parser.js

 'use strict';
 
-const {Transform} = require('stream');
+const Utf8Stream = require('./utils/Utf8Stream');
 
@@ -49,3 +49,3 @@ const patterns = {
 
-class Parser extends Transform {
+class Parser extends Utf8Stream {
   static make(options) {
@@ -56,3 +56,3 @@ return new Parser(options);
   constructor(options) {
-    super(Object.assign({}, options, {writableObjectMode: false, readableObjectMode: true}));
+    super(Object.assign({}, options, {readableObjectMode: true}));
 
@@ -75,3 +75,2 @@ this._packKeys = this._packStrings = this._packNumbers = this._streamKeys = this._streamStrings = this._streamNumbers = true;
 
-    this._buffer = '';
     this._done = false;
@@ -85,29 +84,21 @@ this._expect = this._jsonStreaming ? 'done' : 'value';
 
-  _transform(chunk, _, callback) {
-    this._buffer += chunk.toString();
-    this._processInput(callback);
-  }
-
   _flush(callback) {
     this._done = true;
-    this._processInput(err => {
-      if (err) {
-        callback(err);
-      } else {
-        if (this._open_number) {
-          if (this._streamNumbers) {
-            this.push({name: 'endNumber'});
-          }
-          this._open_number = false;
-          if (this._packNumbers) {
-            this.push({name: 'numberValue', value: this._accumulator});
-            this._accumulator = '';
-          }
+    super._flush(error => {
+      if (error) return callback(error);
+      if (this._open_number) {
+        if (this._streamNumbers) {
+          this.push({name: 'endNumber'});
         }
-        callback(null);
+        this._open_number = false;
+        if (this._packNumbers) {
+          this.push({name: 'numberValue', value: this._accumulator});
+          this._accumulator = '';
+        }
       }
+      callback(null);
     });
   }
 
-  _processInput(callback) {
+  _processBuffer(callback) {
     let match,
@@ -123,3 +114,3 @@ value,
           if (!match) {
-            if (this._done || !index && this._buffer.length > MAX_PATTERN_SIZE) {
+            if (this._done || (!index && this._buffer.length > MAX_PATTERN_SIZE)) {
               if (index < this._buffer.length) return callback(new Error('Parser cannot parse input: expected a value'));
@@ -126,0 +117,0 @@ return callback(new Error('Parser has expected a value'));

README.md

@@ -42,2 +42,3 @@ # stream-json [![NPM version][npm-image]][npm-url]
   * [Verifier](https://github.com/uhop/stream-json/wiki/Verifier) reads a stream and verifies that it is a valid JSON.
+  * [Utf8Stream](https://github.com/uhop/stream-json/wiki/Utf8Stream) sanitizes multibyte `utf8` text input.
 * Special helpers:
@@ -116,2 +117,3 @@ * JSONL AKA [JSON Lines](http://jsonlines.org/):
 
+- 1.7.0 *added `utils/Utf8Stream` to sanitize `utf8` input, all parsers support it automatically. Thx [john30](https://github.com/john30) for the suggestion.*
 - 1.6.1 *the technical release, no need to upgrade.*
@@ -118,0 +120,0 @@ - 1.6.0 *added `jsonl/Parser` and `jsonl/Stringer`.*

utils/Verifier.js

 'use strict';
 
 const {Writable} = require('stream');
+const {StringDecoder} = require('string_decoder');
 
@@ -67,9 +68,27 @@ const patterns = {
   _write(chunk, encoding, callback) {
+    if (typeof chunk == 'string') {
+      this._write = this._writeString;
+    } else {
+      this._stringDecoder = new StringDecoder();
+      this._write = this._writeBuffer;
+    }
+    this._write(chunk, encoding, callback);
+  }
+
+  _writeBuffer(chunk, _, callback) {
+    this._buffer += this._stringDecoder.write(chunk);
+    this._processBuffer(callback);
+  }
+
+  _writeString(chunk, _, callback) {
     this._buffer += chunk.toString();
-    this._processInput(callback);
+    this._processBuffer(callback);
   }
 
   _final(callback) {
+    if (this._stringDecoder) {
+      this._buffer += this._stringDecoder.end();
+    }
     this._done = true;
-    this._processInput(callback);
+    this._processBuffer(callback);
   }
@@ -97,3 +116,3 @@
 
-  _processInput(callback) {
+  _processBuffer(callback) {
     let match,
@@ -100,0 +119,0 @@ value,

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

88967

increased by2.19%

Number of package files

25

increased by4.17%

Lines of code

2271

increased by2.57%

Number of lines in readme file

156

increased by1.3%

No dependency changes