stream-json - npm Package Compare versions

Comparing version 1.4.1 to 1.5.0

Assembler.js

@@ -6,3 +6,3 @@ 'use strict';
 const startObject = Ctr =>
-  function() {
+  function () {
     if (this.done) {
@@ -18,7 +18,7 @@ this.done = false;
 class Assembler extends EventEmitter {
-  static connectTo(stream) {
-    return new Assembler().connectTo(stream);
+  static connectTo(stream, options) {
+    return new Assembler(options).connectTo(stream);
   }
 
-  constructor() {
+  constructor(options) {
     super();
@@ -28,2 +28,8 @@ this.stack = [];
     this.done = true;
+    if (options) {
+      this.reviver = typeof options.reviver == 'function' && options.reviver;
+      if (this.reviver) {
+        this.stringValue = this._saveValue = this._saveValueWithReviver;
+      }
+    }
   }
@@ -122,2 +128,21 @@
   }
+  _saveValueWithReviver(value) {
+    if (this.done) {
+      this.current = this.reviver('', value);
+    } else {
+      if (this.current instanceof Array) {
+        value = this.reviver('' + this.current.length, value);
+        this.current.push(value);
+        if (value === undefined) {
+          delete this.current[this.current.length - 1];
+        }
+      } else {
+        value = this.reviver(this.key, value);
+        if (value !== undefined) {
+          this.current[this.key] = value;
+        }
+        this.key = null;
+      }
+    }
+  }
 }
@@ -124,0 +149,0 @@

Disassembler.js

@@ -28,2 +28,7 @@ 'use strict';
       'streamNumbers' in options && (this._streamNumbers = options.streamNumbers);
+      if (typeof options.replacer == 'function') {
+        this._replacer = options.replacer;
+      } else if (Array.isArray(options.replacer)) {
+        this._replacerDict = options.replacer.reduce((acc, k) => (acc[k] = 1, acc), {});
+      }
     }
@@ -35,27 +40,68 @@ !this._packKeys && (this._streamKeys = true);
 
-  _transform(chunk, encoding, callback) {
-    const stack = [chunk];
+  _transform(chunk, _, callback) {
+    const stack = [],
+      isArray = [];
+    if (chunk && typeof chunk == 'object' && typeof chunk.toJSON == 'function') {
+      chunk = chunk.toJSON('');
+    }
+    if (this._replacer) {
+      chunk = this._replacer('', chunk);
+    }
+    stack.push(chunk);
     while (stack.length) {
       const top = stack.pop();
-      if (top && typeof top == 'object') {
-        if (top instanceof Emit) {
-          if (top.tokenName === 'keyValue') {
-            const key = stack.pop();
-            if (this._streamKeys) {
-              this.push({name: 'startKey'});
-              this.push({name: 'stringChunk', value: key});
-              this.push({name: 'endKey'});
+      main: switch (typeof top) {
+        case 'object':
+          if (top instanceof Emit) {
+            switch (top.tokenName) {
+              case 'keyValue':
+                const key = stack.pop();
+                if (this._streamKeys) {
+                  this.push({name: 'startKey'});
+                  this.push({name: 'stringChunk', value: key});
+                  this.push({name: 'endKey'});
+                }
+                this._packKeys && this.push({name: 'keyValue', value: key});
+                break main;
+              case 'startArray':
+                isArray.push(true);
+                break;
+              case 'startObject':
+                isArray.push(false);
+                break;
+              case 'endArray':
+              case 'endObject':
+                isArray.pop();
+                break;
             }
-            this._packKeys && this.push({name: 'keyValue', value: key});
-          } else {
             this.push({name: top.tokenName});
+            break;
           }
-          continue;
-        } else if (Array.isArray(top)) {
-          stack.push(new Emit('endArray'));
-          for (let i = top.length - 1; i >= 0; --i) {
-            stack.push(top[i]);
+          if (Array.isArray(top)) {
+            stack.push(new Emit('endArray'));
+            for (let i = top.length - 1; i >= 0; --i) {
+              let value = top[i];
+              if (value && typeof value == 'object' && typeof value.toJSON == 'function') {
+                value = value.toJSON('' + i);
+              }
+              if (this._replacer) {
+                value = this._replacer('' + i, value);
+              }
+              switch (typeof value) {
+                case 'function':
+                case 'symbol':
+                case 'undefined':
+                  value = null;
+                  break;
+              }
+              stack.push(value);
+            }
+            stack.push(new Emit('startArray'));
+            break;
           }
-          stack.push(new Emit('startArray'));
-        } else { // all other objects are just objects
+          if (top === null) {
+            this.push({name: 'nullValue', value: null});
+            break;
+          }
+          // all other objects are just objects
           const keys = Object.keys(top);
@@ -65,8 +111,21 @@ stack.push(new Emit('endObject'));
             const key = keys[i];
-            stack.push(top[key], key, new Emit('keyValue'));
+            if (this._replacerDict && this._replacerDict[key] !== 1) continue;
+            let value = top[key];
+            if (value && typeof value == 'object' && typeof value.toJSON == 'function') {
+              value = value.toJSON(key);
+            }
+            if (this._replacer) {
+              value = this._replacer(key, value);
+            }
+            switch (typeof value) {
+              case 'function':
+              case 'symbol':
+              case 'undefined':
+                continue;
+            }
+            stack.push(value, key, new Emit('keyValue'));
           }
           stack.push(new Emit('startObject'));
-        }
-      } else {
-        if (typeof top == 'string') {
+          break;
+        case 'string':
           if (this._streamStrings) {
@@ -78,4 +137,9 @@ this.push({name: 'startString'});
           this._packStrings && this.push({name: 'stringValue', value: top});
-        } else if (typeof top == 'number') {
+          break;
+        case 'number':
           const number = top.toString();
+          if (isNaN(number) || !isFinite(number)) {
+            this.push({name: 'nullValue', value: null});
+            break;
+          }
           if (this._streamNumbers) {
@@ -87,9 +151,17 @@ this.push({name: 'startNumber'});
           this._packNumbers && this.push({name: 'numberValue', value: number});
-        } else if (top === true) {
-          this.push({name: 'trueValue', value: true});
-        } else if (top === false) {
-          this.push({name: 'falseValue', value: false});
-        } else { // everything else is null
-          this.push({name: 'nullValue', value: null});
-        }
+          break;
+        case 'function':
+        case 'symbol':
+        case 'undefined':
+          if (isArray.length && isArray[isArray.length - 1]) {
+            // replace with null inside arrays
+            this.push({name: 'nullValue', value: null});
+          }
+          break;
+        case 'boolean':
+          this.push(top ? {name: 'trueValue', value: true} : {name: 'falseValue', value: false});
+          break;
+        default:
+          // skip everything else
+          break;
       }
@@ -96,0 +168,0 @@ }

package.json

 {
   "name": "stream-json",
-  "version": "1.4.1",
+  "version": "1.5.0",
   "description": "stream-json is the micro-library of Node.js stream components for creating custom JSON processing pipelines with a minimal memory footprint. It can parse JSON files far exceeding available memory streaming individual primitives using a SAX-inspired API. Includes utilities to stream JSON database dumps.",
@@ -18,3 +18,4 @@ "homepage": "http://github.com/uhop/stream-json",
   "scripts": {
-    "test": "node tests/tests.js"
+    "test": "node tests/tests.js",
+    "debug": "node --inspect-brk tests/tests.js"
   },
@@ -21,0 +22,0 @@ "github": "http://github.com/uhop/stream-json",

README.md

@@ -113,7 +113,8 @@ # stream-json
 
-- 1.4.1 *Bugfix: `Stringer` with `makeArray` should produce empty array if no input.*
+- 1.5.0 *`Disassembler` and streamers now follow `JSON.stringify()` and `JSON.parse()` protocols respectively including `replacer` and `reviver`.*
+- 1.4.1 *bugfix: `Stringer` with `makeArray` should produce empty array if no input.*
 - 1.4.0 *added `makeArray` functionality to `Stringer`. Thx all who asked for it!*
-- 1.3.3 *Bugfix: very large/infinite streams with garbage didn't fail. Thx [Arne Marschall](https://github.com/Disco1267)!*
-- 1.3.2 *Bugfix: filters could fail with packed-only token streams. Thx [Trey Brisbane](https://github.com/treybrisbane)!*
-- 1.3.1 *Bugfix: reverted the last bugfix in `Verifier`, a bugfix in tests, thx [Guillermo Ares](https://github.com/guillermoares).*
+- 1.3.3 *bugfix: very large/infinite streams with garbage didn't fail. Thx [Arne Marschall](https://github.com/Disco1267)!*
+- 1.3.2 *bugfix: filters could fail with packed-only token streams. Thx [Trey Brisbane](https://github.com/treybrisbane)!*
+- 1.3.1 *bugfix: reverted the last bugfix in `Verifier`, a bugfix in tests, thx [Guillermo Ares](https://github.com/guillermoares).*
 - 1.3.0 *added `Batch`, a bugfix in `Verifier`.*
@@ -120,0 +121,0 @@ - 1.2.1 *the technical release.*

streamers/StreamBase.js

@@ -27,9 +27,11 @@ 'use strict';
     super(Object.assign({}, options, {writableObjectMode: true, readableObjectMode: true}));
-    this.objectFilter = options && options.objectFilter;
-    this.includeUndecided = options && options.includeUndecided;
+    if (options) {
+      this.objectFilter = options.objectFilter;
+      this.includeUndecided = options.includeUndecided;
+    }
     if (typeof this.objectFilter != 'function') {
       this._filter = this._transform;
     }
-    this._assembler = new Assembler();
     this._transform = this._wait || this._filter;
+    this._assembler = new Assembler(options && {reviver: options.reviver});
   }
@@ -36,0 +38,0 @@

tests/test_array.js

@@ -146,3 +146,25 @@ 'use strict';
     new ReadString(JSON.stringify(input)).pipe(stream.input);
+  },
+  function test_array_with_replacer_and_reviver(t) {
+    const async = t.startAsync('test_array_with_replacer_and_reviver');
+
+    const reviver = (k, v) => {
+      if (/Date$/.test(k) && typeof v == 'string') return new Date(Date.parse(v));
+      return v;
+    };
+
+    const source = [{createdDate: new Date(), updatedDate: new Date(), user: 'bob', life: 42}],
+      json = JSON.stringify(source);
+
+    const stream = StreamArray.withParser({reviver}),
+      result = [];
+
+    stream.output.on('data', object => result.push(object.value));
+    stream.output.on('end', () => {
+      eval(t.TEST('t.unify(result, source)'));
+      async.done();
+    });
+
+    new ReadString(json).pipe(stream.input);
   }
 ]);

tests/test_assembler.js

@@ -87,3 +87,59 @@ 'use strict';
     new ReadString(pattern.map(value => JSON.stringify(value)).join(' ')).pipe(parser);
+  },
+  function test_assembler_reviver(t) {
+    const async = t.startAsync('test_assembler_reviver');
+
+    const reviver = (k, v) => {
+      if (k === 'b' || k === '1') return;
+      return v;
+    };
+
+    const source = [{a: 1, b: 2, c: 3}, {a: 1, b: 2, c: 3}, {a: 1, b: 2, c: 3}],
+      json = JSON.stringify(source),
+      shouldBe = JSON.parse(json, reviver);
+
+    const parser = makeParser({streamValues: false}),
+      assembler = Assembler.connectTo(parser, {reviver});
+
+    parser.on('end', () => {
+      eval(t.TEST('t.unify(assembler.current, shouldBe)'));
+      async.done();
+    });
+
+    new ReadString(json).pipe(parser);
+  },
+  function test_assembler_no_streaming_with_reviver(t) {
+    const async = t.startAsync('test_assembler_no_streaming_with_reviver');
+
+    const reviver = (k, v) => {
+      if (k.charAt(0) === '@' || /^data/.test(k)) return;
+      return v;
+    };
+
+    let object = null;
+    const parser = makeParser({streamValues: false}),
+      assembler = Assembler.connectTo(parser, {reviver});
+
+    parser.on('end', () => {
+      eval(t.TEST('t.unify(assembler.current, object)'));
+      async.done();
+    });
+
+    fs.readFile(path.resolve(__dirname, './sample.json.gz'), (err, data) => {
+      if (err) {
+        throw err;
+      }
+      zlib.gunzip(data, (err, data) => {
+        if (err) {
+          throw err;
+        }
+
+        object = JSON.parse(data.toString(), reviver);
+
+        fs.createReadStream(path.resolve(__dirname, './sample.json.gz'))
+          .pipe(zlib.createGunzip())
+          .pipe(parser);
+      });
+    });
   }
 ]);

tests/test_disassembler.js

@@ -14,4 +14,14 @@ 'use strict';
 
+const sanitize = x => {
+  x = JSON.stringify(x);
+  return typeof x == 'string' ? JSON.parse(x) : x;
+};
+
+const sanitizeWithReplacer = replacer => x => {
+  x = JSON.stringify(x, replacer);
+  return typeof x == 'string' ? JSON.parse(x) : x;
+};
+
 unit.add(module, [
-  function test_assembler(t) {
+  function test_disassembler(t) {
     const async = t.startAsync('test_disassembler');
@@ -22,10 +32,3 @@
 
-    const pipeline = chain([
-      new ReadString(JSON.stringify(input)),
-      parser(),
-      streamArray(),
-      disassembler(),
-      pick({filter: 'value'}),
-      streamValues()
-    ]);
+    const pipeline = chain([new ReadString(JSON.stringify(input)), parser(), streamArray(), disassembler(), pick({filter: 'value'}), streamValues()]);
 
@@ -37,3 +40,272 @@ pipeline.on('data', item => result.push(item.value));
     });
+  },
+  function test_disassembler_bad_top_level(t) {
+    const async = t.startAsync('test_disassembler_bad_top_level');
+
+    const input = [1, () => {}, 2, undefined, 3, Symbol(), 4],
+      result = [];
+
+    const pipeline = chain([disassembler(), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, [1, 2, 3, 4])'));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_bad_values_in_object(t) {
+    const async = t.startAsync('test_disassembler_bad_values_in_object');
+
+    const input = [{a: 1, b: () => {}, c: 2, d: undefined, e: 3, f: Symbol(), g: 4}],
+      result = [];
+
+    const pipeline = chain([disassembler(), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, [{a: 1, c: 2, e: 3, g: 4}])'));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_bad_values_in_array(t) {
+    const async = t.startAsync('test_disassembler_bad_values_in_array');
+
+    const input = [[1, () => {}, 2, undefined, 3, Symbol(), 4]],
+      result = [];
+
+    const pipeline = chain([disassembler(), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, [[1, null, 2, null, 3, null, 4]])'));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_dates(t) {
+    const async = t.startAsync('test_disassembler_dates');
+
+    const date = new Date(),
+      input = [1, date, 2],
+      result = [];
+
+    const pipeline = chain([disassembler(), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, [1, date.toJSON(""), 2])'));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_chained_toJSON(t) {
+    const async = t.startAsync('test_disassembler_chained_toJSON');
+
+    const x = {a: 1};
+
+    const y = {
+      b: 2,
+      toJSON() {
+        return x;
+      }
+    };
+
+    const z = {
+      c: 3,
+      toJSON() {
+        return y;
+      }
+    };
+
+    const input = [x, y, z],
+      shouldBe = input.map(sanitize),
+      result = [];
+
+    const pipeline = chain([disassembler(), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, shouldBe)'));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_custom_toJSON(t) {
+    const async = t.startAsync('test_disassembler_custom_toJSON');
+
+    const x = {
+      a: 1,
+      toJSON(k) {
+        if (k !== '1' && k !== 'b') return 5;
+        // otherwise skip by returning undefined
+      }
+    };
+
+    const input = [x, x, {a: x, b: x}, [x, x]],
+      shouldBe = input.map(sanitize),
+      result = [];
+
+    const pipeline = chain([disassembler(), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, shouldBe)'));
+      async.done();
+    });
+    pipeline.on('error', error => {
+      console.log(error);
+      eval(t.TEST("!'We shouldn't be here.'"));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_custom_toJSON_filter_top_level(t) {
+    const async = t.startAsync('test_disassembler_custom_toJSON_filter_top_level');
+
+    const x = {
+      a: 1,
+      toJSON(k) {
+        if (k !== '') return 5;
+        // otherwise skip by returning undefined
+      }
+    };
+
+    const input = [x, x, {a: x, b: x}, [x, x]],
+      shouldBe = input.map(sanitize).filter(item => item !== undefined),
+      result = [];
+
+    const pipeline = chain([disassembler(), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, shouldBe)'));
+      async.done();
+    });
+    pipeline.on('error', error => {
+      console.log(error);
+      eval(t.TEST("!'We shouldn't be here.'"));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_custom_replacer(t) {
+    const async = t.startAsync('test_disassembler_custom_replacer');
+
+    const replacer = (k, v) => {
+      if (k === '1' || k === 'b') return 5;
+      if (k === '0' || k === 'c') return;
+      return v;
+    };
+
+    const input = [1, 2, {a: 3, b: 4, c: 7}, [5, 6]],
+      shouldBe = input.map(sanitizeWithReplacer(replacer)),
+      result = [];
+
+    const pipeline = chain([disassembler({replacer}), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, shouldBe)'));
+      async.done();
+    });
+    pipeline.on('error', error => {
+      console.log(error);
+      eval(t.TEST("!'We shouldn't be here.'"));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_custom_replacer_filter_top_level(t) {
+    const async = t.startAsync('test_disassembler_custom_replacer_filter_top_level');
+
+    const replacer = (k, v) => {
+      if (k === '' && typeof v == 'number') return;
+      if (k === '1' || k === 'b') return 5;
+      if (k === '0' || k === 'c') return;
+      return v;
+    };
+
+    const input = [1, 2, {a: 3, b: 4, c: 7}, [5, 6]],
+      shouldBe = input.map(sanitizeWithReplacer(replacer)).filter(item => item !== undefined),
+      result = [];
+
+    const pipeline = chain([disassembler({replacer}), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, shouldBe)'));
+      async.done();
+    });
+    pipeline.on('error', error => {
+      console.log(error);
+      eval(t.TEST("!'We shouldn't be here.'"));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
+  },
+  function test_disassembler_custom_replacer_array(t) {
+    const async = t.startAsync('test_disassembler_custom_replacer_array');
+
+    const replacer = ['a', 'b'];
+
+    const input = [1, 2, {a: 3, b: {a: 8, b: 9, c: 10}, c: 7}, [5, 6]],
+      shouldBe = input.map(sanitizeWithReplacer(replacer)),
+      result = [];
+
+    const pipeline = chain([disassembler({replacer}), streamValues()]);
+
+    pipeline.on('data', item => result.push(item.value));
+    pipeline.on('end', () => {
+      eval(t.TEST('t.unify(result, shouldBe)'));
+      async.done();
+    });
+    pipeline.on('error', error => {
+      console.log(error);
+      eval(t.TEST("!'We shouldn't be here.'"));
+      async.done();
+    });
+
+    for (const item of input) {
+      pipeline.write(item);
+    }
+    pipeline.end();
   }
 ]);

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

226858

increased by6.12%

Lines of code

4741

increased by8.79%

Number of lines in readme file

161

increased by0.63%

Worsened metrics

Number of medium supply chain risk alerts

262

increased by7.38%

No dependency changes