avsc - npm Package Compare versions

Comparing version 0.1.1 to 0.1.2

lib/index.js

@@ -5,10 +5,38 @@ /* jshint node: true */
 
-var parse = require('./parse'),
+// TODO: Add append option to `createWriteStream`.
+
+var streams = require('./streams'),
+    types = require('./types'),
     fs = require('fs');
 
+
 /**
+ * Parse a schema and return the corresponding type.
+ *
+ * @param readerSchema {Object|String} Schema (type object or type name
+ * string).
+ * @param opts {Object} Parsing options. The following keys are currently
+ * supported:
+ *
+ * + `namespace` Optional parent namespace.
+ * + `registry` Optional registry of predefined type names.
+ * + `unwrapUnions` By default, Avro expects all unions to be wrapped inside an
+ *   object with a single key. Setting this to `true` will prevent this.
+ *   (Defaults to `false`.)
+ *
+ */
+function parse(schema, opts) {
+
+  if (schema instanceof types.Type) {
+    return schema;
+  }
+  return types.Type.fromSchema(schema, opts);
+
+}
+
+/**
  * Convenience function to parse an Avro schema file (`.avsc` typically).
  *
- * @param path {String} Path to file.
- * @param opts {Object} Parsing options. See `parse` for details.
+ * @param path {String} Path to Avro schema file (stored in JSON format).
+ * @param opts {Object} Parsing options. See `parse` above for details.
  *
@@ -18,10 +46,62 @@ */
 
-  return parse.parse(JSON.parse(fs.readFileSync(path)), opts);
+  return parse(JSON.parse(fs.readFileSync(path)), opts);
 
 }
 
+/**
+ * Convenience method to decode a file.
+ *
+ * @param path {String} Path to object container file or raw Avro file.
+ * @param opts {Object} Options.
+ *
+ * + `raw`
+ * + `writerType`, inferred if reading container file.
+ * + `readerType`, defaults to writer type.
+ * + `includeBuffer`, emit {obj, buf} instead of just decoded object. `buf`
+ *   will contain the original bytes representation (the writer type's
+ *   encoding).
+ *
+ */
+function createReadStream(path, opts) {
+
+  opts = opts || {};
+
+  if (opts.raw === undefined) {
+    opts.raw = !streams.Decoder.isContainerFile(path);
+  }
+
+  var Decoder = opts.raw ? streams.RawDecoder : streams.Decoder;
+  return fs.createReadStream(path).pipe(new Decoder(opts));
+
+}
+
+/**
+ * Convenience method to encode Avro objects.
+ *
+ * @param path {String} Path to object container file or raw Avro file.
+ * @param opts {Object} Options.
+ *
+ * + `raw`
+ * + `writerType`, inferred if piping records or from a decoder stream (from
+ *   the reader type).
+ *
+ */
+function createWriteStream(path, opts) {
+
+  opts = opts || {};
+
+  var Encoder = opts.raw ? streams.RawEncoder : streams.Encoder;
+  var fileOpts = {defaultEncoding: 'binary'};
+  return new Encoder(opts).pipe(fs.createWriteStream(path, fileOpts));
+
+}
+
+
 module.exports = {
-  parse: parse.parse,
+  parse: parse,
   parseFile: parseFile,
-  types: parse.types
+  createReadStream: createReadStream,
+  createWriteStream: createWriteStream,
+  streams: streams,
+  types: types
 };

lib/tap.js

 /* jshint node: true */
 
-// TODO: Implement skip methods.
 // TODO: Allow customizing block size when writing arrays and maps.
@@ -9,2 +8,3 @@ // TODO: Allow configuring when to write the size when writing arrays and maps.
 
+
 /**
@@ -36,8 +36,18 @@ * A tap is simply a buffer which remembers what has been already read.
 
+// Read, skip, write methods start here.
+//
+// These should fail silently when the buffer overflows. Note this is only
+// required to be true when the functions are decoding valid objects. For
+// example errors will still be thrown if a bad count is read, leading to a
+// negative position offset (which will typically cause a failure in
+// `readFixed`).
+
 Tap.prototype.readNull = function () { return null; };
 
-Tap.prototype.writeNull = function () {};
+Tap.prototype.skipNull = Tap.prototype.writeNull = function () {};
 
 Tap.prototype.readBoolean = function () { return !!this.buf[this.pos++]; };
 
+Tap.prototype.skipBoolean = function () { this.pos++; };
+
 Tap.prototype.writeBoolean = function (b) { this.buf[this.pos++] = !!b; };
@@ -47,18 +57,18 @@
 
-  var b = 0;
   var n = 0;
   var k = 0;
   var buf = this.buf;
-  var fk = 268435456; // 2 ** 28.
-  var f;
+  var b, h, f, fk;
 
   do {
     b = buf[this.pos++];
+    h = b & 0x80;
     n |= (b & 0x7f) << k;
     k += 7;
-  } while (b & 0x80 && k < 28);
+  } while (h && k < 28);
 
-  if (b & 0x80) {
+  if (h) {
     // Switch to float arithmetic, otherwise we might overflow.
     f = n;
+    fk = 268435456; // 2 ** 28.
     do {
@@ -70,8 +80,15 @@ b = buf[this.pos++];
     return (f % 2 ? -(f + 1) : f) / 2;
-  } else {
-    return (n >> 1) ^ -(n & 1);
   }
 
+  return (n >> 1) ^ -(n & 1);
+
 };
 
+Tap.prototype.skipInt = Tap.prototype.skipLong = function () {
+
+  var buf = this.buf;
+  while (buf[this.pos++] & 0x80) {}
+
+};
+
 Tap.prototype.writeInt = Tap.prototype.writeLong = function (n) {
@@ -113,2 +130,8 @@
 
+Tap.prototype.skipFloat = function () {
+
+  this.pos += 4;
+
+};
+
 Tap.prototype.writeFloat = function (f) {
@@ -138,2 +161,8 @@
 
+Tap.prototype.skipDouble = function () {
+
+  this.pos += 8;
+
+};
+
 Tap.prototype.writeDouble = function (d) {
@@ -154,8 +183,15 @@
   var len = this.readLong();
-  var s = this.buf.toString(undefined, this.pos, this.pos + len);
+  var pos = this.pos;
   this.pos += len;
-  return s;
+  return this.buf.toString(undefined, pos, pos + len);
 
 };
 
+Tap.prototype.skipString = function () {
+
+  var len = this.readLong();
+  this.pos += len;
+
+};
+
 Tap.prototype.writeString = function (s) {
@@ -187,2 +223,8 @@
 
+Tap.prototype.skipFixed = function (len) {
+
+  this.pos += len;
+
+};
+
 Tap.prototype.writeFixed = function (buf, len) {
@@ -206,2 +248,9 @@
 
+Tap.prototype.skipBytes = function () {
+
+  var len = this.readLong();
+  this.pos += len;
+
+};
+
 Tap.prototype.writeBytes = function (buf) {
@@ -218,9 +267,9 @@
   var arr = [];
-  var i, len;
-  while ((len = this.readLong())) {
-    if (len < 0) {
-      len = -len;
-      this.readLong(); // Skip size.
+  var n;
+  while ((n = this.readLong())) {
+    if (n < 0) {
+      n = -n;
+      this.skipLong(); // Skip size.
     }
-    for (i = 0; i < len; i++) {
+    while (n--) {
       arr.push(fn.call(this));
@@ -233,9 +282,25 @@ }
 
+Tap.prototype.skipArray = function (fn) {
+
+  var len, n;
+  while ((n = this.readLong())) {
+    if (n < 0) {
+      len = this.readLong();
+      this.pos += len;
+    } else {
+      while (n--) {
+        fn.call(this);
+      }
+    }
+  }
+
+};
+
 Tap.prototype.writeArray = function (arr, fn) {
 
-  var len = arr.length;
+  var n = arr.length;
   var i;
-  if (len) {
-    this.writeLong(len);
-    for (i = 0; i < len; i++) {
+  if (n) {
+    this.writeLong(n);
+    for (i = 0; i < n; i++) {
       fn.call(this, arr[i]);
@@ -251,9 +316,9 @@ }
   var obj = {};
-  var i, len;
-  while ((len = this.readLong())) {
-    if (len < 0) {
-      len = -len;
-      this.readLong(); // Skip size.
+  var n;
+  while ((n = this.readLong())) {
+    if (n < 0) {
+      n = -n;
+      this.skipLong(); // Skip size.
     }
-    for (i = 0; i < len; i++) {
+    while (n--) {
       var key = this.readString();
@@ -267,10 +332,26 @@ obj[key] = fn.call(this);
 
+Tap.prototype.skipMap = function (fn) {
+
+  var len, n;
+  while ((n = this.readLong())) {
+    if (n < 0) {
+      len = this.readLong();
+      this.pos += len;
+    }
+    while (n--) {
+      this.skipString();
+      fn.call(this);
+    }
+  }
+
+};
+
 Tap.prototype.writeMap = function (obj, fn) {
 
   var keys = Object.keys(obj);
-  var len = keys.length;
+  var n = keys.length;
   var i, key;
-  if (len) {
-    this.writeLong(len);
-    for (i = 0; i < len; i++) {
+  if (n) {
+    this.writeLong(n);
+    for (i = 0; i < n; i++) {
       key = keys[i];
@@ -277,0 +358,0 @@ this.writeString(key);

package.json

 {
   "name": "avsc",
-  "version": "0.1.1",
-  "description": "Avro serialization.",
+  "version": "0.1.2",
+  "description": "Pure JavaScript implementation of the Avro spec.",
+  "keywords": ["avro", "avsc", "schema", "encoding", "decoding"],
   "main": "./lib",
@@ -10,2 +11,3 @@ "repository": {
   },
+  "directories": ["lib"],
   "scripts": {
@@ -12,0 +14,0 @@ "test": "mocha --no-colors --ui tdd",

README.md

@@ -5,14 +5,16 @@ # Avsc
 
-*Under development.*
 
+## Features
 
-## Examples
++ Pure JavaScript implementation of the [Avro specification](https://avro.apache.org/docs/1.7.7/spec.html) (nearly complete, see below).
++ Fast.
++ No dependencies.
 
-### Fragments
 
+## Example
+
 ```javascript
 var avsc = require('avsc');
 
-// Parsing a schema returns a corresponding Avro type.
-var type = avsc.parse({
+var Person = avsc.parse({
   type: 'record',
@@ -24,45 +26,36 @@ name: 'Person',
   ]
-});
+}).getRecordConstructor();
 
-// For record types, constructors are programmatically generated!
-var Person = recordType.getRecordConstructor();
-
-// This constructor can be used to instantiate records directly.
 var person = new Person('Ann', 25);
-person.name; // == 'Ann' (The record's fields get set appropriately.)
+person.name; // == 'Ann'
 person.age; // == 25
 
-// Or to decode them from an existing buffer.
-Person.decode(buf);
+person.$encode(); // Buffer containing this record's Avro encoding.
+```
 
-// Or even to create random instances.
-var fakePerson = Person.random();
 
-// Records instance also have a few useful properties and methods.
-person.$type; // == type
-person.$encode(); // Returns a buffer with the record's Avro encoding.
-person.$isValid(); // Check that all fields satisfy the schema.
+## Installation
+
+```bash
+$ npm install avsc
 ```
 
 
-### Container files
+## Documentation
 
-(API still undergoing changes.)
+https://github.com/mtth/avsc/wiki/API
 
-```javascript
-var avsc = require('avsc'),
-    fs = require('fs');
 
-// Read an Avro container file.
-fs.createReadStream('input.avro')
-  .pipe(new avsc.Decoder()) // The decoder will infer the type.
-  .on('data', function (record) { console.log(record); });
+## Status
 
-// Writable record stream.
-var stream = new avsc.Encoder();
-stream.pipe(fs.createWriteStream('output.avro', {defaultEncoding: 'binary'}));
-```
+What's already there:
 
-## Documentation
++ Parsing schemas, including recursive ones.
++ Encoding, decoding, validating, and generating data.
++ Resolving schemas (a.k.a. "reader's schemas").
++ Reading container files.
 
-API: https://github.com/mtth/avsc/wiki/API
+Coming up:
+
++ Writing container files.
++ Comparing data.

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Uses eval

Supply chain risk

Package uses dynamic code execution (e.g., eval()), which is a dangerous practice. This can prevent the code from running in certain environments and increases the risk that the code may contain exploits or malicious behavior.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Uses eval

Supply chain risk

Package uses dynamic code execution (e.g., eval()), which is a dangerous practice. This can prevent the code from running in certain environments and increases the risk that the code may contain exploits or malicious behavior.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

66044

increased by25.58%

Number of package files

16

increased by77.78%

Lines of code

2225

increased by26.85%

Worsened metrics

Number of lines in readme file

60

decreased by-10.45%

Number of low supply chain risk alerts

3

increased by50%

Number of medium supply chain risk alerts

8

increased by60%

No dependency changes