pbf - npm Package Compare versions

Comparing version 1.1.4 to 1.2.0

buffer.js

@@ -19,5 +19,15 @@ 'use strict';
 
-    for (var i in BufferMethods) {
-        buf[i] = BufferMethods[i];
-    }
+    buf.readUInt32LE = BufferMethods.readUInt32LE;
+    buf.writeUInt32LE = BufferMethods.writeUInt32LE;
+    buf.readInt32LE = BufferMethods.readInt32LE;
+    buf.writeInt32LE = BufferMethods.writeInt32LE;
+    buf.readFloatLE = BufferMethods.readFloatLE;
+    buf.writeFloatLE = BufferMethods.writeFloatLE;
+    buf.readDoubleLE = BufferMethods.readDoubleLE;
+    buf.writeDoubleLE = BufferMethods.writeDoubleLE;
+    buf.toString = BufferMethods.toString;
+    buf.write = BufferMethods.write;
+    buf.slice = BufferMethods.slice;
+    buf.copy = BufferMethods.copy;
+
     buf._isBuffer = true;
@@ -27,2 +37,4 @@ return buf;
 
+var lastStr, lastStrEncoded;
+
 var BufferMethods = {
@@ -57,4 +69,2 @@ readUInt32LE: function(pos) {
     toString: function(encoding, start, end) {
-        if (typeof TextDecoder !== 'undefined') return new TextDecoder('utf8').decode(this.subarray(start, end));
-
         var str = '',
@@ -69,14 +79,16 @@ tmp = '';
             if (ch <= 0x7F) {
-                str += decodeUtf8Str(tmp) + String.fromCharCode(ch)
+                str += decodeURIComponent(tmp) + String.fromCharCode(ch);
                 tmp = '';
             } else {
-                tmp += '%' + ch.toString(16)
+                tmp += '%' + ch.toString(16);
             }
         }
 
-        return str + decodeUtf8Str(tmp);
+        str += decodeURIComponent(tmp);
+
+        return str;
     },
 
     write: function(str, pos) {
-        var bytes = Buffer._lastStr === str ? Buffer._lastEncoded : encodeString(str);
+        var bytes = str === lastStr ? lastStrEncoded : encodeString(str);
         for (var i = 0; i < bytes.length; i++) {
@@ -102,5 +114,5 @@ this[pos + i] = bytes[i];
 Buffer.byteLength = function(str) {
-    Buffer._lastStr = str;
-    var bytes = Buffer._lastEncoded = encodeString(str);
-    return bytes.length;
+    lastStr = str;
+    lastStrEncoded = encodeString(str);
+    return lastStrEncoded.length;
 };
@@ -113,4 +125,2 @@
 function encodeString(str) {
-    if (typeof TextEncoder !== 'undefined') return new TextEncoder('utf8').encode(str);
-
     var length = str.length,
@@ -147,11 +157,7 @@ bytes = [];
 
-        if (c < 0x80) {
-            bytes.push(c);
-
-        } else {
-            if (c < 0x800) bytes.push(c >> 0x6 | 0xC0);
-            else if (c < 0x10000) bytes.push(c >> 0xC | 0xE0, c >> 0x6 & 0x3F | 0x80);
-            else bytes.push(c >> 0x12 | 0xF0, c >> 0xC & 0x3F | 0x80, c >> 0x6 & 0x3F | 0x80);
-
-            bytes.push(c & 0x3F | 0x80);
+        if (c < 0x80) bytes.push(c);
+        else {
+            if (c < 0x800) bytes.push(c >> 0x6 | 0xC0, c & 0x3F | 0x80);
+            else if (c < 0x10000) bytes.push(c >> 0xC | 0xE0, c >> 0x6 & 0x3F | 0x80, c & 0x3F | 0x80);
+            else bytes.push(c >> 0x12 | 0xF0, c >> 0xC & 0x3F | 0x80, c >> 0x6 & 0x3F | 0x80, c & 0x3F | 0x80);
         }
@@ -161,9 +167,1 @@ }
 }
-
-function decodeUtf8Str(str) {
-    try {
-        return decodeURIComponent(str);
-    } catch (err) {
-        return String.fromCharCode(0xFFFD); // UTF 8 invalid char
-    }
-}

index.js

 'use strict';
 
-module.exports = Protobuf;
+module.exports = Pbf;
 
 var Buffer = global.Buffer || require('./buffer');
 
-function Protobuf(buf) {
+function Pbf(buf) {
     this.buf = !Buffer.isBuffer(buf) ? new Buffer(buf || 0) : buf;
@@ -13,6 +13,6 @@ this.pos = 0;
 
-Protobuf.Varint  = 0; // varint: int32, int64, uint32, uint64, sint32, sint64, bool, enum
-Protobuf.Fixed64 = 1; // 64-bit: double, fixed64, sfixed64
-Protobuf.Bytes   = 2; // length-delimited: string, bytes, embedded messages, packed repeated fields
-Protobuf.Fixed32 = 5; // 32-bit: float, fixed32, sfixed32
+Pbf.Varint  = 0; // varint: int32, int64, uint32, uint64, sint32, sint64, bool, enum
+Pbf.Fixed64 = 1; // 64-bit: double, fixed64, sfixed64
+Pbf.Bytes   = 2; // length-delimited: string, bytes, embedded messages, packed repeated fields
+Pbf.Fixed32 = 5; // 32-bit: float, fixed32, sfixed32
 
@@ -22,3 +22,3 @@ var SHIFT_LEFT_32 = (1 << 16) * (1 << 16),
 
-Protobuf.prototype = {
+Pbf.prototype = {
 
@@ -132,22 +132,56 @@ destroy: function() {
 
-    readPacked: function(type) {
-        var end = this.readVarint() + this.pos,
-            arr = [];
-        if (type === 'Varint')        while (this.pos < end) arr.push(this.readVarint());
-        else if (type === 'SVarint')  while (this.pos < end) arr.push(this.readSVarint());
-        else if (type === 'Float')    while (this.pos < end) arr.push(this.readFloat());
-        else if (type === 'Double')   while (this.pos < end) arr.push(this.readDouble());
-        else if (type === 'Fixed32')  while (this.pos < end) arr.push(this.readFixed32());
-        else if (type === 'SFixed32') while (this.pos < end) arr.push(this.readSFixed32());
-        else if (type === 'Fixed64')  while (this.pos < end) arr.push(this.readFixed64());
-        else if (type === 'SFixed64') while (this.pos < end) arr.push(this.readSFixed64());
+    // verbose for performance reasons; doesn't affect gzipped size
+
+    readPackedVarint: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readVarint());
         return arr;
     },
+    readPackedSVarint: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readSVarint());
+        return arr;
+    },
+    readPackedBoolean: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readBoolean());
+        return arr;
+    },
+    readPackedFloat: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readFloat());
+        return arr;
+    },
+    readPackedDouble: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readDouble());
+        return arr;
+    },
+    readPackedFixed32: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readFixed32());
+        return arr;
+    },
+    readPackedSFixed32: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readSFixed32());
+        return arr;
+    },
+    readPackedFixed64: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readFixed64());
+        return arr;
+    },
+    readPackedSFixed64: function() {
+        var end = this.readVarint() + this.pos, arr = [];
+        while (this.pos < end) arr.push(this.readSFixed64());
+        return arr;
+    },
 
     skip: function(val) {
         var type = val & 0x7;
-        if (type === Protobuf.Varint) while (this.buf[this.pos++] > 0x7f);
-        else if (type === Protobuf.Bytes) this.pos = this.readVarint() + this.pos;
-        else if (type === Protobuf.Fixed32) this.pos += 4;
-        else if (type === Protobuf.Fixed64) this.pos += 8;
+        if (type === Pbf.Varint) while (this.buf[this.pos++] > 0x7f);
+        else if (type === Pbf.Bytes) this.pos = this.readVarint() + this.pos;
+        else if (type === Pbf.Fixed32) this.pos += 4;
+        else if (type === Pbf.Fixed64) this.pos += 8;
         else throw new Error('Unimplemented type: ' + type);
@@ -163,3 +197,3 @@ },
     realloc: function(min) {
-        var length = this.length || 1;
+        var length = this.length || 16;
 
@@ -182,21 +216,2 @@ while (length < this.pos + min) length *= 2;
 
-    writePacked: function(tag, type, items) {
-        if (!items.length) return;
-
-        var message = new Protobuf(),
-            len = items.length,
-            i = 0;
-
-        if (type === 'Varint')        for (; i < len; i++) message.writeVarint(items[i]);
-        else if (type === 'SVarint')  for (; i < len; i++) message.writeSVarint(items[i]);
-        else if (type === 'Float')    for (; i < len; i++) message.writeFloat(items[i]);
-        else if (type === 'Double')   for (; i < len; i++) message.writeDouble(items[i]);
-        else if (type === 'Fixed32')  for (; i < len; i++) message.writeFixed32(items[i]);
-        else if (type === 'SFixed32') for (; i < len; i++) message.writeSFixed32(items[i]);
-        else if (type === 'Fixed64')  for (; i < len; i++) message.writeFixed64(items[i]);
-        else if (type === 'SFixed64') for (; i < len; i++) message.writeSFixed64(items[i]);
-
-        this.writeMessage(tag, message);
-    },
-
     writeFixed32: function(val) {
@@ -234,17 +249,21 @@ this.realloc(4);
             this.buf[this.pos++] = val;
+
         } else if (val <= 0x3fff) {
             this.realloc(2);
-            this.buf[this.pos++] = 0x80 | ((val >>> 0) & 0x7f);
-            this.buf[this.pos++] = 0x00 | ((val >>> 7) & 0x7f);
+            this.buf[this.pos++] = ((val >>> 0) & 0x7f) | 0x80;
+            this.buf[this.pos++] =  (val >>> 7) & 0x7f;
+
         } else if (val <= 0x1fffff) {
             this.realloc(3);
-            this.buf[this.pos++] = 0x80 | ((val >>> 0) & 0x7f);
-            this.buf[this.pos++] = 0x80 | ((val >>> 7) & 0x7f);
-            this.buf[this.pos++] = 0x00 | ((val >>> 14) & 0x7f);
+            this.buf[this.pos++] = ((val >>>  0) & 0x7f) | 0x80;
+            this.buf[this.pos++] = ((val >>>  7) & 0x7f) | 0x80;
+            this.buf[this.pos++] =  (val >>> 14) & 0x7f;
+
         } else if (val <= 0xfffffff) {
             this.realloc(4);
-            this.buf[this.pos++] = 0x80 | ((val >>> 0) & 0x7f);
-            this.buf[this.pos++] = 0x80 | ((val >>> 7) & 0x7f);
-            this.buf[this.pos++] = 0x80 | ((val >>> 14) & 0x7f);
-            this.buf[this.pos++] = 0x00 | ((val >>> 21) & 0x7f);
+            this.buf[this.pos++] = ((val >>>  0) & 0x7f) | 0x80;
+            this.buf[this.pos++] = ((val >>>  7) & 0x7f) | 0x80;
+            this.buf[this.pos++] = ((val >>> 14) & 0x7f) | 0x80;
+            this.buf[this.pos++] =  (val >>> 21) & 0x7f;
+
         } else {
@@ -296,63 +315,83 @@ var pos = this.pos;
         this.realloc(len);
-        for (var i = 0; i < len; i++) {
-            this.buf[this.pos + i] = buffer[i];
+        for (var i = 0; i < len; i++) this.buf[this.pos++] = buffer[i];
+    },
+
+    writeMessage: function(tag, fn, obj) {
+        this.writeTag(tag, Pbf.Bytes);
+
+        this.pos++; // reserve 1 byte for short message length
+
+        // write the message directly to the buffer and see how much was written
+        var startPos = this.pos;
+        fn(obj, this);
+        var len = this.pos - startPos;
+
+        var varintLen =
+            len <= 0x7f ? 1 :
+            len <= 0x3fff ? 2 :
+            len <= 0x1fffff ? 3 :
+            len <= 0xfffffff ? 4 : Math.ceil(Math.log(len) / (Math.LN2 * 7));
+
+        // if 1 byte isn't enough for encoding message length, shift the data to the right
+        if (varintLen > 1) {
+            this.realloc(varintLen - 1);
+            for (var i = this.pos - 1; i >= startPos; i--) this.buf[i + varintLen - 1] = this.buf[i];
         }
+
+        // finally, write the message length in the reserved place and restore the position
+        this.pos = startPos - 1;
+        this.writeVarint(len);
         this.pos += len;
     },
 
-    writeMessage: function(tag, protobuf) {
-        this.writeTag(tag, Protobuf.Bytes);
-        this.writeBytes(protobuf.finish());
-    },
+    writePackedVarint:   function(tag, arr) { this.writeMessage(tag, writePackedVarint, arr);   },
+    writePackedSVarint:  function(tag, arr) { this.writeMessage(tag, writePackedSVarint, arr);  },
+    writePackedBoolean:  function(tag, arr) { this.writeMessage(tag, writePackedBoolean, arr);  },
+    writePackedFloat:    function(tag, arr) { this.writeMessage(tag, writePackedFloat, arr);    },
+    writePackedDouble:   function(tag, arr) { this.writeMessage(tag, writePackedDouble, arr);   },
+    writePackedFixed32:  function(tag, arr) { this.writeMessage(tag, writePackedFixed32, arr);  },
+    writePackedSFixed32: function(tag, arr) { this.writeMessage(tag, writePackedSFixed32, arr); },
+    writePackedFixed64:  function(tag, arr) { this.writeMessage(tag, writePackedFixed64, arr);  },
+    writePackedSFixed64: function(tag, arr) { this.writeMessage(tag, writePackedSFixed64, arr); },
 
     writeBytesField: function(tag, buffer) {
-        this.writeTag(tag, Protobuf.Bytes);
+        this.writeTag(tag, Pbf.Bytes);
         this.writeBytes(buffer);
     },
-
     writeFixed32Field: function(tag, val) {
-        this.writeTag(tag, Protobuf.Fixed32);
+        this.writeTag(tag, Pbf.Fixed32);
         this.writeFixed32(val);
     },
-
     writeSFixed32Field: function(tag, val) {
-        this.writeTag(tag, Protobuf.Fixed32);
+        this.writeTag(tag, Pbf.Fixed32);
         this.writeSFixed32(val);
     },
-
     writeFixed64Field: function(tag, val) {
-        this.writeTag(tag, Protobuf.Fixed64);
+        this.writeTag(tag, Pbf.Fixed64);
         this.writeFixed64(val);
     },
-
     writeSFixed64Field: function(tag, val) {
-        this.writeTag(tag, Protobuf.Fixed64);
+        this.writeTag(tag, Pbf.Fixed64);
         this.writeSFixed64(val);
     },
-
     writeVarintField: function(tag, val) {
-        this.writeTag(tag, Protobuf.Varint);
+        this.writeTag(tag, Pbf.Varint);
         this.writeVarint(val);
     },
-
     writeSVarintField: function(tag, val) {
-        this.writeTag(tag, Protobuf.Varint);
+        this.writeTag(tag, Pbf.Varint);
         this.writeSVarint(val);
     },
-
     writeStringField: function(tag, str) {
-        this.writeTag(tag, Protobuf.Bytes);
+        this.writeTag(tag, Pbf.Bytes);
         this.writeString(str);
     },
-
     writeFloatField: function(tag, val) {
-        this.writeTag(tag, Protobuf.Fixed32);
+        this.writeTag(tag, Pbf.Fixed32);
         this.writeFloat(val);
     },
-
     writeDoubleField: function(tag, val) {
-        this.writeTag(tag, Protobuf.Fixed64);
+        this.writeTag(tag, Pbf.Fixed64);
         this.writeDouble(val);
     },
-
     writeBooleanField: function(tag, val) {
@@ -362,1 +401,11 @@ this.writeVarintField(tag, Boolean(val));
 };
+
+function writePackedVarint(arr, pbf)   { for (var i = 0; i < arr.length; i++) pbf.writeVarint(arr[i]);   }
+function writePackedSVarint(arr, pbf)  { for (var i = 0; i < arr.length; i++) pbf.writeSVarint(arr[i]);  }
+function writePackedFloat(arr, pbf)    { for (var i = 0; i < arr.length; i++) pbf.writeFloat(arr[i]);    }
+function writePackedDouble(arr, pbf)   { for (var i = 0; i < arr.length; i++) pbf.writeDouble(arr[i]);   }
+function writePackedBoolean(arr, pbf)  { for (var i = 0; i < arr.length; i++) pbf.writeBoolean(arr[i]);  }
+function writePackedFixed32(arr, pbf)  { for (var i = 0; i < arr.length; i++) pbf.writeFixed32(arr[i]);  }
+function writePackedSFixed32(arr, pbf) { for (var i = 0; i < arr.length; i++) pbf.writeSFixed32(arr[i]); }
+function writePackedFixed64(arr, pbf)  { for (var i = 0; i < arr.length; i++) pbf.writeFixed64(arr[i]);  }
+function writePackedSFixed64(arr, pbf) { for (var i = 0; i < arr.length; i++) pbf.writeSFixed64(arr[i]); }

package.json

 {
   "name": "pbf",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "description": "a low-level, lightweight protocol buffers implementation in JavaScript",
   "main": "index.js",
   "scripts": {
-    "test": "jshint index.js test/*.js && tape test/index.js | faucet",
-    "cov": "istanbul cover test/index.js && coveralls < ./coverage/lcov.info",
+    "test": "jshint index.js buffer.js test/*.js && tape test/*.test.js | faucet",
+    "cov": "istanbul cover tape test/*.test.js && coveralls < ./coverage/lcov.info",
     "build-min": "browserify index.js -s Pbf | uglifyjs -c -m > pbf.js",
@@ -10,0 +10,0 @@ "build-dev": "browserify index.js -d -s Pbf > pbf-dev.js"

README.md

@@ -16,12 +16,13 @@ # pbf
 ```js
-var pbf = new Pbf(buffer),
-    name, version, layerName;
+var data = new Pbf(buffer).readFields(readData, {});
 
-pbf.readFields(function(tag) {
-    if (tag === 1) name = pbf.readString();
-    else if (tag === 2) version = pbf.readVarint();
-    else if (tag === 3) pbf.readMessage(function(tag) {
-        if (tag === 1) layerName = pbf.readString();
-    });
-});
+function readData(tag, data, pbf) {
+    if (tag === 1) data.name = pbf.readString();
+    else if (tag === 2) data.version = pbf.readVarint();
+    else if (tag === 3) data.layer = pbf.readMessage(readLayer, {});
+}
+function readLayer(tag, layer, pbf) {
+    if (tag === 1) layer.name = pbf.readString();
+    else if (tag === 3) layer.size = pbf.readVarint();
+}
 ```
@@ -32,13 +33,15 @@
 ```js
-var pbf = new Pbf();
+var buffer = writeData(data);
 
-pbf.writeStringField(1, 'Hello world');
-pbf.writeVarintField(2, 300);
-
-var layer = new Pbf();
-layer.writeStringField(1, 'foobar');
-
-pbf.writeMessage(3, layer);
-
-var buffer = pbf.finish();
+function writeData(data) {
+    var pbf = new Pbf();
+    pbf.writeStringField(1, data.name);
+    pbf.writeVarintField(2, data.version);
+    pbf.writeMessage(3, writeLayer, data.layer);
+    return pbf.finish();
+}
+function writeLayer(layer, pbf) {
+    pbf.writeStringField(1, layer.name);
+    pbf.writeVarintField(2, layer.size);
+}
 ```
@@ -122,3 +125,3 @@
 
-Basic reading methods:
+Scalar reading methods:
 
@@ -136,5 +139,18 @@ * `readVarint()`
 * `readBytes()`
-* `readPacked(type)`
+* `readString()`
+* `readBytes()`
 * `skip(value)`
 
+Packed reading methods:
+
+* `readPackedVarint()`
+* `readPackedSVarint()`
+* `readPackedFixed32()`
+* `readPackedFixed64()`
+* `readPackedSFixed32()`
+* `readPackedSFixed64()`
+* `readPackedBoolean()`
+* `readPackedFloat()`
+* `readPackedDouble()`
+
 #### Writing
@@ -149,2 +165,13 @@
 
+Write an embedded message:
+
+```js
+pbf.writeMessage(1, writeObj, obj);
+
+function writeObj(obj, pbf) {
+    pbf.writeStringField(obj.name);
+    pbf.writeVarintField(obj.version);
+}
+```
+
 Field writing methods:
@@ -166,2 +193,12 @@
 
+Packed field writing methods:
+
+* `writePackedVarint(tag, val)`
+* `writePackedSVarint(tag, val)`
+* `writePackedSFixed32(tag, val)`
+* `writePackedSFixed64(tag, val)`
+* `writePackedBoolean(tag, val)`
+* `writePackedFloat(tag, val)`
+* `writePackedDouble(tag, val)`
+
 Scalar writing methods:
@@ -173,6 +210,7 @@
 * `writeSFixed64(val)`
+* `writeBoolean(val)`
 * `writeFloat(val)`
 * `writeDouble(val)`
 * `writeString(val)`
-* `writeBytes(buffer)`
+* `writeBytes(buffer[, start, end])`
 
@@ -190,2 +228,16 @@ Misc methods:
 
+#### 1.2.0 (Jan 5, 2015)
+
+##### Breaking API changes
+
+- Changed `writeMessage` signature to `(tag, fn, obj)` (see example in the docs)
+  for a huge encoding performance improvement.
+- Replaced `readPacked` and `writePacked` methods that accept type as a string
+  with `readPackedVarint`, etc. for each type (better performance and simpler API).
+
+##### Improvements
+
+- 5x faster encoding in Node (vector tile benchmark).
+- 40x faster encoding and 3x faster decoding in the browser (vector tile benchmark).
+
 #### 1.1.4 (Jan 2, 2015)
@@ -192,0 +244,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

563517

increased by242.19%

Number of package files

13

increased by18.18%

Lines of code

2503

increased by173.85%

Number of lines in readme file

272

increased by23.64%

Worsened metrics

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes