pomelo-protobuf - npm Package Compare versions

Comparing version 0.3.1 to 0.3.2

lib/client/protobuf.js

@@ -129,3 +129,2 @@ /* ProtocolBuffer client 0.1.0*/
 
-
   Codec.decodeSInt32 = function(bytes){
@@ -146,3 +145,3 @@ var n = this.decodeUInt32(bytes);
   Codec.decodeFloat = function(bytes, offset){
-    if(!bytes || bytes.length < (offset +4)){
+    if(!bytes || bytes.length < (offset + 4)){
       return null;
@@ -164,3 +163,3 @@ }
   Codec.decodeDouble = function(bytes, offset){
-    if(!bytes || bytes.length < (8 + offset)){
+    if(!bytes || bytes.length < (offset + 8)){
       return null;
@@ -167,0 +166,0 @@ }

lib/encoder.js

@@ -0,56 +1,193 @@
+var codec = require('./codec');
+var constant = require('./constant');
+var util = require('./util');
+
 var Encoder = module.exports;
 
-Encoder.encodeUInt32 = function(num){
-	var n = parseInt(num);
-	if(isNaN(n) || n < 0){
-		console.log(n);
+Encoder.init = function(protos){
+	this.protos = protos || {};
+};
+
+Encoder.encode = function(route, msg){
+	if(!route || !msg){
+		console.warn('Route or msg can not be null! route : %j, msg %j', route, msg);
 		return null;
 	}
 
-	var result = [];
-	do{
-		var tmp = n % 128;
-		var next = Math.floor(n/128);
+	//Get protos from protos map use the route as key
+	var protos = this.protos[route];
 
-		if(next !== 0){
-			tmp = tmp + 128;
+	//Check msg
+	if(!checkMsg(msg, protos)){
+		console.warn('check msg failed! msg : %j, proto : %j', msg, protos);
+		return null;
+	}
+
+	//Set the length of the buffer 2 times bigger to prevent overflow
+	var length = Buffer.byteLength(JSON.stringify(msg))*2;
+
+	//Init buffer and offset
+	var buffer = new Buffer(length);
+	var offset = 0;
+
+	if(!!protos){
+		offset = encodeMsg(buffer, offset, protos, msg);
+		if(offset > 0){
+			return buffer.slice(0, offset);
 		}
-		result.push(tmp);
-		n = next;
-	} while(n !== 0);
+	}
 
-	return result;
+	return null;
 };
 
-Encoder.encodeSInt32 = function(n){
-	var n = parseInt(n);
-	if(isNaN(n)){
-		return null;
+/**
+ * Check if the msg follow the defination in the protos
+ */
+function checkMsg(msg, protos){
+	if(!protos || !msg){
+		console.warn('no protos or msg exist! msg : %j, protos : %j', msg, protos);
+		return false;
 	}
-	n = n<0?(Math.abs(n)*2-1):n*2;
 
-	return Encoder.encodeUInt32(n);
+	for(var name in protos){
+		var proto = protos[name];
+
+		//All required element must exist
+		switch(proto.option){
+			case 'required' :
+				if(typeof(msg[name]) === 'undefined'){
+					console.warn('no property exist for required! name : %j ,proto : %j, msg : %j, ', name, proto);
+					return false;
+				}
+			case 'optional' :
+				if(typeof(msg[name]) !== 'undefined'){
+					if(!!protos.__messages[proto.type]){
+						checkMsg(msg[name], protos.__messages[proto.type]);
+					}
+				}
+			break;
+			case 'repeated' :
+				//Check nest message in repeated elements
+				if(!!msg[name] && !!protos.__messages[proto.type]){
+					for(var i = 0; i < msg[name].length; i++){
+						if(!checkMsg(msg[name][i], protos.__messages[proto.type])){
+							return false;
+						}
+					}
+				}
+			break;
+		}
+	}
+
+	return true;
 }
 
-Encoder.decodeUInt32 = function(bytes){
-	var n = 0;
+function encodeMsg(buffer, offset, protos, msg){
+	for(var name in msg){
+		if(!!protos[name]){
+			var proto = protos[name];
 
-	for(var i = 0; i < bytes.length; i++){
-		var m = parseInt(bytes[i]);
-		n = n + ((m & 0x7f) * Math.pow(2,(7*i)));
-		if(m < 128)
-			return n;
+			switch(proto.option){
+				case 'required' :
+				case 'optional' :
+					offset = writeBytes(buffer, offset, encodeTag(proto.type, proto.tag));
+					offset = encodeProp(msg[name], proto.type, offset, buffer, protos);
+				break;
+				case 'repeated' :
+					if(!!msg[name] && msg[name].length > 0){
+						offset = encodeArray(msg[name], proto, offset, buffer, protos);
+					}
+				break;
+			}
+		}
 	}
 
-	return n;
+	return offset;
 }
 
+function encodeProp(value, type, offset, buffer, protos){
+	var length = 0;
 
-Encoder.decodeSInt32 = function(bytes){
-	var n = this.decodeUInt32(bytes);
-	var flag = ((n%2) === 1)?-1:1;
+	switch(type){
+		case 'uInt32':
+			offset = writeBytes(buffer, offset, codec.encodeUInt32(value));
+		break;
+		case 'int32' :
+		case 'sInt32':
+			offset = writeBytes(buffer, offset, codec.encodeSInt32(value));
+		break;
+		case 'float':
+			buffer.wirteFloatLE(value, offset);
+			offset += 4;
+		break;
+		case 'double':
+			buffer.writeDoubleLE(value, offset);
+			offset += 8;
+		break;
+		case 'string':
+			length = Buffer.byteLength(value);
 
-	n = ((n%2 + n)/2)*flag;
+			//Encode length
+			offset = writeBytes(buffer, offset, codec.encodeUInt32(length));
+			//write string
+			buffer.write(value, offset, length);
+			offset += length;
+		break;
+		default :
+			if(!!protos.__messages[type]){
+				//Use a tmp buffer to build an internal msg
+				var tmpBuffer = new Buffer(Buffer.byteLength(JSON.stringify(value)));
+				length = 0;
 
-	return n;
+				length = encodeMsg(tmpBuffer, length, protos.__messages[type], value);
+				//Encode length
+				offset = writeBytes(buffer, offset, codec.encodeUInt32(length));
+				//contact the object
+				tmpBuffer.copy(buffer, offset, 0, length);
+
+				offset += length;
+			}
+		break;
+	}
+
+	return offset;
 }
+
+/**
+ * Encode reapeated properties, simple msg and object are decode differented
+ */
+function encodeArray(array, proto, offset, buffer, protos){
+	var i = 0;
+
+	if(util.isSimpleType(proto.type)){
+		offset = writeBytes(buffer, offset, encodeTag(proto.type, proto.tag));
+		offset = writeBytes(buffer, offset, codec.encodeUInt32(array.length));
+		for(i = 0; i < array.length; i++){
+			offset = encodeProp(array[i], proto.type, offset, buffer);
+		}
+	}else{
+		for(i = 0; i < array.length; i++){
+			offset = writeBytes(buffer, offset, encodeTag(proto.type, proto.tag));
+			offset = encodeProp(array[i], proto.type, offset, buffer, protos);
+		}
+	}
+
+	return offset;
+}
+
+function writeBytes(buffer, offset, bytes){
+	for(var i = 0; i < bytes.length; i++){
+		buffer.writeUInt8(bytes[i], offset);
+		offset++;
+	}
+
+	return offset;
+}
+
+function encodeTag(type, tag){
+	var value = constant.TYPES[type];
+
+	if(value === undefined) value = 2;
+
+	return codec.encodeUInt32((tag<<3)|value);
+}

lib/protobuf.js

@@ -1,6 +0,13 @@
-var encoder = require('./msgEncoder');
-var decoder = require('./msgDecoder');
-var parser = require('./protoParser');
+var encoder = require('./encoder');
+var decoder = require('./decoder');
+var parser = require('./parser');
 
 var Protobuf = module.exports;
+
+/**
+ * [encode the given message, return a Buffer represent the message encoded by protobuf]
+ * @param  {[type]} key The key to identify the message type.
+ * @param  {[type]} msg The message body, a js object.
+ * @return {[type]} The binary encode result in a Buffer.
+ */
 Protobuf.encode = function(key, msg){
@@ -15,2 +22,3 @@ try{
 
+
 Protobuf.encode2Bytes = function(key, msg){
@@ -17,0 +25,0 @@ var buffer = this.encode(key, msg);

lib/util.js

 var util = module.exports;
 
 util.isSimpleType = function(type){
-	return ( type === 'uInt32' || 
-					 type === 'sInt32' || 
-					 type === 'int32' || 
-					 type === 'uInt64' || 
+	return ( type === 'uInt32' ||
+					 type === 'sInt32' ||
+					 type === 'int32'  ||
+					 type === 'uInt64' ||
 					 type === 'sInt64' ||
-					 type === 'float' ||
+					 type === 'float'  ||
 					 type === 'double');
-}
+};
 
@@ -17,3 +17,3 @@ util.equal = function(obj0, obj1){
 		var n = obj1[key];
-		
+
 		if(typeof(m) === 'object'){
@@ -24,8 +24,7 @@ if(!util.equal(m, n)){
 		}else if(m !== n){
-			console.log('m  %j, n : %j', m , n);
 			return false;
 		}
 	}
-	
+
 	return true;
-}
+};

package.json

 {
   "name": "pomelo-protobuf",
-  "version": "0.3.1",
+  "version": "0.3.2",
   "main": "./lib/protobuf",
@@ -5,0 +5,0 @@ "dependencies": {},

README.md

 #Pomelo-protobuf
-  protobuf protocol is a high efficient binary protocol for data encode, this module implement the protobuf protocol, and used in pomelo for data transfer.
+  Protobuf protocol is a high efficient binary protocol for data encode, this module implement the protobuf protocol, and used in [pomelo](https://github.com/NetEase/pomelo) for data transfer.
+Of course, pomelo-protobuf can also be used independently in other projects.
 ##Architecture
@@ -44,5 +45,5 @@ Unlike the google protobuf, we provide a universal encoder and decoder in pomelo-protobuf. We use protos file as meta data to encode/decode messages, so you do not need to add any code to your project, instead , what you need is to add a protos.json (or two for different encoder and decoder messages) files to define the message need to encode by protobuf.The architecture of pomelo-protobuf is as follow:
 - The server code run in Node.JS environment, use Buffer to represent the binary data.
-- The client side code run on browser, use Buffer to represent the binary data.
+- The client side code run on browser, use ByteArray to represent the binary data.
 
-On average the Server version can be 60% higher than client version, with less memory usage. So we  highly recommend that use the server code on Node.JS for better performance.
+On average, the encode/decode speed of Server version is 60% faster than client version, with less memory usage. So we  highly recommend that use the server code on Node.JS for better performance.
 
@@ -49,0 +50,0 @@ ### Example message

test/client/encoderTest.js

 var should = require('should');
-var encoder = require('../../lib/client/encoder');
+var encoder = require('../../lib/client/protobuf').codec;
 
@@ -8,24 +8,24 @@ describe('client encoder test', function(){
 			var float = Math.random();
-			
+
 			var bytes = encoder.encodeFloat(float);
-			var result = encoder.decodeFloat(bytes);
-			
+			var result = encoder.decodeFloat(bytes, 0);
+
 			var diff = Math.abs(float-result);
 			//console.log('float : %j, result : %j, diff : %j', float, result, diff);
 			diff.should.below(0.0000001);
-			
+
 		}
-	})
-	
+	});
+
 	describe('double test for 10000 times', function(){
 		for(var i = 0; i < 10000; i++){
 			var double = Math.random();
-			
+
 			var bytes = encoder.encodeDouble(double);
-			var result = encoder.decodeDouble(bytes);
-			
+			var result = encoder.decodeDouble(bytes, 0);
+
 			double.should.equal(result);
 		}
-	})
-	
+	});
+
 	describe('utf8 encode & decode test, use 1000 * 1000 test case', function(){
@@ -41,11 +41,11 @@ var num = 1000;
 			//arr = [ 58452, 127, 38641, 25796, 20652, 19237 ];
-			
+
 			var str = String.fromCharCode.apply(null, arr);
-			
+
 			//console.log('old arr : %j', arr);
-			
+
 			var length = encoder.byteLength(str);
 			var buffer = new ArrayBuffer(length);
 			var bytes = new Uint8Array(buffer);
-			
+
 			var offset = encoder.encodeStr(bytes, 0, str);
@@ -55,6 +55,6 @@ //console.log('encode over, offset : %j, length : %j, str length : %j', offset, length, str.length);
 			length.should.equal.offset;
-			
+
 			var result = encoder.decodeStr(bytes, 0, length);
-			
-			
+
+
 			str.length.should.equal(result.length);
@@ -69,11 +69,11 @@ var flag = true;
 			}
-			
+
 			if(!flag)return;
 			//console.log('str : %j, bytes : %j, result : %j', str, bytes, result);
 		}
-	})
-	
+	});
+
 	describe('string decode speed test', function(){
 		var array = [];
-		var length = 1000;
+		var length = 100000;
 		for(var i = 0; i < length; i++,array.push(0));
@@ -89,4 +89,4 @@ var start = Date.now();
 
-		console.log('cost time : %j, length : %j', end-start, str.length);
-		
+		console.log('cost time with fromCharCode method : %j, length : %j', end-start, str.length);
+
 		start = Date.now();
@@ -98,5 +98,5 @@ str = '';
 		end = Date.now();
-		
+
 		console.log('cost time by add string: %j, length : %j', end-start, str.length);
-	})
-})
+	});
+});

test/client/protobufTest.js

@@ -5,3 +5,3 @@ var protobuf = require('../../lib/client/protobuf');
 var codec = protobuf.codec;
-var parser = require('../../lib/protoParser');
+var parser = require('../../lib/parser');
 var util = require('../../lib/util');
@@ -8,0 +8,0 @@ var should = require('should');

test/codecTest.js

@@ -1,2 +0,2 @@
-var encoder = require('../lib/encoder');
+var encoder = require('../lib/codec');
 var should = require('should');
@@ -28,35 +28,2 @@
 	});
-
-	describe('buffer speed test', function(){
-		var n = 100000;
-		var i;
-		var a = '';
-		var start = Date.now();
-
-		for(i = 0; i < n; i++){
-			var b = new Buffer(4);
-
-			b.writeUInt8(0, 0);
-			b.writeUInt8(1, 1);
-			b.writeUInt8(2, 2);
-			b.writeUInt8(3, 3);
-		}
-
-		var end = Date.now();
-		var time1 = end-start;
-
-		start = Date.now();
-		for(i = 0; i < n; i++){
-			var c = [];
-			c.push(0);
-			c.push(1);
-			c.push(2);
-			c.push(3);
-		}
-
-		end = Date.now();
-
-		var time2 = end -start;
-		console.log('time1 : %j, time2 : %j', time1, time2);
-	});
 });

test/protobufTest.js

@@ -15,8 +15,14 @@ var protobuf = require('../lib/protobuf');
 			var buffer = protobuf.encode(route, msg);
+
+console.log(msg);
+			console.log(buffer.length);
+			console.log(buffer)
+
 			var decodeMsg = protobuf.decode(route, buffer);
 
+			console.log(decodeMsg);
+
 			util.equal(msg, decodeMsg).should.equal(true);
 		}
-
 	});
 });

test/stringBufferTest.js

@@ -7,13 +7,13 @@
 		arr.push(i + '');
-		
+
 	var start = Date.now();
 	var str = '';
-	for(var j = 0; j < 1000; j++)
+
 	for(var i = 0; i < num; i++){
 		str += arr[i];
 	}
-	
+
 	var end = Date.now();
 	var time1 = end - start;
-	
+
 	start = Date.now();
@@ -27,3 +27,3 @@ var arr = [];
 	var time2 = end - start;
-	
+
 	console.log('test count : %j, \ncost 1 : %j, \ncost 2 : %j', num, time1, time2);
@@ -33,5 +33,5 @@ }
 joinTest(100);
-//joinTest(50000);
-//joinTest(100000);
-//joinTest(200000);
+joinTest(50000);
+joinTest(100000);
+joinTest(200000);
 //joinTest(500000);

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

77255

increased by65.95%

Number of package files

25

increased by13.64%

Lines of code

2857

increased by96.36%

Number of lines in readme file

117

increased by0.86%

Worsened metrics

Number of low supply chain risk alerts

2

increased by100%

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes