redis-parser - npm Package Compare versions

Comparing version 2.3.0 to 2.4.0

changelog.md

@@ -0,1 +1,15 @@
+## v.2.4.0 - 25 Jan, 2017
+
+Features
+
+-  Added `reset` function to reset the parser to it's initial values
+-  Added `setReturnBuffers` function to reset the returnBuffers option (Only for the JSParser)
+-  Added `setStringNumbers` function to reset the stringNumbers option (Only for the JSParser)
+-  All Errors are now of sub classes of the new `RedisError` class. It is also exported.
+-  Improved bulk string chunked data handling performance
+
+Bugfixes
+
+-  Parsing time for big nested arrays is now linear
+
 ## v.2.3.0 - 25 Nov, 2016
@@ -2,0 +16,0 @@

index.js

@@ -5,1 +5,2 @@ 'use strict'
 module.exports.ReplyError = require('./lib/replyError')
+module.exports.RedisError = require('./lib/redisError')

lib/hiredis.js

@@ -52,2 +52,11 @@ 'use strict'
 
+/**
+ * Reset the parser values to the initial state
+ *
+ * @returns {undefined}
+ */
+HiredisReplyParser.prototype.reset = function () {
+  this.reader = new hiredis.Reader(this.options)
+}
+
 module.exports = HiredisReplyParser

lib/parser.js

@@ -103,3 +103,3 @@ 'use strict'
  */
-function parseSimpleStringViaOffset (parser) {
+function parseSimpleString (parser) {
   var start = parser.offset
@@ -177,3 +177,3 @@ var offset = start
 function parseError (parser) {
-  var string = parseSimpleStringViaOffset(parser)
+  var string = parseSimpleString(parser)
   if (string !== undefined) {
@@ -215,2 +215,15 @@ if (parser.optionReturnBuffers === true) {
 /**
+ * Push a partly parsed array to the stack
+ *
+ * @param parser
+ * @param elem
+ * @param i
+ * @returns {undefined}
+ */
+function pushArrayCache (parser, elem, pos) {
+  parser.arrayCache.push(elem)
+  parser.arrayPos.push(pos)
+}
+
+/**
  * Parse chunked redis array response
@@ -221,3 +234,13 @@ * @param parser
 function parseArrayChunks (parser) {
-  return parseArrayElements(parser, parser.arrayCache, parser.arrayPos)
+  var tmp = parser.arrayCache.pop()
+  var pos = parser.arrayPos.pop()
+  if (parser.arrayCache.length) {
+    var res = parseArrayChunks(parser)
+    if (!res) {
+      pushArrayCache(parser, tmp, pos)
+      return
+    }
+    tmp[pos++] = res
+  }
+  return parseArrayElements(parser, tmp, pos)
 }
@@ -237,5 +260,3 @@
     if (parser.offset >= bufferLength) {
-      parser.arrayCache = responses
-      parser.arrayPos = i
-      parser.offset = offset
+      pushArrayCache(parser, responses, i)
       return
@@ -245,5 +266,6 @@ }
     if (response === undefined) {
-      parser.arrayCache = responses
-      parser.arrayPos = i
-      parser.offset = offset
+      if (!parser.arrayCache.length) {
+        parser.offset = offset
+      }
+      pushArrayCache(parser, responses, i)
       return
@@ -271,3 +293,3 @@ }
     case 43: // +
-      return parseSimpleStringViaOffset(parser)
+      return parseSimpleString(parser)
     case 42: // *
@@ -328,2 +350,11 @@ return parseArray(parser)
   this.name = 'javascript'
+  this.reset()
+}
+
+/**
+ * Reset the parser values to the initial state
+ *
+ * @returns {undefined}
+ */
+JavascriptRedisParser.prototype.reset = function () {
   this.offset = 0
@@ -335,34 +366,30 @@ this.buffer = null
   this.bufferCache = []
-  this.arrayCache = null
-  this.arrayPos = 0
+  this.arrayCache = []
+  this.arrayPos = []
 }
 
 /**
- * Concat a bulk string containing multiple chunks
+ * Set the returnBuffers option
  *
- * Notes:
- * 1) The first chunk might contain the whole bulk string including the \r
- * 2) We are only safe to fully add up elements that are neither the first nor any of the last two elements
+ * @param returnBuffers
+ * @returns {undefined}
+ */
+JavascriptRedisParser.prototype.setReturnBuffers = function (returnBuffers) {
+  if (typeof returnBuffers !== 'boolean') {
+    throw new TypeError('The returnBuffers argument has to be a boolean')
+  }
+  this.optionReturnBuffers = returnBuffers
+}
+
+/**
+ * Set the stringNumbers option
  *
- * @param parser
- * @param buffer
- * @returns {String}
+ * @param stringNumbers
+ * @returns {undefined}
  */
-function concatBulkString (parser) {
-  var list = parser.bufferCache
-  var chunks = list.length
-  var offset = parser.bigStrSize - parser.totalChunkSize
-  parser.offset = offset
-  if (offset === 1) {
-    if (chunks === 2) {
-      return list[0].toString('utf8', parser.bigOffset, list[0].length - 1)
-    }
-    chunks--
+JavascriptRedisParser.prototype.setStringNumbers = function (stringNumbers) {
+  if (typeof stringNumbers !== 'boolean') {
+    throw new TypeError('The stringNumbers argument has to be a boolean')
   }
-  var res = decoder.write(list[0].slice(parser.bigOffset))
-  for (var i = 1; i < chunks - 1; i++) {
-    res += decoder.write(list[i])
-  }
-  res += decoder.end(list[i].slice(0, offset - 2))
-  return res
+  this.optionStringNumbers = stringNumbers
 }
@@ -399,15 +426,12 @@
 /**
- * Concat the collected chunks from parser.bufferCache
- * @param parser
+ * Check if the requested size fits in the current bufferPool.
+ * If it does not, reset and increase the bufferPool accordingly.
+ *
  * @param length
- * @returns {Buffer}
+ * @returns {undefined}
  */
-function concatBuffer (parser, length) {
-  var list = parser.bufferCache
-  var pos = bufferOffset
-  length -= parser.offset
+function resizeBuffer (length) {
   if (bufferPool.length < length + bufferOffset) {
-    // Increase the bufferPool size
     var multiplier = length > 1024 * 1024 * 75 ? 2 : 3
-    if (bufferOffset > 1024 * 1024 * 120) {
+    if (bufferOffset > 1024 * 1024 * 111) {
       bufferOffset = 1024 * 1024 * 50
@@ -418,3 +442,2 @@ }
     counter++
-    pos = 0
     if (interval === null) {
@@ -424,8 +447,63 @@ interval = setInterval(decreaseBufferPool, 50)
   }
-  list[0].copy(bufferPool, pos, parser.offset, list[0].length)
-  pos += list[0].length - parser.offset
-  for (var i = 1; i < list.length; i++) {
+}
+
+/**
+ * Concat a bulk string containing multiple chunks
+ *
+ * Notes:
+ * 1) The first chunk might contain the whole bulk string including the \r
+ * 2) We are only safe to fully add up elements that are neither the first nor any of the last two elements
+ *
+ * @param parser
+ * @returns {String}
+ */
+function concatBulkString (parser) {
+  var list = parser.bufferCache
+  var chunks = list.length
+  var offset = parser.bigStrSize - parser.totalChunkSize
+  parser.offset = offset
+  if (offset === 1) {
+    if (chunks === 2) {
+      return list[0].toString('utf8', parser.bigOffset, list[0].length - 1)
+    }
+    chunks--
+  }
+  var res = decoder.write(list[0].slice(parser.bigOffset))
+  for (var i = 1; i < chunks - 1; i++) {
+    res += decoder.write(list[i])
+  }
+  res += decoder.end(list[i].slice(0, offset - 2))
+  return res
+}
+
+/**
+ * Concat the collected chunks from parser.bufferCache.
+ *
+ * Increases the bufferPool size beforehand if necessary.
+ *
+ * @param parser
+ * @returns {Buffer}
+ */
+function concatBulkBuffer (parser) {
+  var list = parser.bufferCache
+  var chunks = list.length
+  var length = parser.bigStrSize - parser.bigOffset - 2
+  var offset = parser.bigStrSize - parser.totalChunkSize
+  parser.offset = offset
+  if (offset === 1) {
+    if (chunks === 2) {
+      return list[0].slice(parser.bigOffset, list[0].length - 1)
+    }
+    chunks--
+    offset = list[list.length - 1].length + 1
+  }
+  resizeBuffer(length)
+  var pos = bufferOffset
+  list[0].copy(bufferPool, pos, parser.bigOffset, list[0].length)
+  pos += list[0].length - parser.bigOffset
+  for (var i = 1; i < list.length - 1; i++) {
     list[i].copy(bufferPool, pos)
     pos += list[i].length
   }
+  list[i].copy(bufferPool, pos, 0, offset - 2)
   var buffer = bufferPool.slice(bufferOffset, length + bufferOffset)
@@ -442,3 +520,2 @@ bufferOffset += length
 JavascriptRedisParser.prototype.execute = function execute (buffer) {
-  var arr
   if (this.buffer === null) {
@@ -455,4 +532,4 @@ this.buffer = buffer
     this.offset = 0
-    if (this.arrayCache) {
-      arr = parseArrayChunks(this)
+    if (this.arrayCache.length) {
+      var arr = parseArrayChunks(this)
       if (!arr) {
@@ -462,25 +539,17 @@ return
       this.returnReply(arr)
-      this.arrayCache = null
     }
   } else if (this.totalChunkSize + buffer.length >= this.bigStrSize) {
     this.bufferCache.push(buffer)
-    if (this.optionReturnBuffers === false && !this.arrayCache) {
-      this.returnReply(concatBulkString(this))
-      this.buffer = buffer
-    } else {
-      this.buffer = concatBuffer(this, this.totalChunkSize + buffer.length)
-      this.offset = 0
-      if (this.arrayCache) {
-        arr = parseArrayChunks(this)
-        if (!arr) {
-          this.bigStrSize = 0
-          this.bufferCache = []
-          return
-        }
-        this.returnReply(arr)
-        this.arrayCache = null
+    var tmp = this.optionReturnBuffers ? concatBulkBuffer(this) : concatBulkString(this)
+    this.bigStrSize = 0
+    this.bufferCache = []
+    this.buffer = buffer
+    if (this.arrayCache.length) {
+      this.arrayCache[0][this.arrayPos[0]++] = tmp
+      tmp = parseArrayChunks(this)
+      if (!tmp) {
+        return
       }
     }
-    this.bigStrSize = 0
-    this.bufferCache = []
+    this.returnReply(tmp)
   } else {
@@ -497,3 +566,3 @@ this.bufferCache.push(buffer)
     if (response === undefined) {
-      if (!this.arrayCache) {
+      if (!this.arrayCache.length) {
         this.offset = offset
@@ -500,0 +569,0 @@ }

lib/replyError.js

 'use strict'
 
 var util = require('util')
+var RedisError = require('./redisError')
 
@@ -8,12 +9,7 @@ function ReplyError (message, newLimit) {
   Error.stackTraceLimit = newLimit || 2
-  Error.call(this, message)
-  Error.captureStackTrace(this, this.constructor)
+  RedisError.call(this, message)
   Error.stackTraceLimit = limit
-  Object.defineProperty(this, 'message', {
-    value: message || '',
-    writable: true
-  })
 }
 
-util.inherits(ReplyError, Error)
+util.inherits(ReplyError, RedisError)
 
@@ -20,0 +16,0 @@ Object.defineProperty(ReplyError.prototype, 'name', {

package.json

 {
   "name": "redis-parser",
-  "version": "2.3.0",
+  "version": "2.4.0",
   "description": "Javascript Redis protocol (RESP) parser",
@@ -5,0 +5,0 @@ "main": "index.js",

README.md

 [![Build Status](https://travis-ci.org/NodeRedis/node-redis-parser.png?branch=master)](https://travis-ci.org/NodeRedis/node-redis-parser)
-[![Code Climate](https://codeclimate.com/github/NodeRedis/node-redis-parser/badges/gpa.svg)](https://codeclimate.com/github/NodeRedis/node-redis-parser)
 [![Test Coverage](https://codeclimate.com/github/NodeRedis/node-redis-parser/badges/coverage.svg)](https://codeclimate.com/github/NodeRedis/node-redis-parser/coverage)
@@ -24,3 +23,3 @@ [![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/)
 
-### Possible options
+### Options
 
@@ -33,2 +32,13 @@ * `returnReply`: *function*; mandatory
 
+### Functions
+
+* `reset()`: reset the parser to it's initial state
+* `setReturnBuffers(boolean)`: (JSParser only) set the returnBuffers option on/off without resetting the parser
+* `setStringNumbers(boolean)`: (JSParser only) set the stringNumbers option on/off without resetting the parser
+
+### Error classes
+
+All errors returned by the parser are of the class `ReplyError` that is a sub class of `RedisError`.  
+Both types are exported by the parser.
+
 ### Example
@@ -103,46 +113,46 @@
 
-    HIREDIS: $ multiple chunks in a bulk string x 867,643 ops/sec ±1.39% (82 runs sampled)
-    HIREDIS BUF: $ multiple chunks in a bulk string x 591,398 ops/sec ±1.48% (83 runs sampled)
-    JS PARSER: $ multiple chunks in a bulk string x 942,834 ops/sec ±0.87% (90 runs sampled)
-    JS PARSER BUF: $ multiple chunks in a bulk string x 1,081,096 ops/sec ±1.81% (85 runs sampled)
+    HIREDIS: $ multiple chunks in a bulk string x 859,880 ops/sec ±1.22% (82 runs sampled)
+    HIREDIS BUF: $ multiple chunks in a bulk string x 608,869 ops/sec ±1.72% (85 runs sampled)
+    JS PARSER: $ multiple chunks in a bulk string x 910,590 ops/sec ±0.87% (89 runs sampled)
+    JS PARSER BUF: $ multiple chunks in a bulk string x 1,299,507 ops/sec ±2.18% (84 runs sampled)
 
-    HIREDIS: + multiple chunks in a string x 1,785,222 ops/sec ±0.59% (92 runs sampled)
-    HIREDIS BUF: + multiple chunks in a string x 902,391 ops/sec ±1.62% (88 runs sampled)
-    JS PARSER: + multiple chunks in a string x 1,936,709 ops/sec ±1.07% (90 runs sampled)
-    JS PARSER BUF: + multiple chunks in a string x 1,954,798 ops/sec ±0.84% (91 runs sampled)
+    HIREDIS: + multiple chunks in a string x 1,787,203 ops/sec ±0.58% (96 runs sampled)
+    HIREDIS BUF: + multiple chunks in a string x 943,584 ops/sec ±1.62% (87 runs sampled)
+    JS PARSER: + multiple chunks in a string x 2,008,264 ops/sec ±1.01% (91 runs sampled)
+    JS PARSER BUF: + multiple chunks in a string x 2,045,546 ops/sec ±0.78% (91 runs sampled)
 
-    HIREDIS: $ 4mb bulk string x 344 ops/sec ±1.40% (85 runs sampled)
-    HIREDIS BUF: $ 4mb bulk string x 555 ops/sec ±1.85% (80 runs sampled)
-    JS PARSER: $ 4mb bulk string x 834 ops/sec ±1.23% (81 runs sampled)
-    JS PARSER BUF: $ 4mb bulk string x 620 ops/sec ±2.40% (59 runs sampled)
+    HIREDIS: $ 4mb bulk string x 310 ops/sec ±1.58% (75 runs sampled)
+    HIREDIS BUF: $ 4mb bulk string x 471 ops/sec ±2.28% (78 runs sampled)
+    JS PARSER: $ 4mb bulk string x 747 ops/sec ±2.43% (85 runs sampled)
+    JS PARSER BUF: $ 4mb bulk string x 846 ops/sec ±5.52% (72 runs sampled)
 
-    HIREDIS: + simple string x 2,344,042 ops/sec ±1.45% (91 runs sampled)
-    HIREDIS BUF: + simple string x 993,081 ops/sec ±1.87% (83 runs sampled)
-    JS PARSER: + simple string x 4,431,517 ops/sec ±1.86% (88 runs sampled)
-    JS PARSER BUF: + simple string x 5,259,552 ops/sec ±0.61% (96 runs sampled)
+    HIREDIS: + simple string x 2,324,866 ops/sec ±1.61% (90 runs sampled)
+    HIREDIS BUF: + simple string x 1,085,823 ops/sec ±2.47% (82 runs sampled)
+    JS PARSER: + simple string x 4,567,358 ops/sec ±1.97% (81 runs sampled)
+    JS PARSER BUF: + simple string x 5,433,901 ops/sec ±0.66% (93 runs sampled)
 
-    HIREDIS: : integer x 2,376,642 ops/sec ±0.30% (92 runs sampled)
-    JS PARSER: : integer x 17,765,077 ops/sec ±0.53% (93 runs sampled)
-    JS PARSER STR: : integer x 13,110,365 ops/sec ±0.67% (91 runs sampled)
+    HIREDIS: : integer x 2,332,946 ops/sec ±0.47% (93 runs sampled)
+    JS PARSER: : integer x 17,730,449 ops/sec ±0.73% (91 runs sampled)
+    JS PARSER STR: : integer x 12,942,037 ops/sec ±0.51% (92 runs sampled)
 
-    HIREDIS: : big integer x 2,010,124 ops/sec ±0.87% (86 runs sampled)
-    JS PARSER: : big integer x 10,277,063 ops/sec ±0.69% (91 runs sampled)
-    JS PARSER STR: : big integer x 4,492,626 ops/sec ±0.67% (94 runs sampled)
+    HIREDIS: : big integer x 2,012,572 ops/sec ±0.33% (93 runs sampled)
+    JS PARSER: : big integer x 10,210,923 ops/sec ±0.94% (94 runs sampled)
+    JS PARSER STR: : big integer x 4,453,320 ops/sec ±0.52% (94 runs sampled)
 
-    HIREDIS: * array x 43,763 ops/sec ±0.84% (94 runs sampled)
-    HIREDIS BUF: * array x 13,893 ops/sec ±1.05% (85 runs sampled)
-    JS PARSER: * array x 50,825 ops/sec ±1.92% (80 runs sampled)
-    JS PARSER BUF: * array x 72,546 ops/sec ±0.80% (94 runs sampled)
+    HIREDIS: * array x 44,479 ops/sec ±0.55% (94 runs sampled)
+    HIREDIS BUF: * array x 14,391 ops/sec ±1.04% (86 runs sampled)
+    JS PARSER: * array x 53,796 ops/sec ±2.08% (79 runs sampled)
+    JS PARSER BUF: * array x 72,428 ops/sec ±0.72% (93 runs sampled)
 
-    HIREDIS: * big array x 265 ops/sec ±1.46% (86 runs sampled)
-    HIREDIS BUF: * big array x 226 ops/sec ±3.21% (75 runs sampled)
-    JS PARSER: * big array x 201 ops/sec ±0.95% (83 runs sampled)
-    JS PARSER BUF: * big array x 244 ops/sec ±2.65% (81 runs sampled)
+    HIREDIS: * big nested array x 217 ops/sec ±0.97% (83 runs sampled)
+    HIREDIS BUF: * big nested array x 255 ops/sec ±2.28% (77 runs sampled)
+    JS PARSER: * big nested array x 242 ops/sec ±1.10% (85 runs sampled)
+    JS PARSER BUF: * big nested array x 375 ops/sec ±1.21% (88 runs sampled)
 
-    HIREDIS: - error x 81,563 ops/sec ±0.51% (93 runs sampled)
-    JS PARSER: - error x 155,225 ops/sec ±0.57% (95 runs sampled)
+    HIREDIS: - error x 78,821 ops/sec ±0.80% (93 runs sampled)
+    JS PARSER: - error x 143,382 ops/sec ±0.75% (92 runs sampled)
 
     Platform info:
     Ubuntu 16.10
-    Node.js 7.1.0
+    Node.js 7.4.0
     Intel(R) Core(TM) i7-5600U CPU
@@ -149,0 +159,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

29366

increased by11.18%

Lines of code

619

increased by15.92%

Number of lines in readme file

160

increased by6.67%

Worsened metrics

Number of package files

11

decreased by-8.33%

No dependency changes