redis-parser - npm Package Compare versions

Comparing version 1.3.0 to 2.0.0

changelog.md

@@ -0,1 +1,20 @@
+## v.2.0.0 - 29 May, 2016
+
+The javascript parser got completly rewritten by [Michael Diarmid](https://github.com/Salakar) and [Ruben Bridgewater](https://github.com/BridgeAR) and is now a lot faster than the hiredis parser.
+Therefore the hiredis parser was deprecated and should only be used for testing purposes and benchmarking comparison.
+
+All Errors returned by the parser are from now on of class ReplyError
+
+Features
+
+-  Improved performance by up to 15x as fast as before
+-  Improved options validation
+-  Added ReplyError Class
+-  Added parser benchmark
+-  Switched default parser from hiredis to JS, no matter if hiredis is installed or not
+
+Removed
+
+-  Deprecated hiredis support
+
 ## v.1.3.0 - 27 Mar, 2016
@@ -2,0 +21,0 @@

index.js

@@ -1,3 +0,4 @@
-'use strict';
+'use strict'
 
-module.exports = require('./lib/parser');
+module.exports = require('./lib/parser')
+module.exports.ReplyError = require('./lib/replyError')

lib/hiredis.js

@@ -1,36 +0,52 @@
-'use strict';
+'use strict'
 
-var hiredis = require('hiredis');
+var hiredis = require('hiredis')
+var ReplyError = require('../lib/replyError')
 
-function HiredisReplyParser(options) {
-    this.name = 'hiredis';
-    this.options = options;
-    this.reader = new hiredis.Reader(options);
+/**
+ * Parse data
+ * @param parser
+ * @returns {*}
+ */
+function parseData (parser) {
+  try {
+    return parser.reader.get()
+  } catch (err) {
+    // Protocol errors land here
+    // Reset the parser. Otherwise new commands can't be processed properly
+    parser.reader = new hiredis.Reader(parser.options)
+    parser.returnFatalError(new ReplyError(err.message))
+  }
 }
 
-HiredisReplyParser.prototype.parseData = function () {
-    try {
-        return this.reader.get();
-    } catch (err) {
-        // Protocol errors land here
-        // Reset the parser. Otherwise new commands can't be processed properly
-        this.reader = new hiredis.Reader(this.options);
-        this.returnFatalError(err);
-    }
-};
+/**
+ * Hiredis Parser
+ * @param options
+ * @constructor
+ */
+function HiredisReplyParser (options) {
+  this.returnError = options.returnError
+  this.returnFatalError = options.returnFatalError || options.returnError
+  this.returnReply = options.returnReply
+  this.name = 'hiredis'
+  this.options = {
+    return_buffers: !!options.returnBuffers
+  }
+  this.reader = new hiredis.Reader(this.options)
+}
 
 HiredisReplyParser.prototype.execute = function (data) {
-    this.reader.feed(data);
-    var reply = this.parseData();
+  this.reader.feed(data)
+  var reply = parseData(this)
 
-    while (reply !== undefined) {
-        if (reply && reply.name === 'Error') {
-            this.returnError(reply);
-        } else {
-            this.returnReply(reply);
-        }
-        reply = this.parseData();
+  while (reply !== undefined) {
+    if (reply && reply.name === 'Error') {
+      this.returnError(new ReplyError(reply.message))
+    } else {
+      this.returnReply(reply)
     }
-};
+    reply = parseData(this)
+  }
+}
 
-module.exports = HiredisReplyParser;
+module.exports = HiredisReplyParser

lib/parser.js

@@ -1,59 +0,399 @@
-'use strict';
+'use strict'
 
-var parsers = {
-    javascript: require('./javascript')
-};
+var ReplyError = require('./replyError')
+var bufferPool = new Buffer(64 * 1024)
+var interval = null
 
-// Hiredis might not be installed
-try {
-    parsers.hiredis = require('./hiredis');
-} catch (err) { /* ignore errors */ }
+/**
+ * Used for lengths and numbers only, faster perf on arrays / bulks
+ * @param parser
+ * @returns {*}
+ */
+function parseSimpleNumbers (parser) {
+  var offset = parser.offset
+  var length = parser.buffer.length
+  var number = 0
+  var sign = false
 
-function Parser (options) {
-    var parser, msg;
+  if (parser.buffer[offset] === 45) {
+    sign = true
+    offset++
+  }
 
-    if (
-        !options ||
-        typeof options.returnError !== 'function' ||
-        typeof options.returnReply !== 'function'
-    ) {
-        throw new Error('Please provide all return functions while initiating the parser');
+  while (offset < length) {
+    var c1 = parser.buffer[offset++]
+    if (c1 === 13 && parser.buffer[offset] === 10) { // \r\n
+      parser.offset = offset + 1
+      return sign ? -number : number
     }
+    number = (number * 10) + (c1 - 48)
+  }
+}
 
-    if (options.name === 'hiredis') {
-        /* istanbul ignore if: hiredis should always be installed while testing */
-        if (!parsers.hiredis) {
-            msg = 'You explicitly required the hiredis parser but hiredis is not installed. The JS parser is going to be returned instead.';
-        } else if (options.stringNumbers) {
-            msg = 'You explicitly required the hiredis parser in combination with the stringNumbers option. Only the JS parser can handle that and is choosen instead.';
-        }
-    } else if (options.name && !parsers[options.name] && options.name !== 'auto') {
-        msg = 'The requested parser "' + options.name + '" is unkown and the default parser is choosen instead.';
+/**
+ * Used for integer numbers in case of the returnNumbers option
+ * @param parser
+ * @returns {*}
+ */
+function parseStringNumbers (parser) {
+  var offset = parser.offset
+  var length = parser.buffer.length
+  var number = ''
+
+  if (parser.buffer[offset] === 45) {
+    number += '-'
+    offset++
+  }
+
+  while (offset < length) {
+    var c1 = parser.buffer[offset++]
+    if (c1 === 13 && parser.buffer[offset] === 10) { // \r\n
+      parser.offset = offset + 1
+      return number
     }
+    number += c1 - 48
+  }
+}
 
-    if (msg) {
-        console.warn(new Error(msg).stack.replace('Error: ', 'Warning: '));
+/**
+ * Returns a string or buffer of the provided offset start and
+ * end ranges. Checks `optionReturnBuffers`.
+ * @param parser
+ * @param start
+ * @param end
+ * @returns {*}
+ */
+function convertBufferRange (parser, start, end) {
+  // If returnBuffers is active, all return values are returned as buffers besides numbers and errors
+  parser.offset = end + 2
+  if (parser.optionReturnBuffers === true) {
+    return parser.buffer.slice(start, end)
+  }
+
+  return parser.buffer.toString('utf-8', start, end)
+}
+
+/**
+ * Parse a '+' redis simple string response but forward the offsets
+ * onto convertBufferRange to generate a string.
+ * @param parser
+ * @returns {*}
+ */
+function parseSimpleStringViaOffset (parser) {
+  var start = parser.offset
+  var offset = parser.offset
+  var length = parser.buffer.length
+  var buffer = parser.buffer
+
+  while (offset < length) {
+    if (buffer[offset++] === 10) { // \r\n
+      return convertBufferRange(parser, start, offset - 2)
     }
+  }
+}
 
-    options.name = options.name || 'hiredis';
-    options.name = options.name.toLowerCase();
+/**
+ * Returns the string length via parseSimpleNumbers
+ * @param parser
+ * @returns {*}
+ */
+function parseLength (parser) {
+  var string = parseSimpleNumbers(parser)
+  if (string !== undefined) {
+    return +string
+  }
+}
 
-    var innerOptions = {
-        // The hiredis parser expects underscores
-        return_buffers: options.returnBuffers || false,
-        string_numbers: options.stringNumbers || false
-    };
+/**
+ * Parse a ':' redis integer response
+ * @param parser
+ * @returns {*}
+ */
+function parseInteger (parser) {
+  // If stringNumbers is activated the parser always returns numbers as string
+  // This is important for big numbers (number > Math.pow(2, 53)) as js numbers
+  // are 64bit floating point numbers with reduced precision
+  if (parser.optionStringNumbers) {
+    return parseStringNumbers(parser)
+  }
+  return parseSimpleNumbers(parser)
+}
 
-    if (options.name === 'javascript' || !parsers.hiredis || options.stringNumbers) {
-        parser = new parsers.javascript(innerOptions);
+/**
+ * Parse a '$' redis bulk string response
+ * @param parser
+ * @returns {*}
+ */
+function parseBulkString (parser) {
+  var length = parseLength(parser)
+  if (length === undefined) {
+    return
+  }
+  if (length === -1) {
+    return null
+  }
+  var offsetEnd = parser.offset + length
+  if (offsetEnd + 2 > parser.buffer.length) {
+    parser.bigStrSize = offsetEnd + 2
+    parser.bigOffset = parser.offset
+    parser.totalChunkSize = parser.buffer.length
+    parser.bufferCache.push(parser.buffer)
+    return
+  }
+
+  return convertBufferRange(parser, parser.offset, offsetEnd)
+}
+
+/**
+ * Parse a '-' redis error response
+ * @param parser
+ * @returns {Error}
+ */
+function parseError (parser) {
+  var string = parseSimpleStringViaOffset(parser)
+  if (string !== undefined) {
+    if (parser.optionReturnBuffers === true) {
+      string = string.toString()
+    }
+    return new ReplyError(string)
+  }
+}
+
+/**
+ * Parsing error handler, resets parser buffer
+ * @param parser
+ * @param error
+ */
+function handleError (parser, error) {
+  parser.buffer = null
+  parser.returnFatalError(error)
+}
+
+/**
+ * Parse a '*' redis array response
+ * @param parser
+ * @returns {*}
+ */
+function parseArray (parser) {
+  var length = parseLength(parser)
+  if (length === undefined) {
+    return
+  }
+  if (length === -1) {
+    return null
+  }
+
+  var responses = new Array(length)
+  var bufferLength = parser.buffer.length
+  for (var i = 0; i < length; i++) {
+    if (parser.offset >= bufferLength) {
+      return
+    }
+    var response = parseType(parser, parser.buffer[parser.offset++])
+    if (response === undefined) {
+      return
+    }
+    responses[i] = response
+  }
+
+  return responses
+}
+
+/**
+ * Called the appropriate parser for the specified type.
+ * @param parser
+ * @param type
+ * @returns {*}
+ */
+function parseType (parser, type) {
+  switch (type) {
+    case 36: // $
+      return parseBulkString(parser)
+    case 58: // :
+      return parseInteger(parser)
+    case 43: // +
+      return parseSimpleStringViaOffset(parser)
+    case 42: // *
+      return parseArray(parser)
+    case 45: // -
+      return parseError(parser)
+    default:
+      return handleError(parser, new ReplyError('Protocol error, got ' + JSON.stringify(String.fromCharCode(type)) + ' as reply type byte'))
+  }
+}
+
+// All allowed options including their typeof value
+var optionTypes = {
+  returnError: 'function',
+  returnFatalError: 'function',
+  returnReply: 'function',
+  returnBuffers: 'boolean',
+  stringNumbers: 'boolean',
+  name: 'string'
+}
+
+/**
+ * Javascript Redis Parser
+ * @param options
+ * @constructor
+ */
+function JavascriptRedisParser (options) {
+  if (!(this instanceof JavascriptRedisParser)) {
+    return new JavascriptRedisParser(options)
+  }
+  if (!options || !options.returnError || !options.returnReply) {
+    throw new TypeError('Please provide all return functions while initiating the parser')
+  }
+  for (var key in options) {
+    if (typeof options[key] !== optionTypes[key]) {
+      throw new TypeError('The options argument contains the property "' + key + '" that is either unkown or of a wrong type')
+    }
+  }
+  if (options.name === 'hiredis') {
+    /* istanbul ignore next: hiredis is only supported for legacy usage */
+    try {
+      var Hiredis = require('./hiredis')
+      console.error(new TypeError('Using hiredis is discouraged. Please use the faster JS parser by removing the name option.').stack.replace('Error', 'Warning'))
+      return new Hiredis(options)
+    } catch (e) {
+      console.error(new TypeError('Hiredis is not installed. Please remove the `name` option. The (faster) JS parser is used instead.').stack.replace('Error', 'Warning'))
+    }
+  }
+  this.optionReturnBuffers = !!options.returnBuffers
+  this.optionStringNumbers = !!options.stringNumbers
+  this.returnError = options.returnError
+  this.returnFatalError = options.returnFatalError || options.returnError
+  this.returnReply = options.returnReply
+  this.name = 'javascript'
+  this.offset = 0
+  this.buffer = null
+  this.bigStrSize = 0
+  this.bigOffset = 0
+  this.totalChunkSize = 0
+  this.bufferCache = []
+}
+
+/**
+ * Concat a bulk string containing multiple chunks
+ * @param parser
+ * @param buffer
+ * @returns {String}
+ */
+function concatBulkString (parser) {
+  var list = parser.bufferCache
+  // The first chunk might contain the whole bulk string including the \r
+  var chunks = list.length
+  var offset = parser.bigStrSize - parser.totalChunkSize
+  parser.offset = offset
+  if (offset === 1) {
+    if (chunks === 2) {
+      return list[0].toString('utf8', parser.bigOffset, list[0].length - 1)
+    }
+  } else {
+    chunks++
+  }
+  var res = list[0].toString('utf8', parser.bigOffset)
+  for (var i = 1; i < chunks - 2; i++) {
+    // We are only safe to fully add up elements that are neither the first nor any of the last two elements
+    res += list[i].toString()
+  }
+  res += list[i].toString('utf8', 0, offset === 1 ? list[i].length - 1 : offset - 2)
+  return res
+}
+
+/**
+ * Decrease the bufferPool size over time
+ * @returns {undefined}
+ */
+function decreaseBufferPool () {
+  if (bufferPool.length > 96 * 1024) {
+    // Decrease the bufferPool by 16kb
+    bufferPool = bufferPool.slice(0, bufferPool.length - 16 * 1024)
+  } else {
+    clearInterval(interval)
+    interval = null
+  }
+}
+
+/**
+ * Concat the collected chunks from parser.bufferCache
+ * @param parser
+ * @param length
+ * @returns {Buffer}
+ */
+function concatBuffer (parser, length) {
+  var list = parser.bufferCache
+  var pos = 0
+  if (bufferPool.length < length) {
+    bufferPool = new Buffer(length)
+    if (interval === null) {
+      interval = setInterval(decreaseBufferPool, 50)
+    }
+  }
+  for (var i = 0; i < list.length; i++) {
+    list[i].copy(bufferPool, pos)
+    pos += list[i].length
+  }
+  return bufferPool.slice(parser.offset, length)
+}
+
+/**
+ * Parse the redis buffer
+ * @param buffer
+ * @returns {undefined}
+ */
+JavascriptRedisParser.prototype.execute = function (buffer) {
+  if (this.buffer === null) {
+    this.buffer = buffer
+    this.offset = 0
+  } else if (this.bigStrSize === 0) {
+    var oldLength = this.buffer.length
+    var remainingLength = oldLength - this.offset
+    var newLength = remainingLength + buffer.length
+    // ~ 5% speed increase over using new Buffer(length) all the time
+    if (bufferPool.length < newLength) { // We can't rely on the chunk size
+      bufferPool = new Buffer(newLength)
+    }
+    this.buffer.copy(bufferPool, 0, this.offset, oldLength)
+    buffer.copy(bufferPool, remainingLength, 0, buffer.length)
+    this.buffer = bufferPool.slice(0, newLength)
+    this.offset = 0
+  } else if (this.totalChunkSize + buffer.length >= this.bigStrSize) {
+    this.bufferCache.push(buffer)
+    // The returned type might be Array * (42) and in that case we can't improve the parsing currently
+    if (this.optionReturnBuffers === false && this.buffer[this.offset] === 36) {
+      this.returnReply(concatBulkString(this))
+      this.buffer = buffer
+    } else { // This applies for arrays too
+      this.buffer = concatBuffer(this, this.totalChunkSize + buffer.length)
+      this.offset = 0
+    }
+    this.bigStrSize = 0
+    this.totalChunkSize = 0
+    this.bufferCache = []
+  } else {
+    this.bufferCache.push(buffer)
+    this.totalChunkSize += buffer.length
+    return
+  }
+
+  while (this.offset < this.buffer.length) {
+    var offset = this.offset
+    var type = this.buffer[this.offset++]
+    var response = parseType(this, type)
+    if (response === undefined) {
+      this.offset = offset
+      return
+    }
+
+    if (type === 45) {
+      this.returnError(response) // Errors -
     } else {
-        parser = new parsers.hiredis(innerOptions);
+      this.returnReply(response) // Strings + // Integers : // Bulk strings $ // Arrays *
     }
+  }
 
-    parser.returnError = options.returnError;
-    parser.returnFatalError = options.returnFatalError || options.returnError;
-    parser.returnReply = options.returnReply;
-    return parser;
+  this.buffer = null
 }
 
-module.exports = Parser;
+module.exports = JavascriptRedisParser

package.json

 {
   "name": "redis-parser",
-  "version": "1.3.0",
+  "version": "2.0.0",
   "description": "Javascript Redis protocol (RESP) parser",
   "main": "index.js",
   "scripts": {
-    "test": "mocha",
-    "posttest": "jshint . && npm run coverage && npm run coverage:check",
+    "test": "npm run coverage",
+    "benchmark": "node ./benchmark",
+    "posttest": "standard && npm run coverage:check",
     "coverage": "node ./node_modules/istanbul/lib/cli.js cover --preserve-comments ./node_modules/mocha/bin/_mocha -- -R spec",
@@ -31,9 +32,8 @@ "coverage:check": "node ./node_modules/istanbul/lib/cli.js check-coverage --branch 100 --statement 100"
   "devDependencies": {
-    "codeclimate-test-reporter": "^0.1.1",
+    "benchmark": "^2.1.0",
+    "codeclimate-test-reporter": "^0.3.1",
     "intercept-stdout": "^0.1.2",
     "istanbul": "^0.4.0",
-    "jshint": "^2.8.0",
-    "mocha": "^2.3.2"
-  },
-  "optionalDependency": {
+    "standard": "^7.0.1",
+    "mocha": "^2.3.2",
     "hiredis": "^0.4.1"
@@ -40,0 +40,0 @@ },

README.md

 [![Build Status](https://travis-ci.org/NodeRedis/node-redis-parser.png?branch=master)](https://travis-ci.org/NodeRedis/node-redis-parser)
 [![Code Climate](https://codeclimate.com/github/NodeRedis/node-redis-parser/badges/gpa.svg)](https://codeclimate.com/github/NodeRedis/node-redis-parser)
 [![Test Coverage](https://codeclimate.com/github/NodeRedis/node-redis-parser/badges/coverage.svg)](https://codeclimate.com/github/NodeRedis/node-redis-parser/coverage)
+[![js-standard-style](https://img.shields.io/badge/code%20style-standard-brightgreen.svg)](http://standardjs.com/)
 
 # redis-parser
 
-A high performance redis parser solution built for [node_redis](https://github.com/NodeRedis/node_redis) and [ioredis](https://github.com/ioredis/luin).
+A high performance javascript redis parser built for [node_redis](https://github.com/NodeRedis/node_redis) and [ioredis](https://github.com/luin/ioredis). Parses all [RESP](http://redis.io/topics/protocol) data.
 
-Generally all [RESP](http://redis.io/topics/protocol) data will be properly parsed by the parser.
-
 ## Install
@@ -24,3 +23,3 @@
 
-new Parser(options);
+var myParser = new Parser(options);
 ```
@@ -34,4 +33,2 @@
 * `returnBuffers`: *boolean*; optional, defaults to false
-* `name`: *'javascript'|'hiredis'|'auto'|null*; optional, defaults to hiredis and falls back to the js parser if not available or if the stringNumbers option is choosen. Setting this to 'auto' or null is going to automatically determine what parser is available and chooses that one.
-* `stringNumbers`: *boolean*; optional, defaults to false. This is only available for the javascript parser at the moment!
 
@@ -60,4 +57,3 @@ ### Example
         lib.returnFatalError(err);
-    },
-    name: 'auto' // This returns either the hiredis or the js parser instance depending on what's available
+    }
 });
@@ -76,3 +72,3 @@
 
-If you handle big numbers, you should pass the `stringNumbers` option. That case numbers above 2^53 can be handled properly without reduced precision.
+Big numbers that are too large for JS are automatically stringified.
 
@@ -89,4 +85,2 @@ ```js
     },
-    name: 'javascript', // Use the Javascript parser
-    stringNumbers: true, // Return all numbers as string instead of a js number
     returnBuffers: true // All strings are returned as buffer e.g. <Buffer 48 65 6c 6c 6f>
@@ -98,20 +92,9 @@ });
 
-## Further info
-
-The [hiredis](https://github.com/redis/hiredis) parser is still the fasted parser for
-Node.js and therefor used as default in redis-parser if the hiredis parser is available.
-
-Otherwise the pure js NodeRedis parser is choosen that is almost as fast as the
-hiredis parser besides some situations in which it'll be a bit slower.
-
 ## Protocol errors
 
-To handle protocol errors (this is very unlikely to happen) gracefuly you should add the returnFatalError option, reject any still running command (they might have been processed properly but the reply is just wrong), destroy the socket and reconnect.
-Otherwise a chunk might still contain partial data of a following command that was already processed properly but answered in the same chunk as the command that resulted in the protocol error.
+To handle protocol errors (this is very unlikely to happen) gracefully you should add the returnFatalError option, reject any still running command (they might have been processed properly but the reply is just wrong), destroy the socket and reconnect. Note that while doing this no new command may be added, so all new commands have to be buffered in the meantime, otherwise a chunk might still contain partial data of a following command that was already processed properly but answered in the same chunk as the command that resulted in the protocol error.
 
 ## Contribute
 
-The js parser is already optimized but there are likely further optimizations possible.
-Besides running the tests you'll also have to run the change at least against the node_redis benchmark suite and post the improvement in the PR.
-If you want to write a own parser benchmark, that would also be great!
+The parser is highly optimized but there may still be further optimizations possible.
 
@@ -121,10 +104,3 @@ ```
 npm test
-
-# Run node_redis benchmark (let's guess you cloned node_redis in another folder)
-cd ../redis
-npm install
-npm run benchmark parser=javascript > old.log
-# Replace the changed parser in the node_modules
-npm run benchmark parser=javascript > new.log
-node benchmarks/diff_multi_bench_output.js old.log new.log > improvement.log
+npm run benchmark
 ```
@@ -131,0 +107,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Deprecated

Maintenance

The maintainer of the package marked it as deprecated. This could indicate that a single version should not be used, or that the package is no longer maintained and any new vulnerabilities will not be fixed.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Worsened metrics

Total package byte prevSize

19793

decreased by-48.57%

Dev dependency count

7

increased by40%

Number of package files

9

decreased by-30.77%

Lines of code

439

decreased by-34.87%

Number of medium maintenance alerts

1

increased byInfinity%

Number of lines in readme file

104

decreased by-18.75%

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes