redis-clustr - npm Package Compare versions

Comparing version 1.0.2 to 1.1.0

package.json

 {
   "name": "redis-clustr",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "description": "Redis cluster client",
@@ -17,4 +17,4 @@ "main": "src/redisClustr.js",
   "repository": {
-    "type" : "git",
-    "url" : "git://github.com/gosquared/redis-clustr.git"
+    "type": "git",
+    "url": "git://github.com/gosquared/redis-clustr.git"
   },
@@ -25,3 +25,3 @@ "scripts": {
   "dependencies": {
-    "redis": "^0.12.1"
+    "redis": "^2.3.0"
   },
@@ -28,0 +28,0 @@ "optionalDependencies": {

README.md

@@ -50,14 +50,18 @@ # redis-clustr
 
-## Supported functionality/limitations
+## Supported functionality
 
 ### Slot reallocation
 
-Supported - when a response is given with a `MOVED` error, we will immediately re-issue the command on the other server and run another `cluster slots` to get the new slot allocations. `ASK` redirection is also supported - we wil re-issue the command without updating the slots.
+Supported - when a response is given with a `MOVED` error, we will immediately re-issue the command on the other server and run another `cluster slots` to get the new slot allocations. `ASK` redirection is also supported - we wil re-issue the command without updating the slots. `TRYAGAIN` responses will be retried automatically.
 
-### Multi / Exec
+### Multi / Exec (Batch)
 
-Multi commands are *supported* but treated as a batch of commands (not an actual multi) and the response is recreated in the original order.
+Multi commands are *supported* but treated as a batch of commands (not an actual multi) and the response is recreated in the original order. Commands are grouped by node and sent as [node_redis batches](https://github.com/NodeRedis/node_redis#clientbatchcommands)
 
 ### Multi-key commands (`del`, `mget`)
 
-Multi-key commands are also supported and will split into individual commands then have the response recreated as an array. This means that `del` will get a response of `[ 1, 1 ]`  when deleting two keys instead of `2`.
+Multi-key commands are also supported and will be split into individual commands (using a batch) then have the response recreated as an array. This means that `del` will get a response of `[ 1, 1 ]`  when deleting two keys instead of `2`.
+
+### Errors
+
+Just like node_redis, listen to the `error` event to stop your application from crashing due to errors. We automatically intercept connection errors and try to reconnect to the server.

src/crc16-xmodem.js

@@ -43,5 +43,5 @@ // quickly ported from https://github.com/antirez/redis-rb-cluster/blob/master/crc16.rb
   for (var i = 0; i < buf.length; i++) {
-    crc = ((crc<<8) & 0xffff) ^ lookup[((crc>>8)^buf[i]) & 0xff]
+    crc = ((crc << 8) & 0xffff) ^ lookup[((crc >> 8) ^ buf[i]) & 0xff];
   }
   return crc;
 };

src/RedisBatch.js

@@ -24,4 +24,16 @@ var setupCommands = require('./setupCommands');
 
-  if (!cb) cb = function(){};
+  if (!cb) {
+    cb = function(err) {
+      if (err) self.cluster.emit('error', err);
+    };
+  }
 
+  if (!self.cluster.slots.length) {
+    self.cluster.getSlots(function(err) {
+      if (err) return cb(err);
+      self.exec(cb);
+    });
+    return;
+  }
+
   var todo = self.queue.length;
@@ -37,8 +49,12 @@ var resp = new Array(self.queue.length);
 
+  var batches = {};
+
   self.queue.forEach(function(op, index) {
-    var cmd = self.cluster[op[0]];
+    var cmd = op[0];
     var keys = Array.prototype.slice.call(op[1]);
 
-    var cb = false;
-    if (typeof keys[keys.length -1] === 'function') cb = keys.pop();
+    var cb = function(err) {
+      if (err) self.cluster.emit('error', err);
+    };
+    if (typeof keys[keys.length - 1] === 'function') cb = keys.pop();
 
@@ -50,4 +66,7 @@ var first = keys[0];
 
-    keys.push(function(err, res) {
-      if (cb) cb.apply(this, arguments);
+    var cli = self.cluster.selectClient(keys);
+    var b = batches[cli.address] || (batches[cli.address] = cli.batch());
+
+    self.cluster.commandCallback(cli, cmd, keys, function(err, res) {
+      cb.apply(this, arguments);
       if (err) {
@@ -61,4 +80,6 @@ if (!errors) errors = [];
 
-    cmd.apply(self.cluster, keys);
+    b[cmd].apply(b, keys);
   });
+
+  for (var i in batches) batches[i].exec();
 };

src/redisClustr.js

@@ -51,3 +51,9 @@ var setupCommands = require('./setupCommands');
   cli.on('error', function(err) {
-    if (/Redis connection to .* failed.*/.test(err.message)) {
+    if (
+      err.code === 'CONNECTION_BROKEN' ||
+      err.code === 'UNCERTAIN_STATE' ||
+      /Redis connection to .* failed.*/.test(err.message)
+    ) {
+      // broken connection so force a new client to be created, otherwise node_redis will reconnect
+      if (err.code === 'CONNECTION_BROKEN') self.connections[name] = null;
       self.emit('connectionError', err, cli);
@@ -62,3 +68,4 @@ self.getSlots();
 
-  return self.connections[name] = cli;
+  self.connections[name] = cli;
+  return cli;
 };
@@ -90,6 +97,8 @@
     self._slotQ = false;
-  }
+  };
 
   var exclude = [];
   var tryClient = function() {
+    if (self.quitting) return runCbs(new Error('cluster is quitting'));
+
     var client = self.getRandomConnection(exclude);
@@ -141,12 +150,14 @@ if (!client) return runCbs(new Error('couldn\'t get slot allocation'));
   if (Array.isArray(key)) key = key[0];
+  if (Buffer.isBuffer(key)) key = key.toString();
 
   // support for hash tags to keep keys on the same slot
-  // http://redis.io/topics/cluster-spec#multiple-keys-operations
+  // http://redis.io/topics/cluster-spec#keys-hash-tags
   var openKey = key.indexOf('{');
   if (openKey !== -1) {
-    var closeKey = key.indexOf('}');
+    var tmpKey = key.substring(openKey + 1);
+    var closeKey = tmpKey.indexOf('}');
 
     // } in key and it's not {}
-    if (closeKey !== -1 && closeKey !== openKey + 1) {
-      key = key.substring(openKey + 1, closeKey);
+    if (closeKey > 0) {
+      key = tmpKey.substring(0, closeKey);
     }
@@ -167,10 +178,34 @@ }
 
-  var cb = function(){};
-  if (typeof args[args.length - 1] === 'function') cb = args.pop();
+  var cb = function(err) {
+    if (err) self.emit('error', err);
+  };
 
+  var argsCb = typeof args[args.length - 1] === 'function';
+  if (argsCb) {
+    cb = args[args.length - 1];
+  }
+
   if (!key) return cb(new Error('no key for command: ' + cmd));
 
+  if (!self.slots.length) {
+    self.getSlots(function(err) {
+      if (err) return cb(err);
+      self.command(cmd, args);
+    });
+    return;
+  }
+
+  // now take cb off args so we can attach our own callback wrapper
+  if (argsCb) args.pop();
+
   var r = self.selectClient(key);
   if (!r) return cb(new Error('couldn\'t get client'));
 
+  self.commandCallback(r, cmd, args, cb);
+  r[cmd].apply(r, args);
+};
+
+RedisClustr.prototype.commandCallback = function(cli, cmd, args, cb) {
+  var self = this;
+
   // number of attempts/redirects when we get connection errors
@@ -183,7 +218,7 @@ // or when we get MOVED/ASK responses
     if (err && err.message && retries--) {
-      var moved = err.message.substr(0, 6) === 'MOVED ';
-      var ask = err.message.substr(0, 4) === 'ASK ';
+      var msg = err.message;
+      var ask = msg.substr(0, 4) === 'ASK ';
+      var moved = !ask && msg.substr(0, 6) === 'MOVED ';
 
       if (moved || ask) {
-
         // key has been moved!
@@ -195,3 +230,3 @@ // lets refetch slots from redis to get an up to date allocation
         var addr = err.message.split(' ')[2];
-        var saddr = addr.split(':')
+        var saddr = addr.split(':');
         var c = self.getClient(saddr[1], saddr[0]);
@@ -203,12 +238,8 @@ if (ask) c.send_command('asking', []);
 
-      if (/Redis connection to .* failed.*/.test(err.message)) {
-        self.emit('connectionError', err, r);
-
-        // get slots and try again
-        self.getSlots(function(err) {
-          if (err) return cb(err);
-          var r = self.selectClient(key);
-          if (!r) return cb(new Error('couldn\'t get client'));
-          r[cmd].apply(r, args);
-        });
+      var tryAgain = msg.substr(0, 8) === 'TRYAGAIN';
+      if (tryAgain || err.code === 'CLUSTERDOWN') {
+        // TRYAGAIN response or cluster down, retry with backoff up to 1280ms
+        setTimeout(function() {
+          cli[cmd].apply(cli, args);
+        }, Math.pow(2, 16 - Math.max(retries, 9)) * 10);
         return;
@@ -220,4 +251,2 @@ }
   });
-
-  r[cmd].apply(r, args);
 };
@@ -229,6 +258,8 @@
 
-  var cb = function(){};
+  var cb = function(err) {
+    if (err) self.emit('error', err);
+  };
 
   var keys = Array.prototype.slice.call(args);
-  if (typeof keys[keys.length -1] === 'function') cb = keys.pop();
+  if (typeof keys[keys.length - 1] === 'function') cb = keys.pop();
 
@@ -264,8 +295,12 @@ var first = keys[0];
   var todo = Object.keys(self.connections).length;
+  self.quitting = true;
 
-  for (var i in self.connections) {
-    self.connections[i].quit(function() {
-      if (!--todo && cb) cb();
-    });
-  }
+  var errs = null;
+  var quitCb = function(err) {
+    if (err && !errs) errs = [];
+    if (err) errs.push(err);
+    if (!--todo && cb) cb(errs);
+  };
+
+  for (var i in self.connections) self.connections[i].quit(quitCb);
 };

test/cluster.js

 // not really tests but being used to ensure things are working
-var RedisClustr = require('../index');
+var RedisClustr = require('../src/redisClustr');
 
 var c = new RedisClustr({
-  clients: [
+  servers: [
     {
-      port: 7000,
+      port: 7007,
       host: '127.0.0.1'
@@ -38,17 +38,27 @@ },
   b.get('hi' + runs);
+  b.set('hi' + (runs + 1), runs);
   b.exec(function(err, resp) {
     console.log(err, resp);
-    c.del('hi' + (runs - 1));
+    c.del('hi' + (runs - 1), 'hi' + (runs - 2));
     runs++;
     run();
-  })
+  });
 };
 
+// c.getSlots(run);
 run();
 
+// setTimeout(function() {
+//   c.quit(function() {
+//     console.log('QUIIITTT', arguments);
+//   });
+// }, 1500);
 
-c.del('hi', 'test', 'lol', function() {
-  c.del('hi', 'test', 'lol', console.log);
-});
+c.on('error', console.log.bind(console, 'Error'));
 
 
+// c.del('hi', 'test', 'lol', function() {
+//   c.del('hi', 'test', 'lol', console.log);
+// });
+
+

test/main.js

 // not really tests but being used to ensure things are working
-var RedisClustr = require('../index');
+var RedisClustr = require('../src/redisClustr');
 
 var c = new RedisClustr({
-  clients: [
+  servers: [
     {
@@ -7,0 +7,0 @@ port: 6380,

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

22559

increased by16.66%

Number of package files

12

increased by9.09%

Lines of code

647

increased by9.48%

Number of lines in readme file

67

increased by6.35%

Dependency changes

• + Addeddouble-ended-queue@2.1.0-0(transitive)

• + Addedredis@2.8.0(transitive)

• + Addedredis-commands@1.7.0(transitive)

• + Addedredis-parser@2.6.0(transitive)

• - Removedredis@0.12.1(transitive)

• Updatedredis@^2.3.0