memcached - npm Package Compare versions

Comparing version 0.2.3 to 0.2.4

CHANGELOG.md

@@ -1,2 +0,9 @@
-0.2.3
+### 0.2.4
+  - Tons of fixes have been made to the way we do error handling and failover,
+    this includes better reconnect, server failure detection, timeout handling
+    and much more.
+  - Introduction of a new `idle` timeout option.
+  - Documentation improvements.
+
+### 0.2.3
   - Added documentation for public api's
@@ -6,48 +13,48 @@ - new namespace option added that namespaces all your keys.
 
-0.2.2
+### 0.2.2
  - Support for touch command #86
  - Fix for chunked responses from the server #84
 
-0.2.1
+### 0.2.1
  - Supports for a queued callback limit so it would crash the process when we queue
    to much callbacks. #81
 
-0.2.0
+### 0.2.0
  - [breaking] We are now returning Error instances instead of strings for errors
  - Dependency bump for a critical bug in our connection pool.
 
-0.1.5
+### 0.1.5
  - Don't execute callbacks multiple times if the connection fails
- - Parser fix for handling server responses that contain Memcached Procotol
+ - Parser fix for handling server responses that contain Memcached Protocol
    keywords
  - Make sure that the retry option is set correctly
 
-0.1.4
+### 0.1.4
  - Added missing error listener to the 3rd-Eden/jackpot module, this prevents crashes
    when it's unable to connect to a server.
 
-0.1.3
+### 0.1.3
  - Handle Memcached responses that contain no value.
  - Travis CI integration.
 
-0.1.2
- - Returning an error when the Memcached server issues a NOT_STORED response.
+### 0.1.2
+ - Returning an error when the Memcached server issues a `NOT_STORED` response.
 
-0.1.1
+### 0.1.1
  - Now using 3rd-Eden/jackpot as connection pool, this should give a more stable
    connection.
 
-0.1.0
+### 0.1.0
  - Storing numeric values are now returned as numeric values, they are no
    longer strings.
 
-0.0.12
- - Added .setEncoding to the connections, this way UTF-8 chars will not be
+### 0.0.12
+ - Added `.setEncoding` to the connections, this way UTF-8 chars will not be
    chopped in to multiple pieces and breaking binary stored data or UTF-8 text
 
-0.0.11
+### 0.0.11
  - Added more useful error messages instead of returning false. Please note
    that they are still strings instead of Error instances (legacy)
 
-0.0.10
+### 0.0.10
  - Compatibility with Node.js 0.8
@@ -57,4 +64,4 @@ - Don't saturate the Node process by retrying to connect if pool is full #43
 
-0.0.9
- - Codestyle refactor, named the functions, removed tabs
+### 0.0.9
+ - Code style refactor, named the functions, removed tabs
  - Added Mocha test suite

lib/connection.js

@@ -28,4 +28,6 @@ "use strict";
   this.failed = false;
+  this.locked = false;
+  this.isScheduledToReconnect = false;
 
-  this.totalRetries = 0;
+  this.totalFailures = 0;
   this.retry = 0;
@@ -46,4 +48,4 @@ this.totalReconnectsAttempted = 0;
   this.messages.push(message || 'No message specified');
-
-  if (this.retries) {
+  if (this.failures && !this.locked) {
+    this.locked = true;
     setTimeout(issue.attemptRetry.bind(issue), this.retry);
@@ -55,3 +57,6 @@ return this.emit('issue', this.details);
 
-  setTimeout(issue.attemptReconnect.bind(issue), this.reconnect);
+  if (!this.isScheduledToReconnect) {
+      this.isScheduledToReconnect = true;
+      setTimeout(issue.attemptReconnect.bind(issue), this.reconnect);
+  }
 };
@@ -63,9 +68,9 @@
 
-    res.server = this.serverAddress;
+    res.server = this.server;
     res.tokens = this.tokens;
     res.messages = this.messages;
 
-    if (this.retries) {
-      res.retries = this.retries;
-      res.totalRetries = this.totalRetries;
+    if (this.failures) {
+      res.failures = this.failures;
+      res.totalFailures = this.totalFailures;
     } else {
@@ -75,3 +80,3 @@ res.totalReconnectsAttempted = this.totalReconnectsAttempted;
       res.totalReconnectsFailed = this.totalReconnectsAttempted - this.totalReconnectsSuccess;
-      res.totalDownTime = (res.totalReconnectsFailed * this.reconnect) + (this.totalRetries * this.retry);
+      res.totalDownTime = (res.totalReconnectsFailed * this.reconnect) + (this.totalFailures * this.retry);
     }
@@ -84,5 +89,6 @@
 issues.attemptRetry = function attemptRetry () {
-  this.totalRetries++;
-  this.retries--;
+  this.totalFailures++;
+  this.failures--;
   this.failed = false;
+  this.locked = false;
 };
@@ -108,2 +114,3 @@
     issue.failed = false;
+    issue.isScheduledToReconnect = false;
 
@@ -110,0 +117,0 @@ // we connected again, so we are going through the whole cycle again

lib/memcached.js

@@ -6,4 +6,3 @@ "use strict";
  */
-var EventEmitter = require('events').EventEmitter
-  , Stream = require('net').Stream
+var Stream = require('net').Stream
   , Socket = require('net').Socket;
@@ -62,3 +61,2 @@
   Utils.merge(this, options);
-  EventEmitter.call(this);
 
@@ -81,6 +79,13 @@ this.servers = servers;
   , poolSize: 10            // maximal parallel connections
+  , retries: 5              // Connection pool retries to pull connection from pool
+  , factor: 3               // Connection pool retry exponential backoff factor
+  , minTimeout: 1000        // Connection pool retry min delay before retrying
+  , maxTimeout: 60000       // Connection pool retry max delay before retrying
+  , randomize: false        // Connection pool retry timeout randomization
+
   , reconnect: 18000000     // if dead, attempt reconnect each xx ms
   , timeout: 5000           // after x ms the server should send a timeout if we can't connect
-  , retries: 5              // amount of retries before server is dead
-  , retry: 30000            // timeout between retries, all call will be marked as cache miss
+  , failures: 5             // Number of times a server can have an issue before marked dead
+  , retry: 30000            // When a server has an error, wait this amount of time before retrying
+  , idle: 5000              // Remove connection from pool when no I/O after `idle` ms
   , remove: false           // remove server if dead if false, we will attempt to reconnect
@@ -104,3 +109,5 @@ , redundancy: false       // allows you do re-distribute the keys over a x amount of servers
 
-  var memcached = nMemcached.prototype = new EventEmitter
+  nMemcached.prototype.__proto__ = require('events').EventEmitter.prototype;
+
+  var memcached = nMemcached.prototype
     , privates = {}
@@ -140,3 +147,9 @@ , undefined;
     manager.retries = memcached.retries;
+    manager.factor = memcached.factor;
+    manager.minTimeout = memcached.minTimeout;
+    manager.maxTimeout = memcached.maxTimeout;
+    manager.randomize = memcached.randomize;
 
+    manager.setMaxListeners(0);
+
     manager.factory(function factory() {
@@ -146,3 +159,10 @@ var S = Array.isArray(serverTokens)
           : new Socket
-        , Manager = this;
+        , Manager = this
+        , idleTimeout = function() {
+            Manager.remove(this);
+          }
+        , streamError = function(e) {
+            memcached.connectionIssue(e.toString(), S);
+            Manager.remove(this);
+        };
 
@@ -167,4 +187,10 @@ // config the Stream
         , data: curry(memcached, privates.buffer, S)
-        , timeout: function streamTimeout() {
-            Manager.remove(this);
+        , connect: function streamConnect() {
+            // Jackpot handles any pre-connect timeouts by calling back
+            // with the error object.
+            this.setTimeout(this.memcached.idle, idleTimeout);
+            // Jackpot handles any pre-connect errors, but does not handle errors
+            // once a connection has been made, nor does Jackpot handle releasing
+            // connections if an error occurs post-connect
+            this.on('error', streamError);
           }
@@ -272,4 +298,6 @@ , end: S.end
 
-    // check if the server is still alive
-    if (server in this.issues && this.issues[server].failed) {
+    // check if any server exists or and if the server is still alive
+    // a server may not exist if the manager was never able to connect
+    // to any server.
+    if (!server || (server in this.issues && this.issues[server].failed)) {
       return query.callback && memcached.makeCallback(query.callback,new Error('Server not available'));
@@ -286,7 +314,17 @@ }
       // check for issues
-      if (error) return query.callback && memcached.makeCallback(query.callback,error);
-      if (!S) return query.callback && memcached.makeCallback(query.callback,new Error('Connect did not give a server'));
+      if (error) {
+        memcached.connectionIssue(error.toString(), S);
+        return query.callback && memcached.makeCallback(query.callback,error);
+      }
 
+      if (!S) {
+        var message = 'Connect did not give a server';
+        memcached.connectionIssue(message);
+        return query.callback && memcached.makeCallback(query.callback,new Error(message));
+      }
+
       if (S.readyState !== 'open') {
-        return query.callback && memcached.makeCallback(query.callback,new Error('Connection readyState is set to ' + S.readySate));
+        var message = 'Connection readyState is set to ' + S.readyState;
+        memcached.connectionIssue(message, S);
+        return query.callback && memcached.makeCallback(query.callback,new Error(message));
       }
@@ -338,3 +376,3 @@
         , reconnect: this.reconnect
-        , retries: this.retries
+        , failures: this.failures
         , retry: this.retry
@@ -367,2 +405,3 @@ , remove: this.remove
               memcached.HashRing.removeServer(server);
+              memcached.emit('failure', details);
             }
@@ -369,0 +408,0 @@ }

package.json

 {
     "name": "memcached"
-  , "version": "0.2.3"
+  , "version": "0.2.4"
   , "author": "Arnout Kazemier"
@@ -38,3 +38,3 @@ , "description": "A fully featured Memcached API client, supporting both single and clustered Memcached servers through consistent hashing and failover/failure. Memcached is rewrite of nMemcached, which will be deprecated in the near future."
         "hashring": "0.0.x"
-      , "jackpot": ">=0.0.2"
+      , "jackpot": ">=0.0.6"
     }
@@ -41,0 +41,0 @@ , "devDependencies": {

README.md

@@ -1,2 +0,2 @@
-#Memcached [![Build Status](https://secure.travis-ci.org/3rd-Eden/node-memcached.png?branch=master)](http://travis-ci.org/3rd-Eden/node-memcached)
+# Memcached [![Build Status](https://secure.travis-ci.org/3rd-Eden/node-memcached.png?branch=master)](http://travis-ci.org/3rd-Eden/node-memcached)
 
@@ -101,4 +101,6 @@ `memcached` is a fully featured Memcached client for Node.js. `memcached` is
   connect. This will also be used close the connection if we are idle.
-* `retries`: *5*, amount of tries before we mark the server as dead.
-* `retry`: *30000*, timeout between each retry in x milliseconds.
+* `retries`: *5*, How many times to retry socket allocation for given request
+* `failures`: *5*, Number of times a server may have issues before marked dead.
+* `retry`: *30000*, time to wait between failures before putting server back in
+  service.
 * `remove`: *false*, when the server is marked as dead you can remove it from
@@ -111,2 +113,3 @@ the pool so all other will receive the keys instead.
   maxKeySize option.
+* `idle`: *5000*, the idle timeout for the connections.
 
@@ -208,3 +211,3 @@ Example usage:
 ```js
-memcached.set('foo', 10, 'bar', function (err) {
+memcached.set('foo', 'bar', 10, function (err) {
   // stuff
@@ -226,3 +229,3 @@ });
 ```js
-memcached.replaces('foo', 10, 'bar', function (err) {
+memcached.replace('foo', 'bar', 10, function (err) {
   // stuff
@@ -542,5 +545,5 @@ });
 
-* `retries`: the amount of retries left before we mark the server as dead.
-* `totalRetries`: the total amount of retries we did on this server, as when the
-  server has been reconnected after it's dead the `retries` will be rest to
+* `failures`: the amount of failures left before we mark the server as dead.
+* `totalFailures`: the total amount of failures that occurred on this server, as when the
+  server has been reconnected after it's dead the `failures` will be rest to
   defaults and messages will be removed.
@@ -575,1 +578,14 @@
 ```
+
+# Contributors
+
+This project wouldn't be possible without the hard work of our amazing
+contributors. See the contributors tab in Github for an up to date list of
+[contributors](/3rd-Eden/node-memcached/graphs/contributors).
+
+Thanks for all your hard work on this project!
+
+# License
+
+The driver is released under the MIT license. See the
+[LICENSE](/3rd-Eden/node-memcached/blob/master/LICENSE) for more information.

test/memcached-connections.test.js

@@ -36,2 +36,199 @@ //global it
   });
+  it('should remove a failed server', function(done) {
+    var memcached = new Memcached('127.0.1:1234', {
+      timeout: 1000,
+      retries: 0,
+      failures: 0,
+      retry: 100,
+      remove: true });
+
+    this.timeout(60000);
+
+    memcached.get('idontcare', function (err) {
+        function noserver() {
+          memcached.get('idontcare', function(err) {
+              throw err;
+          });
+        };
+        assert.throws(noserver, /Server not available/);
+        memcached.end();
+        done();
+    });
+  });
+  it('should rebalance to remaining healthy server', function(done) {
+    var memcached = new Memcached(['127.0.1:1234', common.servers.single], {
+      timeout: 1000,
+      retries: 0,
+      failures: 0,
+      retry: 100,
+      remove: true,
+      redundancy: true });
+
+    this.timeout(60000);
+
+    // 'a' goes to fake server. first request will cause server to be removed
+    memcached.get('a', function (err) {
+      // second request should be rebalanced to healthy server
+      memcached.get('a', function (err) {
+        assert.ifError(err);
+        memcached.end();
+        done();
+      });
+    });
+  });
+  it('should properly schedule failed server retries', function(done) {
+    var server = '127.0.0.1:1234';
+    var memcached = new Memcached(server, {
+      retries: 0,
+      failures: 5,
+      retry: 100 });
+
+    // First request will schedule a retry attempt, and lock scheduling
+    memcached.get('idontcare', function (err) {
+      assert.throws(function() { throw err }, /connect ECONNREFUSED/);
+      assert.deepEqual(memcached.issues[server].failures, 5);
+      assert.deepEqual(memcached.issues[server].locked, true);
+      assert.deepEqual(memcached.issues[server].failed, true);
+      // Immediate request should not decrement failures
+      memcached.get('idontcare', function(err) {
+        assert.throws(function() { throw err }, /Server not available/);
+        assert.deepEqual(memcached.issues[server].failures, 5);
+      assert.deepEqual(memcached.issues[server].locked, true);
+      assert.deepEqual(memcached.issues[server].failed, true);
+        // Once `retry` time has passed, failures should decrement by one
+        setTimeout(function() {
+          // Server should be back in action
+          assert.deepEqual(memcached.issues[server].locked, false);
+          assert.deepEqual(memcached.issues[server].failed, false);
+          memcached.get('idontcare', function(err) {
+            // Server should be marked healthy again, though we'll get this error
+            assert.throws(function() { throw err }, /connect ECONNREFUSED/);
+            assert.deepEqual(memcached.issues[server].failures, 4);
+            memcached.end();
+            done();
+          });
+        }, 100); // `retry` is 100 so wait 100
+      });
+    });
+  });
+  it('should properly schedule server reconnection attempts', function(done) {
+    var server = '127.0.0.1:1234'
+    , memcached = new Memcached(server, {
+      retries: 3,
+      minTimeout: 0,
+      maxTimeout: 100,
+      failures: 0,
+      reconnect: 100 })
+    , reconnectAttempts = 0;
+
+    memcached.on('reconnecting', function() {
+      reconnectAttempts++;
+    });
+
+    // First request will mark server dead and schedule reconnect
+    memcached.get('idontcare', function (err) {
+      assert.throws(function() { throw err }, /connect ECONNREFUSED/);
+      // Second request should not schedule another reconnect
+      memcached.get('idontcare', function (err) {
+        assert.throws(function() { throw err }, /Server not available/);
+        // Allow enough time to pass for a connection retries to occur
+        setTimeout(function() {
+          assert.deepEqual(reconnectAttempts, 1);
+          memcached.end();
+          done();
+        }, 400);
+      });
+    });
+  });
+  it('should reset failures after reconnecting to failed server', function(done) {
+    var server = '127.0.0.1:1234'
+    , memcached = new Memcached(server, {
+      retries: 0,
+      minTimeout: 0,
+      maxTimeout: 100,
+      failures: 1,
+      retry: 1,
+      reconnect: 100 })
+
+    this.timeout(60000);
+
+    // First request will mark server failed
+    memcached.get('idontcare', function(err) {
+      assert.throws(function() { throw err }, /connect ECONNREFUSED/);
+      // Wait 10ms, server should be back online
+      setTimeout(function() {
+        // Second request will mark server dead
+        memcached.get('idontcare', function(err) {
+          assert.throws(function() { throw err }, /connect ECONNREFUSED/);
+            // Third request should find no servers
+            memcached.get('idontcare', function(err) {
+            assert.throws(function() { throw err }, /Server not available/);
+              // Give enough time for server to reconnect
+              setTimeout(function() {
+                // Server should be reconnected, but expect ECONNREFUSED
+                memcached.get('idontcare', function(err) {
+                  assert.throws(function() { throw err }, /connect ECONNREFUSED/);
+                  assert.deepEqual(memcached.issues[server].failures,
+                    memcached.issues[server].config.failures);
+                  memcached.end();
+                  done();
+                });
+              }, 150);
+            });
+          });
+      },10);
+    });
+  });
+  it('should return error on connection timeout', function(done) {
+    // Use a non routable IP
+    var server = '10.255.255.255:1234'
+    , memcached = new Memcached(server, {
+      retries: 0,
+      timeout: 100,
+      idle: 1000,
+      failures: 0 });
+
+    memcached.get('idontcare', function(err) {
+      assert.throws(function() { throw err }, /Timed out while trying to establish connection/);
+      memcached.end();
+      done();
+    });
+  });
+  it('should remove connection when idle', function(done) {
+    var memcached = new Memcached(common.servers.single, {
+      retries: 0,
+      timeout: 100,
+      idle: 100,
+      failures: 0 });
+
+    memcached.get('idontcare', function(err) {
+      assert.deepEqual(memcached.connections[common.servers.single].pool.length, 1);
+      setTimeout(function() {
+        assert.deepEqual(memcached.connections[common.servers.single].pool.length, 0);
+        memcached.end();
+        done();
+      }, 100);
+    });
+  });
+  it('should remove server if error occurs after connection established', function(done) {
+    var memcached = new Memcached(common.servers.single, {
+      poolSize: 1,
+      retries: 0,
+      timeout: 1000,
+      idle: 5000,
+      failures: 0 });
+
+    // Should work fine
+    memcached.get('idontcare', function(err) {
+      assert.ifError(err);
+      // Fake an error on the connected socket which should mark server failed
+      var S = memcached.connections[common.servers.single].pool.pop();
+      S.emit('error', new Error('Dummy error'));
+      memcached.get('idontcare', function(err) {
+        assert.throws(function() { throw err; }, /Server not available/);
+        done();
+      });
+    });
+
+  });
 });

New alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Found 1 instance in 1 package

Fixed alerts

URL strings

Supply chain risk

Package contains fragments of external URLs or IP addresses, which the package may be accessing at runtime.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

495259

8.71%

Lines of code

3763

39.47%

Number of lines in readme file

586

2.81%

Worsened metrics

Number of medium supply chain risk alerts

7

40%

Dependency changes

• Updated

  jackpot@>=0.0.6