limitd-client - npm Package Compare versions

Comparing version 2.9.1 to 2.10.0

package.json

 {
   "name": "limitd-client",
-  "version": "2.9.1",
+  "version": "2.10.0",
   "description": "limitd client for node.js",
@@ -15,2 +15,3 @@ "main": "index.js",
     "disyuntor": "^2.0.0",
+    "hashring": "^3.2.0",
     "length-prefixed-message": "^3.0.3",
@@ -20,3 +21,3 @@ "length-prefixed-stream": "~1.4.0",
     "lodash": "~3.7.0",
-    "murmurhash3js": "^3.0.1",
+    "ms": "^1.0.0",
     "reconnect-net": "0.0.0",
@@ -31,3 +32,4 @@ "retry": "^0.10.1",
     "nyc": "^10.2.0",
-    "open": "github:jfromaniello/node-open#add_bin"
+    "open": "github:jfromaniello/node-open#add_bin",
+    "proxyquire": "^1.7.11"
   },
@@ -42,2 +44,2 @@ "homepage": "https://github.com/limitd/node-client",
   }
-}
+}

shard_client.js

 const LimitdClient = require('./client');
 const EventEmitter = require('events').EventEmitter;
+const Hashring = require('hashring');
 
 const _      = require('lodash');
-const murmur = require('murmurhash3js').x86.hash32;
 const async  = require('async');
@@ -11,3 +11,3 @@ const dns    = require('dns');
 
-const REFRESH_AFTER_MS = 1000 * 60 * 5;
+const ms = require('ms');
 
@@ -29,13 +29,15 @@ const defaults = {
   if (Array.isArray(this._options.shard.hosts)) {
-    this.clients = _.sortBy(this._options.shard.hosts).map(host => {
+    this.clients = this._options.shard.hosts.reduce((result, host) => {
       if (url.parse(host).protocol === null) {
-        return this.createClient(`limitd://${host}:${this._options.port}`);
+        result[`${host}:${this._options.port}`] = this.createClient(`limitd://${host}:${this._options.port}`);
       } else {
-        return this.createClient(host);
+        result[host.replace(/^(.*)\/\//, '')] = this.createClient(host);
       }
-    });
+      return result;
+    }, {});
+    this.ring = new Hashring(Object.keys(this.clients));
   } else if (this._options.shard.autodiscover) {
-    this.autodiscover = this._options.shard.autodiscover;
-    this.clients = [];
-    this.currentHosts = [];
+    this.autodiscover = _.extend({ refreshInterval: ms('5m') }, this._options.shard.autodiscover);
+    this.clients = {};
+    this.ring = new Hashring([]);
     this.discover();
@@ -67,4 +69,4 @@ } else {
 ShardClient.prototype.discover = function() {
-  dns.resolve(this.autodiscover.address, this.autodiscover.type || 'A', (err, addresses) => {
-    setTimeout(() => this.discover(), REFRESH_AFTER_MS);
+  dns.resolve(this.autodiscover.address, this.autodiscover.type || 'A', (err, ips) => {
+    setTimeout(() => this.discover(), this.autodiscover.refreshInterval);
     if (err) {
@@ -74,15 +76,22 @@ return this.emit('error', err);
 
-    const newList = _.sortBy(addresses)
-                     .map(ip => `limitd://${ip}:${this._options.port}`);
+    ips.filter(ip => !this.ring.servers.some(s => s.host === ip && s.port === this._options.port))
+      .forEach(newIp => {
+        const ipPort = `${newIp}:${this._options.port}`;
+        this.ring.add(ipPort);
+        this.clients[ipPort] = this.createClient(`limitd://${ipPort}`);
+        this.emit('new client', this.clients[ipPort]);
+      });
 
-    if (_.isEqual(newList, this.currentHosts)) {
-      //the list hasn't changed
-      return;
-    }
-
-    this.currentHosts = newList;
-
-    this.clients.forEach(c => c.disconnect());
-
-    this.clients = this.currentHosts.map(host => this.createClient(host));
+    this.ring.servers
+        .filter(server => !ips.some(ip => server.host === ip))
+        .forEach(oldServer => {
+          const ipPort = `${oldServer.host}:${oldServer.port}`;
+          if (this.clients[ipPort]) {
+            this.ring.remove(ipPort);
+            const client = this.clients[ipPort];
+            client.disconnect();
+            delete this.clients[ipPort];
+            this.emit('removed client', client);
+          }
+        });
   });
@@ -95,4 +104,3 @@ };
   }
-  const index =  murmur(`${type}:${key}`) % this.clients.length;
-  return this.clients[index];
+  return this.clients[this.ring.get(`${type}:${key}`)];
 };
@@ -99,0 +107,0 @@

test/shard_client.tests.js

 const ShardClient = require('../shard_client');
 const assert      = require('chai').assert;
 const _           = require('lodash');
-const mockuire    = require('mockuire')(module);
+const proxyquire  = require('proxyquire');
 
+
 describe('ShardClient', function() {
@@ -10,6 +11,7 @@ it('should fail if shard is not specified', function() {
     assert.throws(() => new ShardClient({}), /shard is required/);
+    assert.throws(() => new ShardClient({ shard: {} }), /unsupported shard configuration/);
   });
 
   function ShardClientCtor(client) {
-    return mockuire('../shard_client', {
+    return proxyquire('../shard_client', {
       './client': client
@@ -19,2 +21,19 @@ });
 
+  it('should fail when no shards are available', function(done) {
+    const client = function(params) {
+      this.host = params.host;
+    };
+
+    const ShardClient = ShardClientCtor(client);
+
+    const shardClient = new ShardClient({
+      shard: { hosts: [ ] }
+    });
+
+    shardClient.put('test', 'foo', (err) => {
+      assert.match(err.message, /no shard available/);
+      done();
+    });
+  });
+
   it('should work when full url are provided', function() {
@@ -31,4 +50,6 @@ const client = function(params) {
 
-    assert.equal(shardClient.clients[0].host, 'limitd://host-1:9231');
-    assert.equal(shardClient.clients[1].host, 'limitd://host-2:9231');
+    assert.equal(shardClient.clients['host-1:9231'].host, 'limitd://host-1:9231');
+    assert.equal(shardClient.clients['host-2:9231'].host, 'limitd://host-2:9231');
+    assert.ok(shardClient.ring.servers.some(s => s.host === 'host-1' && s.port === 9231));
+    assert.ok(shardClient.ring.servers.some(s => s.host === 'host-2' && s.port === 9231));
   });
@@ -47,4 +68,4 @@
 
-    assert.equal(shardClient.clients[0].host, 'limitd://host-1:9231');
-    assert.equal(shardClient.clients[1].host, 'limitd://host-2:9231');
+    assert.equal(shardClient.clients['host-1:9231'].host, 'limitd://host-1:9231');
+    assert.equal(shardClient.clients['host-2:9231'].host, 'limitd://host-2:9231');
   });
@@ -67,3 +88,3 @@
       this.put = function(type, key, count, callback) {
-        assert.equal(this.host, 'limitd://host-2:9231');
+        assert.equal(this.host, 'limitd://host-1:9231');
         assert.equal(type, 'ip');
@@ -231,3 +252,3 @@ assert.equal(key, '10.0.0.1');
 
-    const SharedClient = mockuire('../shard_client', {
+    const SharedClient = proxyquire('../shard_client', {
       './client': client,
@@ -245,7 +266,121 @@ 'dns': dns
 
-    assert.equal(shardClient.clients[0].host, 'limitd://host-a:9231');
-    assert.equal(shardClient.clients[1].host, 'limitd://host-b:9231');
+    assert.equal(shardClient.clients['host-a:9231'].host, 'limitd://host-a:9231');
+    assert.equal(shardClient.clients['host-b:9231'].host, 'limitd://host-b:9231');
+    assert.ok(shardClient.ring.servers.some(s => s.host === 'host-b' && s.port === 9231));
+    assert.ok(shardClient.ring.servers.some(s => s.host === 'host-a' && s.port === 9231));
   });
 
+  it('should add new shards', function(done) {
+    var clientsCreated = 0;
+    const client = function(params) {
+      clientsCreated++;
+      this.host = params.host;
+    };
 
+    var resolveCalls = 0;
+
+    const dns = {
+      resolve: (address, type, callback) => {
+        assert.equal(address, 'foo.bar.company.example.com');
+        assert.equal(type, 'A');
+        resolveCalls++;
+        if (resolveCalls === 1)  {
+          callback(null, [ 'host-b', 'host-a' ]);
+        } else if (resolveCalls === 2)  {
+          callback(null, [ 'host-c', 'host-b', 'host-a' ]);
+        }
+      }
+    };
+
+    const SharedClient = proxyquire('../shard_client', {
+      './client': client,
+      'dns': dns
+    });
+
+    const shardClient = new SharedClient({
+      shard: {
+        autodiscover: {
+          refreshInterval: 10,
+          address: 'foo.bar.company.example.com'
+        }
+      }
+    });
+
+
+    shardClient.on('new client', () => {
+      if (resolveCalls === 1) {
+        assert.equal(shardClient.ring.servers.length, 2);
+        assert.equal(clientsCreated, 2);
+      } else {
+        assert.equal(shardClient.ring.servers.length, 3);
+        assert.equal(clientsCreated, 3);
+        assert.equal(shardClient.clients['host-a:9231'].host, 'limitd://host-a:9231');
+        assert.equal(shardClient.clients['host-b:9231'].host, 'limitd://host-b:9231');
+        assert.equal(shardClient.clients['host-c:9231'].host, 'limitd://host-c:9231');
+        assert.ok(shardClient.ring.servers.some(s => s.host === 'host-c' && s.port === 9231));
+        assert.ok(shardClient.ring.servers.some(s => s.host === 'host-b' && s.port === 9231));
+        assert.ok(shardClient.ring.servers.some(s => s.host === 'host-a' && s.port === 9231));
+        done();
+      }
+    });
+  });
+
+  it('should remove shards', function(done) {
+    var clientsCreated = 0;
+    var clients = [];
+    const client = function(params) {
+      clientsCreated++;
+      this.host = params.host;
+      this.disconnect = () => this.disconnected = true;
+      clients.push(this);
+    };
+
+    var resolveCalls = 0;
+
+    const dns = {
+      resolve: (address, type, callback) => {
+        assert.equal(address, 'foo.bar.company.example.com');
+        assert.equal(type, 'A');
+        resolveCalls++;
+        if (resolveCalls === 1)  {
+          callback(null, [ 'host-b', 'host-a' ]);
+        } else if (resolveCalls === 2)  {
+          callback(null, [ 'host-a' ]);
+        }
+      }
+    };
+
+    const SharedClient = proxyquire('../shard_client', {
+      './client': client,
+      'dns': dns
+    });
+
+    const shardClient = new SharedClient({
+      shard: {
+        autodiscover: {
+          refreshInterval: 10,
+          address: 'foo.bar.company.example.com'
+        }
+      }
+    });
+
+
+    shardClient.once('new client', () => {
+      assert.equal(shardClient.ring.servers.length, 2);
+      assert.equal(clientsCreated, 2);
+    }).once('removed client', () => {
+      assert.equal(shardClient.ring.servers.length, 1);
+      assert.equal(clientsCreated, 2);
+      assert.equal(shardClient.clients['host-a:9231'].host, 'limitd://host-a:9231');
+      assert.notOk(shardClient.ring.servers.some(s => s.host === 'host-b' && s.port === 9231));
+      assert.ok(shardClient.ring.servers.some(s => s.host === 'host-a' && s.port === 9231));
+      assert.ok(clients.some(c => c.disconnected && c.host === 'limitd://host-b:9231'));
+      assert.ok(clients.some(c => !c.disconnected && c.host === 'limitd://host-a:9231'));
+      done();
+    });
+  });
+
+
+
+
 });

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

51729

increased by10.83%

Lines of code

1357

increased by10.15%

Worsened metrics

Dependency count

11

increased by10%

Dev dependency count

6

increased by20%

Dependency changes

• + Addedhashring@^3.2.0

• + Addedms@^1.0.0

• + Addedconnection-parse@0.0.7(transitive)

• + Addedhashring@3.2.0(transitive)

• + Addedms@1.0.0(transitive)

• + Addedsimple-lru-cache@0.0.2(transitive)

• - Removedmurmurhash3js@^3.0.1

• - Removedmurmurhash3js@3.0.1(transitive)