cueball - npm Package Compare versions

Comparing version 2.9.0 to 2.10.0

lib/connection-fsm.js

@@ -194,2 +194,4 @@ /*
 
+	this.sm_delaySpread = initialRecov.delaySpread || 0.2;
+
 	if (this.sm_monitor === true) {
@@ -281,3 +283,4 @@ var mult = 1 << this.sm_retries;
 	S.on(this.sm_socket, 'error', function socketMgrErrorListener(err) {
-		self.sm_lastError = err;
+		self.sm_lastError = new mod_errors.ConnectionError(
+		    self.sm_backend, 'error', 'connect', err);
 		S.gotoState('error');
@@ -288,3 +291,4 @@ self.sm_log.trace(err, 'emitted error while connecting');
 	S.on(this.sm_socket, 'connectError', function (err) {
-		self.sm_lastError = err;
+		self.sm_lastError = new mod_errors.ConnectionError(
+		    self.sm_backend, 'connectError', 'connect', err);
 		S.gotoState('error');
@@ -295,3 +299,4 @@ self.sm_log.trace(err, 'emitted connectError while connecting');
 	S.on(this.sm_socket, 'close', function () {
-		self.sm_lastError = new mod_errors.ConnectionClosedError(self);
+		self.sm_lastError = new mod_errors.ConnectionClosedError(
+		    self.sm_backend);
 		S.gotoState('error');
@@ -302,3 +307,4 @@ self.sm_log.trace('closed while connecting');
 	S.on(this.sm_socket, 'timeout', function () {
-		self.sm_lastError = new mod_errors.ConnectionTimeoutError(self);
+		self.sm_lastError = new mod_errors.ConnectionTimeoutError(
+		    self.sm_backend);
 		S.gotoState('error');
@@ -309,3 +315,4 @@ self.sm_log.trace('timed out while connecting');
 	S.on(this.sm_socket, 'connectTimeout', function () {
-		self.sm_lastError = new mod_errors.ConnectionTimeoutError(self);
+		self.sm_lastError = new mod_errors.ConnectionTimeoutError(
+		    self.sm_backend);
 		S.gotoState('error');
@@ -332,3 +339,4 @@ self.sm_log.trace('timed out while connecting');
 	S.on(this.sm_socket, 'error', function socketMgrErrorListener(err) {
-		self.sm_lastError = err;
+		self.sm_lastError = new mod_errors.ConnectionError(
+		    self.sm_backend, 'error', 'operation', err);
 		S.gotoState('error');
@@ -373,3 +381,3 @@ self.sm_pool._incrCounter('error-while-connected');
 
-	var delay = this.sm_delay;
+	var delay = mod_utils.delay(this.sm_delay, this.sm_delaySpread);
 
@@ -376,0 +384,0 @@ if (this.sm_retries !== Infinity) {

lib/errors.js

@@ -16,3 +16,4 @@ /*
 	PoolStoppingError: PoolStoppingError,
-	ClaimHandleMisusedError: ClaimHandleMisusedError
+	ClaimHandleMisusedError: ClaimHandleMisusedError,
+	ConnectionError: ConnectionError
 };
@@ -22,73 +23,92 @@
 const mod_assert = require('assert-plus');
+const mod_verror = require('verror');
+const VError = mod_verror.VError;
 
 function ClaimHandleMisusedError() {
-	if (Error.captureStackTrace)
-		Error.captureStackTrace(this, ClaimHandleMisusedError);
-	this.name = 'ClaimHandleMisusedError';
-	this.message = 'CueBall claim handle used as if it was a ' +
-	    'socket. Check the order and number of arguments in ' +
-	    'your claim callbacks.';
+	var opts = {};
+	opts.constructorOpt = ClaimHandleMisusedError;
+	VError.call(this, opts, 'CueBall claim handle used as if it was a ' +
+	    'socket (Check the order and number of arguments in ' +
+	    'your claim callbacks)');
 }
-mod_util.inherits(ClaimHandleMisusedError, Error);
+mod_util.inherits(ClaimHandleMisusedError, VError);
+ClaimHandleMisusedError.prototype.name = 'ClaimHandleMisusedError';
 
 function ClaimTimeoutError(pool) {
-	if (Error.captureStackTrace)
-		Error.captureStackTrace(this, ClaimTimeoutError);
+	var opts = {};
+	opts.constructorOpt = ClaimTimeoutError;
 	this.pool = pool;
-	this.name = 'ClaimTimeoutError';
-	this.message = 'Timed out while waiting for connection in pool ' +
-	    pool.p_uuid + ' (' + pool.p_domain + ')';
+	VError.call(this, opts, 'Timed out while waiting for connection in ' +
+	    'pool %s (%s)', pool.p_uuid, pool.p_domain);
 }
-mod_util.inherits(ClaimTimeoutError, Error);
+mod_util.inherits(ClaimTimeoutError, VError);
+ClaimTimeoutError.prototype.name = 'ClaimTimeoutError';
 
-function NoBackendsError(pool) {
-	if (Error.captureStackTrace)
-		Error.captureStackTrace(this, NoBackendsError);
+function NoBackendsError(pool, cause) {
+	var opts = {};
+	opts.constructorOpt = NoBackendsError;
+	opts.cause = cause;
 	this.pool = pool;
-	this.name = 'NoBackendsError';
-	this.message = 'No backends available in pool ' + pool.p_uuid +
-	    ' (' + pool.p_domain + ')';
+	VError.call(this, opts, 'No backends available in pool %s (%s)',
+	    pool.p_uuid, pool.p_domain);
 }
-mod_util.inherits(NoBackendsError, Error);
+mod_util.inherits(NoBackendsError, VError);
+NoBackendsError.prototype.name = 'NoBackendsError';
 
-function PoolFailedError(pool) {
-	if (Error.captureStackTrace)
-		Error.captureStackTrace(this, PoolFailedError);
+function PoolFailedError(pool, cause) {
+	var opts = {};
+	opts.constructorOpt = PoolFailedError;
+	opts.cause = cause;
 	this.pool = pool;
-	this.name = 'PoolFailedError';
-	this.message = 'Pool ' + pool.p_uuid + ' (' + pool.p_domain + ') ' +
-	    'has failed and cannot take new requests.';
+	var dead = Object.keys(pool.p_dead).length;
+	var avail = pool.p_keys.length;
+	VError.call(this, opts, 'Connections to backends of pool %s (%s) are ' +
+	    'persistently failing; request aborted (%d of %d declared dead, ' +
+	    'in state "failed")', pool.p_uuid.split('-')[0], pool.p_domain,
+	    dead, avail);
 }
-mod_util.inherits(PoolFailedError, Error);
+mod_util.inherits(PoolFailedError, VError);
+PoolFailedError.prototype.name = 'PoolFailedError';
 
 function PoolStoppingError(pool) {
-	if (Error.captureStackTrace)
-		Error.captureStackTrace(this, PoolStoppingError);
+	var opts = {};
+	opts.constructorOpt = PoolStoppingError;
 	this.pool = pool;
-	this.name = 'PoolStoppingError';
-	this.message = 'Pool ' + pool.p_uuid + ' (' + pool.p_domain + ') ' +
-	    'is stopping and cannot take new requests.';
+	VError.call(this, opts, 'Pool %s (%s) is stopping and cannot take ' +
+	    'new requests', pool.p_uuid.split('-')[0], pool.p_domain);
 }
-mod_util.inherits(PoolStoppingError, Error);
+mod_util.inherits(PoolStoppingError, VError);
+PoolStoppingError.prototype.name = 'PoolStoppingError';
 
-function ConnectionTimeoutError(fsm) {
-	if (Error.captureStackTrace)
-		Error.captureStackTrace(this, ConnectionTimeoutError);
-	this.fsm = fsm;
-	this.backend = fsm.cf_backend;
-	this.name = 'ConnectionTimeoutError';
-	this.message = 'Connection timed out to backend ' +
-	    JSON.stringify(this.backend);
+function ConnectionError(backend, event, state, cause) {
+	var opts = {};
+	opts.constructorOpt = ConnectionError;
+	opts.cause = cause;
+	this.backend = backend;
+	VError.call(this, opts, 'Connection to backend %s (%s:%d) emitted ' +
+	    '"%s" during %s', backend.name || backend.key, backend.address,
+	    backend.port, event, state);
 }
-mod_util.inherits(ConnectionTimeoutError, Error);
+mod_util.inherits(ConnectionError, VError);
+ConnectionError.prototype.name = 'ConnectionError';
 
-function ConnectionClosedError(fsm) {
-	if (Error.captureStackTrace)
-		Error.captureStackTrace(this, ConnectionClosedError);
-	this.fsm = fsm;
-	this.backend = fsm.cf_backend;
-	this.name = 'ConnectionClosedError';
-	this.message = 'Connection closed unexpectedly to backend ' +
-	    JSON.stringify(this.backend);
+function ConnectionTimeoutError(backend) {
+	var opts = {};
+	opts.constructorOpt = ConnectionTimeoutError;
+	this.backend = backend;
+	VError.call(this, opts, 'Connection timed out to backend %s (%s:%d)',
+	    backend.name || backend.key, backend.address, backend.port);
 }
-mod_util.inherits(ConnectionClosedError, Error);
+mod_util.inherits(ConnectionTimeoutError, VError);
+ConnectionTimeoutError.prototype.name = 'ConnectionTimeoutError';
+
+function ConnectionClosedError(backend) {
+	var opts = {};
+	opts.constructorOpt = ConnectionClosedError;
+	this.backend = backend;
+	VError.call(this, opts, 'Connection closed unexpectedly to backend ' +
+	    '%s (%s:%d)', backend.name || backend.key, backend.address,
+	    backend.port);
+}
+mod_util.inherits(ConnectionClosedError, VError);
+ConnectionClosedError.prototype.name = 'ConnectionClosedError';

lib/pool-monitor.js

@@ -19,2 +19,3 @@ /*
 const mod_os = require('os');
+const mod_resolver = require('./resolver');
 
@@ -24,2 +25,3 @@ function CueBallPoolMonitor() {
 	this.pm_sets = {};
+	this.pm_dnsRes = {};
 }
@@ -49,2 +51,13 @@
 
+CueBallPoolMonitor.prototype.registerDnsResolver = function (res) {
+	mod_assert.ok(res instanceof mod_resolver.DNSResolver);
+	this.pm_dnsRes[res.r_uuid] = res;
+};
+
+CueBallPoolMonitor.prototype.unregisterDnsResolver = function (res) {
+	mod_assert.ok(res instanceof mod_resolver.DNSResolver);
+	mod_assert.ok(this.pm_dnsRes[res.r_uuid]);
+	delete (this.pm_dnsRes[res.r_uuid]);
+};
+
 CueBallPoolMonitor.prototype.toKangOptions = function () {
@@ -54,3 +67,3 @@ var self = this;
 	function listTypes() {
-		return (['pool', 'set']);
+		return (['pool', 'set', 'dns_res']);
 	}
@@ -63,2 +76,4 @@
 			return (Object.keys(self.pm_sets));
+		} else if (type === 'dns_res') {
+			return (Object.keys(self.pm_dnsRes));
 		} else {
@@ -74,2 +89,4 @@ throw (new Error('Invalid type "' + type + '"'));
 			return (getSet(id));
+		} else if (type === 'dns_res') {
+			return (getDnsResolver(id));
 		} else {
@@ -169,2 +186,24 @@ throw (new Error('Invalid type "' + type + '"'));
 
+	function getDnsResolver(id) {
+		var res = self.pm_dnsRes[id];
+		mod_assert.object(res);
+
+		var obj = {};
+		obj.domain = res.r_domain;
+		obj.service = res.r_service;
+		obj.resolvers = res.r_resolvers;
+		obj.defaultPort = res.r_defport;
+		obj.state = res.getState();
+		obj.next = {};
+		if (res.r_nextService)
+			obj.next.srv = res.r_nextService.toISOString();
+		if (res.r_nextV6)
+			obj.next.v6 = res.r_nextV6.toISOString();
+		if (res.r_nextV4)
+			obj.next.v4 = res.r_nextV4.toISOString();
+		obj.backends = res.r_backends;
+		obj.counters = res.r_counters;
+		return (obj);
+	}
+
 	function stats() {
@@ -171,0 +210,0 @@ return ({});

lib/pool.js

@@ -24,2 +24,3 @@ /*
 const mod_errors = require('./errors');
+const mod_verror = require('verror');
 
@@ -327,3 +328,5 @@ const mod_codel = require('./codel');
 		    'pool will start up in "failed" state');
-		this.p_lastError = this.p_resolver.getLastError();
+		this.p_lastError = new mod_verror.VError(
+		    this.p_resolver.getLastError(),
+		    'Pool resolver entered state "failed"');
 		S.gotoState('failed');
@@ -337,3 +340,5 @@ return;
 			    'pool to "failed" state');
-			self.p_lastError = self.p_resolver.getLastError();
+			self.p_lastError = new mod_verror.VError(
+			    self.p_resolver.getLastError(),
+			    'Pool resolver entered state "failed"');
 			S.gotoState('failed');
@@ -399,4 +404,6 @@ }
 		var hdl = this.p_waiters.shift();
-		if (hdl.isInState('waiting'))
-			hdl.fail(new mod_errors.PoolFailedError(self));
+		if (hdl.isInState('waiting')) {
+			hdl.fail(new mod_errors.PoolFailedError(self,
+			    self.p_lastError));
+		}
 	}
@@ -898,4 +905,6 @@ };
 		setImmediate(function () {
-			if (!done)
-				cb(new mod_errors.PoolFailedError(self));
+			if (!done) {
+				cb(new mod_errors.PoolFailedError(self,
+				    self.p_lastError));
+			}
 			done = true;
@@ -950,3 +959,4 @@ });
 		if (errOnEmpty && self.p_resolver.count() < 1) {
-			var err = new mod_errors.NoBackendsError(self);
+			var err = new mod_errors.NoBackendsError(self,
+			    self.p_resolver.getLastError());
 			handle.fail(err);
@@ -953,0 +963,0 @@ }

lib/resolver.js

@@ -38,2 +38,4 @@ /*
 const mod_os = require('os');
+const mod_uuid = require('uuid');
+const mod_monitor = require('./pool-monitor');
 
@@ -253,2 +255,3 @@ const FSM = mod_mooremachine.FSM;
 
+	this.r_uuid = mod_uuid.v4();
 	this.r_resolvers = options.resolvers || [];
@@ -312,2 +315,3 @@ this.r_domain = options.domain;
 		delay: dnsSrvRecov.delay,
+		delaySpread: dnsSrvRecov.delaySpread || 0.2,
 		maxDelay: dnsSrvRecov.maxDelay || Infinity
@@ -322,2 +326,3 @@ };
 		delay: dnsRecov.delay,
+		delaySpread: dnsRecov.delaySpread || 0.2,
 		maxDelay: dnsRecov.maxDelay || Infinity
@@ -345,2 +350,7 @@ };
 	this.r_lastSrvTtl = 60;
+	/*
+	 * And last TTL we saw at all -- we'll use this as an effective
+	 * max delay if we exhaust our retry policy.
+	 */
+	this.r_lastTtl = 60;
 
@@ -398,2 +408,4 @@ this.r_lastError = undefined;
 
+	this.r_counters = {};
+
 	FSM.call(this, 'init');
@@ -409,2 +421,17 @@
 
+CueBallDNSResolver.prototype._incrCounter = function (counter) {
+	if (this.r_counters[counter] === undefined)
+		this.r_counters[counter] = 0;
+	++this.r_counters[counter];
+};
+
+CueBallDNSResolver.prototype._hwmCounter = function (counter, val) {
+	if (this.r_counters[counter] === undefined) {
+		this.r_counters[counter] = val;
+		return;
+	}
+	if (this.r_counters[counter] < val)
+		this.r_counters[counter] = val;
+};
+
 CueBallDNSResolver.prototype.start = function () {
@@ -434,2 +461,3 @@ this.emit('startAsserted');
 	this.r_stopping = false;
+	mod_monitor.monitor.registerDnsResolver(this);
 	if (this.r_bootstrap !== undefined) {
@@ -539,2 +567,3 @@ if (--this.r_bootstrap.r_refCount <= 0)
 		self.r_lastSrvTtl = ttl;
+		self.r_lastTtl = ttl;
 		self.r_haveSeenSRV = true;
@@ -566,3 +595,5 @@
 	S.on(req, 'error', function (err) {
-		self.r_lastError = err;
+		self.r_lastError = new mod_verror.VError(err,
+		    'SRV lookup for "%s" failed', name);
+		self._incrCounter('srv-failure');
 
@@ -608,2 +639,4 @@ if (NoRecordsError.isInstance(err) ||
 
+			self._incrCounter('srv-skipped');
+
 			S.gotoState('aaaa');
@@ -630,3 +663,4 @@
 	if (--r.count > 0) {
-		S.timeout(r.delay, function () {
+		var delay = mod_utils.delay(r);
+		S.timeout(delay, function () {
 			S.gotoState('srv_try');
@@ -640,3 +674,3 @@ });
 	} else {
-		self.r_log.trace({ err: self.r_lastError },
+		self.r_log.trace(self.r_lastError,
 		    'repeated error during SRV resolution for service %s, ' +
@@ -789,2 +823,3 @@ 'will retry in %d sec', self.r_service, self.r_lastSrvTtl);
 		}
+		self.r_lastTtl = ttl;
 
@@ -826,3 +861,4 @@ self.r_haveSeenAddr = true;
 		}
-		self.r_lastError = err;
+		self.r_lastError = new mod_verror.VError(err,
+		     'IPv6 (AAAA) lookup failed for "%s"', srv.name);
 		S.gotoState('aaaa_error');
@@ -837,3 +873,4 @@ });
 	if (--r.count > 0) {
-		S.timeout(r.delay, function () {
+		var delay = mod_utils.delay(r);
+		S.timeout(delay, function () {
 			S.gotoState('aaaa_try');
@@ -847,3 +884,3 @@ });
 	} else {
-		self.r_log.trace({ err: self.r_lastError },
+		self.r_log.trace(self.r_lastError,
 		    'repeated error during AAAA resolution for name %s, ' +
@@ -913,2 +950,3 @@ 'proceeding', self.r_srv.name);
 			self.r_nextV4 = d;
+		self.r_lastTtl = ttl;
 
@@ -953,3 +991,4 @@ self.r_haveSeenAddr = true;
 		}
-		self.r_lastError = err;
+		self.r_lastError = new mod_verror.VError(err,
+		    'IPv4 (A) lookup for "%s" failed', srv.name);
 		S.gotoState('a_error');
@@ -964,3 +1003,4 @@ });
 	if (--r.count > 0) {
-		S.timeout(r.delay, function () {
+		var delay = mod_utils.delay(r);
+		S.timeout(delay, function () {
 			S.gotoState('a_try');
@@ -974,3 +1014,3 @@ });
 	} else {
-		self.r_log.debug({ err: self.r_lastError },
+		self.r_log.debug(self.r_lastError,
 		    'repeated error during A resolution for name %s, ' +
@@ -985,3 +1025,3 @@ 'proceeding', self.r_srv.name);
 		 */
-		d.setTime(d.getTime() + 1000*60);
+		d.setTime(d.getTime() + 1000*self.r_lastTtl);
 		if (self.r_nextV4 === undefined || d <= self.r_nextV4)
@@ -1025,2 +1065,3 @@ self.r_nextV4 = d;
 		    this.r_service, this.r_domain);
+		this._incrCounter('empty-set');
 		this.r_log.warn(err, 'finished processing');
@@ -1051,2 +1092,3 @@ this.emit('updated', err);
 		self.emit('removed', k);
+		self._incrCounter('backend-removed');
 	});
@@ -1056,2 +1098,3 @@ added.forEach(function (k) {
 		self.emit('added', k, newBackends[k]);
+		self._incrCounter('backend-added');
 	});
@@ -1101,8 +1144,20 @@
 
+	this._hwmCounter('max-sleep', minDelay);
+
 	if (minDelay < 0) {
 		S.gotoState(state);
 	} else {
-		self.r_log.trace({state: state, delay: minDelay},
+		/*
+		 * Unlike the regular logic used by mod_utils.delay(), for
+		 * DNS TTL timeouts there's no point spreading them backwards,
+		 * as we will just get the same answer again from a cache.
+		 *
+		 * So we spread it forwards in time only (1.0 to 1.0 + spread).
+		 */
+		var delay = Math.round(minDelay *
+		    (1 + Math.random() * this.r_retry.delaySpread));
+
+		self.r_log.trace({state: state, delay: delay},
 		    'sleeping until next TTL expiry');
-		S.timeout(minDelay, function () {
+		S.timeout(delay, function () {
 			S.gotoState(state);
@@ -1197,2 +1252,6 @@ });
 			errs.forEach(function (e) {
+				if (e.name === 'TimeoutError') {
+					self._incrCounter('timeout');
+					return;
+				}
 				if (e.code === undefined)
@@ -1203,2 +1262,4 @@ return;
 				++codes[e.code];
+				self._incrCounter('rcode-' +
+				    e.code.toLowerCase());
 			});
@@ -1220,2 +1281,6 @@ var sorted = Object.keys(codes).sort(function (a, b) {
 		if (err) {
+			if (err.code) {
+				self._incrCounter('rcode-' +
+				    err.code.toLowerCase());
+			}
 			em.emit('error', err);
@@ -1227,2 +1292,3 @@ return;
 		var minTTL = undefined;
+		self._incrCounter('rcode-ok');
 		if (type === 'A' || type === 'AAAA') {
@@ -1233,4 +1299,7 @@ ans = [];
 					if (a.type === 'CNAME' ||
-					    a.type === 'DNAME')
+					    a.type === 'DNAME') {
+						self._incrCounter('cname');
 						return;
+					}
+					self._incrCounter('unknown-rrtype');
 					self.r_log.warn('got unsupported ' +
@@ -1259,2 +1328,3 @@ 'answer rrtype: %s', a.type);
 						return;
+					self._incrCounter('unknown-rrtype');
 					self.r_log.warn('got unsupported ' +
@@ -1278,4 +1348,7 @@ 'additional rrtype: %s', rr.type);
 					if (a.type === 'CNAME' ||
-					    a.type === 'DNAME')
+					    a.type === 'DNAME') {
+						self._incrCounter('cname');
 						return;
+					}
+					self._incrCounter('unknown-rrtype');
 					self.r_log.warn('got unsupported ' +
@@ -1290,4 +1363,6 @@ 'answer rrtype: %s', a.type);
 				var obj = { name: a.target, port: a.port };
-				if (cache[a.target])
+				if (cache[a.target]) {
+					self._incrCounter('additionals-used');
 					obj.additionals = cache[a.target];
+				}
 				ans.push(obj);
@@ -1294,0 +1369,0 @@ });

lib/utils.js

@@ -19,3 +19,4 @@ /*
 	createErrorMetrics: createErrorMetrics,
-	updateErrorMetrics: updateErrorMetrics
+	updateErrorMetrics: updateErrorMetrics,
+	delay: genDelay
 };
@@ -155,2 +156,8 @@
 	delete (ks.maxDelay);
+	mod_assert.optionalNumber(obj.delaySpread, name + '.delaySpread');
+	if (obj.delaySpread !== undefined && obj.delaySpread !== null) {
+		mod_assert.ok(obj.delaySpread >= 0.0 && obj.delaySpread <= 1.0,
+		    name + '.delaySpread must be between 0.0 and 1.0');
+	}
+	delete (ks.delaySpread);
 	mod_assert.deepEqual(Object.keys(ks), []);
@@ -440,1 +447,18 @@
 }
+
+function genDelay(recovOrDelay, spread) {
+	var base = recovOrDelay;
+	if (typeof (recovOrDelay) === 'object' && spread === undefined) {
+		base = recovOrDelay.delay;
+		spread = recovOrDelay.delaySpread;
+	}
+	mod_assert.number(base, 'base delay');
+	mod_assert.optionalNumber(spread, 'spread factor');
+	if (spread === undefined || spread === null)
+		spread = 0.2;
+	/*
+	 * delaySpread = 0.2 means 20% spread (so choose a random delay
+	 * between 0.9*delay and 1.1*delay).
+	 */
+	return (Math.round(base * (1 - spread / 2 + Math.random() * spread)));
+}

package.json

 {
   "name": "cueball",
-  "version": "2.9.0",
+  "version": "2.10.0",
   "description": "manage a pool of connections to a multi-node service where nodes are listed in DNS",
@@ -5,0 +5,0 @@ "main": "lib/index.js",

No alert changes

Improved metrics

Total package byte prevSize

204075

3.77%

Lines of code

5471

3.07%

No dependency changes