reliable-get - npm Package Compare versions

Comparing version 0.1.1 to 0.1.2

index.js

@@ -6,23 +6,25 @@ 'use strict';
 var url = require('url');
+var util = require('util');
+var utils = require('./lib/utils');
+var EventEmitter = require('events').EventEmitter;
 var CircuitBreaker = require('./lib/CircuitBreaker');
 var CacheFactory = require('./lib/cache/cacheFactory');
 
-function createClient(config) {
+function ReliableGet(config) {
 
-    var cache = CacheFactory.getCache(config.cache),
-        eventHandler = config.eventHandler || {
-            logger: function() {},
-            stats: function() {}
-        };
+    this.get = function(options, next) {
 
-    return function(options, next) {
+        var self = this,
+            start = Date.now(),
+            cache = CacheFactory.getCache(config.cache),
+            hasCacheControl = function(res, value) {
+                if (typeof value === 'undefined') { return res.headers['cache-control']; }
+                return (res.headers['cache-control'] || '').indexOf(value) !== -1;
+            };
 
-        var start = Date.now(), hasCacheControl = function(res, value) {
-            if (typeof value === 'undefined') { return res.headers['cache-control']; }
-            return (res.headers['cache-control'] || '').indexOf(value) !== -1;
-        };
-
+        // Defaults
         options.headers = options.headers || config.headers || {};
+        if(!options.cacheKey) { options.cacheKey = utils.urlToCacheKey(options.url); }
 
-        function pipeAndCacheContent(next) {
+        var pipeAndCacheContent = function(cb) {
 
@@ -38,3 +40,4 @@ var content = '', start = Date.now(), inErrorState = false, res;
                     });
-                    next({statusCode: statusCode, message: message});
+                    self.emit('stat', 'error', 'FAIL ' + message, {tracer:options.tracer, statusCode: statusCode, type:options.type});
+                    cb({statusCode: statusCode || 500, message: message});
                 }
@@ -62,6 +65,6 @@ }
                     res.content = content;
-                    next(null, res);
+                    cb(null, res);
                     var timing = Date.now() - start;
-                    eventHandler.logger('debug', 'OK ' + options.url,{tracer:options.tracer, responseTime: timing, pcType:options.type});
-                    eventHandler.stats('timing', options.statsdKey + '.responseTime', timing);
+                    self.emit('log', 'debug', 'OK ' + options.url, {tracer:options.tracer, responseTime: timing, type:options.type});
+                    self.emit('stat', 'timing', options.statsdKey + '.responseTime', timing);
                 });
@@ -78,10 +81,10 @@
                     var timing = Date.now() - start;
-                    eventHandler.logger('debug', 'CACHE HIT for key: ' + options.cacheKey,{tracer:options.tracer, responseTime: timing, pcType:options.type});
-                    eventHandler.stats('increment', options.statsdKey + '.cacheHit');
-                    next(null, {content: cacheData.content, headers: cacheData.headers});
+                    self.emit('log','debug', 'CACHE HIT for key: ' + options.cacheKey,{tracer:options.tracer, responseTime: timing, type:options.type});
+                    self.emit('stat', 'increment', options.statsdKey + '.cacheHit');
+                    next(null, {statusCode: 200, content: cacheData.content, headers: cacheData.headers});
                     return;
                 }
 
-                eventHandler.logger('debug', 'CACHE MISS for key: ' + options.cacheKey,{tracer:options.tracer,pcType:options.type});
-                eventHandler.stats('increment', options.statsdKey + '.cacheMiss');
+                self.emit('log', 'debug', 'CACHE MISS for key: ' + options.cacheKey,{tracer:options.tracer, type:options.type});
+                self.emit('stat', 'increment', options.statsdKey + '.cacheMiss');
 
@@ -93,3 +96,3 @@ if(options.url == 'cache') {
 
-                new CircuitBreaker(options, config, eventHandler, pipeAndCacheContent, function(err, res) {
+                new CircuitBreaker(self, options, config, pipeAndCacheContent, function(err, res) {
 
@@ -101,5 +104,4 @@ if (err) {
 
-                    // Honor fragment cache control headers in a simplistic way
                     if (hasCacheControl(res, 'no-cache') || hasCacheControl(res, 'no-store')) {
-                        next(null, {content: res.content, headers: res.headers});
+                        next(null, {statusCode: 200, content: res.content, headers: res.headers});
                         return;
@@ -111,6 +113,5 @@ }
 
-                    next(null, {content: res.content, headers:res.headers});
-
                     cache.set(options.cacheKey, {content: res.content, headers: res.headers}, options.cacheTTL, function() {
-                        eventHandler.logger('debug', 'CACHE SET for key: ' + options.cacheKey + ' @ TTL: ' + options.cacheTTL,{tracer:options.tracer,pcType:options.type});
+                        next(null, {statusCode: 200, content: res.content, headers:res.headers});
+                        self.emit('log','debug', 'CACHE SET for key: ' + options.cacheKey + ' @ TTL: ' + options.cacheTTL,{tracer:options.tracer,type:options.type});
                     });
@@ -123,10 +124,10 @@
 
-            new CircuitBreaker(options, config, eventHandler, pipeAndCacheContent, function(err, res) {
+            new CircuitBreaker(self, options, config, pipeAndCacheContent, function(err, res) {
                 if (err) { return next(err); }
-                // Force no store
                 res.headers['cache-control'] = 'no-store';
-                next(null, {content: res.content, headers: res.headers});
+                next(null, {statusCode: res.statusCode, content: res.content, headers: res.headers});
             });
 
         }
+
     }
@@ -136,2 +137,4 @@
 
-module.exports = createClient;
+util.inherits(ReliableGet, EventEmitter);
+
+module.exports = ReliableGet;

lib/CircuitBreaker.js

@@ -15,3 +15,3 @@ /*
 
-module.exports = function(options, config, eventHandler, command, next) {
+module.exports = function(rg, options, config, command, next) {
 
@@ -34,2 +34,3 @@ var parsedUrl = url.parse(options.url);
       });
+      rg.emit('log','debug', 'CB ENGAGED for service: ' + cbKey + ' @ TTL: ' + options.cacheTTL, {tracer:options.tracer, type:options.type});
       next({statusCode: 500, message:message});
@@ -63,3 +64,3 @@ };
     function onCircuitOpen(metrics) {
-        eventHandler.logger('error', 'CIRCUIT BREAKER OPEN for host ' + cbKey,{
+        rg.emit('log', 'error', 'CIRCUIT BREAKER OPEN for host ' + cbKey, {
             tracer:options.tracer,
@@ -73,3 +74,3 @@ pcType:options.type,
     function onCircuitClose(metrics) {
-        eventHandler.logger('info', 'CIRCUIT BREAKER CLOSED for host ' + cbKey,{
+        rg.emit('log', 'error', 'CIRCUIT BREAKER CLOSED for host ' + cbKey, {
             tracer:options.tracer,
@@ -76,0 +77,0 @@ pcType:options.type,

lib/utils.js

 'use strict';
 
-var _ = require('lodash');
 var url = require('url');
 
-function timeToMillis(timeString) {
-
-	var matched = new RegExp('(\\d+)(.*)').exec(timeString),
-		num = matched[1],
-		period = matched[2] || 'ms',
-		value = 0;
-
-	switch(period) {
-		case 'ms':
-			value = parseInt(num);
-			break;
-		case 's':
-			value = parseInt(num)*1000;
-			break;
-		case 'm':
-			value = parseInt(num)*1000*60;
-			break;
-		case 'h':
-			value = parseInt(num)*1000*60*60;
-			break;
-		case 'd':
-			value = parseInt(num)*1000*60*60*24;
-			break;
-		default:
-			value = parseInt(num);
-	}
-
-	return value;
-
-}
-
 function cacheKeytoStatsd(key) {
@@ -51,11 +19,2 @@ key = key.replace(/\./g,'_');
 
-function createTag(tagname, attribs) {
-    var attribArray = [], attribLength = attribs.length, attribCounter = 0;
-    _.forIn(attribs, function(value, key) {
-        attribCounter++;
-        attribArray.push(' ' + key + '=\'' + value + '\'');
-    });
-    return ['<',tagname,(attribLength > 0 ? ' ' : '')].concat(attribArray).concat(['>']).join('');
-}
-
 function parseRedisConnectionString(connectionString) {
@@ -71,7 +30,5 @@ var params = url.parse(connectionString, true);
 module.exports = {
-	timeToMillis: timeToMillis,
 	urlToCacheKey: urlToCacheKey,
 	cacheKeytoStatsd: cacheKeytoStatsd,
-	createTag: createTag,
 	parseRedisConnectionString: parseRedisConnectionString
 };

package.json

 {
   "name": "reliable-get",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "A circuit breaker and cached wrapper for GET requests (enables reliable external service interaction).",
@@ -41,2 +41,5 @@ "main": "index.js",
     "cheerio": "^0.17.0",
+    "connect": "^3.3.3",
+    "connect-route": "^0.1.4",
+    "cookie-parser": "^1.3.3",
     "expect.js": "~0.3.1",
@@ -43,0 +46,0 @@ "istanbul": "^0.3.2",

readme.md

 Reliable HTTP get wrapper (cache and circuit breaker), best wrapped around things you dont trust very much.
 
 [![Build Status](https://travis-ci.org/tes/reliable-get.svg)](https://travis-ci.org/tes/reliable-get)
+
+Example options from Compoxure backend request:
+
+```
+var options = {
+          url: targetUrl,
+          cacheKey: targetCacheKey,
+          cacheTTL: targetCacheTTL,
+          timeout: utils.timeToMillis(backend.timeout || DEFAULT_LOW_TIMEOUT),
+          headers: backendHeaders,
+          tracer: req.tracer,
+          statsdKey: 'backend_' + utils.urlToCacheKey(host),
+          eventHandler: eventHandler
+        };
+```
+
+From compoxure fragment request:
+
+```
+var options = {
+      url: url,
+      timeout: timeout,
+      cacheKey: cacheKey,
+      cacheTTL: cacheTTL,
+      explicitNoCache: explicitNoCache,
+      headers: optionsHeaders,
+      tracer: req.tracer,
+      statsdKey: statsdKey,
+      eventHandler: eventHandler
+  }
+```
+
+Property|Description|Example / Default|Required
+---------|----------|-------------|-------
+url|Service to get|http://my-service.tes.co.uk|Yes
+timeout|Timeout for service|5000|No
+cacheKey|Key to store cached value against|my-service_tes_co_uk|No
+cacheTTL|TTL of cached value|1 minute|No
+explicitNoCache|Do not cache under any circumstances|false|No
+headers|Headers to send with request||No
+tracer|Unique value to pass with request||No
+type|Type of request, used for statsd and logging||No
+statsdKey|Key that statsd events will be posted to||No
+eventHandler|Object (see below) for logging and stats||No
+
+Event Handler
+=============
+
+To allow Reliable Get to report back on status, at the moment we require you to pass in a simple object:
+
+```
+var eventHandler = {
+  logger: function(level, message, data) {},
+  stats: function(type, key, value) {}
+}
+```
+
+This will likely get replaced with a more standard EventEmitter at some point when we get around to it (this is a legacy of the extraction of this code from another project for now).
+

tests/cache/RedisCacheTest.js

@@ -45,2 +45,14 @@ 'use strict';
 
+
+    it('should set and get values from cache', function(done) {
+        withCache({engine:'redis'}, function(err, cache) {
+            cache.set('bar:123', {content:'content', headers:{'header':'1'}}, 1000, function(err) {
+                expect(err).to.be(null);
+                assertCachedValue(cache, 'bar:123', 'content', function() {
+                    assertHeaderValue(cache, 'bar:123', 'header', '1', done);
+                });
+            });
+        });
+    });
+
     it('should bypass cache is redis is unavailable', function(done) {
@@ -55,2 +67,29 @@ var cache = cacheFactory.getCache({engine:'redis', hostname: 'foobar.acuminous.co.uk'});
 
+    it('should parse url structures with host, port and db', function(done) {
+        var cache = cacheFactory.getCache({engine:'redis', url: 'redis://localhost:6379?db=1'});
+        cache.get('anything', function(err, data) {
+            expect(err).to.be(undefined);
+            expect(data).to.be(undefined);
+            done();
+        });
+    });
+
+    it('should parse url structures with host and port', function(done) {
+        var cache = cacheFactory.getCache({engine:'redis', url: 'redis://localhost:6379'});
+        cache.get('anything', function(err, data) {
+            expect(err).to.be(undefined);
+            expect(data).to.be(undefined);
+            done();
+        });
+    });
+
+    it('should parse url structures with host', function(done) {
+        var cache = cacheFactory.getCache({engine:'redis', url: 'redis://localhost'});
+        cache.get('anything', function(err, data) {
+            expect(err).to.be(undefined);
+            expect(data).to.be(undefined);
+            done();
+        });
+    });
+
     function withCache(config, next) {
@@ -57,0 +96,0 @@ var cache = cacheFactory.getCache(config);

tests/reliable-get.test.js

@@ -5,17 +5,182 @@ 'use strict';
 var ReliableGet = require('..');
+var async = require('async');
+var _ = require('lodash');
 
-describe("Core caching", function() {
+describe("Reliable Get", function() {
 
-  it('should request something', function(done) {
+  before(function(done) {
+    var stubServer = require('./stub/server');
+    stubServer.init(5001, done);
+  });
 
-      var config = {cache:{engine:'memory'}};
-      var rg = ReliableGet(config);
-      rg({url:'http://www.google.com'}, function(err, response) {
+  it('NO CACHE: should be able to make a simple request', function(done) {
+      var config = {cache:{engine:'nocache'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001'}, function(err, response) {
+          expect(err).to.be(null);
+          expect(response.statusCode).to.be(200);
           done();
       });
+  });
 
+  it('NO CACHE: should fail if it calls a service that is broken', function(done) {
+      var config = {cache:{engine:'nocache'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001/broken'}, function(err, response) {
+          expect(err.statusCode).to.be(500);
+          done();
+      });
   });
 
+  it('NO CACHE: should fail if it calls a service that breaks after a successful request', function(done) {
+      var config = {cache:{engine:'nocache'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001/faulty?faulty=false'}, function(err, response) {
+          expect(err).to.be(null);
+          expect(response.statusCode).to.be(200);
+          rg.get({url:'http://localhost:5001/faulty?faulty=true'}, function(err, response) {
+            expect(err.statusCode).to.be(500);
+            done();
+          });
+      });
+  });
+
+  it('MEMORY CACHE: should serve from cache after initial request', function(done) {
+      var config = {cache:{engine:'memorycache'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001/faulty?faulty=false', cacheKey: 'memory-faulty-1', cacheTTL: 200}, function(err, response) {
+          expect(err).to.be(null);
+          expect(response.statusCode).to.be(200);
+          rg.get({url:'http://localhost:5001/faulty?faulty=false', cacheKey: 'memory-faulty-1', cacheTTL: 200}, function(err, response) {
+            expect(err).to.be(null);
+            expect(response.statusCode).to.be(200);
+            done();
+          });
+      });
+  });
+
+  it('MEMORY CACHE: should serve cached content if it calls a service that breaks after a successful request', function(done) {
+      var config = {cache:{engine:'memorycache'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001/faulty?faulty=false', cacheKey: 'memory-faulty-2', cacheTTL: 200}, function(err, response) {
+          expect(err).to.be(null);
+          expect(response.statusCode).to.be(200);
+          rg.get({url:'http://localhost:5001/faulty?faulty=true', cacheKey: 'memory-faulty-2', cacheTTL: 200}, function(err, response) {
+            expect(err).to.be(null);
+            expect(response.statusCode).to.be(200);
+            done();
+          });
+      });
+  });
+
+  it('MEMORY CACHE: should serve stale content if it calls a service that breaks after a successful request and ttl expired', function(done) {
+      var config = {cache:{engine:'memorycache'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001/faulty?faulty=false', cacheKey: 'memory-faulty-3', cacheTTL: 200}, function(err, response) {
+          expect(err).to.be(null);
+          expect(response.statusCode).to.be(200);
+          setTimeout(function() {
+            rg.get({url:'http://localhost:5001/faulty?faulty=true', cacheKey: 'memory-faulty-3', cacheTTL: 200}, function(err, response) {
+              expect(err.statusCode).to.be(500);
+              expect(response.stale.content).to.be('Faulty service managed to serve good content!');
+              done();
+            });
+          }, 500);
+      });
+  });
+
+  it('REDIS CACHE: should serve from cache after initial request', function(done) {
+      var config = {cache:{engine:'redis'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001/faulty?faulty=false', cacheKey: 'redis-faulty-1', cacheTTL: 200}, function(err, response) {
+          expect(err).to.be(null);
+          expect(response.statusCode).to.be(200);
+          rg.get({url:'http://localhost:5001/faulty?faulty=true', cacheKey: 'redis-faulty-1', cacheTTL: 200}, function(err, response) {
+            expect(err).to.be(null);
+            expect(response.statusCode).to.be(200);
+            done();
+          });
+      });
+  });
+
+  it('REDIS CACHE: should serve cached content if it calls a service that breaks after a successful request', function(done) {
+      var config = {cache:{engine:'redis'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001/faulty?faulty=false', cacheKey: 'redis-faulty-2', cacheTTL: 200}, function(err, response) {
+          expect(err).to.be(null);
+          expect(response.statusCode).to.be(200);
+          rg.get({url:'http://localhost:5001/faulty?faulty=true', cacheKey: 'redis-faulty-2', cacheTTL: 200}, function(err, response) {
+            expect(err).to.be(null);
+            expect(response.statusCode).to.be(200);
+            done();
+          });
+      });
+  });
+
+  it('REDIS CACHE: should serve stale content if it calls a service that breaks after a successful request and ttl expired', function(done) {
+      var config = {cache:{engine:'redis'}};
+      var rg = new ReliableGet(config);
+      rg.get({url:'http://localhost:5001/faulty?faulty=false', cacheKey: 'redis-faulty-3', cacheTTL: 200}, function(err, response) {
+          expect(err).to.be(null);
+          expect(response.statusCode).to.be(200);
+          setTimeout(function() {
+            rg.get({url:'http://localhost:5001/faulty?faulty=true', cacheKey: 'redis-faulty-3', cacheTTL: 200}, function(err, response) {
+              expect(err.statusCode).to.be(500);
+              expect(response.stale.content).to.be('Faulty service managed to serve good content!');
+              done();
+            });
+          }, 500);
+      });
+  });
+
+  it('CIRCUIT BREAKER: should invoke circuit breaker if configured and then open again after window', function(done) {
+
+      this.timeout(20000);
+
+      var config = {cache:{engine:'memorycache'},
+        'circuitbreaker':{
+            'windowDuration':5000,
+            'numBuckets': 5,
+            'errorThreshold': 20,
+            'volumeThreshold': 3,
+            'includePath': true
+        }
+      };
+      var rg = new ReliableGet(config);
+      var cbOpen = false;
+
+      rg.on('log', function(level, message) {
+        if(_.contains(message, 'CIRCUIT BREAKER OPEN for host')) {
+          cbOpen = true;
+        }
+        if(_.contains(message, 'CIRCUIT BREAKER CLOSED for host')) {
+          cbOpen = false;
+        }
+      });
+
+      async.whilst(function() {
+        return !cbOpen;
+      }, function(next) {
+        rg.get({url:'http://localhost:5001/cb-faulty?faulty=true', cacheKey: 'circuit-breaker', explicitNoCache: true}, function(err, response) {
+          expect(err.statusCode).to.be(500);
+          next();
+        });
+      }, function() {
+        setTimeout(function() {
+          async.whilst(function() {
+              return cbOpen;
+            }, function(next) {
+              rg.get({url:'http://localhost:5001/cb-faulty?faulty=false', cacheKey: 'circuit-breaker', explicitNoCache: true}, function(err, response) {
+                setTimeout(next, 500);
+              });
+            }, function() {
+              done();
+            }
+          );
+        }, 5000);
+      });
+  });
+
 });
 
 

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

35494

increased by46.54%

Number of package files

21

increased by10.53%

Lines of code

760

increased by33.33%

Number of lines in readme file

63

increased by1475%

Worsened metrics

Dev dependency count

9

increased by50%

Number of low supply chain risk alerts

1

increased byInfinity%

No dependency changes