zcache - npm Package Compare versions

Comparing version 0.4.0-alpha to 0.4.0-beta

index.js

 module.exports = {
   CacheCluster: require('./lib/CacheCluster'),
   CacheInstance: require('./lib/CacheInstance'),
-  ConnectionPool: require('./lib/ConnectionPool'),
   ConnectionWrapper: require('./lib/ConnectionWrapper'),
@@ -6,0 +5,0 @@ MemcacheConnection: require('./lib/MemcacheConnection'),

lib/CacheCluster.js

@@ -125,3 +125,2 @@ // Copyright 2014 The Obvious Corporation.
       .fail(setError.bind(null, errors, keysOnInstance))
-      .end()
     )
@@ -179,3 +178,2 @@ }
       })))
-      .end()
     )
@@ -182,0 +180,0 @@ }

lib/RedisConnection.js

 var redis = require('redis')
 var util = require('util')
 var Q = require('kew')
+var snappy = require('snappy')
 
@@ -15,3 +16,3 @@ var CacheInstance = require('./CacheInstance')
  * @param {number} port The port that the redis-server listens to
- * @param {{requestTimeoutMs: (number|undefined)}=} options Additional options for this connection.
+ * @param {{requestTimeoutMs: (number|undefined), compressionEnabled: (boolean|undefined)}=} options Additional options for this connection.
  *     'requestTimeoutMs' specifies the timeout of a Redis request.
@@ -31,2 +32,11 @@ * @extends CacheInstance
   this._bound_onEnd = this._onEnd.bind(this)
+  
+  // Controls if we turn on compression or not. 
+  // All cache values which are longer than the pivot are eligible for compression
+  // Pivot and encoding prefix are hardcoded for now. Will revisit after
+  // we know we are using snappy for sure
+  this._snappyPivot = 750
+  this._compressedPrefix = '@snappy@'
+  this._uncompressedPrefix = '@orig@'
+  this._compressionEnabled = (options && options.compressionEnabled) || false
 }
@@ -43,3 +53,3 @@ util.inherits(RedisConnection, CacheInstance)
   var deferred = Q.defer()
-  var params = [key, val, 'PX', maxAgeMs]
+  var params = [key, this._compress(val), 'PX', maxAgeMs]
   if (setWhenNotExist) params.push('NX')
@@ -60,3 +70,3 @@ this._client.set(params, this._makeNodeResolverWithTimeout(deferred, 'set', 'Redis [set] key: ' + key))
     for (i = 0, l = items.length; i < l; i++) {
-      commands.push(['set', items[i].key, items[i].value, 'PX', maxAgeMs, 'NX'])
+      commands.push(['set', items[i].key, this._compress(items[i].value), 'PX', maxAgeMs, 'NX'])
     }
@@ -70,3 +80,3 @@ } else {
       // Append key value arguments to the set command.
-      msetCommand.push(key, items[i].value)
+      msetCommand.push(key, this._compress(items[i].value))
       // Append an expire command.
@@ -100,3 +110,3 @@ commands.push(['EXPIRE', key, Math.floor(maxAgeMs / 1000)])
   if (!keys || !keys.length) return Q.resolve([])
-
+  var self = this
   var deferred = Q.defer()
@@ -114,3 +124,8 @@ this._client.mget(keys,
       for (var i = 0; i < vals.length; i++) {
-        if (null === vals[i]) vals[i] = undefined
+        if (null === vals[i]) {
+          vals[i] = undefined
+        } else {
+          //for real values determine if you need to decompress
+          vals[i] = self._decompress(vals[i])
+        }
       }
@@ -250,2 +265,53 @@ return vals
 /**
+ * Private method controls how all cache values are encoded.
+ *
+ * @param {string|undefined|null} value Original cache value
+ * @return {string|undefined|null} Value encoded appropriately for the cache
+ */
+RedisConnection.prototype._compress = function (value) {
+  if (!value || !this._compressionEnabled) {
+    return value
+  }
+
+  if (value.length > this._snappyPivot) {
+    try {
+      var compressed = snappy.compressSync(value)
+      return this._compressedPrefix + compressed.toString('base64')
+    } catch (e) {
+      console.warn("Compression failed: " + e.message)
+      return this._uncompressedPrefix + value 
+    }
+  } else {
+    return this._uncompressedPrefix + value
+  }
+}
+
+/**
+ * Private Method that knows how to parsed encoded cache value and decode.
+ *
+ * @param {string|undefined|null} value Possibly encoded value retrieved from the cache.
+ * @return {string|undefined|null} The original input value
+ */
+RedisConnection.prototype._decompress = function (value) {
+  if (!value) return value
+
+  // Note: always check prefixes even if compression is disabled, as there might
+  // be entries from prior to disabling compression
+  if (value.indexOf(this._compressedPrefix) === 0) {
+    try {
+      var compressedBuf = new Buffer(value.substring(this._compressedPrefix.length), 'base64')
+      var orig = snappy.decompressSync(compressedBuf, snappy.parsers.string)
+      return orig
+    } catch (e) {
+      console.warn("Decompression failed: " + e.message)
+      return undefined 
+    }  
+  } else if (value.indexOf(this._uncompressedPrefix) === 0) {
+    return value.substring(this._uncompressedPrefix.length)
+  } else {
+    return value
+  }
+}
+
+/**
  * Return the first result from a result set
@@ -252,0 +318,0 @@ * @param  {Array.<Object>} results the results

package.json

 {
   "name": "zcache",
   "description": "AWS zone-aware multi-layer cache",
-  "version": "0.4.0-alpha",
+  "version": "0.4.0-beta",
   "homepage": "https://github.com/Medium/zcache",
@@ -24,3 +24,4 @@ "authors": [
     "metrics": "0.1.6",
-    "hashring": "1.0.3"
+    "hashring": "1.0.3",
+    "snappy": "2.1.2"
   },
@@ -31,3 +32,4 @@ "devDependencies": {
     "logg": "0.2.2",
-    "closure-npc": "0.1.3"
+    "closure-npc": "0.1.3",
+    "sinon": "git://github.com/Medium/Sinon.JS.git#xiao-fix-clearTimeout-for-nodejs"
   },
@@ -40,3 +42,4 @@ "externDependencies": {
     "generic-pool": "./externs/generic-pool.js",
-    "metrics": "./externs/metrics.js"
+    "metrics": "./externs/metrics.js",
+    "snappy": "./externs/snappy.js"
   },
@@ -43,0 +46,0 @@ "scripts": {

test/test_ConnectionWrapper.js

@@ -5,3 +5,3 @@ var zcache = require('../index')
 exports.testConnectionWrapper = function (test) {
-  var wrappedCacheInstance = new zcache.MemcacheConnection("localhost", 11212)
+  var wrappedCacheInstance = new zcache.MemcacheConnection("localhost", 11211)
   var cacheInstance = new zcache.ConnectionWrapper(wrappedCacheInstance)
@@ -8,0 +8,0 @@

test/test_InMemoryCache.js

 var zcache = require('../index')
 var Q = require('kew')
+var sinon = require('sinon')
 
 // TODO: these test cases should be using nodeunitq
+var clock
 
 exports.setUp = function (callback) {
+  clock = sinon.useFakeTimers()
   this.cI = new zcache.InMemoryCache()
@@ -16,2 +19,3 @@ this.cI.connect()
   this.cI.destroy()
+  clock.restore()
   callback()
@@ -74,2 +78,4 @@ }
   }.bind(this), 2)
+
+  clock.tick(2)
 }
@@ -92,7 +98,9 @@
 
-  // the item should be reaped after the reaper interval of 5000 ms
+  // the item should be reaped after the reaper interval of 3000 ms
   setTimeout(function () {
     test.deepEqual(this.cI._data['foo'], undefined, 'foo should not be in the cache')
     test.done()
-  }.bind(this), 3001)
+  }.bind(this), 3000)
+
+  clock.tick(3000)
 }
@@ -115,2 +123,4 @@
   }, 750)
+
+  clock.tick(750)
 }
@@ -206,2 +216,4 @@
   }, 1101)
+
+  clock.tick(1101)
 }
@@ -208,0 +220,0 @@

test/test_MemcacheConnection.js

@@ -5,3 +5,3 @@ var zcache = require('../index')
 exports.testMemcacheConnection = function (test) {
-  var cacheInstance = new zcache.MemcacheConnection("localhost", 11212)
+  var cacheInstance = new zcache.MemcacheConnection("localhost", 11211)
 
@@ -65,3 +65,3 @@ test.equal(cacheInstance.isAvailable(), false, "Connection should not be available")
 exports.testMemcacheConnectionBase64 = function (test) {
-  var cacheInstance = new zcache.MemcacheConnection("localhost", 11212, 'base64')
+  var cacheInstance = new zcache.MemcacheConnection("localhost", 11211, 'base64')
 
@@ -68,0 +68,0 @@ test.equal(cacheInstance.isAvailable(), false, "Connection should not be available")

test/test_RedisConnection.js

 var zcache = require('../index')
 var ServerInfo = require('../lib/ServerInfo')
 var Q = require('kew')
+var nodeunitq = require('nodeunitq')
+var builder = new nodeunitq.Builder(exports)
+var redis = require('redis')
 
-exports.testRedisConnection = function (test) {
+builder.add(function testRedisConnection(test) {
   var cacheInstance = new zcache.RedisConnection('localhost', 6379)
@@ -49,2 +52,3 @@
       .then(function () {
+
         return cacheInstance.mset([{
@@ -106,5 +110,5 @@ key: 'a',
   cacheInstance.connect()
-}
+})
 
-exports.testSetNotExist = function (test) {
+builder.add(function testSetNotExist(test) {
   var cacheInstance = new zcache.RedisConnection('localhost', 6379)
@@ -140,5 +144,5 @@
   cacheInstance.connect()
-}
+})
 
-exports.testMsetNotExist = function (test) {
+builder.add(function testMsetNotExist(test) {
   var cacheInstance = new zcache.RedisConnection('localhost', 6379)
@@ -188,3 +192,3 @@
   cacheInstance.connect()
-}
+})
 
@@ -195,3 +199,3 @@ // Test .set() with TTL
 //  (3) get the key, and it should return 'undefined'.
-exports.testSetTimeout = function (test) {
+builder.add(function testSetTimeout(test) {
   var cacheInstance = new zcache.RedisConnection('localhost', 6379)
@@ -229,3 +233,3 @@
   cacheInstance.connect()
-}
+})
 
@@ -237,3 +241,3 @@ // Test .mset() with 'setWhenNotExist' set and TTL
 //  (4) the two new keys should have expired and the two old keys should still exist and have the old value
-exports.testMsetNotExistTimeout = function (test) {
+builder.add(function testMsetNotExistTimeout(test) {
   var cacheInstance = new zcache.RedisConnection('localhost', 6379)
@@ -286,5 +290,5 @@
   cacheInstance.connect()
-}
+})
 
-exports.testCounts = function (test) {
+builder.add(function testCounts(test) {
   var cacheInstance = new zcache.RedisConnection('localhost', 6379)
@@ -326,3 +330,194 @@
   cacheInstance.connect()
+})
+
+
+// Helper function to test all the scenarios of compression, off, on, and dual mode
+function runCommonTest(cacheInstancePut, cacheInstanceGet, test, compressionFlag) {
+  var bigDeferred = Q.defer()
+
+  // longVal will have a length of 1180 after the for loop is executed, making it longer than the pivot
+  var longVal = 'A long string that should be compressed because it is greater than 750 chars'
+  for (i = 0; i < 4; i++) { 
+   longVal = longVal.concat(longVal) 
+  }
+  var longValOn = '@snappy@wAnwPEEgbG9uZyBzdHJpbmcgdGhhdCBzaG91bGQgYmUgY29tcHJlc3NlZCBiZWNhdXNlIGl0IGlzIGdyZWF0ZXIBMChuIDc1MCBjaGFyc/5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAP5MAM5MAA=='
+  var tinyVal = 'tiny'
+  var tinyValOn = '@orig@tiny'
+  var nullVal = 'null'
+  var undefinedVal = 'undefined'
+    
+  var items = [
+    {key: 'longValue2', value: longVal},
+    {key: 'tinyValue2', value: tinyVal}
+  ]
+   
+  var testKeys = ['longValue', 'tinyValue', 'longValue2', 'tinyValue2', 'nullValue', 'undefinedValue']
+  var expValsWithoutCompression = [longVal, tinyVal, longVal, tinyVal, nullVal, undefinedVal]
+  var expValsWithCompression = [longValOn, tinyValOn, longValOn, tinyValOn, nullVal, undefinedVal]
+  var expVals = compressionFlag ? expValsWithCompression : expValsWithoutCompression
+
+  var populateSetterFuncs = function() {
+    return [ 
+      cacheInstancePut.set('longValue', longVal, 100000),
+      cacheInstancePut.set('tinyValue', tinyVal, 100000),
+      cacheInstancePut.set('nullValue', null, 100000),
+      cacheInstancePut.set('undefinedValue', undefined, 100000),
+      cacheInstancePut.mset(items, 100000) 
+    ]
+  }
+
+  var redisClient = redis.createClient(6379, 'localhost')
+  redisClient.on('error', function (err) {
+     console.log("error event - " + err)
+  })
+
+  var destroyRedisClient = function () {
+    redisClient.quit()
+    delete redisClient
+    return bigDeferred.resolve()
+  }
+
+  Q.all(populateSetterFuncs()).then(function () {
+    //Retrieve all items from redis directly to inspect format
+    var deferred = Q.defer()
+    redisClient.mget(testKeys, function(err, value) {
+          return deferred.resolve(value)
+    })
+    return deferred.promise
+        
+   }).then(function (vals) {
+       // confirm cache entries look good
+       test.deepEqual(expVals, vals)
+       return Q.resolve() 
+   }).then(function () { return cacheInstanceGet.get('longValue')
+   }).then(function (val) {
+       // confirm get works   
+       test.equal(longVal, val)  
+       return cacheInstanceGet.mget(testKeys)
+   }).then(function (vals) {
+       // confirm mget works
+       test.deepEqual(expValsWithoutCompression, vals)
+       destroyRedisClient()
+   }) 
+   .fail(function (e) {
+       console.error(e)
+       test.fail(e.message)
+       destroyRedisClient()
+   })  
+
+   return bigDeferred.promise  
 }
 
+// Test 1: Compression Off
+builder.add(function testCompressionOff(test) {
+  var cacheInstance = new zcache.RedisConnection('localhost', 6379, {requestTimeoutMs : 100})
+
+  cacheInstance.on('connect', function () {
+    cacheInstance.removeAllListeners('connect')
+    test.equal(cacheInstance.isAvailable(), true, 'Connection should be available')
+    runCommonTest(cacheInstance, cacheInstance, test, false)
+      .fin(function () {
+        cacheInstance.destroy()
+      })  
+  })
+
+  cacheInstance.on('destroy', function () {
+    test.done()
+  })
+
+  cacheInstance.connect()
+})
+
+// Test 2: Compression On
+builder.add(function testCompressionOn(test) {
+  var cacheInstance = new zcache.RedisConnection('localhost', 6379, {compressionEnabled : true, requestTimeoutMs : 100})
+
+  cacheInstance.on('connect', function () {
+    cacheInstance.removeAllListeners('connect')
+    test.equal(cacheInstance.isAvailable(), true, 'Connection should be available')
+    runCommonTest(cacheInstance, cacheInstance, test, true)
+      .fin(function () {
+         cacheInstance.destroy()
+      })
+  })
+
+  cacheInstance.on('destroy', function () {
+    test.done()
+  })
+
+  cacheInstance.connect()
+})
+
+// Test 3: Writing Client compression Off, Reading Client compression On
+builder.add(function testCompressionPutOffGetOn(test) {
+  var cacheInstancePut = new zcache.RedisConnection('localhost', 6379, {requestTimeoutMs : 100})
+  var cacheInstanceGet = new zcache.RedisConnection('localhost', 6379, {compressionEnabled : true, requestTimeoutMs : 100})
+
+  cacheInstancePut.on('connect', function () {
+    cacheInstancePut.removeAllListeners('connect')
+    test.equal(cacheInstancePut.isAvailable(), true, 'Connection should be available')
+    cacheInstanceGet.connect()
+  })
+
+  cacheInstanceGet.on('connect', function () {
+    cacheInstanceGet.removeAllListeners('connect')
+    test.equal(cacheInstanceGet.isAvailable(), true, 'Connection should be available')
+    runCommonTest(cacheInstancePut, cacheInstanceGet, test, false)
+      .fin(function () {
+         cacheInstancePut.destroy()
+         cacheInstanceGet.destroy()
+      }) 
+  })
+
+  var count = 0
+  var destroy = function () { 
+    if (++count === 2) test.done() 
+  }
+
+  cacheInstancePut.on('destroy', function() {
+    destroy()
+  })
+  cacheInstanceGet.on('destroy', function () {
+     destroy()
+  })
+
+  cacheInstancePut.connect()
+})
+
+// Test 4: Writing Client compression On, Reading Client compression Off
+builder.add(function testCompressionPutOnGetOff(test) {
+  var cacheInstanceGet = new zcache.RedisConnection('localhost', 6379, {requestTimeoutMs : 100})
+  var cacheInstancePut = new zcache.RedisConnection('localhost', 6379, {compressionEnabled : true, requestTimeoutMs : 100})
+
+  cacheInstancePut.on('connect', function () {
+    cacheInstancePut.removeAllListeners('connect')
+    test.equal(cacheInstancePut.isAvailable(), true, 'Connection should be available')
+    cacheInstanceGet.connect()
+  })
+
+  cacheInstanceGet.on('connect', function () {
+    cacheInstanceGet.removeAllListeners('connect')
+    test.equal(cacheInstanceGet.isAvailable(), true, 'Connection should be available')
+    runCommonTest(cacheInstancePut, cacheInstanceGet, test, true)
+      .fin(function () {
+         cacheInstancePut.destroy()
+         cacheInstanceGet.destroy()         
+      })
+  })
+ 
+  var count = 0
+  var destroy = function () { 
+   if (++count === 2) test.done() 
+  }
+
+  cacheInstancePut.on('destroy', function() {
+    destroy()
+  })
+  cacheInstanceGet.on('destroy', function () {
+    destroy()
+  })
+
+  cacheInstancePut.connect()
+})
+
+

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

165684

increased by0.26%

Number of package files

39

increased by2.63%

Worsened metrics

Dependency count

8

increased by14.29%

Dev dependency count

5

increased by25%

Lines of code

4091

decreased by-0.87%

Number of lines in readme file

5

decreased by-96.24%

Dependency changes

• + Addedsnappy@2.1.2

• + Addednan@0.6.0(transitive)

• + Addedsnappy@2.1.2(transitive)