cacheskin - npm Package Compare versions

Comparing version 0.0.1-dev to 0.1.0

index.js

@@ -5,3 +5,5 @@ /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */
 
-module.exports = require(__dirname + '/lib/cache.js');
+var LIBPATH = __dirname + (process.env.CACHE_SKIN_COV ? '/lib-cov' : '/lib');
 
+module.exports = require(LIBPATH + '/cacheskin.js');
+

lib/cacheskin.js

@@ -5,9 +5,61 @@ /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */
 
-exports.create = function (options, storage, taginfo) {
+var TimeoutError  = function (msg) {
+  var e = new Error(msg);
+  e.name = 'TimeoutError';
+  return e;
+};
 
+var BuildCacheKey = function (key) {
+  var hash1 = 5381;
+  var hash2 = 0;
+
+  key = (key instanceof Buffer) ? key : new Buffer(key);
+  for (var i = 0, len = key.length; i < len; i++) {
+    hash1 = (hash1 << 5 + hash1) + key[i];
+    hash2 = (hash2 << 4) ^ (hash2 >> 28) ^ key[i];
+  }
+
+  return ([hash1, hash2, key.length]).join(':');
+};
+
+var trim = function (str) {
+  var m = str.length;
+  for (var i = 0; i < m; i++) {
+    if (str.charCodeAt(i) > 32) {
+      break;
+    }
+  }
+
+  for (var j = m - 1; j > i; j--) {
+    if (str.charCodeAt(j) > 32) {
+      break;
+    }
+  }
+
+  return str.slice(i, j + 1);
+};
+
+var pack = function (data) {
+  return JSON.stringify(data);
+};
+
+var unpack = function (data) {
+  var _me = {};
+  try {
+    _me = JSON.parse(data);
+  } catch (e) {
+  }
+
+  return _me;
+};
+
+exports.create = function (options, storage, tagsync) {
+
   /**
-   *
+   * @ 配置参数
    */
   var _options = {
-    'timeout' : 500,          /**<  读写超时时间(ms)  */
+    'timeout'   : 500,          /**<  读写超时时间(ms)  */
+    'lifetime'  : 86400000,     /**<  默认存活时间(ms)  */
+    'tag_sync_interval' : 1000, /**<  TAG 同步周期(ms)  */
   };
@@ -18,17 +70,140 @@ for (var i in options) {
 
+  /**
+   * @ TAG信息
+   */
+  var _tags = {};
+
+  var _modifiedtags = {};
+
+  /* {{{ private function _sync_tags_info() */
+  (function _sync_tags_info() {
+    if (!tagsync) {
+      return;
+    }
+
+    for (var i in _modifiedtags) {
+      (function () {
+        var t = _modifiedtags[i];
+        tagsync.write(i, t, function (error) {
+          if (error && (!_modifiedtags[i] || t > _modifiedtags[i])) {
+            _modifiedtags[i] = t;
+          }
+        });
+      })();
+      delete _modifiedtags[i];
+    }
+
+    tagsync.reload(function (error, res) {
+      setTimeout(_sync_tags_info, _options.tag_sync_interval);
+      if (error) {
+        return;
+      }
+
+      var map = {};
+      for (var i in res) {
+        map[i] = _tags[i] ? Math.max(res[i], _tags[i]) : res[i];
+        if (_modifiedtags[i] && _modifiedtags[i] <= res[i]) {
+          delete _modifiedtags[i];
+        }
+      }
+      _tags = map;
+    });
+  })();
+  /* }}} */
+
   var _me = {};
 
-  _me.set = function (key, value, config, callback) {
+  /* {{{ public function set() */
+  /**
+   * @ options : {Object} with:
+   *  lifetime:
+   *  tags :
+   */
+  _me.set = function (key, value, options, callback) {
+
+    key = trim(key);
+    if ('function' === (typeof options)) {
+      callback = options;
+      options = {};
+    }
+
+    var tm1 = setTimeout(function () {
+      callback(TimeoutError('cache set timeout after ' + _options.timeout + ' ms.'));
+      callback = function () {};
+    }, _options.timeout);
+
+    var now = Date.now();
+    var ttl = options.lifetime || _options.lifetime;
+    var val = pack({
+      'i' : now,
+        'e' : now + ttl,
+        'k' : key,
+        'v' : value,
+        't' : options.tags ? (Array.isArray(options.tags) ? options.tags : [options.tags]) : [],
+    });
+    storage.set(BuildCacheKey(key), val, function (error, res) {
+      clearTimeout(tm1);
+      tm1 = null;
+      callback(error, res);
+    }, ttl);
   };
+  /* }}} */
 
+  /* {{{ public function get() */
   _me.get = function (key, callback) {
-  };
+    key = trim(key);
+    var tm1 = setTimeout(function () {
+      callback(TimeoutError('cache get timeout after ' + _options.timeout + ' ms.'));
+      callback = function () {};
+    }, _options.timeout);
 
-  _me.remove = function (key, callback) {
+    storage.get(BuildCacheKey(key), function (error, res) {
+      clearTimeout(tm1);
+      tm1 = null;
+
+      if (error) {
+        return callback(error, null);
+      }
+
+      var now = Date.now();
+      res = unpack(res);
+
+      if (!res.i || !res.k || key !== res.k || !res.e || res.e < now) {
+        return callback(null, null);
+      }
+
+      var the = Array.isArray(res.t) ? res.t : [];
+      var len = the.unshift('__global__');
+
+      for (var i = 0; i < len; i++) {
+        var idx = trim(the[i]);
+        if (_tags[idx] && res.i <= _tags[idx]) {
+          return callback(null, null);
+        }
+      }
+
+      callback(null, res.v, res.e - now);
+    });
   };
+  /* }}} */
 
-  _me.cleanByTag = function (tag, callback) {
+  /* {{{ public function removeByTag() */
+  _me.removeByTag = function (tag, delay, flush) {
+    if ('string' !== (typeof tag)) {
+      return;
+    }
+
+    var now = Date.now() + (delay ? parseInt(delay, 10) : 0);
+
+    tag = trim(tag);
+    _modifiedtags[tag] = now;
+    _tags[tag] = now;
+    if (flush && tagsync) {
+      tagsync.write(tag, now, function (error) {});
+    }
   };
+  /* }}} */
 
   return _me;
 };
+

package.json

 {
   "name": "cacheskin",
-    "version": "0.0.1-dev",
+    "version": "0.1.0",
     "author": "Aleafs Zhang ([e-mail])",
@@ -14,4 +14,4 @@ "contributors": [
     "repository": "git://github.com/aleafs/cacheskin.git",
-    "description": "Http service switch on/off filter.",
-    "keywords": [ "node", "vip", "onoff" ],
+    "description": "cache consistency manager in Node.js.",
+    "keywords": [ "cache", "consistency", "manager" ],
     "dependencies": {},
@@ -23,3 +23,4 @@ "engines": {
       "should"  : ">=0.4.2",
-      "mocha"   : ">=0.9.0"
+      "mocha"   : ">=0.9.0",
+      "visionmedia-jscoverage" : ">=1.0.0"
     },
@@ -26,0 +27,0 @@ "main"  : "./index.js",

README.md

@@ -1,6 +0,10 @@
-[![Build Status](https://secure.travis-ci.org/aleafs/cache.png?branch=master)](http://travis-ci.org/aleafs/cache)
+[![Build Status](https://secure.travis-ci.org/aleafs/cacheskin.png?branch=master)](http://travis-ci.org/aleafs/cacheskin)
 
 ## About
 
-缓存一致性管理组件，正在开发
+* `cacheskin`是对缓存访问的一个基础封装，重点为解决分布式系统中的数据一致性问题，支持按时间及tag进行缓存的失效检查；
+* 支持缓存读写的超时控制；
+* 由于采用时间戳来作为数据版本号，`cacheskin`并没有从理论上严格保证缓存与后台数据的一致性，但在大多数工业场景下，`cacheskin`足以满足需求；
+* 原理与异常场景请参考[《分布式系统缓存设计浅析》](cacheskin/blob/master/doc/cache_designer.pdf)；
+* `cacheskin` 没有要求你采用什么方案存储缓存和TAG信息，这给使用者最大的灵活度。但为了接入`cacheskin`，开发者仍然需要满足缓存Storage以及TAG信息符合给定的接口，详细情况可参考[interface](cacheskin/blob/master/lib/interface.js)。
 
@@ -10,6 +14,6 @@ ## Install
 ```bash
-$ npm install cache
+$ npm install cacheskin
 ```
 
-## Usage
+## API
 

test/cache.test.js

@@ -6,24 +6,26 @@ /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */
 
-/* {{{ private function Handle() */
+/* {{{ private function Storage() */
 /**
  * @缓存存储引擎
  */
-var Handle  = function() {
+var Storage = function (latency) {
 
-  var _obj  = {};
+  var _obj = {};
 
-  var _me   = {};
+  var _me = {};
 
-  _me.set   = function(key, value, callback) {
-    _obj[key]   = value;
-    callback(null);
-  };
+  _me.set = function(key, value, callback) {
+    var fn = function () {
+      _obj[key] = value;
+      callback(null);
+    }
 
-  _me.get   = function(key, callback) {
-    callback(null, _obj[key]);
+    latency ? setTimeout(fn, latency) : fn();
   };
 
-  _me.delete    = function(key, callback) {
-    delete _obj[key];
-    callback(null);
+  _me.get = function(key, callback) {
+    var fn = function () {
+      callback(null, _obj[key]);
+    }
+    latency ? setTimeout(fn, latency) : fn();
   };
@@ -35,7 +37,7 @@
 
-/* {{{ private function Keeper() */
+/* {{{ private function Tagmeta() */
 /**
  * @模拟的tag info存储引擎
  */
-var Keeper  = function() {
+var Tagmeta  = function() {
 
@@ -69,3 +71,3 @@ var _info = {
   };
-  _me.load  = function(callback, _after) {
+  _me.reload = function(callback) {
     callback(_info.error ? (new Error('TestError')) : null, _info);
@@ -78,10 +80,13 @@ };
 
-describe('cache management', function() {
+describe('cache skin', function() {
 
-  /* {{{ should_cache_set_and_get_and_unset_works_fine() */
-  it('should_cache_set_and_get_and_unset_works_fine', function(done) {
+  /* {{{ should_cache_set_and_get_works_fine() */
+  it('should_cache_set_and_get_works_fine', function(done) {
     var num = 2;
-    var _me = cache.create('test1', Handle());
+    var _me = cache.create({}, Storage());
 
-    _me.set('key1', {'a' : 'val1', 'b' : [2]}, function(error) {
+    var options = {
+      'lifetime'  : 2,
+    };
+    _me.set('key1', {'a' : 'val1', 'b' : [2]}, options, function(error) {
       should.ok(!error);
@@ -92,16 +97,9 @@ _me.get('key1', function(error, value, expire) {
         expire.should.be.within(0,2);
-        _me.unset('key1', function(error) {
-          should.ok(!error);
-          _me.get('key1', function(error, value, expire) {
-            should.ok(!error);
-            should.ok(!value);
-            if ((--num) <= 0) {
-              done();
-            }
-          });
-        });
+
+        if ((--num) <= 0) {
+          done();
+        }
       });
+    });
 
-    }, 2);
-
     _me.get('i am not exists', function(error, value, expire) {
@@ -117,20 +115,6 @@ should.ok(!error);
 
-  /* {{{ should_unexpected_cache_works_fine() */
-  it('should_unexpected_cache_works_fine', function(done) {
-    var res = Handle();
-    var _me = cache.create('test2', res);
-
-    res.set(cache.getkey('test2#key1'), JSON.stringify({'a' : 'fwekksgeg'}), function(error) {
-      _me.get('key1', function(error, value, expire) {
-        error.toString().should.eql('Error: UnExpectCacheValue');
-        done();
-      });
-    });
-  });
-  /* }}} */
-
   /* {{{ should_cache_expire_works_fine() */
   it('should_cache_expire_works_fine', function(done) {
     var num = 1;
-    var _me = cache.create('test2', Handle());
+    var _me = cache.create({'lifetime' : 1}, Storage());
     _me.set('key1', 'val1', function(error) {
@@ -140,20 +124,23 @@
         value.should.eql('val1');
+        setTimeout(function() {
+          _me.get('key1', function(error, value, expire) {
+            should.ok(!error);
+            should.ok(null === value);
+            if ((--num) <= 0) {
+              done();
+            }
+          });
+        }, 2);
       });
+    });
 
-      setTimeout(function() {
-        _me.get('key1', function(error, value, expire) {
-          should.ok(!error);
-          should.ok(null === value);
-          if ((--num) <= 0) {
-            done();
-          }
-        });
-      }, 2);
-    }, 1);
-
     ++num;
-    _me.set('key2', 'val2', function(error) {
+    var options = {
+      'lifetime' : 86400000,
+      'tags' : ['table1', ' table2']
+    };
+    _me.set('key2', 'val2', options, function(error) {
 
       should.ok(!error);
-      _me.tagrm('table3');
+      _me.removeByTag('table3');
 
@@ -166,3 +153,3 @@ _me.get('key2', function(error, value, expire) {
         setTimeout(function() {
-          _me.tagrm('table2');
+          _me.removeByTag('table2');
           _me.get('key2', function(error, value, expire) {
@@ -177,22 +164,2 @@ should.ok(!error);
       });
-    }, null, ['table1', 'table2']);
-  });
-  /* }}} */
-
-  /* {{{ should_gzip_with_buffer_works_fine() */
-  it('should_gzip_with_buffer_works_fine', function(done) {
-    var Zlib    = require('zlib');
-    Zlib.gzip(JSON.stringify({
-      '_me' : 'abcdefghijklmnopqrstuvwxyz0123456',
-      '_hi' : 0x2312312,
-    }), function(error, value) {
-      should.ok(!error);
-      Zlib.gunzip(value, function(error, data) {
-        should.ok(!error);
-
-        var _me = JSON.parse(data);
-        _me.should.have.property('_me', 'abcdefghijklmnopqrstuvwxyz0123456');
-        _me.should.have.property('_hi', 0x2312312);
-        done();
-      });
     });
@@ -206,80 +173,72 @@ });
     // XXX: 同一个keeper，两个cache对象（模拟两台机器，或者两个进程）的taginfo会共享
-    var kep = Keeper();
+    var kep = Tagmeta();
 
-    var me1 = cache.create('test6_1', Handle(), kep, {
-      'tag_flush_interval'  : 1,
-    });
-    var me2 = cache.create('test6_2', Handle(), kep, {
-      'tag_flush_interval'  : 1,
-    });
+    var tagmeta = Tagmeta();
+    var options = {
+      'tag_sync_interval' : 10
+    };
 
-    me1.set('key1', 'val1', function(error) {
+    var me1 = cache.create(options, Storage(), tagmeta);
+    var me2 = cache.create(options, Storage(), tagmeta);
+
+    var options = {
+      'tags'  : ['right','delay']
+    };
+
+    me1.set('key1', 'val1', options, function(error) {
       me1.get('key1', function(error, value) {
         value.should.eql('val1');
-        me2.set('key1', 'val2', function(error) {
+        me2.set('key1', 'val2', options, function(error) {
           me2.get('key1', function(error, value) {
             value.should.eql('val2');
+            me2.removeByTag('delay');
+            me2.get('key1', function(error, value) {
 
-            setTimeout(function() {
-              me2.tagrm('delay');
-              me2.get('key1', function(error, value) {
+              // me2 立即生效, 并且在1ms后向keeper同步tag info
+              should.ok(!error);
+              should.ok(null === value);
 
-                // me2 立即生效, 并且在1ms后向keeper同步tag info
-                should.ok(!error);
-                should.ok(null === value);
-
-                // me1 要等到同步完成后才能见到效果
-                me1.get('key1', function(error, value) {
-                  value.should.eql('val1');
-                  setTimeout(function() {
-                    me1.get('key1', function(error, value) {
-                      should.ok(!error);
-                      should.ok(null === value);
-                      done();
-                    });
-                  }, 80);
-                });
+              // me1 要等到同步完成后才能见到效果
+              me1.get('key1', function(error, value) {
+                value.should.eql('val1');
+                setTimeout(function() {
+                  me1.get('key1', function(error, value) {
+                    should.ok(!error);
+                    should.ok(null === value);
+                    done();
+                  });
+                }, 25);
               });
-            }, 1);
+            });
           });
-        }, null, ['right', 'delay']);
+        });
       });
-    }, null, ['right', 'delay']);
-
-
+    });
   });
   /* }}} */
 
-  /* {{{ should_cache_protected_after_update_works_fine() */
-  it('should_cache_protected_after_update_works_fine', function(done) {
-    var _me = cache.create('test7', Handle(), null);
+  it('should_cache_get_and_set_timeout_works_fine', function (_done) {
+    var _me = cache.create({'timeout' : 15}, Storage(20));
+    var num = 2;
+    var done = function () {
+      if (0 === (--num)) {
+        _done();
+      }
+    };
 
-    _me.set('key1', 'val1', function(error) {
-      should.ok(!error);
-      _me.get('key1', function(error, result, expire) {
-        should.ok(!error);
-        result.should.eql('val1');
+    _me.get('key1', function (error, res) {
+      should.ok(!res);
+      error.should.have.property('name', 'TimeoutError');
+      error.should.have.property('message', 'cache get timeout after 15 ms.');
+      setTimeout(done, 25);
+    });
 
-        _me.tagrm('tag1', 3);
-        _me.set('key1', 'val1', function(error) {
-          should.ok(!error);
-          _me.get('key1', function(error, result, expire) {
-            should.ok(!error);
-            should.ok(null === result);
-            setTimeout(function() {
-              _me.set('key1', 'val1', function(error){
-                _me.get('key1', function(error, result, expire) {
-                  result.should.eql('val1');
-                  done();
-                });
-              }, null, ['tag1']);
-            }, 7);
-          });
-        }, null, ['tag1']);
-      });
-    }, null, ['tag1']);
+    _me.set('key1', 'val1', function (error) {
+      error.should.have.property('name', 'TimeoutError');
+      error.should.have.property('message', 'cache set timeout after 15 ms.');
+      setTimeout(done, 25);
+    });
   });
-  /* }}} */
 
 });
 

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

927015

increased by5812.46%

Number of package files

12

increased by9.09%

Number of lines in readme file

19

increased by26.67%

Worsened metrics

Dev dependency count

3

increased by50%

Lines of code

400

decreased by-13.61%

Number of low supply chain risk alerts

4

increased by33.33%

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes