disk-memoizer - npm Package Compare versions

Comparing version 2.2.0 to 2.2.3

.eslintrc.js

@@ -172,3 +172,3 @@ module.exports = {
     "no-spaced-func": "error",
-    "no-sync": "off",
+    "no-sync": "error",
     "no-tabs": "error",
@@ -175,0 +175,0 @@ "no-template-curly-in-string": "error",

dist/config.js

@@ -22,3 +22,3 @@ "use strict";
   CACHE_DIR: DISK_MEMOIZER_CACHE_DIR || os.tmpdir() + "/disk-memoizer",
-  LOCK_STALE_MS: +(DISK_MEMOIZER_LOCK_STALE_MS || 30 * 1000),
+  LOCK_STALE_MS: +(DISK_MEMOIZER_LOCK_STALE_MS || 5 * 1000),
   GC: cluster.isMaster && DISK_MEMOIZER_GC !== "false",
@@ -25,0 +25,0 @@ GC_INTERVAL: +(DISK_MEMOIZER_GC_INTERVAL || 1000 * 60 * 5),

dist/disk_memoizer.js

@@ -154,23 +154,36 @@ "use strict";
 
-  var isLocked = lockFile.checkSync(lockPath, { stale: lockStale });
+  lockFile.check(lockPath, { stale: lockStale }, function (err, isLocked) {
+    if (err) {
+      return callback(err);
+    }
 
-  if (isLocked) {
-    // We'll wait until the lock
-    waitForLockRelease({
-      lockPath: lockPath,
-      key: key,
-      cachePath: cachePath,
-      unmemoizedFn: unmemoizedFn,
-      marshaller: marshaller,
-      lockStale: lockStale,
-      memoryCache: memoryCache,
-      type: type
-    }, callback);
-    return;
-  }
+    if (isLocked) {
+      return delayedRead({
+        key: key,
+        cachePath: cachePath,
+        unmemoizedFn: unmemoizedFn,
+        marshaller: marshaller,
+        memoryCache: memoryCache,
+        lockStale: lockStale,
+        type: type
+      }, callback);
+    }
 
-  lockFile.lockSync(lockPath, { stale: lockStale });
+    lockFile.lock(lockPath, { stale: lockStale }, function (err) {
+      if (err) {
+        // A concurrent lock? We'll try to read again in a bit
+        return delayedRead({
+          key: key,
+          cachePath: cachePath,
+          unmemoizedFn: unmemoizedFn,
+          marshaller: marshaller,
+          memoryCache: memoryCache,
+          lockStale: lockStale,
+          type: type
+        }, callback);
+      }
+      unmemoizedFn.apply(undefined, _toConsumableArray(args.concat(grabAndCacheCallback)));
+    });
+  });
 
-  unmemoizedFn.apply(undefined, _toConsumableArray(args.concat(grabAndCacheCallback)));
-
   function grabAndCacheCallback(err, unmarshalledData) {
@@ -198,17 +211,8 @@ if (err) {
 
-        // We'll save the file on a temporary file to avoid in-flight files
-        // from being taken as complete
-        var tmpCachePath = cachePath + ".tmp";
-        fs.writeFile(tmpCachePath, data, function (err) {
+        fs.writeFile(cachePath, data, function (err) {
           if (err) {
-            debug("[error] Failed saving %s. Got error: %s", tmpCachePath, err.message);
+            debug("[error] Failed saving %s. Got error: %s", cachePath, err.message);
             return callback(err);
           }
-          // Rename the file once it's persisted in disk to make it available
-          // to any queued cache
-          fs.rename(tmpCachePath, cachePath, function (err) {
-            if (err) {
-              debug("[error] Failed saving %s. Got error: %s", cachePath, err.message);
-              return callback(err);
-            }
+          lockFile.unlock(lockPath, function () {
 
@@ -218,8 +222,3 @@ debug("[info] Saved cache for %s on %s", key, cachePath);
 
-            lockFile.unlock(lockPath, function (err) {
-              if (err) {
-                return callback(err);
-              }
-              callback(null, unmarshalledData);
-            });
+            callback(null, unmarshalledData);
           });
@@ -232,42 +231,13 @@ });
 
-var lockWatchers = {};
-function waitForLockRelease(_ref4, callback) {
-  var lockPath = _ref4.lockPath,
-      key = _ref4.key,
+function delayedRead(_ref4, callback) {
+  var key = _ref4.key,
       cachePath = _ref4.cachePath,
       unmemoizedFn = _ref4.unmemoizedFn,
       marshaller = _ref4.marshaller,
+      memoryCache = _ref4.memoryCache,
       lockStale = _ref4.lockStale,
-      memoryCache = _ref4.memoryCache,
       type = _ref4.type;
 
-  // We only want to keep one lock watcher per lock path
-  var hasWatcher = !!lockWatchers[lockPath];
-
-  lockWatchers[lockPath] = lockWatchers[lockPath] || [];
-  var currentWatchList = lockWatchers[lockPath];
-
-  if (hasWatcher) {
-    // If there's already a watcher there's no need to register a new
-    // one, we'll defer the execution of the task until the watcher notifies
-    // about changes on the lock
-    currentWatchList.push(runOnLockReleased);
-  } else {
-
-    // Register a singleton watcher for a particular lock file
-
-    var _watcherFn = null;
-    _watcherFn = function watcherFn() {
-      runOnLockReleased();
-      // Run all the watchers
-      currentWatchList.forEach(function (watcher) {
-        return watcher();
-      });
-      fs.unwatchFile(lockPath, _watcherFn);
-      currentWatchList.splice(0, currentWatchList.length);
-    };
-    fs.watch(lockPath, _watcherFn);
-  }
-
-  function runOnLockReleased() {
+  // We'll wait until the lock
+  setTimeout(function () {
     useCachedFile({
@@ -278,7 +248,7 @@ key: key,
       marshaller: marshaller,
+      memoryCache: memoryCache,
       lockStale: lockStale,
-      memoryCache: memoryCache,
       type: type
     }, callback);
-  }
+  }, 10);
 }
@@ -303,6 +273,12 @@
 
-  fs.readFile(cachePath, function (err, dataFromCache) {
-
+  var lockPath = getLockPath(cachePath);
+  lockFile.check(lockPath, { stale: lockStale }, function (err, isLocked) {
     if (err) {
-      return grabAndCache({
+      return callback(err);
+    }
+    if (isLocked) {
+      // If we've got this far and there's still a lock file, we've
+      // probably hit a race condition with another concurrent process.
+      // We'll retry when the lock is released.
+      return delayedRead({
         key: key,
@@ -312,2 +288,3 @@ cachePath: cachePath,
         marshaller: marshaller,
+        memoryCache: memoryCache,
         lockStale: lockStale,
@@ -318,13 +295,28 @@ type: type
 
-    debug("[info] Using disk cache for %s from %s", key, cachePath);
+    fs.readFile(cachePath, function (err, dataFromCache) {
 
-    marshaller.unmarshall(dataFromCache, function (err, data) {
       if (err) {
-        debug("[warning] Not caching %s. Failed marshalling data. Got error %s", key, err.message);
-        unmemoizedFn(key, callback);
-        return;
+        debug("[warning] Failed reading file %s from cache %s", key, cachePath);
+        return grabAndCache({
+          key: key,
+          cachePath: cachePath,
+          unmemoizedFn: unmemoizedFn,
+          marshaller: marshaller,
+          lockStale: lockStale,
+          type: type
+        }, callback);
       }
 
-      memoryCache.set(key, data);
-      callback(null, data);
+      debug("[info] Using disk cache for %s from %s", key, cachePath);
+
+      marshaller.unmarshall(dataFromCache, function (err, data) {
+        if (err) {
+          debug("[warning] Not caching %s. Failed marshalling data. Got error %s", key, err.message);
+          unmemoizedFn(key, callback);
+          return;
+        }
+
+        memoryCache.set(key, data);
+        callback(null, data);
+      });
     });
@@ -331,0 +323,0 @@ });

lib/config.js

@@ -16,3 +16,3 @@ // Environment variables
   // Time for a lock to be considered stale.
-  // Defaults to 30000 ms (30s)
+  // Defaults to 5000 ms (5s)
   DISK_MEMOIZER_LOCK_STALE_MS,
@@ -44,3 +44,3 @@
     `${os.tmpdir()}/disk-memoizer`,
-  LOCK_STALE_MS: +(DISK_MEMOIZER_LOCK_STALE_MS || 30 * 1000),
+  LOCK_STALE_MS: +(DISK_MEMOIZER_LOCK_STALE_MS || 5 * 1000),
   GC: cluster.isMaster && (DISK_MEMOIZER_GC !== "false"),
@@ -47,0 +47,0 @@ GC_INTERVAL: +(DISK_MEMOIZER_GC_INTERVAL || 1000 * 60 * 5),

lib/disk_memoizer.js

@@ -188,23 +188,39 @@ // Simple disk memoization and in memory LRU cache for high
 
-  const isLocked = lockFile.checkSync(lockPath, {stale: lockStale});
+  lockFile.check(lockPath, {stale: lockStale}, (err, isLocked) => {
+    if (err) {
+      return callback(err);
+    }
 
-  if (isLocked) {
-    // We'll wait until the lock
-    waitForLockRelease({
-      lockPath,
-      key,
-      cachePath,
-      unmemoizedFn,
-      marshaller,
-      lockStale,
-      memoryCache,
-      type
-    }, callback);
-    return;
-  }
+    if (isLocked) {
+      return delayedRead({
+        key,
+        cachePath,
+        unmemoizedFn,
+        marshaller,
+        memoryCache,
+        lockStale,
+        type
+      }, callback);
+    }
 
-  lockFile.lockSync(lockPath, {stale: lockStale});
 
-  unmemoizedFn(...args.concat(grabAndCacheCallback));
+    lockFile.lock(lockPath, {stale: lockStale}, (err) => {
+      if (err) {
+        // A concurrent lock? We'll try to read again in a bit
+        return delayedRead({
+          key,
+          cachePath,
+          unmemoizedFn,
+          marshaller,
+          memoryCache,
+          lockStale,
+          type
+        }, callback);
+      }
+      unmemoizedFn(...args.concat(grabAndCacheCallback));
+    });
 
+  });
+
+
   function grabAndCacheCallback(err, unmarshalledData) {
@@ -232,9 +248,6 @@ if (err) {
 
-        // We'll save the file on a temporary file to avoid in-flight files
-        // from being taken as complete
-        const tmpCachePath = `${cachePath}.tmp`;
-        fs.writeFile(tmpCachePath, data, (err) => {
+        fs.writeFile(cachePath, data, (err) => {
           if (err) {
             debug("[error] Failed saving %s. Got error: %s",
-              tmpCachePath,
+              cachePath,
               err.message
@@ -244,10 +257,3 @@ );
           }
-          // Rename the file once it's persisted in disk to make it available
-          // to any queued cache
-          fs.rename(tmpCachePath, cachePath, (err) => {
-            if (err) {
-              debug("[error] Failed saving %s. Got error: %s",
-                cachePath, err.message);
-              return callback(err);
-            }
+          lockFile.unlock(lockPath, () => {
 
@@ -257,8 +263,3 @@ debug("[info] Saved cache for %s on %s", key, cachePath);
 
-            lockFile.unlock(lockPath, (err) => {
-              if (err) {
-                return callback(err);
-              }
-              callback(null, unmarshalledData);
-            });
+            callback(null, unmarshalledData);
           });
@@ -272,5 +273,4 @@ });
 
-const lockWatchers = {};
-function waitForLockRelease({
-  lockPath,
+
+function delayedRead({
   key,
@@ -280,33 +280,8 @@ cachePath,
   marshaller,
+  memoryCache,
   lockStale,
-  memoryCache,
   type
 }, callback) {
-  // We only want to keep one lock watcher per lock path
-  const hasWatcher = !!lockWatchers[lockPath];
-
-  lockWatchers[lockPath] = lockWatchers[lockPath] || [];
-  const currentWatchList = lockWatchers[lockPath];
-
-  if (hasWatcher) {
-    // If there's already a watcher there's no need to register a new
-    // one, we'll defer the execution of the task until the watcher notifies
-    // about changes on the lock
-    currentWatchList.push(runOnLockReleased);
-  } else {
-
-    // Register a singleton watcher for a particular lock file
-
-    let watcherFn = null;
-    watcherFn = () => {
-      runOnLockReleased();
-      // Run all the watchers
-      currentWatchList.forEach((watcher) => watcher());
-      fs.unwatchFile(lockPath, watcherFn);
-      currentWatchList.splice(0, currentWatchList.length);
-    };
-    fs.watch(lockPath, watcherFn);
-  }
-
-  function runOnLockReleased() {
+  // We'll wait until the lock
+  setTimeout(() => {
     useCachedFile({
@@ -317,9 +292,10 @@ key,
       marshaller,
+      memoryCache,
       lockStale,
-      memoryCache,
       type
     }, callback);
-  }
+  }, 10);
 }
 
+
 function useCachedFile({
@@ -340,6 +316,12 @@ key,
 
-  fs.readFile(cachePath, (err, dataFromCache) => {
-
+  const lockPath = getLockPath(cachePath);
+  lockFile.check(lockPath, {stale: lockStale}, (err, isLocked) => {
     if (err) {
-      return grabAndCache({
+      return callback(err);
+    }
+    if (isLocked) {
+      // If we've got this far and there's still a lock file, we've
+      // probably hit a race condition with another concurrent process.
+      // We'll retry when the lock is released.
+      return delayedRead({
         key,
@@ -349,2 +331,3 @@ cachePath,
         marshaller,
+        memoryCache,
         lockStale,
@@ -355,15 +338,31 @@ type
 
-    debug("[info] Using disk cache for %s from %s", key, cachePath);
+    fs.readFile(cachePath, (err, dataFromCache) => {
 
-    marshaller.unmarshall(dataFromCache, (err, data) => {
       if (err) {
-        debug("[warning] Not caching %s. Failed marshalling data. Got error %s",
+        debug("[warning] Failed reading file %s from cache %s", key, cachePath);
+        return grabAndCache({
           key,
-          err.message);
-        unmemoizedFn(key, callback);
-        return;
+          cachePath,
+          unmemoizedFn,
+          marshaller,
+          lockStale,
+          type
+        }, callback);
       }
 
-      memoryCache.set(key, data);
-      callback(null, data);
+      debug("[info] Using disk cache for %s from %s", key, cachePath);
+
+      marshaller.unmarshall(dataFromCache, (err, data) => {
+        if (err) {
+          debug(
+            "[warning] Not caching %s. Failed marshalling data. Got error %s",
+            key,
+            err.message);
+          unmemoizedFn(key, callback);
+          return;
+        }
+
+        memoryCache.set(key, data);
+        callback(null, data);
+      });
     });
@@ -375,4 +374,4 @@ });
   return maxAge && creationTime
-  ? ((new Date().getTime()) - maxAge) > creationTime.getTime()
-   : false;
+    ? ((new Date().getTime()) - maxAge) > creationTime.getTime()
+    : false;
 }
@@ -420,3 +419,6 @@
 
-function getMarshaller({type, marshaller}) {
+function getMarshaller({
+  type,
+  marshaller
+}) {
   if (marshallers[type]) {
@@ -428,3 +430,3 @@ marshaller = marshallers[type];
 
-function fakeLruCache () {
+function fakeLruCache() {
   return {
@@ -431,0 +433,0 @@ has: () => false,

package.json

 {
   "name": "disk-memoizer",
-  "version": "2.2.0",
+  "version": "2.2.3",
   "description": "Simple disk memoization and in memory LRU cache for high latency IO responses",
   "main": "dist/disk_memoizer.js",
   "scripts": {
-    "test": "mocha",
+    "test": "mocha && ./test/integration/concurrent_read",
     "test:watch": "npm run test -- -w -G",
@@ -54,4 +54,5 @@ "test:coverate-report": "nyc mocha && nyc report --reporter=html && echo 'Coverage report available on ./coverage/index.html'",
     "mkdirp": "^0.5.1",
-    "reltime": "^0.0.2"
+    "reltime": "^0.0.2",
+    "request": "^2.81.0"
   }
 }

README.md

@@ -104,3 +104,3 @@ # disk-memoizer
 | DISK_MEMOIZER_GC_LAST_ACCESS | 1h | When removing old files only those that have not been accessed for the specified time will be removed. |
-| DISK_MEMOIZER_LOCK_STALE_MS | 30000 | Milliseconds for the cache lock to be considerer stale. |
+| DISK_MEMOIZER_LOCK_STALE_MS | 5000 | Milliseconds for the cache lock to be considerer stale. |
 
@@ -107,0 +107,0 @@

No alert changes

Improved metrics

Total package byte prevSize

137508

increased by1.64%

Worsened metrics

Dependency count

9

increased by12.5%

Number of package files

24

decreased by-4%

Lines of code

1576

decreased by-0.06%

Dependency changes

• + Addedrequest@^2.81.0

• + Addedajv@6.12.6(transitive)

• + Addedasn1@0.2.6(transitive)

• + Addedassert-plus@1.0.0(transitive)

• + Addedasynckit@0.4.0(transitive)

• + Addedaws-sign2@0.7.0(transitive)

• + Addedaws4@1.13.2(transitive)

• + Addedbcrypt-pbkdf@1.0.2(transitive)

• + Addedcaseless@0.12.0(transitive)

• + Addedcombined-stream@1.0.8(transitive)

• + Addedcore-util-is@1.0.2(transitive)

• + Addeddashdash@1.14.1(transitive)

• + Addeddelayed-stream@1.0.0(transitive)

• + Addedecc-jsbn@0.1.2(transitive)

• + Addedextend@3.0.2(transitive)

• + Addedextsprintf@1.3.0(transitive)

• + Addedfast-deep-equal@3.1.3(transitive)

• + Addedfast-json-stable-stringify@2.1.0(transitive)

• + Addedforever-agent@0.6.1(transitive)

• + Addedform-data@2.3.3(transitive)

• + Addedgetpass@0.1.7(transitive)

• + Addedhar-schema@2.0.0(transitive)

• + Addedhar-validator@5.1.5(transitive)

• + Addedhttp-signature@1.2.0(transitive)

• + Addedis-typedarray@1.0.0(transitive)

• + Addedisstream@0.1.2(transitive)

• + Addedjsbn@0.1.1(transitive)

• + Addedjson-schema@0.4.0(transitive)

• + Addedjson-schema-traverse@0.4.1(transitive)

• + Addedjson-stringify-safe@5.0.1(transitive)

• + Addedjsprim@1.4.2(transitive)

• + Addedmime-db@1.52.0(transitive)

• + Addedmime-types@2.1.35(transitive)

• + Addedoauth-sign@0.9.0(transitive)

• + Addedperformance-now@2.1.0(transitive)

• + Addedpsl@1.15.0(transitive)

• + Addedpunycode@2.3.1(transitive)

• + Addedqs@6.5.3(transitive)

• + Addedrequest@2.88.2(transitive)

• + Addedsafe-buffer@5.2.1(transitive)

• + Addedsafer-buffer@2.1.2(transitive)

• + Addedsshpk@1.18.0(transitive)

• + Addedtough-cookie@2.5.0(transitive)

• + Addedtunnel-agent@0.6.0(transitive)

• + Addedtweetnacl@0.14.5(transitive)

• + Addeduri-js@4.4.1(transitive)

• + Addeduuid@3.4.0(transitive)

• + Addedverror@1.10.0(transitive)