@nozbe/lokijs - npm Package Compare versions

Comparing version 1.5.10-wmelon3 to 1.5.11-wmelon-idb-fix

package.json

 {
   "name": "@nozbe/lokijs",
-  "version": "1.5.10-wmelon3",
+  "version": "1.5.11-wmelon-idb-fix",
   "description": "Nozbe's temporary fork of LokiJS - used for WatermelonDB purposes to work around NPM issues",
@@ -5,0 +5,0 @@ "homepage": "https://techfort.github.io/LokiJS/",

README.md

 # LokiJS
 
-LokiJS is being sponsored by the following tool; please help to support us by taking a look and signing up to a free trial
-<a href="https://tracking.gitads.io/?repo=lokijs"> <img src="https://images.gitads.io/lokijs" alt="GitAds"/> </a>
-
 [![Join the chat at https://gitter.im/techfort/LokiJS](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/techfort/LokiJS?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
@@ -7,0 +4,0 @@ ![alt CI-badge](https://travis-ci.org/techfort/LokiJS.svg?branch=master)

src/incremental-indexeddb-adapter.js

@@ -52,2 +52,4 @@ (function(root, factory) {
       this.idb = null; // will be lazily loaded on first operation that needs it
+      this._prevLokiVersionId = null;
+      this._prevCollectionVersionIds = {};
     }
@@ -102,3 +104,3 @@
 
-      // TODO: remove sanity checks when everything is fully tested
+      // verify
       var firstElement = collection.data[firstDataPosition];
@@ -118,3 +120,2 @@ if (!(firstElement && firstElement.$loki >= minId && firstElement.$loki <= maxId)) {
 
-      // TODO: remove sanity checks when everything is fully tested
       if (chunkData.length > this.chunkSize) {
@@ -138,18 +139,138 @@ throw new Error("broken invariant - chunk size");
      * @param {string} dbname - the name to give the serialized database
-     * @param {object} dbcopy - copy of the Loki database
+     * @param {function} getLokiCopy - returns copy of the Loki database
      * @param {function} callback - (Optional) callback passed obj.success with true or false
      * @memberof IncrementalIndexedDBAdapter
      */
-    IncrementalIndexedDBAdapter.prototype.saveDatabase = function(dbname, loki, callback) {
+    IncrementalIndexedDBAdapter.prototype.saveDatabase = function(dbname, getLokiCopy, callback) {
       var that = this;
-      DEBUG && console.log("exportDatabase - begin");
-      DEBUG && console.time("exportDatabase");
 
-      var chunksToSave = [];
-      var savedLength = 0;
+      if (!this.idb) {
+        this._initializeIDB(dbname, callback, function() {
+          that.saveDatabase(dbname, getLokiCopy, callback);
+        });
+        return;
+      }
 
+      if (this.operationInProgress) {
+        throw new Error("Error while saving to database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
+      }
+      this.operationInProgress = true;
+
+      DEBUG && console.log("saveDatabase - begin");
+      DEBUG && console.time("saveDatabase");
+      function finish(e) {
+        DEBUG && e && console.error(e);
+        DEBUG && console.timeEnd("saveDatabase");
+        that.operationInProgress = false;
+        callback(e);
+      }
+
+      var updatePrevVersionIds = function () {
+        console.error('Unexpected successful tx - cannot update previous version ids');
+      };
+
+      DEBUG && console.log("save tx: begin");
+      var tx = this.idb.transaction(['LokiIncrementalData'], "readwrite");
+      tx.oncomplete = function() {
+        updatePrevVersionIds();
+        DEBUG && console.log("save tx: complete");
+        return finish();
+      };
+
+      tx.onerror = function(e) {
+        DEBUG && console.log("save tx: error");
+        return finish(e);
+      };
+
+      tx.onabort = function(e) {
+        DEBUG && console.log("save tx: abort");
+        return finish(e);
+      };
+
+      var store = tx.objectStore('LokiIncrementalData');
+
+      function performSave(maxChunkIds) {
+        var incremental = !maxChunkIds;
+        var chunkInfo = that._putInChunks(store, getLokiCopy(), incremental, maxChunkIds);
+        updatePrevVersionIds = function() {
+          that._prevLokiVersionId = chunkInfo.lokiVersionId;
+          chunkInfo.collectionVersionIds.forEach(function (collectionInfo) {
+            that._prevCollectionVersionIds[collectionInfo.name] = collectionInfo.versionId;
+          });
+        };
+        DEBUG && console.log('chunks saved');
+        tx.commit && tx.commit();
+      }
+
+      function getAllKeysThenSave() {
+        idbReq(store.getAllKeys(), function(e) {
+          var maxChunkIds = getMaxChunkIds(e.target.result);
+          performSave(maxChunkIds);
+        }, function(e) {
+          console.error('Getting all keys failed: ', e);
+          tx.abort();
+        });
+      }
+
+      function getLokiThenSave() {
+        idbReq(store.get('loki'), function(e) {
+          if (lokiChunkVersionId(e.target.result) === that._prevLokiVersionId) {
+            performSave();
+          } else {
+            DEBUG && console.warn('--------> LOKI CHANGED!!! [slow path]');
+            // TODO: Get collection metadata chunks
+            getAllKeysThenSave();
+          }
+        }, function(e) {
+          console.error('Getting loki chunk failed: ', e);
+          tx.abort();
+        });
+      }
+
+      getLokiThenSave();
+    };
+
+    // gets current largest chunk ID for each collection
+    function getMaxChunkIds(allKeys) {
+      var maxChunkIds = {};
+
+      allKeys.forEach(function (key) {
+        var keySegments = key.split(".");
+        // table.chunk.2317
+        if (keySegments.length === 3 && keySegments[1] === "chunk") {
+          var collection = keySegments[0];
+          var chunkId = parseInt(keySegments[2]) || 0;
+          var currentMax = maxChunkIds[collection];
+
+          if (!currentMax || chunkId > currentMax) {
+            maxChunkIds[collection] = chunkId;
+          }
+        }
+      });
+      return maxChunkIds;
+    }
+
+    function lokiChunkVersionId(chunk) {
+      try {
+        if (chunk) {
+          var loki = JSON.parse(chunk.value);
+          return loki.idbVersionId || null;
+        } else {
+          return null;
+        }
+      } catch (e) {
+        console.error('Error while parsing loki chunk', e);
+        return null;
+      }
+    }
+
+    IncrementalIndexedDBAdapter.prototype._putInChunks = function(idbStore, loki, incremental, maxChunkIds) {
+      var that = this;
+      var collectionVersionIds = [];
+      var savedSize = 0;
+
       var prepareCollection = function (collection, i) {
         // Find dirty chunk ids
         var dirtyChunks = new Set();
-        collection.dirtyIds.forEach(function(lokiId) {
+        incremental && collection.dirtyIds.forEach(function(lokiId) {
           var chunkId = (lokiId / that.chunkSize) | 0;
@@ -167,6 +288,7 @@ dirtyChunks.add(chunkId);
           // we must stringify now, because IDB is asynchronous, and underlying objects are mutable
-          // (and it's faster for some reason)
+          // In general, it's also faster to stringify, because we need serialization anyway, and
+          // JSON.stringify is much better optimized than IDB's structured clone
           chunkData = JSON.stringify(chunkData);
-          savedLength += chunkData.length;
-          chunksToSave.push({
+          savedSize += chunkData.length;
+          idbStore.put({
             key: collection.name + ".chunk." + chunkId,
@@ -176,12 +298,32 @@ value: chunkData,
         };
-        dirtyChunks.forEach(prepareChunk);
+        if (incremental) {
+          dirtyChunks.forEach(prepareChunk);
+        } else {
+          // add all chunks
+          var maxChunkId = (collection.maxId / that.chunkSize) | 0;
+          for (var j = 0; j <= maxChunkId; j += 1) {
+            prepareChunk(j);
+          }
 
+          // delete chunks with larger ids than what we have
+          // NOTE: we don't have to delete metadata chunks as they will be absent from loki anyway
+          // NOTE: failures are silently ignored, so we don't have to worry about holes
+          var persistedMaxChunkId = maxChunkIds[collection.name] || 0;
+          for (var k = maxChunkId + 1; k <= persistedMaxChunkId; k += 1) {
+            var deletedChunkName = collection.name + ".chunk." + k;
+            idbStore.delete(deletedChunkName);
+            DEBUG && console.warn('Deleted chunk: ' + deletedChunkName);
+          }
+        }
+
         // save collection metadata as separate chunk (but only if changed)
-        if (collection.dirty) {
+        if (collection.dirty || dirtyChunks.size || !incremental) {
           collection.idIndex = []; // this is recreated lazily
           collection.data = [];
+          collection.idbVersionId = randomVersionId();
+          collectionVersionIds.push({ name: collection.name, versionId: collection.idbVersionId });
 
           var metadataChunk = JSON.stringify(collection);
-          savedLength += metadataChunk.length;
-          chunksToSave.push({
+          savedSize += metadataChunk.length;
+          idbStore.put({
             key: collection.name + ".metadata",
@@ -197,10 +339,13 @@ value: metadataChunk,
 
+      loki.idbVersionId = randomVersionId();
       var serializedMetadata = JSON.stringify(loki);
-      savedLength += serializedMetadata.length;
-      loki = null; // allow GC of the DB copy
+      savedSize += serializedMetadata.length;
 
-      chunksToSave.push({ key: "loki", value: serializedMetadata });
+      idbStore.put({ key: "loki", value: serializedMetadata });
 
-      DEBUG && console.log("saved size: " + savedLength);
-      that._saveChunks(dbname, chunksToSave, callback);
+      DEBUG && console.log("saved size: " + savedSize);
+      return {
+        lokiVersionId: loki.idbVersionId,
+        collectionVersionIds: collectionVersionIds,
+      };
     };
@@ -225,106 +370,111 @@
       var that = this;
-      DEBUG && console.log("loadDatabase - begin");
-      DEBUG && console.time("loadDatabase");
-      this._getAllChunks(dbname, function(chunks) {
-        if (!Array.isArray(chunks)) {
-          // we got an error
-          DEBUG && console.timeEnd("loadDatabase");
-          callback(chunks);
-        }
 
-        if (!chunks.length) {
-          DEBUG && console.timeEnd("loadDatabase");
-          callback(null);
-          return;
-        }
+      if (this.operationInProgress) {
+        throw new Error("Error while loading database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
+      }
 
-        DEBUG && console.log("Found chunks:", chunks.length);
+      this.operationInProgress = true;
 
-        that._sortChunksInPlace(chunks);
+      DEBUG && console.log("loadDatabase - begin");
+      DEBUG && console.time("loadDatabase");
 
-        // repack chunks into a map
-        var loki;
-        var chunkCollections = {};
+      var finish = function (value) {
+        DEBUG && console.timeEnd("loadDatabase");
+        that.operationInProgress = false;
+        callback(value);
+      };
 
-        chunks.forEach(function(object) {
-          var key = object.key;
-          var value = object.value;
-          if (key === "loki") {
-            loki = value;
-            return;
-          } else if (key.includes(".")) {
-            var keySegments = key.split(".");
-            if (keySegments.length === 3 && keySegments[1] === "chunk") {
-              var colName = keySegments[0];
-              if (chunkCollections[colName]) {
-                chunkCollections[colName].dataChunks.push(value);
-              } else {
-                chunkCollections[colName] = {
-                  metadata: null,
-                  dataChunks: [value],
-                };
-              }
-              return;
-            } else if (keySegments.length === 2 && keySegments[1] === "metadata") {
-              var name = keySegments[0];
-              if (chunkCollections[name]) {
-                chunkCollections[name].metadata = value;
-              } else {
-                chunkCollections[name] = { metadata: value, dataChunks: [] };
-              }
-              return;
-            }
+      this._getAllChunks(dbname, function(chunks) {
+        try {
+          if (!Array.isArray(chunks)) {
+            throw chunks; // we have an error
           }
 
-          console.error("Unknown chunk " + key);
-          callback(new Error("Invalid database - unknown chunk found"));
-        });
-        chunks = null;
+          if (!chunks.length) {
+            return finish(null);
+          }
 
-        if (!loki) {
-          callback(new Error("Invalid database - missing database metadata"));
-        }
+          DEBUG && console.log("Found chunks:", chunks.length);
 
-        // parse Loki object
-        loki = JSON.parse(loki);
+          // repack chunks into a map
+          chunks = chunksToMap(chunks);
+          var loki = JSON.parse(chunks.loki);
+          chunks.loki = null; // gc
 
-        // populate collections with data
-        that._populate(loki, chunkCollections);
-        chunkCollections = null;
+          // populate collections with data
+          var deserializeChunk = that.options.deserializeChunk;
+          populateLoki(loki, chunks.chunkMap, deserializeChunk);
+          chunks = null; // gc
 
-        DEBUG && console.timeEnd("loadDatabase");
-        callback(loki);
+          // remember previous version IDs
+          that._prevLokiVersionId = loki.idbVersionId || null;
+          that._prevCollectionVersionIds = {};
+          loki.collections.forEach(function (collection) {
+            that._prevCollectionVersionIds[collection.name] = collection.idbVersionId || null;
+          });
+
+          return finish(loki);
+        } catch (error) {
+          that._prevLokiVersionId = null;
+          that._prevCollectionVersionIds = {};
+          return finish(error);
+        }
       });
     };
 
-    IncrementalIndexedDBAdapter.prototype._sortChunksInPlace = function(chunks) {
-      // sort chunks in place to load data in the right order (ascending loki ids)
-      // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100...
-      var getSortKey = function(object) {
+    function chunksToMap(chunks) {
+      var loki;
+      var chunkMap = {};
+
+      sortChunksInPlace(chunks);
+
+      chunks.forEach(function(object) {
         var key = object.key;
-        if (key.includes(".")) {
-          var segments = key.split(".");
-          if (segments.length === 3 && segments[1] === "chunk") {
-            return parseInt(segments[2], 10);
+        var value = object.value;
+        if (key === "loki") {
+          loki = value;
+          return;
+        } else if (key.includes(".")) {
+          var keySegments = key.split(".");
+          if (keySegments.length === 3 && keySegments[1] === "chunk") {
+            var colName = keySegments[0];
+            if (chunkMap[colName]) {
+              chunkMap[colName].dataChunks.push(value);
+            } else {
+              chunkMap[colName] = {
+                metadata: null,
+                dataChunks: [value],
+              };
+            }
+            return;
+          } else if (keySegments.length === 2 && keySegments[1] === "metadata") {
+            var name = keySegments[0];
+            if (chunkMap[name]) {
+              chunkMap[name].metadata = value;
+            } else {
+              chunkMap[name] = { metadata: value, dataChunks: [] };
+            }
+            return;
           }
         }
 
-        return -1; // consistent type must be returned
-      };
-      chunks.sort(function(a, b) {
-        var aKey = getSortKey(a),
-          bKey = getSortKey(b);
-        if (aKey < bKey) return -1;
-        if (aKey > bKey) return 1;
-        return 0;
+        console.error("Unknown chunk " + key);
+        throw new Error("Corrupted database - unknown chunk found");
       });
-    };
 
-    IncrementalIndexedDBAdapter.prototype._populate = function(loki, chunkCollections) {
-      var that = this;
+      if (!loki) {
+        throw new Error("Corrupted database - missing database metadata");
+      }
+
+      return { loki: loki, chunkMap: chunkMap };
+    }
+
+    function populateLoki(loki, chunkMap, deserializeChunk) {
       loki.collections.forEach(function(collectionStub, i) {
-        var chunkCollection = chunkCollections[collectionStub.name];
+        var chunkCollection = chunkMap[collectionStub.name];
 
         if (chunkCollection) {
-          // TODO: What if metadata is missing?
+          if (!chunkCollection.metadata) {
+            throw new Error("Corrupted database - missing metadata chunk for " + collectionStub.name);
+          }
           var collection = JSON.parse(chunkCollection.metadata);
@@ -341,4 +491,4 @@ chunkCollection.metadata = null;
 
-            if (that.options.deserializeChunk) {
-              chunk = that.options.deserializeChunk(collection.name, chunk);
+            if (deserializeChunk) {
+              chunk = deserializeChunk(collection.name, chunk);
             }
@@ -352,3 +502,3 @@
       });
-    };
+    }
 
@@ -381,5 +531,6 @@ IncrementalIndexedDBAdapter.prototype._initializeIDB = function(dbname, onError, onSuccess) {
         that.idbInitInProgress = false;
-        that.idb = e.target.result;
+        var db = e.target.result;
+        that.idb = db;
 
-        if (!that.idb.objectStoreNames.contains('LokiIncrementalData')) {
+        if (!db.objectStoreNames.contains('LokiIncrementalData')) {
           onError(new Error("Missing LokiIncrementalData"));
@@ -393,3 +544,3 @@ // Attempt to recover (after reload) by deleting database, since it's damaged anyway
 
-        that.idb.onversionchange = function(versionChangeEvent) {
+        db.onversionchange = function(versionChangeEvent) {
           DEBUG && console.log('IDB version change', versionChangeEvent);
@@ -402,3 +553,3 @@ // This function will be called if another connection changed DB version
           // to force logout
-          that.idb.close();
+          db.close();
           if (that.options.onversionchange) {
@@ -419,3 +570,3 @@ that.options.onversionchange(versionChangeEvent);
         that.idbInitInProgress = false;
-        console.error("IndexeddB open error", e);
+        console.error("IndexedDB open error", e);
         onError(e);
@@ -425,41 +576,2 @@ };
 
-    IncrementalIndexedDBAdapter.prototype._saveChunks = function(dbname, chunks, callback) {
-      var that = this;
-      if (!this.idb) {
-        this._initializeIDB(dbname, callback, function() {
-          that._saveChunks(dbname, chunks, callback);
-        });
-        return;
-      }
-
-      if (this.operationInProgress) {
-        throw new Error("Error while saving to database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
-      }
-
-      this.operationInProgress = true;
-
-      var tx = this.idb.transaction(['LokiIncrementalData'], "readwrite");
-      tx.oncomplete = function() {
-        that.operationInProgress = false;
-        DEBUG && console.timeEnd("exportDatabase");
-        callback();
-      };
-
-      tx.onerror = function(e) {
-        that.operationInProgress = false;
-        callback(e);
-      };
-
-      tx.onabort = function(e) {
-        that.operationInProgress = false;
-        callback(e);
-      };
-
-      var store = tx.objectStore('LokiIncrementalData');
-
-      chunks.forEach(function(object) {
-        store.put(object);
-      });
-    };
-
     IncrementalIndexedDBAdapter.prototype._getAllChunks = function(dbname, callback) {
@@ -474,8 +586,2 @@ var that = this;
 
-      if (this.operationInProgress) {
-        throw new Error("Error while loading database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
-      }
-
-      this.operationInProgress = true;
-
       var tx = this.idb.transaction(['LokiIncrementalData'], "readonly");
@@ -485,3 +591,2 @@
       request.onsuccess = function(e) {
-        that.operationInProgress = false;
         var chunks = e.target.result;
@@ -492,3 +597,2 @@ callback(chunks);
       request.onerror = function(e) {
-        that.operationInProgress = false;
         callback(e);
@@ -527,2 +631,5 @@ };
 
+      this._prevLokiVersionId = null;
+      this._prevCollectionVersionIds = {};
+
       if (this.idb) {
@@ -554,4 +661,41 @@ this.idb.close();
 
+    function randomVersionId() {
+      // Appears to have enough entropy for chunk version IDs
+      // (Only has to be different than enough of its own previous versions that there's no writer
+      // that thinks a new version is the same as an earlier one, not globally unique)
+      return Math.random().toString(36).substring(2);
+    }
+
+    function _getSortKey(object) {
+      var key = object.key;
+      if (key.includes(".")) {
+        var segments = key.split(".");
+        if (segments.length === 3 && segments[1] === "chunk") {
+          return parseInt(segments[2], 10);
+        }
+      }
+
+      return -1; // consistent type must be returned
+    }
+
+    function sortChunksInPlace(chunks) {
+      // sort chunks in place to load data in the right order (ascending loki ids)
+      // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100...
+      chunks.sort(function(a, b) {
+        var aKey = _getSortKey(a),
+          bKey = _getSortKey(b);
+        if (aKey < bKey) return -1;
+        if (aKey > bKey) return 1;
+        return 0;
+      });
+    }
+
+    function idbReq(request, onsuccess, onerror) {
+      request.onsuccess = onsuccess;
+      request.onerror = onerror;
+      return request;
+    }
+
     return IncrementalIndexedDBAdapter;
   })();
 });

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

3065923

increased by0.19%

Lines of code

9874

increased by1.43%

Worsened metrics

Number of lines in readme file

88

decreased by-3.3%

No dependency changes