lokijs - npm Package Compare versions

Comparing version 1.5.8 to 1.5.9

CHANGELOG.md

 # Changelog
 
+* add IncrementalIndexedDBAdapter
+* make ensureIndex faster
+* LokiEventEmitter.emit faster
+* make batch inserts faster (add option to overrideAdaptiveIndices)
+
 ## 1.5.6
@@ -4,0 +9,0 @@ * added support for indexes on nested properties (#718)

package.json

 {
   "name": "lokijs",
-  "version": "1.5.8",
+  "version": "1.5.9",
   "description": "Fast document oriented javascript in-memory database",
-  "homepage": "http://lokijs.org",
+  "homepage": "https://techfort.github.io/LokiJS/",
   "main": "src/lokijs.js",
@@ -7,0 +7,0 @@ "directories": {

src/incremental-indexeddb-adapter.js

@@ -16,6 +16,25 @@ (function(root, factory) {
 
-    // TODO: db name, etc.
-
-    function IncrementalIndexedDBAdapter() {
+    /**
+     * An improved Loki persistence adapter for IndexedDB (not compatible with LokiIndexedAdapter)
+     *     Unlike LokiIndexedAdapter, the database is saved not as one big JSON blob, but split into
+     *     small chunks with individual collection documents. When saving, only the chunks with changed
+     *     documents (and database metadata) is saved to IndexedDB. This speeds up small incremental
+     *     saves by an order of magnitude on large (tens of thousands of records) databases. It also
+     *     avoids Safari 13 bug that would cause the database to balloon in size to gigabytes
+     *
+     *     The `appname` argument is not provided - to distinguish between multiple app on the same
+     *     domain, simply use a different Loki database name
+     *
+     * @example
+     * var adapter = new IncrementalIndexedDBAdapter();
+     *
+     * @constructor IncrementalIndexedDBAdapter
+     *
+     * @param {object=} options Configuration options for the adapter
+     * @param {boolean} options.onversionchange Function to call on `IDBDatabase.onversionchange` event
+     *     (most likely database deleted from another browser tab)
+     */
+    function IncrementalIndexedDBAdapter(options) {
       this.mode = "incremental";
+      this.options = options || {};
       this.chunkSize = 100;
@@ -50,3 +69,3 @@ this.idb = null; // will be lazily loaded on first operation that needs it
 
-      if (max === min && idIndex[min] >= minId) {
+      if (max === min && idIndex[min] >= minId && idIndex[min] <= maxId) {
         firstDataPosition = min;
@@ -95,5 +114,20 @@ }
 
+    /**
+     * Incrementally saves the database to IndexedDB
+     *
+     * @example
+     * var idbAdapter = new IncrementalIndexedDBAdapter();
+     * var db = new loki('test', { adapter: idbAdapter });
+     * var coll = db.addCollection('testColl');
+     * coll.insert({test: 'val'});
+     * db.saveDatabase();
+     *
+     * @param {string} dbname - the name to give the serialized database
+     * @param {object} dbcopy - copy of the Loki database
+     * @param {function} callback - (Optional) callback passed obj.success with true or false
+     * @memberof IncrementalIndexedDBAdapter
+     */
     IncrementalIndexedDBAdapter.prototype.saveDatabase = function(dbname, loki, callback) {
       var that = this;
-      console.log("-- exportDatabase - begin");
+      console.log("exportDatabase - begin");
       console.time("exportDatabase");
@@ -103,5 +137,4 @@
 
-      console.time("makeChunks");
       loki.collections.forEach(function(collection, i) {
-        console.time("get dirty chunk ids");
+        // Find dirty chunk ids
         var dirtyChunks = new Set();
@@ -113,8 +146,7 @@ collection.dirtyIds.forEach(function(lokiId) {
         collection.dirtyIds = [];
-        console.timeEnd("get dirty chunk ids");
 
-        console.time("get chunks&serialize");
+        // Serialize chunks to save
         dirtyChunks.forEach(function(chunkId) {
           var chunkData = that._getChunk(collection, chunkId);
-          // we must stringify, because IDB is asynchronous, and underlying objects are mutable
+          // we must stringify now, because IDB is asynchronous, and underlying objects are mutable
           chunksToSave.push({
@@ -125,3 +157,2 @@ key: collection.name + ".chunk." + chunkId,
         });
-        console.timeEnd("get chunks&serialize");
 
@@ -140,3 +171,2 @@ collection.data = [];
       });
-      console.timeEnd("makeChunks");
 
@@ -146,20 +176,30 @@ var serializedMetadata = JSON.stringify(loki);
 
-      // console.log(chunksToSave)
-      // console.log(chunkIdsToRemove)
-      // console.log(JSON.parse(serializedMetadata))
-
       chunksToSave.push({ key: "loki", value: serializedMetadata });
 
-      // TODO: Clear out lokiChangedIds flags on original database
-
-      that._saveChunks(chunksToSave, callback);
+      that._saveChunks(dbname, chunksToSave, callback);
     };
 
+    /**
+     * Retrieves a serialized db string from the catalog.
+     *
+     * @example
+     * // LOAD
+     * var idbAdapter = new IncrementalIndexedDBAdapter();
+     * var db = new loki('test', { adapter: idbAdapter });
+     * db.loadDatabase(function(result) {
+     *   console.log('done');
+     * });
+     *
+     * @param {string} dbname - the name of the database to retrieve.
+     * @param {function} callback - callback should accept string param containing serialized db string.
+     * @memberof IncrementalIndexedDBAdapter
+     */
     IncrementalIndexedDBAdapter.prototype.loadDatabase = function(dbname, callback) {
       var that = this;
-      console.log("-- loadDatabase - begin");
+      console.log("loadDatabase - begin");
       console.time("loadDatabase");
-      this._getAllChunks(function(chunks) {
+      this._getAllChunks(dbname, function(chunks) {
         if (!Array.isArray(chunks)) {
           // we got an error
+          console.timeEnd("loadDatabase");
           callback(chunks);
@@ -169,3 +209,3 @@ }
         if (!chunks.length) {
-          console.log("No chunks");
+          console.timeEnd("loadDatabase");
           callback(null);
@@ -183,3 +223,2 @@ return;
 
-        // console.time('repack')
         chunks.forEach(function(object) {
@@ -219,4 +258,2 @@ var key = object.key;
         chunks = null;
-        // console.timeEnd('repack')
-        // console.log('chunkCollections', chunkCollections)
 
@@ -228,15 +265,9 @@ if (!loki) {
         // parse Loki object
-        // console.time('parse')
         loki = JSON.parse(loki);
-        // console.timeEnd('parse')
-        // console.log('Parsed loki object', loki)
 
         // populate collections with data
-        console.time("populate");
         that._populate(loki, chunkCollections);
         chunkCollections = null;
-        console.timeEnd("populate");
 
         console.timeEnd("loadDatabase");
-        // console.log('Loaded Loki database!', loki)
         callback(loki);
@@ -249,3 +280,2 @@ });
       // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100...
-      // console.time('sort')
       var getSortKey = function(object) {
@@ -269,4 +299,2 @@ var key = object.key;
       });
-      // console.timeEnd('sort')
-      // console.log('Sorted chunks', chunks)
     };
@@ -299,3 +327,3 @@
 
-    IncrementalIndexedDBAdapter.prototype._initializeIDB = function(callback) {
+    IncrementalIndexedDBAdapter.prototype._initializeIDB = function(dbname, onError, onSuccess) {
       var that = this;
@@ -309,13 +337,15 @@ console.log("initializing idb");
 
-      var openRequest = indexedDB.open("IncrementalAdapterIDB", 1);
+      var openRequest = indexedDB.open(dbname, 1);
 
       openRequest.onupgradeneeded = function(e) {
-        console.log("onupgradeneeded");
         var db = e.target.result;
-        if (db.objectStoreNames.contains("Store2")) {
-          throw new Error("todo");
-          // TODO: Finish this
+        console.log('onupgradeneeded, old version: ' + e.oldVersion);
+
+        if (e.oldVersion < 1) {
+          // Version 1 - Initial - Create database
+          db.createObjectStore('LokiIncrementalData', { keyPath: "key" });
+        } else {
+          // Unknown version
+          throw new Error("Invalid old version " + e.oldVersion + " for IndexedDB upgrade");
         }
-
-        var store = db.createObjectStore("Store2", { keyPath: "key" });
       };
@@ -325,5 +355,28 @@
         that.idbInitInProgress = false;
+        that.idb = e.target.result;
+
+        if (!that.idb.objectStoreNames.contains('LokiIncrementalData')) {
+          onError(new Error("Missing LokiIncrementalData"));
+          // Attempt to recover (after reload) by deleting database, since it's damaged anyway
+          that.deleteDatabase(dbname);
+          return;
+        }
+
         console.log("init success");
-        that.idb = e.target.result;
-        callback();
+
+        that.idb.onversionchange = function(versionChangeEvent) {
+          console.log('IDB version change', versionChangeEvent);
+          // This function will be called if another connection changed DB version
+          // (Most likely database was deleted from another browser tab, unless there's a new version
+          // of this adapter, or someone makes a connection to IDB outside of this adapter)
+          // We must close the database to avoid blocking concurrent deletes.
+          // The database will be unusable after this. Be sure to supply `onversionchange` option
+          // to force logout
+          that.idb.close();
+          if (that.options.onversionchange) {
+            that.options.onversionchange(versionChangeEvent);
+          }
+        };
+
+        onSuccess();
       };
@@ -333,3 +386,3 @@
         console.error("IndexedDB open is blocked", e);
-        throw new Error("IndexedDB open is blocked by open connection");
+        onError(new Error("IndexedDB open is blocked by open connection"));
       };
@@ -340,11 +393,11 @@
         console.error("IndexeddB open error", e);
-        throw e;
+        onError(e);
       };
     };
 
-    IncrementalIndexedDBAdapter.prototype._saveChunks = function(chunks, callback) {
+    IncrementalIndexedDBAdapter.prototype._saveChunks = function(dbname, chunks, callback) {
       var that = this;
       if (!this.idb) {
-        this._initializeIDB(function() {
-          that._saveChunks(chunks, callback);
+        this._initializeIDB(dbname, callback, function() {
+          that._saveChunks(dbname, chunks, callback);
         });
@@ -354,4 +407,2 @@ return;
 
-      console.time("save chunks to idb");
-
       if (this.operationInProgress) {
@@ -363,6 +414,5 @@ throw new Error("Error while saving to database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
 
-      var tx = this.idb.transaction(["Store2"], "readwrite");
+      var tx = this.idb.transaction(['LokiIncrementalData'], "readwrite");
       tx.oncomplete = function() {
         that.operationInProgress = false;
-        console.timeEnd("save chunks to idb");
         console.timeEnd("exportDatabase");
@@ -374,3 +424,2 @@ callback();
         that.operationInProgress = false;
-        console.error("Error while saving data to database", e);
         callback(e);
@@ -381,26 +430,20 @@ };
         that.operationInProgress = false;
-        console.error("Abort while saving data to database", e);
         callback(e);
       };
 
-      var store = tx.objectStore("Store2");
+      var store = tx.objectStore('LokiIncrementalData');
 
-      console.time("put");
-      // console.log(chunks)
       chunks.forEach(function(object) {
         store.put(object);
       });
-      console.timeEnd("put");
     };
 
-    IncrementalIndexedDBAdapter.prototype._getAllChunks = function(callback) {
+    IncrementalIndexedDBAdapter.prototype._getAllChunks = function(dbname, callback) {
       var that = this;
       if (!this.idb) {
-        this._initializeIDB(function() {
-          that._getAllChunks(callback);
+        this._initializeIDB(dbname, callback, function() {
+          that._getAllChunks(dbname, callback);
         });
         return;
       }
-      console.log("getting all chunks");
-      console.time("getChunks");
 
@@ -413,9 +456,8 @@ if (this.operationInProgress) {
 
-      var tx = this.idb.transaction(["Store2"], "readonly");
+      var tx = this.idb.transaction(['LokiIncrementalData'], "readonly");
 
-      var request = tx.objectStore("Store2").getAll();
+      var request = tx.objectStore('LokiIncrementalData').getAll();
       request.onsuccess = function(e) {
         that.operationInProgress = false;
         var chunks = e.target.result;
-        console.timeEnd("getChunks");
         callback(chunks);
@@ -426,3 +468,2 @@ };
         that.operationInProgress = false;
-        console.error("Error while fetching data from IndexedDB", e);
         callback(e);
@@ -432,7 +473,17 @@ };
 
+    /**
+     * Deletes a database from IndexedDB
+     *
+     * @example
+     * // DELETE DATABASE
+     * // delete 'finance'/'test' value from catalog
+     * idbAdapter.deleteDatabase('test', function {
+     *   // database deleted
+     * });
+     *
+     * @param {string} dbname - the name of the database to delete from IDB
+     * @param {function=} callback - (Optional) executed on database delete
+     * @memberof IncrementalIndexedDBAdapter
+     */
     IncrementalIndexedDBAdapter.prototype.deleteDatabase = function(dbname, callback) {
-      var that = this;
-      console.log("deleteDatabase");
-      console.time("deleteDatabase");
-
       if (this.operationInProgress) {
@@ -444,2 +495,6 @@ throw new Error("Error while deleting database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
 
+      var that = this;
+      console.log("deleteDatabase - begin");
+      console.time("deleteDatabase");
+
       if (this.idb) {
@@ -450,3 +505,3 @@ this.idb.close();
 
-      var request = indexedDB.deleteDatabase("IncrementalAdapterIDB");
+      var request = indexedDB.deleteDatabase(dbname);
 
@@ -456,3 +511,2 @@ request.onsuccess = function() {
         console.timeEnd("deleteDatabase");
-        console.log("deleteDatabase done");
         callback({ success: true });
@@ -467,3 +521,7 @@ };
 
-      console.log("deleteDatabase - exit fn");
+      request.onblocked = function(e) {
+        // We can't call callback with failure status, because this will be called even if we
+        // succeed in just a moment
+        console.error("Deleting database failed because it's blocked by another connection", e);
+      };
     };
@@ -470,0 +528,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

3056587

increased by0.48%

Number of package files

32

increased by3.23%

Lines of code

9665

increased by2.57%

No dependency changes