nedb - npm Package Compare versions

Comparing version 0.5.0 to 0.5.1

lib/datastore.js

@@ -1,7 +0,1 @@
-/**
- * The datastore itself
- * TODO
- * Update and removes should only modify the corresponding part of the database (or use much faster append-only format)
- */
-
 var fs = require('fs')
@@ -27,7 +21,8 @@ , path = require('path')
  * Load the database
- * For now this means pulling data out of the data file or creating it
- * if it doesn't exist
+ * This means pulling data out of the data file or creating it if it doesn't exist
+ * Also, all data is persisted right away, which has the effect of compacting the database file
+ * This operation is very quick at startup for a big collection (60ms for ~10k docs)
  * @param {Function} cb Optional callback, signature: err
  *
- * @api private
+ * @api private Use loadDatabase
  */
@@ -48,3 +43,3 @@ Datastore.prototype._loadDatabase = function (cb) {
           self.data = Datastore.treatRawData(rawData);
-          return callback();
+          self.persistCachedDatabase(callback);
         });
@@ -96,2 +91,24 @@ }
 /**
+ * Persist cached database
+ * This serves as a compaction function since the cache always contains only the number of documents in the collection
+ * while the data file is append-only so it may grow larger
+ * @param {Function} cb Optional callback, signature: err
+ */
+Datastore.prototype.persistCachedDatabase = function (cb) {
+  var callback = cb || function () {}
+    , toPersist = ''
+    ;
+
+  this.data.forEach(function (doc) {
+    toPersist += model.serialize(doc) + '\n';
+  });
+
+  if (toPersist.length === 0) { return callback(); }
+
+  fs.writeFile(this.filename, toPersist, function (err) { return callback(err); });
+
+};
+
+
+/**
  * Insert a new document
@@ -178,32 +195,4 @@ * @param {Function} cb Optional callback, signature: err, insertedDoc
 /**
- * Persist the whole database
- * @param {Array} data Optional data to persist. If not set, we use the database data
- * @param {Function} cb Optional callback, signature: err
- */
-Datastore.prototype.persistWholeDatabase = function (data, cb) {
-  var callback
-    , self = this
-    , newContents = '';
-
-  if (!data) {
-    data = self.data;
-  }
-
-  if (typeof data === 'function') {
-    cb = data;
-    data = self.data;
-  }
-
-  callback = cb || function () {};
-
-  data.forEach(function (d) {
-    newContents += model.serialize(d) + '\n';
-  });
-
-  fs.writeFile(self.filename, newContents, 'utf8', callback);
-};
-
-
-/**
  * Persist new state for the given newDocs (can be update or removal)
+ * Use an append-only format
  * @param {Array} newDocs Can be empty if no doc was updated/removed
@@ -210,0 +199,0 @@ * @param {Function} cb Optional, signature: err

lib/model.js

@@ -58,2 +58,5 @@ /**
  * Serialize an object to be persisted to a one-line string
+ * For serialization/deserialization, we use the native JSON parser and not eval or Function
+ * That gives us less freedom but data entered in the database may come from users
+ * so eval and the like are not safe
  * Accepted primitive types: Number, String, Boolean, Date, null
@@ -60,0 +63,0 @@ * Accepted secondary types: Objects, Arrays

package.json

 {
   "name": "nedb",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "author": {
@@ -5,0 +5,0 @@ "name": "tldr.io",

README.md

@@ -276,4 +276,4 @@ # NeDB (Node embedded database)
 * An insert takes 0.1 ms
-* A read takes 5.7 ms
-* An update takes 10.9 ms
+* A read takes 6.4 ms
+* An update takes 9.2 ms
 * A deletion takes 8.1 ms  
@@ -294,3 +294,3 @@
 Connect and Express, backed by nedb
-* I'm planning on making an export tool to get all your data in an nedb
+* I'm planning on making an export tool to get all your data in a nedb
   database in a Mongo database
@@ -297,0 +297,0 @@

test/db.test.js

@@ -39,3 +39,3 @@ var Datastore = require('../lib/datastore')
 
-  describe('Loading the database data from file', function () {
+  describe('Loading the database data from file and persistence', function () {
 
@@ -128,5 +128,33 @@ it('Every line represents a document', function () {
 
-  });   // ==== End of 'Loading the database data from file' ==== //
+    it('Compact database on load', function (done) {
+      d.insert({ a: 2 }, function () {
+        d.insert({ a: 4 }, function () {
+          d.remove({ a: 2 }, {}, function () {
+            // Here, the underlying file is 3 lines long for only one document
+            var data = fs.readFileSync(d.filename, 'utf8').split('\n')
+              , filledCount = 0;
 
+            data.forEach(function (item) { if (item.length > 0) { filledCount += 1; } });
+            filledCount.should.equal(3);
 
+            d.loadDatabase(function (err) {
+              assert.isNull(err);
+
+              // Now, the file has been compacted and is only 1 line long
+              var data = fs.readFileSync(d.filename, 'utf8').split('\n')
+                , filledCount = 0;
+
+              data.forEach(function (item) { if (item.length > 0) { filledCount += 1; } });
+              filledCount.should.equal(1);
+
+              done();
+            });
+          })
+        });
+      });
+    });
+
+  });   // ==== End of 'Loading the database data from file and persistence' ==== //
+
+
   describe('Insert', function () {
@@ -742,3 +770,3 @@
               d.loadDatabase(function (err) {
-                assert.isUndefined(err);
+                assert.isNull(err);
 
@@ -776,3 +804,3 @@ d.find({}, function (err, docs) {
                 d.loadDatabase(function (err) {
-                  assert.isUndefined(err);
+                  assert.isNull(err);
 
@@ -898,3 +926,3 @@ d.find({}, function (err, docs) {
                 d.loadDatabase(function (err) {
-                  assert.isUndefined(err);
+                  assert.isNull(err);
 
@@ -930,3 +958,3 @@ d.find({}, function (err, docs) {
                 d.loadDatabase(function (err) {
-                  assert.isUndefined(err);
+                  assert.isNull(err);
 
@@ -933,0 +961,0 @@ d.find({}, function (err, docs) {

No alert changes

Improved metrics

Total package byte prevSize

109581

increased by1.19%

Lines of code

2335

increased by0.69%

No dependency changes