nedb - npm Package Compare versions

Comparing version 0.4.2 to 0.5.0

lib/datastore.js

@@ -58,2 +58,3 @@ /**
 
+
 /**
@@ -65,2 +66,3 @@ * From a database's raw data, return the corresponding
   var data = rawData.split('\n')
+    , dataById = {}
     , res = []
@@ -74,3 +76,9 @@ , i;
       doc = model.deserialize(data[i]);
-      if (doc._id) {res.push(doc); }
+      if (doc._id) {
+        if (doc.$$deleted === true) {
+          delete dataById[doc._id];
+        } else {
+          dataById[doc._id] = doc;
+        }
+      }
     } catch (e) {
@@ -80,2 +88,6 @@ }
 
+  Object.keys(dataById).forEach(function (k) {
+    res.push(dataById[k]);
+  });
+
   return res;
@@ -196,2 +208,25 @@ };
 /**
+ * Persist new state for the given newDocs (can be update or removal)
+ * @param {Array} newDocs Can be empty if no doc was updated/removed
+ * @param {Function} cb Optional, signature: err
+ */
+Datastore.prototype.persistNewState = function (newDocs, cb) {
+  var self = this
+    , toPersist = ''
+    , callback = cb || function () {}
+    ;
+
+  newDocs.forEach(function (doc) {
+    toPersist += model.serialize(doc) + '\n';
+  });
+
+  if (toPersist.length === 0) { return callback(); }
+
+  fs.appendFile(self.filename, toPersist, 'utf8', function (err) {
+    return callback(err);
+  });
+};
+
+
+/**
  * Update all docs matching query
@@ -213,3 +248,5 @@ * For now, very naive implementation (recalculating the whole database)
     , multi, upsert
-    , newData = [];
+    , updatedDocs = []
+    , i
+    ;
 
@@ -241,10 +278,9 @@ if (typeof options === 'function') { cb = options; options = {}; }
     try {
-      self.data.forEach(function (d) {
-        if (model.match(d, query) && (multi || numReplaced === 0)) {
+      for (i = 0; i < self.data.length; i += 1) {
+        if (model.match(self.data[i], query) && (multi || numReplaced === 0)) {
           numReplaced += 1;
-          newData.push(model.modify(d, updateQuery));
-        } else {
-          newData.push(d);
+          self.data[i] = model.modify(self.data[i], updateQuery);
+          updatedDocs.push(self.data[i]);
         }
-      });
+      }
     } catch (err) {
@@ -254,5 +290,4 @@ return callback(err);
 
-    self.persistWholeDatabase(newData, function (err) {
+    self.persistNewState(updatedDocs, function (err) {
       if (err) { return callback(err); }
-      self.data = newData;
       return callback(null, numReplaced);
@@ -283,3 +318,5 @@ });
     , multi
-    , newData = [];
+    , newData = []
+    , removedDocs = []
+    ;
 
@@ -294,2 +331,3 @@ if (typeof options === 'function') { cb = options; options = {}; }
         numRemoved += 1;
+        removedDocs.push({ $$deleted: true, _id: d._id });
       } else {
@@ -303,3 +341,3 @@ newData.push(d);
 
-  self.persistWholeDatabase(newData, function (err) {
+  self.persistNewState(removedDocs, function (err) {
     if (err) { return callback(err); }
@@ -306,0 +344,0 @@ self.data = newData;

lib/model.js

@@ -21,3 +21,3 @@ /**
  * @param {Model} v value, needed to treat the Date edge case
- * Non-treatable edge case here: if part of the object if of the form { $$date: number }
+ * Non-treatable edge cases here: if part of the object if of the form { $$date: number } or { $$deleted: true }
  * Its serialized-then-deserialized version it will transformed into a Date object
@@ -27,3 +27,3 @@ * But you really need to want it to trigger such behaviour, even when warned not to use '$' at the beginning of the field names...
 function checkKey (k, v) {
-  if (k[0] === '$' && !(k === '$$date' && typeof v === 'number')) {
+  if (k[0] === '$' && !(k === '$$date' && typeof v === 'number') && !(k === '$$deleted' && v === true)) {
     throw 'Field names cannot begin with the $ character';
@@ -30,0 +30,0 @@ }

package.json

 {
   "name": "nedb",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "author": {
@@ -5,0 +5,0 @@ "name": "tldr.io",

README.md

@@ -5,3 +5,3 @@ # NeDB (Node embedded database)
 
-**Embedded persistent database for Node.js, with no dependency** (except npm
+**Embedded persistent database for Node.js, written in Javascript, with no dependency** (except npm
 modules of course). You can **think of it as a SQLite for Node.js projects**, which
@@ -275,9 +275,9 @@ can be used with a simple `require` statement. The API is a subset of MongoDB's.
 documents max). On my machine (3 years old, no SSD), with a collection
-containing 10,000 documents:  
-* An insert takes 0.1ms
-* A read takes 5.7ms
-* An update takes 58ms
-* A deletion takes 57ms  
+containing 10,000 documents and with no index (they are not implemented yet):  
+* An insert takes 0.1 ms
+* A read takes 5.7 ms
+* An update takes 10.9 ms
+* A deletion takes 8.1 ms  
 
-You can run the simple benchmarks I use by executing the scripts in the `benchmarks` folder. They all take an optional parameter which is the size of the dataset to use (default is 10,000). Most of the time spent during update and remove operations is IO, and I will work on optimizing this in the future (probably by switching to an append-only format).
+You can run the simple benchmarks I use by executing the scripts in the `benchmarks` folder. They all take an optional parameter which is the size of the dataset to use (default is 10,000).
 
@@ -284,0 +284,0 @@ ### Memory footprint

test/db.test.js

@@ -49,2 +49,3 @@ var Datastore = require('../lib/datastore')
 
+      treatedData.sort(function (a, b) { return a._id - b._id; });
       treatedData.length.should.equal(3);
@@ -64,2 +65,3 @@ _.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true);
 
+      treatedData.sort(function (a, b) { return a._id - b._id; });
       treatedData.length.should.equal(2);
@@ -78,2 +80,3 @@ _.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true);
 
+      treatedData.sort(function (a, b) { return a._id - b._id; });
       treatedData.length.should.equal(2);
@@ -84,2 +87,45 @@ _.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true);
 
+    it('If two lines concern the same doc (= same _id), the last one is the good version', function () {
+      var now = new Date()
+        , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
+                    model.serialize({ _id: "2", hello: 'world' }) + '\n' +
+                    model.serialize({ _id: "1", nested: { today: now } })
+        , treatedData = Datastore.treatRawData(rawData)
+        ;
+
+      treatedData.sort(function (a, b) { return a._id - b._id; });
+      treatedData.length.should.equal(2);
+      _.isEqual(treatedData[0], { _id: "1", nested: { today: now } }).should.equal(true);
+      _.isEqual(treatedData[1], { _id: "2", hello: 'world' }).should.equal(true);
+    });
+
+    it('If a doc contains $$deleted: true, that means we need to remove it from the data', function () {
+      var now = new Date()
+        , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
+                    model.serialize({ _id: "2", hello: 'world' }) + '\n' +
+                    model.serialize({ _id: "1", $$deleted: true }) + '\n' +
+                    model.serialize({ _id: "3", today: now })
+        , treatedData = Datastore.treatRawData(rawData)
+        ;
+
+      treatedData.sort(function (a, b) { return a._id - b._id; });
+      treatedData.length.should.equal(2);
+      _.isEqual(treatedData[0], { _id: "2", hello: 'world' }).should.equal(true);
+      _.isEqual(treatedData[1], { _id: "3", today: now }).should.equal(true);
+    });
+
+    it('If a doc contains $$deleted: true, no error is thrown if the doc wasnt in the list before', function () {
+      var now = new Date()
+        , rawData = model.serialize({ _id: "1", a: 2, ages: [1, 5, 12] }) + '\n' +
+                    model.serialize({ _id: "2", $$deleted: true }) + '\n' +
+                    model.serialize({ _id: "3", today: now })
+        , treatedData = Datastore.treatRawData(rawData)
+        ;
+
+      treatedData.sort(function (a, b) { return a._id - b._id; });
+      treatedData.length.should.equal(2);
+      _.isEqual(treatedData[0], { _id: "1", a: 2, ages: [1, 5, 12] }).should.equal(true);
+      _.isEqual(treatedData[1], { _id: "3", today: now }).should.equal(true);
+    });
+
   });   // ==== End of 'Loading the database data from file' ==== //
@@ -112,3 +158,2 @@
 
-
                 done();
@@ -686,2 +731,68 @@ });
 
+    it('Non-multi updates are persistent', function (done) {
+      d.insert({ a:1, hello: 'world' }, function (err, doc1) {
+        d.insert({ a:2, hello: 'earth' }, function (err, doc2) {
+          d.update({ a: 2 }, { $set: { hello: 'changed' } }, {}, function (err) {
+            assert.isNull(err);
+
+            d.find({}, function (err, docs) {
+              docs.sort(function (a, b) { return a.a - b.a; });
+              docs.length.should.equal(2);
+              _.isEqual(docs[0], { _id: doc1._id, a:1, hello: 'world' }).should.equal(true);
+              _.isEqual(docs[1], { _id: doc2._id, a:2, hello: 'changed' }).should.equal(true);
+
+              // Even after a reload the database state hasn't changed
+              d.loadDatabase(function (err) {
+                assert.isUndefined(err);
+
+                d.find({}, function (err, docs) {
+                  docs.sort(function (a, b) { return a.a - b.a; });
+                  docs.length.should.equal(2);
+                  _.isEqual(docs[0], { _id: doc1._id, a:1, hello: 'world' }).should.equal(true);
+                  _.isEqual(docs[1], { _id: doc2._id, a:2, hello: 'changed' }).should.equal(true);
+
+                  done();
+                });
+              });
+            });
+          });
+        });
+      });
+    });
+
+    it('Multi updates are persistent', function (done) {
+      d.insert({ a:1, hello: 'world' }, function (err, doc1) {
+        d.insert({ a:2, hello: 'earth' }, function (err, doc2) {
+          d.insert({ a:5, hello: 'pluton' }, function (err, doc3) {
+            d.update({ a: { $in: [1, 2] } }, { $set: { hello: 'changed' } }, { multi: true }, function (err) {
+              assert.isNull(err);
+
+              d.find({}, function (err, docs) {
+                docs.sort(function (a, b) { return a.a - b.a; });
+                docs.length.should.equal(3);
+                _.isEqual(docs[0], { _id: doc1._id, a:1, hello: 'changed' }).should.equal(true);
+                _.isEqual(docs[1], { _id: doc2._id, a:2, hello: 'changed' }).should.equal(true);
+                _.isEqual(docs[2], { _id: doc3._id, a:5, hello: 'pluton' }).should.equal(true);
+
+                // Even after a reload the database state hasn't changed
+                d.loadDatabase(function (err) {
+                  assert.isUndefined(err);
+
+                  d.find({}, function (err, docs) {
+                    docs.sort(function (a, b) { return a.a - b.a; });
+                    docs.length.should.equal(3);
+                    _.isEqual(docs[0], { _id: doc1._id, a:1, hello: 'changed' }).should.equal(true);
+                    _.isEqual(docs[1], { _id: doc2._id, a:2, hello: 'changed' }).should.equal(true);
+                    _.isEqual(docs[2], { _id: doc3._id, a:5, hello: 'pluton' }).should.equal(true);
+
+                    done();
+                  });
+                });
+              });
+            });
+          });
+        });
+      });
+    });
+
   });   // ==== End of 'Update' ==== //
@@ -774,2 +885,64 @@
 
+    it('Non-multi removes are persistent', function (done) {
+      d.insert({ a:1, hello: 'world' }, function (err, doc1) {
+        d.insert({ a:2, hello: 'earth' }, function (err, doc2) {
+          d.insert({ a:3, hello: 'moto' }, function (err, doc3) {
+            d.remove({ a: 2 }, {}, function (err) {
+              assert.isNull(err);
+
+              d.find({}, function (err, docs) {
+                docs.sort(function (a, b) { return a.a - b.a; });
+                docs.length.should.equal(2);
+                _.isEqual(docs[0], { _id: doc1._id, a:1, hello: 'world' }).should.equal(true);
+                _.isEqual(docs[1], { _id: doc3._id, a:3, hello: 'moto' }).should.equal(true);
+
+                // Even after a reload the database state hasn't changed
+                d.loadDatabase(function (err) {
+                  assert.isUndefined(err);
+
+                  d.find({}, function (err, docs) {
+                    docs.sort(function (a, b) { return a.a - b.a; });
+                    docs.length.should.equal(2);
+                    _.isEqual(docs[0], { _id: doc1._id, a:1, hello: 'world' }).should.equal(true);
+                    _.isEqual(docs[1], { _id: doc3._id, a:3, hello: 'moto' }).should.equal(true);
+
+                    done();
+                  });
+                });
+              });
+            });
+          });
+        });
+      });
+    });
+
+    it('Multi removes are persistent', function (done) {
+      d.insert({ a:1, hello: 'world' }, function (err, doc1) {
+        d.insert({ a:2, hello: 'earth' }, function (err, doc2) {
+          d.insert({ a:3, hello: 'moto' }, function (err, doc3) {
+            d.remove({ a: { $in: [1, 3] } }, { multi: true }, function (err) {
+              assert.isNull(err);
+
+              d.find({}, function (err, docs) {
+                docs.length.should.equal(1);
+                _.isEqual(docs[0], { _id: doc2._id, a:2, hello: 'earth' }).should.equal(true);
+
+                // Even after a reload the database state hasn't changed
+                d.loadDatabase(function (err) {
+                  assert.isUndefined(err);
+
+                  d.find({}, function (err, docs) {
+                    docs.length.should.equal(1);
+                    _.isEqual(docs[0], { _id: doc2._id, a:2, hello: 'earth' }).should.equal(true);
+
+                    done();
+                  });
+                });
+              });
+            });
+          });
+        });
+      });
+    });
+
   });   // ==== End of 'Remove' ==== //
@@ -776,0 +949,0 @@

test/model.test.js

@@ -112,12 +112,16 @@ var model = require('../lib/model')
 
-    it('Reject field names beginning with a $ sign', function (done) {
-      var a = { $something: 'totest' }
+    it('Reject field names beginning with a $ sign or containing a dot, except the two edge cases', function () {
+      var a1 = { $something: 'totest' }
+        , a2 = { "with.dot": 'totest' }
+        , e1 = { $$date: 4321 }
+        , e2 = { $$deleted: true }
         , b;
 
-      try {
-        b = model.serialize(a);
-        return done('An error should have been thrown');
-      } catch (e) {
-        return done();
-      }
+      // Normal cases
+      (function () { b = model.serialize(a1); }).should.throw();
+      (function () { b = model.serialize(a2); }).should.throw();
+
+      // Edge cases
+      b = model.serialize(e1);
+      b = model.serialize(e2);
     });
@@ -124,0 +128,0 @@

No alert changes

Improved metrics

Total package byte prevSize

108287

increased by8.97%

Lines of code

2319

increased by8.52%

No dependency changes