nedb - npm Package Compare versions

Comparing version 0.8.12 to 0.8.13

lib/customUtils.js

@@ -1,2 +0,4 @@
-var crypto = require('crypto');
+var crypto = require('crypto')
+  , fs = require('fs')
+  ;
 
@@ -19,2 +21,15 @@ /**
 
+/**
+ * Callback signature: err
+ */
+function ensureFileDoesntExist (file, callback) {
+  fs.exists(file, function (exists) {
+    if (!exists) { return callback(null); }
+    
+    fs.unlink(file, function (err) { return callback(err); });
+  });
+}
+
+
 module.exports.uid = uid;
+module.exports.ensureFileDoesntExist = ensureFileDoesntExist;

lib/datastore.js

@@ -543,4 +543,4 @@ var customUtils = require('./customUtils')
 };
+      
 
-
 module.exports = Datastore;

lib/persistence.js

@@ -13,2 +13,3 @@ /**
   , mkdirp = require('mkdirp')
+  , customUtils = require('./customUtils')
   ;
@@ -27,2 +28,11 @@
   this.filename = this.db.filename;
+  
+  if (!this.inMemoryOnly && this.filename) {
+    if (this.filename.charAt(this.filename.length - 1) === '~') {
+      throw "The datafile name can't end with a ~, which is reserved for automatic backup files";
+    } else {
+      this.tempFilename = this.filename + '~';
+      this.oldFilename = this.filename + '~~';
+    }
+  }
 
@@ -90,9 +100,31 @@ // For NW apps, store data in the same directory where NW stores application data
     , toPersist = ''
+    , self = this
     ;
 
+  if (this.inMemoryOnly) { return callback(null); } 
+
   this.db.getAllData().forEach(function (doc) {
     toPersist += model.serialize(doc) + '\n';
   });
-
-  fs.writeFile(this.filename, toPersist, function (err) { return callback(err); });
+  
+  async.waterfall([
+    async.apply(customUtils.ensureFileDoesntExist, self.tempFilename)
+  , async.apply(customUtils.ensureFileDoesntExist, self.oldFilename)
+  , function (cb) {
+      fs.exists(self.filename, function (exists) {
+        if (exists) {
+          fs.rename(self.filename, self.oldFilename, function (err) { return cb(err); });
+        } else {
+          return cb();
+        }
+      });  
+  }
+  , function (cb) {
+      fs.writeFile(self.tempFilename, toPersist, function (err) { return cb(err); });
+    }
+  , function (cb) {
+      fs.rename(self.tempFilename, self.filename, function (err) { return cb(err); });
+    }
+  , async.apply(customUtils.ensureFileDoesntExist, self.oldFilename)
+  ], function (err) { if (err) { return callback(err); } else { return callback(null); } })
 };
@@ -195,2 +227,26 @@
 /**
+ * Ensure that this.filename contains the most up-to-date version of the data
+ * Even if a loadDatabase crashed before
+ */
+Persistence.prototype.ensureDatafileIntegrity = function (callback) {
+  var self = this  ;
+
+  fs.exists(self.filename, function (filenameExists) {
+    // Write was successful
+    if (filenameExists) { return callback(null); }
+  
+    fs.exists(self.oldFilename, function (oldFilenameExists) {
+      // New database
+      if (!oldFilenameExists) {
+        return fs.writeFile(self.filename, '', 'utf8', function (err) { callback(err); });            
+      }
+    
+      // Write failed, use old version
+      fs.rename(self.oldFilename, self.filename, function (err) { return callback(err); });
+    });
+  });
+};
+
+
+/**
  * Load the database
@@ -215,5 +271,3 @@ * This means pulling data out of the data file or creating it if it doesn't exist
       Persistence.ensureDirectoryExists(path.dirname(self.filename), function (err) {
-        fs.exists(self.filename, function (exists) {
-          if (!exists) { return fs.writeFile(self.filename, '', 'utf8', function (err) { cb(err); }); }
-
+        self.ensureDatafileIntegrity(function (exists) {
           fs.readFile(self.filename, 'utf8', function (err, rawData) {
@@ -220,0 +274,0 @@ if (err) { return cb(err); }

package.json

 {
   "name": "nedb",
-  "version": "0.8.12",
+  "version": "0.8.13",
   "author": {
@@ -5,0 +5,0 @@ "name": "tldr.io",

README.md

@@ -38,3 +38,3 @@ # NeDB (Node embedded database)
 
-* `filename` (optional): path to the file where the data is persisted. If left blank, the datastore is automatically considered in-memory only.
+* `filename` (optional): path to the file where the data is persisted. If left blank, the datastore is automatically considered in-memory only. It cannot end with a `~` which is used in the temporary files NeDB uses to perform crash-safe writes
 * `inMemoryOnly` (optional, defaults to false): as the name implies.
@@ -41,0 +41,0 @@ * `autoload` (optional, defaults to false): if used, the database will

test/model.test.js

@@ -7,2 +7,4 @@ var model = require('../lib/model')
   , util = require('util')
+  , Datastore = require('../lib/datastore')
+  , fs = require('fs')
   ;
@@ -129,3 +131,32 @@
     });
+    
+    it('Can serialize string fields with a new line without breaking the DB', function (done) {
+      var db1, db2
+        , badString = "world\r\nearth\nother\rline"
+        ;
+      
+      if (fs.existsSync('workspace/test1.db')) { fs.unlinkSync('workspace/test1.db'); }
+      fs.existsSync('workspace/test1.db').should.equal(false);
+      db1 = new Datastore({ filename: 'workspace/test1.db' });
+      
+      db1.loadDatabase(function (err) {
+        assert.isNull(err);
+        db1.insert({ hello: badString }, function (err) {
+          assert.isNull(err);
+        
+          db2 = new Datastore({ filename: 'workspace/test1.db' });
+          db2.loadDatabase(function (err) {
+            assert.isNull(err);
+            db2.find({}, function (err, docs) {
+              assert.isNull(err);
+              docs.length.should.equal(1);
+              docs[0].hello.should.equal(badString);
 
+              done();
+            });
+          });
+        });
+      });
+    });
+
   });   // ==== End of 'Serialization, deserialization' ==== //
@@ -132,0 +163,0 @@

test/persistence.test.js

@@ -9,4 +9,6 @@ var should = require('chai').should()
   , model = require('../lib/model')
+  , customUtils = require('../lib/customUtils')
   , Datastore = require('../lib/datastore')
   , Persistence = require('../lib/persistence')
+  , child_process = require('child_process')
   ;
@@ -251,3 +253,352 @@
   });
+  
+  
+  describe('Prevent dataloss when persisting data', function () {
 
+    it('Creating a datastore with in memory as true and a bad filename wont cause an error', function () {
+      new Datastore({ filename: 'workspace/bad.db~', inMemoryOnly: true });
+    })
+    
+    it('Creating a persistent datastore with a bad filename will cause an error', function () {
+      (function () { new Datastore({ filename: 'workspace/bad.db~' }); }).should.throw();
+    })  
+  
+    it('If no file exists, ensureDatafileIntegrity creates an empty datafile', function (done) {
+      var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
+    
+      if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
+      if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
+      
+      fs.existsSync('workspace/it.db').should.equal(false);
+      fs.existsSync('workspace/it.db~~').should.equal(false);      
+      
+      p.ensureDatafileIntegrity(function (err) {
+        assert.isNull(err);
+        
+        fs.existsSync('workspace/it.db').should.equal(true);
+        fs.existsSync('workspace/it.db~~').should.equal(false);
+        
+        fs.readFileSync('workspace/it.db', 'utf8').should.equal('');
+        
+        done();
+      });
+    });
+  
+    it('If only datafile exists, ensureDatafileIntegrity will use it', function (done) {
+      var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
+    
+      if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
+      if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
+      
+      fs.writeFileSync('workspace/it.db', 'something', 'utf8');
+
+      fs.existsSync('workspace/it.db').should.equal(true);
+      fs.existsSync('workspace/it.db~~').should.equal(false);      
+      
+      p.ensureDatafileIntegrity(function (err) {
+        assert.isNull(err);
+
+        fs.existsSync('workspace/it.db').should.equal(true);
+        fs.existsSync('workspace/it.db~~').should.equal(false);
+        
+        fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
+        
+        done();
+      });
+    });
+    
+    it('If old datafile exists and datafile doesnt, ensureDatafileIntegrity will use it', function (done) {
+      var p = new Persistence({ db: { inMemoryOnly: false, filename: 'workspace/it.db' } });
+    
+      if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
+      if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
+      
+      fs.writeFileSync('workspace/it.db~~', 'something', 'utf8');
+      
+      fs.existsSync('workspace/it.db').should.equal(false);
+      fs.existsSync('workspace/it.db~~').should.equal(true);      
+      
+      p.ensureDatafileIntegrity(function (err) {
+        assert.isNull(err);
+        
+        fs.existsSync('workspace/it.db').should.equal(true);
+        fs.existsSync('workspace/it.db~~').should.equal(false);
+        
+        fs.readFileSync('workspace/it.db', 'utf8').should.equal('something');
+        
+        done();
+      });
+    });
+    
+    it('If both old and current datafiles exist, ensureDatafileIntegrity will use the datafile, it means step 4 of persistence failed', function (done) {
+      var theDb = new Datastore({ filename: 'workspace/it.db' });
+    
+      if (fs.existsSync('workspace/it.db')) { fs.unlinkSync('workspace/it.db'); }
+      if (fs.existsSync('workspace/it.db~~')) { fs.unlinkSync('workspace/it.db~~'); }
+      
+      fs.writeFileSync('workspace/it.db', '{"_id":"0","hello":"world"}', 'utf8');
+      fs.writeFileSync('workspace/it.db~~', '{"_id":"0","hello":"other"}', 'utf8');
+      
+      fs.existsSync('workspace/it.db').should.equal(true);
+      fs.existsSync('workspace/it.db~~').should.equal(true);      
+      
+      theDb.persistence.ensureDatafileIntegrity(function (err) {
+        assert.isNull(err);
+        
+        fs.existsSync('workspace/it.db').should.equal(true);
+        fs.existsSync('workspace/it.db~~').should.equal(true);
+        
+        fs.readFileSync('workspace/it.db', 'utf8').should.equal('{"_id":"0","hello":"world"}');
+        
+        theDb.loadDatabase(function (err) {
+          assert.isNull(err);
+          theDb.find({}, function (err, docs) {
+            assert.isNull(err);
+            docs.length.should.equal(1);
+            docs[0].hello.should.equal("world");
+            done();
+          });
+        });
+      });
+    });
+  
+    it('persistCachedDatabase should update the contents of the datafile and leave a clean state', function (done) {
+      d.insert({ hello: 'world' }, function () {
+        d.find({}, function (err, docs) {
+          docs.length.should.equal(1);
+          
+          if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
+          if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
+          if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
+          fs.existsSync(testDb).should.equal(false);
+          
+          fs.writeFileSync(testDb + '~', 'something', 'utf8');
+          fs.writeFileSync(testDb + '~~', 'something else', 'utf8');
+          fs.existsSync(testDb + '~').should.equal(true);
+          fs.existsSync(testDb + '~~').should.equal(true);
+          
+          d.persistence.persistCachedDatabase(function (err) {
+            var contents = fs.readFileSync(testDb, 'utf8');
+            assert.isNull(err);
+            fs.existsSync(testDb).should.equal(true);
+            fs.existsSync(testDb + '~').should.equal(false);            
+            fs.existsSync(testDb + '~~').should.equal(false);            
+            if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
+              throw "Datafile contents not as expected";
+            }
+            done();
+          });
+        });
+      });
+    });
+    
+    it('After a persistCachedDatabase, there should be no temp or old filename', function (done) {
+      d.insert({ hello: 'world' }, function () {
+        d.find({}, function (err, docs) {
+          docs.length.should.equal(1);
+          
+          if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
+          if (fs.existsSync(testDb + '~')) { fs.unlinkSync(testDb + '~'); }
+          if (fs.existsSync(testDb + '~~')) { fs.unlinkSync(testDb + '~~'); }
+          fs.existsSync(testDb).should.equal(false);
+          
+          fs.writeFileSync(testDb + '~', 'bloup', 'utf8');
+          fs.writeFileSync(testDb + '~~', 'blap', 'utf8');
+          fs.existsSync(testDb + '~').should.equal(true);
+          fs.existsSync(testDb + '~~').should.equal(true);
+          
+          d.persistence.persistCachedDatabase(function (err) {
+            var contents = fs.readFileSync(testDb, 'utf8');
+            assert.isNull(err);
+            fs.existsSync(testDb).should.equal(true);
+            fs.existsSync(testDb + '~').should.equal(false);            
+            fs.existsSync(testDb + '~~').should.equal(false);            
+            if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
+              throw "Datafile contents not as expected";
+            }
+            done();
+          });
+        });
+      });    
+    });
+    
+    it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
+      d.insert({ hello: 'world' }, function () {
+        d.find({}, function (err, docs) {
+          docs.length.should.equal(1);
+          
+          if (fs.existsSync(testDb)) { fs.unlinkSync(testDb); }
+          fs.writeFileSync(testDb + '~', 'blabla', 'utf8');
+          fs.writeFileSync(testDb + '~~', 'bloblo', 'utf8');
+          fs.existsSync(testDb).should.equal(false);
+          fs.existsSync(testDb + '~').should.equal(true);
+          fs.existsSync(testDb + '~~').should.equal(true);
+          
+          d.persistence.persistCachedDatabase(function (err) {
+            var contents = fs.readFileSync(testDb, 'utf8');
+            assert.isNull(err);
+            fs.existsSync(testDb).should.equal(true);
+            fs.existsSync(testDb + '~').should.equal(false);            
+            fs.existsSync(testDb + '~~').should.equal(false);            
+            if (!contents.match(/^{"hello":"world","_id":"[0-9a-zA-Z]{16}"}\n$/)) {
+              throw "Datafile contents not as expected";
+            }
+            done();
+          });
+        });
+      });
+    });
+    
+    it('persistCachedDatabase should update the contents of the datafile and leave a clean state even if there is a temp or old datafile', function (done) {
+      var dbFile = 'workspace/test2.db', theDb;
+    
+      if (fs.existsSync(dbFile)) { fs.unlinkSync(dbFile); }
+      if (fs.existsSync(dbFile + '~')) { fs.unlinkSync(dbFile + '~'); }
+      if (fs.existsSync(dbFile + '~~')) { fs.unlinkSync(dbFile + '~~'); }
+      
+      theDb = new Datastore({ filename: dbFile });
+      
+      theDb.loadDatabase(function (err) {
+        var contents = fs.readFileSync(dbFile, 'utf8');
+        assert.isNull(err);
+        fs.existsSync(dbFile).should.equal(true);
+        fs.existsSync(dbFile + '~').should.equal(false);            
+        fs.existsSync(dbFile + '~~').should.equal(false);            
+        if (contents != "") {
+          throw "Datafile contents not as expected";
+        }
+        done();
+      });
+    });
+
+    it('Persistence works as expected when everything goes fine', function (done) {
+      var dbFile = 'workspace/test2.db', theDb, theDb2, doc1, doc2;
+      
+      async.waterfall([
+        async.apply(customUtils.ensureFileDoesntExist, dbFile)
+      , async.apply(customUtils.ensureFileDoesntExist, dbFile + '~')
+      , async.apply(customUtils.ensureFileDoesntExist, dbFile + '~~')
+      , function (cb) {
+        theDb = new Datastore({ filename: dbFile });
+        theDb.loadDatabase(cb);
+      }
+      , function (cb) {
+        theDb.find({}, function (err, docs) {
+          assert.isNull(err);
+          docs.length.should.equal(0);
+          return cb();
+        });
+      }
+      , function (cb) {
+        theDb.insert({ a: 'hello' }, function (err, _doc1) {
+          assert.isNull(err);
+          doc1 = _doc1;
+          theDb.insert({ a: 'world' }, function (err, _doc2) {
+            assert.isNull(err);
+            doc2 = _doc2;
+            return cb();
+          });
+        });
+      }
+      , function (cb) {
+        theDb.find({}, function (err, docs) {
+          assert.isNull(err);
+          docs.length.should.equal(2);
+          _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
+          _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
+          return cb();
+        });
+      }
+      , function (cb) {
+        theDb.loadDatabase(cb);
+      }
+      , function (cb) {   // No change
+        theDb.find({}, function (err, docs) {
+          assert.isNull(err);
+          docs.length.should.equal(2);
+          _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
+          _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
+          return cb();
+        });
+      }
+      , function (cb) {
+        fs.existsSync(dbFile).should.equal(true);
+        fs.existsSync(dbFile + '~').should.equal(false);
+        fs.existsSync(dbFile + '~~').should.equal(false);
+        return cb();
+      }
+      , function (cb) {
+        theDb2 = new Datastore({ filename: dbFile });
+        theDb2.loadDatabase(cb);
+      }  
+      , function (cb) {   // No change in second db
+        theDb2.find({}, function (err, docs) {
+          assert.isNull(err);
+          docs.length.should.equal(2);
+          _.find(docs, function (item) { return item._id === doc1._id }).a.should.equal('hello');
+          _.find(docs, function (item) { return item._id === doc2._id }).a.should.equal('world');
+          return cb();
+        });
+      }
+      , function (cb) {
+        fs.existsSync(dbFile).should.equal(true);
+        fs.existsSync(dbFile + '~').should.equal(false);
+        fs.existsSync(dbFile + '~~').should.equal(false);
+        return cb();
+      } 
+      ], done);
+    });
+    
+  
+    // This test is a bit complicated since it depends on the time I/O actions take to execute
+    // That depends on the machine and the load on the machine when the tests are run
+    // It is timed for my machine with nothing else running but may not work as expected on others (it will not fail but may not be a proof)
+    // Every new version of NeDB passes it on my machine before rtelease
+    it('If system crashes during a loadDatabase, the former version is not lost', function (done) {
+      var cp, N = 150000, toWrite = "", i;
+      
+      // Ensuring the state is clean
+      if (fs.existsSync('workspace/lac.db')) { fs.unlinkSync('workspace/lac.db'); }
+      if (fs.existsSync('workspace/lac.db~')) { fs.unlinkSync('workspace/lac.db~'); }
+
+      // Creating a db file with 150k records (a bit long to load)
+      for (i = 0; i < N; i += 1) {
+        toWrite += model.serialize({ _id: customUtils.uid(16), hello: 'world' }) + '\n';
+      }        
+      fs.writeFileSync('workspace/lac.db', toWrite, 'utf8');
+      
+      // Loading it in a separate process that we will crash before finishing the loadDatabase
+      cp = child_process.fork('test_lac/loadAndCrash.test')
+      
+      // Kill the child process when we're at step 3 of persistCachedDatabase (during write to datafile)
+      setTimeout(function() {
+        cp.kill('SIGINT');
+        
+        // If the timing is correct, only the temp datafile contains data
+        // The datafile was in the middle of being written and is empty
+        
+        // Let the process crash be finished then load database without a crash, and test we didn't lose data
+        setTimeout(function () {
+          var db = new Datastore({ filename: 'workspace/lac.db' });
+          db.loadDatabase(function (err) {
+            assert.isNull(err);
+            
+            db.count({}, function (err, n) {
+              // Data has not been lost
+              assert.isNull(err);
+              n.should.equal(150000);
+              
+              // State is clean, the temp datafile has been erased and the datafile contains all the data
+              fs.existsSync('workspace/lac.db').should.equal(true);
+              fs.existsSync('workspace/lac.db~').should.equal(false);
+              
+              done();
+            });
+          });
+        }, 100);        
+      }, 2000);
+    });
+  
+  });
+
 });

No alert changes

Improved metrics

Total package byte prevSize

626448

increased by3.21%

Number of package files

36

increased by5.88%

Lines of code

14997

increased by2.75%

Worsened metrics

Number of low supply chain risk alerts

31

increased by10.71%

Number of medium supply chain risk alerts

4

increased by33.33%

No dependency changes