Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
@seald-io/nedb
Advanced tools
@seald-io/nedb - npm Package Compare versions
Comparing version 2.1.0-3 to 2.1.0-test.actions.1
@@ -71,7 +71,2 @@ /** | ||
| const crashSafeWriteFileLines = (filename, lines, callback) => { | ||
| lines.push('') // Add final new line | ||
| writeFile(filename, lines.join('\n'), callback) | ||
| } | ||
| // Interface | ||
@@ -81,3 +76,3 @@ module.exports.exists = exists | ||
| module.exports.writeFile = writeFile | ||
| module.exports.crashSafeWriteFileLines = crashSafeWriteFileLines | ||
| module.exports.crashSafeWriteFile = writeFile // No need for a crash safe function in the browser | ||
| module.exports.appendFile = appendFile | ||
@@ -84,0 +79,0 @@ module.exports.readFile = readFile |
@@ -9,17 +9,2 @@ # Changelog | ||
| ## [2.1.0-2] - 2021-10-14 | ||
| ### Changed | ||
| - properly streaming writing the database file | ||
| ## [2.1.0-1] - 2021-10-07 | ||
| ### Changed | ||
| - fixed package.json browser field for byline.js | ||
| - last minute improvements on [PR](https://github.com/seald/nedb/pull/5) | ||
| ## [2.1.0-0] - 2021-10-05 | ||
| Thank [@eliot-akira](https://github.com/eliot-akira) for the amazing work on this. | ||
| ### Changed | ||
| - [implement file streaming of the database](https://github.com/seald/nedb/pull/5) like [a PR on the original repo](https://github.com/louischatriot/nedb/pull/463) did; | ||
| - internalize [`byline`](https://github.com/jahewson/node-byline) package because it is unmaintained. | ||
| ## [2.0.4] - 2021-07-12 | ||
@@ -26,0 +11,0 @@ ### Fixed |
@@ -9,3 +9,2 @@ /** | ||
| const async = require('async') | ||
| const byline = require('./byline') | ||
| const customUtils = require('./customUtils.js') | ||
@@ -77,3 +76,3 @@ const Index = require('./indexes.js') | ||
| persistCachedDatabase (callback = () => {}) { | ||
| const lines = [] | ||
| let toPersist = '' | ||
@@ -83,7 +82,7 @@ if (this.inMemoryOnly) return callback(null) | ||
| this.db.getAllData().forEach(doc => { | ||
| lines.push(this.afterSerialization(model.serialize(doc))) | ||
| toPersist += this.afterSerialization(model.serialize(doc)) + '\n' | ||
| }) | ||
| Object.keys(this.db.indexes).forEach(fieldName => { | ||
| if (fieldName !== '_id') { // The special _id index is managed by datastore.js, the others need to be persisted | ||
| lines.push(this.afterSerialization(model.serialize({ | ||
| toPersist += this.afterSerialization(model.serialize({ | ||
| $$indexCreated: { | ||
@@ -94,7 +93,7 @@ fieldName: fieldName, | ||
| } | ||
| }))) | ||
| })) + '\n' | ||
| } | ||
| }) | ||
| storage.crashSafeWriteFileLines(this.filename, lines, err => { | ||
| storage.crashSafeWriteFile(this.filename, toPersist, err => { | ||
| if (err) return callback(err) | ||
@@ -163,5 +162,4 @@ this.db.emit('compaction.done') | ||
| const dataById = {} | ||
| const tdata = [] | ||
| const indexes = {} | ||
| // Last line of every data file is usually blank so not really corrupt | ||
| let corruptItems = -1 | ||
@@ -188,3 +186,3 @@ | ||
| const tdata = Object.values(dataById) | ||
| tdata.push(...Object.values(dataById)) | ||
@@ -195,49 +193,2 @@ return { data: tdata, indexes: indexes } | ||
| /** | ||
| * From a database's raw stream, return the corresponding | ||
| * machine understandable collection | ||
| */ | ||
| treatRawStream (rawStream, cb) { | ||
| const dataById = {} | ||
| const indexes = {} | ||
| // Last line of every data file is usually blank so not really corrupt | ||
| let corruptItems = -1 | ||
| const lineStream = byline(rawStream, { keepEmptyLines: true }) | ||
| let length = 0 | ||
| lineStream.on('data', (line) => { | ||
| try { | ||
| const doc = model.deserialize(this.beforeDeserialization(line)) | ||
| if (doc._id) { | ||
| if (doc.$$deleted === true) delete dataById[doc._id] | ||
| else dataById[doc._id] = doc | ||
| } else if (doc.$$indexCreated && doc.$$indexCreated.fieldName != null) indexes[doc.$$indexCreated.fieldName] = doc.$$indexCreated | ||
| else if (typeof doc.$$indexRemoved === 'string') delete indexes[doc.$$indexRemoved] | ||
| } catch (e) { | ||
| corruptItems += 1 | ||
| } | ||
| length++ | ||
| }) | ||
| lineStream.on('end', () => { | ||
| // A bit lenient on corruption | ||
| if (length > 0 && corruptItems / length > this.corruptAlertThreshold) { | ||
| const err = new Error(`More than ${Math.floor(100 * this.corruptAlertThreshold)}% of the data file is corrupt, the wrong beforeDeserialization hook may be used. Cautiously refusing to start NeDB to prevent dataloss`) | ||
| cb(err, null) | ||
| return | ||
| } | ||
| const data = Object.values(dataById) | ||
| cb(null, { data, indexes: indexes }) | ||
| }) | ||
| lineStream.on('error', function (err) { | ||
| cb(err) | ||
| }) | ||
| } | ||
| /** | ||
| * Load the database | ||
@@ -266,4 +217,10 @@ * 1) Create all indexes | ||
| // TODO: handle error | ||
| const treatedDataCallback = (err, treatedData) => { | ||
| storage.readFile(this.filename, 'utf8', (err, rawData) => { | ||
| if (err) return cb(err) | ||
| let treatedData | ||
| try { | ||
| treatedData = this.treatRawData(rawData) | ||
| } catch (e) { | ||
| return cb(e) | ||
| } | ||
@@ -284,21 +241,2 @@ // Recreate all indexes in the datafile | ||
| this.db.persistence.persistCachedDatabase(cb) | ||
| } | ||
| if (storage.readFileStream) { | ||
| // Server side | ||
| const fileStream = storage.readFileStream(this.filename, { encoding: 'utf8' }) | ||
| this.treatRawStream(fileStream, treatedDataCallback) | ||
| return | ||
| } | ||
| // Browser | ||
| storage.readFile(this.filename, 'utf8', (err, rawData) => { | ||
| if (err) return cb(err) | ||
| try { | ||
| const treatedData = this.treatRawData(rawData) | ||
| treatedDataCallback(null, treatedData) | ||
| } catch (e) { | ||
| return cb(e) | ||
| } | ||
| }) | ||
@@ -305,0 +243,0 @@ }) |
@@ -13,3 +13,2 @@ /** | ||
| const storage = {} | ||
| const { Readable } = require('stream') | ||
@@ -23,3 +22,2 @@ // eslint-disable-next-line node/no-callback-literal | ||
| storage.readFile = fs.readFile | ||
| storage.readFileStream = fs.createReadStream | ||
| storage.mkdir = fs.mkdir | ||
@@ -77,36 +75,8 @@ | ||
| /** | ||
| * Fully write or rewrite the datafile | ||
| * @param {String} filename | ||
| * @param {String[]} lines | ||
| * @param {Function} callback | ||
| */ | ||
| storage.writeFileLines = (filename, lines, callback = () => {}) => { | ||
| try { | ||
| const stream = fs.createWriteStream(filename) | ||
| const readable = Readable.from(lines) | ||
| readable.on('data', (line) => { | ||
| try { | ||
| stream.write(line) | ||
| stream.write('\n') | ||
| } catch (err) { | ||
| callback(err) | ||
| } | ||
| }) | ||
| readable.on('end', () => { | ||
| stream.close(callback) | ||
| }) | ||
| readable.on('error', callback) | ||
| stream.on('error', callback) | ||
| } catch (err) { | ||
| callback(err) | ||
| } | ||
| } | ||
| /** | ||
| * Fully write or rewrite the datafile, immune to crashes during the write operation (data will not be lost) | ||
| * @param {String} filename | ||
| * @param {String[]} lines | ||
| * @param {String} data | ||
| * @param {Function} callback Optional callback, signature: err | ||
| */ | ||
| storage.crashSafeWriteFileLines = (filename, lines, callback = () => {}) => { | ||
| storage.crashSafeWriteFile = (filename, data, callback = () => {}) => { | ||
| const tempFilename = filename + '~' | ||
@@ -123,3 +93,3 @@ | ||
| cb => { | ||
| storage.writeFileLines(tempFilename, lines, cb) | ||
| storage.writeFile(tempFilename, data, err => cb(err)) | ||
| }, | ||
@@ -126,0 +96,0 @@ async.apply(storage.flushToStorage, tempFilename), |
| { | ||
| "name": "@seald-io/nedb", | ||
| "version": "2.1.0-3", | ||
| "version": "2.1.0-test.actions.1", | ||
| "files": [ | ||
@@ -23,7 +23,2 @@ "lib/**/*.js", | ||
| "url": "https://www.seald.io/" | ||
| }, | ||
| { | ||
| "name": "Eliot Akira", | ||
| "email": "me@eliotakira.com", | ||
| "url": "https://eliotakira.com/" | ||
| } | ||
@@ -82,4 +77,3 @@ ], | ||
| "./lib/customUtils.js": "./browser-version/lib/customUtils.js", | ||
| "./lib/storage.js": "./browser-version/lib/storage.js", | ||
| "./lib/byline.js": "./browser-version/lib/byline.js" | ||
| "./lib/storage.js": "./browser-version/lib/storage.js" | ||
| }, | ||
@@ -86,0 +80,0 @@ "license": "MIT", |
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
New alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Fixed alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Worsened metrics
- Total package byte prevSize
- decreased by-2.31%
497934
- Number of package files
- decreased by-10%
18
- Lines of code
- decreased by-2.76%
9616
No dependency changes