node-persist - npm Package Compare versions

Comparing version 3.1.0 to 3.1.1

package.json

 {
   "name": "node-persist",
-  "version": "3.1.0",
+  "version": "3.1.1",
   "description": "Super-easy (and fast) persistent data structures in Node.js, modeled after HTML5 localStorage",
@@ -5,0 +5,0 @@ "main": "./src/node-persist.js",

README.md

@@ -36,2 +36,9 @@ # node-persist
 
+## 3.1.1 change logs
+
+backward changes
+
+* Added the `writeQueue*` options, trying to resolve [issue#108](https://github.com/simonlast/node-persist/issues/108), see the API Documentation below.
+
+
 ## 3.0.0 change logs
@@ -89,12 +96,23 @@
 
-	logging: false,  // can also be custom logging function
+	// can also be custom logging function
+	logging: false,  
 
-	ttl: false, // ttl* [NEW], can be true for 24h default or a number in MILLISECONDS or a valid Javascript Date object
+	// ttl* [NEW], can be true for 24h default or a number in MILLISECONDS or a valid Javascript Date object
+	ttl: false,
 
-	expiredInterval: 2 * 60 * 1000, // every 2 minutes the process will clean-up the expired cache
+	// every 2 minutes the process will clean-up the expired cache
+	expiredInterval: 2 * 60 * 1000, 
 
     // in some cases, you (or some other service) might add non-valid storage files to your
     // storage dir, i.e. Google Drive, make this true if you'd like to ignore these files and not throw an error
-    forgiveParseErrors: false
-
+    forgiveParseErrors: false,
+	
+	// instead of writing to file immediately, each "file" will have its own mini queue to avoid corrupted files, keep in mind that this would not properly work in multi-process setting.
+	writeQueue: true, 
+	
+	// how often to check for pending writes, don't worry if you feel like 1s is a lot, it actually tries to process every time you setItem as well
+	writeQueueIntervalMs: 1000, 
+	
+	// if you setItem() multiple times to the same key, only the last one would be set, BUT the others would still resolve with the results of the last one, if you turn this to false, each one will execute, but might slow down the writing process.
+	writeQueueWriteOnlyLast: true, 
 });
@@ -121,9 +139,9 @@
 #### `async updateItem(key, value, [options])`
-This function updates a 'key' in your database with a new 'value' without touching the `ttl`, however, if the `key` was not found of it was `expired` a new item will get set
+This function updates a 'key' in your database with a new 'value' without touching the `ttl`, however, if the `key` was not found or if it was `expired` a new item will get set
 
 ```js
-await storage.setItem(42,'the answer to life, the universe, and everything.', {ttl: 1000*60*10 /* 10 minutes */ });
-await storage.setItem(42,'means nothing, do not trust wikipedia'); // ttl is still the same, will expired in 10 minutes since it was first set
+await storage.updateItem(42,'the answer to life, the universe, and everything.', {ttl: 1000*60*10 /* 10 minutes */ });
+await storage.updateItem(42,'means nothing, do not trust wikipedia'); // ttl is still the same, will expired in 10 minutes since it was first set
 ```
-\* The only option available when calling `setItem(key, value, option)` is `{ttl: Number|Date}`
+\* The only option available when calling `updateItem(key, value, option)` is `{ttl: Number|Date}`
 
@@ -130,0 +148,0 @@ #### `async removeItem(key)`

src/local-storage.js

@@ -10,12 +10,16 @@ /*
 const pkg = require('../package.json');
+const { nextTick } = require('process');
 
 const defaults = {
-	dir: '.' + pkg.name + '/storage',
+	ttl: false,
+	logging: false,
+	encoding: 'utf8',
+	parse: JSON.parse,
 	stringify: JSON.stringify,
-	parse: JSON.parse,
-	encoding: 'utf8',
-	logging: false,
+	forgiveParseErrors: false,
 	expiredInterval: 2 * 60 * 1000, /* every 2 minutes */
-	forgiveParseErrors: false,
-	ttl: false
+	dir: '.' + pkg.name + '/storage',
+	writeQueue: true,
+	writeQueueIntervalMs: 1000,
+	writeQueueWriteOnlyLast: true,
 };
@@ -86,2 +90,4 @@
 		}
+		this.q = {}
+		this.startWriteQueueInterval();
 		return this.options;
@@ -164,7 +170,5 @@ },
 		let ttl = this.calcTTL(options.ttl);
-		if (this.logging) {
-			this.log(`set ('${key}': '${this.stringify(value)}')`);
-		}
-		let datum = {key: key, value: value, ttl: ttl};
-		return this.writeFile(this.getDatumPath(key), datum);
+		this.log(`set ('${key}': '${this.stringify(value)}')`);
+		let datum = { key, value, ttl };
+		return this.queueWriteFile(this.getDatumPath(key), datum);
 	},
@@ -186,7 +190,5 @@
 			}
-			if (this.logging) {
-				this.log(`update ('${key}': '${this.stringify(newDatumValue)}')`);
-			}
-			let datum = {key: key, value: newDatumValue, ttl: ttl};
-			return this.writeFile(this.getDatumPath(key), datum);
+			this.log(`update ('${key}': '${this.stringify(newDatumValue)}')`);
+			let datum = { key, value: newDatumValue, ttl };
+			return this.queueWriteFile(this.getDatumPath(key), datum);
 		} else {
@@ -325,5 +327,72 @@ return this.setItem(key, datumValue, options);
 
-	writeFile: function (file, content) {
+	queueWriteFile: async function (file, content) {
+		if (this.options.writeQueue === false) {
+			return this.writeFile(file, content)			
+		}
+		this.q[file] = this.q[file] || []
+		nextTick(() => {
+			this.startWriteQueueInterval()
+		})
 		return new Promise((resolve, reject) => {
-			fs.writeFile(file, this.stringify(content), this.options.encoding, (err) => {
+			this.q[file].push({ content, resolve, reject })
+		})
+	},
+	
+	processWriteQueue: async function () {
+		if (this.processingWriteQueue) {
+			this.log('Still processing write queue, waiting...');
+			return
+		}
+		this.processingWriteQueue = true
+		const promises = Object.keys(this.q).map(async file => {
+			let writeItem
+			if (this.options.writeQueueWriteOnlyLast) {
+				// lifo
+				writeItem = this.q[file].pop()
+			} else {
+				// fifo
+				writeItem = this.q[file].shift()
+			}
+			try {
+				const ret = await this.writeFile(file, writeItem.content)
+				if (this.options.writeQueueWriteOnlyLast) {
+					while (this.q[file].length) {
+						const writeItem0 = this.q[file].shift()
+						writeItem0.resolve(ret)
+					}
+				}
+				writeItem.resolve(ret)
+			} catch (e) {
+				while (this.q[file].length) {
+					const writeItem0 = this.q[file].shift()
+					writeItem0.reject(e)
+				}
+				writeItem.reject(e)
+			}
+			if (!this.q[file]?.length) {
+				delete this.q[file]
+			}
+		})
+		try {
+			await Promise.all(promises)
+		} finally {
+			this.processingWriteQueue = false
+		}
+	},
+	
+	startWriteQueueInterval: function () {
+		this.processWriteQueue()
+		if (!this._writeQueueInterval) {
+			this._writeQueueInterval = setInterval(() => this.processWriteQueue(), this.options.writeQueueIntervalMs || 1000)
+			this._writeQueueInterval.unref && this._writeQueueInterval.unref();
+		}
+	},
+
+	stopWriteQueueInterval: function () {
+		clearInterval(this._writeQueueInterval);
+	},
+
+	writeFile: async function (file, content) {
+		return new Promise((resolve, reject) => {
+			fs.writeFile(file, this.stringify(content), this.options.encoding, async (err) => {
 				if (err) {
@@ -406,3 +475,2 @@ return reject(err);
 			ttl = this.options.ttl;
-		} else {
 		}
@@ -409,0 +477,0 @@

tests/index.js

@@ -91,3 +91,5 @@
 			dir: randDir(),
-			// logging: true
+			// logging: true,
+			writeQueue: true,
+			writeQueueWriteOnlyLast: true
 		};
@@ -102,4 +104,11 @@ let storage = nodePersist.create();
 		};
+		let itemKeys = Object.keys(items);
 
-		let KEYS = Object.keys(items);
+		const generatedItemsLength = 100;
+		const generatedItemsParallel = 10;
+		let generatedItems = {};
+		for (let i = 0; i < generatedItemsLength; i++) {
+			generatedItems['generated' + i] = i			
+		}
+		let generatedItemsKeys = Object.keys(generatedItems);
 
@@ -117,2 +126,39 @@ describe('general items operations', function() {
 			});
+			
+			it(`should write ${generatedItemsLength * generatedItemsParallel} times, with writeQueueWriteOnlyLast=true,  in parallel setItem() then read them back`, async function() {				
+				let writePromises = [];
+				for (let i = 0; i < generatedItemsParallel; i++) {
+					writePromises = writePromises.concat(generatedItemsKeys.map(k => storage.setItem(k, i < generatedItemsParallel - 1 ? generatedItems[k] * i : generatedItems[k])))
+				}
+				await Promise.all(writePromises);
+				let readPromises = generatedItemsKeys.map(async (k) => { 
+					return assert.equal(await storage.getItem(k), generatedItems[k])
+				});
+				
+				await Promise.all(readPromises);
+			});
+			
+			it(`should write ${generatedItemsLength * generatedItemsParallel} times, with writeQueueWriteOnlyLast=false, in parallel setItem() then read them back`, async function() {
+				this.timeout(30000)
+				storage.setOptions({ 
+					...options, 
+					writeQueueWriteOnlyLast: false 
+				});
+				
+				let writePromises = [];
+				for (let i = 0; i < generatedItemsParallel; i++) {
+					writePromises = writePromises.concat(generatedItemsKeys.map(k => storage.setItem(k, i < generatedItemsParallel - 1 ? generatedItems[k] * i : generatedItems[k])))
+				}
+				await Promise.all(writePromises);
+				let readPromises = generatedItemsKeys.map(async (k) => { 
+					return assert.equal(await storage.getItem(k), generatedItems[k])
+				});
+				
+				await Promise.all(readPromises);
+				
+				storage.setOptions({ 
+					...options, 
+					writeQueueWriteOnlyLast: true 
+				});
+			});
 
@@ -153,3 +199,3 @@ it('should setItem() with ttl as a Date Object', async function() {
 				let value = await storage.valuesWithKeyMatch('item');
-				assert.equal(value.length, KEYS.length);
+				assert.equal(value.length, itemKeys.length);
 			});
@@ -159,3 +205,3 @@
 				let value = await storage.valuesWithKeyMatch(/item/);
-				assert.equal(value.length, KEYS.length);
+				assert.equal(value.length, itemKeys.length);
 			});
@@ -162,0 +208,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

36532

increased by14.31%

Lines of code

845

increased by14.04%

Number of lines in readme file

216

increased by9.09%

No dependency changes