lmdb-store - npm Package Compare versions

Comparing version 0.3.0 to 0.3.1

index.js

@@ -10,5 +10,3 @@ const fs = require('fs-extra')
 const RANGE_BATCH_SIZE = 100
-const STARTING_ARRAY = [null]
-const VALUE_OVERFLOW_THRESHOLD = 2048
-const DEFAULT_SYNC_BATCH_THRESHOLD = 3000
+const DEFAULT_SYNC_BATCH_THRESHOLD = 20000000
 const AS_STRING = {
@@ -34,3 +32,2 @@ asBuffer: false
 let env
-const EXTENSION = '.mdpack'
 exports.open = open
@@ -79,4 +76,2 @@ function open(path, options) {
 			this.env = env
-			this.bytesRead = 0
-			this.bytesWritten = 0
 			this.reads = 0
@@ -147,4 +142,2 @@ this.writes = 0
 					result = undefined
-				else
-					this.bytesRead += result.length || 1
 				try {
@@ -167,4 +160,7 @@ if (copy)
 				scheduledOperations = []
+				scheduledOperations.bytes = 0
 			}
 			let index = scheduledOperations.push([this.db, id, value, ifValue]) - 1
+			// track the size of the scheduled operations (and include the approx size of the array structure too)
+			scheduledOperations.bytes += id.length + (value && value.length || 0) + (ifValue && ifValue.length || 0) + 200
 			let commit = this.scheduleCommit()
@@ -181,3 +177,2 @@ return ifValue === undefined ? commit.unconditionalResults :
 				}
-				this.bytesWritten += value && value.length || 0
 				this.writes++
@@ -225,4 +220,6 @@ let startCpu = process.cpuUsage()
 				scheduledOperations = []
+				scheduledOperations.bytes = 0
 			}
 			let index = scheduledOperations.push([this.db, id, undefined, ifValue]) - 1
+			scheduledOperations.bytes += id.length + (ifValue && ifValue.length || 0) + 200
 			let commit = this.scheduleCommit()
@@ -378,10 +375,10 @@ return ifValue === undefined ? commit.unconditionalResults :
 					results: whenCommitted,
-					unconditionalResults: whenCommitted.then(() => true) // for returning for non-conditional
+					unconditionalResults: whenCommitted.then(() => true) // for returning from non-conditional operations
 				}
 			}
-			if (scheduledOperations && scheduledOperations.length >= this.syncBatchThreshold && this.runNextBatch) {
+			if (scheduledOperations && scheduledOperations.bytes >= this.syncBatchThreshold && this.runNextBatch) {
 				// past a certain threshold, run it immediately and synchronously
 				let batch = this.pendingBatch
 				try {
-					let results = this.batchSync()
+					let results = this.commitBatchNow()
 					this.runNextBatch(results)
@@ -395,3 +392,3 @@ } catch (error) {
 		}
-		batchSync() {
+		commitBatchNow() {
 			let value
@@ -427,3 +424,3 @@ let results = new Array(scheduledOperations.length)
 			})
-			console.log('batchSync', scheduledOperations.length)
+			console.log('commitBatchNow', scheduledOperations.bytes, scheduledOperations.length)
 			scheduledOperations = null
@@ -434,19 +431,12 @@ return results
 			this.writes += operations.length
-			this.bytesWritten += operations.reduce((a, b) => a + (b.value && b.value.length || 0), 0)
+			if (!scheduledOperations) {
+				scheduledOperations = []
+				scheduledOperations.bytes = 0
+			}
 			for (let operation of operations) {
 				if (typeof operation.key != 'object')
 					throw new Error('non-buffer key')
-				try {
-					let value = operation.value
-					if (!scheduledOperations) {
-						scheduledOperations = []
-					}
-					scheduledOperations.push([this.db, operation.key, value])
-				} catch (error) {
-					if (error.message.startsWith('MDB_NOTFOUND')) {
-						// not an error
-					} else {
-						throw error
-					}
-				}
+				let value = operation.value
+				scheduledOperations.push([this.db, operation.key, value])
+				scheduledOperations.bytes += operation.key.length + (value && value.length || 0) + 200
 			}
@@ -453,0 +443,0 @@ return this.scheduleCommit().unconditionalResults

package.json

 {
   "name": "lmdb-store",
   "author": "Kris Zyp",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Simple, effiecent, scalable data store wrapper for LDMB",
@@ -6,0 +6,0 @@ "license": "MIT",

util/ArrayLikeIterable.js

@@ -19,10 +19,27 @@ const when = require('./when')
 			return {
-				next() {
+				next(resolvedResult) {
 					let result
 					do {
-						result = iterator.next()
-						if (result.done === true) {
-							return result
+						let iteratorResult
+						if (resolvedResult) {
+							iteratorResult = resolvedResult
+							resolvedResult = null // don't go in this branch on next iteration
+						} else {
+							iteratorResult = iterator.next()
+							if (iteratorResult.then) {
+								return iteratorResult.then(iteratorResult => this.next(iteratorResult))
+							}
 						}
-						result = func(result.value)
+						if (iteratorResult.done === true) {
+							return iteratorResult
+						}
+						result = func(iteratorResult.value)
+						if (result && result.then) {
+							return result.then(result =>
+								result == SKIP ?
+									this.next() :
+									{
+										value: result
+									})
+						}
 					} while(result == SKIP)
@@ -49,12 +66,5 @@ return {
 	}
-	toJSON() {
-		if (this._asArray && this._asArray.forEach) {
-			return this._asArray
-		}
-		throw new Error('Can not serialize async iteratables without first calling resolveJSON')
-		//return Array.from(this)
-	}
+
 	forEach(callback) {
 		let iterator = this[Symbol.iterator]()
-		let array = []
 		let result
@@ -65,8 +75,30 @@ while ((result = iterator.next()).done !== true) {
 	}
+	toJSON() {
+		if (this.asArray && this.asArray.forEach) {
+			return this.asArray
+		}
+		throw new Error('Can not serialize async iteratables without first calling resolveJSON')
+		//return Array.from(this)
+	}
 	get asArray() {
 		if (this._asArray)
 			return this._asArray
-		let array = []
-		this.forEach((value) => array.push(value))
-		return this._asArray = array
+		let promise = new Promise((resolve, reject) => {
+			let iterator = this[Symbol.iterator](true)
+			let array = []
+			let iterable = this
+			function next(result) {
+				while (result.done !== true) {
+					if (result.then) {
+						return result.then(next)
+					} else {
+						array.push(result.value)
+					}
+					result = iterator.next()
+				}
+				resolve(iterable._asArray = array)
+			}
+			next(iterator.next())
+		})
+		return this._asArray || (this._asArray = promise)
 	}
@@ -77,2 +109,2 @@ resolveData() {
 }
-exports.ArrayLikeIterable = ArrayLikeIterable
+exports.ArrayLikeIterable = ArrayLikeIterable

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

101930

increased by0.96%

Lines of code

1071

increased by2.1%

No dependency changes