async-flumelog - npm Package Compare versions

Comparing version 1.0.5 to 1.0.6

index.js

@@ -93,4 +93,3 @@ const Cache = require('hashlru')
       debug("getting offset %d from cache", offset)
-      // we use setImmediate to avoid the stack blowing up
-      setImmediate(() => cb(null, cachedBlock))
+      cb(null, cachedBlock)
     } else {
@@ -97,0 +96,0 @@ debug("getting offset %d from disc", offset)

package.json

 {
   "name": "async-flumelog",
   "description": "An async flumelog",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "homepage": "https://github.com/flumedb/async-flumelog",
@@ -14,2 +14,3 @@ "repository": {
     "lodash.debounce": "^4.0.8",
+    "looper": "^4.0.0",
     "ltgt": "^2.2.1",
@@ -16,0 +17,0 @@ "obz": "^1.0.2",

README.md

 # Async flumelog
 
 This module is heavily inspired by [flumelog-aligned-offset]. It is an
-attempt to write something simpler that is easier to reason
-about. Flumelog is the lowest part of the SSB stack, so it should
+attempt to implement the same concept but in a simpler fashion,
+making it easier to reason about the code.
+Flumelog is the lowest part of the SSB stack, so it should
 extremly stable while still maintaining good performance.
 
-An async flumelog consists of a number of blocks, that contain a
-number of records. The records are simply length + data. A record must
-be in one and only one block, which means there probably will be some
-empty space at the end of a block. Blocks are always written in full.
+An async flumelog consists of a number of `blocks`, that contain a
+number of `record`s. A `record` is simply it's `length`, as a 16-bit unsigned integer,
+followed by the `data` bytes. A record must be in one and only one block,
+which means there probably will be some empty space at the end of a block.
+Blocks are always written in full.
 
@@ -16,4 +18,4 @@ ```
   <record
-    <record.length: UInt16LE>
-    <record.data>
+    <length: UInt16LE>
+    <data: Bytes>
   </record>*
@@ -23,13 +25,13 @@ </block>*
 
-Contrasting to flumelog-aligned-offset there is no length after the
-data in a record and no pointer at the end of a block. These were to
-be able to run the log in reverse, but I have never seen the need for
+In contrast to flumelog-aligned-offset there is no additional `length` after the
+`data` in a `record` and no pointer at the end of a `block`. These were there to
+be able to iterate over the log in reverse, but I have never seen the need for
 that.
 
 Writing to the log is always async. Note this is different from
-[flumelog-offset] and [flumelog-aligned-offfset]. The since observable
-will be updated once the data is written. `onDrain` can be used to
+[flumelog-offset] and [flumelog-aligned-offfset]. The `since` observable
+will be updated once the data is written. The `onDrain` callback can be used to
 know when data has been written if needed. Streaming will only emit
 values that have been written to storage. This is to ensure that a
-view will never to ahead of the main log and thus end up in a bad
+view will never get ahead of the main log and thus end up in a bad
 state if the system crashes before data is written. `get` will return
@@ -41,6 +43,7 @@ values that have not been written to disk yet.
 Running [bench-flumelog] reveals the following numbers. Async flumelog
-is faster in all tests except random, where flumelog aligned offset is
-a lot faster. The `append` test is very slow for flumelog-aligned-offset
-because it writes every message synchronously. The most important numbers 
-are append (used for onboarding) and stream (used for building indexes).
+is faster that regular flumelog-offset in all categories. The most
+important numbers are append (used for onboarding) and stream (used
+for building indexes). Flumelog-aligned-offset is not included in the
+benchmarks, as it writes every message synchronously rendering the
+results invalid.
 
@@ -51,17 +54,8 @@ ```
 name, ops/second, mb/second, ops, total-mb, seconds
-append, 672175.964, 100.417, 3361552, 502.188, 5.001
-stream, 1276215.641, 190.656, 3361552, 502.188, 2.634
-stream no cache, 1355464.516, 202.495, 3361552, 502.188, 2.48
-stream10, 2046797.202, 305.777, 10244220, 1530.418, 5.005
-random, 21936.533, 3.277, 110604, 16.522, 5.042
+append, 923964.807, 138.002, 4620748, 690.149, 5.001
+stream, 1059075.865, 158.182, 4620748, 690.149, 4.363
+stream no cache, 1102803.818, 164.713, 4620748, 690.149, 4.19
+stream10, 2540947.641, 379.51, 12714902, 1899.068, 5.004
+random, 39715.656, 5.931, 198618, 29.664, 5.001
 
-flumelog aligned offset:
-
-name, ops/second, mb/second, ops, total-mb, seconds
-append, 769.2, 0.114, 3856, 0.576, 5.013
-stream, 128533.333, 19.207, 3856, 0.576, 0.03
-stream no cache, 124387.096, 18.587, 3856, 0.576, 0.031
-stream10, 428444.444, 64.023, 38560, 5.762, 0.09
-random, 907996.6, 135.684, 4540891, 678.556, 5.001
-
 flumelog offset:
@@ -80,4 +74,8 @@
 
+[JITDB] results for more real world benchmarks are available as [jitdb-results].
+
 [flumelog-aligned-offset]: https://github.com/flumedb/flumelog-aligned-offset/
 [flumelog-offset]: https://github.com/flumedb/flumelog-offset/
 [bench-flumelog]: https://github.com/flumedb/bench-flumelog
+[JITDB]: https://github.com/arj03/jitdb/
+[jitdb-results]: https://github.com/arj03/jitdb/blob/master/bench.txt

stream.js

 var ltgt = require('ltgt')
+var looper = require('looper')
+
 module.exports = Stream
@@ -108,3 +110,3 @@
 
-Stream.prototype.resume = function () {
+Stream.prototype._resume = function () {
   if (!this.sink || this.sink.paused) return
@@ -130,3 +132,3 @@
       this.cursor = this.blocks.getNextBlockIndex(this.cursor)
-      this.resume()
+      this._next()
     }
@@ -138,2 +140,7 @@ else if (this.live !== true)
 
+Stream.prototype.resume = function () {
+  this._next = looper(this._resume.bind(this))
+  this._next()
+}
+
 Stream.prototype.abort = function (err) {
@@ -140,0 +147,0 @@ //only thing to do is unsubscribe from live stream.

No alert changes

Improved metrics

Total package byte prevSize

37482

increased by0.25%

Lines of code

1048

increased by0.38%

Worsened metrics

Dependency count

9

increased by12.5%

Number of lines in readme file

77

decreased by-2.53%

Dependency changes

• + Addedlooper@^4.0.0