aligned-block-file - npm Package Compare versions

Comparing version 1.1.5 to 1.2.0

bench/append.js

@@ -12,3 +12,3 @@
   require('./')(ABF, {
-    data: new Buffer(1024*16),
+    data: new Buffer.alloc(1024*16),
     time:10e3, size: 100e6,
@@ -15,0 +15,0 @@ onUpdate: log

bench/readUInt64BE.js

@@ -6,3 +6,3 @@ var looper = require('looper')
     if(i == n) return cb(null, blocks)
-    blocks.append(Buffer.alloc(10).fill(0), function (err) {
+    blocks.append(Buffer.alloc(10), function (err) {
       if(err) cb(err)
@@ -9,0 +9,0 @@ else next(i + 1)

blocks.js

@@ -63,6 +63,7 @@ var fs = require('fs')
 
-        if(start < end) next(i+1)
-        else {
+        if (start < end) {
+          next(i+1)
+        } else {
           var buffer = bufs.length == 1 ? bufs[0] : Buffer.concat(bufs)
-          if(!buffer.length)
+          if (!buffer.length)
             return cb(new Error('read an empty buffer at:'+start + ' to ' + end + '\n'+
@@ -150,4 +151,3 @@ JSON.stringify({
             if(null == b) {
-              b = new Buffer(block_size)
-              b.fill(0)
+              b = Buffer.alloc(block_size)
               cache.set(i, b)
@@ -179,2 +179,18 @@ }
     },
+    /**
+     * Writes a buffer directly to a position in the file.
+     * This wraps `file.write()` and removes the block cache after the file
+     * write finishes to avoid having the item re-cached during the write.
+     *
+     * @param {buffer} buf - the data to write to the file
+     * @param {number} pos - position in the file to write the buffer
+     * @param {function} cb - callback that returns any error as an argument
+     */
+    write: (buf, pos, cb) => {
+      const i = Math.floor(pos/block_size)
+      file.write(buf, pos, (err) => {
+        cache.remove(i)
+        cb(err)
+      })
+    },
     //we arn't specifically clearing the buffers,
@@ -181,0 +197,0 @@ //but they should get updated anyway.

file.js

@@ -7,11 +7,27 @@ var fs = require('fs')
 module.exports = function (file, block_size, flags) {
-  var self
-  var fd = Obv()
+  flags = flags || 'r+'
+  var fd
   var offset = Obv()
-  //fs.openSync(file, flags || 'r+')
+  var writing = false
+  var waitingForWrite = []
+
+  function readyToWrite () {
+    if(!writing) throw new Error('should be writing')
+    writing = false
+    while(waitingForWrite.length)
+      waitingForWrite.shift()()
+  }
+
   mkdirp(path.dirname(file), function () {
-    fs.open(file, flags || 'r+', function (err, _fd) {
-      fd.set(_fd || err)
-      fs.stat(file, function (err, stat) {
-        offset.set(err ? 0 : stat.size)
+    //r+ opens the file for reading and writing, but errors if file does not exist.
+    //to open the file for reading and writing and not error if it does not exist.
+    //we need to open and close the file for append first.
+    fs.open(file, 'a', function (_, _fd) {
+      fs.close(_fd, function (_) {
+        fs.open(file, flags, function (err, _fd) {
+          fd = _fd
+          fs.stat(file, function (err, stat) {
+            offset.set(err ? 0 : stat.size)
+          })
+        })
       })
@@ -21,30 +37,34 @@ })
 
-  var writing = 0
+  // This variable *only* tracks appends, not positional writes.
+  var appending = 0
 
-  return self = {
+  return {
     get: function (i, cb) {
       offset.once(function (_offset) {
-        var max = ~~(_offset / block_size)
-        if(i > max)
-          return cb(new Error('aligned-block-file/file.get: requested block index was greater than max, got:'+i+', expected less than or equal to:'+max))
+        function onReady () {
+          var max = ~~(_offset / block_size)
+          if(i > max)
+            return cb(new Error('aligned-block-file/file.get: requested block index was greater than max, got:'+i+', expected less than or equal to:'+max))
 
-        var buf = new Buffer(block_size)
-        buf.fill(0) //security
+          var buf = Buffer.alloc(block_size)
 
-        fs.read(fd.value, buf, 0, block_size, i*block_size, function (err, bytes_read) {
-          if(err) cb(err)
-          else if(
-            //if bytes_read is wrong
-            i < max &&
-            buf.length !== bytes_read &&
-            //unless this is the very last block and it is incomplete.
-            !((i*block_size + bytes_read) == offset.value)
-          )
-            cb(new Error(
-              'aligned-block-file/file.get: did not read whole block, expected length:'+
-              block_size+' but got:'+bytes_read
-            ))
-          else
-            cb(null, buf, bytes_read)
-        })
+          fs.read(fd, buf, 0, block_size, i*block_size, function (err, bytes_read) {
+            if(err) cb(err)
+            else if(
+              //if bytes_read is wrong
+              i < max &&
+              buf.length !== bytes_read &&
+              //unless this is the very last block and it is incomplete.
+              !((i*block_size + bytes_read) == offset.value)
+            )
+              cb(new Error(
+                'aligned-block-file/file.get: did not read whole block, expected length:'+
+                block_size+' but got:'+bytes_read
+              ))
+            else
+              cb(null, buf, bytes_read)
+          })
+        }
+        if(!writing) onReady()
+        else waitingForWrite.push(onReady)
       })
@@ -55,9 +75,6 @@ },
     append: function (buf, cb) {
-      if(writing++) throw new Error('already writing to this file')
-      fd.once(function (_fd) {
-        if('object' === typeof _fd)
-          return cb(_fd)
+      if(appending++) throw new Error('already appending to this file')
         offset.once(function (_offset) {
-          fs.write(_fd, buf, 0, buf.length, _offset, function (err, written) {
-            writing = 0
+          fs.write(fd, buf, 0, buf.length, _offset, function (err, written) {
+            appending = 0
             if(err) return cb(err)
@@ -69,18 +86,48 @@ if(written !== buf.length) return cb(new Error('wrote less bytes than expected:'+written+', but wanted:'+buf.length))
         })
+    },
+    /**
+     * Writes a buffer directly to a position in the file. This opens the file
+     * with another file descriptor so that the main file descriptor can just
+     * append and read without doing any positional writes.
+     *
+     * @param {buffer} buf - the data to write to the file
+     * @param {number} pos - position in the file to write the buffer
+     * @param {function} cb - callback that returns any error as an argument
+     */
+    write: (buf, pos, cb) => {
+      if(flags !== 'r+') throw new Error('file opened with flags:'+flags+' refusing to write unless flags are:r+')
+      offset.once((_offset) => {
+        const endPos = pos + buf.length
+        const isPastOffset = endPos > _offset
+
+        if (isPastOffset) {
+          return cb(new Error(`cannot write past offset: ${endPos} > ${_offset}`))
+        }
+
+        function onReady (_writing) {
+          writing = true
+          fs.write(fd, buf, 0, buf.length, pos, (err, written) => {
+            readyToWrite()
+            if (err == null && written !== buf.length) {
+              cb(new Error('wrote less bytes than expected:'+written+', but wanted:'+buf.length))
+            } else {
+              cb(err)
+            }
+          })
+        }
+
+        if(!writing) onReady()
+        else waitingForWrite.push(onReady)
       })
     },
     truncate: function (len, cb) {
-      if(writing++) throw new Error('already writing, cannot truncate')
-      fd.once(function (_fd) {
-        if('object' === typeof _fd)
-          return cb(_fd)
-        offset.once(function (_offset) {
-          if(_offset <= len) return cb()
-          fs.ftruncate(_fd, len, function (err) {
-            if(err) cb(err)
-            else {
-              offset.set(len)
-              cb(null, offset.value)
-            }
-          })
+      if(appending) throw new Error('already appending, cannot truncate')
+      offset.once(function (_offset) {
+        if(_offset <= len) return cb()
+        fs.ftruncate(fd, len, function (err) {
+          if(err) cb(err)
+          else {
+            offset.set(len)
+            cb(null, offset.value)
+          }
         })
@@ -92,1 +139,2 @@ })
 
+

package.json

 {
   "name": "aligned-block-file",
   "description": "",
-  "version": "1.1.5",
+  "version": "1.2.0",
   "homepage": "https://github.com/flumedb/aligned-block-file",
@@ -12,10 +12,10 @@ "repository": {
     "hashlru": "^2.1.0",
-    "int53": "^0.2.4",
+    "int53": "^1.0.0",
     "mkdirp": "^0.5.1",
-    "obv": "0.0.1",
-    "uint48be": "^1.0.1"
+    "obv": "^0.0.1",
+    "uint48be": "^2.0.1"
   },
   "devDependencies": {
     "looper": "^4.0.0",
-    "tape": "^4.6.0"
+    "tape": "^4.10.1"
   },
@@ -22,0 +22,0 @@ "scripts": {

test/append.js

@@ -19,17 +19,9 @@
 
-  var a = new Buffer(32)
-  a.fill('a')
+  var a = Buffer.alloc(32, 'a')
+  var b = Buffer.alloc(32, 'b')
+  var c = Buffer.alloc(32, 'c')
+  var d = Buffer.alloc(32, 'd')
+  var e = Buffer.alloc(24, 'e')
+  var f = Buffer.alloc(64, 'f')
 
-  var b = new Buffer(32)
-  b.fill('b')
-
-  var c = new Buffer(32)
-  c.fill('c')
-  var d = new Buffer(32)
-  d.fill('d')
-  var e = new Buffer(24)
-  e.fill('e')
-  var f = new Buffer(64)
-  f.fill('f')
-
   tape('append one block', function (t) {
@@ -36,0 +28,0 @@ blocks.append(a, function (err, offset) {

test/simple.js

@@ -8,8 +8,5 @@ var fs = require('fs')
 
-var a = new Buffer(32)
-a.fill('a')
-var b = new Buffer(32)
-b.fill('b')
-var c = new Buffer(32)
-c.fill('c')
+var a = Buffer.alloc(32, 'a')
+var b = Buffer.alloc(32, 'b')
+var c = Buffer.alloc(32, 'c')
 
@@ -98,3 +95,36 @@ function Cache () {
 
+tape('overwrite previous data', function (t) {
+  var file = '/tmp/test_block-reader_'+Date.now()
+  var bufs = Blocks(File(file, 32, 'r+'), 32)
+  bufs.append(a, function (err) {
+    t.error(err)
+    bufs.read(0, 32, function (err, bufA) {
+      t.error(err)
+      t.deepEqual(bufA, a)
+      bufs.write(b, 0, function (err) {
+        t.error(err)
+        bufs.read(0, 32, function (err, bufB) {
+          t.error(err)
+          t.deepEqual(bufB, b)
+          bufs.write(b, 1, function (err) {
+            t.ok(err, 'error if writing past last offset')
 
+            // let's make a race condition!
+            // first we'll start writing...
+            bufs.write(c, 0, function (err) {
+              t.error(err)
+            })
 
+            // and we'll start reading before it's done
+            bufs.read(0, 32, (err, bufC) => {
+              t.error(err)
+              t.deepEqual(bufC, c)
+              t.end()
+            })
+          })
+        })
+      })
+    })
+  })
+})
+

New alerts

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

28838

increased by13%

Lines of code

799

increased by11.28%

Worsened metrics

Number of medium supply chain risk alerts

2

increased by100%

Dependency changes

• + Addedint53@1.0.0(transitive)

• + Addeduint48be@2.0.1(transitive)

• - Removedint53@0.2.4(transitive)

• - Removeduint48be@1.0.2(transitive)

• Updatedint53@^1.0.0

• Updatedobv@^0.0.1

• Updateduint48be@^2.0.1