@andrewosh/hyperdrive - npm Package Compare versions

Comparing version 0.2.0 to 0.3.0

archive.js

@@ -33,3 +33,3 @@ var inherits = require('inherits')
   this.drive = drive
-  this.live = this.options.live = !!this.options.live
+  this.live = this.options.live = !key && (this.options.live !== false)
   this.metadata = drive.core.createFeed(key, this.options)
@@ -68,3 +68,3 @@ this.content = null
     if (err) return stream.destroy(err)
-    if (self.content.key) self.content.replicate({stream: stream})
+    if (self.content && self.content.key) self.content.replicate({stream: stream})
   })
@@ -268,4 +268,10 @@
       entry.blocks = self.content.blocks - start
-      if (self.options.storage) self.options.storage.closeAppend(done)
-      else done(null)
+      if (self.options.storage) {
+        self.options.storage.end(bytesOffset, entry, function (err) {
+          if (err) return cb(err)
+          self.options.storage.closeAppend(done)
+        })
+      } else {
+        done(null)
+      }
 
@@ -293,12 +299,19 @@ function done (err) {
   var cur = null
-  var start = 0
-  var end = 0
+  var destroyed = false
 
-  return from(read)
+  var stream = from(read)
 
+  stream.on('end', cleanup)
+  stream.on('close', cleanup)
+
+  return stream
+
+  function cleanup () {
+    destroyed = true
+    if (cur) cur.destroy()
+  }
+
   function read (size, cb) {
     if (!opened) return open(size, cb)
-    if (cur) return cur.next(cb)
-    if (start >= end) return cb(null, null)
-    self.content.get(start++, cb)
+    cur.next(cb)
   }
@@ -310,5 +323,4 @@
       if (err) return cb(err)
-      start = startBlock
-      end = endBlock
-      if (opts.start) cur = self.createByteCursor(latest, opts)
+      if (destroyed) return
+      cur = self.createByteCursor(latest, opts)
       read(size, cb)
@@ -367,2 +379,14 @@ })
 
+    if (start === end && entry.type === 'file') {
+      var storage = self.options.storage
+      if (storage) {
+        storage.openAppend(entry.name, true)
+        storage.write(0, Buffer(0), function (err) {
+          if (err) return cb(err)
+          self.options.storage.closeAppend(cb)
+        })
+        return
+      }
+    }
+
     self.content.on('download', kick)
@@ -390,2 +414,7 @@ kick()
 
+  if (entry && entry.content && entry.blocks !== undefined) {
+    return process.nextTick(function () {
+      cb(null, entry.content.blockOffset, entry.content.blockOffset + entry.blocks, entry)
+    })
+  }
   this.get(entry, function (err, result) {
@@ -467,2 +496,4 @@ if (err) return cb(err)
 
+    self.emit('content')
+
     if (self.metadata.live && !index) self._writeIndex(opened)
@@ -469,0 +500,0 @@ else opened(null)

cursor.js

@@ -13,2 +13,4 @@ var thunky = require('thunky')
   this._block = 0
+  this._endBlock = 0
+  this._range = null
 
@@ -52,2 +54,15 @@ this.archive = archive
 
+Cursor.prototype.destroy = function (cb) {
+  if (!cb) cb = noop
+  var self = this
+  this.open(function (err) {
+    self._clear()
+    cb(err)
+  })
+}
+
+Cursor.prototype._clear = function () {
+  if (this._range) this.archive.content.unprioritize(this._range)
+}
+
 Cursor.prototype._seek = function (offset, cb) {
@@ -59,7 +74,10 @@ var self = this
   if (offset >= this.end) {
-    self.position = this.end
+    this._clear()
+    this.position = this.end
     return cb(null)
   }
 
-  this.archive.content.seek(offset, function (err, block, rel) {
+  var opts = {start: this.entry.content.blockOffset, end: this._endBlock}
+
+  this.archive.content.seek(offset, opts, function (err, block, rel) {
     if (err) return cb(err)
@@ -71,2 +89,5 @@
 
+    self._clear()
+    self._range = self.archive.content.prioritize({start: block, end: self._endBlock, priority: 4, linear: true})
+
     cb(null)
@@ -81,3 +102,6 @@ })
 
-  if (self.position >= self.end) return cb(null, null)
+  if (this.position >= this.end) {
+    this._clear()
+    return cb(null, null)
+  }
 
@@ -118,5 +142,9 @@ this.archive.content.get(block, function (err, data) {
     self._block = entry.content.blockOffset
+    self._endBlock = entry.content.blockOffset + entry.blocks
 
-    if (self.start !== entry.content.bytesOffset) self._seek(self.start, done)
-    else done(null)
+    if (self.start !== entry.content.bytesOffset) return self._seek(self.start, done)
+
+    self.position = self.start
+    self._range = self.archive.content.prioritize({start: self._block, end: self._endBlock, priority: 4, linear: true})
+    done(null)
   })
@@ -123,0 +151,0 @@

package.json

 {
   "name": "@andrewosh/hyperdrive",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "A file sharing network based on rabin file chunking and append only feeds of data verified by merkle trees.",
@@ -18,3 +18,3 @@ "main": "index.js",
     "from2": "^2.1.1",
-    "hypercore": "^4.2.4",
+    "hypercore": "^4.7.0",
     "inherits": "^2.0.1",
@@ -32,5 +32,6 @@ "protocol-buffers": "^3.1.6",
     "memdb": "^1.3.1",
-    "random-access-file": "^1.2.0",
+    "random-access-file": "^1.3.0",
     "standard": "^6.0.8",
-    "tape": "^4.4.0"
+    "tape": "^4.4.0",
+    "tmp": "0.0.28"
   },
@@ -37,0 +38,0 @@ "repository": {

README.md

@@ -11,3 +11,3 @@ # hyperdrive
 
-If you are interested in learning how hyperdrive works on a technical level a specification is available in the [Dat docs repo](https://github.com/datproject/docs/blob/master/hyperdrive.md)
+If you are interested in learning how hyperdrive works on a technical level a specification is available in the [Dat docs repo](https://github.com/datproject/docs/blob/master/docs/hyperdrive_spec.md)
 
@@ -33,13 +33,11 @@ ## Usage
 
-archive.finalize(function () { // finalize the archive
-  var link = archive.key.toString('hex')
-  console.log(link, '<-- this is your hyperdrive link')
+var link = archive.key.toString('hex')
+console.log(link, '<-- this is your hyperdrive link')
 
-  // the archive is now ready for sharing.
-  // we can use swarm to replicate it to other peers
-  swarm.listen()
-  swarm.join(new Buffer(link, 'hex'))
-  swarm.on('connection', function (connection) {
-    connection.pipe(archive.replicate()).pipe(connection)
-  })
+// the archive is now ready for sharing.
+// we can use swarm to replicate it to other peers
+swarm.listen()
+swarm.join(new Buffer(link, 'hex'))
+swarm.on('connection', function (connection) {
+  connection.pipe(archive.replicate()).pipe(connection)
 })
@@ -103,2 +101,3 @@ ```
   live: false, // set this to share the archive without finalizing it
+  sparse: false, // set this to only download the pieces of the feed you are requesting / prioritizing
   file: function (name) {
@@ -120,2 +119,6 @@ // set this to determine how file data is stored.
 
+#### `archive.live`
+
+Boolean whether archive is live. `true` by default. Note that its only populated after archive.open(cb) has been fired.
+
 #### `archive.append(entry, callback)`
@@ -144,3 +147,3 @@
 
-Finalize the archive. You need to do this before sharing it if the archive is not live.
+Finalize the archive. You need to do this before sharing it if the archive is not live (it is live per default).
 
@@ -147,0 +150,0 @@ #### `archive.get(index, callback)`

storage.js

@@ -48,2 +48,16 @@ module.exports = Storage
 
+Storage.prototype.end = function (offset, options, cb) {
+  var s = this._get(offset)
+  if (!s) {
+    var self = this
+    return this._open(offset, 0, null, function (err) {
+      if (err) return cb(err)
+      self.end(offset, options, cb)
+    })
+  }
+  if (this._readonly && s === this._appending) return cb(null)
+  if (s.storage.end) return s.storage.end(options, cb)
+  cb()
+}
+
 Storage.prototype._get = function (offset) {
@@ -50,0 +64,0 @@ for (var i = 0; i < this._opened.length; i++) {

test/misc.js

@@ -12,2 +12,3 @@ var fs = require('fs')
   var archive = drive.createArchive({
+    live: false,
     file: function (name) {
@@ -38,2 +39,3 @@ return raf(path.join(__dirname, name), {readable: true, writable: false})
   var archive = drive.createArchive({
+    live: false,
     file: function (name) {
@@ -157,1 +159,31 @@ return raf(path.join(__dirname, name), {readable: true, writable: false})
 })
+
+tape('live by default', function (t) {
+  var drive = hyperdrive(memdb())
+  var archive = drive.createArchive()
+
+  t.ok(archive.live, 'live')
+  t.end()
+})
+
+tape('mtime preserved', function (t) {
+  t.plan(2)
+  var txt = '/tmp/mtime.txt'
+  var mtime = new Date(1000 *
+    Math.round((Date.now() + 1000 * 60 * 60 * 10) / 1000))
+  var drive = hyperdrive(memdb())
+  var archive = drive.createArchive({
+    file: function () { return raf(txt) }
+  })
+  archive.createFileWriteStream({
+    name: 'mtime.txt',
+    mtime: mtime
+  })
+  .on('finish', function () {
+    fs.stat(txt, function (err, stat) {
+      t.error(err)
+      t.deepEqual(stat.mtime, mtime)
+    })
+  })
+  .end('hyper hyper')
+})

test/overwrite.js

@@ -9,3 +9,3 @@ var test = require('tape')
   var drive = hyperdrive(memdb())
-  var archive = drive.createArchive(undefined, { live: true })
+  var archive = drive.createArchive()
 
@@ -12,0 +12,0 @@ var w1 = archive.createFileWriteStream('hello.txt')

test/replicates.js

@@ -5,2 +5,3 @@ var tape = require('tape')
 var fs = require('fs')
+var tmp = require('tmp')
 var raf = require('random-access-file')
@@ -55,4 +56,3 @@ var hyperdrive = require('../')
 
-  var ws = archive.createFileWriteStream('empty.txt')
-  ws.end()
+  archive.createFileWriteStream('empty.txt').end()
 
@@ -116,1 +116,109 @@ archive.finalize(function (err) {
 })
+
+tape('downloads empty files to fs', function (t) {
+  var drive = hyperdrive(memdb())
+  var driveClone = hyperdrive(memdb())
+  var tmpdir = tmp.dirSync()
+
+  var archive = drive.createArchive()
+
+  archive.createFileWriteStream('empty.txt').end()
+
+  archive.finalize(function (err) {
+    t.error(err, 'no error')
+
+    var clone = driveClone.createArchive(archive.key, {
+      file: function (name) {
+        return raf(path.join(tmpdir.name, name))
+      }
+    })
+
+    clone.get(0, function (err, entry) {
+      t.error(err, 'no error')
+      t.same(entry.name, 'empty.txt')
+      t.same(entry.length, 0, 'empty')
+
+      clone.download(entry, function (err) {
+        var fileList = fs.readdirSync(tmpdir.name).join(' ')
+        t.ok(fileList.indexOf('empty.txt') > -1, 'has empty file')
+        t.error(err, 'no error')
+        t.end()
+      })
+    })
+
+    var stream = archive.replicate()
+    var streamClone = clone.replicate()
+
+    stream.pipe(streamClone).pipe(stream)
+  })
+})
+
+tape('replicates static archives', function (t) {
+  var drive = hyperdrive(memdb())
+  var driveClone = hyperdrive(memdb())
+
+  var archive = drive.createArchive({
+    live: false
+  })
+
+  archive.createFileWriteStream('empty.txt').end(function () {
+    archive.finalize(function () {
+      var clone = driveClone.createArchive(archive.key)
+
+      clone.download(0, function () {
+        t.pass('archive replicated')
+        t.end()
+      })
+
+      var stream = archive.replicate()
+      var streamClone = clone.replicate()
+
+      stream.pipe(streamClone).pipe(stream)
+    })
+  })
+})
+
+tape('downloads empty directories to fs', function (t) {
+  var drive = hyperdrive(memdb())
+  var driveClone = hyperdrive(memdb())
+  var tmpdir = tmp.dirSync()
+
+  var archive = drive.createArchive()
+
+  archive.append({
+    type: 'directory',
+    name: 'a-dir'
+  })
+
+  archive.finalize(function (err) {
+    t.error(err, 'no error')
+
+    var clone = driveClone.createArchive(archive.key, {
+      file: function (name) {
+        return raf(path.join(tmpdir.name, name))
+      }
+    })
+
+    clone.get(0, function (err, entry) {
+      t.error(err, 'no error')
+        // TODO: change these to t.same
+      t.same(entry.type, 'directory')
+      t.same(entry.name, 'a-dir')
+      t.same(entry.length, 0, 'empty')
+
+      clone.download(entry, function (err) {
+        var fileList = fs.readdirSync(tmpdir.name).join(' ')
+        // TODO: change this to t.ok
+        t.skip(fileList.indexOf('a-dir') > -1, 'has empty dir')
+        // TODO: test a-dir is actually a directory
+        t.error(err, 'no error')
+        t.end()
+      })
+    })
+
+    var stream = archive.replicate()
+    var streamClone = clone.replicate()
+
+    stream.pipe(streamClone).pipe(stream)
+  })
+})

test/rw.js

@@ -119,3 +119,3 @@ var test = require('tape')
   var arr = []
-  for (var i = 0; i < 10000 * 2; i++) {
+  for (var i = 0; i < 10000 * 4; i++) {
     arr.push(Math.floor(Math.random() * 100))
@@ -140,3 +140,2 @@ }
     archive.get('hello.txt', function (err, entry) {
-      console.log('entry.blocks', entry.blocks)
       t.error(err, 'no error')
@@ -211,1 +210,42 @@ verifyBlock(0, cursor, archive.content, function () {
 })
+
+test('read previous entries', function (t) {
+  t.plan(5)
+  var drive = hyperdrive(memdb())
+  var archive = drive.createArchive(null, { live: true })
+  var data = [
+    { 'hello.txt': 'HI' },
+    { 'hello.txt': 'WHAT', 'index.html': '<h1>hey</h1>' },
+    { 'hello.txt': '!' }
+  ]
+  var expected = [
+    { name: 'hello.txt', body: 'HI' },
+    { name: 'hello.txt', body: 'WHAT' },
+    { name: 'index.html', body: '<h1>hey</h1>' },
+    { name: 'hello.txt', body: '!' }
+  ]
+  ;(function next () {
+    if (data.length === 0) return check()
+    var files = data.shift()
+    var pending = 1
+    Object.keys(files).forEach(function (key) {
+      pending++
+      archive.createFileWriteStream(key)
+        .once('finish', done)
+        .end(files[key])
+    })
+    done()
+    function done () { if (--pending === 0) next() }
+  })()
+  function check () {
+    archive.list({ live: false }, function (err, files) {
+      t.error(err)
+      files.forEach(function (file) {
+        var e = expected.shift()
+        archive.createFileReadStream(file).pipe(concat(function (body) {
+          t.equal(body.toString(), e.body, e.name)
+        }))
+      })
+    })
+  }
+})

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

476892

increased by1.36%

Lines of code

1403

increased by18%

Number of lines in readme file

228

increased by1.33%

Worsened metrics

Dev dependency count

6

increased by20%

Dependency changes

• Updatedhypercore@^4.7.0