Socket
Socket
Sign inDemoInstall

hypertrie

Package Overview
Dependencies
Maintainers
1
Versions
45
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

hypertrie - npm Package Compare versions

Comparing version 1.0.0 to 2.0.0

5

example.js
const hypertrie = require('./')
const hypercore = require('hypercore')
const ram = require('random-access-memory')
const db = hypertrie(
hypercore(ram)
)
const db = hypertrie(ram)

@@ -9,0 +6,0 @@ const batch = new Array(200)

3

index.js

@@ -37,2 +37,3 @@ const Node = require('./lib/node')

this.discoveryKey = null
this.metadata = opts.metadata || null
this.feed = opts.feed || hypercore(storage, key, {sparse: opts.sparse})

@@ -74,3 +75,3 @@ this.opened = false

if (self.feed.length || !self.feed.writable) return done(null)
self.feed.append(Header.encode({protocol: 'hypertrie'}), done)
self.feed.append(Header.encode({type: 'hypertrie', metadata: self.metadata}), done)

@@ -77,0 +78,0 @@ function done (err) {

const Nanoiterator = require('nanoiterator')
const inherits = require('inherits')
const Node = require('./node')

@@ -16,3 +17,6 @@ module.exports = Diff

this._callback = null
this._left = []
this._right = []
this._onnode = (opts && opts.onnode) || null
this._needsCheck = []
}

@@ -35,34 +39,17 @@

Diff.prototype._get = function (seq, top, left) {
const self = this
this._pending++
this._db.getBySeq(seq, function (err, node) {
if (self._onnode && node) self._onnode(node)
if (node) set(top, left, node)
else if (err) self._error = err
if (!--self._pending) self._finalize()
})
}
Diff.prototype._finalize = function () {
const callback = this._callback
if (!callback) return
const err = this._error
this._callback = this._error = null
if (err) return callback(err)
this._next(callback)
}
Diff.prototype._push = function (seq, top, node, left) {
if (node && seq === node.seq) {
set(top, left, node)
return true
while (this._needsCheck.length) {
const end = this._needsCheck.pop()
const start = this._needsCheck.pop()
this._maybeCollides(start, end)
}
if (seq) {
this._get(seq, top, left)
return true
}
return false
this._next(callback)
}

@@ -86,4 +73,2 @@

// TODO: collisions, yada yada
if (doneLeft && doneRight) return call(cb, left, right)

@@ -97,12 +82,42 @@

for (var j = 0; j < 5; j++) {
const leftSeq = leftVal === j ? left.seq : (leftBucket[j] || 0)
const rightSeq = rightVal === j ? right.seq : (rightBucket[j] || 0)
const leftSeq = leftVal === j ? left.seq : 0
const rightSeq = rightVal === j ? right.seq : 0
const len = this._stack.length
var leftLen = this._stack.length
var rightLen = this._stack.length
var val
if (leftSeq === rightSeq) continue
if (leftSeq !== rightSeq) {
if (!doneLeft && leftSeq && notInBucket(j, leftSeq, rightBucket)) {
set(this._pushStack(leftLen++, i + 1), true, left)
}
if (!doneRight && rightSeq && notInBucket(j, rightSeq, leftBucket)) {
set(this._pushStack(rightLen++, i + 1), false, right)
}
}
const top = {i: i + 1, left: null, right: null}
const pushLeft = !doneLeft && this._push(leftSeq, top, left, true)
const pushRight = !doneRight && this._push(rightSeq, top, right, false)
if (!doneLeft) {
for (val = j; val < leftBucket.length; val += 5) {
const seq = leftBucket[val]
if (!seq) break
if (seq !== rightSeq && notInBucket(j, seq, rightBucket)) {
this._getNode(seq, this._pushStack(leftLen++, i + 1), true)
}
}
}
if (pushLeft || pushRight) this._stack.push(top)
if (!doneRight) {
for (val = j; val < rightBucket.length; val += 5) {
const seq = rightBucket[val]
if (!seq) break
if (seq !== leftSeq && notInBucket(j, seq, leftBucket)) {
this._getNode(seq, this._pushStack(rightLen++, i + 1), false)
}
}
}
if (Node.terminator(i) && this._stack.length > len) {
if (!this._pending) this._maybeCollides(len, this._stack.length)
else this._needsCheck.push(len, this._stack.length)
}
}

@@ -121,2 +136,65 @@

Diff.prototype._maybeCollides = function (start, end) {
// all nodes, start -> end, share the same hash
// we need to check that there are no collisions
// much simpler and *much* more likely - only one node
if (end - start === 1) {
const top = this._stack[start]
if (collides(top)) {
this._stack.push({i: top.i, left: null, right: top.right})
top.right = null
}
return
}
// very unlikely, but multiple collisions or a trie reordering
// due to a collision being deleted
for (var i = start; i < end; i++) {
const top = this._stack[i]
if (collides(top) || !top.left) {
const right = top.right
for (var j = start; j < end; j++) {
const other = this._stack[j]
if (other.left && !other.left.collides(right)) {
top.right = other.right
other.right = right
i-- // revisit top again, as it might still collide
break
}
}
if (top.right === right && top.left) {
this._stack.push({i: top.i, left: null, right})
top.right = null
}
}
}
}
Diff.prototype._pushStack = function (len, i) {
if (this._stack.length === len) this._stack.push({i, left: null, right: null})
return this._stack[len]
}
Diff.prototype._getNode = function (seq, top, left) {
const self = this
this._pending++
this._db.getBySeq(seq, onnode)
function onnode (err, node) {
if (self._onnode && node) self._onnode(node)
if (node) set(top, left, node)
else if (err) self._error = err
if (!--self._pending) self._finalize()
}
}
function notInBucket (val, seq, bucket) {
for (; val < bucket.length; val += 5) {
if (bucket[val] === seq) return false
}
return true
}
function set (top, left, node) {

@@ -146,1 +224,6 @@ if (left) top.left = node

}
function collides (top) {
if (!top.left || !top.right) return false
return top.left.collides(top.right, top.i)
}

@@ -26,22 +26,2 @@ const Node = require('./node')

Get.prototype._collision = function (head) {
const self = this
const bucket = head.trie[this._length - 1] || []
var error = null
var node = null
var missing = bucket.length - 4 + 1
for (var i = 4; i < bucket.length; i++) {
this._db.getBySeq(bucket[i], onnode)
}
onnode(null, null)
function onnode (err, n) {
if (err) error = err
else if (n && n.key === self._node.key) node = n
if (!--missing) self._callback(error, error ? null : node)
}
}
Get.prototype._update = function (i, head) {

@@ -55,16 +35,44 @@ if (!head) return this._callback(null, null)

const val = node.path(i)
if (head.path(i) === val) continue
const checkCollision = Node.terminator(i)
if (head.path(i) === val) {
if (!checkCollision || !node.collides(head, i)) continue
}
const bucket = head.trie[i] || []
if (checkCollision) return this._updateHeadCollides(i, bucket, val)
const seq = bucket[val]
if (!seq) return this._callback(null, null)
if (seq) this._updateHead(i, seq)
else this._callback(null, null)
return
return this._updateHead(i, seq)
}
if (this._prefix || head.key === node.key) return this._callback(null, head)
this._callback(null, head)
}
// hash matches but key doesn't - we have a collision
this._collision(head)
Get.prototype._updateHeadCollides = function (i, bucket, val) {
const self = this
var missing = 1
var error = null
var node = null
for (var j = val; j < bucket.length; j += 5) {
const seq = bucket[j]
if (!seq) break
missing++
this._db.getBySeq(seq, onnode)
}
onnode(null, null)
function onnode (err, n) {
if (err) error = err
else if (n && !n.collides(self._node, i)) node = n
if (--missing) return
if (!node || error) return self._callback(error, null)
self._update(i + 1, node)
}
}

@@ -71,0 +79,0 @@

@@ -5,2 +5,5 @@ const Nanoiterator = require('nanoiterator')

const SORT_ORDER = [4, 0, 1, 2, 3].reverse()
const REVERSE_SORT_ORDER = SORT_ORDER.slice(0).reverse()
module.exports = Iterator

@@ -13,2 +16,3 @@

this._recursive = !opts || opts.recursive !== false
this._order = (opts && opts.reverse) ? REVERSE_SORT_ORDER : SORT_ORDER
this._start = 0

@@ -22,2 +26,3 @@ this._end = 0

this._gt = !!(opts && opts.gt)
this._needsSort = []
}

@@ -49,17 +54,21 @@

if (i >= len) {
if (this._prefix && !isPrefix(top.node.key, this._prefix)) continue
return cb(null, top.node)
}
if (i >= len) return cb(null, top.node)
const bucket = top.node.trie[i] || []
// 3, 2, 1, 0, 4
for (j = 3; j >= 0; j--) this._push(i + 1, bucket[j] || 0)
if (!this._gt || i !== this._start) {
// TODO: consistent order across dbs when there are collisions
for (j = 4; j < bucket.length; j++) this._push(i + 1, bucket[j] || 0)
for (j = 0; j < this._order.length; j++) {
var val = this._order[j]
if (val !== 4 || !this._gt || i !== this._start) {
const len = this._stack.length
if (top.node.path(i) === val) this._stack.push(top)
for (; val < bucket.length; val += 5) {
const seq = bucket[val]
if (seq) this._push(i + 1, seq)
}
if (this._stack.length - len > 1) {
this._needsSort.push(len, this._stack.length)
}
}
}
this._stack.push(top)
if (!this._pending) continue

@@ -74,14 +83,26 @@ this._callback = cb

Iterator.prototype._push = function (i, seq) {
if (!seq) return
const self = this
const top = {i, node: null}
this._pending++
this._db.getBySeq(seq, function (err, node) {
if (node) self._stack.push({i, node})
this._stack.push(top)
this._db.getBySeq(seq, onnode)
function onnode (err, node) {
if (node) top.node = node
else if (err) self._error = err
if (!--self._pending) self._continue()
})
}
}
Iterator.prototype._sort = function () {
// only ran when there are potential collisions to make sure
// the iterator sorts consistently
while (this._needsSort.length) {
const end = this._needsSort.pop()
const start = this._needsSort.pop()
sort(this._stack, start, end)
}
}
Iterator.prototype._continue = function () {

@@ -92,9 +113,17 @@ const callback = this._callback

if (err) return callback(err)
if (this._needsSort.length) this._sort()
this._next(callback)
}
function isPrefix (key, prefix) {
if (!key.startsWith(prefix)) return false
if (prefix.length === key.length) return true
return key[prefix.length] === '/'
function sort (list, from, to) {
// only ran on short lists so the simple o(n^2) algo is fine
for (var i = from + 1; i < to; i++) {
for (var j = i; j > from; j--) {
const a = list[j]
const b = list[j - 1]
if (a.node.key <= b.node.key) break
list[j] = b
list[j - 1] = a
}
}
}

@@ -20,2 +20,9 @@ // This file is auto generated by the protocol-buffers cli tool

var Metadata = exports.Metadata = {
buffer: true,
encodingLength: null,
encode: null,
decode: null
}
var Node = exports.Node = {

@@ -29,2 +36,3 @@ buffer: true,

defineHeader()
defineMetadata()
defineNode()

@@ -34,3 +42,4 @@

var enc = [
encodings.string
encodings.string,
Metadata
]

@@ -44,5 +53,10 @@

var length = 0
if (!defined(obj.protocol)) throw new Error("protocol is required")
var len = enc[0].encodingLength(obj.protocol)
if (!defined(obj.type)) throw new Error("type is required")
var len = enc[0].encodingLength(obj.type)
length += 1 + len
if (defined(obj.metadata)) {
var len = enc[1].encodingLength(obj.metadata)
length += varint.encodingLength(len)
length += 1 + len
}
return length

@@ -55,6 +69,13 @@ }

var oldOffset = offset
if (!defined(obj.protocol)) throw new Error("protocol is required")
if (!defined(obj.type)) throw new Error("type is required")
buf[offset++] = 10
enc[0].encode(obj.protocol, buf, offset)
enc[0].encode(obj.type, buf, offset)
offset += enc[0].encode.bytes
if (defined(obj.metadata)) {
buf[offset++] = 18
varint.encode(enc[1].encodingLength(obj.metadata), buf, offset)
offset += varint.encode.bytes
enc[1].encode(obj.metadata, buf, offset)
offset += enc[1].encode.bytes
}
encode.bytes = offset - oldOffset

@@ -70,3 +91,4 @@ return buf

var obj = {
protocol: ""
type: "",
metadata: null
}

@@ -85,6 +107,12 @@ var found0 = false

case 1:
obj.protocol = enc[0].decode(buf, offset)
obj.type = enc[0].decode(buf, offset)
offset += enc[0].decode.bytes
found0 = true
break
case 2:
var len = varint.decode(buf, offset)
offset += varint.decode.bytes
obj.metadata = enc[1].decode(buf, offset, offset + len)
offset += enc[1].decode.bytes
break
default:

@@ -97,2 +125,61 @@ offset = skip(prefix & 7, buf, offset)

function defineMetadata () {
var enc = [
encodings.bytes
]
Metadata.encodingLength = encodingLength
Metadata.encode = encode
Metadata.decode = decode
function encodingLength (obj) {
var length = 0
if (defined(obj.contentFeed)) {
var len = enc[0].encodingLength(obj.contentFeed)
length += 1 + len
}
return length
}
function encode (obj, buf, offset) {
if (!offset) offset = 0
if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj))
var oldOffset = offset
if (defined(obj.contentFeed)) {
buf[offset++] = 10
enc[0].encode(obj.contentFeed, buf, offset)
offset += enc[0].encode.bytes
}
encode.bytes = offset - oldOffset
return buf
}
function decode (buf, offset, end) {
if (!offset) offset = 0
if (!end) end = buf.length
if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid")
var oldOffset = offset
var obj = {
contentFeed: null
}
while (true) {
if (end <= offset) {
decode.bytes = offset - oldOffset
return obj
}
var prefix = varint.decode(buf, offset)
offset += varint.decode.bytes
var tag = prefix >> 3
switch (tag) {
case 1:
obj.contentFeed = enc[0].decode(buf, offset)
offset += enc[0].decode.bytes
break
default:
offset = skip(prefix & 7, buf, offset)
}
}
}
}
function defineNode () {

@@ -99,0 +186,0 @@ var enc = [

const sodium = require('sodium-universal')
const inspect = require('inspect-custom-symbol')
const messages = require('./messages')

@@ -13,3 +14,4 @@ const trie = require('./trie')

this.value = data.value !== undefined ? data.value : ((data.valueBuffer && enc) ? enc.decode(data.valueBuffer) : data.valueBuffer)
this.hash = hash(this.key)
this.keySplit = split(this.key)
this.hash = hash(this.keySplit)
this.trie = data.trieBuffer ? trie.decode(data.trieBuffer) : (data.trie || [])

@@ -22,6 +24,11 @@ this.trieBuffer = null

Node.prototype[inspect] = function (depth, opts) {
return opts.stylize({seq: this.seq, key: this.key, value: this.value}, 'object')
}
Node.prototype.path = function (i) {
const hash = this.hash
if (i >= hash.length * 4) return 4
return (hash[i >> 2] >> (2 * (i & 3))) & 3
const j = i >> 2
if (j >= hash.length) return 4
return (hash[j] >> (2 * (i & 3))) & 3
}

@@ -35,2 +42,8 @@

Node.prototype.collides = function (node, i) {
if (i === this.length - 1) return this.key !== node.key
const j = Math.floor(i / 32)
return this.keySplit[j] !== node.keySplit[j]
}
Node.decode = function (buf, seq, enc) {

@@ -40,7 +53,9 @@ return new Node(messages.Node.decode(buf), seq, enc)

Node.terminator = function (i) {
return ((i + 1) & 31) === 0
}
Node.normalizeKey = normalizeKey
function hash (keys) {
if (typeof keys === 'string') keys = split(keys)
const buf = Buffer.allocUnsafe(8 * keys.length)

@@ -47,0 +62,0 @@

@@ -70,3 +70,3 @@ const Node = require('./node')

Put.prototype._pushEnd = function (i, val, seq) {
Put.prototype._pushCollidable = function (i, val, seq) {
if (seq === this._del) return

@@ -78,3 +78,3 @@

if (err) this._error = err
else if (node.key !== self._node.key) push(self._node.trie, i, val, seq)
else if (node.collides(self._node, i)) push(self._node.trie, i, val, seq)
if (!--self._pending && self._finalized) self._finalize(null)

@@ -87,4 +87,8 @@ })

for (; i < this._node.length; i++) {
const val = this._node.path(i)
const node = this._node
for (; i < node.length; i++) {
// check for collision at the end (4) or if it's a prefix terminator
const checkCollision = Node.terminator(i)
const val = node.path(i)
const bucket = head.trie[i] || []

@@ -94,17 +98,26 @@ const headVal = head.path(i)

for (var j = 0; j < bucket.length; j++) {
if (j === val && val !== 4) continue
// if same hash prefix, if no collision check is needed just continue
if (j === val && !checkCollision) continue
const seq = bucket[j]
if (!seq) continue
if (val === 4) this._pushEnd(i, j, seq)
else this._push(i, j, seq)
if (!seq) continue // skip no-ops
if (!checkCollision) { // TODO: can prob optimise this with a || j !== val
this._push(i, j, seq)
} else {
this._pushCollidable(i, j, seq)
}
}
if (headVal === val && (headVal < 4 || head.key === this._node.key)) continue
// we copied the head bucket, if this is still the closest node, continue
// if no collision is possible
if (headVal === val && (!checkCollision || !node.collides(head, i))) continue
this._push(i, headVal, head.seq)
const seq = bucket[val] // TODO: handle val === 4
if (!seq) return this._finalize(null)
this._updateHead(i, seq)
return
if (checkCollision) return this._updateHeadCollidable(i, bucket, val)
const seq = bucket[val]
if (!seq) break
return this._updateHead(i, seq)
}

@@ -121,2 +134,27 @@

Put.prototype._updateHeadCollidable = function (i, bucket, val) {
const self = this
var missing = 1
var error = null
var node = null
for (var j = val; j < bucket.length; j += 5) {
const seq = bucket[j]
if (!seq) break
missing++
this._get(seq, onnode)
}
onnode(null, null)
function onnode (err, n) {
if (err) error = err
else if (n && !n.collides(self._node, i)) node = n
if (--missing) return
if (!node) return self._finalize(error)
self._update(i + 1, node)
}
}
Put.prototype._updateHead = function (i, seq) {

@@ -133,8 +171,8 @@ const self = this

function push (trie, i, val, seq) {
while (val >= 5) val -= 5
const bucket = trie[i] || (trie[i] = [])
if (val >= 4 && bucket.length >= 5) {
if (bucket.indexOf(seq, 4) === -1) bucket.push(seq)
} else {
bucket[val] = seq
}
while (bucket.length > val && bucket[val]) val += 5
if (bucket.indexOf(seq) === -1) bucket[val] = seq
}
{
"name": "hypertrie",
"version": "1.0.0",
"version": "2.0.0",
"description": "Distributed single writer key/value store",

@@ -11,2 +11,3 @@ "main": "index.js",

"inherits": "^2.0.3",
"inspect-custom-symbol": "^1.1.0",
"is-options": "^1.0.1",

@@ -23,2 +24,5 @@ "mutexify": "^1.2.0",

"compare": "^2.0.0",
"protocol-buffers": "^4.0.4",
"random-access-memory": "^3.0.0",
"standard": "^11.0.1",
"stream-collector": "^1.0.1",

@@ -25,0 +29,0 @@ "tape": "^4.9.0"

@@ -237,3 +237,3 @@ const tape = require('tape')

tape('race works', async function (t) {
tape('race works', function (t) {
t.plan(40)

@@ -240,0 +240,0 @@

@@ -90,1 +90,86 @@ const tape = require('tape')

})
tape('two prefixes with same key', function (t) {
const db = create()
db.put('idgcmnmna/a', 'a', function () {
db.put('mpomeiehc/a', 'a', function () {
const ite = db.iterator({recursive: false})
ite.next(function (err, node) {
t.error(err, 'no error')
t.same(node.value, 'a')
})
ite.next(function (err, node) {
t.error(err, 'no error')
t.same(node.value, 'a')
t.end()
})
})
})
})
tape('sorts based on key when colliding', function (t) {
const db1 = create()
const db2 = create()
db1.batch([
{key: 'idgcmnmna'},
{key: 'mpomeiehc'},
{key: 'a'},
{key: 'b'},
{key: 'c'}
], function () {
db2.batch([
{key: 'b'},
{key: 'mpomeiehc'},
{key: 'a'},
{key: 'idgcmnmna'},
{key: 'c'}
], function () {
const i1 = db1.iterator()
const i2 = db2.iterator()
i1.next(function loop (err, n1) {
t.error(err, 'no error')
i2.next(function (err, n2) {
t.error(err, 'no error')
if (!n1 && !n2) return t.end()
t.same(n1.key, n2.key)
i1.next(loop)
})
})
})
})
})
tape('two keys with same siphash (diff)', function (t) {
const db = create()
db.batch([
{key: 'idgcmnmna'},
{key: 'mpomeiehc'},
{key: 'a'},
{key: 'b'},
{key: 'c'}
], function () {
const ite = db.diff(0)
const found = {}
ite.next(function loop (err, node) {
t.error(err, 'no error')
if (!node) {
t.same(found, {
idgcmnmna: true,
mpomeiehc: true,
a: true,
b: true,
c: true
})
return t.end()
}
found[node.key] = true
ite.next(loop)
})
})
})
const ram = require('random-access-memory')
const hypertrie = require('hypertrie')
const hypertrie = require('../../')

@@ -4,0 +4,0 @@ module.exports = function (key) {

Sorry, the diff of this file is not supported yet

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc