Socket
Socket
Sign inDemoInstall

scattered-store

Package Overview
Dependencies
Maintainers
1
Versions
8
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

scattered-store - npm Package Compare versions

Comparing version 0.1.2 to 0.1.3

spec/persistence.spec.js

227

benchmark/benchmark.js

@@ -5,11 +5,13 @@ "use strict";

var Q = require('q');
var scatteredStore = require('..');
var _ = require('underscore');
var jetpack = require('fs-jetpack');
var scatteredStore = require('..');
var store;
var path = os.tmpdir() + '/scattered-store-benchmark';
var numberOfOperations = 20000;
var itemSize = 1024 * 50;
var itemsTotal = 100000;
var readsPerTest = 10000;
var itemSize = 1000 * 25;
var startTime;
var keys = [];

@@ -24,83 +26,162 @@ var testObj = new Buffer(itemSize);

var start = function (message) {
process.stdout.write(message);
startTime = Date.now();
}
var start = function (message, totalOps) {
var startTime = Date.now();
var doneOps = 0;
var currPerc;
var stop = function () {
var endTime = Date.now();
var duration = (endTime - startTime) / 1000;
var opsPerSec = Math.round(numberOfOperations / duration);
console.log(' ' + opsPerSec + " items/s");
var progress = function (moreDone) {
doneOps += moreDone;
var perc = Math.floor(doneOps / totalOps * 100);
if (currPerc !== perc) {
currPerc = perc;
process.stdout.clearLine();
process.stdout.cursorTo(0);
process.stdout.write(message + ' [' + currPerc + '%]');
}
};
var stop = function () {
var endTime = Date.now();
var duration = (endTime - startTime) / 1000;
var opsPerSec = Math.round(totalOps / duration);
process.stdout.clearLine();
process.stdout.cursorTo(0);
console.log(message + ' ' + opsPerSec + " items/s");
};
progress(0);
return {
progress: progress,
stop: stop,
}
}
// clean before benchmark
jetpack.dir(path, { exists: false });
var prepare = function () {
var deferred = Q.defer();
console.log('Testing scattered-store performance: ' + numberOfOperations +
' items, ' + (itemSize / 1024) + 'KB each, ' +
Math.round(numberOfOperations * itemSize / (1024 * 1024)) +
'MB combined.');
jetpack.dir(path, { exists: false });
var store = scatteredStore.create(path, function (err) {
if (!err) {
run();
console.log('Testing scattered-store performance: ' + itemsTotal +
' items, ' + (itemSize / 1000) + 'KB each, ' +
(itemsTotal * itemSize / (1000 * 1000 * 1000)).toFixed(1) +
'GB combined.');
store = scatteredStore.create(path, function (err) {
if (err) {
console.log(err);
deferred.reject();
} else {
deferred.resolve();
}
});
return deferred.promise;
};
var testSet = function () {
var test = start('set', itemsTotal);
var deferred = Q.defer();
var oneMore = function () {
if (keys.length < itemsTotal) {
store.set(generateKey(), testObj)
.then(function () {
test.progress(1);
oneMore();
});
} else {
test.stop();
// After finished insertion shuffle keys array to simulate
// random access to stored entries.
keys = _.shuffle(keys);
deferred.resolve();
}
}
});
oneMore();
return deferred.promise;
};
var run = function () {
start('set...');
for (var i = 0; i < numberOfOperations; i += 1) {
store.set(generateKey(), testObj)
}
// order of operations is preserved,
// so we know that after finish of this one all are finished
store.set(generateKey(), testObj)
.then(function () {
stop();
start('get...');
for (var i = 0; i < keys.length; i += 1) {
var testGet = function () {
var test = start('get', readsPerTest);
var deferred = Q.defer();
var i = 0;
var oneMore = function () {
if (i < readsPerTest) {
store.get(keys[i])
.then(function () {
test.progress(1);
oneMore();
});
} else {
test.stop();
deferred.resolve();
}
return store.get("none");
i += 1;
}
oneMore();
return deferred.promise;
};
var testGetMany = function () {
var test = start('getMany', readsPerTest);
var deferred = Q.defer();
var stream = store.getMany(keys.slice(0, readsPerTest))
.on('readable', function () {
stream.read();
test.progress(1);
})
.then(function () {
stop();
start('getAll...');
var deferred = Q.defer();
var stream = store.getAll()
.on('readable', function () {
stream.read();
})
.on('end', deferred.resolve);
return deferred.promise;
.on('error', deferred.reject)
.on('end', function () {
test.stop();
deferred.resolve();
});
return deferred.promise;
};
var testGetAll = function () {
var test = start('getAll', keys.length);
var deferred = Q.defer();
var stream = store.getAll()
.on('readable', function () {
stream.read();
test.progress(1);
})
.then(function () {
stop();
start('delete...');
for (var i = 0; i < keys.length; i += 1) {
.on('end', function () {
test.stop();
deferred.resolve();
});
return deferred.promise;
};
var testDelete = function () {
var test = start('delete', keys.length);
var deferred = Q.defer();
var i = 0;
var oneMore = function () {
if (i < keys.length) {
store.delete(keys[i])
.then(function () {
test.progress(1);
oneMore();
});
} else {
test.stop();
deferred.resolve();
}
return store.delete("none");
})
.then(function () {
stop();
// clean after benchmark
jetpack.dir(path, { exists: false });
});
};
i += 1;
}
oneMore();
return deferred.promise;
};
var clean = function () {
jetpack.dir(path, { exists: false });
};
prepare()
.then(testSet)
.then(testGet)
.then(testGetMany)
.then(testGetAll)
.then(testDelete)
.then(clean);

82

lib/lister.js

@@ -13,2 +13,3 @@ // Readable stream. Discovers all file paths inside storage folder.

this._basePath = basePath;
this._subdirsToGo = null;
this._pathsToGive = [];

@@ -19,42 +20,40 @@ };

Lister.prototype._ensureBaseDirListed = function () {
Lister.prototype._loadMorePaths = function () {
var deferred = Q.defer();
var that = this;
if (this._subdirsInBaseDir === undefined) {
jetpack.listAsync(this._basePath)
.then(function (dirs) {
that._subdirsInBaseDir = dirs;
deferred.resolve();
});
} else {
deferred.resolve();
}
return deferred.promise;
};
Lister.prototype._ensureWeHavePathToGive = function () {
var deferred = Q.defer();
var that = this;
this._ensureBaseDirListed()
.then(function () {
if (that._pathsToGive.length > 0) {
deferred.resolve();
} else if (that._subdirsInBaseDir.length > 0) {
var subdir = that._subdirsInBaseDir.pop();
var listNextSubdir = function () {
if (that._subdirsToGo.length === 0) {
// No more directories to list!
deferred.reject('endOfPaths');
} else {
var subdir = that._subdirsToGo.pop();
var path = jetpack.path(that._basePath, subdir);
jetpack.listAsync(path)
.then(function (filenames) {
// Generate absolute paths from filenames.
that._pathsToGive = filenames.map(function (filename) {
return jetpack.path(path, filename);
});
deferred.resolve();
if (that._pathsToGive.length > 0) {
// Yep. We have paths. Done!
deferred.resolve();
} else {
// This directory was apparently empty. Go for next one.
listNextSubdir();
}
});
} else {
deferred.reject('endOfPaths');
}
});
};
if (this._subdirsToGo === null) {
jetpack.listAsync(this._basePath)
.then(function (dirs) {
that._subdirsToGo = dirs;
listNextSubdir();
});
} else {
listNextSubdir();
}
return deferred.promise;

@@ -64,15 +63,18 @@ };

Lister.prototype._read = function() {
var that = this;
this._ensureWeHavePathToGive()
.then(function () {
that.push(that._pathsToGive.pop());
})
.catch(function (err) {
if (err === 'endOfPaths') {
that.push(null);
}
});
if (this._pathsToGive.length > 0) {
this.push(this._pathsToGive.pop());
} else {
var that = this;
this._loadMorePaths()
.then(function () {
that.push(that._pathsToGive.pop());
})
.catch(function (err) {
if (err === 'endOfPaths') {
that.push(null);
}
});
}
};
module.exports = Lister;

44

lib/persistence.js

@@ -25,3 +25,3 @@ "use strict";

var data;
if (Buffer.isBuffer(value)) {

@@ -34,3 +34,3 @@ type = 'binary';

}
var fileHeaderStr = JSON.stringify({

@@ -41,3 +41,3 @@ type: type,

var fileHeader = new Buffer(fileHeaderStr + String.fromCharCode(newLineCode));
return Buffer.concat([fileHeader, data]);

@@ -57,10 +57,10 @@ };

var fileHeader = JSON.parse(fileHeaderBuf.toString());
var dataBuf = buf.slice(i + 1); // Skip the new line character...
// ... and everything after new line is data:
var entry = {
key: fileHeader.key
};
if (fileHeader.type === 'binary') {

@@ -71,3 +71,3 @@ entry.value = dataBuf;

}
return entry;

@@ -84,6 +84,6 @@ };

module.exports.create = function (storageDir, callback) {
// ----------------------------------------------
// Initialization
if (typeof storageDir !== 'string' || storageDir === '') {

@@ -112,6 +112,6 @@ callback(new Error('Path to storage directory not specified'));

}
// ----------------------------------------------
// Utils
var transformKeyToFilePath = function (key) {

@@ -125,6 +125,6 @@ var sha = crypto.createHash('sha1');

};
// ----------------------------------------------
// Actions on storage
var set = function (key, value) {

@@ -136,3 +136,3 @@ var filePath = transformKeyToFilePath(key);

set.asyncInterfaceType = 'promise';
var get = function (key) {

@@ -152,3 +152,3 @@ var deferred = Q.defer();

get.asyncInterfaceType = 'promise';
var del = function (key) {

@@ -164,3 +164,3 @@ var deferred = Q.defer();

del.asyncInterfaceType = 'promise';
var getMany = function (keys) {

@@ -175,19 +175,19 @@ var filePaths = keys.map(function (key) {

var reader = new ParallelReader(8, decodeFromStorage);
giver.pipe(reader);
return reader;
};
getMany.asyncInterfaceType = 'stream';
var getAll = function () {
var lister = new DirLister(storageDir.path());
var reader = new ParallelReader(8, decodeFromStorage);
lister.pipe(reader);
return reader;
};
getAll.asyncInterfaceType = 'stream';
return {

@@ -194,0 +194,0 @@ set: set,

19

lib/reader.js

@@ -1,2 +0,3 @@

// Parallel transform stream. Gets the path to file and returns content of this file processed with decoderFn.
// Parallel transform stream. Gets the path to file and returns
// content of this file processed with decodeDataFn.

@@ -25,3 +26,3 @@ 'use strict';

this._running += 1;
if (typeof item === 'string') {

@@ -34,7 +35,7 @@ // Given item is just path to file

}
jetpack.readAsync(path, 'buf', { safe: true })
.then(function (buf) {
that._running -= 1;
if (Buffer.isBuffer(buf)) {

@@ -46,12 +47,12 @@ that.push(that._decodeDataFn(buf));

}
if (callback) {
callback();
}
if (that._running === 0 && that._done) {
that._done();
}
});
});
if (this._running < this._maxParallel) {

@@ -71,2 +72,2 @@ callback();

module.exports = ParallelReader;
module.exports = ParallelReader;

2

package.json
{
"name": "scattered-store",
"description": "Key-value store for large datasets",
"version": "0.1.2",
"version": "0.1.3",
"author": "Jakub Szwacz <jakub@szwacz.com>",

@@ -6,0 +6,0 @@ "keywords": [

45

README.md

@@ -144,3 +144,3 @@ scattered-store

## whenIdle()
Hook to know when all queued tasks has been executed and store is idle. Useful e.g. if you want to terminate the process, and want to make sure no dataloss will occur.
Hook to know when all queued tasks has been executed and store is idle. Useful e.g. if you want to terminate the process, and want to make sure no dataloss will occur.
**Returns:** promise

@@ -161,29 +161,26 @@

```
Here are results of this test on few machines for comparison:
Desktop PC (HDD 7200rpm)
```
Testing scattered-store performance: 20000 items, 50KB each, 977MB combined.
set... 2522 items/s
get... 4471 items/s
getAll... 8428 items/s
delete... 5605 items/s
```
Here are results of this test on MacBook Pro with SSD. Tested with 10K, 100K and 1M items in store.
MacBook Pro (SSD)
```
Testing scattered-store performance: 20000 items, 50KB each, 977MB combined.
set... 1694 items/s
get... 4018 items/s
getAll... 6416 items/s
delete... 4030 items/s
```
Testing scattered-store performance: 10000 items, 25KB each, 0.3GB combined.
set 1737 items/s
get 4170 items/s
getMany 7018 items/s
getAll 6817 items/s
delete 4073 items/s
Mac Mini (HDD 5400rpm)
Testing scattered-store performance: 100000 items, 25KB each, 2.5GB combined.
set 1684 items/s
get 3926 items/s
getMany 6671 items/s
getAll 6644 items/s
delete 3733 items/s
Testing scattered-store performance: 1000000 items, 25KB each, 25.0GB combined.
set 1132 items/s
get 1259 items/s
getMany 5139 items/s
getAll 3574 items/s
delete 1348 items/s
```
Testing scattered-store performance: 20000 items, 50KB each, 977MB combined.
set... 726 items/s
get... 3860 items/s
getAll... 5071 items/s
delete... 1130 items/s
```

86

spec/api.spec.js
"use strict";
describe('api', function () {
var _ = require('underscore');

@@ -15,5 +15,5 @@ var pathUtil = require('path');

var testDir = pathUtil.resolve(utils.workingDir, 'test');
describe('get & set', function () {
it('writes and reads string', function (done) {

@@ -32,3 +32,3 @@ var key = "ąż"; // utf8 test

});
it('writes and reads object', function (done) {

@@ -50,3 +50,3 @@ var key = "ąż"; // utf8 test

});
it('writes and reads array', function (done) {

@@ -65,3 +65,3 @@ var key = "a";

});
it('writes and reads binary data', function (done) {

@@ -82,3 +82,3 @@ var key = "a";

});
it("returns null if key doesn't exist", function (done) {

@@ -93,3 +93,3 @@ var key = "a";

});
it("throws if key of length 0", function (done) {

@@ -101,7 +101,7 @@ var value = { a: "a" };

})
expect(function () {
store.get('');
}).toThrow(err);
expect(function () {

@@ -111,3 +111,3 @@ store.set('', value);

});
it("throws if key of different type than string", function (done) {

@@ -119,3 +119,3 @@ var value = { a: "a" };

});
expect(function () {

@@ -130,3 +130,3 @@ store.get(null);

}).toThrow(err);
expect(function () {

@@ -141,9 +141,9 @@ store.set(null, value);

}).toThrow(err);
});
});
describe('delete', function () {
it('can delete value for a key', function (done) {

@@ -165,3 +165,3 @@ var key = "a";

});
it("attempt to delete non-existent key does nothing", function (done) {

@@ -174,7 +174,7 @@ scatteredStore.create(testDir)

});
});
describe('getMany', function () {
it('throws if different argument than array passed', function (done) {

@@ -185,3 +185,3 @@ var err = new Error('Malformed array of keys');

});
expect(function () {

@@ -197,3 +197,3 @@ store.getMany();

});
it('terminates gracefully when empty collection passed', function (done) {

@@ -207,3 +207,3 @@ scatteredStore.create(testDir)

});
it('gives back entries with passed keys', function (done) {

@@ -215,3 +215,3 @@ var count = 0;

];
var store = scatteredStore.create(testDir);

@@ -235,3 +235,3 @@ store.set("b", "2"); // Should not be returned, although is in collection.

});
it('gives null for nonexistent key', function (done) {

@@ -250,7 +250,7 @@ var deliveredItem;

});
});
describe('getAll', function () {
it('terminates gracefully when store empty', function (done) {

@@ -264,3 +264,3 @@ scatteredStore.create(testDir)

});
it('iterates through all stored entries', function (done) {

@@ -273,3 +273,3 @@ var count = 0;

];
var store = scatteredStore.create(testDir);

@@ -293,7 +293,7 @@ store.set(dataset[0].key, dataset[0].value);

});
});
describe('write edge cases', function () {
it("can write empty string", function (done) {

@@ -312,3 +312,3 @@ var key = "a";

});
it("can write empty object", function (done) {

@@ -327,3 +327,3 @@ var key = "a";

});
it("can write empty array", function (done) {

@@ -342,3 +342,3 @@ var key = "a";

});
it("can write buffer of length 0", function (done) {

@@ -358,7 +358,7 @@ var key = "a";

});
});
describe('preventing data loss', function () {
it("whenIdle fires when all tasks done", function (done) {

@@ -377,3 +377,3 @@ var setCallbackFired = false;

});
it("whenIdle is called if already idle", function (done) {

@@ -390,5 +390,5 @@ var store = scatteredStore.create(testDir, function () {

});
});
});
SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc