Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More
Socket
Sign inDemoInstall
Socket

@iobroker/db-states-jsonl

Package Overview
Dependencies
Maintainers
6
Versions
417
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

@iobroker/db-states-jsonl - npm Package Compare versions

Comparing version 4.0.0-alpha.5-20210903-ac21ada4 to 4.0.0-alpha.50-20220123-bedb0512

138

lib/states/statesInMemJsonlDB.js

@@ -18,4 +18,7 @@ /**

const StatesInMemoryFileDB = require('@iobroker/db-states-file').StatesInMemoryFileDB;
const { JsonlDB } = require('@alcalzone/jsonl-db');
const path = require('path');
const { JsonlDB } = require('@alcalzone/jsonl-db');
const path = require('path');
const fs = require('fs');
const os = require('os');
const { tools } = require('@iobroker/js-controller-common');

@@ -48,3 +51,2 @@ // settings = {

class StatesInMemoryJsonlDB extends StatesInMemoryFileDB {
constructor(settings) {

@@ -57,7 +59,9 @@ settings = settings || {};

};
super(settings);
/** @type {import("@alcalzone/jsonl-db").JsonlDBOptions<any>} */
const jsonlOptions = {
const jsonlOptions = this.settings.connection.jsonlOptions || {
autoCompress: {
sizeFactor: 2,
sizeFactorMinimumSize: 1000
sizeFactor: 10,
sizeFactorMinimumSize: 50000
},

@@ -71,16 +75,13 @@ ignoreReadErrors: true,

settings.jsonlDB = {
fileName: 'states.jsonl',
options: jsonlOptions
fileName: 'states.jsonl'
};
super(settings);
/** @type {JsonlDB<any>} */
this._db = new JsonlDB(
path.join(this.dataDir, settings.jsonlDB.fileName),
jsonlOptions
);
this._db = new JsonlDB(path.join(this.dataDir, settings.jsonlDB.fileName), jsonlOptions);
}
async open() {
await this._db.open();
if (!(await this._maybeMigrateFileDB())) {
await this._db.open();
}

@@ -122,4 +123,86 @@ // Create an object-like wrapper around the internal Map

});
if (this.settings.backup && this.settings.backup.period && !this.settings.backup.disabled) {
this._backupInterval = setInterval(() => {
this.saveBackup();
}, this.settings.backup.period);
}
}
/**
* Checks if an existing file DB should be migrated to JSONL
* @returns {Promise<boolean>} true if the file DB was migrated. false if not.
* If this returns true, the jsonl DB was opened and doesn't need to be opened again.
*/
async _maybeMigrateFileDB() {
const jsonlFileName = path.join(this.dataDir, this.settings.jsonlDB.fileName);
const jsonFileName = path.join(this.dataDir, this.settings.fileDB.fileName);
const bakFileName = path.join(this.dataDir, this.settings.fileDB.fileName + '.bak');
// Check the timestamps of each file, defaulting to 0 if they don't exist
let jsonlTimeStamp = 0;
let jsonTimeStamp = 0;
let bakTimeStamp = 0;
try {
const stat = fs.statSync(jsonlFileName);
if (stat.isFile()) {
jsonlTimeStamp = stat.mtime;
}
} catch {
// ignore
}
try {
const stat = fs.statSync(jsonFileName);
if (stat.isFile()) {
jsonTimeStamp = stat.mtime;
}
} catch {
// ignore
}
try {
const stat = fs.statSync(bakFileName);
if (stat.isFile()) {
bakTimeStamp = stat.mtime;
}
} catch {
// ignore
}
// Figure out which file needs to be imported
/** @type {string} */
let importFilename;
if (jsonTimeStamp > 0 && jsonTimeStamp >= bakTimeStamp && jsonTimeStamp >= jsonlTimeStamp) {
importFilename = jsonFileName;
} else if (bakTimeStamp > 0 && bakTimeStamp >= jsonTimeStamp && bakTimeStamp >= jsonlTimeStamp) {
importFilename = bakFileName;
} else {
// None of the File DB files are newer than the JSONL file
// There is nothing to restore, we're done
return false;
}
await this._db.open();
this._db.clear();
await this._db.importJson(importFilename);
// And rename the existing files to avoid redoing the work next time
if (fs.existsSync(jsonFileName)) {
try {
fs.renameSync(jsonFileName, `${jsonFileName}.migrated`);
} catch {
// ignore
}
}
if (fs.existsSync(bakFileName)) {
try {
fs.renameSync(bakFileName, `${bakFileName}.migrated`);
} catch {
// ignore
}
}
// Signal to the caller that the DB is already open
return true;
}
async saveState() {

@@ -129,2 +212,26 @@ // Nothing to do, the DB saves behind the scenes

// Is regularly called and stores a compressed backup of the DB
async saveBackup() {
const now = Date.now();
const tmpBackupFileName = path.join(os.tmpdir(), `${this.getTimeStr(now)}_${this.settings.jsonlDB.fileName}`);
const backupFileName = path.join(
this.backupDir,
`${this.getTimeStr(now)}_${this.settings.jsonlDB.fileName}.gz`
);
try {
if (fs.existsSync(backupFileName)) {
return;
}
// Create a DB dump
await this._db.dump(tmpBackupFileName);
// and zip it
await tools.compressFileGZip(tmpBackupFileName, backupFileName, { deleteInput: true });
// figure out if older files need to be deleted
this.deleteOldBackupFiles(this.settings.jsonlDB.fileName);
} catch (e) {
this.log.error(`${this.namespace} Cannot save backup ${backupFileName}: ${e.message}`);
}
}
async destroy() {

@@ -134,2 +241,5 @@ if (this._db) {

}
if (this._backupInterval) {
clearInterval(this._backupInterval);
}
}

@@ -136,0 +246,0 @@ }

32

lib/states/statesInMemServerClass.js

@@ -18,30 +18,14 @@ /**

const StatesInRedisClient = require('@iobroker/db-states-redis').Client;
const StatesInMemServer = require('./statesInMemServerRedis');
const StatesInMemServer = require('./statesInMemServerRedis');
class StatesInMemoryServerClass extends StatesInRedisClient {
constructor(settings) {
settings.autoConnect = false; // delay Client connection to when we need it
// hack around testing problem where subscribe was called before connect
// Should be removed for a later release
const origConnected = settings.connected;
settings.connected = () => {
this.clientConnected = true;
if (Array.isArray(this.storedSubscribes) && this.storedSubscribes.length) {
this.log.warn(`${this.namespace} Replay ${this.storedSubscribes.length} subscription calls for States Server that were done before the client was connected initially`);
this.storedSubscribes.forEach((s => this.subscribe(s.pattern, s.options, s.callback)));
this.storedSubscribes = [];
}
origConnected();
};
super(settings);
this.clientConnected = false;
this.storedSubscribes = [];
const serverSettings = {
namespace: settings.namespace ? `${settings.namespace}-Server` : 'Server',
namespace: settings.namespace ? `${settings.namespace}-Server` : 'Server',
connection: settings.connection,
logger: settings.logger,
hostname: settings.hostname,
logger: settings.logger,
hostname: settings.hostname,
connected: () => {

@@ -62,12 +46,4 @@ this.connectDb(); // now that server is connected also connect client

}
async subscribe(pattern, options, callback) {
if (!this.clientConnected) {
this.storedSubscribes.push({pattern, options, callback}); // we ignore the promise return because not used for this testing issue we work around here
} else {
await super.subscribe(pattern, options, callback);
}
}
}
module.exports = StatesInMemoryServerClass;

@@ -16,6 +16,6 @@ /**

'use strict';
const net = require('net');
const net = require('net');
const { inspect } = require('util');
const RedisHandler = require('@iobroker/db-base').redisHandler;
const RedisHandler = require('@iobroker/db-base').redisHandler;
const StatesInMemoryJsonlDB = require('./statesInMemJsonlDB');

@@ -57,23 +57,34 @@

this.serverConnections = {};
this.namespaceStates = (this.settings.redisNamespace || 'io') + '.';
this.namespaceMsg = (this.settings.namespaceMsg || 'messagebox') + '.';
this.namespaceLog = (this.settings.namespaceLog || 'log') + '.';
this.namespaceSession = (this.settings.namespaceSession || 'session') + '.';
this.namespaceStates = (this.settings.redisNamespace || 'io') + '.';
this.namespaceMsg = (this.settings.namespaceMsg || 'messagebox') + '.';
this.namespaceLog = (this.settings.namespaceLog || 'log') + '.';
this.namespaceSession = (this.settings.namespaceSession || 'session') + '.';
//this.namespaceStatesLen = this.namespaceStates.length;
this.namespaceMsgLen = this.namespaceMsg.length;
this.namespaceLogLen = this.namespaceLog.length;
this.namespaceMsgLen = this.namespaceMsg.length;
this.namespaceLogLen = this.namespaceLog.length;
//this.namespaceSessionlen = this.namespaceSession.length;
this.metaNamespace = (this.settings.metaNamespace || 'meta') + '.';
this.metaNamespaceLen = this.metaNamespace.length;
this.open().then(() => {
return this._initRedisServer(this.settings.connection);
}).then(() => {
this.log.debug(`${this.namespace} ${settings.secure ? 'Secure ' : ''} Redis inMem-states listening on port ${this.settings.port || 9000}`);
this.open()
.then(() => {
return this._initRedisServer(this.settings.connection);
})
.then(() => {
this.log.debug(
`${this.namespace} ${settings.secure ? 'Secure ' : ''} Redis inMem-states listening on port ${
this.settings.port || 9000
}`
);
if (typeof this.settings.connected === 'function') {
setImmediate(() => this.settings.connected());
}
}).catch(e => {
this.log.error(`${this.namespace} Cannot start inMem-states on port ${this.settings.port || 9000}: ${e.message}`);
process.exit(24); // todo: replace it with exitcode
});
if (typeof this.settings.connected === 'function') {
setImmediate(() => this.settings.connected());
}
})
.catch(e => {
this.log.error(
`${this.namespace} Cannot start inMem-states on port ${this.settings.port || 9000}: ${e.message}`
);
process.exit(24); // todo: replace it with exitcode
});
}

@@ -94,3 +105,3 @@

idWithNamespace.forEach(el => {
const {id, namespace} = this._normalizeId(el);
const { id, namespace } = this._normalizeId(el);
ids.push(id);

@@ -105,3 +116,3 @@ ns = namespace; // we ignore the pot. case from arrays with different namespaces

ns = idWithNamespace.substr(0, pointIdx + 1);
if (ns === this.namespaceStates) {
if (ns === this.namespaceStates || ns === this.metaNamespace) {
id = idWithNamespace.substr(pointIdx + 1);

@@ -111,3 +122,3 @@ }

}
return {id: id, namespace: ns};
return { id: id, namespace: ns };
}

@@ -132,15 +143,22 @@

if (found) {
let objString;
try {
objString = JSON.stringify(obj);
} catch (e) {
// mainly catch circular structures - thus log object with inspect
this.log.error(`${this.namespace} Error on publishing state: ${id}=${inspect(obj)}: ${e.message}`);
return 0;
if (type === 'meta') {
this.log.silly(`${this.namespace} Redis Publish Meta ${id}=${obj}`);
const sendPattern = this.metaNamespace + found.pattern;
const sendId = this.metaNamespace + id;
client.sendArray(null, ['pmessage', sendPattern, sendId, obj]);
} else {
let objString;
try {
objString = JSON.stringify(obj);
} catch (e) {
// mainly catch circular structures - thus log object with inspect
this.log.error(`${this.namespace} Error on publishing state: ${id}=${inspect(obj)}: ${e.message}`);
return 0;
}
this.log.silly(`${this.namespace} Redis Publish State ${id}=${objString}`);
const sendPattern = (type === 'state' ? '' : this.namespaceStates) + found.pattern;
const sendId = (type === 'state' ? '' : this.namespaceStates) + id;
client.sendArray(null, ['pmessage', sendPattern, sendId, objString]);
}
this.log.silly(`${this.namespace} Redis Publish State ${id}=${objString}`);
const sendPattern = (type === 'state' ? '' : this.namespaceStates) + found.pattern;
const sendId = (type === 'state' ? '' : this.namespaceStates) + id;
client.sendArray(null, ['pmessage', sendPattern, sendId, objString]);
return 1;

@@ -172,3 +190,5 @@ }

infoString += '# Keyspace\r\n';
infoString += `db0:keys=${Object.keys(this.dataset).length},expires=${Object.keys(this.stateExpires).length + Object.keys(this.sessionExpires).length},avg_ttl=98633637897`;
infoString += `db0:keys=${Object.keys(this.dataset).length},expires=${
Object.keys(this.stateExpires).length + Object.keys(this.sessionExpires).length
},avg_ttl=98633637897`;
handler.sendBulk(responseId, infoString);

@@ -186,4 +206,5 @@ });

handler.on('publish', (data, responseId) => {
const {id, namespace} = this._normalizeId(data[0]);
if (namespace === this.namespaceStates) { // a "set" always comes afterwards, so do not publish
const { id, namespace } = this._normalizeId(data[0]);
if (namespace === this.namespaceStates || namespace === this.metaNamespace) {
// a "set" always comes afterwards, so do not publish
return void handler.sendInteger(responseId, 0); // do not publish for now

@@ -200,3 +221,3 @@ }

}
const {id, namespace} = this._normalizeId(data);
const { id, namespace } = this._normalizeId(data);

@@ -206,3 +227,3 @@ if (namespace === this.namespaceStates) {

const states = this._getStates(id);
const result = states.map(el => el ? JSON.stringify(el) : null);
const result = states.map(el => (el ? JSON.stringify(el) : null));
handler.sendArray(responseId, result);

@@ -213,3 +234,6 @@ } catch (err) {

} else {
handler.sendError(responseId, new Error(`MGET-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
handler.sendError(
responseId,
new Error(`MGET-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
);
}

@@ -220,3 +244,3 @@ });

handler.on('get', (data, responseId) => {
const {id, namespace} = this._normalizeId(data[0]);
const { id, namespace } = this._normalizeId(data[0]);
if (namespace === this.namespaceStates) {

@@ -240,4 +264,14 @@ const result = this._getState(id);

}
} else if (namespace === this.metaNamespace) {
const result = this.getMeta(id);
if (result === undefined || result === null) {
handler.sendNull(responseId);
} else {
handler.sendBulk(responseId, result);
}
} else {
handler.sendError(responseId, new Error(`GET-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
handler.sendError(
responseId,
new Error(`GET-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
);
}

@@ -248,3 +282,3 @@ });

handler.on('set', (data, responseId) => {
const {id, namespace} = this._normalizeId(data[0]);
const { id, namespace } = this._normalizeId(data[0]);
if (namespace === this.namespaceStates) {

@@ -255,3 +289,4 @@ try {

state = JSON.parse(data[1].toString('utf-8'));
} catch { // No JSON, so handle as binary data and set as Buffer
} catch {
// No JSON, so handle as binary data and set as Buffer
this._setBinaryState(id, data[1]);

@@ -265,4 +300,10 @@ return void handler.sendString(responseId, 'OK');

}
} else if (namespace === this.metaNamespace) {
this.setMeta(id, data[1].toString('utf-8'));
handler.sendString(responseId, 'OK');
} else {
handler.sendError(responseId, new Error(`SET-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
handler.sendError(
responseId,
new Error(`SET-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
);
}

@@ -273,3 +314,3 @@ });

handler.on('setex', (data, responseId) => {
const {id, namespace} = this._normalizeId(data[0]);
const { id, namespace } = this._normalizeId(data[0]);
if (namespace === this.namespaceStates) {

@@ -280,3 +321,4 @@ try {

state = JSON.parse(data[2].toString('utf-8'));
} catch { // No JSON, so handle as binary data and set as Buffer
} catch {
// No JSON, so handle as binary data and set as Buffer
state = data[2];

@@ -286,3 +328,6 @@ }

if (isNaN(expire)) {
return void handler.sendError(responseId, new Error(`ERROR parsing expire value ${data[1].toString('utf-8')}`));
return void handler.sendError(
responseId,
new Error(`ERROR parsing expire value ${data[1].toString('utf-8')}`)
);
}

@@ -299,3 +344,6 @@ this._setStateDirect(id, state, expire);

if (isNaN(expire)) {
return void handler.sendError(responseId, new Error(`ERROR parsing expire value ${data[1].toString('utf-8')}`));
return void handler.sendError(
responseId,
new Error(`ERROR parsing expire value ${data[1].toString('utf-8')}`)
);
}

@@ -308,3 +356,6 @@ this._setSession(id, expire, state);

} else {
handler.sendError(responseId, new Error(`SETEX-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
handler.sendError(
responseId,
new Error(`SETEX-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
);
}

@@ -315,3 +366,3 @@ });

handler.on('del', (data, responseId) => {
const {id, namespace} = this._normalizeId(data[0]);
const { id, namespace } = this._normalizeId(data[0]);
if (namespace === this.namespaceStates) {

@@ -324,3 +375,6 @@ this._delState(id);

} else {
handler.sendError(responseId, new Error(`DEL-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
handler.sendError(
responseId,
new Error(`DEL-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
);
}

@@ -334,3 +388,3 @@ });

}
const {id, namespace} = this._normalizeId(data[0]);
const { id, namespace } = this._normalizeId(data[0]);
if (namespace === this.namespaceStates) {

@@ -346,3 +400,6 @@ // special case because of simulation of redis

} else {
handler.sendError(responseId, new Error(`KEYS-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
handler.sendError(
responseId,
new Error(`KEYS-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
);
}

@@ -353,3 +410,3 @@ });

handler.on('psubscribe', (data, responseId) => {
const {id, namespace} = this._normalizeId(data[0]);
const { id, namespace } = this._normalizeId(data[0]);
if (namespace === this.namespaceMsg) {

@@ -364,4 +421,10 @@ this._subscribeMessageForClient(handler, id.substr(this.namespaceMsgLen));

handler.sendArray(responseId, ['psubscribe', data[0], 1]);
} else if (namespace === this.metaNamespace) {
this._subscribeMeta(handler, id);
handler.sendArray(responseId, ['psubscribe', data[0], 1]);
} else {
handler.sendError(responseId, new Error(`PSUBSCRIBE-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
handler.sendError(
responseId,
new Error(`PSUBSCRIBE-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
);
}

@@ -372,3 +435,3 @@ });

handler.on('punsubscribe', (data, responseId) => {
const {id, namespace} = this._normalizeId(data[0]);
const { id, namespace } = this._normalizeId(data[0]);
if (namespace === this.namespaceMsg) {

@@ -384,3 +447,6 @@ this._unsubscribeMessageForClient(handler, id.substr(this.namespaceMsgLen));

} else {
handler.sendError(responseId, new Error(`PUNSUBSCRIBE-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`));
handler.sendError(
responseId,
new Error(`PUNSUBSCRIBE-UNSUPPORTED for namespace ${namespace}: Data=${JSON.stringify(data)}`)
);
}

@@ -427,4 +493,3 @@ });

handler.on('error', err =>
this.log.warn(`${namespaceLog} Redis states: ${err}`));
handler.on('error', err => this.log.warn(`${namespaceLog} Redis states: ${err}`));
}

@@ -452,13 +517,15 @@

return /** @type {Promise<void>} */ (new Promise(resolve => {
if (!this.server) {
return void resolve();
}
try {
this.server.close(() => resolve());
} catch (e) {
console.log(e.message);
resolve();
}
}));
return /** @type {Promise<void>} */ (
new Promise(resolve => {
if (!this.server) {
return void resolve();
}
try {
this.server.close(() => resolve());
} catch (e) {
console.log(e.message);
resolve();
}
})
);
}

@@ -473,3 +540,4 @@ }

_initSocket(socket) {
this.settings.connection.enhancedLogging && this.log.silly(`${this.namespace} Handling new Redis States connection`);
this.settings.connection.enhancedLogging &&
this.log.silly(`${this.namespace} Handling new Redis States connection`);

@@ -508,3 +576,8 @@ const options = {

this.server.on('error', err =>
this.log.info(`${this.namespace} ${settings.secure ? 'Secure ' : ''} Error inMem-objects listening on port ${settings.port || 9001}: ${err}`));
this.log.info(
`${this.namespace} ${settings.secure ? 'Secure ' : ''} Error inMem-objects listening on port ${
settings.port || 9001
}: ${err}`
)
);
this.server.on('connection', socket => this._initSocket(socket));

@@ -511,0 +584,0 @@

{
"name": "@iobroker/db-states-jsonl",
"version": "4.0.0-alpha.5-20210903-ac21ada4",
"version": "4.0.0-alpha.50-20220123-bedb0512",
"engines": {

@@ -8,6 +8,6 @@ "node": ">=12.0.0"

"dependencies": {
"@alcalzone/jsonl-db": "^2.1.0",
"@iobroker/db-base": "4.0.0-alpha.5-20210903-ac21ada4",
"@iobroker/db-states-file": "4.0.0-alpha.5-20210903-ac21ada4",
"@iobroker/db-states-redis": "4.0.0-alpha.5-20210903-ac21ada4"
"@alcalzone/jsonl-db": "~2.4.1",
"@iobroker/db-base": "4.0.0-alpha.50-20220123-bedb0512",
"@iobroker/db-states-file": "4.0.0-alpha.50-20220123-bedb0512",
"@iobroker/db-states-redis": "4.0.0-alpha.50-20220123-bedb0512"
},

@@ -35,3 +35,7 @@ "keywords": [

},
"gitHead": "747dba053c5838df01bb90d04101665f449cec8a"
"files": [
"lib/",
"index.js"
],
"gitHead": "6c23f1520df68de8eb5450e81ee94ec286063720"
}

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc