@iobroker/db-base - npm Package Compare versions

Comparing version 4.0.0-alpha.7-20210908-26fd46db to 4.0.0-alpha.70-20220202-ee665b37

lib/inMemFileDB.js

@@ -17,5 +17,5 @@ /**
 
-const fs                = require('fs-extra');
-const path              = require('path');
-const tools             = require('./tools.js');
+const fs = require('fs-extra');
+const path = require('path');
+const tools = require('./tools.js');
 
@@ -48,3 +48,2 @@ // settings = {
 class InMemoryFileDB {
-
     constructor(settings) {
@@ -62,11 +61,11 @@ this.settings = settings || {};
 
-        this.settings.backup = this.settings.backup || {
-            disabled: false,  // deactivates
-            files: 24,     // minimum number of files
-            hours: 48,     // hours
-            period: 120,    // minutes
-            path: ''      // use default path
+        this.settings.backup = this.settings.connection.backup || {
+            disabled: false, // deactivates
+            files: 24, // minimum number of files
+            hours: 48, // hours
+            period: 120, // minutes
+            path: '' // use default path
         };
 
-        this.dataDir = (this.settings.connection.dataDir || tools.getDefaultDataDir());
+        this.dataDir = this.settings.connection.dataDir || tools.getDefaultDataDir();
         if (!path.isAbsolute(this.dataDir)) {
@@ -83,3 +82,3 @@ this.dataDir = path.normalize(path.join(tools.getControllerDir(), this.dataDir));
 
-        this.backupDir = this.settings.backup.path || (path.join(this.dataDir, this.settings.fileDB.backupDirName));
+        this.backupDir = this.settings.backup.path || path.join(this.dataDir, this.settings.fileDB.backupDirName);
 
@@ -124,2 +123,17 @@ if (!this.settings.backup.disabled) {
             ret = await this.loadDatasetFile(datasetName);
+
+            // loading worked, make sure that "bak" File is not broken
+            try {
+                await fs.readJSON(`${datasetName}.bak`);
+            } catch (e) {
+                this.log.info(
+                    `${this.namespace} Rewrite bak file, because error on verify ${datasetName}.bak: ${e.message}`
+                );
+                try {
+                    const jsonString = JSON.stringify(ret);
+                    await fs.writeFile(`${datasetName}.bak`, jsonString);
+                } catch (e) {
+                    this.log.error(`${this.namespace} Cannot save ${datasetName}.bak: ${e.message}`);
+                }
+            }
         } catch (err) {
@@ -129,3 +143,3 @@ this.log.error(`${this.namespace} Cannot load ${datasetName}: ${err.message}. We try last Backup!`);
             try {
-                ret = await this.loadDatasetFile(datasetName + '.bak');
+                ret = await this.loadDatasetFile(`${datasetName}.bak`);
 
@@ -136,5 +150,7 @@ // it worked, lets overwrite old file and store the broken one for pot. forensic check
                         try {
-                            await fs.move(datasetName, `${datasetName}.broken`, {overwrite: true});
+                            await fs.move(datasetName, `${datasetName}.broken`, { overwrite: true });
                         } catch (e) {
-                            this.log.error(`${this.namespace} Cannot copy the broken file ${datasetName} to ${datasetName}.broken ${e.message}`);
+                            this.log.error(
+                                `${this.namespace} Cannot copy the broken file ${datasetName} to ${datasetName}.broken ${e.message}`
+                            );
                         }
@@ -144,3 +160,5 @@ try {
                         } catch (e) {
-                            this.log.error(`${this.namespace} Cannot restore backup file as new main ${datasetName}: ${e.message}`);
+                            this.log.error(
+                                `${this.namespace} Cannot restore backup file as new main ${datasetName}: ${e.message}`
+                            );
                         }
@@ -152,4 +170,8 @@ }
             } catch (err) {
-                this.log.error(`${this.namespace} Cannot load ${datasetName}.bak: ${err.message}. Continue with empty dataset!`);
-                this.log.error(`${this.namespace} If this is no Migration or initial start please restore the last backup from ${this.backupDir}`);
+                this.log.error(
+                    `${this.namespace} Cannot load ${datasetName}.bak: ${err.message}. Continue with empty dataset!`
+                );
+                this.log.error(
+                    `${this.namespace} If this is no Migration or initial start please restore the last backup from ${this.backupDir}`
+                );
             }
@@ -163,3 +185,4 @@ }
         // Interval in minutes => to milliseconds
-        this.settings.backup.period = this.settings.backup.period === undefined ? 120 : parseInt(this.settings.backup.period);
+        this.settings.backup.period =
+            this.settings.backup.period === undefined ? 120 : parseInt(this.settings.backup.period);
         if (isNaN(this.settings.backup.period)) {
@@ -170,3 +193,4 @@ this.settings.backup.period = 120;
 
-        this.settings.backup.files = this.settings.backup.files === undefined ? 24 : parseInt(this.settings.backup.files);
+        this.settings.backup.files =
+            this.settings.backup.files === undefined ? 24 : parseInt(this.settings.backup.files);
         if (isNaN(this.settings.backup.files)) {
@@ -176,3 +200,4 @@ this.settings.backup.files = 24;
 
-        this.settings.backup.hours = this.settings.backup.hours === undefined ? 48 : parseInt(this.settings.backup.hours);
+        this.settings.backup.hours =
+            this.settings.backup.hours === undefined ? 48 : parseInt(this.settings.backup.hours);
         if (isNaN(this.settings.backup.hours)) {
@@ -201,7 +226,7 @@ this.settings.backup.hours = 48;
 
-                s.push({pattern: pattern, regex: new RegExp(tools.pattern2RegEx(pattern)), options: options});
+                s.push({ pattern: pattern, regex: new RegExp(tools.pattern2RegEx(pattern)), options: options });
             });
         } else {
             if (!s.find(sub => sub.pattern === pattern)) {
-                s.push({pattern: pattern, regex: new RegExp(tools.pattern2RegEx(pattern)), options: options});
+                s.push({ pattern: pattern, regex: new RegExp(tools.pattern2RegEx(pattern)), options: options });
             }
@@ -242,3 +267,3 @@ }
 
-    deleteOldBackupFiles() {
+    deleteOldBackupFiles(baseFilename) {
         // delete files only if settings.backupNumber is not 0
@@ -249,3 +274,3 @@ let files = fs.readdirSync(this.backupDir);
 
-        files = files.filter(f => f.endsWith(this.settings.fileDB.fileName + '.gz'));
+        files = files.filter(f => f.endsWith(baseFilename + '.gz'));
 
@@ -263,3 +288,5 @@ while (files.length > this.settings.backup.files) {
                 } catch (e) {
-                    this.log.error(`${this.namespace} Cannot delete file "${path.join(this.backupDir, file)}: ${e.message}`);
+                    this.log.error(
+                        `${this.namespace} Cannot delete file "${path.join(this.backupDir, file)}: ${e.message}`
+                    );
                 }
@@ -338,3 +365,3 @@ }
                 try {
-                    await fs.move(this.datasetName, `${this.datasetName}.bak`, {overwrite: true});
+                    await fs.move(this.datasetName, `${this.datasetName}.bak`, { overwrite: true });
                 } catch (e) {
@@ -353,5 +380,7 @@ bakOk = false;
         try {
-            await fs.move(`${this.datasetName}.new`, this.datasetName, {overwrite: true});
+            await fs.move(`${this.datasetName}.new`, this.datasetName, { overwrite: true });
         } catch (e) {
-            this.log.error(`${this.namespace} Cannot move ${this.datasetName}.new to ${this.datasetName}: ${e.message}. Try direct write as fallback`);
+            this.log.error(
+                `${this.namespace} Cannot move ${this.datasetName}.new to ${this.datasetName}: ${e.message}. Try direct write as fallback`
+            );
             try {
@@ -363,6 +392,6 @@ await fs.writeFile(this.datasetName, jsonString);
             }
-
         }
 
-        if (!bakOk) { // it seems the bak File is not successfully there, write current content again
+        if (!bakOk) {
+            // it seems the bak File is not successfully there, write current content again
             try {
@@ -390,3 +419,6 @@ await fs.writeFile(`${this.datasetName}.bak`, jsonString);
             this.lastSave = now;
-            const backFileName = path.join(this.backupDir, this.getTimeStr(now) + '_' + this.settings.fileDB.fileName + '.gz');
+            const backFileName = path.join(
+                this.backupDir,
+                this.getTimeStr(now) + '_' + this.settings.fileDB.fileName + '.gz'
+            );
 
@@ -409,3 +441,3 @@ try {
                     // analyse older files
-                    this.deleteOldBackupFiles();
+                    this.deleteOldBackupFiles(this.settings.fileDB.fileName);
                 }
@@ -419,3 +451,3 @@ } catch (e) {
     getStatus() {
-        return {type: 'file', server: true};
+        return { type: 'file', server: true };
     }
@@ -444,7 +476,10 @@
         // local subscriptions
-        if (this.change && this.callbackSubscriptionClient._subscribe && this.callbackSubscriptionClient._subscribe[type]) {
+        if (
+            this.change &&
+            this.callbackSubscriptionClient._subscribe &&
+            this.callbackSubscriptionClient._subscribe[type]
+        ) {
             for (let j = 0; j < this.callbackSubscriptionClient._subscribe[type].length; j++) {
                 if (this.callbackSubscriptionClient._subscribe[type][j].regex.test(id)) {
-                    setImmediate(() =>
-                        this.change(id, obj));
+                    setImmediate(() => this.change(id, obj));
                     break;
@@ -451,0 +486,0 @@ }

lib/redisHandler.js

 const Resp = require('respjs');
 const { EventEmitter } = require('events');
+const { QUEUED_STR_BUF, OK_STR_BUF } = require('./constants');
 
@@ -26,6 +27,7 @@ /**
 
-        this.socketId = this.logScope + socket.remoteAddress + ':' + socket.remotePort;
+        this.socketId = `${this.logScope + socket.remoteAddress}:${socket.remotePort}`;
         this.initialized = false;
         this.stop = false;
 
+        this.activeMultiCalls = [];
         this.writeQueue = [];
@@ -44,3 +46,3 @@
             if (this.initialized) {
-                this.sendError(null,new Error('PARSER ERROR ' + err)); // TODO
+                this.sendError(null, new Error(`PARSER ERROR ${err}`)); // TODO
             } else {
@@ -55,3 +57,12 @@ this.close();
             if (this.options.enhancedLogging) {
-                this.log.silly(`${this.socketId} New Redis request: ${(data.length > 1024) ? data.toString().replace(/[\r\n]+/g, '').substring(0, 100) + ' -- ' + data.length + ' bytes' : data.toString().replace(/[\r\n]+/g, '')}`);
+                this.log.silly(
+                    `${this.socketId} New Redis request: ${
+                        data.length > 1024
+                            ? `${data
+                                  .toString()
+                                  .replace(/[\r\n]+/g, '')
+                                  .substring(0, 100)} -- ${data.length} bytes`
+                            : data.toString().replace(/[\r\n]+/g, '')
+                    }`
+                );
             }
@@ -93,15 +104,49 @@ this.resp.write(data);
         }
+
+        const t = process.hrtime();
+        const responseId = t[0] * 1e3 + t[1] / 1e6;
+
+        if (this.options.enhancedLogging) {
+            this.log.silly(
+                `${this.socketId} Parser result: id=${responseId}, command=${command}, data=${
+                    JSON.stringify(data).length > 1024
+                        ? `${JSON.stringify(data).substring(0, 100)} -- ${JSON.stringify(data).length} bytes`
+                        : JSON.stringify(data)
+                }`
+            );
+        }
+
+        if (command === 'multi') {
+            if (this.activeMultiCalls.length && !this.activeMultiCalls[0].execCalled) {
+                // should never happen
+                this.log.warn(`${this.socketId} Conflicting multi call`);
+            }
+            this._handleMulti();
+            return;
+        }
+
+        // multi active and exec not called yet
+        if (this.activeMultiCalls.length && !this.activeMultiCalls[0].execCalled && command !== 'exec') {
+            // store all response ids so we know which need to be in the multi call
+            this.activeMultiCalls[0].responseIds.push(responseId);
+            // add it for the correct order will be overwritten with correct response
+            this.activeMultiCalls[0].responseMap.set(responseId, null);
+        } else {
+            // multi response ids should not be pushed - we will answer combined
+            this.writeQueue.push({ id: responseId, data: false });
+        }
+
+        if (command === 'exec') {
+            this._handleExec(responseId);
+            return;
+        }
+
         if (command === 'info') {
             this.initialized = true;
         }
-        const t = process.hrtime();
-        const responseId = (t[0] * 1e3) + (t[1] / 1e6);
-        if (this.options.enhancedLogging) {
-            this.log.silly(`${this.socketId} Parser result: id=${responseId}, command=${command}, data=${(JSON.stringify(data).length > 1024) ? JSON.stringify(data).substring(0, 100) + ' -- ' + JSON.stringify(data).length + ' bytes' : JSON.stringify(data)}`);
-        }
-        this.writeQueue.push({id: responseId, data: false});
+
         if (this.listenerCount(command) !== 0) {
             setImmediate(() => this.emit(command, data, responseId));
         } else {
-            this.sendError(responseId, new Error(command + ' NOT SUPPORTED'));
+            this.sendError(responseId, new Error(`${command} NOT SUPPORTED`));
         }
@@ -119,2 +164,3 @@ }
         let idx = 0;
+
         while (this.writeQueue.length && idx < this.writeQueue.length) {
@@ -135,4 +181,11 @@ // we found the queue entry that matches with the responseId, so store the data so be sent out
                 if (this.options.enhancedLogging) {
-                    this.log.silly(`${this.socketId} Redis response (${response.id}): ${(response.data.length > 1024) ? data.length + ' bytes' : response.data.toString().replace(/[\r\n]+/g, '')}`);
+                    this.log.silly(
+                        `${this.socketId} Redis response (${response.id}): ${
+                            response.data.length > 1024
+                                ? `${data.length} bytes`
+                                : response.data.toString().replace(/[\r\n]+/g, '')
+                        }`
+                    );
                 }
+
                 this._write(response.data);
@@ -184,4 +237,5 @@ // We sended out first queue entry but no further response is ready
             this.log.warn(`${this.socketId} Not able to write ${JSON.stringify(data)}`);
-            data = Resp.encodeError(new Error('INVALID RESPONSE: ' + JSON.stringify(data)));
+            data = Resp.encodeError(new Error(`INVALID RESPONSE: ${JSON.stringify(data)}`));
         }
+
         setImmediate(() => this._sendQueued(responseId, data));
@@ -212,2 +266,9 @@ }
     sendNull(responseId) {
+        for (const i in this.activeMultiCalls) {
+            if (this.activeMultiCalls[i].responseIds.includes(responseId)) {
+                this._handleMultiResponse(responseId, i, Resp.encodeNull());
+                return;
+            }
+        }
+
         this.sendResponse(responseId, Resp.encodeNull());
@@ -221,2 +282,9 @@ }
     sendNullArray(responseId) {
+        for (const i in this.activeMultiCalls) {
+            if (this.activeMultiCalls[i].responseIds.includes(responseId)) {
+                this._handleMultiResponse(responseId, i, Resp.encodeNullArray());
+                return;
+            }
+        }
+
         this.sendResponse(responseId, Resp.encodeNullArray());
@@ -231,2 +299,9 @@ }
     sendString(responseId, str) {
+        for (const i in this.activeMultiCalls) {
+            if (this.activeMultiCalls[i].responseIds.includes(responseId)) {
+                this._handleMultiResponse(responseId, i, Resp.encodeString(str));
+                return;
+            }
+        }
+
         this.sendResponse(responseId, Resp.encodeString(str));
@@ -242,2 +317,10 @@ }
         this.log.warn(`${this.socketId} Error from InMemDB: ${error}`);
+
+        for (const i in this.activeMultiCalls) {
+            if (this.activeMultiCalls[i].responseIds.includes(responseId)) {
+                this._handleMultiResponse(responseId, i, Resp.encodeError(error));
+                return;
+            }
+        }
+
         this.sendResponse(responseId, Resp.encodeError(error));
@@ -252,2 +335,9 @@ }
     sendInteger(responseId, num) {
+        for (const i in this.activeMultiCalls) {
+            if (this.activeMultiCalls[i].responseIds.includes(responseId)) {
+                this._handleMultiResponse(responseId, i, Resp.encodeInteger(num));
+                return;
+            }
+        }
+
         this.sendResponse(responseId, Resp.encodeInteger(num));
@@ -262,2 +352,9 @@ }
     sendBulk(responseId, str) {
+        for (const i in this.activeMultiCalls) {
+            if (this.activeMultiCalls[i].responseIds.includes(responseId)) {
+                this._handleMultiResponse(responseId, i, Resp.encodeBulk(str));
+                return;
+            }
+        }
+
         this.sendResponse(responseId, Resp.encodeBulk(str));
@@ -272,2 +369,9 @@ }
     sendBufBulk(responseId, buf) {
+        for (const i in this.activeMultiCalls) {
+            if (this.activeMultiCalls[i].responseIds.includes(responseId)) {
+                this._handleMultiResponse(responseId, i, Resp.encodeBufBulk(buf));
+                return;
+            }
+        }
+
         this.sendResponse(responseId, Resp.encodeBufBulk(buf));
@@ -300,10 +404,81 @@ }
      * Encode a array values to buffers and send out
-     * @param responseId ID of the response
-     * @param arr Array to send out
+     * @param {number} responseId ID of the response
+     * @param {any[]} arr Array to send out
      */
     sendArray(responseId, arr) {
+        for (const i in this.activeMultiCalls) {
+            if (this.activeMultiCalls[i].responseIds.includes(responseId)) {
+                this._handleMultiResponse(responseId, i, Resp.encodeArray(this.encodeRespArray(arr)));
+                return;
+            }
+        }
+
         this.sendResponse(responseId, Resp.encodeArray(this.encodeRespArray(arr)));
     }
+
+    /**
+     * Handles a 'multi' command
+     *
+     * @private
+     */
+    _handleMulti() {
+        this.activeMultiCalls.unshift({
+            responseIds: [],
+            execCalled: false,
+            responseCount: 0,
+            responseMap: new Map()
+        });
+    }
+
+    /**
+     * Handles an 'exec' command
+     *
+     * @param {number} responseId ID of the response
+     * @private
+     */
+    _handleExec(responseId) {
+        this.activeMultiCalls[0].execId = responseId;
+        this.activeMultiCalls[0].execCalled = true;
+
+        // maybe we have all fullfilled yet
+        if (this.activeMultiCalls[0].responseCount === this.activeMultiCalls[0].responseIds.length) {
+            const multiRespObj = this.activeMultiCalls.shift();
+            this._sendExecResponse(multiRespObj);
+        }
+    }
+
+    /**
+     * Builds up the exec response and sends it
+     * @param {Record<string, any>} multiObj the multi object to send out
+     *
+     * @private
+     */
+    _sendExecResponse(multiObj) {
+        // collect all 'QUEUED' answers
+        const queuedStrArr = new Array(multiObj.responseCount).fill(QUEUED_STR_BUF);
+
+        this._sendQueued(
+            multiObj.execId,
+            Buffer.concat([OK_STR_BUF, ...queuedStrArr, Resp.encodeArray(Array.from(multiObj.responseMap.values()))])
+        );
+    }
+
+    /**
+     * Handles a multi response
+     *
+     * @param {number} responseId ID of the response
+     * @param {number} index index of the multi call
+     * @param {Buffer} buf buffer to include in response
+     * @private
+     */
+    _handleMultiResponse(responseId, index, buf) {
+        this.activeMultiCalls[index].responseMap.set(responseId, buf);
+        this.activeMultiCalls[index].responseCount++;
+        if (this.activeMultiCalls[index].responseCount === this.activeMultiCalls[index].responseIds.length) {
+            const multiRespObj = this.activeMultiCalls.splice(index, 1)[0];
+            this._sendExecResponse(multiRespObj);
+        }
+    }
 }
 
 module.exports = RedisHandler;

lib/tools.js

@@ -21,2 +21,2 @@ const { tools } = require('@iobroker/js-controller-common');
     return module.exports.maybeCallbackWithError(callback, error, ...args);
-};
+};

package.json

 {
   "name": "@iobroker/db-base",
-  "version": "4.0.0-alpha.7-20210908-26fd46db",
+  "version": "4.0.0-alpha.70-20220202-ee665b37",
   "engines": {
@@ -8,6 +8,5 @@ "node": ">=12.0.0"
   "dependencies": {
-    "@iobroker/js-controller-common": "4.0.0-alpha.7-20210908-26fd46db",
+    "@iobroker/js-controller-common": "4.0.0-alpha.70-20220202-ee665b37",
     "deep-clone": "^3.0.3",
     "fs-extra": "^10.0.0",
-    "node.extend": "^2.0.2",
     "respjs": "^4.2.0"
@@ -40,3 +39,3 @@ },
   ],
-  "gitHead": "4f40df93de6713580bf8f03248b775bb8ccf2443"
+  "gitHead": "5ed03c73e9998cb9d1400ad1597a86f2e64f5738"
 }

README.md

@@ -7,3 +7,3 @@ # Base DB classes for ioBroker
 
-Copyright (c) 2014-2020 bluefox <[e-mail]>,
+Copyright (c) 2014-2021 bluefox <[e-mail]>,
 Copyright (c) 2014      hobbyquaker

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

36189

increased by23.33%

Dependency count

4

decreased by-20%

Number of package files

8

increased by14.29%

Lines of code

865

increased by28.53%

Dependency changes

• + Added@alcalzone/pak@0.7.0(transitive)

• + Added@datalust/winston-seq@1.0.2(transitive)

• + Added@iobroker/js-controller-common@4.0.0-alpha.70-20220202-ee665b37(transitive)

• + Addedbuffer-equal-constant-time@1.0.1(transitive)

• + Addedci-info@3.9.0(transitive)

• + Addedecdsa-sig-formatter@1.0.11(transitive)

• + Addedjsonwebtoken@8.5.1(transitive)

• + Addedjwa@1.4.1(transitive)

• + Addedjws@3.2.2(transitive)

• + Addedlodash.includes@4.3.0(transitive)

• + Addedlodash.isboolean@3.0.3(transitive)

• + Addedlodash.isinteger@4.0.4(transitive)

• + Addedlodash.isnumber@3.0.3(transitive)

• + Addedlodash.isplainobject@4.0.6(transitive)

• + Addedlodash.isstring@4.0.1(transitive)

• + Addedlodash.once@4.1.1(transitive)

• + Addednode-forge@1.3.1(transitive)

• + Addedsemver@5.7.2(transitive)

• + Addedseq-logging@1.1.2(transitive)

• - Removednode.extend@^2.0.2

• - Removed@alcalzone/pak@0.6.0(transitive)

• - Removed@iobroker/js-controller-common@4.0.0-alpha.7-20210908-26fd46db(transitive)

• - Removednode-forge@0.10.0(transitive)

• - Removedseq-logging@0.4.6(transitive)

• - Removedwinston-seq-updated@1.0.4(transitive)

• Updated@iobroker/js-controller-common@4.0.0-alpha.70-20220202-ee665b37