errsole-sqlite - npm Package Compare versions

Comparing version 1.0.0 to 1.0.1

lib/index.js

@@ -1,5 +0,46 @@
-const sqlite3 = require('sqlite3');
+/**
+ * @typedef {Object} Log
+ * @property {number} [id]
+ * @property {string} hostname
+ * @property {number} pid
+ * @property {string} source
+ * @property {Date} timestamp
+ * @property {string} level
+ * @property {string} message
+ * @property {string} [meta]
+ */
+
+/**
+ * @typedef {Object} LogFilter
+ * @property {string} [hostname]
+ * @property {number} [pid]
+ * @property {{source: string, level: string}[]} [level_json]
+ * @property {string[]} [sources]
+ * @property {string[]} [levels]
+ * @property {number} [lt_id]
+ * @property {number} [gt_id]
+ * @property {Date} [lte_timestamp]
+ * @property {Date} [gte_timestamp]
+ * @property {number} [limit=100]
+ */
+
+/**
+ * @typedef {Object} Config
+ * @property {number} id
+ * @property {string} key
+ * @property {string} value
+ */
+
+/**
+ * @typedef {Object} User
+ * @property {number} id
+ * @property {string} name
+ * @property {string} email
+ * @property {string} role
+ */
+
 const bcrypt = require('bcryptjs');
+const { EventEmitter } = require('events');
 const cron = require('node-cron');
-const { EventEmitter } = require('events');
+const sqlite3 = require('sqlite3');
 
@@ -13,2 +54,6 @@ class ErrsoleSQLite extends EventEmitter {
 
+    this.pendingLogs = [];
+    this.batchSize = 100;
+    this.flushInterval = 1000;
+
     this.isConnectionInProgress = true;
@@ -26,16 +71,15 @@ this.db = new sqlite3.Database(filename, err => {
     this.emit('ready');
-    cron.schedule('0 * * * *', () => {
-      this.deleteExpiredLogs();
-    });
+    setInterval(() => this.flushLogs(), this.flushInterval);
+    cron.schedule('0 * * * *', () => this.deleteExpiredLogs());
   }
 
   async setCacheSize () {
-    const desiredSize = 8 * 1024; // Desired cache size in pages, where each page is approximately 1.5 KB (12 MB total)
+    const DESIRED_CACHE_SIZE = 8 * 1024; // Desired cache size in pages, where each page is approximately 1.5 KB (12 MB total)
     const currentSize = await this.getCacheSize();
 
-    if (currentSize >= desiredSize) {
+    if (currentSize >= DESIRED_CACHE_SIZE) {
       return Promise.resolve(); // No need to update cache size
     }
 
-    const query = `PRAGMA cache_size = ${desiredSize}`;
+    const query = `PRAGMA cache_size = ${DESIRED_CACHE_SIZE}`;
     return new Promise((resolve, reject) => {
@@ -97,20 +141,8 @@ this.db.run(query, err => {
 
-  /**
-   * Ensures that the Time To Live (TTL) configuration for logs is set.
-   *
-   * @async
-   * @function ensureLogsTTL
-   * @returns {Promise<{}>} - A promise that resolves with an empty object once the TTL configuration is confirmed or updated.
-   */
   async ensureLogsTTL () {
-    const defaultTTL = 30 * 24 * 60 * 60 * 1000; // 30 days in milliseconds
-    try {
-      const configResult = await this.getConfig('logsTTL');
-      if (!configResult.item) {
-        await this.setConfig('logsTTL', defaultTTL.toString());
-      }
-    } catch (err) {
-      console.error(err);
+    const DEFAULT_LOGS_TTL = 30 * 24 * 60 * 60 * 1000; // 30 days in milliseconds
+    const configResult = await this.getConfig('logsTTL');
+    if (!configResult.item) {
+      await this.setConfig('logsTTL', DEFAULT_LOGS_TTL.toString());
     }
-    return {};
   }
@@ -178,11 +210,24 @@
   /**
-   * Adds log entries to the database.
+   * Adds log entries to the pending logs and flushes them if the batch size is reached.
    *
+   * @param {Log[]} logEntries - An array of log entries to be added to the pending logs.
+   * @returns {Object} - An empty object.
+   */
+  postLogs (logEntries) {
+    this.pendingLogs.push(...logEntries);
+    if (this.pendingLogs.length >= this.batchSize) {
+      this.flushLogs();
+    }
+    return {};
+  }
+
+  /**
+   * Flushes pending logs to the database.
+   *
    * @async
-   * @function postLogs
-   * @param {Log[]} logEntries - An array of log entries to be added to the database.
+   * @function flushLogs
    * @returns {Promise<{}>} - A Promise that resolves with an empty object.
    * @throws {Error} - Throws an error if the operation fails.
    */
-  async postLogs (logEntries) {
+  async flushLogs () {
     while (this.isConnectionInProgress) {
@@ -192,3 +237,5 @@ await new Promise(resolve => setTimeout(resolve, 100));
 
-    const values = logEntries.map(logEntry => [
+    const logsToPost = this.pendingLogs.splice(0, this.pendingLogs.length);
+    const values = logsToPost.map(logEntry => [
+      new Date(logEntry.timestamp),
       logEntry.hostname,
@@ -202,10 +249,10 @@ logEntry.pid,
 
-    const placeholders = values.map(() => '(?, ?, ?, ?, ?, ?)').join(',');
+    const placeholders = values.map(() => '(?, ?, ?, ?, ?, ?, ?)').join(', ');
+    const query = `INSERT INTO errsole_logs_v1 (timestamp, hostname, pid, source, level, message, meta) VALUES ${placeholders}`;
 
-    const query = `INSERT INTO errsole_logs_v1 (hostname, pid, source, level, message, meta) VALUES ${placeholders}`;
+    const flatValues = values.reduce((accumulator, Logvalues) => accumulator.concat(Logvalues), []);
 
-    const flattenedValues = values.flat();
-
     return new Promise((resolve, reject) => {
-      this.db.run(query, flattenedValues, () => {
+      this.db.run(query, flatValues, function (err) {
+        if (err) return reject(err);
         resolve({});
@@ -226,6 +273,7 @@ });
   async getLogs (filters = {}) {
+    const DEFAULT_LOGS_LIMIT = 100;
+    filters.limit = filters.limit || DEFAULT_LOGS_LIMIT;
+
     const whereClauses = [];
     const values = [];
-    const defaultLimit = 100;
-    filters.limit = filters.limit || defaultLimit;
     let sortOrder = 'DESC';
@@ -271,3 +319,3 @@ let shouldReverse = true;
         whereClauses.push('timestamp <= ?');
-        values.push(new Date(filters.lte_timestamp).toISOString());
+        values.push(new Date(filters.lte_timestamp));
         sortOrder = 'DESC';
@@ -278,3 +326,3 @@ shouldReverse = true;
         whereClauses.push('timestamp >= ?');
-        values.push(new Date(filters.gte_timestamp).toISOString());
+        values.push(new Date(filters.gte_timestamp));
         sortOrder = 'ASC';
@@ -305,10 +353,15 @@ shouldReverse = false;
    * @param {LogFilter} [filters] - Filters to refine the search.
-   * @returns {Promise<{items: Log[]}>} - A promise that resolves with an object containing an array of log items.
+   * @returns {Promise<{items: Log[], filters: LogFilter[]}>} - A promise that resolves with an object containing an array of log items.
    * @throws {Error} - Throws an error if the operation fails.
    */
   async searchLogs (searchTerms, filters = {}) {
+    const DEFAULT_LOGS_LIMIT = 100;
+    filters.limit = filters.limit || DEFAULT_LOGS_LIMIT;
+
     const whereClauses = searchTerms.map(() => 'message LIKE ?');
     const values = searchTerms.map(term => `%${term}%`);
-    filters.limit = filters.limit || 100;
+    let sortOrder = 'DESC';
+    let shouldReverse = true;
 
+    // Apply filters
     if (filters.hostname) {
@@ -322,13 +375,14 @@ whereClauses.push('hostname = ?');
     }
-    if (filters.sources) {
-      whereClauses.push('source IN (?)');
-      values.push(filters.sources);
+    if (filters.sources && filters.sources.length > 0) {
+      whereClauses.push(`source IN (${filters.sources.map(() => '?').join(', ')})`);
+      values.push(...filters.sources);
     }
-    if (filters.levels) {
-      whereClauses.push('level IN (?)');
-      values.push(filters.levels);
+    if (filters.levels && filters.levels.length > 0) {
+      whereClauses.push(`level IN (${filters.levels.map(() => '?').join(', ')})`);
+      values.push(...filters.levels);
     }
-    if (filters.level_json) {
+    if (filters.level_json && filters.level_json.length > 0) {
+      const levelConditions = filters.level_json.map(() => '(source = ? AND level = ?)');
+      whereClauses.push(`(${levelConditions.join(' OR ')})`);
       filters.level_json.forEach(levelObj => {
-        whereClauses.push('(source = ? AND level = ?)');
         values.push(levelObj.source, levelObj.level);
@@ -340,19 +394,43 @@ });
       values.push(filters.lt_id);
-    } else if (filters.gt_id) {
+      sortOrder = 'DESC';
+      shouldReverse = true;
+    }
+    if (filters.gt_id) {
       whereClauses.push('id > ?');
       values.push(filters.gt_id);
-    } else if (filters.lte_timestamp || filters.gte_timestamp) {
+      sortOrder = 'ASC';
+      shouldReverse = false;
+    }
+    if (filters.lte_timestamp || filters.gte_timestamp) {
       if (filters.lte_timestamp) {
         whereClauses.push('timestamp <= ?');
-        values.push(new Date(filters.lte_timestamp).toISOString());
+        values.push(new Date(filters.lte_timestamp));
+        sortOrder = 'DESC';
+        shouldReverse = true;
       }
       if (filters.gte_timestamp) {
         whereClauses.push('timestamp >= ?');
-        values.push(new Date(filters.gte_timestamp).toISOString());
+        values.push(new Date(filters.gte_timestamp));
+        sortOrder = 'ASC';
+        shouldReverse = false;
       }
+      if (filters.lte_timestamp && !filters.gte_timestamp) {
+        filters.lte_timestamp = new Date(filters.lte_timestamp);
+        const gteTimestamp = new Date(filters.lte_timestamp.getTime() - 24 * 60 * 60 * 1000);
+        whereClauses.push('timestamp >= ?');
+        values.push(gteTimestamp);
+        filters.gte_timestamp = gteTimestamp;
+      }
+      if (filters.gte_timestamp && !filters.lte_timestamp) {
+        filters.gte_timestamp = new Date(filters.gte_timestamp);
+        const lteTimestamp = new Date(filters.gte_timestamp.getTime() + 24 * 60 * 60 * 1000);
+        whereClauses.push('timestamp <= ?');
+        values.push(lteTimestamp);
+        filters.lte_timestamp = lteTimestamp;
+      }
     }
 
     const whereClause = whereClauses.length ? `WHERE ${whereClauses.join(' AND ')}` : '';
-    const query = `SELECT id, hostname, pid, source, timestamp, level, message FROM errsole_logs_v1 ${whereClause} ORDER BY id DESC LIMIT ?`;
-    values.push(filters.limit);
+    const limitClause = `LIMIT ${filters.limit}`;
+    const query = `SELECT * FROM errsole_logs_v1 ${whereClause} ORDER BY id ${sortOrder} ${limitClause}`;
 
@@ -362,3 +440,4 @@ return new Promise((resolve, reject) => {
         if (err) return reject(err);
-        resolve({ items: rows });
+        if (shouldReverse) rows.reverse();
+        resolve({ items: rows, filters });
       });
@@ -399,13 +478,13 @@ });
 
-    const defaultLogsTTL = 30 * 24 * 60 * 60 * 1000; // 30 days in milliseconds
+    const DEFAULT_LOGS_TTL = 30 * 24 * 60 * 60 * 1000; // 30 days in milliseconds
 
     try {
-      let logsTTL = defaultLogsTTL;
+      let logsTTL = DEFAULT_LOGS_TTL;
       const configResult = await this.getConfig('logsTTL');
       if (configResult.item) {
         const parsedTTL = parseInt(configResult.item.value, 10);
-        logsTTL = isNaN(parsedTTL) ? defaultLogsTTL : parsedTTL;
+        logsTTL = isNaN(parsedTTL) ? DEFAULT_LOGS_TTL : parsedTTL;
       }
       let expirationTime = new Date(Date.now() - logsTTL);
-      expirationTime = new Date(expirationTime).toISOString();
+      expirationTime = new Date(expirationTime);
       let deletedRowCount;
@@ -412,0 +491,0 @@ do {

package.json

 {
   "name": "errsole-sqlite",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "SQLite storage plugin for Errsole",
@@ -5,0 +5,0 @@ "main": "lib/index.js",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

26972

increased by10.8%

Lines of code

642

increased by12.63%

No dependency changes