clickhouse-buffer - npm Package Compare versions

Comparing version 2.0.0 to 2.0.1

lib/index.d.ts

@@ -45,7 +45,6 @@ import { ClickhouseClient } from "@watchdg/clickhouse-client";
     private lastLoadDate;
-    private statRowsInFiles;
+    private statsRowsInFiles;
     private readonly insertStatement;
     readonly clickhouseClient: ClickhouseClient;
     private static fmtRow;
-    private static calcBytes;
     private static maxTimeHandler;
@@ -52,0 +51,0 @@ private static isConditionMet;

lib/index.js

@@ -9,3 +9,2 @@ "use strict";
 const promises_1 = require("fs/promises");
-const process_1 = require("process");
 const clickhouse_client_1 = require("@watchdg/clickhouse-client");
@@ -19,2 +18,3 @@ var clickhouse_client_2 = require("@watchdg/clickhouse-client");
 const encoder_decoder_2 = require("./encoder_decoder");
+const rows_to_files_1 = require("./rows_to_files");
 function removeFiles(paths) {
@@ -36,7 +36,2 @@ return Promise.all(paths.map(path => (0, promises_1.rm)(path, { force: true })));
     }
-    static calcBytes(rows) {
-        return rows.reduce(function (bytes, row) {
-            return bytes + Buffer.byteLength(row);
-        }, 0);
-    }
     static maxTimeHandler(self) {
@@ -50,26 +45,13 @@ const rows = self.resetRows();
         return (self.conditions?.maxTime && (Date.now() - self.lastLoadDate) >= self.conditions.maxTime) ||
-            (self.conditions?.maxRows && self.statRowsInFiles >= self.conditions.maxRows);
+            (self.conditions?.maxRows && self.statsRowsInFiles >= self.conditions.maxRows);
     }
     static async flushToFiles(self, rows, checkConditions = false) {
-        const rowsLength = rows.length;
-        const files = [];
-        const sortKeys = `${Date.now() / 1000 | 0}_${(process_1.hrtime.bigint() % 10000000000n).toString(10)}`;
-        const parts = Math.ceil(rowsLength / self.maxRowsPerFile);
-        for (let part = 0; part < parts; part++) {
-            const numRowsToFile = rows.length >= self.maxRowsPerFile ? self.maxRowsPerFile : rows.length;
-            const rowsToFile = rows.splice(0, numRowsToFile);
-            const numBytesToFile = ClickhouseBuffer.calcBytes(rowsToFile);
-            let dataToFile = rowsToFile.join('\n') + '\n';
-            let filename = `${sortKeys}_p${part}_r${numRowsToFile}_b${numBytesToFile}`;
-            const encoder = (0, encoder_decoder_2.getEncoder)(self.compressedFiles);
-            if (encoder) {
-                encoder.end(dataToFile);
-                dataToFile = encoder;
-                filename += `.${self.compressedFiles}`;
-            }
-            await (0, promises_1.writeFile)(path_1.default.join(self.directoryPath, filename), dataToFile, { mode: self.fsMode });
-            files.push(filename);
-        }
+        const { files, stats } = await (0, rows_to_files_1.rowsToFiles)(rows, {
+            directory: self.directoryPath,
+            fsMode: self.fsMode,
+            maxRowsPerFile: self.maxRowsPerFile,
+            compression: self.compressedFiles
+        });
         self.files.push(...files);
-        self.statRowsInFiles += rowsLength;
+        self.statsRowsInFiles += stats.rows;
         if (checkConditions && ClickhouseBuffer.isConditionMet(self)) {
@@ -131,3 +113,3 @@ const files = self.resetFiles();
         this.lastLoadDate = Date.now();
-        this.statRowsInFiles = 0;
+        this.statsRowsInFiles = 0;
         if (options.maxRowsInMemory) {
@@ -177,3 +159,3 @@ this.maxRowsInMemory = options.maxRowsInMemory;
         this.files = [];
-        this.statRowsInFiles = 0;
+        this.statsRowsInFiles = 0;
         return files;
@@ -210,5 +192,5 @@ }
     rowsInFiles() {
-        return this.statRowsInFiles;
+        return this.statsRowsInFiles;
     }
 }
 exports.ClickhouseBuffer = ClickhouseBuffer;

package.json

 {
   "name": "clickhouse-buffer",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "license": "MIT",
@@ -5,0 +5,0 @@ "author": {

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

19555

increased by7.2%

Number of package files

13

increased by18.18%

Lines of code

433

increased by10.74%

Worsened metrics

Number of low supply chain risk alerts

4

increased by33.33%

No dependency changes