clickhouse-buffer - npm Package Compare versions

Comparing version 1.0.0 to 1.1.0

lib/index.d.ts

@@ -12,2 +12,3 @@ import { ClickhouseClient } from "@watchdg/clickhouse-client";
     maxRowsPerFile?: number;
+    maxFilesPerLoad?: number;
     directoryPath: string;
@@ -24,2 +25,3 @@ fsMode?: number;
     private readonly maxRowsInMemory;
+    private readonly maxFilesPerLoad;
     private rows;
@@ -39,2 +41,3 @@ private readonly files;
     private static loadToDatabase;
+    private static flushToFilesAndLoadToDatabase;
     static prepareDirectoryPath(mainDirectoryPath: string, database: string, table: string, fsMode: number): Promise<string>;
@@ -45,3 +48,5 @@ constructor(options: Options);
     release(): void;
+    fileNamesInMemory(): number;
+    rowsInMemory(): number;
 }
 export {};

lib/index.js

@@ -19,2 +19,3 @@ "use strict";
         this.maxRowsInMemory = 1000;
+        this.maxFilesPerLoad = 100;
         this.rows = [];
@@ -25,12 +26,15 @@ this.files = [];
         this.lastLoadDate = Date.now();
-        if (options?.maxRowsInMemory) {
+        if (options.maxRowsInMemory) {
             this.maxRowsInMemory = options.maxRowsInMemory;
         }
-        if (options?.maxRowsPerFile) {
+        if (options.maxRowsPerFile) {
             this.maxRowsPerFile = options.maxRowsPerFile;
         }
-        if (options?.fsMode) {
+        if (options.maxFilesPerLoad) {
+            this.maxFilesPerLoad = options.maxFilesPerLoad;
+        }
+        if (options.fsMode) {
             this.fsMode = options.fsMode;
         }
-        if (options?.conditions) {
+        if (options.conditions) {
             this.conditions = options.conditions;
@@ -63,15 +67,8 @@ }
     static isConditionMet(self) {
-        return self.conditions?.maxTime && Date.now() - self.lastLoadDate >= self.conditions.maxTime;
+        return self.conditions?.maxTime && (Date.now() - self.lastLoadDate) >= self.conditions.maxTime;
     }
     static maxTimeHandler(self) {
-        const rows = self.rows.splice(0, self.rows.length);
-        ClickhouseBuffer.flushToFiles(self, rows)
-            .finally(function () {
-            const numOfFiles = self.files.length;
-            if (!(numOfFiles > 0)) {
-                return;
-            }
-            const files = self.files.splice(0, numOfFiles);
-            ClickhouseBuffer.loadToDatabase(self, files).finally();
-        });
+        if (ClickhouseBuffer.isConditionMet(self)) {
+            ClickhouseBuffer.flushToFilesAndLoadToDatabase(self).finally();
+        }
     }
@@ -110,2 +107,15 @@ static async flushToFiles(self, rows) {
     }
+    static async flushToFilesAndLoadToDatabase(self) {
+        const rows = self.rows;
+        self.rows = [];
+        ClickhouseBuffer.flushToFiles(self, rows)
+            .then(function () {
+            const numOfFiles = self.files.length >= self.maxFilesPerLoad ? self.maxFilesPerLoad : self.files.length;
+            if (!(numOfFiles > 0)) {
+                return;
+            }
+            const files = self.files.splice(0, numOfFiles);
+            ClickhouseBuffer.loadToDatabase(self, files).finally();
+        });
+    }
     static async prepareDirectoryPath(mainDirectoryPath, database, table, fsMode) {
@@ -137,3 +147,9 @@ const directoryPath = path_1.default.join(mainDirectoryPath, database, table);
     }
+    fileNamesInMemory() {
+        return this.files.length;
+    }
+    rowsInMemory() {
+        return this.rows.length;
+    }
 }
 exports.ClickhouseBuffer = ClickhouseBuffer;

lib/mutex.d.ts

 export declare class Mutex {
-    mutex: Uint8Array;
+    private readonly mutex;
     acquire(): boolean;
     release(): void;
 }

package.json

 {
   "name": "clickhouse-buffer",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "license": "MIT",
@@ -18,2 +18,3 @@ "author": {
     "lint": "yarn eslint . --ext .ts",
+    "test": "yarn jest",
     "build": "Yarn clean && yarn tsc",
@@ -23,12 +24,15 @@ "prepare": "yarn run lint && yarn run build"
   "dependencies": {
-    "@watchdg/clickhouse-client": "^1.2.0"
+    "@watchdg/clickhouse-client": "^1.3.0"
   },
   "devDependencies": {
-    "@types/node": "^17.0.24",
-    "@typescript-eslint/eslint-plugin": "^5.19.0",
-    "@typescript-eslint/parser": "^5.19.0",
-    "eslint": "^8.13.0",
+    "@types/jest": "^27.5.1",
+    "@types/node": "^17.0.35",
+    "@typescript-eslint/eslint-plugin": "^5.26.0",
+    "@typescript-eslint/parser": "^5.26.0",
+    "eslint": "^8.16.0",
+    "jest": "^28.1.0",
     "rimraf": "^3.0.2",
-    "typescript": "^4.6.3"
+    "ts-jest": "^28.0.3",
+    "typescript": "^4.7.2"
   }
 }

README.md

@@ -1,1 +0,44 @@
-# node-clickhouse-buffer
+# clickhouse-buffer
+
+## Install
+
+```shell
+yarn add clickhouse-buffer
+# or
+npm install clickhouse-buffer
+```
+
+## How to use
+
+```typescript
+import {ClickhouseBuffer, DEFAULT_DATABASE} from "clickhouse-buffer";
+
+(async () => {
+    const database = DEFAULT_DATABASE;
+    const table = 'events';
+
+    const clickhouseBuffer = new ClickhouseBuffer({
+        directoryPath: await ClickhouseBuffer.prepareDirectoryPath('buffer', database, table, 0o777),
+        database,
+        table
+    });
+
+    await clickhouseBuffer.clickhouseClient.query(`CREATE TABLE IF NOT EXISTS ${database}.${table} (id UInt16) ENGINE = MergeTree() ORDER BY id`);
+
+    await clickhouseBuffer.loadFilesToDatabase();
+
+    for (let i = 0; i <= 9; i++) {
+        clickhouseBuffer.push([i]);
+    }
+
+    setTimeout(() => {
+        for (let i = 10; i <= 19; i++) {
+            clickhouseBuffer.push([i]);
+        }
+    }, 5000)
+
+    setTimeout(() => {
+        clickhouseBuffer.release();
+    }, 15000);
+})();
+```

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

12898

increased by16.1%

Lines of code

262

increased by8.71%

Number of lines in readme file

44

increased by4300%

Worsened metrics

Dev dependency count

9

increased by50%

Dependency changes

• Updated@watchdg/clickhouse-client@^1.3.0