clickhouse-buffer
Advanced tools
Comparing version 1.9.6 to 1.9.8
@@ -25,3 +25,3 @@ import { ClickhouseClient } from "@watchdg/clickhouse-client"; | ||
} | ||
declare type columnType = string | number | Date | boolean; | ||
type columnType = string | number | Date | boolean; | ||
export declare class ClickhouseBuffer { | ||
@@ -28,0 +28,0 @@ private readonly directoryPath; |
101
lib/index.js
@@ -16,50 +16,6 @@ "use strict"; | ||
const mutex_1 = require("./mutex"); | ||
function removeFiles(paths) { | ||
return Promise.all(paths.map(path => (0, promises_1.rm)(path, { force: true }))); | ||
} | ||
class ClickhouseBuffer { | ||
constructor(options) { | ||
this.fsMode = 0o777; | ||
this.maxRowsPerFile = 1000; | ||
this.maxRowsInMemory = 1000; | ||
this.maxFilesPerLoad = 100; | ||
this.rows = []; | ||
this.files = []; | ||
this.conditions = { maxTime: 5000 }; | ||
this.loadToDatabaseMutex = new mutex_1.Mutex(); | ||
this.lastLoadDate = Date.now(); | ||
this.statRowsInFiles = 0; | ||
if (options.maxRowsInMemory) { | ||
this.maxRowsInMemory = options.maxRowsInMemory; | ||
} | ||
if (options.maxRowsPerFile) { | ||
this.maxRowsPerFile = options.maxRowsPerFile; | ||
} | ||
if (options.maxFilesPerLoad) { | ||
this.maxFilesPerLoad = options.maxFilesPerLoad; | ||
} | ||
if (options.fsMode) { | ||
this.fsMode = options.fsMode; | ||
} | ||
if (options.conditions) { | ||
this.conditions = options.conditions; | ||
} | ||
this.directoryPath = options.directoryPath; | ||
this.clickhouseClient = new clickhouse_client_1.ClickhouseClient(options.clickhouseClient); | ||
if (options.fields) { | ||
this.fields = options.fields; | ||
} | ||
if (options.compressed) { | ||
this.compressed = options.compressed; | ||
} | ||
if (options.compressedFiles) { | ||
this.compressedFiles = options.compressedFiles; | ||
} | ||
this.database = options.database ?? clickhouse_client_1.DEFAULT_DATABASE; | ||
this.table = options.table; | ||
const columns = this.fields ? ' (' + this.fields.map(function (fieldSettings) { | ||
return `"${fieldSettings.name}"`; | ||
}).join(',') + ')' : ''; | ||
this.insertStatement = `INSERT INTO "${this.database}"."${this.table}"${columns} FORMAT JSONCompactEachRow`; | ||
if (this.conditions.maxTime) { | ||
this.maxTimeTimer = setInterval(ClickhouseBuffer.maxTimeHandler, this.conditions.maxTime, this).unref(); | ||
} | ||
} | ||
static fmtRow(row) { | ||
@@ -160,5 +116,3 @@ for (let i = 0, l = row.length; i < l; i++) { | ||
}); | ||
for (const path of paths) { | ||
await (0, promises_1.rm)(path, { force: true }); | ||
} | ||
await removeFiles(paths); | ||
}) | ||
@@ -185,2 +139,49 @@ .finally(function () { | ||
} | ||
constructor(options) { | ||
this.fsMode = 0o777; | ||
this.maxRowsPerFile = 1000; | ||
this.maxRowsInMemory = 1000; | ||
this.maxFilesPerLoad = 100; | ||
this.rows = []; | ||
this.files = []; | ||
this.conditions = { maxTime: 5000 }; | ||
this.loadToDatabaseMutex = new mutex_1.Mutex(); | ||
this.lastLoadDate = Date.now(); | ||
this.statRowsInFiles = 0; | ||
if (options.maxRowsInMemory) { | ||
this.maxRowsInMemory = options.maxRowsInMemory; | ||
} | ||
if (options.maxRowsPerFile) { | ||
this.maxRowsPerFile = options.maxRowsPerFile; | ||
} | ||
if (options.maxFilesPerLoad) { | ||
this.maxFilesPerLoad = options.maxFilesPerLoad; | ||
} | ||
if (options.fsMode) { | ||
this.fsMode = options.fsMode; | ||
} | ||
if (options.conditions) { | ||
this.conditions = options.conditions; | ||
} | ||
this.directoryPath = options.directoryPath; | ||
this.clickhouseClient = new clickhouse_client_1.ClickhouseClient(options.clickhouseClient); | ||
if (options.fields) { | ||
this.fields = options.fields; | ||
} | ||
if (options.compressed) { | ||
this.compressed = options.compressed; | ||
} | ||
if (options.compressedFiles) { | ||
this.compressedFiles = options.compressedFiles; | ||
} | ||
this.database = options.database ?? clickhouse_client_1.DEFAULT_DATABASE; | ||
this.table = options.table; | ||
const columns = this.fields ? ' (' + this.fields.map(function (fieldSettings) { | ||
return `"${fieldSettings.name}"`; | ||
}).join(',') + ')' : ''; | ||
this.insertStatement = `INSERT INTO "${this.database}"."${this.table}"${columns} FORMAT JSONCompactEachRow`; | ||
if (this.conditions.maxTime) { | ||
this.maxTimeTimer = setInterval(ClickhouseBuffer.maxTimeHandler, this.conditions.maxTime, this).unref(); | ||
} | ||
} | ||
resetRows() { | ||
@@ -187,0 +188,0 @@ const rows = this.rows; |
{ | ||
"name": "clickhouse-buffer", | ||
"version": "1.9.6", | ||
"version": "1.9.8", | ||
"license": "MIT", | ||
@@ -23,9 +23,9 @@ "author": { | ||
"dependencies": { | ||
"@watchdg/clickhouse-client": "^1.13.3" | ||
"@watchdg/clickhouse-client": "^1.13.4" | ||
}, | ||
"devDependencies": { | ||
"@types/jest": "^29.2.0", | ||
"@types/node": "^18.11.8", | ||
"@typescript-eslint/eslint-plugin": "^5.41.0", | ||
"@typescript-eslint/parser": "^5.41.0", | ||
"@types/jest": "^29.2.4", | ||
"@types/node": "^18.11.15", | ||
"@typescript-eslint/eslint-plugin": "^5.46.1", | ||
"@typescript-eslint/parser": "^5.46.1", | ||
"eslint": "^8.26.0", | ||
@@ -35,4 +35,4 @@ "jest": "^29.2.2", | ||
"ts-jest": "^29.0.3", | ||
"typescript": "^4.8.4" | ||
"typescript": "^4.9.4" | ||
} | ||
} |
@@ -0,0 +0,0 @@ # clickhouse-buffer |
17422
380