clickhouse-ts

clickhouse-ts

This module is used by LookFor.sale which produces WB Analytics

Module ⚠ Using this module can lead to some unhandled errors. Please, report Me about issues you face!

How to contact me

• [VK] - https://vk.com/bytdnel
• TG - @daniel_byta
• Email- [e-mail]

TS client for clickhouse database with using redis/nodejs caching rows for bulk insertion

import { Clickhouse } from 'clickhouse-ts'
import fs from 'fs'
import dayjs from 'dayjs'
import { serializeError } from 'serialize-error'
import Redis from 'ioredis' // for correct work use only this package

const client = new Clickhouse(
  {
    url: 'url',
    port: 8443,
    user: 'user',
    password: 'password',
    database: 'database',
    ca: fs.readFileSync('cert.crt')
  },
  {
    /*
      cacheClient: 'system' | 'redis' | undefined
      if 'system', Node.js will accumulate insert rows filtered by table name and insert by interval or overflowing
      if 'redis' Redis client will do the same
      if nothing package will not use caching
    */
    cache: {
      client: 'redis',
      redis: {
        instance: new Redis(),
        options: {
          ttlMs: dayjs().add(2, 'hours').unix()
        }
      },
      chunkSize: 100_000,
      chunkCollectorIntervalMs: 100_000
    },
    defaultResponseFormat: 'JSON',
    clickhouseOptions: {
      /*
        any clickhouse options
        https://clickhouse.tech/docs/en/operations/settings/settings/
      */
      send_progress_in_http_headers: '1'
    }
  }
)

client
  .query('WITH now() as t SELECT t', {
    responseFormat: 'TSV',
    // ...another query options
  })
  .then(result => result.data.data)
  .catch(e => serializeError(e))

client.query(`
  CREATE TABLE strings (
    date DateTime('UTC'),
    string String
  ) Engine = ReplacingMergeTree()
  PARTITION BY toMonday(date)
  ORDER BY (date, string)
`)

const response = client
  .insert(
    'strings',
    [{ date: '2021-01-01', string: 'str1' }, { date: '2021-01-02', string: 'str2' }],
    {
      responseFormat: 'CSVWithNames' // or another format
      // another query options
    }
  )
  .then(response => response)
  .catch(e => serializeError(e))

response 
/*
  {
    r: 1,
    result: {
      inserted: 2,
      data: [...data],
      chunk: 'strings-1624746778066-4ebdfda07dc6e73a73a3c87490b8ebf0',
      isNew: true
    }
  }
*/