clickhouse-ts

clickhouse-ts

This module is used by LookFor.sale which produces WB Analytics

⚠️ Using this module might lead to some unhandled errors. Please, report Me about issues you faced!

This module is not released yet. Wait for version 2.0.0 to be release.

How to contact me

• [VK] - https://vk.com/bytdnel
• Telegram - @daniel_byta
• Email- [e-mail]

TS client for clickhouse database with using redis caching rows for bulk insertion

import { Clickhouse } from 'clickhouse-ts'
import fs from 'fs'
import dayjs from 'dayjs'
import { serializeError } from 'serialize-error'
import Redis from 'ioredis' // for correct work use only this package


const clickhouseInstance = new Clickhouse(
  {
    url: 'url',
    port: 8443,
    user: 'user',
    password: 'password',
    database: 'database',
    ca: fs.readFileSync('cert.crt')
  },
  {
    /* debug may be undefined */
    debug: {
      mode: true,
      /* List of providers to exclude from logging */
      exclude: [...providers]
    },
    /* cache may be undefined */
    cache: {
      /* after this time chunk will be completed */
      chunkTTLSeconds: 3600,
      /* interval of checking chunks */
      chunkResolverIntervalSeconds: 180,
      /* count of rows in one chunk */
      chunkSizeLimit: 10_000,
      /*
        'events': on completed chunk emits event 'chunk'. You can save rows as you want
        'autoInsert': on completed chunk inserts rows automatically 
      */
      chunkResolveType: 'events'
    },
    defaultResponseFormat: 'JSON',
    /*
      any clickhouse options
      https://clickhouse.tech/docs/en/operations/settings/settings/
    */
    clickhouseOptions: {
      send_progress_in_http_headers: '1'
    }
  }
)

clickhouseInstance.useCaching('redis', new Redis())

clickhouseInstance.onChunk((chunkId, table, rows) => {
  // do want you want
})

Cache

const response = clickhouseInstance
  .cache(
    'strings',
    [
      { date: '2021-01-01', string: 'str1' },
      { date: '2021-01-02', string: 'str2' }
    ],
    {
      responseFormat: 'CSVWithNames' // or another format
      // another query options
    }
  )
  .then(response => response)
  .catch(e => serializeError(e))

response 
/*
  {
    cached: 2,
    chunk: chunkId
  }
*/

Insert

const response = clickhouseInstance
  .insert(
    'strings',
    [
      { date: '2021-01-01', string: 'str1' },
      { date: '2021-01-02', string: 'str2' }
    ],
    {
      responseFormat: 'CSVWithNames' // or another format
      // another query options
    }
  )
  .then(response => response)
  .catch(e => serializeError(e))

response 
/*
  {
    inserted: 2,
    data: [...rows]
  }
*/

Query

clickhouseInstance
  .query('WITH now() as t SELECT t', {
    responseFormat: 'TSV',
    // ...another query options
  })
  .then(result => result.data.data)
  .catch(e => serializeError(e))

clickhouseInstance.query(`
  CREATE TABLE strings (
    date DateTime('UTC'),
    string String
  ) Engine = ReplacingMergeTree()
  PARTITION BY toMonday(date)
  ORDER BY (date, string)
`)