clickhouse-ts

clickhouse-ts

This module is used by LookFor.sale which produces WB Analytics

⚠️ Module Using this module might lead to some unhandled errors. Please, report Me about issues you faced!

How to contact me

• [VK] - https://vk.com/bytdnel
• Telegram - @daniel_byta
• Email- [e-mail]

TS client for clickhouse database with using redis caching rows for bulk insertion

import { Clickhouse } from 'clickhouse-ts'
import fs from 'fs'
import dayjs from 'dayjs'
import { serializeError } from 'serialize-error'
import Redis from 'ioredis' // for correct work use only this package


const clickhouseInstance = new Clickhouse(
  {
    url: 'url',
    port: 8443,
    user: 'user',
    password: 'password',
    database: 'database',
    ca: fs.readFileSync('cert.crt')
  },
  {
    cache: { // cache can be undefined
      chunkTTLSeconds: 3600,
      /* after this time chunk will be completed */
      chunkResolverIntervalSeconds: 180,
      /* interval of checking chunks */
      chunkSizeLimit: 10_000,
      /* count of rows in one chunk */
      chunkResolveType: 'events'
      /*
        'events': on completed chunk emits event 'chunk'. You can save rows as you want
        'autoInsert': on completed chunk inserts rows automatically 
      */
    },
    defaultResponseFormat: 'JSON',
    clickhouseOptions: {
      /*
        any clickhouse options
        https://clickhouse.tech/docs/en/operations/settings/settings/
      */
      send_progress_in_http_headers: '1'
    }
  }
)
clickhouseInstance.useCaching('redis', new Redis())

clickhouseInstance.onChunk((chunkId, table, rows) => {
  // do want you want
})

clickhouseInstance
  .query('WITH now() as t SELECT t', {
    responseFormat: 'TSV',
    // ...another query options
  })
  .then(result => result.data.data)
  .catch(e => serializeError(e))

clickhouseInstance.query(`
  CREATE TABLE strings (
    date DateTime('UTC'),
    string String
  ) Engine = ReplacingMergeTree()
  PARTITION BY toMonday(date)
  ORDER BY (date, string)
`)

const response = clickhouseInstance
  .cache(
    'strings',
    [{ date: '2021-01-01', string: 'str1' }, { date: '2021-01-02', string: 'str2' }],
    {
      responseFormat: 'CSVWithNames' // or another format
      // another query options
    }
  )
  .then(response => response)
  .catch(e => serializeError(e))

response 
/*
  {
    cached: 2,
    chunk: 'strings-1624746778066-4ebdfda07dc6e73a73a3c87490b8ebf0'
  }
*/

const response = clickhouseInstance
  .insert(
    'strings',
    [{ date: '2021-01-01', string: 'str1' }, { date: '2021-01-02', string: 'str2' }],
    {
      responseFormat: 'CSVWithNames' // or another format
      // another query options
    }
  )
  .then(response => response)
  .catch(e => serializeError(e))

response 
/*
  {
    inserted: 2,
    data: [...rows]
  }
*/