Socket
Socket
Sign inDemoInstall

clickhouse-ts

Package Overview
Dependencies
Maintainers
1
Versions
70
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

clickhouse-ts - npm Package Compare versions

Comparing version 1.11.1 to 2.0.0

lib/jest.config.d.ts

14

CHANGELOG.md

@@ -7,2 +7,16 @@ ### Changelog

#### [v1.11.1](https://github.com/bytadaniel/clickhouse-ts/compare/v1.11.0...v1.11.1)
> 13 October 2022
- import clickcache [`f7e8823`](https://github.com/bytadaniel/clickhouse-ts/commit/f7e88237fef0c0be4a187401bc46fd5dcaa09c3c)
#### [v1.11.0](https://github.com/bytadaniel/clickhouse-ts/compare/v1.10.0...v1.11.0)
> 12 October 2022
- refactor code [`64d1fc9`](https://github.com/bytadaniel/clickhouse-ts/commit/64d1fc9afe68fffd61154a21b72a7ae7d5c780f0)
- up to date all dependencies [`c7721f9`](https://github.com/bytadaniel/clickhouse-ts/commit/c7721f9c18c5662ea31f311d6cdd9605c155ce06)
- release version [`9233a28`](https://github.com/bytadaniel/clickhouse-ts/commit/9233a286c47b36cf48a1ec22ab5e293f60124ce5)
#### [v1.10.0](https://github.com/bytadaniel/clickhouse-ts/compare/v1.9.0...v1.10.0)

@@ -9,0 +23,0 @@

42

lib/src/clickhouse/clickhouse.d.ts

@@ -1,32 +0,36 @@

import { InsertRows, ConnectionConfig, InstanceOptions, InsertQueryOptions, SelectQueryOptions } from './clickhouse.interface';
import { InsertRows, ConnectionConfig, InstanceOptions, QueryOptions } from './interface';
import { HttpClientResponse } from '../httpClient';
/**
* Clickhouse is a simple client for making queries and getting responses
*/
export declare class Clickhouse {
#private;
/**
* Create Clickhouse instance
*
* @param {ConnectionConfig} context
* @param {InstanceOptions} options
*/
constructor(context: ConnectionConfig, options: InstanceOptions);
private getCacheManager;
private formatInsertRows;
private formatInsertValue;
/**
* Make insert query
* Auto validating data
*
* @param query database
* @param rows [{ value1: 'text' }, {value2: number}]
* @param {string} table table name
* @param {InsertRows} rows insert rows in JSON format
* @param {InsertQueryOptions} options insert options
*
* @returns {Promise<number>} insert count
*/
insert(table: string, rows: InsertRows, options?: InsertQueryOptions): Promise<{
inserted: number;
data: InsertRows;
}>;
insert(table: string, rows: InsertRows, options?: QueryOptions): Promise<number>;
/**
* Select query for getting results
* There is no anu wrapper for response. So, you can do what you want with response
*
* @param query WITH now() as time SELECT time;
* @param {string} query WITH now() as time SELECT time;
* @param {SelectQueryOptions} options select options
*
* @returns {Promise<HttpClientResponse>}
*/
query<T>(query: string, options?: SelectQueryOptions): Promise<HttpClientResponse<T>>;
useCaching(): void;
onChunk(onChunkCb: (chunkId: string, table: string, rows: InsertRows) => void): void;
cache(table: string, rows: InsertRows): Promise<{
cached: number;
chunk: string;
}>;
cleanupChunks(): Promise<void>;
query<T>(query: string, options?: QueryOptions): Promise<HttpClientResponse<T>>;
}

@@ -13,127 +13,41 @@ "use strict";

};
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
var _Clickhouse_httpClient, _Clickhouse_options, _Clickhouse_defaultValues, _Clickhouse_cacheManager, _Clickhouse_onChunkCb;
var _Clickhouse_httpClient, _Clickhouse_options;
Object.defineProperty(exports, "__esModule", { value: true });
exports.Clickhouse = void 0;
const sqlstring_1 = __importDefault(require("sqlstring"));
const lodash_1 = require("lodash");
const httpClient_1 = require("../httpClient");
const debug_1 = require("../debug");
const cache_1 = require("../cache");
const errors_1 = require("../errors");
const utils_1 = require("../utils");
/**
* Clickhouse is a simple client for making queries and getting responses
*/
class Clickhouse {
/**
* Create Clickhouse instance
*
* @param {ConnectionConfig} context
* @param {InstanceOptions} options
*/
constructor(context, options) {
_Clickhouse_httpClient.set(this, void 0);
_Clickhouse_options.set(this, void 0);
_Clickhouse_defaultValues.set(this, void 0);
_Clickhouse_cacheManager.set(this, void 0);
_Clickhouse_onChunkCb.set(this, void 0);
__classPrivateFieldSet(this, _Clickhouse_onChunkCb, [], "f");
__classPrivateFieldSet(this, _Clickhouse_options, options, "f");
__classPrivateFieldSet(this, _Clickhouse_defaultValues, options.hooks?.useDefaultValue?.() ?? {}, "f");
__classPrivateFieldSet(this, _Clickhouse_httpClient, new httpClient_1.HttpClient({ context, options: options.clickhouseOptions }), "f");
__classPrivateFieldSet(this, _Clickhouse_cacheManager, this.getCacheManager(__classPrivateFieldGet(this, _Clickhouse_options, "f").cache?.provider ?? 'none'), "f");
if (__classPrivateFieldGet(this, _Clickhouse_options, "f").debug != null) {
debug_1.debug.setDebugMode(__classPrivateFieldGet(this, _Clickhouse_options, "f").debug.mode);
debug_1.debug.excludeDebugProviders(__classPrivateFieldGet(this, _Clickhouse_options, "f").debug.exclude ?? []);
}
__classPrivateFieldSet(this, _Clickhouse_httpClient, new httpClient_1.HttpClient({
context,
options: __classPrivateFieldGet(this, _Clickhouse_options, "f").clickhouseOptions
}), "f");
}
getCacheManager(provider) {
if (provider === 'none')
return;
const options = {
chunkTTLSeconds: __classPrivateFieldGet(this, _Clickhouse_options, "f").cache?.chunkTTLSeconds ?? 60,
chunkExpireTimeSeconds: __classPrivateFieldGet(this, _Clickhouse_options, "f").cache?.chunkExpireTimeSeconds ?? 120,
chunkSizeLimit: __classPrivateFieldGet(this, _Clickhouse_options, "f").cache?.chunkSizeLimit ?? 1000,
chunkResolverIntervalSeconds: __classPrivateFieldGet(this, _Clickhouse_options, "f").cache?.chunkResolverIntervalSeconds ?? 10,
chunkResolveType: __classPrivateFieldGet(this, _Clickhouse_options, "f").cache?.chunkResolveType ?? 'events',
useInsert: async (table, rows) => {
debug_1.debug.log('hook.useInsert', { table, rows });
await this.insert(table, rows);
}
};
if (provider === 'nodejs') {
return new cache_1.ProcessMemoryCache(options);
}
}
formatInsertRows(rows) {
const keysArr = Object.keys(rows[0]);
const valuesSqlArr = rows.map(row => `(${keysArr.map(key => {
try {
// eslint-disable-next-line @typescript-eslint/no-unsafe-return
return this.formatInsertValue(key, row[key], __classPrivateFieldGet(this, _Clickhouse_defaultValues, "f")[key]);
}
catch (e) {
debug_1.debug.log('row.row', row);
throw e;
}
}).join(',')})`);
return {
keysArr,
valuesSqlFormat: valuesSqlArr.join(',')
};
}
formatInsertValue(rowKey, rowValue, defaultValue) {
/**
* Check if column value not exists
*/
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
const fixedRowValue = rowValue === undefined ? defaultValue : rowValue;
if (fixedRowValue === undefined) {
throw new errors_1.PreprocessInsertQueryError(
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
`Cannot find value of column and has not default ${rowKey}:${fixedRowValue}`);
}
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
debug_1.debug.log('row.value', { fixedRowValue });
/**
* is Array
*/
if (Array.isArray(fixedRowValue)) {
// eslint-disable-next-line @typescript-eslint/unbound-method
return `[${fixedRowValue.map(this.formatInsertValue).join(',')}]`;
}
/**
* is Map
*/
if ((0, lodash_1.isObject)(fixedRowValue)) {
const mapValues = Object
.entries(fixedRowValue)
.map(([mapKey, mapValue]) => ([sqlstring_1.default.escape(mapKey), sqlstring_1.default.escape(mapValue)]));
return `map(${mapValues.join(',')})`;
}
/**
* is Number
*/
if (typeof fixedRowValue === 'number') {
return fixedRowValue;
}
/**
* is String
*/
if (typeof fixedRowValue === 'string') {
return sqlstring_1.default.escape(fixedRowValue);
}
/**
* is Null
*/
if ((0, lodash_1.isNull)(fixedRowValue)) {
return sqlstring_1.default.escape('NULL');
}
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
throw new errors_1.PreprocessInsertQueryError(`Unknown type of row [${rowKey}:${fixedRowValue}]`);
}
/**
* Make insert query
* Auto validating data
*
* @param query database
* @param rows [{ value1: 'text' }, {value2: number}]
* @param {string} table table name
* @param {InsertRows} rows insert rows in JSON format
* @param {InsertQueryOptions} options insert options
*
* @returns {Promise<number>} insert count
*/
async insert(table, rows, options = {}) {
if (rows.length === 0) {
return { inserted: rows.length, data: rows };
return 0;
}
const { keysArr, valuesSqlFormat } = this.formatInsertRows(rows);
const { keysArr, valuesSqlFormat } = (0, utils_1.formatInsertRows)(rows);
const keys = keysArr.join(',');

@@ -145,57 +59,27 @@ await __classPrivateFieldGet(this, _Clickhouse_httpClient, "f").request({

});
return { inserted: rows.length, data: rows };
return rows.length;
}
/**
* Select query for getting results
* There is no anu wrapper for response. So, you can do what you want with response
*
* @param query WITH now() as time SELECT time;
* @param {string} query WITH now() as time SELECT time;
* @param {SelectQueryOptions} options select options
*
* @returns {Promise<HttpClientResponse>}
*/
async query(query, options = {}) {
const format = (options.noFormat ?? false)
? ''
: `FORMAT ${options.responseFormat ?? __classPrivateFieldGet(this, _Clickhouse_options, "f").defaultResponseFormat}`;
let format;
const { noFormat = false, responseFormat = __classPrivateFieldGet(this, _Clickhouse_options, "f").defaultResponseFormat } = options;
if (noFormat) {
format = '';
}
else {
format = `FORMAT ${responseFormat}`;
}
const request = `${query} ${format}`;
debug_1.debug.log('ch.query', request);
return await __classPrivateFieldGet(this, _Clickhouse_httpClient, "f").request({ data: request });
}
useCaching() {
if (__classPrivateFieldGet(this, _Clickhouse_cacheManager, "f") == null) {
throw new Error('Cache manager is not initialized!');
}
__classPrivateFieldGet(this, _Clickhouse_cacheManager, "f").on('chunk', (chunkId, table, rows) => {
debug_1.debug.log('ch.useCaching', 'received event \'chunk\'', { chunkId, table, rowsCount: rows.length, firstRow: rows[0] });
__classPrivateFieldGet(this, _Clickhouse_onChunkCb, "f").forEach(cb => cb(chunkId, table, rows));
});
}
onChunk(onChunkCb) {
__classPrivateFieldGet(this, _Clickhouse_onChunkCb, "f").push(onChunkCb);
}
async cache(table, rows) {
if (__classPrivateFieldGet(this, _Clickhouse_cacheManager, "f") == null) {
throw new Error('CacheClient is not implemented');
}
/**
* Simple replacing values if can
*/
const rowsWithDefaults = rows.map(row => Object.fromEntries(Object.entries(row).map(([key, value]) => {
return [
key,
value === undefined ? __classPrivateFieldGet(this, _Clickhouse_defaultValues, "f")[key] : value
];
})));
const hasUndefined = rowsWithDefaults.some(row => Object.values(row).some(value => value === undefined));
if (hasUndefined)
throw new Error('Reject caching rows because some of them is including undefinded and can\'t replace it');
const result = await __classPrivateFieldGet(this, _Clickhouse_cacheManager, "f")
.cache(table, rowsWithDefaults.map(row => JSON.stringify(row)));
return result;
}
async cleanupChunks() {
if (__classPrivateFieldGet(this, _Clickhouse_cacheManager, "f") == null) {
throw new Error('CacheClient is not implemented');
}
await __classPrivateFieldGet(this, _Clickhouse_cacheManager, "f").gracefulShutdown();
}
}
exports.Clickhouse = Clickhouse;
_Clickhouse_httpClient = new WeakMap(), _Clickhouse_options = new WeakMap(), _Clickhouse_defaultValues = new WeakMap(), _Clickhouse_cacheManager = new WeakMap(), _Clickhouse_onChunkCb = new WeakMap();
_Clickhouse_httpClient = new WeakMap(), _Clickhouse_options = new WeakMap();
export * from './clickhouse';
export * as Types from './clickhouse.types';
export * from './clickhouse.interface';
export * from './interface';

@@ -13,21 +13,7 @@ "use strict";

}));
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
Object.defineProperty(o, "default", { enumerable: true, value: v });
}) : function(o, v) {
o["default"] = v;
});
var __exportStar = (this && this.__exportStar) || function(m, exports) {
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
};
var __importStar = (this && this.__importStar) || function (mod) {
if (mod && mod.__esModule) return mod;
var result = {};
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
__setModuleDefault(result, mod);
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.Types = void 0;
__exportStar(require("./clickhouse"), exports);
exports.Types = __importStar(require("./clickhouse.types"));
__exportStar(require("./clickhouse.interface"), exports);
__exportStar(require("./interface"), exports);
import { AxiosError, AxiosResponse } from 'axios';
/**
* ClickhouseHttpError is a custom error for handling Axios Crashes or Clickhouse bad responses
*/
export declare class ClickhouseHttpError extends Error {

@@ -7,3 +10,8 @@ status?: number;

headers?: AxiosResponse['headers'];
/**
* Create ClickhouseHttpError instance
*
* @param {AxiosError} error axios
*/
constructor(error: AxiosError['response']);
}
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.ClickhouseHttpError = void 0;
/**
* ClickhouseHttpError is a custom error for handling Axios Crashes or Clickhouse bad responses
*/
class ClickhouseHttpError extends Error {
/**
* Create ClickhouseHttpError instance
*
* @param {AxiosError} error axios
*/
constructor(error) {

@@ -6,0 +14,0 @@ super();

@@ -0,2 +1,5 @@

/**
* This error means you failed prevalidation of insert request
*/
export declare class PreprocessInsertQueryError extends Error {
}
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.PreprocessInsertQueryError = void 0;
/**
* This error means you failed prevalidation of insert request
*/
class PreprocessInsertQueryError extends Error {
}
exports.PreprocessInsertQueryError = PreprocessInsertQueryError;
import { HttpClientConstructor, HttpClientRequest, HttpClientResponse } from './http-client.interface';
/**
* HttpClient wraps Axios and provides transparent data transfering between your code and clickhouse server
* It uses HTTP/1 protocol
*/
export declare class HttpClient {
#private;
/**
* Create HttpClient instance
*
* @param {HttpClientConstructor} options
*/
constructor({ context, options }: HttpClientConstructor);
/**
* Make full axios request and get full Clickhouse HTTP response
*
* @param {HttpClientRequest} config request config
* @returns {Promise<HttpClientResponse>}
*/
request<T>({ params, data, requestOptions }: HttpClientRequest): Promise<HttpClientResponse<T>>;
}
/// <reference types="node" />
import { AxiosResponse } from 'axios';
import { InsertQueryOptions, SelectQueryOptions } from '../clickhouse';
import { QueryOptions } from '../clickhouse';
export interface HttpClientConstructor {

@@ -18,3 +18,3 @@ context: {

data: string;
requestOptions?: InsertQueryOptions | SelectQueryOptions;
requestOptions?: QueryOptions;
}

@@ -21,0 +21,0 @@ export interface HttpClientResponse<T> {

@@ -23,5 +23,13 @@ "use strict";

const axios_1 = __importDefault(require("axios"));
const debug_1 = require("../debug");
const errors_1 = require("../errors");
/**
* HttpClient wraps Axios and provides transparent data transfering between your code and clickhouse server
* It uses HTTP/1 protocol
*/
class HttpClient {
/**
* Create HttpClient instance
*
* @param {HttpClientConstructor} options
*/
constructor({ context, options = {} }) {

@@ -45,2 +53,8 @@ _HttpClient_axios.set(this, axios_1.default);

}
/**
* Make full axios request and get full Clickhouse HTTP response
*
* @param {HttpClientRequest} config request config
* @returns {Promise<HttpClientResponse>}
*/
async request({ params, data = '', requestOptions = {} }) {

@@ -63,7 +77,5 @@ const config = {

};
debug_1.debug.log('http.request', 'Http request', { config });
const response = await __classPrivateFieldGet(this, _HttpClient_axios, "f")
.request(config)
.catch((error) => {
console.log(error);
throw new errors_1.ClickhouseHttpError(error.response);

@@ -70,0 +82,0 @@ });

{
"name": "clickhouse-ts",
"author": "@daniel_byta Daniel Byta",
"author": "@bytadaniel Daniel Byta",
"main": "lib/index.js",

@@ -10,16 +10,12 @@ "files": [

],
"version": "1.11.1",
"license": "ISC",
"description": "Clickhouse queries with in-memory bulk insert and TypeScript!",
"version": "2.0.0",
"license": "MIT",
"description": "Clickhouse client",
"keywords": [
"clickhouse",
"bulk",
"batch",
"ts",
"insert",
"typescript",
"clickhouse-client",
"clickhouse-ts",
"client",
"clickhouse-client",
"caching",
"database"

@@ -36,5 +32,5 @@ ],

"scripts": {
"build": "rm -rf lib && tsc",
"test": "echo test",
"lint": "eslint 'src/**' --fix"
"build": "tsc -d",
"test": "jest",
"lint": "eslint . --ext .ts"
},

@@ -44,3 +40,3 @@ "dependencies": {

"chvalid": "^0.2.1",
"clickcache": "^1.0.2",
"clickcache": "^1.0.4",
"dayjs": "^1.11.5",

@@ -51,2 +47,3 @@ "lodash": "^4.17.21",

"devDependencies": {
"@types/jest": "^29.1.2",
"@types/lodash": "^4.14.186",

@@ -61,2 +58,4 @@ "@types/node": "^18.8.4",

"eslint-plugin-promise": "^6.0.1",
"jest": "^29.1.2",
"ts-jest": "^29.0.3",
"ts-node": "^10.9.1",

@@ -63,0 +62,0 @@ "typescript": "^4.8.4"

# clickhouse-ts by @bytadaniel
![Travis (.org)](https://img.shields.io/travis/bytadaniel/clickhouse-ts?style=for-the-badge)
![Libraries.io dependency status for GitHub repo](https://img.shields.io/librariesio/github/bytadaniel/clickhouse-ts?style=for-the-badge)
![npms.io (final)](https://img.shields.io/npms-io/final-score/clickhouse-ts?style=for-the-badge)
![GitHub issues](https://img.shields.io/github/issues/bytadaniel/clickhouse-ts?style=for-the-badge)
![Travis (.org)](https://img.shields.io/travis/bytadaniel/clickhouse-ts)
![Libraries.io dependency status for GitHub repo](https://img.shields.io/librariesio/github/bytadaniel/clickhouse-ts)
![npms.io (final)](https://img.shields.io/npms-io/final-score/clickhouse-ts)
![GitHub issues](https://img.shields.io/github/issues/bytadaniel/clickhouse-ts)
[![Join the chat at https://gitter.im/bytadaniel/clickhouse-ts](https://badges.gitter.im/bytadaniel/clickhouse-ts.svg)](https://gitter.im/bytadaniel/clickhouse-ts?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
![GitHub followers](https://img.shields.io/github/followers/bytadaniel?style=social)

@@ -18,3 +20,3 @@

This package is written in TypeScript because Node.js typing development is already an industry standard.
### 🖐 Bulk insert
### 🖐 Batch insert*
It has been empirically verified that in-memory collecting rows is the most efficient and consistent way to insert into Clickhouse. To work with built-in caching, you just need to call the useCaching() method

@@ -34,11 +36,12 @@ ### 💪 Transparent and Stability

## *How can I insert in-memory batches?
Starting from version 2.0.0 [the caching module](https://www.npmjs.com/package/clickcache) should be imported separately.
This is because clickcache package, like clickhouse-ts, is going to be part of my Clickhouse Node.js ecosystem.
In addition, it planned to introduce [data validation](https://www.npmjs.com/package/chvalid), as in Joi and model generation, as in mongodb/mongoose.
## Documentation
## Usage
```js
import { Clickhouse } from 'clickhouse-ts'
import fs from 'fs'
const clickhouseInstance = new Clickhouse(
const client = new Clickhouse(
{

@@ -53,17 +56,2 @@ url: 'url',

{
debug: {
mode: true,
/* List of providers to exclude from logging */
exclude: [...providers]
},
cache: {
/* after this time chunk will be completed */
chunkTTLSeconds: 3600,
/* interval of checking chunks */
chunkResolverIntervalSeconds: 180,
/* count of rows in one chunk */
chunkSizeLimit: 10_000,
/* 'events': on completed chunk emits event 'chunk'. You can save rows as you want */
chunkResolveType: 'events'
},
defaultResponseFormat: 'JSON',

@@ -77,35 +65,13 @@ clickhouseOptions: {

clickhouseInstance.useCaching()
clickhouseInstance.onChunk((chunkId, table, rows) => {
// handle insertion
})
```
## Cache
```js
const response = clickhouseInstance.cache(
'table_strings',
[{ date: '2021-01-01', string: 'str1' }],
{
responseFormat: 'CSVWithNames' // or other format
// other query options
}
)
```
## Insert
```js
const response = await clickhouseInstance.insert(
'table_strings',
[{ date: '2021-01-01', string: 'str1' }],
{
responseFormat: 'CSVWithNames' // or other format
// other query options
}
)
const response = await client.insert('table_strings', rows, {
responseFormat: 'CSVWithNames' // or other format
// other query options
})
```
## Query
## Select
```js

@@ -116,3 +82,6 @@ await clickhouseInstance.query<{ t: string }>('WITH now() as t SELECT t', {

})
```
## Create
```js
await clickhouseInstance.query(`

@@ -126,2 +95,2 @@ CREATE TABLE strings (

`)
```
```

@@ -10,4 +10,4 @@ {

},
"include": ["src", "index.ts"],
"include": ["src", "index.ts", "jest.config.ts", "tests", "tests"],
"exclude": ["node_modules", "**/__tests__/*"]
}

Sorry, the diff of this file is too big to display

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc