clickhouse-ts - npm Package Compare versions

Comparing version 1.4.2 to 1.4.3

dist/src/caching/NodeJSCacheManager.js

@@ -31,7 +31,7 @@ "use strict";
         _NodeJSCacheManager_checkChunks.set(this, async () => {
-            Debug_1.debug.log('tableChunks_profiling', { tableChunks: __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f") });
+            Debug_1.debug.log('chunksContent', { tableChunks: __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f") });
             for await (const [table, chunkIds] of Object.entries(__classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f"))) {
                 for await (const chunkId of Object.keys(chunkIds)) {
                     const chunkLen = __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f")[table][chunkId].length;
-                    Debug_1.debug.log(`chunk.${table}.${chunkId}`, { chunkLen });
+                    Debug_1.debug.log('chunkInfo', { table, chunkId, chunkLen });
                 }
@@ -42,13 +42,5 @@ }
                 for (const chunkNamespace of Object.keys(chunkNamespaces)) {
-                    Debug_1.debug.log('RedisCacheClientManager.#checkChunk', { table, chunkNamespace });
+                    Debug_1.debug.log('checkChunk', { table, chunkNamespace });
                     const [_chunk_, _table, _id, _strExpiresAtUnix] = chunkNamespace.split(__classPrivateFieldGet(this, _NodeJSCacheManager_splitter, "f"));
                     const expiresAt = Number(_strExpiresAtUnix);
-                    // const chunkNamespaceTTL = await this.#instance!.ttl(chunkNamespace)
-                    // if (chunkNamespaceTTL < 0) {
-                    //   await this.#instance!.expireat(
-                    //     chunkNamespace,
-                    //     now.add(this.#options.chunkExpireTimeSeconds ?? 86400, 's').unix()
-                    //   )
-                    // }
-                    // const chunkFull = await this.#instance!.llen(chunkNamespace) >= this.#options.chunkSizeLimit
                     const chunkTooOld = now.unix() > expiresAt;
@@ -77,5 +69,2 @@ if (chunkTooOld) {
             const now = dayjs_1.default();
-            // if (!this.#tableChunks[table]) {
-            //   this.#tableChunks[table] = []
-            // }
             const id = crypto_1.default
@@ -88,12 +77,8 @@ .createHash('md5')
             __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f")[table][newChunk] = [];
-            Debug_1.debug.log('RedisCacheClientManager.#createChunk', { table, newChunk, tableChunks: __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f") });
+            Debug_1.debug.log('createChunk', { table, newChunk, tableChunks: __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f") });
             return newChunk;
         });
         _NodeJSCacheManager_deleteChunk.set(this, (table, chunk) => {
-            // if (!this.#tableChunks[table]) {
-            // this.#tableChunks[table] = []
-            // }
-            // this.#tableChunks[table] = this.#tableChunks[table].filter(tableChunk => tableChunk !== chunk)
             delete __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f")[table][chunk];
-            Debug_1.debug.log('RedisCacheClientManager.#deleteChunk', { table, chunk, tableChunks: __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f") });
+            Debug_1.debug.log('deleteChunk', { table, chunk, tableChunks: __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f") });
         });
@@ -135,7 +120,3 @@ _NodeJSCacheManager_getChunk.set(this, (table) => {
         __classPrivateFieldGet(this, _NodeJSCacheManager_tableChunks, "f")[table][chunkCandidate].push(...items);
-        // const cached = await this.#instance!.rpush(chunkCandidate, ...items)
-        Debug_1.debug.log('RedisCacheClientManager.cache', {
-            cached: items.length,
-            chunk: chunkCandidate
-        });
+        Debug_1.debug.log('cache', { cached: items.length, chunk: chunkCandidate });
         return {
@@ -142,0 +123,0 @@ cached: items.length,

dist/src/caching/RedisCacheManager.js

@@ -33,7 +33,7 @@ "use strict";
             this.checkInstance();
-            Debug_1.debug.log('tableChunks_profiling', { tableChunks: __classPrivateFieldGet(this, _RedisCacheManager_tableChunks, "f") });
+            Debug_1.debug.log('chunksContent', { tableChunks: __classPrivateFieldGet(this, _RedisCacheManager_tableChunks, "f") });
             for await (const [table, chunkIds] of Object.entries(__classPrivateFieldGet(this, _RedisCacheManager_tableChunks, "f"))) {
                 for await (const chunkId of chunkIds) {
                     const chunkLen = await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").llen(chunkId);
-                    Debug_1.debug.log(`chunk.${table}.${chunkId}`, { chunkLen });
+                    Debug_1.debug.log('chunkInfo', { table, chunkId, chunkLen });
                 }
@@ -44,3 +44,3 @@ }
                 for (const chunkNamespace of chunkNamespaces) {
-                    Debug_1.debug.log('RedisCacheClientManager.#checkChunk', { table, chunkNamespace });
+                    Debug_1.debug.log('checkChunk', { table, chunkNamespace });
                     const [_chunk_, _table, _id, _strExpiresAtUnix] = chunkNamespace.split(__classPrivateFieldGet(this, _RedisCacheManager_splitter, "f"));
@@ -91,3 +91,3 @@ const expiresAt = Number(_strExpiresAtUnix);
             __classPrivateFieldGet(this, _RedisCacheManager_tableChunks, "f")[table].push(newChunk);
-            Debug_1.debug.log('RedisCacheClientManager.#createChunk', { table, newChunk, tableChunks: __classPrivateFieldGet(this, _RedisCacheManager_tableChunks, "f") });
+            Debug_1.debug.log('createChunk', { table, newChunk, tableChunks: __classPrivateFieldGet(this, _RedisCacheManager_tableChunks, "f") });
             return newChunk;
@@ -101,3 +101,3 @@ });
             await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").del(chunk);
-            Debug_1.debug.log('RedisCacheClientManager.#deleteChunk', { table, chunk, tableChunks: __classPrivateFieldGet(this, _RedisCacheManager_tableChunks, "f") });
+            Debug_1.debug.log('deleteChunk', { table, chunk, tableChunks: __classPrivateFieldGet(this, _RedisCacheManager_tableChunks, "f") });
         });
@@ -134,3 +134,3 @@ _RedisCacheManager_getChunk.set(this, (table) => {
         const cachedChunkTables = await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").keys(`chunk${__classPrivateFieldGet(this, _RedisCacheManager_splitter, "f")}*`);
-        Debug_1.debug.log('RedisCacheClientManager.useRedisInstance', { cachedChunkTables });
+        Debug_1.debug.log('useRedisInstance', { cachedChunkTables });
         for await (const chunkTable of cachedChunkTables) {
@@ -153,3 +153,3 @@ const [_, table] = chunkTable.split(__classPrivateFieldGet(this, _RedisCacheManager_splitter, "f"));
         const cached = await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").rpush(chunkCandidate, ...items);
-        Debug_1.debug.log('RedisCacheClientManager.cache', { cached, chunk: chunkCandidate });
+        Debug_1.debug.log('cache', { cached, chunk: chunkCandidate });
         return {
@@ -156,0 +156,0 @@ cached,

dist/src/clickhouse/Clickhouse.d.ts

@@ -0,3 +1,3 @@
+import { Redis } from 'ioredis';
 import { ClickhouseNamespace } from './interface';
-import { Redis } from 'ioredis';
 export declare class Clickhouse {
@@ -4,0 +4,0 @@ #private;

dist/src/clickhouse/Clickhouse.js

@@ -19,7 +19,8 @@ "use strict";
 exports.Clickhouse = void 0;
+const sqlstring_1 = __importDefault(require("sqlstring"));
 const ClickhouseHttpClient_1 = require("../httpClient/ClickhouseHttpClient");
-const sqlstring_1 = __importDefault(require("sqlstring"));
 const RedisCacheManager_1 = require("../caching/RedisCacheManager");
 const Debug_1 = require("../debug/Debug");
 const NodeJSCacheManager_1 = require("../caching/NodeJSCacheManager");
+const PreprocessInsertQueryError_1 = require("../errors/PreprocessInsertQueryError");
 class Clickhouse {
@@ -39,13 +40,35 @@ constructor(context, options) {
         };
-        this.formatInsertValue = (value) => {
-            if (Array.isArray(value)) {
-                return `[${value.map(this.formatInsertValue).join(',')}]`;
+        this.formatInsertValue = (rowValue) => {
+            /**
+             * is Array
+             */
+            if (Array.isArray(rowValue)) {
+                return `[${rowValue.map(this.formatInsertValue).join(',')}]`;
             }
-            if (typeof value === 'object') {
-                const mapValues = Object.entries(value).map((v) => {
-                    return [sqlstring_1.default.escape(v[0]), this.formatInsertValue(v[1])];
+            /**
+             * is Map
+             */
+            if (typeof rowValue === 'object') {
+                const mapValues = Object
+                    .entries(rowValue)
+                    .map(([mapKey, mapValue]) => {
+                    if (typeof mapValue === 'object')
+                        throw new PreprocessInsertQueryError_1.PreprocessInsertQueryError(`Value of map() has denied type, ${mapValue}, ${typeof mapValue}`);
+                    return [sqlstring_1.default.escape(mapKey), sqlstring_1.default.escape(mapValue)];
                 });
                 return `map(${mapValues.join(',')})`;
             }
-            return sqlstring_1.default.escape(value);
+            /**
+             * is Number
+             */
+            if (typeof rowValue === 'number') {
+                return rowValue;
+            }
+            /**
+             * is String
+             */
+            if (typeof rowValue === 'string') {
+                return sqlstring_1.default.escape(rowValue);
+            }
+            throw new PreprocessInsertQueryError_1.PreprocessInsertQueryError('Unknown type of row: ' + rowValue);
         };
@@ -52,0 +75,0 @@ __classPrivateFieldSet(this, _Clickhouse_isFirstInsert, true, "f");

dist/src/clickhouse/interface.d.ts

@@ -14,3 +14,3 @@ /// <reference types="node" />
             mode: boolean;
-            exclude?: ('RedisCacheClientManager.#deleteChunk' | 'RedisCacheClientManager.#checkChunk' | 'RedisCacheClientManager.#createChunk' | 'RedisCacheClientManager.useRedisInstance' | 'RedisCacheClientManager.cache' | 'hooks.useInsert' | 'Clickhouse.insert' | 'Clickhouse.query' | 'Clickhouse.cache' | 'Clickhouse.useCaching' | 'ClickhouseHttpClient.request')[];
+            exclude?: ('deleteChunk' | 'checkChunk' | 'createChunk' | 'chunkProfiler' | 'useRedisInstance' | 'cache' | 'hooks.useInsert' | 'Clickhouse.insert' | 'Clickhouse.query' | 'Clickhouse.cache' | 'Clickhouse.useCaching' | 'ClickhouseHttpClient.request')[];
         };
@@ -35,2 +35,9 @@ clickhouseOptions?: Record<string, string>;
     } & Record<string, any>;
+    namespace Types {
+        type Number = number;
+        type String = string;
+        type Array<T = unknown> = T[];
+        type Map = Record<string, any>;
+    }
+    type FormattedRowType = Types.String | Types.Number;
 }

dist/src/debug/Debug.d.ts

@@ -0,1 +1,2 @@
+export declare type DebugProvider = 'deleteChunk' | 'checkChunk' | 'createChunk' | 'chunksContent' | 'chunkInfo' | 'useRedisInstance' | 'cache' | 'hooks.useInsert' | 'Clickhouse.insert' | 'Clickhouse.query' | 'Clickhouse.cache' | 'Clickhouse.useCaching' | 'ClickhouseHttpClient.request';
 export declare class Debug {
@@ -6,4 +7,4 @@ #private;
     excludeDebugProviders(providers: string[]): void;
-    log(provider: string, ...args: (string | number | Object | Function)[]): void;
+    log(provider: DebugProvider, ...args: (string | number | Object | Function)[]): void;
 }
 export declare const debug: Debug;

package.json

@@ -12,3 +12,3 @@ {
   ],
-  "version": "1.4.2",
+  "version": "1.4.3",
   "license": "ISC",
@@ -43,5 +43,7 @@ "description": "Clickhouse client on TypeScript using redis caching queries",
     "ioredis": "^4.27.6",
+    "lodash": "^4.17.21",
     "sqlstring": "^2.3.2"
   },
   "devDependencies": {
+    "@types/lodash": "^4.14.181",
     "@types/sqlstring": "^2.3.0",
@@ -48,0 +50,0 @@ "ts-node": "^10.0.0",

src/caching/NodeJSCacheManager.ts

@@ -41,3 +41,3 @@ import dayjs from "dayjs";
   readonly #checkChunks = async () => {
-    debug.log('tableChunks_profiling', { tableChunks: this.#tableChunks })
+    debug.log('chunksContent', { tableChunks: this.#tableChunks })
 
@@ -47,3 +47,3 @@ for await (const [table, chunkIds] of Object.entries(this.#tableChunks)) {
         const chunkLen = this.#tableChunks[table][chunkId].length
-        debug.log(`chunk.${table}.${chunkId}`, { chunkLen })
+        debug.log('chunkInfo', { table, chunkId, chunkLen });
       }
@@ -55,23 +55,6 @@ }
       for (const chunkNamespace of Object.keys(chunkNamespaces)) {
-
-        debug.log(
-          'RedisCacheClientManager.#checkChunk',
-          { table, chunkNamespace }
-        )
-
+        debug.log('checkChunk', { table, chunkNamespace })
         const [_chunk_, _table, _id, _strExpiresAtUnix] = chunkNamespace.split(this.#splitter)
         const expiresAt = Number(_strExpiresAtUnix)
-
-        // const chunkNamespaceTTL = await this.#instance!.ttl(chunkNamespace)
-
-        // if (chunkNamespaceTTL < 0) {
-        //   await this.#instance!.expireat(
-        //     chunkNamespace,
-        //     now.add(this.#options.chunkExpireTimeSeconds ?? 86400, 's').unix()
-        //   )
-        // }
-
-        // const chunkFull = await this.#instance!.llen(chunkNamespace) >= this.#options.chunkSizeLimit
         const chunkTooOld = now.unix() > expiresAt
-
         if (chunkTooOld) {
@@ -102,6 +85,2 @@ await this.#resolveChunk(chunkNamespace, table)
 
-    // if (!this.#tableChunks[table]) {
-    //   this.#tableChunks[table] = []
-    // }
-
     const id = crypto
@@ -118,6 +97,3 @@ .createHash('md5')
 
-    debug.log(
-      'RedisCacheClientManager.#createChunk',
-      { table, newChunk, tableChunks: this.#tableChunks }
-    )
+    debug.log('createChunk', { table, newChunk, tableChunks: this.#tableChunks })
 
@@ -128,14 +104,4 @@ return newChunk
   readonly #deleteChunk = (table: string, chunk: string) => {
-    // if (!this.#tableChunks[table]) {
-      // this.#tableChunks[table] = []
-    // }
-
-    // this.#tableChunks[table] = this.#tableChunks[table].filter(tableChunk => tableChunk !== chunk)
-
     delete this.#tableChunks[table][chunk]
-
-    debug.log(
-      'RedisCacheClientManager.#deleteChunk',
-      { table, chunk, tableChunks: this.#tableChunks }
-    )
+    debug.log('deleteChunk', { table, chunk, tableChunks: this.#tableChunks })
   }
@@ -177,12 +143,4 @@
 
-    // const cached = await this.#instance!.rpush(chunkCandidate, ...items)
+    debug.log('cache', { cached: items.length, chunk: chunkCandidate })
 
-    debug.log(
-      'RedisCacheClientManager.cache',
-      {
-        cached: items.length,
-        chunk: chunkCandidate
-      }
-    )
-
     return {
@@ -189,0 +147,0 @@ cached: items.length,

src/caching/RedisCacheManager.ts

@@ -41,3 +41,3 @@ import dayjs from "dayjs";
 
-    debug.log('tableChunks_profiling', { tableChunks: this.#tableChunks })
+    debug.log('chunksContent', { tableChunks: this.#tableChunks })
 
@@ -47,3 +47,3 @@ for await (const [table, chunkIds] of Object.entries(this.#tableChunks)) {
         const chunkLen = await this.#instance!.llen(chunkId)
-        debug.log(`chunk.${table}.${chunkId}`, { chunkLen })
+        debug.log('chunkInfo', { table, chunkId, chunkLen })
       }
@@ -56,6 +56,3 @@ }
         
-        debug.log(
-          'RedisCacheClientManager.#checkChunk',
-          { table, chunkNamespace }
-        )
+        debug.log('checkChunk', { table, chunkNamespace })
 
@@ -118,12 +115,5 @@ const [_chunk_, _table, _id, _strExpiresAtUnix] = chunkNamespace.split(this.#splitter)
     const ttl = now.add(this.#options.chunkTTLSeconds, 'second').unix()
-    
     const newChunk = ['chunk', table, id, ttl].join(this.#splitter)
-    
     this.#tableChunks[table].push(newChunk)
-
-    debug.log(
-      'RedisCacheClientManager.#createChunk',
-      { table, newChunk, tableChunks: this.#tableChunks }
-    )
-
+    debug.log('createChunk', { table, newChunk, tableChunks: this.#tableChunks })
     return newChunk
@@ -140,6 +130,3 @@ }
 
-    debug.log(
-      'RedisCacheClientManager.#deleteChunk',
-      { table, chunk, tableChunks: this.#tableChunks }
-    )
+    debug.log('deleteChunk', { table, chunk, tableChunks: this.#tableChunks })
   }
@@ -178,3 +165,3 @@
 
-    debug.log('RedisCacheClientManager.useRedisInstance', { cachedChunkTables })
+    debug.log('useRedisInstance', { cachedChunkTables })
 
@@ -206,8 +193,3 @@ for await (const chunkTable of cachedChunkTables) {
     const cached = await this.#instance!.rpush(chunkCandidate, ...items)
-
-    debug.log(
-      'RedisCacheClientManager.cache',
-      { cached, chunk: chunkCandidate }
-    )
-
+    debug.log('cache', { cached, chunk: chunkCandidate })
     return {
@@ -214,0 +196,0 @@ cached,

src/clickhouse/Clickhouse.ts

@@ -0,11 +1,14 @@
+import sqlstring from 'sqlstring'
+import { Redis } from 'ioredis'
+import _ from 'lodash'
 import { ClickhouseHttpClient } from '../httpClient/ClickhouseHttpClient'
-import sqlstring from 'sqlstring'
 import { ClickhouseNamespace } from './interface'
-import { Redis } from 'ioredis'
 import { RedisCacheManager } from '../caching/RedisCacheManager'
 import { debug } from '../debug/Debug'
 import { NodeJSCacheManager } from '../caching/NodeJSCacheManager'
+import { PreprocessInsertQueryError } from '../errors/PreprocessInsertQueryError'
 
 
 
+
 export class Clickhouse {
@@ -65,15 +68,38 @@ // DI
 
-  private formatInsertValue = (value: any): string => {
-    if (Array.isArray(value)) {
-      return `[${value.map(this.formatInsertValue).join(',')}]`
+  private formatInsertValue = (rowValue: any): ClickhouseNamespace.FormattedRowType => {
+    /**
+     * is Array
+     */
+    if (Array.isArray(rowValue)) {
+      return `[${rowValue.map(this.formatInsertValue).join(',')}]`
     }
 
-    if (typeof value === 'object') {
-      const mapValues = Object.entries(value).map((v) => {
-        return [sqlstring.escape(v[0]), this.formatInsertValue(v[1])]
-      })
+    /**
+     * is Map
+     */
+    if (typeof rowValue === 'object') {
+      const mapValues = Object
+        .entries(rowValue)
+        .map(([mapKey, mapValue]) => {
+          if (typeof mapValue === 'object') throw new PreprocessInsertQueryError(`Value of map() has denied type, ${mapValue}, ${typeof mapValue}`)
+          return [sqlstring.escape(mapKey), sqlstring.escape(mapValue)]
+        })
       return `map(${mapValues.join(',')})`
     }
 
-    return sqlstring.escape(value)
+    /**
+     * is Number
+     */
+    if (typeof rowValue === 'number') {
+      return rowValue as number
+    }
+
+    /**
+     * is String
+     */
+    if (typeof rowValue === 'string') {
+      return sqlstring.escape(rowValue) as string
+    }
+
+    throw new PreprocessInsertQueryError('Unknown type of row: ' + rowValue)
   }
@@ -80,0 +106,0 @@

src/clickhouse/interface.ts

 import { Redis } from 'ioredis'
-
 export namespace ClickhouseNamespace {
@@ -17,7 +16,8 @@ export type Constructor = {
       exclude?: (
-        'RedisCacheClientManager.#deleteChunk' |
-        'RedisCacheClientManager.#checkChunk' |
-        'RedisCacheClientManager.#createChunk' |
-        'RedisCacheClientManager.useRedisInstance' |
-        'RedisCacheClientManager.cache' |
+        'deleteChunk' |
+        'checkChunk' |
+        'createChunk' |
+        'chunkProfiler' |
+        'useRedisInstance' |
+        'cache' |
         'hooks.useInsert' |
@@ -51,2 +51,11 @@ 'Clickhouse.insert' |
   } & Record<string, any>
+
+  export namespace Types {
+    export type Number = number
+    export type String = string
+    export type Array<T = unknown> = T[]
+    export type Map = Record<string, any>
+  }
+
+  export type FormattedRowType = Types.String | Types.Number
 }

src/debug/Debug.ts

@@ -0,1 +1,17 @@
+
+export type DebugProvider =
+  'deleteChunk' |
+  'checkChunk' |
+  'createChunk' |
+  'chunksContent' |
+  'chunkInfo' |
+  'useRedisInstance' |
+  'cache' |
+  'hooks.useInsert' |
+  'Clickhouse.insert' |
+  'Clickhouse.query' |
+  'Clickhouse.cache' |
+  'Clickhouse.useCaching' |
+  'ClickhouseHttpClient.request'
+
 export class Debug {
@@ -18,3 +34,3 @@ #isDebugMode: boolean
 
-  public log(provider: string, ...args: (string | number | Object | Function)[]) {
+  public log(provider: DebugProvider, ...args: (string | number | Object | Function)[]) {
     if (
@@ -21,0 +37,0 @@ this.#isDebugMode &&

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Worsened metrics

Total package byte prevSize

69663

decreased by-20.72%

Dependency count

7

increased by16.67%

Dev dependency count

4

increased by33.33%

Number of package files

29

decreased by-14.71%

Lines of code

1392

decreased by-18.02%

Dependency changes

• + Addedlodash@^4.17.21

• + Addedlodash@4.17.21(transitive)