clickhouse-ts - npm Package Compare versions

Comparing version 1.3.13 to 1.3.14

dist/src/caching/RedisCacheManager.d.ts

@@ -16,5 +16,5 @@ /// <reference types="node" />
     cache(table: string, items: string[]): Promise<{
+        cached: number;
         chunk: string;
-        cached: number;
     }>;
 }

dist/src/caching/RedisCacheManager.js

@@ -16,3 +16,3 @@ "use strict";
 };
-var _RedisCacheManager_tableChunks, _RedisCacheManager_instance, _RedisCacheManager_chunkResolver, _RedisCacheManager_splitter, _RedisCacheManager_options, _RedisCacheManager_checkChunks, _RedisCacheManager_createChunk, _RedisCacheManager_deleteChunk, _RedisCacheManager_getChunk;
+var _RedisCacheManager_tableChunks, _RedisCacheManager_instance, _RedisCacheManager_chunkResolver, _RedisCacheManager_splitter, _RedisCacheManager_options, _RedisCacheManager_checkChunks, _RedisCacheManager_resolveChunk, _RedisCacheManager_createChunk, _RedisCacheManager_deleteChunk, _RedisCacheManager_getChunk;
 Object.defineProperty(exports, "__esModule", { value: true });
@@ -22,3 +22,3 @@ exports.RedisCacheManager = void 0;
 const stream_1 = require("stream");
-const uuid_1 = require("uuid");
+const crypto_1 = __importDefault(require("crypto"));
 const Debug_1 = require("../debug/Debug");
@@ -31,3 +31,3 @@ class RedisCacheManager extends stream_1.EventEmitter {
         _RedisCacheManager_chunkResolver.set(this, void 0);
-        _RedisCacheManager_splitter.set(this, '&&&');
+        _RedisCacheManager_splitter.set(this, '-');
         _RedisCacheManager_options.set(this, void 0);
@@ -46,17 +46,6 @@ _RedisCacheManager_checkChunks.set(this, async () => {
                     }
-                    const chunkFull = await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").llen(chunkNamespace) >= __classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkSizeLimit;
+                    // const chunkFull = await this.#instance!.llen(chunkNamespace) >= this.#options.chunkSizeLimit
                     const chunkTooOld = now.unix() > expiresAt;
-                    if (chunkFull || chunkTooOld) {
-                        const raw = await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").lrange(chunkNamespace, 0, -1);
-                        const rows = raw.map(str => JSON.parse(str));
-                        if (__classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkResolveType === 'autoInsert') {
-                            await __classPrivateFieldGet(this, _RedisCacheManager_options, "f").useInsert(table, rows);
-                        }
-                        else if (__classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkResolveType === 'events') {
-                            this.emit('chunk', chunkNamespace, table, rows);
-                        }
-                        else {
-                            throw new Error('resolveType is not correct!');
-                        }
-                        __classPrivateFieldGet(this, _RedisCacheManager_deleteChunk, "f").call(this, table, chunkNamespace);
+                    if (chunkTooOld) {
+                        await __classPrivateFieldGet(this, _RedisCacheManager_resolveChunk, "f").call(this, chunkNamespace, table);
                     }
@@ -66,2 +55,16 @@ }
         });
+        _RedisCacheManager_resolveChunk.set(this, async (chunkId, table) => {
+            const raw = await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").lrange(chunkId, 0, -1);
+            const rows = raw.map(str => JSON.parse(str));
+            if (__classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkResolveType === 'autoInsert') {
+                await __classPrivateFieldGet(this, _RedisCacheManager_options, "f").useInsert(table, rows);
+            }
+            else if (__classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkResolveType === 'events') {
+                this.emit('chunk', chunkId, table, rows);
+            }
+            else {
+                throw new Error('resolveType is not correct!');
+            }
+            await __classPrivateFieldGet(this, _RedisCacheManager_deleteChunk, "f").call(this, table, chunkId);
+        });
         this.checkInstance = () => {
@@ -77,3 +80,6 @@ if (!__classPrivateFieldGet(this, _RedisCacheManager_instance, "f")) {
             }
-            const id = uuid_1.v4();
+            const id = crypto_1.default
+                .createHash('md5')
+                .update((Math.random() * 9e5).toString())
+                .digest('hex');
             const ttl = now.add(__classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkTTLSeconds, 'second').unix();
@@ -114,2 +120,5 @@ const newChunk = ['chunk', table, id, ttl].join(__classPrivateFieldGet(this, _RedisCacheManager_splitter, "f"));
     async useRedisInstance(instance) {
+        if (__classPrivateFieldGet(this, _RedisCacheManager_instance, "f")) {
+            return;
+        }
         const intervalTime = 1000 * __classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkResolverIntervalSeconds;
@@ -130,9 +139,20 @@ __classPrivateFieldSet(this, _RedisCacheManager_instance, instance, "f");
         this.checkInstance();
-        const chunk = __classPrivateFieldGet(this, _RedisCacheManager_getChunk, "f").call(this, table);
-        const cached = await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").rpush(chunk, ...items);
-        Debug_1.debug.log('RedisCacheClientManager.cache', { cached, chunk });
-        return { chunk, cached };
+        let chunkCandidate = __classPrivateFieldGet(this, _RedisCacheManager_getChunk, "f").call(this, table);
+        while (await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").llen(chunkCandidate) >= __classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkSizeLimit) {
+            console.log({
+                chunkCandidate,
+                limit: __classPrivateFieldGet(this, _RedisCacheManager_options, "f").chunkSizeLimit
+            });
+            await __classPrivateFieldGet(this, _RedisCacheManager_resolveChunk, "f").call(this, chunkCandidate, table);
+            chunkCandidate = __classPrivateFieldGet(this, _RedisCacheManager_getChunk, "f").call(this, table);
+        }
+        const cached = await __classPrivateFieldGet(this, _RedisCacheManager_instance, "f").rpush(chunkCandidate, ...items);
+        Debug_1.debug.log('RedisCacheClientManager.cache', { cached, chunk: chunkCandidate });
+        return {
+            cached,
+            chunk: chunkCandidate,
+        };
     }
 }
 exports.RedisCacheManager = RedisCacheManager;
-_RedisCacheManager_tableChunks = new WeakMap(), _RedisCacheManager_instance = new WeakMap(), _RedisCacheManager_chunkResolver = new WeakMap(), _RedisCacheManager_splitter = new WeakMap(), _RedisCacheManager_options = new WeakMap(), _RedisCacheManager_checkChunks = new WeakMap(), _RedisCacheManager_createChunk = new WeakMap(), _RedisCacheManager_deleteChunk = new WeakMap(), _RedisCacheManager_getChunk = new WeakMap();
+_RedisCacheManager_tableChunks = new WeakMap(), _RedisCacheManager_instance = new WeakMap(), _RedisCacheManager_chunkResolver = new WeakMap(), _RedisCacheManager_splitter = new WeakMap(), _RedisCacheManager_options = new WeakMap(), _RedisCacheManager_checkChunks = new WeakMap(), _RedisCacheManager_resolveChunk = new WeakMap(), _RedisCacheManager_createChunk = new WeakMap(), _RedisCacheManager_deleteChunk = new WeakMap(), _RedisCacheManager_getChunk = new WeakMap();

dist/src/clickhouse/Clickhouse.d.ts

@@ -30,5 +30,5 @@ import { ClickhouseNamespace } from './interface';
     cache(table: string, rows: ClickhouseNamespace.InsertRows): Promise<{
+        cached: number;
         chunk: string;
-        cached: number;
     }>;
 }

dist/src/clickhouse/Clickhouse.js

@@ -101,4 +101,4 @@ "use strict";
             Debug_1.debug.log('Clickhouse.cache', 'Implementing redis cache instance');
+            __classPrivateFieldSet(this, _Clickhouse_isFirstInsert, false, "f");
             await __classPrivateFieldGet(this, _Clickhouse_cacheManager, "f").useRedisInstance(__classPrivateFieldGet(this, _Clickhouse_redisClient, "f"));
-            __classPrivateFieldSet(this, _Clickhouse_isFirstInsert, false, "f");
         }
@@ -105,0 +105,0 @@ const result = await __classPrivateFieldGet(this, _Clickhouse_cacheManager, "f")

package.json

@@ -8,4 +8,7 @@ {
   "main": "./dist/src/index.js",
-  "files": [ "dist", "src" ],
-  "version": "1.3.13",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "version": "1.3.14",
   "license": "ISC",
@@ -40,4 +43,3 @@ "description": "Clickhouse client on TypeScript using redis caching queries",
     "ioredis": "^4.27.6",
-    "sqlstring": "^2.3.2",
-    "uuid": "^8.3.2"
+    "sqlstring": "^2.3.2"
   },
@@ -44,0 +46,0 @@ "devDependencies": {

src/caching/RedisCacheManager.ts

 import dayjs from "dayjs";
 import { Redis } from "ioredis";
 import { EventEmitter } from "stream";
-import { v4 as uuidv4 } from 'uuid'
+import crypto from 'crypto'
 import { debug } from "../debug/Debug";
@@ -11,3 +11,3 @@
   #chunkResolver?: NodeJS.Timeout
-  readonly #splitter = '&&&'
+  readonly #splitter = '-'
   #options: {
@@ -54,18 +54,7 @@ chunkTTLSeconds: number;
 
-        const chunkFull = await this.#instance!.llen(chunkNamespace) >= this.#options.chunkSizeLimit
+        // const chunkFull = await this.#instance!.llen(chunkNamespace) >= this.#options.chunkSizeLimit
         const chunkTooOld = now.unix() > expiresAt
 
-        if (chunkFull || chunkTooOld) {
-          const raw = await this.#instance!.lrange(chunkNamespace, 0, -1)
-          const rows = raw.map(str => JSON.parse(str))
-  
-          if (this.#options.chunkResolveType === 'autoInsert') {
-            await this.#options.useInsert(table, rows)
-          } else if (this.#options.chunkResolveType === 'events') {
-            this.emit('chunk', chunkNamespace, table, rows)
-          } else {
-            throw new Error('resolveType is not correct!')
-          }
-  
-          this.#deleteChunk(table, chunkNamespace)
+        if (chunkTooOld) {
+          await this.#resolveChunk(chunkNamespace, table)
         }
@@ -76,2 +65,17 @@ }
 
+  readonly #resolveChunk = async (chunkId: string, table: string) => {
+    const raw = await this.#instance!.lrange(chunkId, 0, -1)
+    const rows = raw.map(str => JSON.parse(str))
+
+    if (this.#options.chunkResolveType === 'autoInsert') {
+      await this.#options.useInsert(table, rows)
+    } else if (this.#options.chunkResolveType === 'events') {
+      this.emit('chunk', chunkId, table, rows)
+    } else {
+      throw new Error('resolveType is not correct!')
+    }
+
+    await this.#deleteChunk(table, chunkId)
+  }
+
   readonly checkInstance = () => {
@@ -89,5 +93,8 @@ if (!this.#instance) {
     }
+    
+    const id = crypto
+      .createHash('md5')
+      .update((Math.random() * 9e5).toString())
+      .digest('hex')
 
-    
-    const id = uuidv4()
     const ttl = now.add(this.#options.chunkTTLSeconds, 'second').unix()
@@ -142,4 +149,10 @@
   public async useRedisInstance(instance: Redis) {
+    if (this.#instance) {
+      return
+    }
+
     const intervalTime = 1000 * this.#options.chunkResolverIntervalSeconds
+
     this.#instance = instance
+
     const cachedChunkTables = await this.#instance.keys(`chunk${this.#splitter}*`)
@@ -165,10 +178,26 @@
 
-    const chunk = this.#getChunk(table)
+    let chunkCandidate = this.#getChunk(table)
+    while (
+      await this.#instance!.llen(chunkCandidate) >= this.#options.chunkSizeLimit
+    ) {
+      console.log({
+        chunkCandidate,
+        limit: this.#options.chunkSizeLimit
+      })
+      await this.#resolveChunk(chunkCandidate, table)
+      chunkCandidate = this.#getChunk(table)
+    }
 
-    const cached = await this.#instance!.rpush(chunk, ...items)
+    const cached = await this.#instance!.rpush(chunkCandidate, ...items)
 
-    debug.log('RedisCacheClientManager.cache', { cached, chunk })
+    debug.log(
+      'RedisCacheClientManager.cache',
+      { cached, chunk: chunkCandidate }
+    )
 
-    return { chunk, cached }
+    return {
+      cached,
+      chunk: chunkCandidate,
+    }
   }
 }

src/clickhouse/Clickhouse.ts

@@ -129,4 +129,5 @@ import { ClickhouseHttpClient } from '../httpClient/ClickhouseHttpClient'
       debug.log('Clickhouse.cache', 'Implementing redis cache instance')
+
+      this.#isFirstInsert = false
       await this.#cacheManager.useRedisInstance(this.#redisClient!)
-      this.#isFirstInsert = false
     }
@@ -133,0 +134,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

68002

increased by2.3%

Dependency count

6

decreased by-14.29%

Lines of code

1287

increased by3.62%

Dependency changes

• - Removeduuid@^8.3.2

• - Removeduuid@8.3.2(transitive)