@cubejs-backend/query-orchestrator - npm Package Compare versions

Comparing version 0.11.0 to 0.11.6

CHANGELOG.md

@@ -6,2 +6,13 @@ # Change Log
 
+## [0.11.6](https://github.com/statsbotco/cubejs-client/compare/v0.11.5...v0.11.6) (2019-10-17)
+
+
+### Bug Fixes
+
+* TypeError: Cannot read property 'table_name' of undefined: Drop orphaned tables implementation drops recent tables in cluster environments ([84ea78a](https://github.com/statsbotco/cubejs-client/commit/84ea78a))
+
+
+
+
+
 # [0.11.0](https://github.com/statsbotco/cubejs-client/compare/v0.10.62...v0.11.0) (2019-10-15)
@@ -8,0 +19,0 @@

orchestrator/LocalCacheDriver.js

@@ -25,4 +25,8 @@ const store = {};
   }
+
+  async keysStartingWith(prefix) {
+    return Object.keys(this.store).filter(k => k.indexOf(prefix) === 0);
+  }
 }
 
 module.exports = LocalCacheDriver;

orchestrator/PreAggregations.js

@@ -370,3 +370,3 @@ const crypto = require('crypto');
   async dropOrphanedTables(client, justCreatedTable) {
-    this.flushUsedTables();
+    await this.preAggregations.addTableUsed(justCreatedTable);
     const actualTables = await client.getTablesQuery(this.preAggregation.preAggregationsSchema);
@@ -380,3 +380,3 @@ const versionEntries = tablesToVersionEntries(this.preAggregation.preAggregationsSchema, actualTables);
     const tablesToSave =
-      Object.keys(this.preAggregations.tablesUsedInQuery)
+      (await this.preAggregations.tablesUsed())
         .concat(versionEntriesToSave.map(v => this.targetTableName(v)))
@@ -390,9 +390,2 @@ .concat([justCreatedTable]);
   }
-
-  flushUsedTables() {
-    this.preAggregations.tablesUsedInQuery = R.filter(
-      timeStamp => new Date().getTime() - timeStamp.getTime() < 10 * 60 * 1000,
-      this.preAggregations.tablesUsedInQuery
-    );
-  }
 }
@@ -408,3 +401,2 @@
     this.refreshErrors = {}; // TODO should be in redis
-    this.tablesUsedInQuery = {}; // TODO should be in redis
     this.cacheDriver = options.cacheAndQueueDriver === 'redis' ?
@@ -416,2 +408,15 @@ new RedisCacheDriver() :
 
+  tablesUsedRedisKey(tableName) {
+    return `SQL_PRE_AGGREGATIONS_${this.redisPrefix}_TABLES_USED_${tableName}`;
+  }
+
+  async addTableUsed(tableName) {
+    return this.cacheDriver.set(this.tablesUsedRedisKey(tableName), true, 600);
+  }
+
+  async tablesUsed() {
+    return (await this.cacheDriver.keysStartingWith(this.tablesUsedRedisKey('')))
+      .map(k => k.replace(this.tablesUsedRedisKey(''), ''));
+  }
+
   loadAllPreAggregationsIfNeeded(queryBody) {
@@ -432,4 +437,4 @@ const preAggregations = queryBody.preAggregations || [];
       );
-      const preAggregationPromise = () => loader.loadPreAggregation().then(tempTableName => {
-        this.tablesUsedInQuery[tempTableName] = new Date();
+      const preAggregationPromise = () => loader.loadPreAggregation().then(async tempTableName => {
+        await this.addTableUsed(tempTableName);
         return [p.tableName, tempTableName];
@@ -436,0 +441,0 @@ });

orchestrator/RedisCacheDriver.js

@@ -20,4 +20,8 @@ const createRedisClient = require('./RedisFactory');
   }
+
+  keysStartingWith(prefix) {
+    return this.redisClient.keysAsync(`${prefix}*`);
+  }
 }
 
 module.exports = RedisCacheDriver;

orchestrator/RedisFactory.js

@@ -18,3 +18,3 @@ const redis = require('redis');
 
-  ['brpop', 'del', 'get', 'hget', 'rpop', 'set', 'zadd', 'zrange', 'zrangebyscore'].forEach(
+  ['brpop', 'del', 'get', 'hget', 'rpop', 'set', 'zadd', 'zrange', 'zrangebyscore', 'keys'].forEach(
     k => {
@@ -21,0 +21,0 @@ client[`${k}Async`] = promisify(client[k]);

package.json

@@ -5,3 +5,3 @@ {
   "author": "Statsbot, Inc.",
-  "version": "0.11.0",
+  "version": "0.11.6",
   "engines": {
@@ -24,3 +24,3 @@ "node": ">=8.11.1"
   "license": "Apache-2.0",
-  "gitHead": "b62d0e18bc793a1182fb2e3d26baf114a9cd848c"
+  "gitHead": "0613d7784955a898fe193137a6cba66c3285244d"
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

82532

increased by0.85%

Lines of code

1666

increased by0.54%

No dependency changes