@cubejs-backend/query-orchestrator - npm Package Compare versions

Comparing version 0.14.0 to 0.15.0

CHANGELOG.md

@@ -6,2 +6,14 @@ # Change Log
 
+# [0.15.0](https://github.com/cube-js/cube.js/compare/v0.14.3...v0.15.0) (2020-01-18)
+
+
+### Features
+
+* New refreshKeyRenewalThresholds and foreground renew defaults ([9fb0abb](https://github.com/cube-js/cube.js/commit/9fb0abb))
+* Slow Query Warning and scheduled refresh for cube refresh keys ([8768b0e](https://github.com/cube-js/cube.js/commit/8768b0e))
+
+
+
+
+
 # [0.14.0](https://github.com/cube-js/cube.js/compare/v0.13.12...v0.14.0) (2020-01-16)
@@ -8,0 +20,0 @@

orchestrator/PreAggregations.js

@@ -112,3 +112,3 @@ const crypto = require('crypto');
 
-  async keyQueryResult(keyQuery, waitForRenew, priority) {
+  async keyQueryResult(keyQuery, waitForRenew, priority, renewalThreshold) {
     if (!this.queryResults[this.queryCache.queryRedisKey(keyQuery)]) {
@@ -121,3 +121,6 @@ this.queryResults[this.queryCache.queryRedisKey(keyQuery)] = await this.queryCache.cacheQueryResult(
         {
-          renewalThreshold: this.queryCache.options.refreshKeyRenewalThreshold || 2 * 60,
+          renewalThreshold:
+            this.queryCache.options.refreshKeyRenewalThreshold ||
+            renewalThreshold ||
+            2 * 60,
           renewalKey: keyQuery,
@@ -314,3 +317,10 @@ waitForRenew,
       (this.preAggregation.invalidateKeyQueries || [])
-        .map(keyQuery => this.loadCache.keyQueryResult(keyQuery, this.waitForRenew, this.priority(10)))
+        .map(
+          (keyQuery, i) => this.loadCache.keyQueryResult(
+            keyQuery,
+            this.waitForRenew,
+            this.priority(10),
+            (this.preAggregation.refreshKeyRenewalThresholds || [])[i]
+          )
+        )
     );
@@ -317,0 +327,0 @@ }

orchestrator/QueryCache.js

@@ -40,2 +40,4 @@ const crypto = require('crypto');
     const renewalThreshold = queryBody.cacheKeyQueries && queryBody.cacheKeyQueries.renewalThreshold;
+    const refreshKeyRenewalThresholds = queryBody.cacheKeyQueries &&
+      queryBody.cacheKeyQueries.refreshKeyRenewalThresholds;
 
@@ -58,6 +60,26 @@ const expireSecs = queryBody.expireSecs || 24 * 3600;
         external: queryBody.external,
-        requestId: queryBody.requestId
+        requestId: queryBody.requestId,
+        refreshKeyRenewalThresholds
       });
     }
 
+    if (!this.options.backgroundRenew) {
+      const resultPromise = this.renewQuery(query, values, cacheKeyQueries, expireSecs, cacheKey, renewalThreshold, {
+        external: queryBody.external,
+        requestId: queryBody.requestId,
+        refreshKeyRenewalThresholds,
+        skipRefreshKeyWaitForRenew: true
+      });
+
+      this.startRenewCycle(query, values, cacheKeyQueries, expireSecs, cacheKey, renewalThreshold, {
+        external: queryBody.external,
+        requestId: queryBody.requestId,
+        refreshKeyRenewalThresholds
+      });
+
+      return resultPromise;
+    }
+
+    this.logger('Background fetch', { cacheKey, requestId: queryBody.requestId });
+
     const mainPromise = this.cacheQueryResult(
@@ -78,3 +100,4 @@ query, values,
         external: queryBody.external,
-        requestId: queryBody.requestId
+        requestId: queryBody.requestId,
+        refreshKeyRenewalThresholds
       });
@@ -203,3 +226,3 @@ }
     return Promise.all(
-      cacheKeyQueries.map(q => this.cacheQueryResult(
+      cacheKeyQueries.map((q, i) => this.cacheQueryResult(
         Array.isArray(q) ? q[0] : q,
@@ -210,5 +233,8 @@ Array.isArray(q) ? q[1] : [],
         {
-          renewalThreshold: this.options.refreshKeyRenewalThreshold || 2 * 60,
+          renewalThreshold:
+            this.options.refreshKeyRenewalThreshold ||
+            (options.refreshKeyRenewalThresholds || [])[i] ||
+            2 * 60,
           renewalKey: q,
-          waitForRenew: true,
+          waitForRenew: !options.skipRefreshKeyWaitForRenew,
           requestId: options.requestId
@@ -332,2 +358,14 @@ }
 
+  async resultFromCacheIfExists(queryBody) {
+    const cacheKey = QueryCache.queryCacheKey(queryBody);
+    const cachedValue = await this.cacheDriver.get(this.queryRedisKey(cacheKey));
+    if (cachedValue) {
+      return {
+        data: cachedValue.result,
+        lastRefreshTime: new Date(cachedValue.time)
+      };
+    }
+    return null;
+  }
+
   queryRedisKey(cacheKey) {
@@ -334,0 +372,0 @@ return `SQL_QUERY_RESULT_${this.redisPrefix}_${crypto.createHash('md5').update(JSON.stringify(cacheKey)).digest("hex")}`;

orchestrator/QueryOrchestrator.js

@@ -82,4 +82,8 @@ const R = require('ramda');
   }
+
+  resultFromCacheIfExists(queryBody) {
+    return this.queryCache.resultFromCacheIfExists(queryBody);
+  }
 }
 
 module.exports = QueryOrchestrator;

package.json

@@ -5,3 +5,3 @@ {
   "author": "Statsbot, Inc.",
-  "version": "0.14.0",
+  "version": "0.15.0",
   "repository": {
@@ -29,3 +29,3 @@ "type": "git",
   "license": "Apache-2.0",
-  "gitHead": "d40bf2a4f174f5ef79726c2d9bf18cc1233959f0"
+  "gitHead": "6242b369545c02ed5990327a5c32226209e52943"
 }

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

94504

increased by2.2%

Lines of code

1899

increased by2.48%

No dependency changes