@cubejs-backend/query-orchestrator - npm Package Compare versions

Comparing version 0.13.9 to 0.14.0

CHANGELOG.md

@@ -6,2 +6,19 @@ # Change Log
 
+# [0.14.0](https://github.com/cube-js/cube.js/compare/v0.13.12...v0.14.0) (2020-01-16)
+
+
+### Bug Fixes
+
+* Cannot read property 'requestId' of null ([d087837](https://github.com/cube-js/cube.js/commit/d087837)), closes [#347](https://github.com/cube-js/cube.js/issues/347)
+
+
+### Features
+
+* Scheduled refresh for pre-aggregations ([c87b525](https://github.com/cube-js/cube.js/commit/c87b525))
+* Scheduled Refresh REST API ([472a0c3](https://github.com/cube-js/cube.js/commit/472a0c3))
+
+
+
+
+
 ## [0.13.9](https://github.com/cube-js/cube.js/compare/v0.13.8...v0.13.9) (2020-01-03)
@@ -8,0 +25,0 @@

orchestrator/PreAggregations.js

@@ -112,3 +112,3 @@ const crypto = require('crypto');
 
-  async keyQueryResult(keyQuery) {
+  async keyQueryResult(keyQuery, waitForRenew, priority) {
     if (!this.queryResults[this.queryCache.queryRedisKey(keyQuery)]) {
@@ -120,3 +120,8 @@ this.queryResults[this.queryCache.queryRedisKey(keyQuery)] = await this.queryCache.cacheQueryResult(
         60 * 60,
-        { renewalThreshold: 5 * 60, renewalKey: keyQuery }
+        {
+          renewalThreshold: this.queryCache.options.refreshKeyRenewalThreshold || 2 * 60,
+          renewalKey: keyQuery,
+          waitForRenew,
+          priority
+        }
       );
@@ -270,3 +275,3 @@ }
         });
-        await this.executeInQueue(invalidationKeys, 10, newVersionEntry);
+        await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry);
         return mostRecentTargetTableName();
@@ -279,3 +284,3 @@ } else if (versionEntry.content_version !== newVersionEntry.content_version) {
           });
-          await this.executeInQueue(invalidationKeys, 0, newVersionEntry);
+          await this.executeInQueue(invalidationKeys, this.priority(0), newVersionEntry);
           return mostRecentTargetTableName();
@@ -298,3 +303,3 @@ } else {
       });
-      await this.executeInQueue(invalidationKeys, 10, newVersionEntry);
+      await this.executeInQueue(invalidationKeys, this.priority(10), newVersionEntry);
       return mostRecentTargetTableName();
@@ -305,6 +310,10 @@ }
 
+  priority(defaultValue) {
+    return this.preAggregation.priority != null ? this.preAggregation.priority : defaultValue;
+  }
+
   getInvalidationKeyValues() {
     return Promise.all(
       (this.preAggregation.invalidateKeyQueries || [])
-        .map(keyQuery => this.loadCache.keyQueryResult(keyQuery))
+        .map(keyQuery => this.loadCache.keyQueryResult(keyQuery, this.waitForRenew, this.priority(10)))
     );
@@ -318,3 +327,3 @@ }
     });
-    this.executeInQueue(invalidationKeys, 0, newVersionEntry)
+    this.executeInQueue(invalidationKeys, this.priority(0), newVersionEntry)
       .then(() => {
@@ -321,0 +330,0 @@ delete this.preAggregations.refreshErrors[newVersionEntry.table_name];

orchestrator/QueryOrchestrator.js

@@ -38,2 +38,8 @@ const R = require('ramda');
       .then(async preAggregationsTablesToTempTables => {
+        const usedPreAggregations = R.fromPairs(preAggregationsTablesToTempTables);
+        if (!queryBody.query) {
+          return {
+            usedPreAggregations
+          };
+        }
         const result = await this.queryCache.cachedQueryResult(
@@ -44,3 +50,3 @@ queryBody, preAggregationsTablesToTempTables
           ...result,
-          usedPreAggregations: R.fromPairs(preAggregationsTablesToTempTables)
+          usedPreAggregations
         };
@@ -47,0 +53,0 @@ });

orchestrator/QueryQueue.js

@@ -40,4 +40,4 @@ const R = require('ramda');
       }
-      if (!(priority >= 0 && priority <= 100)) {
-        throw new Error('Priority should be between 0 and 100');
+      if (!(priority >= -10000 && priority <= 10000)) {
+        throw new Error('Priority should be between -10000 and 10000');
       }
@@ -49,3 +49,3 @@ let result = await redisClient.getResult(queryKey);
       const time = new Date().getTime();
-      const keyScore = time + (100 - priority) * 1E14;
+      const keyScore = time + (10000 - priority) * 1E14;
 
@@ -253,4 +253,3 @@ // eslint-disable-next-line no-unused-vars
             queryKey,
-            queuePrefix: this.redisQueuePrefix,
-            requestId: query.requestId
+            queuePrefix: this.redisQueuePrefix
           });
@@ -257,0 +256,0 @@ await redisClient.removeQuery(queryKey);

package.json

@@ -5,3 +5,3 @@ {
   "author": "Statsbot, Inc.",
-  "version": "0.13.9",
+  "version": "0.14.0",
   "repository": {
@@ -29,3 +29,3 @@ "type": "git",
   "license": "Apache-2.0",
-  "gitHead": "7a7a3d206d8e18a64e53716fb46d6ef9f5677313"
+  "gitHead": "d40bf2a4f174f5ef79726c2d9bf18cc1233959f0"
 }

test/QueryQueueTest.js

@@ -91,2 +91,15 @@ /* globals describe,it */
 
+    it('negative priority', async () => {
+      delayCount = 0;
+      const results = [];
+      await Promise.all([
+        queue.executeInQueue('delay', '31', { delay: 100, result: '4' }, -10).then(r => results.push(r)),
+        queue.executeInQueue('delay', '32', { delay: 100, result: '3' }, -9).then(r => results.push(r)),
+        queue.executeInQueue('delay', '33', { delay: 100, result: '2' }, -8).then(r => results.push(r)),
+        queue.executeInQueue('delay', '34', { delay: 100, result: '1' }, -7).then(r => results.push(r))
+      ]);
+
+      should(results).be.eql(['10', '21', '32', '43']);
+    });
+
     it('orphaned', async () => {
@@ -93,0 +106,0 @@ for (let i = 1; i <= 4; i++) {

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

92473

increased by1.75%

Lines of code

1853

increased by1.31%

No dependency changes