dataloader-codegen - npm Package Compare versions

Comparing version 0.3.1 to 0.4.1

lib/codegen.js

@@ -68,2 +68,3 @@ "use strict";
             defaultErrorHandler,
+            getBatchKeysForPartitionItems,
             partitionItems,
@@ -73,2 +74,3 @@ resultsDictToList,
             unPartitionResults,
+            unPartitionResultsByBatchKeyPartition,
         } from '${runtimeHelpers}';
@@ -75,0 +77,0 @@

lib/config.d.ts

@@ -20,2 +20,4 @@ export interface GlobalConfig {
     isBatchKeyASet?: boolean;
+    propertyBatchKey?: string;
+    responseKey?: string;
 }
@@ -22,0 +24,0 @@ export interface NonBatchResourceConfig {

lib/implementation.js

@@ -87,3 +87,4 @@ "use strict";
 function getBatchLoader(resourceConfig, resourcePath) {
-  (0, _assert.default)(resourceConfig.isBatchResource === true, `${(0, _runtimeHelpers.errorPrefix)(resourcePath)} Expected getBatchLoader to be called with a batch resource config`); // The reference at runtime to where the underlying resource lives
+  (0, _assert.default)(resourceConfig.isBatchResource === true, `${(0, _runtimeHelpers.errorPrefix)(resourcePath)} Expected getBatchLoader to be called with a batch resource config`);
+  (0, _assert.default)(typeof resourceConfig.batchKey === 'string' && typeof resourceConfig.newKey === 'string', `${(0, _runtimeHelpers.errorPrefix)(resourcePath)} Expected both batchKey and newKey for a batch resource`); // The reference at runtime to where the underlying resource lives
 
@@ -155,7 +156,7 @@ const resourceReference = ['resources', ...resourcePath].join('.');
              * \`\`\`js
-             * partitionItems([
+             * partitionItems('bar_id', [
              *   { bar_id: 7, include_extra_info: true },
              *   { bar_id: 8, include_extra_info: false },
              *   { bar_id: 9, include_extra_info: true },
-             * ], 'bar_id')
+             * ])
              * \`\`\`
@@ -166,6 +167,30 @@ *
              *
+             * We could also have more than one batch key.
+             *
+             * Example:
+             *
+             * \`\`\`js
+             * partitionItems(['bar_id', 'properties'], [
+             *   { bar_id: 7, properties: ['property_1'], include_extra_info: true },
+             *   { bar_id: 8, properties: ['property_2'], include_extra_info: false },
+             *   { bar_id: 9, properties: ['property_3'], include_extra_info: true },
+             * ])
+             * \`\`\`
+             *
+             * Returns:
+             * \`[ [ 0, 2 ], [ 1 ] ]\`
+             *
              * We'll refer to each element in the group as a "request ID".
              */
-            const requestGroups = partitionItems('${resourceConfig.newKey}', keys);
+            let requestGroups;
 
+            if (${typeof resourceConfig.propertyBatchKey === 'string'}) {
+                requestGroups = partitionItems([
+                    '${resourceConfig.newKey}',
+                    '${resourceConfig.propertyBatchKey}'
+                ], keys);
+            } else {
+                requestGroups = partitionItems('${resourceConfig.newKey}', keys);
+            }
+
             // Map the request groups to a list of Promises - one for each request
@@ -186,3 +211,4 @@ const groupedResults = await Promise.all(requestGroups.map(async requestIDs => {
       newKey,
-      commaSeparatedBatchKey
+      commaSeparatedBatchKey,
+      propertyBatchKey
     } = resourceConfig;
@@ -283,6 +309,13 @@ let batchKeyParam = `['${batchKey}']: requests.map(k => k['${newKey}'])`;
       reorderResultsByKey,
-      isResponseDictionary
+      isResponseDictionary,
+      propertyBatchKey
     } = resourceConfig;
 
-    if (!isResponseDictionary && reorderResultsByKey == null) {
+    if (!isResponseDictionary && reorderResultsByKey == null &&
+    /**
+     * When there's propertyBatchKey and propertyNewKey, the resource might
+     * contain less number of items that we requested. It's valid, so we
+     * should skip the check.
+     */
+    !(typeof propertyBatchKey === 'string')) {
       return `
@@ -399,4 +432,31 @@ if (!(response instanceof Error)) {
 
-            // Split the results back up into the order that they were requested
-            return unPartitionResults(requestGroups, groupedResults);
+            /**
+             *  When there's propertyBatchKey, the resource might contain less number of items that we requested.
+             *  We need the value of batchKey and propertyBatchKey in requests group to help us split the results
+             *  back up into the order that they were requested.
+             */
+            if (${typeof resourceConfig.propertyBatchKey === 'string'}) {
+                const batchKeyPartition = getBatchKeysForPartitionItems(
+                    '${resourceConfig.newKey}',
+                    ['${resourceConfig.newKey}', '${resourceConfig.propertyBatchKey}'],
+                    keys
+                );
+                const propertyBatchKeyPartiion = getBatchKeysForPartitionItems(
+                    '${resourceConfig.propertyBatchKey}',
+                    ['${resourceConfig.newKey}', '${resourceConfig.propertyBatchKey}'],
+                    keys
+                );
+                return unPartitionResultsByBatchKeyPartition(
+                    '${resourceConfig.newKey}',
+                    '${resourceConfig.propertyBatchKey}',
+                    '${resourceConfig.responseKey}',
+                    batchKeyPartition,
+                    propertyBatchKeyPartiion,
+                    requestGroups,
+                    groupedResults
+                );
+            } else {
+                // Split the results back up into the order that they were requested
+                return unPartitionResults(requestGroups, groupedResults);
+            }
          },
@@ -403,0 +463,0 @@ {

lib/runtimeHelpers.d.ts

@@ -37,3 +37,3 @@ /**
  * Take in all objects passed to .load(), and bucket them by the non
- * batchKey attributes.
+ * batch keys (i.e. `batchKey` and `propertyBatchKey`) attributes.
  *
@@ -44,7 +44,7 @@ * We use this to chunk up the requests to the resource.
  * ```js
- * partitionItems([
+ * partitionItems('bar_id', [
  *   { bar_id: 2, include_extra_info: true },
  *   { bar_id: 3, include_extra_info: false },
  *   { bar_id: 4, include_extra_info: true },
- * ], 'bar_id')
+ * ])
  * ```
@@ -57,4 +57,30 @@ *
  */
-export declare function partitionItems(ignoreKey: string, items: ReadonlyArray<object>): ReadonlyArray<ReadonlyArray<number>>;
+export declare function partitionItems(ignoreKeys: Array<string> | string, items: ReadonlyArray<object>): ReadonlyArray<ReadonlyArray<number>>;
 /**
+ * Take in all objects passed to .load(), and bucket them by the non
+ * batch keys (i.e. `batchKey` and `propertyBatchKey`) attributes.
+ * Return batch keys value for each partition items.
+ *
+ * This function is only called when we have propertyBatchKey, and it's
+ * used to map result to the order of requests.
+ *
+ * Example:
+ * ```js
+ * getBatchKeyForPartitionItems(
+ *    'bar_id',
+ *    ['bar_id', 'properties'],
+ *    [
+ *      { bar_id: 2, properties: ['name'], include_extra_info: true },
+ *      { bar_id: 3, properties: ['rating'], include_extra_info: false },
+ *      { bar_id: 2, properties: ['rating'], include_extra_info: true },
+ *    ])
+ * ```
+ *
+ * Returns:
+ * `[ [ 2, 2 ], [ 3 ] ]`
+ *
+ * TODO: add generic instead of 'object' for the items array argument
+ */
+export declare function getBatchKeysForPartitionItems(batchKey: string, ignoreKeys: Array<string>, items: ReadonlyArray<any>): ReadonlyArray<ReadonlyArray<any>>;
+/**
  * Utility function to sort array of objects by a list of corresponding IDs
@@ -124,2 +150,55 @@ *
 /**
+ * Perform the inverse mapping from partitionItems on the nested results we get
+ * back from the service. This function is only called when we have propertyBatchKey.
+ * We currently only support one specific response contract.
+ *
+ * propertyBatchKey is not returned in a nested object, but spread at top level as well.
+ * If we have 'id' as responseKey and 'properties' as propertyBatchKey,
+ * the resultGroups should look like this:
+ * [
+ *      [ { id: 2, name: 'Burger King', rating: 3 } ],
+ *      [ { id: 1, name: 'In N Out', rating: 4 } ]
+ * ],
+ *
+ *
+ * IMPORTANT NOTE: The contract must have a one-to-one correspondence between the input propertyBatchKey and the output propertyBatchKey.
+ * i.e. if we have property: 'name' in the request, the response must have 'name' in it, and no extra data associated with it.
+ *
+ * Example
+ * Request args:
+ * [
+ *      { bar_id: 2, properties: ['name'], include_extra_info: true },
+ *      { bar_id: 1, properties: ['rating'], include_extra_info: false },
+ *      { bar_id: 2, properties: ['rating'], include_extra_info: true },
+ * ]
+ *
+ * ```js
+ * unPartitionResultsByBatchKeyPartition(
+ *   newKey = 'bar_id',
+ *   propertyBatchKey =  'properties',
+ *   responseKey = 'id'
+ *   batchKeyPartition = [ [2, 2], [1] ],
+ *   propertyBatchKeyPartion = [ [['name'], ['rating']], [['rating']] ],
+ *   requestGroups = [ [0, 2], [1] ],
+ *   resultGroups = [
+ *      [ { id: 2, name: 'Burger King', rating: 3 } ],
+ *      [ { id: 1, name: 'In N Out', rating: 4 } ]
+ *   ],
+ * )
+ * ```
+ *
+ * Returns:
+ * ```
+ * [
+ *   { id: 2, name: 'Burger King' }，
+ *   { id: 1, rating: 4 }，
+ *   { id: 2, rating: 3 }，
+ * ]
+ */
+export declare function unPartitionResultsByBatchKeyPartition<T extends Record<string, any>>(newKey: string, propertyBatchKey: string, responseKey: string, batchKeyPartition: ReadonlyArray<ReadonlyArray<any>>, propertyBatchKeyPartion: ReadonlyArray<ReadonlyArray<any>>, 
+/** Should be a nested array of IDs, as generated by partitionItems */
+requestGroups: ReadonlyArray<ReadonlyArray<number>>, 
+/** The results back from the service, in the same shape as groups */
+resultGroups: ReadonlyArray<ReadonlyArray<T | CaughtResourceError>>): ReadonlyArray<T | Error>;
+/**
  * Turn a dictionary of results into an ordered list
@@ -126,0 +205,0 @@ *

lib/runtimeHelpers.js

@@ -8,4 +8,6 @@ "use strict";
 exports.partitionItems = partitionItems;
+exports.getBatchKeysForPartitionItems = getBatchKeysForPartitionItems;
 exports.sortByKeys = sortByKeys;
 exports.unPartitionResults = unPartitionResults;
+exports.unPartitionResultsByBatchKeyPartition = unPartitionResultsByBatchKeyPartition;
 exports.resultsDictToList = resultsDictToList;
@@ -88,3 +90,3 @@ exports.defaultErrorHandler = defaultErrorHandler;
  * Take in all objects passed to .load(), and bucket them by the non
- * batchKey attributes.
+ * batch keys (i.e. `batchKey` and `propertyBatchKey`) attributes.
  *
@@ -95,7 +97,7 @@ * We use this to chunk up the requests to the resource.
  * ```js
- * partitionItems([
+ * partitionItems('bar_id', [
  *   { bar_id: 2, include_extra_info: true },
  *   { bar_id: 3, include_extra_info: false },
  *   { bar_id: 4, include_extra_info: true },
- * ], 'bar_id')
+ * ])
  * ```
@@ -111,6 +113,6 @@ *
 
-function partitionItems(ignoreKey, items) {
+function partitionItems(ignoreKeys, items) {
   const groups = {};
   items.forEach((item, i) => {
-    const hash = (0, _objectHash.default)(_lodash.default.omit(item, ignoreKey), {
+    const hash = (0, _objectHash.default)(_lodash.default.omit(item, ignoreKeys), {
       algorithm: 'passthrough'
@@ -124,2 +126,40 @@ });
 /**
+ * Take in all objects passed to .load(), and bucket them by the non
+ * batch keys (i.e. `batchKey` and `propertyBatchKey`) attributes.
+ * Return batch keys value for each partition items.
+ *
+ * This function is only called when we have propertyBatchKey, and it's
+ * used to map result to the order of requests.
+ *
+ * Example:
+ * ```js
+ * getBatchKeyForPartitionItems(
+ *    'bar_id',
+ *    ['bar_id', 'properties'],
+ *    [
+ *      { bar_id: 2, properties: ['name'], include_extra_info: true },
+ *      { bar_id: 3, properties: ['rating'], include_extra_info: false },
+ *      { bar_id: 2, properties: ['rating'], include_extra_info: true },
+ *    ])
+ * ```
+ *
+ * Returns:
+ * `[ [ 2, 2 ], [ 3 ] ]`
+ *
+ * TODO: add generic instead of 'object' for the items array argument
+ */
+
+
+function getBatchKeysForPartitionItems(batchKey, ignoreKeys, items) {
+  const groups = {};
+  items.forEach((item, i) => {
+    const hash = (0, _objectHash.default)(_lodash.default.omit(item, ignoreKeys), {
+      algorithm: 'passthrough'
+    });
+    groups[hash] = groups[hash] || [];
+    groups[hash].push(items[i][batchKey]);
+  });
+  return Object.values(groups);
+}
+/**
  * Utility function to sort array of objects by a list of corresponding IDs
@@ -264,2 +304,126 @@ *
 /**
+ * Perform the inverse mapping from partitionItems on the nested results we get
+ * back from the service. This function is only called when we have propertyBatchKey.
+ * We currently only support one specific response contract.
+ *
+ * propertyBatchKey is not returned in a nested object, but spread at top level as well.
+ * If we have 'id' as responseKey and 'properties' as propertyBatchKey,
+ * the resultGroups should look like this:
+ * [
+ *      [ { id: 2, name: 'Burger King', rating: 3 } ],
+ *      [ { id: 1, name: 'In N Out', rating: 4 } ]
+ * ],
+ *
+ *
+ * IMPORTANT NOTE: The contract must have a one-to-one correspondence between the input propertyBatchKey and the output propertyBatchKey.
+ * i.e. if we have property: 'name' in the request, the response must have 'name' in it, and no extra data associated with it.
+ *
+ * Example
+ * Request args:
+ * [
+ *      { bar_id: 2, properties: ['name'], include_extra_info: true },
+ *      { bar_id: 1, properties: ['rating'], include_extra_info: false },
+ *      { bar_id: 2, properties: ['rating'], include_extra_info: true },
+ * ]
+ *
+ * ```js
+ * unPartitionResultsByBatchKeyPartition(
+ *   newKey = 'bar_id',
+ *   propertyBatchKey =  'properties',
+ *   responseKey = 'id'
+ *   batchKeyPartition = [ [2, 2], [1] ],
+ *   propertyBatchKeyPartion = [ [['name'], ['rating']], [['rating']] ],
+ *   requestGroups = [ [0, 2], [1] ],
+ *   resultGroups = [
+ *      [ { id: 2, name: 'Burger King', rating: 3 } ],
+ *      [ { id: 1, name: 'In N Out', rating: 4 } ]
+ *   ],
+ * )
+ * ```
+ *
+ * Returns:
+ * ```
+ * [
+ *   { id: 2, name: 'Burger King' }，
+ *   { id: 1, rating: 4 }，
+ *   { id: 2, rating: 3 }，
+ * ]
+ */
+
+
+function unPartitionResultsByBatchKeyPartition(newKey, propertyBatchKey, responseKey, batchKeyPartition, propertyBatchKeyPartion,
+/** Should be a nested array of IDs, as generated by partitionItems */
+requestGroups,
+/** The results back from the service, in the same shape as groups */
+resultGroups) {
+  /**
+   * e.g. with our inputs, produce:
+   * ```js
+   * [
+   *   [
+   *      { order: 0, result: { id: 2, name: 'Burger King' },
+   *      { order: 2, result: { id: 2, rating: 3 } },
+   *   ],
+   *   [
+   *      { order: 1, result: { id: 1, rating: 4 } },
+   *   ]
+   * ]
+   * ```
+   */
+  const zippedGroups = requestGroups.map((ids, i) => {
+    return ids.map((id, j) => {
+      let result = null;
+
+      for (const resultElement of Object.values(resultGroups)[i]) {
+        // There's error in the result we should return
+        if (resultElement instanceof CaughtResourceError) {
+          result = resultElement;
+          break;
+        } // Find the response that matches the requested batchKey
+
+
+        if (resultElement[responseKey] === batchKeyPartition[i][j]) {
+          // Only responseKey and the requested properties will be in the final result.
+          result = _lodash.default.pick(resultElement, [responseKey, ...propertyBatchKeyPartion[i][j]]);
+          break;
+        }
+      } // If requested property doesn't exist in resultElement, we should throw BatchItemNotFoundError error.
+
+
+      if (result === null) {
+        return {
+          order: id,
+          result: new BatchItemNotFoundError([`Could not find ${responseKey} = ${batchKeyPartition[i][j]} in the response dict.`, `Or your endpoint does not follow the contract we support.`, `Please read https://github.com/Yelp/dataloader-codegen/blob/master/API_DOCS.md.`].join(' '))
+        };
+      } else {
+        return {
+          order: id,
+          result
+        };
+      }
+    });
+  });
+  /**
+   * Flatten and sort the groups - e.g.:
+   * ```js
+   * [
+   *   { order: 0, result: { id: 2, name: 'Burger King' } },
+   *   { order: 1, result: { id: 1, rating: 4 } },
+   *   { order: 2, result: { id: 2, rating: 3 } }
+   * ]
+   * ```
+   */
+
+  const sortedResults = _lodash.default.sortBy(_lodash.default.flatten(zippedGroups), ['order']); // Now that we have a sorted array, return the actual results!
+
+
+  return sortedResults.map(r => r.result).map(result => {
+    if (result instanceof CaughtResourceError) {
+      return result.cause;
+    } else {
+      return result;
+    }
+  });
+}
+/**
  * Turn a dictionary of results into an ordered list
@@ -266,0 +430,0 @@ *

package.json

 {
   "name": "dataloader-codegen",
-  "version": "0.3.1",
+  "version": "0.4.1",
   "description": "dataloader-codegen is an opinionated JavaScript library for automatically generating DataLoaders over a set of resources (e.g. HTTP endpoints)",
@@ -5,0 +5,0 @@ "main": "lib/index.js",

schema.json

@@ -100,3 +100,14 @@ {
                     "description": "(Optional) Set to true if the interface of the resource takes the batch key as a set (rather than an array). For example, when using a generated clientlib based on swagger where `uniqueItems: true` is set for the batchKey parameter. Default: false"
+                },
+                "propertyBatchKey": {
+                    "type": "string",
+                    "description": "(Optional) The argument to the resource that represents the optional properties we want to fetch. (e.g. usually 'properties' or 'features')"
+                },
+                "responseKey": {
+                    "type": "string",
+                    "description": "(Non-optional when propertyBatchKey is used) The key in the response objects corresponds to `batchKey`. This should be the only field that are marked as required in your swagger endpoint response, except nestedPath."
                 }
+            },
+            "dependencies": {
+                "propertyBatchKey": { "required": ["responseKey"] }
             }
@@ -103,0 +114,0 @@ },

No alert changes

Improved metrics

Total package byte prevSize

85671

22.12%

Lines of code

1671

22.33%

No dependency changes