dataloader-codegen
Advanced tools
Comparing version
@@ -68,2 +68,3 @@ "use strict"; | ||
defaultErrorHandler, | ||
getBatchKeysForPartitionItems, | ||
partitionItems, | ||
@@ -73,2 +74,3 @@ resultsDictToList, | ||
unPartitionResults, | ||
unPartitionResultsByBatchKeyPartition, | ||
} from '${runtimeHelpers}'; | ||
@@ -75,0 +77,0 @@ |
@@ -20,2 +20,4 @@ export interface GlobalConfig { | ||
isBatchKeyASet?: boolean; | ||
propertyBatchKey?: string; | ||
responseKey?: string; | ||
} | ||
@@ -22,0 +24,0 @@ export interface NonBatchResourceConfig { |
@@ -87,3 +87,4 @@ "use strict"; | ||
function getBatchLoader(resourceConfig, resourcePath) { | ||
(0, _assert.default)(resourceConfig.isBatchResource === true, `${(0, _runtimeHelpers.errorPrefix)(resourcePath)} Expected getBatchLoader to be called with a batch resource config`); // The reference at runtime to where the underlying resource lives | ||
(0, _assert.default)(resourceConfig.isBatchResource === true, `${(0, _runtimeHelpers.errorPrefix)(resourcePath)} Expected getBatchLoader to be called with a batch resource config`); | ||
(0, _assert.default)(typeof resourceConfig.batchKey === 'string' && typeof resourceConfig.newKey === 'string', `${(0, _runtimeHelpers.errorPrefix)(resourcePath)} Expected both batchKey and newKey for a batch resource`); // The reference at runtime to where the underlying resource lives | ||
@@ -155,7 +156,7 @@ const resourceReference = ['resources', ...resourcePath].join('.'); | ||
* \`\`\`js | ||
* partitionItems([ | ||
* partitionItems('bar_id', [ | ||
* { bar_id: 7, include_extra_info: true }, | ||
* { bar_id: 8, include_extra_info: false }, | ||
* { bar_id: 9, include_extra_info: true }, | ||
* ], 'bar_id') | ||
* ]) | ||
* \`\`\` | ||
@@ -166,6 +167,30 @@ * | ||
* | ||
* We could also have more than one batch key. | ||
* | ||
* Example: | ||
* | ||
* \`\`\`js | ||
* partitionItems(['bar_id', 'properties'], [ | ||
* { bar_id: 7, properties: ['property_1'], include_extra_info: true }, | ||
* { bar_id: 8, properties: ['property_2'], include_extra_info: false }, | ||
* { bar_id: 9, properties: ['property_3'], include_extra_info: true }, | ||
* ]) | ||
* \`\`\` | ||
* | ||
* Returns: | ||
* \`[ [ 0, 2 ], [ 1 ] ]\` | ||
* | ||
* We'll refer to each element in the group as a "request ID". | ||
*/ | ||
const requestGroups = partitionItems('${resourceConfig.newKey}', keys); | ||
let requestGroups; | ||
if (${typeof resourceConfig.propertyBatchKey === 'string'}) { | ||
requestGroups = partitionItems([ | ||
'${resourceConfig.newKey}', | ||
'${resourceConfig.propertyBatchKey}' | ||
], keys); | ||
} else { | ||
requestGroups = partitionItems('${resourceConfig.newKey}', keys); | ||
} | ||
// Map the request groups to a list of Promises - one for each request | ||
@@ -186,3 +211,4 @@ const groupedResults = await Promise.all(requestGroups.map(async requestIDs => { | ||
newKey, | ||
commaSeparatedBatchKey | ||
commaSeparatedBatchKey, | ||
propertyBatchKey | ||
} = resourceConfig; | ||
@@ -283,6 +309,13 @@ let batchKeyParam = `['${batchKey}']: requests.map(k => k['${newKey}'])`; | ||
reorderResultsByKey, | ||
isResponseDictionary | ||
isResponseDictionary, | ||
propertyBatchKey | ||
} = resourceConfig; | ||
if (!isResponseDictionary && reorderResultsByKey == null) { | ||
if (!isResponseDictionary && reorderResultsByKey == null && | ||
/** | ||
* When there's propertyBatchKey and propertyNewKey, the resource might | ||
* contain less number of items that we requested. It's valid, so we | ||
* should skip the check. | ||
*/ | ||
!(typeof propertyBatchKey === 'string')) { | ||
return ` | ||
@@ -399,4 +432,31 @@ if (!(response instanceof Error)) { | ||
// Split the results back up into the order that they were requested | ||
return unPartitionResults(requestGroups, groupedResults); | ||
/** | ||
* When there's propertyBatchKey, the resource might contain less number of items that we requested. | ||
* We need the value of batchKey and propertyBatchKey in requests group to help us split the results | ||
* back up into the order that they were requested. | ||
*/ | ||
if (${typeof resourceConfig.propertyBatchKey === 'string'}) { | ||
const batchKeyPartition = getBatchKeysForPartitionItems( | ||
'${resourceConfig.newKey}', | ||
['${resourceConfig.newKey}', '${resourceConfig.propertyBatchKey}'], | ||
keys | ||
); | ||
const propertyBatchKeyPartiion = getBatchKeysForPartitionItems( | ||
'${resourceConfig.propertyBatchKey}', | ||
['${resourceConfig.newKey}', '${resourceConfig.propertyBatchKey}'], | ||
keys | ||
); | ||
return unPartitionResultsByBatchKeyPartition( | ||
'${resourceConfig.newKey}', | ||
'${resourceConfig.propertyBatchKey}', | ||
'${resourceConfig.responseKey}', | ||
batchKeyPartition, | ||
propertyBatchKeyPartiion, | ||
requestGroups, | ||
groupedResults | ||
); | ||
} else { | ||
// Split the results back up into the order that they were requested | ||
return unPartitionResults(requestGroups, groupedResults); | ||
} | ||
}, | ||
@@ -403,0 +463,0 @@ { |
@@ -37,3 +37,3 @@ /** | ||
* Take in all objects passed to .load(), and bucket them by the non | ||
* batchKey attributes. | ||
* batch keys (i.e. `batchKey` and `propertyBatchKey`) attributes. | ||
* | ||
@@ -44,7 +44,7 @@ * We use this to chunk up the requests to the resource. | ||
* ```js | ||
* partitionItems([ | ||
* partitionItems('bar_id', [ | ||
* { bar_id: 2, include_extra_info: true }, | ||
* { bar_id: 3, include_extra_info: false }, | ||
* { bar_id: 4, include_extra_info: true }, | ||
* ], 'bar_id') | ||
* ]) | ||
* ``` | ||
@@ -57,4 +57,30 @@ * | ||
*/ | ||
export declare function partitionItems(ignoreKey: string, items: ReadonlyArray<object>): ReadonlyArray<ReadonlyArray<number>>; | ||
export declare function partitionItems(ignoreKeys: Array<string> | string, items: ReadonlyArray<object>): ReadonlyArray<ReadonlyArray<number>>; | ||
/** | ||
* Take in all objects passed to .load(), and bucket them by the non | ||
* batch keys (i.e. `batchKey` and `propertyBatchKey`) attributes. | ||
* Return batch keys value for each partition items. | ||
* | ||
* This function is only called when we have propertyBatchKey, and it's | ||
* used to map result to the order of requests. | ||
* | ||
* Example: | ||
* ```js | ||
* getBatchKeyForPartitionItems( | ||
* 'bar_id', | ||
* ['bar_id', 'properties'], | ||
* [ | ||
* { bar_id: 2, properties: ['name'], include_extra_info: true }, | ||
* { bar_id: 3, properties: ['rating'], include_extra_info: false }, | ||
* { bar_id: 2, properties: ['rating'], include_extra_info: true }, | ||
* ]) | ||
* ``` | ||
* | ||
* Returns: | ||
* `[ [ 2, 2 ], [ 3 ] ]` | ||
* | ||
* TODO: add generic instead of 'object' for the items array argument | ||
*/ | ||
export declare function getBatchKeysForPartitionItems(batchKey: string, ignoreKeys: Array<string>, items: ReadonlyArray<any>): ReadonlyArray<ReadonlyArray<any>>; | ||
/** | ||
* Utility function to sort array of objects by a list of corresponding IDs | ||
@@ -124,2 +150,55 @@ * | ||
/** | ||
* Perform the inverse mapping from partitionItems on the nested results we get | ||
* back from the service. This function is only called when we have propertyBatchKey. | ||
* We currently only support one specific response contract. | ||
* | ||
* propertyBatchKey is not returned in a nested object, but spread at top level as well. | ||
* If we have 'id' as responseKey and 'properties' as propertyBatchKey, | ||
* the resultGroups should look like this: | ||
* [ | ||
* [ { id: 2, name: 'Burger King', rating: 3 } ], | ||
* [ { id: 1, name: 'In N Out', rating: 4 } ] | ||
* ], | ||
* | ||
* | ||
* IMPORTANT NOTE: The contract must have a one-to-one correspondence between the input propertyBatchKey and the output propertyBatchKey. | ||
* i.e. if we have property: 'name' in the request, the response must have 'name' in it, and no extra data associated with it. | ||
* | ||
* Example | ||
* Request args: | ||
* [ | ||
* { bar_id: 2, properties: ['name'], include_extra_info: true }, | ||
* { bar_id: 1, properties: ['rating'], include_extra_info: false }, | ||
* { bar_id: 2, properties: ['rating'], include_extra_info: true }, | ||
* ] | ||
* | ||
* ```js | ||
* unPartitionResultsByBatchKeyPartition( | ||
* newKey = 'bar_id', | ||
* propertyBatchKey = 'properties', | ||
* responseKey = 'id' | ||
* batchKeyPartition = [ [2, 2], [1] ], | ||
* propertyBatchKeyPartion = [ [['name'], ['rating']], [['rating']] ], | ||
* requestGroups = [ [0, 2], [1] ], | ||
* resultGroups = [ | ||
* [ { id: 2, name: 'Burger King', rating: 3 } ], | ||
* [ { id: 1, name: 'In N Out', rating: 4 } ] | ||
* ], | ||
* ) | ||
* ``` | ||
* | ||
* Returns: | ||
* ``` | ||
* [ | ||
* { id: 2, name: 'Burger King' }, | ||
* { id: 1, rating: 4 }, | ||
* { id: 2, rating: 3 }, | ||
* ] | ||
*/ | ||
export declare function unPartitionResultsByBatchKeyPartition<T extends Record<string, any>>(newKey: string, propertyBatchKey: string, responseKey: string, batchKeyPartition: ReadonlyArray<ReadonlyArray<any>>, propertyBatchKeyPartion: ReadonlyArray<ReadonlyArray<any>>, | ||
/** Should be a nested array of IDs, as generated by partitionItems */ | ||
requestGroups: ReadonlyArray<ReadonlyArray<number>>, | ||
/** The results back from the service, in the same shape as groups */ | ||
resultGroups: ReadonlyArray<ReadonlyArray<T | CaughtResourceError>>): ReadonlyArray<T | Error>; | ||
/** | ||
* Turn a dictionary of results into an ordered list | ||
@@ -126,0 +205,0 @@ * |
@@ -8,4 +8,6 @@ "use strict"; | ||
exports.partitionItems = partitionItems; | ||
exports.getBatchKeysForPartitionItems = getBatchKeysForPartitionItems; | ||
exports.sortByKeys = sortByKeys; | ||
exports.unPartitionResults = unPartitionResults; | ||
exports.unPartitionResultsByBatchKeyPartition = unPartitionResultsByBatchKeyPartition; | ||
exports.resultsDictToList = resultsDictToList; | ||
@@ -88,3 +90,3 @@ exports.defaultErrorHandler = defaultErrorHandler; | ||
* Take in all objects passed to .load(), and bucket them by the non | ||
* batchKey attributes. | ||
* batch keys (i.e. `batchKey` and `propertyBatchKey`) attributes. | ||
* | ||
@@ -95,7 +97,7 @@ * We use this to chunk up the requests to the resource. | ||
* ```js | ||
* partitionItems([ | ||
* partitionItems('bar_id', [ | ||
* { bar_id: 2, include_extra_info: true }, | ||
* { bar_id: 3, include_extra_info: false }, | ||
* { bar_id: 4, include_extra_info: true }, | ||
* ], 'bar_id') | ||
* ]) | ||
* ``` | ||
@@ -111,6 +113,6 @@ * | ||
function partitionItems(ignoreKey, items) { | ||
function partitionItems(ignoreKeys, items) { | ||
const groups = {}; | ||
items.forEach((item, i) => { | ||
const hash = (0, _objectHash.default)(_lodash.default.omit(item, ignoreKey), { | ||
const hash = (0, _objectHash.default)(_lodash.default.omit(item, ignoreKeys), { | ||
algorithm: 'passthrough' | ||
@@ -124,2 +126,40 @@ }); | ||
/** | ||
* Take in all objects passed to .load(), and bucket them by the non | ||
* batch keys (i.e. `batchKey` and `propertyBatchKey`) attributes. | ||
* Return batch keys value for each partition items. | ||
* | ||
* This function is only called when we have propertyBatchKey, and it's | ||
* used to map result to the order of requests. | ||
* | ||
* Example: | ||
* ```js | ||
* getBatchKeyForPartitionItems( | ||
* 'bar_id', | ||
* ['bar_id', 'properties'], | ||
* [ | ||
* { bar_id: 2, properties: ['name'], include_extra_info: true }, | ||
* { bar_id: 3, properties: ['rating'], include_extra_info: false }, | ||
* { bar_id: 2, properties: ['rating'], include_extra_info: true }, | ||
* ]) | ||
* ``` | ||
* | ||
* Returns: | ||
* `[ [ 2, 2 ], [ 3 ] ]` | ||
* | ||
* TODO: add generic instead of 'object' for the items array argument | ||
*/ | ||
function getBatchKeysForPartitionItems(batchKey, ignoreKeys, items) { | ||
const groups = {}; | ||
items.forEach((item, i) => { | ||
const hash = (0, _objectHash.default)(_lodash.default.omit(item, ignoreKeys), { | ||
algorithm: 'passthrough' | ||
}); | ||
groups[hash] = groups[hash] || []; | ||
groups[hash].push(items[i][batchKey]); | ||
}); | ||
return Object.values(groups); | ||
} | ||
/** | ||
* Utility function to sort array of objects by a list of corresponding IDs | ||
@@ -264,2 +304,126 @@ * | ||
/** | ||
* Perform the inverse mapping from partitionItems on the nested results we get | ||
* back from the service. This function is only called when we have propertyBatchKey. | ||
* We currently only support one specific response contract. | ||
* | ||
* propertyBatchKey is not returned in a nested object, but spread at top level as well. | ||
* If we have 'id' as responseKey and 'properties' as propertyBatchKey, | ||
* the resultGroups should look like this: | ||
* [ | ||
* [ { id: 2, name: 'Burger King', rating: 3 } ], | ||
* [ { id: 1, name: 'In N Out', rating: 4 } ] | ||
* ], | ||
* | ||
* | ||
* IMPORTANT NOTE: The contract must have a one-to-one correspondence between the input propertyBatchKey and the output propertyBatchKey. | ||
* i.e. if we have property: 'name' in the request, the response must have 'name' in it, and no extra data associated with it. | ||
* | ||
* Example | ||
* Request args: | ||
* [ | ||
* { bar_id: 2, properties: ['name'], include_extra_info: true }, | ||
* { bar_id: 1, properties: ['rating'], include_extra_info: false }, | ||
* { bar_id: 2, properties: ['rating'], include_extra_info: true }, | ||
* ] | ||
* | ||
* ```js | ||
* unPartitionResultsByBatchKeyPartition( | ||
* newKey = 'bar_id', | ||
* propertyBatchKey = 'properties', | ||
* responseKey = 'id' | ||
* batchKeyPartition = [ [2, 2], [1] ], | ||
* propertyBatchKeyPartion = [ [['name'], ['rating']], [['rating']] ], | ||
* requestGroups = [ [0, 2], [1] ], | ||
* resultGroups = [ | ||
* [ { id: 2, name: 'Burger King', rating: 3 } ], | ||
* [ { id: 1, name: 'In N Out', rating: 4 } ] | ||
* ], | ||
* ) | ||
* ``` | ||
* | ||
* Returns: | ||
* ``` | ||
* [ | ||
* { id: 2, name: 'Burger King' }, | ||
* { id: 1, rating: 4 }, | ||
* { id: 2, rating: 3 }, | ||
* ] | ||
*/ | ||
function unPartitionResultsByBatchKeyPartition(newKey, propertyBatchKey, responseKey, batchKeyPartition, propertyBatchKeyPartion, | ||
/** Should be a nested array of IDs, as generated by partitionItems */ | ||
requestGroups, | ||
/** The results back from the service, in the same shape as groups */ | ||
resultGroups) { | ||
/** | ||
* e.g. with our inputs, produce: | ||
* ```js | ||
* [ | ||
* [ | ||
* { order: 0, result: { id: 2, name: 'Burger King' }, | ||
* { order: 2, result: { id: 2, rating: 3 } }, | ||
* ], | ||
* [ | ||
* { order: 1, result: { id: 1, rating: 4 } }, | ||
* ] | ||
* ] | ||
* ``` | ||
*/ | ||
const zippedGroups = requestGroups.map((ids, i) => { | ||
return ids.map((id, j) => { | ||
let result = null; | ||
for (const resultElement of Object.values(resultGroups)[i]) { | ||
// There's error in the result we should return | ||
if (resultElement instanceof CaughtResourceError) { | ||
result = resultElement; | ||
break; | ||
} // Find the response that matches the requested batchKey | ||
if (resultElement[responseKey] === batchKeyPartition[i][j]) { | ||
// Only responseKey and the requested properties will be in the final result. | ||
result = _lodash.default.pick(resultElement, [responseKey, ...propertyBatchKeyPartion[i][j]]); | ||
break; | ||
} | ||
} // If requested property doesn't exist in resultElement, we should throw BatchItemNotFoundError error. | ||
if (result === null) { | ||
return { | ||
order: id, | ||
result: new BatchItemNotFoundError([`Could not find ${responseKey} = ${batchKeyPartition[i][j]} in the response dict.`, `Or your endpoint does not follow the contract we support.`, `Please read https://github.com/Yelp/dataloader-codegen/blob/master/API_DOCS.md.`].join(' ')) | ||
}; | ||
} else { | ||
return { | ||
order: id, | ||
result | ||
}; | ||
} | ||
}); | ||
}); | ||
/** | ||
* Flatten and sort the groups - e.g.: | ||
* ```js | ||
* [ | ||
* { order: 0, result: { id: 2, name: 'Burger King' } }, | ||
* { order: 1, result: { id: 1, rating: 4 } }, | ||
* { order: 2, result: { id: 2, rating: 3 } } | ||
* ] | ||
* ``` | ||
*/ | ||
const sortedResults = _lodash.default.sortBy(_lodash.default.flatten(zippedGroups), ['order']); // Now that we have a sorted array, return the actual results! | ||
return sortedResults.map(r => r.result).map(result => { | ||
if (result instanceof CaughtResourceError) { | ||
return result.cause; | ||
} else { | ||
return result; | ||
} | ||
}); | ||
} | ||
/** | ||
* Turn a dictionary of results into an ordered list | ||
@@ -266,0 +430,0 @@ * |
{ | ||
"name": "dataloader-codegen", | ||
"version": "0.3.1", | ||
"version": "0.4.1", | ||
"description": "dataloader-codegen is an opinionated JavaScript library for automatically generating DataLoaders over a set of resources (e.g. HTTP endpoints)", | ||
@@ -5,0 +5,0 @@ "main": "lib/index.js", |
@@ -100,3 +100,14 @@ { | ||
"description": "(Optional) Set to true if the interface of the resource takes the batch key as a set (rather than an array). For example, when using a generated clientlib based on swagger where `uniqueItems: true` is set for the batchKey parameter. Default: false" | ||
}, | ||
"propertyBatchKey": { | ||
"type": "string", | ||
"description": "(Optional) The argument to the resource that represents the optional properties we want to fetch. (e.g. usually 'properties' or 'features')" | ||
}, | ||
"responseKey": { | ||
"type": "string", | ||
"description": "(Non-optional when propertyBatchKey is used) The key in the response objects corresponds to `batchKey`. This should be the only field that are marked as required in your swagger endpoint response, except nestedPath." | ||
} | ||
}, | ||
"dependencies": { | ||
"propertyBatchKey": { "required": ["responseKey"] } | ||
} | ||
@@ -103,0 +114,0 @@ }, |
Sorry, the diff of this file is not supported yet
85671
22.12%1671
22.33%