@arranger/mapping-utils - npm Package Compare versions

Comparing version 0.4.13 to 0.4.14

dist/resolveHits.js

@@ -12,2 +12,4 @@ "use strict";
 
+var _lodash = require("lodash");
+
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
@@ -49,2 +51,4 @@
 var hitsToEdges = function hitsToEdges(_ref3) {
+  var _process, _process$env;
+
   var hits = _ref3.hits,
@@ -54,105 +58,120 @@ nestedFields = _ref3.nestedFields,
       _ref3$copyToSourceFie = _ref3.copyToSourceFields,
-      copyToSourceFields = _ref3$copyToSourceFie === void 0 ? {} : _ref3$copyToSourceFie;
-  //Parallel.spawn output has a .then but it's not returning an actual promise
-  return new Promise(function (resolve) {
-    new Parallel({
-      hits: hits,
-      nestedFields: nestedFields,
-      copyToSourceFields: copyToSourceFields
-    }).spawn(function (_ref4) {
-      var hits = _ref4.hits,
-          nestedFields = _ref4.nestedFields,
-          copyToSourceFields = _ref4.copyToSourceFields;
+      copyToSourceFields = _ref3$copyToSourceFie === void 0 ? {} : _ref3$copyToSourceFie,
+      _ref3$systemCores = _ref3.systemCores,
+      systemCores = _ref3$systemCores === void 0 ? ((_process = process) === null || _process === void 0 ? void 0 : (_process$env = _process.env) === null || _process$env === void 0 ? void 0 : _process$env.SYSTEM_CORES) || 2 : _ref3$systemCores;
 
-      /*
-        everthing inside spawn is executed in a separate thread, so we have
-        to use good old ES5 and require for run-time dependecy bundling.
-      */
-      var _require = require('lodash'),
-          isObject = _require.isObject,
-          flattenDeep = _require.flattenDeep;
+  /*
+    If there's a large request, we'll trigger ludicrous mode and do some parallel
+    map-reduce based on # of cores available. Otherwise, only one child-process
+    is spawn for compute
+  */
+  var dataSize = hits.hits.length;
+  var chunks = (0, _lodash.chunk)(hits.hits, dataSize > 1000 ? dataSize / systemCores + dataSize % systemCores : dataSize);
+  return Promise.all(chunks.map(function (chunk) {
+    return (//Parallel.spawn output has a .then but it's not returning an actual promise
+      new Promise(function (resolve) {
+        new Parallel({
+          hits: chunk,
+          nestedFields: nestedFields,
+          copyToSourceFields: copyToSourceFields
+        }).spawn(function (_ref4) {
+          var hits = _ref4.hits,
+              nestedFields = _ref4.nestedFields,
+              copyToSourceFields = _ref4.copyToSourceFields;
 
-      var jp = require('jsonpath/jsonpath.min');
+          /*
+            everthing inside spawn is executed in a separate thread, so we have
+            to use good old ES5 and require for run-time dependecy bundling.
+          */
+          var _require = require('lodash'),
+              isObject = _require.isObject,
+              flattenDeep = _require.flattenDeep;
 
-      var resolveCopiedTo = function resolveCopiedTo(_ref5) {
-        var node = _ref5.node;
-        var foundValues = Object.entries(copyToSourceFields).reduce(function (acc, pair) {
-          var copyToField = pair[0];
-          var sourceField = pair[1];
-          var found = {};
-          found[copyToField] = flattenDeep(sourceField.map(function (path) {
-            return jp.query(node, path.split('.').reduce(function (acc, part, index) {
-              return index === 0 ? "$.".concat(part) : "".concat(acc, "..").concat(part);
-            }, ''));
-          }));
-          return found;
-        }, {});
-        return foundValues;
-      };
+          var jp = require('jsonpath/jsonpath.min');
 
-      return hits.hits.map(function (x) {
-        var joinParent = function joinParent(parent, field) {
-          return parent ? "".concat(parent, ".").concat(field) : field;
-        };
+          var resolveCopiedTo = function resolveCopiedTo(_ref5) {
+            var node = _ref5.node;
+            var foundValues = Object.entries(copyToSourceFields).reduce(function (acc, pair) {
+              var copyToField = pair[0];
+              var sourceField = pair[1];
+              var found = {};
+              found[copyToField] = flattenDeep(sourceField.map(function (path) {
+                return jp.query(node, path.split('.').reduce(function (acc, part, index) {
+                  return index === 0 ? "$.".concat(part) : "".concat(acc, "..").concat(part);
+                }, ''));
+              }));
+              return found;
+            }, {});
+            return foundValues;
+          };
 
-        var resolveNested = function resolveNested(_ref6) {
-          var node = _ref6.node,
-              nestedFields = _ref6.nestedFields,
-              _ref6$parent = _ref6.parent,
-              parent = _ref6$parent === void 0 ? '' : _ref6$parent;
-          if (!isObject(node) || !node) return node;
-          return Object.entries(node).reduce(function (acc, pair) {
-            var field = pair[0];
-            var hits = pair[1]; // TODO: inner hits query if necessary
+          return hits.map(function (x) {
+            var joinParent = function joinParent(parent, field) {
+              return parent ? "".concat(parent, ".").concat(field) : field;
+            };
 
-            var fullPath = joinParent(parent, field);
-            var resolvedNested = {};
-            resolvedNested[field] = nestedFields.includes(fullPath) ? {
-              hits: {
-                edges: hits.map(function (node) {
-                  return {
-                    node: Object.assign({}, node, resolveNested({
-                      node: node,
-                      nestedFields: nestedFields,
-                      parent: fullPath
-                    }))
-                  };
-                }),
-                total: hits.length
-              }
-            } : isObject(hits) && hits ? Object.assign(hits.constructor(), resolveNested({
-              node: hits,
-              nestedFields: nestedFields,
-              parent: fullPath
-            })) : resolveNested({
-              node: hits,
-              nestedFields: nestedFields,
-              parent: fullPath
+            var resolveNested = function resolveNested(_ref6) {
+              var node = _ref6.node,
+                  nestedFields = _ref6.nestedFields,
+                  _ref6$parent = _ref6.parent,
+                  parent = _ref6$parent === void 0 ? '' : _ref6$parent;
+              if (!isObject(node) || !node) return node;
+              return Object.entries(node).reduce(function (acc, pair) {
+                var field = pair[0];
+                var hits = pair[1]; // TODO: inner hits query if necessary
+
+                var fullPath = joinParent(parent, field);
+                acc[field] = nestedFields.includes(fullPath) ? {
+                  hits: {
+                    edges: hits.map(function (node) {
+                      return {
+                        node: Object.assign({}, node, resolveNested({
+                          node: node,
+                          nestedFields: nestedFields,
+                          parent: fullPath
+                        }))
+                      };
+                    }),
+                    total: hits.length
+                  }
+                } : isObject(hits) && hits ? Object.assign(hits.constructor(), resolveNested({
+                  node: hits,
+                  nestedFields: nestedFields,
+                  parent: fullPath
+                })) : resolveNested({
+                  node: hits,
+                  nestedFields: nestedFields,
+                  parent: fullPath
+                });
+                return acc;
+              }, {});
+            };
+
+            var source = x._source;
+            var nested_nodes = resolveNested({
+              node: source,
+              nestedFields: nestedFields
             });
-            return Object.assign({}, acc, resolvedNested);
-          }, {});
-        };
-
-        var source = x._source;
-        var nested_nodes = resolveNested({
-          node: source,
-          nestedFields: nestedFields
-        });
-        var copied_to_nodes = resolveCopiedTo({
-          node: source
-        });
-        return {
-          searchAfter: x.sort ? x.sort.map(function (x) {
-            return Number.isInteger(x) && !Number.isSafeInteger(x) ? // TODO: figure out a way to inject ES_CONSTANTS in here from @arranger/middleware
-            // ? ES_CONSTANTS.ES_MAX_LONG //https://github.com/elastic/elasticsearch-js/issues/662
-            "-9223372036854775808" //https://github.com/elastic/elasticsearch-js/issues/662
-            : x;
-          }) : [],
-          node: Object.assign({
-            id: x._id
-          }, source, nested_nodes, copied_to_nodes)
-        };
-      });
-    }).then(function (edges) {
-      resolve(edges);
+            var copied_to_nodes = resolveCopiedTo({
+              node: source
+            });
+            return {
+              searchAfter: x.sort ? x.sort.map(function (x) {
+                return Number.isInteger(x) && !Number.isSafeInteger(x) ? // TODO: figure out a way to inject ES_CONSTANTS in here from @arranger/middleware
+                // ? ES_CONSTANTS.ES_MAX_LONG //https://github.com/elastic/elasticsearch-js/issues/662
+                "-9223372036854775808" //https://github.com/elastic/elasticsearch-js/issues/662
+                : x;
+              }) : [],
+              node: Object.assign(source, // we're not afraid of mutating source here!
+              {
+                id: x._id
+              }, nested_nodes, copied_to_nodes)
+            };
+          });
+        }).then(resolve);
+      })
+    );
+  })).then(function (chunks) {
+    return chunks.reduce(function (acc, chunk) {
+      return acc.concat(chunk);
     });
@@ -159,0 +178,0 @@ });

package.json

 {
   "name": "@arranger/mapping-utils",
-  "version": "0.4.13",
+  "version": "0.4.14",
   "description": "Transform Elasticsearch mappings",
@@ -25,3 +25,3 @@ "main": "dist/index.js",
   "dependencies": {
-    "@arranger/middleware": "^0.4.13",
+    "@arranger/middleware": "^0.4.14",
     "babel-polyfill": "^6.26.0",
@@ -28,0 +28,0 @@ "elasticsearch": "^14.0.0",

No alert changes

Improved metrics

Total package byte prevSize

162256

increased by0.75%

Lines of code

3183

increased by0.51%

Dependency changes

• Updated@arranger/middleware@^0.4.14