contexture-elasticsearch - npm Package Compare versions

Comparing version 1.26.0 to 1.27.0

CHANGELOG.md

 # Changelog
 
+## 1.27.0
+
+### Minor Changes
+
+- 2f4e5039: Remove `subFields` configuration in the schema. Instead only send fields of type
+  `text` for highlighting. This both simplifies the API and reduces payload to
+  elastic, as well as fixing an issue where non-text top-level fields such as
+  `keyword` type fields were being highlighted when they should not be.
+
 ## 1.26.0
@@ -4,0 +13,0 @@

dist/cjs/example-types/results/highlighting/request.js

@@ -32,3 +32,2 @@ var __create = Object.create;
   getAllHighlightFields: () => getAllHighlightFields,
-  getHighlightGroupFieldsPaths: () => getHighlightGroupFieldsPaths,
   getRequestHighlightFields: () => getRequestHighlightFields
@@ -40,7 +39,6 @@ });
 var import_minimatch = require("minimatch");
-var import_js_combinatorics = require("js-combinatorics");
 var import_util = require("./util.js");
 let expandGlobs = (schema, globs) => {
   let fieldsNames = import_fp.default.keys(schema.fields);
-  let expandGlob = (glob) => (0, import_util.isLeafField)(schema.fields[glob]) ? [glob] : import_minimatch.minimatch.match(fieldsNames, `${glob}*`);
+  let expandGlob = (glob) => (0, import_util.getFieldType)(schema.fields[glob]) ? [glob] : import_minimatch.minimatch.match(fieldsNames, `${glob}*`);
   return import_fp.default.flow(
@@ -75,49 +73,56 @@ import_fp.default.flatMap(expandGlob),
 };
-let getHighlightSubFieldsNames = (schema) => import_fp.default.keys(import_fp.default.pickBy("highlight", schema.elasticsearch?.subFields));
-let getHighlightGroupFieldsPaths = import_fp.default.memoize((schema) => {
-  let subFieldsNames = getHighlightSubFieldsNames(schema);
-  return import_fp.default.flatMap((field) => {
-    let copy_to = field.elasticsearch?.mapping?.copy_to;
-    if (import_fp.default.isEmpty(copy_to))
-      return [];
-    let subFieldTuples = [...new import_js_combinatorics.CartesianProduct(copy_to, subFieldsNames)];
-    let product = [...copy_to, ...import_fp.default.map(import_fp.default.join("."), subFieldTuples)];
-    return product;
-  }, schema.fields);
-}, import_fp.default.get("elasticsearch.index"));
-let isGroupFieldPath = import_fp.default.curry(
-  (schema, path) => import_fp.default.find(import_fp.default.eq(path), getHighlightGroupFieldsPaths(schema))
+let getFieldSubFields = (field) => import_futil.default.mapValuesIndexed(
+  (subField, subFieldName) => import_futil.default.omitBlank({
+    // Reuse the parent multi-field `subType` so that we can generate the
+    // correct highlighting configuration.
+    subType: field.subType,
+    elasticsearch: import_futil.default.omitBlank({
+      mapping: import_futil.default.omitBlank({
+        ...subField,
+        copy_to: import_fp.default.map(
+          (path) => `${path}.${subFieldName}`,
+          field.elasticsearch.mapping?.copy_to
+        )
+      })
+    })
+  }),
+  field.elasticsearch?.mapping?.fields
 );
-let getAllHighlightFields = import_fp.default.memoize((schema) => {
-  let subFieldsNames = getHighlightSubFieldsNames(schema);
-  return import_futil.default.reduceIndexed(
-    (acc, field, path) => {
-      if (!(0, import_util.isLeafField)(field) || isGroupFieldPath(schema, path)) {
-        return acc;
-      }
-      acc[path] = field;
-      let subFields = import_fp.default.pick(
-        subFieldsNames,
-        field.elasticsearch?.mapping?.fields
-      );
-      for (let name in subFields) {
-        acc[`${path}.${name}`] = import_futil.default.omitBlank({
-          subType: field.subType,
-          elasticsearch: import_futil.default.omitBlank({
-            mapping: import_futil.default.omitBlank({
-              ...subFields[name],
-              copy_to: import_fp.default.map(
-                (path2) => `${path2}.${name}`,
-                field.elasticsearch.mapping?.copy_to
-              )
-            })
-          })
-        });
-      }
-      return acc;
-    },
-    {},
+let getSchemaSubFields = (schema) => import_futil.default.reduceIndexed(
+  (acc, field, path) => import_futil.default.mergeOn(
+    acc,
+    import_fp.default.mapKeys((k) => `${path}.${k}`, getFieldSubFields(field))
+  ),
+  {},
+  schema.fields
+);
+let getSchemaGroupFields = import_fp.default.memoize((schema) => {
+  let groupFields = import_fp.default.pick(
+    import_fp.default.uniq(
+      import_fp.default.flatMap(
+        (field) => field.elasticsearch?.mapping?.copy_to ?? [],
+        schema.fields
+      )
+    ),
     schema.fields
   );
+  return {
+    ...groupFields,
+    ...getSchemaSubFields({ fields: groupFields })
+  };
 }, import_fp.default.get("elasticsearch.index"));
+let getAllHighlightFields = import_fp.default.memoize((schema) => {
+  let groupFields = getSchemaGroupFields(schema);
+  let canHighlightField = (field, path) => (
+    // Only highlight text fields.
+    (0, import_util.getFieldType)(field) === "text" && // Omit group fields from highlighting. We assume users want to
+    // highlight fields that were copied over instead of the group fields
+    // themselves.
+    !import_fp.default.has(path, groupFields)
+  );
+  return import_futil.default.pickByIndexed(canHighlightField, {
+    ...schema.fields,
+    ...getSchemaSubFields(schema)
+  });
+}, import_fp.default.get("elasticsearch.index"));
 let collectKeysAndValues = (f, coll) => import_futil.default.reduceTree()(
@@ -133,4 +138,5 @@ (acc, val, key) => f(val) ? import_futil.default.push(val, acc) : f(key) ? import_futil.default.push(key, acc) : acc,
 let getRequestHighlightFields = (schema, node) => {
+  let groupFields = getSchemaGroupFields(schema);
   let groupFieldsInQuery = collectKeysAndValues(
-    isGroupFieldPath(schema),
+    import_futil.default.getIn(groupFields),
     node._meta?.relevantFilters
@@ -161,4 +167,3 @@ );
   getAllHighlightFields,
-  getHighlightGroupFieldsPaths,
   getRequestHighlightFields
 });

dist/cjs/example-types/results/highlighting/response.js

@@ -50,5 +50,4 @@ var __create = Object.create;
 );
-let groupByArrayOfObjectsFields = import_fp.default.curry((schema, highlight) => {
-  let arrayOfObjectsPaths = import_fp.default.keys((0, import_util.getArrayOfObjectsPathsMap)(schema));
-  return import_futil.default.reduceIndexed(
+let groupByArrayOfObjectsFields = import_fp.default.curry(
+  (arrayOfObjectsPaths, highlight) => import_futil.default.reduceIndexed(
     (acc, fragments, path) => {
@@ -64,4 +63,4 @@ let arrayPath = (0, import_util.findByPrefixIn)(arrayOfObjectsPaths, path);
     highlight
-  );
-});
+  )
+);
 let getIndexedFragments = (tags, source, fragments, nestedPath) => {
@@ -92,32 +91,29 @@ let fragmentsMap = import_fp.default.groupBy((0, import_util.stripTags)(tags), fragments);
 );
-let getResponseHighlight = (schema, hit, tags, copySourcePaths) => {
-  let pathsMap = (0, import_util.getNestedPathsMap)(schema, copySourcePaths);
-  return import_fp.default.flow(
-    groupByMultiField(schema),
-    import_fp.default.mapValues(import_fp.default.flatten),
-    groupByArrayOfObjectsFields(schema),
-    import_futil.default.mapValuesIndexed((fragments, path) => {
-      let field = schema.fields[path];
-      if ((0, import_util.isBlobField)(field)) {
-        return fragments;
-      }
-      if ((0, import_util.isArrayOfScalarsField)(field)) {
-        let sourceArray = import_fp.default.get(path, hit._source);
-        return getArrayOfScalarsFragments(tags, sourceArray, fragments);
-      }
-      if ((0, import_util.isArrayOfObjectsField)(field)) {
-        let sourceArray = import_fp.default.get(path, hit._source);
-        let result = getArrayOfObjectsFragments(tags, sourceArray, fragments);
-        let copyPaths = pathsMap[path];
-        if (import_fp.default.isEmpty(copyPaths))
-          return result;
-        return import_futil.default.mapValuesIndexed(
-          (to, index) => import_fp.default.merge(import_fp.default.pick(copyPaths, sourceArray[index]), to),
-          result
-        );
-      }
-      return (0, import_util.mergeHighlights)(tags, fragments);
-    })
-  )(hit.highlight);
-};
+let getResponseHighlight = (schema, hit, tags, nestedPathsMap) => import_fp.default.flow(
+  groupByMultiField(schema),
+  import_fp.default.mapValues(import_fp.default.flatten),
+  groupByArrayOfObjectsFields(import_fp.default.keys((0, import_util.getArrayOfObjectsPathsMap)(schema))),
+  import_futil.default.mapValuesIndexed((fragments, path) => {
+    let field = schema.fields[path];
+    if ((0, import_util.isBlobField)(field)) {
+      return fragments;
+    }
+    if ((0, import_util.isArrayOfScalarsField)(field)) {
+      let sourceArray = import_fp.default.get(path, hit._source);
+      return getArrayOfScalarsFragments(tags, sourceArray, fragments);
+    }
+    if ((0, import_util.isArrayOfObjectsField)(field)) {
+      let sourceArray = import_fp.default.get(path, hit._source);
+      let result = getArrayOfObjectsFragments(tags, sourceArray, fragments);
+      let copyPaths = nestedPathsMap?.[path];
+      if (import_fp.default.isEmpty(copyPaths))
+        return result;
+      return import_futil.default.mapValuesIndexed(
+        (to, index) => import_fp.default.merge(import_fp.default.pick(copyPaths, sourceArray[index]), to),
+        result
+      );
+    }
+    return (0, import_util.mergeHighlights)(tags, fragments);
+  })
+)(hit.highlight);
 let removePathsFromSource = (schema, hit, paths) => {
@@ -124,0 +120,0 @@ if (import_fp.default.isEmpty(paths))

dist/cjs/example-types/results/highlighting/search.js

@@ -64,5 +64,8 @@ var __create = Object.create;
   });
+  const nestedPathsMap = (0, import_util.getNestedPathsMap)(
+    schema,
+    node.highlight?.copySourcePaths
+  );
   for (let hit of response.hits.hits) {
-    let copySourcePaths = node.highlight?.copySourcePaths;
-    hit.highlight = (0, import_response.getResponseHighlight)(schema, hit, tags, copySourcePaths);
+    hit.highlight = (0, import_response.getResponseHighlight)(schema, hit, tags, nestedPathsMap);
     (0, import_response.removePathsFromSource)(schema, hit, addedPaths);
@@ -69,0 +72,0 @@ (0, import_response.mergeHighlightsOnSource)(schema, hit);

dist/cjs/example-types/results/highlighting/testSchema.js

@@ -24,8 +24,2 @@ var __defProp = Object.defineProperty;
 let schema = {
-  elasticsearch: {
-    subFields: {
-      keyword: { highlight: false },
-      subfield: { highlight: true }
-    }
-  },
   fields: {
@@ -32,0 +26,0 @@ groupField: {

dist/cjs/example-types/results/highlighting/util.js

@@ -32,2 +32,3 @@ var __create = Object.create;
   getArrayOfObjectsPathsMap: () => getArrayOfObjectsPathsMap,
+  getFieldType: () => getFieldType,
   getNestedPathsMap: () => getNestedPathsMap,
@@ -38,3 +39,2 @@ isArrayField: () => isArrayField,
   isBlobField: () => isBlobField,
-  isLeafField: () => isLeafField,
   mergeHighlights: () => mergeHighlights,
@@ -47,7 +47,7 @@ stripParentPath: () => stripParentPath,
 var import_futil = __toESM(require("futil"));
-let isLeafField = (field) => !!field?.elasticsearch?.dataType || !!field?.elasticsearch?.mapping?.type;
-let isBlobField = (field) => field?.subType === "blob" && isLeafField(field);
+let getFieldType = (field) => field?.elasticsearch?.dataType || field?.elasticsearch?.mapping?.type;
+let isBlobField = (field) => field?.subType === "blob" && !!getFieldType(field);
 let isArrayField = (field) => field?.subType === "array";
-let isArrayOfScalarsField = (field) => isArrayField(field) && isLeafField(field);
-let isArrayOfObjectsField = (field) => isArrayField(field) && !isLeafField(field);
+let isArrayOfScalarsField = (field) => isArrayField(field) && !!getFieldType(field);
+let isArrayOfObjectsField = (field) => isArrayField(field) && !getFieldType(field);
 let stripParentPath = import_fp.default.curry(
@@ -121,2 +121,3 @@ (parentPath, path) => import_fp.default.startsWith(`${parentPath}.`, path) ? path.slice(parentPath.length + 1) : void 0
   getArrayOfObjectsPathsMap,
+  getFieldType,
   getNestedPathsMap,
@@ -127,3 +128,2 @@ isArrayField,
   isBlobField,
-  isLeafField,
   mergeHighlights,
@@ -130,0 +130,0 @@ stripParentPath,

dist/esm/example-types/results/highlighting/request.js

 import _ from "lodash/fp.js";
 import F from "futil";
 import { minimatch } from "minimatch";
-import { CartesianProduct } from "js-combinatorics";
-import { isLeafField, isBlobField, isArrayOfObjectsField } from "./util.js";
+import { getFieldType, isBlobField, isArrayOfObjectsField } from "./util.js";
 let expandGlobs = (schema, globs) => {
   let fieldsNames = _.keys(schema.fields);
-  let expandGlob = (glob) => isLeafField(schema.fields[glob]) ? [glob] : minimatch.match(fieldsNames, `${glob}*`);
+  let expandGlob = (glob) => getFieldType(schema.fields[glob]) ? [glob] : minimatch.match(fieldsNames, `${glob}*`);
   return _.flow(
@@ -38,49 +37,56 @@ _.flatMap(expandGlob),
 };
-let getHighlightSubFieldsNames = (schema) => _.keys(_.pickBy("highlight", schema.elasticsearch?.subFields));
-let getHighlightGroupFieldsPaths = _.memoize((schema) => {
-  let subFieldsNames = getHighlightSubFieldsNames(schema);
-  return _.flatMap((field) => {
-    let copy_to = field.elasticsearch?.mapping?.copy_to;
-    if (_.isEmpty(copy_to))
-      return [];
-    let subFieldTuples = [...new CartesianProduct(copy_to, subFieldsNames)];
-    let product = [...copy_to, ..._.map(_.join("."), subFieldTuples)];
-    return product;
-  }, schema.fields);
-}, _.get("elasticsearch.index"));
-let isGroupFieldPath = _.curry(
-  (schema, path) => _.find(_.eq(path), getHighlightGroupFieldsPaths(schema))
+let getFieldSubFields = (field) => F.mapValuesIndexed(
+  (subField, subFieldName) => F.omitBlank({
+    // Reuse the parent multi-field `subType` so that we can generate the
+    // correct highlighting configuration.
+    subType: field.subType,
+    elasticsearch: F.omitBlank({
+      mapping: F.omitBlank({
+        ...subField,
+        copy_to: _.map(
+          (path) => `${path}.${subFieldName}`,
+          field.elasticsearch.mapping?.copy_to
+        )
+      })
+    })
+  }),
+  field.elasticsearch?.mapping?.fields
 );
-let getAllHighlightFields = _.memoize((schema) => {
-  let subFieldsNames = getHighlightSubFieldsNames(schema);
-  return F.reduceIndexed(
-    (acc, field, path) => {
-      if (!isLeafField(field) || isGroupFieldPath(schema, path)) {
-        return acc;
-      }
-      acc[path] = field;
-      let subFields = _.pick(
-        subFieldsNames,
-        field.elasticsearch?.mapping?.fields
-      );
-      for (let name in subFields) {
-        acc[`${path}.${name}`] = F.omitBlank({
-          subType: field.subType,
-          elasticsearch: F.omitBlank({
-            mapping: F.omitBlank({
-              ...subFields[name],
-              copy_to: _.map(
-                (path2) => `${path2}.${name}`,
-                field.elasticsearch.mapping?.copy_to
-              )
-            })
-          })
-        });
-      }
-      return acc;
-    },
-    {},
+let getSchemaSubFields = (schema) => F.reduceIndexed(
+  (acc, field, path) => F.mergeOn(
+    acc,
+    _.mapKeys((k) => `${path}.${k}`, getFieldSubFields(field))
+  ),
+  {},
+  schema.fields
+);
+let getSchemaGroupFields = _.memoize((schema) => {
+  let groupFields = _.pick(
+    _.uniq(
+      _.flatMap(
+        (field) => field.elasticsearch?.mapping?.copy_to ?? [],
+        schema.fields
+      )
+    ),
     schema.fields
   );
+  return {
+    ...groupFields,
+    ...getSchemaSubFields({ fields: groupFields })
+  };
 }, _.get("elasticsearch.index"));
+let getAllHighlightFields = _.memoize((schema) => {
+  let groupFields = getSchemaGroupFields(schema);
+  let canHighlightField = (field, path) => (
+    // Only highlight text fields.
+    getFieldType(field) === "text" && // Omit group fields from highlighting. We assume users want to
+    // highlight fields that were copied over instead of the group fields
+    // themselves.
+    !_.has(path, groupFields)
+  );
+  return F.pickByIndexed(canHighlightField, {
+    ...schema.fields,
+    ...getSchemaSubFields(schema)
+  });
+}, _.get("elasticsearch.index"));
 let collectKeysAndValues = (f, coll) => F.reduceTree()(
@@ -96,4 +102,5 @@ (acc, val, key) => f(val) ? F.push(val, acc) : f(key) ? F.push(key, acc) : acc,
 let getRequestHighlightFields = (schema, node) => {
+  let groupFields = getSchemaGroupFields(schema);
   let groupFieldsInQuery = collectKeysAndValues(
-    isGroupFieldPath(schema),
+    F.getIn(groupFields),
     node._meta?.relevantFilters
@@ -123,4 +130,3 @@ );
   getAllHighlightFields,
-  getHighlightGroupFieldsPaths,
   getRequestHighlightFields
 };

dist/esm/example-types/results/highlighting/response.js

@@ -23,5 +23,4 @@ import _ from "lodash/fp.js";
 );
-let groupByArrayOfObjectsFields = _.curry((schema, highlight) => {
-  let arrayOfObjectsPaths = _.keys(getArrayOfObjectsPathsMap(schema));
-  return F.reduceIndexed(
+let groupByArrayOfObjectsFields = _.curry(
+  (arrayOfObjectsPaths, highlight) => F.reduceIndexed(
     (acc, fragments, path) => {
@@ -37,4 +36,4 @@ let arrayPath = findByPrefixIn(arrayOfObjectsPaths, path);
     highlight
-  );
-});
+  )
+);
 let getIndexedFragments = (tags, source, fragments, nestedPath) => {
@@ -65,32 +64,29 @@ let fragmentsMap = _.groupBy(stripTags(tags), fragments);
 );
-let getResponseHighlight = (schema, hit, tags, copySourcePaths) => {
-  let pathsMap = getNestedPathsMap(schema, copySourcePaths);
-  return _.flow(
-    groupByMultiField(schema),
-    _.mapValues(_.flatten),
-    groupByArrayOfObjectsFields(schema),
-    F.mapValuesIndexed((fragments, path) => {
-      let field = schema.fields[path];
-      if (isBlobField(field)) {
-        return fragments;
-      }
-      if (isArrayOfScalarsField(field)) {
-        let sourceArray = _.get(path, hit._source);
-        return getArrayOfScalarsFragments(tags, sourceArray, fragments);
-      }
-      if (isArrayOfObjectsField(field)) {
-        let sourceArray = _.get(path, hit._source);
-        let result = getArrayOfObjectsFragments(tags, sourceArray, fragments);
-        let copyPaths = pathsMap[path];
-        if (_.isEmpty(copyPaths))
-          return result;
-        return F.mapValuesIndexed(
-          (to, index) => _.merge(_.pick(copyPaths, sourceArray[index]), to),
-          result
-        );
-      }
-      return mergeHighlights(tags, fragments);
-    })
-  )(hit.highlight);
-};
+let getResponseHighlight = (schema, hit, tags, nestedPathsMap) => _.flow(
+  groupByMultiField(schema),
+  _.mapValues(_.flatten),
+  groupByArrayOfObjectsFields(_.keys(getArrayOfObjectsPathsMap(schema))),
+  F.mapValuesIndexed((fragments, path) => {
+    let field = schema.fields[path];
+    if (isBlobField(field)) {
+      return fragments;
+    }
+    if (isArrayOfScalarsField(field)) {
+      let sourceArray = _.get(path, hit._source);
+      return getArrayOfScalarsFragments(tags, sourceArray, fragments);
+    }
+    if (isArrayOfObjectsField(field)) {
+      let sourceArray = _.get(path, hit._source);
+      let result = getArrayOfObjectsFragments(tags, sourceArray, fragments);
+      let copyPaths = nestedPathsMap?.[path];
+      if (_.isEmpty(copyPaths))
+        return result;
+      return F.mapValuesIndexed(
+        (to, index) => _.merge(_.pick(copyPaths, sourceArray[index]), to),
+        result
+      );
+    }
+    return mergeHighlights(tags, fragments);
+  })
+)(hit.highlight);
 let removePathsFromSource = (schema, hit, paths) => {
@@ -97,0 +93,0 @@ if (_.isEmpty(paths))

dist/esm/example-types/results/highlighting/search.js

 import _ from "lodash/fp.js";
 import F from "futil";
-import { getArrayOfObjectsPathsMap } from "./util.js";
+import { getArrayOfObjectsPathsMap, getNestedPathsMap } from "./util.js";
 import {
@@ -39,5 +39,8 @@ addPathsToRequestSource,
   });
+  const nestedPathsMap = getNestedPathsMap(
+    schema,
+    node.highlight?.copySourcePaths
+  );
   for (let hit of response.hits.hits) {
-    let copySourcePaths = node.highlight?.copySourcePaths;
-    hit.highlight = getResponseHighlight(schema, hit, tags, copySourcePaths);
+    hit.highlight = getResponseHighlight(schema, hit, tags, nestedPathsMap);
     removePathsFromSource(schema, hit, addedPaths);
@@ -44,0 +47,0 @@ mergeHighlightsOnSource(schema, hit);

dist/esm/example-types/results/highlighting/testSchema.js

 let schema = {
-  elasticsearch: {
-    subFields: {
-      keyword: { highlight: false },
-      subfield: { highlight: true }
-    }
-  },
   fields: {
@@ -9,0 +3,0 @@ groupField: {

dist/esm/example-types/results/highlighting/util.js

 import _ from "lodash/fp.js";
 import F from "futil";
-let isLeafField = (field) => !!field?.elasticsearch?.dataType || !!field?.elasticsearch?.mapping?.type;
-let isBlobField = (field) => field?.subType === "blob" && isLeafField(field);
+let getFieldType = (field) => field?.elasticsearch?.dataType || field?.elasticsearch?.mapping?.type;
+let isBlobField = (field) => field?.subType === "blob" && !!getFieldType(field);
 let isArrayField = (field) => field?.subType === "array";
-let isArrayOfScalarsField = (field) => isArrayField(field) && isLeafField(field);
-let isArrayOfObjectsField = (field) => isArrayField(field) && !isLeafField(field);
+let isArrayOfScalarsField = (field) => isArrayField(field) && !!getFieldType(field);
+let isArrayOfObjectsField = (field) => isArrayField(field) && !getFieldType(field);
 let stripParentPath = _.curry(
@@ -75,2 +75,3 @@ (parentPath, path) => _.startsWith(`${parentPath}.`, path) ? path.slice(parentPath.length + 1) : void 0
   getArrayOfObjectsPathsMap,
+  getFieldType,
   getNestedPathsMap,
@@ -81,3 +82,2 @@ isArrayField,
   isBlobField,
-  isLeafField,
   mergeHighlights,
@@ -84,0 +84,0 @@ stripParentPath,

package.json

 {
   "name": "contexture-elasticsearch",
-  "version": "1.26.0",
+  "version": "1.27.0",
   "description": "ElasticSearch Provider for Contexture",
@@ -5,0 +5,0 @@ "type": "module",

No alert changes

Worsened metrics

Total package byte prevSize

1573635

-0.01%

Lines of code

44220

-0.01%

No dependency changes