mongo2elastic - npm Package Compare versions

Comparing version 0.19.0 to 0.20.0

CHANGELOG.md

@@ -0,4 +1,9 @@
+# 0.20.0
+
+- Added `passthrough` option to allow for customization of field mappings.
+- Use `_.merge` instead of flattened key path.
+
 # 0.19.0
 
-- Use flattened key path for ignore_malformed.
+- Use flattened key path for `ignore_malformed`.
 
@@ -5,0 +10,0 @@ # 0.18.0

dist/convertSchema.js

@@ -15,15 +15,8 @@ import _ from 'lodash/fp.js';
 };
-const expandedTextType = {
-    type: 'text',
-    fields: {
-        keyword: {
-            type: 'keyword',
-            ignore_above: 256,
-        },
-    },
-};
-const convertSchemaNode = (obj) => {
+const convertSchemaNode = (obj, options) => {
+    const field = options.passthrough ? _.pick(options.passthrough, obj) : {};
     if (obj.bsonType === 'object') {
+        // Use flattened type since object can have arbitrary keys
         if (obj?.additionalProperties !== false) {
-            return { type: 'flattened' };
+            return { type: 'flattened', ...field };
         }
@@ -34,10 +27,20 @@ return _.pick(['properties'], obj);
     if (obj.bsonType === 'string' && obj.enum) {
-        return { type: 'keyword' };
+        return { type: 'keyword', ...field };
     }
     const elasticType = bsonTypeToElastic[obj.bsonType];
+    // Add keyword sub-field to text type automatically
     if (elasticType === 'text') {
-        return expandedTextType;
+        return {
+            type: 'text',
+            fields: {
+                keyword: {
+                    type: 'keyword',
+                    ignore_above: 256,
+                },
+            },
+            ...field,
+        };
     }
     if (elasticType) {
-        return { type: elasticType };
+        return { type: elasticType, ...field };
     }
@@ -80,5 +83,5 @@ };
                 // Unwrap arrays since ES doesn't support explicit array fields
-                return convertSchemaNode(val.items);
+                return convertSchemaNode(val.items, options);
             }
-            return convertSchemaNode(val);
+            return convertSchemaNode(val, options);
         }
@@ -85,0 +88,0 @@ return val;

dist/syncData.js

@@ -14,6 +14,9 @@ import _ from 'lodash/fp.js';
             body: {
-                settings: {
-                    'index.mapping.ignore_malformed': true,
-                    ...settings,
-                },
+                settings: _.merge({
+                    index: {
+                        mapping: {
+                            ignore_malformed: true,
+                        },
+                    },
+                }, settings),
             },
@@ -20,0 +23,0 @@ };

dist/types.d.ts

@@ -13,3 +13,4 @@ import type { Document } from 'mongodb';
     overrides?: Override[];
+    passthrough?: string[];
 }
 export declare type Events = 'process' | 'error';

package.json

 {
   "name": "mongo2elastic",
-  "version": "0.19.0",
+  "version": "0.20.0",
   "description": "Sync MongoDB collections to Elasticsearch",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

src/convertSchema.test.ts

@@ -37,20 +37,27 @@ import { convertSchema } from './convertSchema.js'
                   bsonType: 'string',
+                  copy_to: 'full_address',
                 },
                 address2: {
                   bsonType: 'string',
+                  copy_to: 'full_address',
                 },
                 city: {
                   bsonType: 'string',
+                  copy_to: 'full_address',
                 },
                 county: {
                   bsonType: 'string',
+                  copy_to: 'full_address',
                 },
                 state: {
                   bsonType: 'string',
+                  copy_to: 'full_address',
                 },
                 zip: {
                   bsonType: 'string',
+                  copy_to: 'full_address',
                 },
                 country: {
                   bsonType: 'string',
+                  copy_to: 'full_address',
                 },
@@ -199,2 +206,3 @@ latitude: {
       ],
+      passthrough: ['copy_to'],
     }
@@ -222,2 +230,3 @@ expect(convertSchema(schema, options)).toEqual({
                   fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                  copy_to: 'full_address',
                 },
@@ -227,2 +236,3 @@ address2: {
                   fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                  copy_to: 'full_address',
                 },
@@ -232,2 +242,3 @@ city: {
                   fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                  copy_to: 'full_address',
                 },
@@ -237,2 +248,3 @@ county: {
                   fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                  copy_to: 'full_address',
                 },
@@ -242,2 +254,3 @@ state: {
                   fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                  copy_to: 'full_address',
                 },
@@ -247,2 +260,3 @@ zip: {
                   fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                  copy_to: 'full_address',
                 },
@@ -252,2 +266,3 @@ country: {
                   fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                  copy_to: 'full_address',
                 },
@@ -254,0 +269,0 @@ latitude: { type: 'double' },

src/convertSchema.ts

@@ -19,16 +19,11 @@ import _ from 'lodash/fp.js'
 
-const expandedTextType = {
-  type: 'text',
-  fields: {
-    keyword: {
-      type: 'keyword',
-      ignore_above: 256,
-    },
-  },
-}
-
-const convertSchemaNode = (obj: Record<string, any>) => {
+const convertSchemaNode = (
+  obj: Record<string, any>,
+  options: ConvertOptions
+) => {
+  const field = options.passthrough ? _.pick(options.passthrough, obj) : {}
   if (obj.bsonType === 'object') {
+    // Use flattened type since object can have arbitrary keys
     if (obj?.additionalProperties !== false) {
-      return { type: 'flattened' }
+      return { type: 'flattened', ...field }
     }
@@ -39,10 +34,21 @@ return _.pick(['properties'], obj)
   if (obj.bsonType === 'string' && obj.enum) {
-    return { type: 'keyword' }
+    return { type: 'keyword', ...field }
   }
+
   const elasticType = bsonTypeToElastic[obj.bsonType]
+  // Add keyword sub-field to text type automatically
   if (elasticType === 'text') {
-    return expandedTextType
+    return {
+      type: 'text',
+      fields: {
+        keyword: {
+          type: 'keyword',
+          ignore_above: 256,
+        },
+      },
+      ...field,
+    }
   }
   if (elasticType) {
-    return { type: elasticType }
+    return { type: elasticType, ...field }
   }
@@ -91,5 +97,5 @@ }
         // Unwrap arrays since ES doesn't support explicit array fields
-        return convertSchemaNode(val.items)
+        return convertSchemaNode(val.items, options)
       }
-      return convertSchemaNode(val)
+      return convertSchemaNode(val, options)
     }
@@ -96,0 +102,0 @@ return val

src/syncData.ts

@@ -31,6 +31,12 @@ import _ from 'lodash/fp.js'
       body: {
-        settings: {
-          'index.mapping.ignore_malformed': true,
-          ...settings,
-        },
+        settings: _.merge(
+          {
+            index: {
+              mapping: {
+                ignore_malformed: true,
+              },
+            },
+          },
+          settings
+        ),
       },
@@ -37,0 +43,0 @@ }

src/types.ts

@@ -16,4 +16,5 @@ import type { Document } from 'mongodb'
   overrides?: Override[]
+  passthrough?: string[]
 }
 
 export type Events = 'process' | 'error'

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

32352

increased by5.17%

Lines of code

869

increased by4.2%

No dependency changes