mongo2elastic - npm Package Compare versions

Comparing version 0.15.0 to 0.16.0

dist/convertSchema.d.ts

@@ -1,4 +0,2 @@
-import { JSONSchema } from 'mongochangestream';
-export declare const convertSchema: (jsonSchema: JSONSchema) => {
-    mappings: object;
-};
+import { estypes } from '@elastic/elasticsearch';
+export declare const convertSchema: (jsonSchema: object) => Record<string, estypes.MappingProperty>;

dist/convertSchema.js

@@ -24,16 +24,17 @@ import _ from 'lodash/fp.js';
 };
-const getElasticType = (obj) => {
-    if (obj.bsonType === 'object' && obj?.additionalProperties !== false) {
-        return 'flattened';
+const convertSchemaNode = (obj) => {
+    if (obj.bsonType === 'object') {
+        if (obj?.additionalProperties !== false) {
+            return { type: 'flattened' };
+        }
+        return _.pick(['properties'], obj);
     }
     const elasticType = bsonTypeToElastic[obj.bsonType];
-    return elasticType === 'text' ? expandedTextType : elasticType;
+    if (elasticType === 'text') {
+        return expandedTextType;
+    }
+    if (elasticType) {
+        return { type: elasticType };
+    }
 };
-const convertSchemaNode = (jsonSchema) => {
-    const elasticType = getElasticType(jsonSchema);
-    return {
-        ..._.pick(['properties'], jsonSchema),
-        ...(elasticType && { type: elasticType }),
-    };
-};
 export const convertSchema = (jsonSchema) => {
@@ -56,4 +57,3 @@ const mapper = (node) => {
     // Recursively convert the schema
-    const mappings = map(jsonSchema, mapper);
-    return { mappings };
+    return map(jsonSchema, mapper);
 };

dist/syncData.d.ts

@@ -28,2 +28,6 @@ import type { Collection, Document } from 'mongodb';
     ignoreMalformed: () => Promise<void>;
+    /**
+     * Create mapping from MongoDB JSON schema
+     */
+    createMappingFromSchema: (jsonSchema: object) => Promise<elasticsearch.estypes.IndicesResponseBase>;
     keys: {
@@ -30,0 +34,0 @@ scanCompletedKey: string;

dist/syncData.js

 import _ from 'lodash/fp.js';
 import mongoChangeStream from 'mongochangestream';
 import { indexFromCollection } from './util.js';
+import { convertSchema } from './convertSchema.js';
 import EventEmitter from 'eventemitter3';
@@ -24,2 +25,6 @@ export const initSync = (redis, collection, elastic, options = {}) => {
     };
+    const createMappingFromSchema = async (jsonSchema) => {
+        const mappings = convertSchema(jsonSchema);
+        return elastic.indices.putMapping({ index, ...mappings });
+    };
     /**
@@ -103,2 +108,6 @@ * Process a change stream event.
         ignoreMalformed,
+        /**
+         * Create mapping from MongoDB JSON schema
+         */
+        createMappingFromSchema,
         keys: sync.keys,
@@ -105,0 +114,0 @@ reset: sync.reset,

package.json

 {
   "name": "mongo2elastic",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "Sync MongoDB collections to Elasticsearch",
@@ -5,0 +5,0 @@ "main": "dist/index.js",

src/convertSchema.test.ts

@@ -95,5 +95,2 @@ import { convertSchema } from './convertSchema.js'
               },
-              subscriptionStatus: {
-                bsonType: 'string',
-              },
             },
@@ -116,151 +113,78 @@ },
     expect(convertSchema(schema)).toEqual({
-      mappings: {
-        properties: {
-          parentId: { type: 'keyword' },
-          name: {
-            type: {
-              type: 'text',
-              fields: { keyword: { type: 'keyword', ignore_above: 256 } },
-            },
-          },
-          subType: {
-            type: {
-              type: 'text',
-              fields: { keyword: { type: 'keyword', ignore_above: 256 } },
-            },
-          },
-          numberOfEmployees: {
-            type: {
-              type: 'text',
-              fields: { keyword: { type: 'keyword', ignore_above: 256 } },
-            },
-          },
-          addresses: {
-            properties: {
-              address: {
-                properties: {
-                  address1: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
-                  address2: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
-                  city: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
-                  county: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
-                  state: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
-                  zip: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
-                  country: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
-                  latitude: { type: 'long' },
-                  longitude: { type: 'long' },
-                  timezone: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
+      properties: {
+        parentId: { type: 'keyword' },
+        name: {
+          type: 'text',
+          fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+        },
+        subType: {
+          type: 'text',
+          fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+        },
+        numberOfEmployees: {
+          type: 'text',
+          fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+        },
+        addresses: {
+          properties: {
+            address: {
+              properties: {
+                address1: {
+                  type: 'text',
+                  fields: { keyword: { type: 'keyword', ignore_above: 256 } },
                 },
-              },
-              name: {
-                type: {
+                address2: {
                   type: 'text',
                   fields: { keyword: { type: 'keyword', ignore_above: 256 } },
                 },
-              },
-              isPrimary: { type: 'boolean' },
-            },
-          },
-          logo: {
-            type: {
-              type: 'text',
-              fields: { keyword: { type: 'keyword', ignore_above: 256 } },
-            },
-          },
-          verified: { type: 'boolean' },
-          partner: {
-            type: {
-              type: 'text',
-              fields: { keyword: { type: 'keyword', ignore_above: 256 } },
-            },
-          },
-          integrations: {
-            properties: {
-              stripe: {
-                properties: {
-                  priceId: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
-                  subscriptionStatus: {
-                    type: {
-                      type: 'text',
-                      fields: {
-                        keyword: { type: 'keyword', ignore_above: 256 },
-                      },
-                    },
-                  },
+                city: {
+                  type: 'text',
+                  fields: { keyword: { type: 'keyword', ignore_above: 256 } },
                 },
-                type: 'flattened',
+                county: {
+                  type: 'text',
+                  fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                },
+                state: {
+                  type: 'text',
+                  fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                },
+                zip: {
+                  type: 'text',
+                  fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                },
+                country: {
+                  type: 'text',
+                  fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                },
+                latitude: { type: 'long' },
+                longitude: { type: 'long' },
+                timezone: {
+                  type: 'text',
+                  fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+                },
               },
             },
-            type: 'flattened',
-          },
-          createdAt: { type: 'date' },
-          permissions: {
-            type: {
+            name: {
               type: 'text',
               fields: { keyword: { type: 'keyword', ignore_above: 256 } },
             },
+            isPrimary: { type: 'boolean' },
           },
         },
+        logo: {
+          type: 'text',
+          fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+        },
+        verified: { type: 'boolean' },
+        partner: {
+          type: 'text',
+          fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+        },
+        integrations: { type: 'flattened' },
+        createdAt: { type: 'date' },
+        permissions: {
+          type: 'text',
+          fields: { keyword: { type: 'keyword', ignore_above: 256 } },
+        },
       },
@@ -267,0 +191,0 @@ })

src/convertSchema.ts

 import _ from 'lodash/fp.js'
 import { map, Node } from 'obj-walker'
-import { JSONSchema } from 'mongochangestream'
+import { estypes } from '@elastic/elasticsearch'
 
@@ -28,19 +28,19 @@ const bsonTypeToElastic: Record<string, string> = {
 
-const getElasticType = (obj: Record<string, any>) => {
-  if (obj.bsonType === 'object' && obj?.additionalProperties !== false) {
-    return 'flattened'
+const convertSchemaNode = (obj: Record<string, any>) => {
+  if (obj.bsonType === 'object') {
+    if (obj?.additionalProperties !== false) {
+      return { type: 'flattened' }
+    }
+    return _.pick(['properties'], obj)
   }
   const elasticType = bsonTypeToElastic[obj.bsonType]
-  return elasticType === 'text' ? expandedTextType : elasticType
-}
-
-const convertSchemaNode = (jsonSchema: JSONSchema) => {
-  const elasticType = getElasticType(jsonSchema)
-  return {
-    ..._.pick(['properties'], jsonSchema),
-    ...(elasticType && { type: elasticType }),
+  if (elasticType === 'text') {
+    return expandedTextType
   }
+  if (elasticType) {
+    return { type: elasticType }
+  }
 }
 
-export const convertSchema = (jsonSchema: JSONSchema) => {
+export const convertSchema = (jsonSchema: object) => {
   const mapper = (node: Node) => {
@@ -62,4 +62,3 @@ const { key, val, parents } = node
   // Recursively convert the schema
-  const mappings = map(jsonSchema, mapper)
-  return { mappings }
+  return map(jsonSchema, mapper) as Record<string, estypes.MappingProperty>
 }

src/syncData.ts

@@ -14,2 +14,3 @@ import _ from 'lodash/fp.js'
 import { indexFromCollection } from './util.js'
+import { convertSchema } from './convertSchema.js'
 import EventEmitter from 'eventemitter3'
@@ -42,2 +43,7 @@
   }
+
+  const createMappingFromSchema = async (jsonSchema: object) => {
+    const mappings = convertSchema(jsonSchema)
+    return elastic.indices.putMapping({ index, ...mappings })
+  }
   /**
@@ -122,2 +128,6 @@ * Process a change stream event.
     ignoreMalformed,
+    /**
+     * Create mapping from MongoDB JSON schema
+     */
+    createMappingFromSchema,
     keys: sync.keys,
@@ -124,0 +134,0 @@ reset: sync.reset,

No alert changes

Worsened metrics

Total package byte prevSize

24488

decreased by-5.51%

Lines of code

671

decreased by-7.7%

No dependency changes