@medable/mdctl-import-adapter - npm Package Compare versions

Comparing version 1.0.0 to 1.0.1

index.js

 const { Transform } = require('stream'),
       EventEmitter = require('events'),
+      globby = require('globby'),
       mime = require('mime-types'),
@@ -9,6 +10,12 @@ uuid = require('uuid'),
       { ImportSection } = require('@medable/mdctl-core/streams/section'),
-      { stringifyContent, parseString } = require('@medable/mdctl-core-utils/values'),
+      { parseString } = require('@medable/mdctl-core-utils/values'),
       { md5FileHash } = require('@medable/mdctl-core-utils/crypto'),
       { privatesAccessor } = require('@medable/mdctl-core-utils/privates'),
-      { OutputStream } = require('./chunk-stream')
+      { Fault } = require('@medable/mdctl-core'),
+      { OutputStream } = require('@medable/mdctl-core/streams/chunk-stream'),
+      KNOWN_FILES = {
+        data: 'data/**/*.{json,yaml}',
+        objects: 'env/**/*.{json,yaml}',
+        manifest: 'manifest.{json,yaml}'
+      }
 
@@ -32,5 +39,9 @@ class ImportFileTransformStream extends Transform {
   _transform(chunk, enc, callback) {
-    const { metadata, basePath, file } = privatesAccessor(this),
-          content = parseString(chunk, metadata.format)
-    this.push(new ImportSection(content, content.object, file, basePath))
+    const { metadata, basePath, file } = privatesAccessor(this)
+    try {
+      const content = parseString(chunk, metadata.format)
+      this.push(new ImportSection(content, content.object, file, basePath))
+    } catch (e) {
+      console.log(e, chunk.toString())
+    }
     callback()
@@ -51,5 +62,4 @@ }
       metadata: {},
-      blobs: [],
       index: 0,
-      blobIndex: 0
+      preparedChunks: []
     })
@@ -69,11 +79,8 @@
 
-  getAssetStream(ef) {
-    const { metadata } = privatesAccessor(this),
-          outS = new OutputStream({
-            ndjson: false,
-            template: ef
-          })
-    outS.write(stringifyContent(ef, metadata.format))
-    outS.end()
-    return outS
+  static getAssetStream(ef) {
+    const outS = new OutputStream({
+      ndjson: false,
+      template: ef
+    })
+    return ef.data.pipe(outS)
   }
@@ -92,61 +99,69 @@
 
+  async loadFileContent(f) {
+    const section = await this.loadFile(f)
+    await this.loadFacets(section)
+    await this.loadScripts(section)
+    await this.loadTemplates(section)
+    let results = [],
+        blobResults = []
+    results.push(section.content)
 
-  get blobs() {
-    return privatesAccessor(this).blobs
+    if (section && section.facets && section.facets.length) {
+      results = _.concat(
+        results,
+        section.facets
+      )
+      if (section.extraFiles && section.extraFiles.length) {
+        // const blobs = this.getBlobData(section.extraFiles)
+        blobResults = _.concat(blobResults, _.map(section.extraFiles, (ef) => {
+          return ImportFileTreeAdapter.getAssetStream(ef)
+        }))
+      }
+    }
+    return { results, blobResults }
   }
 
+  async prepareChunks() {
+    const { files, preparedChunks } = privatesAccessor(this),
+          promises = []
+    if (preparedChunks.length) {
+      return Promise.resolve(preparedChunks)
+    }
+    files.forEach((f) => {
+      promises.push(this.loadFileContent(f))
+    })
+    return Promise.all(promises).then((res) => {
+      const results = _.flatten(_.map(res, 'results')),
+            blobs = _.flatten(_.flatten(_.map(res, 'blobResults'))),
+            data = _.concat(results, blobs)
+      privatesAccessor(this, 'preparedChunks', data)
+      return data
+    })
+  }
+
   async getChunks() {
-    const { files, index } = privatesAccessor(this),
-          result = {
-            done: false,
-            value: []
-          }
-    let { blobs } = privatesAccessor(this)
-    if (files.length > index) {
-      // Increment index processing
+    const { index } = privatesAccessor(this),
+          chunks = await this.prepareChunks()
+    if (chunks.length > index) {
       privatesAccessor(this, 'index', index + 1)
-
-      const f = files[index],
-            section = await this.loadFile(f)
-
-      await this.loadFacets(section)
-      await this.loadScripts(section)
-      await this.loadTemplates(section)
-      result.value.push(section.content)
-
-      if (section && section.facets && section.facets.length) {
-        result.value = _.concat(
-          result.value,
-          section.facets
-        )
-        if (section.extraFiles && section.extraFiles.length) {
-          blobs = _.concat(blobs, section.extraFiles)
-        }
-        privatesAccessor(this, 'blobs', blobs)
-      }
-
-      return result
-
+      return Promise.resolve({
+        done: false,
+        value: chunks[index]
+      })
     }
-    return {
-      value: null,
-      done: true
-    }
+    return Promise.resolve({
+      done: true,
+      value: null
+    })
   }
 
   walkFiles(dir) {
-    const files = fs.readdirSync(dir)
-    files.forEach((f) => {
-      if (f.indexOf('.') !== 0) {
-        const pathFile = `${dir}/${f}`
-        if (fs.statSync(pathFile).isDirectory()) {
-          this.walkFiles(pathFile)
-        } else {
-          const type = mime.lookup(pathFile)
-          if (type === 'application/json' || ['text/yaml', 'application/yaml'].indexOf(type) > -1) {
-            privatesAccessor(this, 'files').push(pathFile)
-          }
-        }
-      }
-    })
+    const { format } = privatesAccessor(this),
+          files = globby.sync([KNOWN_FILES.manifest, KNOWN_FILES.objects], { cwd: dir }),
+          mappedFiles = _.map(files, f => `${dir}/${f}`),
+          existsManifest = _.find(files, f => f.indexOf(`manifest.${format}`))
+    if (!existsManifest) {
+      throw Fault.create({ code: 'KMissingManifest', reason: 'There is no manifest file present on folder' })
+    }
+    privatesAccessor(this, 'files', mappedFiles)
   }
@@ -209,3 +224,3 @@
                     streamId: resourceKey,
-                    data: fs.readFileSync(localFile),
+                    data: fs.createReadStream(localFile),
                     object: 'stream'
@@ -233,6 +248,5 @@ }
       if (!_.isObject(n.value)) {
-        const parent = this.getParentFromPath(chunk, n.path)
-        if (parent.script.indexOf('/env') === 0) {
-          const jsFile = `${basePath}${parent.script}`
-          parent.script = fs.readFileSync(jsFile).toString()
+        if (n.value.indexOf('/env') === 0) {
+          const jsFile = `${basePath}${n.value}`
+          jp.value(content, jp.stringify(n.path), fs.readFileSync(jsFile).toString())
         }
@@ -239,0 +253,0 @@ }

package.json

 {
   "name": "@medable/mdctl-import-adapter",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "Medable Developer Client Tools :: Import Adapter",
@@ -28,5 +28,6 @@ "repository": {
   "dependencies": {
-    "@medable/mdctl-core": "^1.0.0",
-    "@medable/mdctl-core-utils": "^1.0.0",
+    "@medable/mdctl-core": "^1.0.1",
+    "@medable/mdctl-core-utils": "^1.0.1",
     "clone": "^2.1.2",
+    "globby": "^9.1.0",
     "jsonpath": "^1.0.0",
@@ -51,3 +52,3 @@ "lodash": "^4.17.11",
   },
-  "gitHead": "5357404ee705e2db77860f94a43352db8c871205"
+  "gitHead": "3cc9bffc20b1026350a2fd2399f126c53a20cd35"
 }

No alert changes

Worsened metrics

Total package byte prevSize

11125

decreased by-25.17%

Dependency count

9

increased by12.5%

Number of package files

4

decreased by-20%

Lines of code

244

decreased by-32.6%

Dependency changes

• + Addedglobby@^9.1.0

• + Added@mrmlnc/readdir-enhanced@2.2.1(transitive)

• + Added@nodelib/fs.stat@1.1.3(transitive)

• + Added@types/glob@7.2.0(transitive)

• + Added@types/minimatch@5.1.2(transitive)

• + Added@types/node@20.12.12(transitive)

• + Addedarr-diff@4.0.0(transitive)

• + Addedarr-flatten@1.1.0(transitive)

• + Addedarr-union@3.1.0(transitive)

• + Addedarray-union@1.0.2(transitive)

• + Addedarray-uniq@1.0.3(transitive)

• + Addedarray-unique@0.3.2(transitive)

• + Addedassign-symbols@1.0.0(transitive)

• + Addedatob@2.1.2(transitive)

• + Addedbalanced-match@1.0.2(transitive)

• + Addedbase@0.11.2(transitive)

• + Addedbrace-expansion@1.1.11(transitive)

• + Addedbraces@2.3.2(transitive)

• + Addedcache-base@1.0.1(transitive)

• + Addedcall-me-maybe@1.0.2(transitive)

• + Addedclass-utils@0.3.6(transitive)

• + Addedcollection-visit@1.0.0(transitive)

• + Addedcomponent-emitter@1.3.1(transitive)

• + Addedconcat-map@0.0.1(transitive)

• + Addedcopy-descriptor@0.1.1(transitive)

• + Addeddebug@2.6.9(transitive)

• + Addeddecode-uri-component@0.2.2(transitive)

• + Addeddefine-property@0.2.51.0.02.0.2(transitive)

• + Addeddir-glob@2.2.2(transitive)

• + Addedexpand-brackets@2.1.4(transitive)

• + Addedextend-shallow@2.0.13.0.2(transitive)

• + Addedextglob@2.0.4(transitive)

• + Addedfast-glob@2.2.7(transitive)

• + Addedfill-range@4.0.0(transitive)

• + Addedfor-in@1.0.2(transitive)

• + Addedfragment-cache@0.2.1(transitive)

• + Addedfs.realpath@1.0.0(transitive)

• + Addedfunction-bind@1.1.2(transitive)

• + Addedget-value@2.0.6(transitive)

• + Addedglob@7.2.3(transitive)

• + Addedglob-parent@3.1.0(transitive)

• + Addedglob-to-regexp@0.3.0(transitive)

• + Addedglobby@9.2.0(transitive)

• + Addedhas-value@0.3.11.0.0(transitive)

• + Addedhas-values@0.1.41.0.0(transitive)

• + Addedhasown@2.0.2(transitive)

• + Addedignore@4.0.6(transitive)

• + Addedinflight@1.0.6(transitive)

• + Addedis-accessor-descriptor@1.0.1(transitive)

• + Addedis-buffer@1.1.6(transitive)

• + Addedis-data-descriptor@1.0.1(transitive)

• + Addedis-descriptor@0.1.71.0.3(transitive)

• + Addedis-extendable@0.1.11.0.1(transitive)

• + Addedis-extglob@2.1.1(transitive)

• + Addedis-glob@3.1.04.0.3(transitive)

• + Addedis-number@3.0.0(transitive)

• + Addedis-plain-object@2.0.4(transitive)

• + Addedis-windows@1.0.2(transitive)

• + Addedisobject@2.1.03.0.1(transitive)

• + Addedkind-of@3.2.24.0.06.0.3(transitive)

• + Addedmap-cache@0.2.2(transitive)

• + Addedmap-visit@1.0.0(transitive)

• + Addedmerge2@1.4.1(transitive)

• + Addedmicromatch@3.1.10(transitive)

• + Addedminimatch@3.1.2(transitive)

• + Addedmixin-deep@1.3.2(transitive)

• + Addedms@2.0.0(transitive)

• + Addednanomatch@1.2.13(transitive)

• + Addedobject-copy@0.1.0(transitive)

• + Addedobject-visit@1.0.1(transitive)

• + Addedobject.pick@1.3.0(transitive)

• + Addedonce@1.4.0(transitive)

• + Addedpascalcase@0.1.1(transitive)

• + Addedpath-dirname@1.0.2(transitive)

• + Addedpath-is-absolute@1.0.1(transitive)

• + Addedpath-type@3.0.0(transitive)

• + Addedpify@3.0.04.0.1(transitive)

• + Addedposix-character-classes@0.1.1(transitive)

• + Addedregex-not@1.0.2(transitive)

• + Addedrepeat-element@1.1.4(transitive)

• + Addedrepeat-string@1.6.1(transitive)

• + Addedresolve-url@0.2.1(transitive)

• + Addedret@0.1.15(transitive)

• + Addedsafe-regex@1.1.0(transitive)

• + Addedset-value@2.0.1(transitive)

• + Addedslash@2.0.0(transitive)

• + Addedsnapdragon@0.8.2(transitive)

• + Addedsnapdragon-node@2.1.1(transitive)

• + Addedsnapdragon-util@3.0.1(transitive)

• + Addedsource-map@0.5.7(transitive)

• + Addedsource-map-resolve@0.5.3(transitive)

• + Addedsource-map-url@0.4.1(transitive)

• + Addedsplit-string@3.1.0(transitive)

• + Addedstatic-extend@0.1.2(transitive)

• + Addedto-object-path@0.3.0(transitive)

• + Addedto-regex@3.0.2(transitive)

• + Addedto-regex-range@2.1.1(transitive)

• + Addedundici-types@5.26.5(transitive)

• + Addedunion-value@1.0.1(transitive)

• + Addedunset-value@1.0.0(transitive)

• + Addedurix@0.1.0(transitive)

• + Addeduse@3.1.1(transitive)

• + Addedwrappy@1.0.2(transitive)

• Updated@medable/mdctl-core@^1.0.1

• Updated@medable/mdctl-core-utils@^1.0.1