@wmfs/xml-flatten2csv - npm Package Compare versions

Comparing version 1.0.1 to 1.1.0

lib/index.js

@@ -116,3 +116,4 @@
         xmllang: options.xmllang,
-        transform: options.transform
+        transform: options.transform,
+        required: options.required
       }
@@ -119,0 +120,0 @@ )

lib/xml-transform-to-csv.js

@@ -8,25 +8,39 @@
 
-function * processPivot (tree, pivotPath, selectPaths) {
+function pivotFields (tree, pivotPath, selectPaths, requiredPaths) {
   const csv = flattenJson(tree, pivotPath, selectPaths)
 
-  yield (csv)
+  if (requiredPaths) {
+    const requiredFields = flattenJson(tree, pivotPath, requiredPaths)
+      .filter(f => f)
+    if (requiredFields.length !== requiredPaths.length) { // uh-oh
+      return null
+    }
+  }
+
+  return csv
+}
+
+function * processPivot (tree, pivotPath, selectPaths, requiredPaths) {
+  const csv = pivotFields(tree, pivotPath, selectPaths, requiredPaths)
+
+  if (csv) yield (csv)
 } // processPivot
 
-function * processPivotArray (tree, pivotPath, count, selectPaths) {
+function * processPivotArray (tree, pivotPath, count, selectPaths, requiredPaths) {
   for (let i = 0; i !== count; ++i) {
     const contextPath = `${pivotPath}[${i}]`
 
-    const csv = flattenJson(tree, contextPath, selectPaths)
+    const csv = pivotFields(tree, contextPath, selectPaths, requiredPaths)
 
-    yield (csv)
+    if (csv) yield (csv)
   }
 } // processPivotArray
 
-function * processSteppedArray (tree, pivotPath, selectPaths) {
+function * processSteppedArray (tree, pivotPath, selectPaths, requiredPaths) {
   const pivotPaths = jp.paths(tree, pivotPath).map(path => jp.stringify(path))
 
   for (const contextPath of pivotPaths) {
-    const csv = flattenJson(tree, contextPath, selectPaths)
+    const csv = pivotFields(tree, contextPath, selectPaths, requiredPaths)
 
-    yield (csv)
+    if (csv) yield (csv)
   }
@@ -38,3 +52,4 @@ } // processSteppedArray
   pivotPath,
-  selectPaths
+  selectPaths,
+  requiredPaths
 ) {
@@ -46,7 +61,7 @@ const cleanTree = simplifyJson(subTree)
   if (pivots.length > 1) {
-    yield * processSteppedArray(cleanTree, pivotPath, selectPaths)
+    yield * processSteppedArray(cleanTree, pivotPath, selectPaths, requiredPaths)
   } else if (Array.isArray(pivots[0])) {
-    yield * processPivotArray(cleanTree, pivotPath, pivots[0].length, selectPaths)
+    yield * processPivotArray(cleanTree, pivotPath, pivots[0].length, selectPaths, requiredPaths)
   } else if (pivots[0]) {
-    yield * processPivot(cleanTree, pivotPath, selectPaths)
+    yield * processPivot(cleanTree, pivotPath, selectPaths, requiredPaths)
   }
@@ -66,2 +81,6 @@ } // processSubtree
 
+  const required = (options && Array.isArray(options.required) && options.required.length !== 0)
+    ? options.required
+    : null
+
   return new EachPromise((each, resolve, reject) => {
@@ -71,3 +90,3 @@ xmlSubtreeProcessor(inputStream, elementName, options)
         const transformedSubTree = treeTransform(subTree)
-        for (const line of processSubtree(transformedSubTree, pivotPath, selectPaths)) {
+        for (const line of processSubtree(transformedSubTree, pivotPath, selectPaths, required)) {
           each(line)
@@ -74,0 +93,0 @@ }

package.json

 {
   "name": "@wmfs/xml-flatten2csv",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Flattened XML file to CSV file",
@@ -29,11 +29,14 @@ "author": "West Midlands Fire Service",
   "devDependencies": {
-    "chai": "4.1.2",
-    "chai-string": "^1.4.0",
-    "codecov": "3.0.3",
+    "chai": "4.2.0",
+    "chai-string": "1.5.0",
+    "codecov": "3.1.0",
+    "conventional-changelog-metahub": "2.0.2",
     "cz-conventional-changelog": "2.1.0",
     "dirty-chai": "2.0.1",
     "mocha": "5.2.0",
-    "nyc": "12.0.2",
-    "semantic-release": "^15.5.2",
-    "standard": "11.0.1"
+    "nyc": "13.1.0",
+    "semantic-release": "15.13.1",
+    "standard": "12.0.1",
+    "@semantic-release/changelog": "3.0.1",
+    "@semantic-release/git": "7.0.6"
   },
@@ -44,3 +47,3 @@ "scripts": {
     "coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov",
-    "semantic-release": "semantic-release pre && npm publish && semantic-release post"
+    "semantic-release": "semantic-release"
   },
@@ -54,20 +57,3 @@ "config": {
     "access": "public"
-  },
-  "release": {
-    "analyzeCommits": {
-      "preset": "angular",
-      "releaseRules": [
-        {
-          "type": "build",
-          "scope": "deps",
-          "release": "minor"
-        },
-        {
-          "type": "build",
-          "scope": "deps-dev",
-          "release": "patch"
-        }
-      ]
-    }
   }
 }

test/xml-flatten2csv-tests.js

@@ -21,4 +21,4 @@ /* eslint-env mocha */
       ['@.Name', 'name', 'string'],
-      [{test: '@.Age<=16', value: 'yes'}, 'child', 'string'],
-      [{test: '@.Age>16', select: '@.Age'}, 'age', 'integer'],
+      [{ test: '@.Age<=16', value: 'yes' }, 'child', 'string'],
+      [{ test: '@.Age>16', select: '@.Age' }, 'age', 'integer'],
       ['@.Siblings[?(@.Sister === "Nediana")].Sister', 'okely-dokely', 'string'],
@@ -29,5 +29,5 @@ ['@.Siblings[?(@.Brother === "Bart")].Brother', 'eat-my-shorts', 'string']
       ['$.Title', 'title', 'string'],
-      [{select: '@.Name', transform: v => v.toUpperCase()}, 'name', 'string'],
-      [{test: '@.Age<=16', value: 'yes'}, 'child', 'string'],
-      [{test: '@.Age>16', select: '@.Age', transform: v => `${v} years old`}, 'age', 'string']
+      [{ select: '@.Name', transform: v => v.toUpperCase() }, 'name', 'string'],
+      [{ test: '@.Age<=16', value: 'yes' }, 'child', 'string'],
+      [{ test: '@.Age>16', select: '@.Age', transform: v => `${v} years old` }, 'age', 'string']
     ] ],
@@ -60,23 +60,70 @@ ['xml to csv, with top-level transform', 'simpsons-node-transforms.csv', [
       it(title, async () => {
-        const sourceFile = path.resolve(__dirname, 'fixtures', 'simpsons.xml')
-        const outputFile = path.resolve(__dirname, 'output', filename)
-        const expectedFile = path.resolve(__dirname, 'expected', filename)
+        await test(
+          'simpsons.xml',
+          filename,
+          'Episode',
+          '$.People.Person',
+          headerMap,
+          transformFn
+        )
+      })
+    }
 
-        if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
+    it('gml extract', async () => {
+      const root = 'Street'
+      const pivot = '$..StreetDescriptiveIdentifier'
+      const headerMap = [
+        ['$.usrn', 'usrn', 'number'],
+        ['$.changeType', 'changeType', 'string'],
+        ['$.state', 'state', 'number'],
+        ['$.stateDate', 'state_date', 'date'],
+        ['@.streetDescription.en', 'description', 'string', 'required'],
+        ['@.locality.en', 'locality', 'string', 'required'],
+        ['@.townName.en', 'town_name', 'string', 'required'],
+        ['@.administrativeArea.en', 'administrative_area', 'string']
+      ]
 
-        await xmlFlatten2csv({
-          xmlPath: sourceFile,
-          csvPath: outputFile,
-          rootXMLElement: 'Episode',
-          pivotPath: '$.People.Person',
-          headerMap: headerMap,
-          transform: transformFn
-        })
+      await test(
+        'ST2065.gml',
+        'gml-extract.csv',
+        root,
+        pivot,
+        headerMap,
+        null,
+        ['@.streetDescription.en']
+      )
+    })
 
-        const output = fs.readFileSync(outputFile, {encoding: 'utf8'}).split('\n').map(s => s.trim())
-        const expected = fs.readFileSync(expectedFile, {encoding: 'utf8'}).split('\n').map(s => s.trim())
+    async function test (
+      inputFilename,
+      outputFilename,
+      root,
+      pivot,
+      headerMap,
+      transformFn,
+      required
+    ) {
+      const sourceFile = path.resolve(__dirname, 'fixtures', inputFilename)
+      const outputFile = path.resolve(__dirname, 'output', outputFilename)
+      const expectedFile = path.resolve(__dirname, 'expected', outputFilename)
 
-        expect(output).to.eql(expected)
+      if (fs.existsSync(outputFile)) fs.unlinkSync(outputFile)
+
+      await xmlFlatten2csv({
+        xmlPath: sourceFile,
+        csvPath: outputFile,
+        rootXMLElement: root,
+        pivotPath: pivot,
+        headerMap: headerMap,
+        transform: transformFn,
+        namespace: 'strip',
+        xmllang: 'wrap',
+        required: required
       })
-    }
+
+      const output = fs.readFileSync(outputFile, { encoding: 'utf8' }).split('\n').map(s => s.trim())
+      const expected = fs.readFileSync(expectedFile, { encoding: 'utf8' }).split('\n').map(s => s.trim())
+
+      expect(output).to.eql(expected)
+    } // test
   })
@@ -83,0 +130,0 @@

test/xml-subtree-tests.js

@@ -56,3 +56,3 @@ /* eslint-env mocha */
       expect(tree).to.exist()
-      expect(tree).to.eql({'#text': 'Hello'})
+      expect(tree).to.eql({ '#text': 'Hello' })
     })
@@ -71,3 +71,3 @@
       expect(tree).to.exist()
-      expect(tree).to.eql([{'#text': 'Hello'}, {'#text': 'World!'}])
+      expect(tree).to.eql([{ '#text': 'Hello' }, { '#text': 'World!' }])
     })
@@ -88,4 +88,4 @@
         line: [
-          {lineItem: [{'#text': 'Hello'}]},
-          {lineItem: [{'#text': 'World!'}]}
+          { lineItem: [{ '#text': 'Hello' }] },
+          { lineItem: [{ '#text': 'World!' }] }
         ]
@@ -112,3 +112,3 @@ })
       expect(tree).to.eql({
-        line: [{'#text': 'Hello'}, {'#text': 'World!'}]
+        line: [{ '#text': 'Hello' }, { '#text': 'World!' }]
       })
@@ -129,3 +129,3 @@ })
       expect(tree).to.eql({
-        line: [{'#text': 'Hello ', strong: [{'#text': 'World!'}]}]
+        line: [{ '#text': 'Hello ', strong: [{ '#text': 'World!' }] }]
       })
@@ -146,3 +146,3 @@ })
       expect(tree).to.eql({
-        'a:line': [{'#text': 'Hello ', strong: [{'#text': 'World!'}]}]
+        'a:line': [{ '#text': 'Hello ', strong: [{ '#text': 'World!' }] }]
       })
@@ -162,3 +162,3 @@ })
       expect(tree).to.eql({
-        line: [{'#text': 'Hello ', strong: [{'#text': 'World!'}]}]
+        line: [{ '#text': 'Hello ', strong: [{ '#text': 'World!' }] }]
       })
@@ -178,3 +178,3 @@ })
       expect(tree).to.eql({
-        a__line: [{'#text': 'Hello ', b__strong: [{'#text': 'World!'}]}]
+        a__line: [{ '#text': 'Hello ', b__strong: [{ '#text': 'World!' }] }]
       })
@@ -195,3 +195,3 @@ })
       expect(tree).to.eql({
-        line: [{'#text': 'Hello ', strong: [{'#text': 'World!'}]}]
+        line: [{ '#text': 'Hello ', strong: [{ '#text': 'World!' }] }]
       })
@@ -211,3 +211,3 @@ })
       expect(tree).to.eql({
-        en: [{line: [{'#text': 'Hello ', strong: [{'#text': 'World!'}]}]}]
+        en: [{ line: [{ '#text': 'Hello ', strong: [{ '#text': 'World!' }] }] }]
       })
@@ -227,3 +227,3 @@ })
       expect(tree).to.eql({
-        line: [{'en': [{'#text': 'Hello ', strong: [{'#text': 'World!'}]}]}]
+        line: [{ 'en': [{ '#text': 'Hello ', strong: [{ '#text': 'World!' }] }] }]
       })
@@ -244,4 +244,4 @@ })
         line: [
-          {'en': [{'#text': 'Hello ', strong: [{'#text': 'World!'}]}]},
-          {'fr': [{'#text': 'Bonjour ', strong: [{'#text': 'tout le monde!'}]}]}
+          { 'en': [{ '#text': 'Hello ', strong: [{ '#text': 'World!' }] }] },
+          { 'fr': [{ '#text': 'Bonjour ', strong: [{ '#text': 'tout le monde!' }] }] }
         ]
@@ -248,0 +248,0 @@ })

test/xml-transform-tests.js

@@ -67,3 +67,3 @@ /* eslint-env mocha */
   '@.description',
-  {test: '@.description=="Pomelo"', value: 'LOVE IT'},
+  { test: '@.description=="Pomelo"', value: 'LOVE IT' },
   '$.price'
@@ -83,3 +83,3 @@ ]
   '@.description..en',
-  {test: '@.description.en=="Pomelo"', value: 'LOVE IT'},
+  { test: '@.description.en=="Pomelo"', value: 'LOVE IT' },
   '$.price'
@@ -86,0 +86,0 @@ ]

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

80540

increased by70.1%

Number of package files

30

increased by50%

Lines of code

1280

increased by7.83%

Worsened metrics

Dev dependency count

12

increased by33.33%

No dependency changes