@wmfs/supercopy - npm Package Compare versions

Comparing version 1.38.0 to 1.39.0

CHANGELOG.md

@@ -0,1 +1,49 @@
+# [1.39.0](https://github.com/wmfs/supercopy/compare/v1.38.0...v1.39.0) (2020-08-31)
+
+
+### ✨ Features
+
+* Added directoryNames option. ([956777b](https://github.com/wmfs/supercopy/commit/956777b2d5f6afddf365b6e2de07feafd29f52ec))
+* Added option.quote to change the quote default from " to whatever. ([fa41172](https://github.com/wmfs/supercopy/commit/fa4117206b88843c7ae0b179d1a9f6cb94e51b91))
+
+
+### 🐛 Bug Fixes
+
+* Tweak preprocess so it creates parent directories. ([3353676](https://github.com/wmfs/supercopy/commit/3353676a99b571f53f98645f8849e4f4b48a8fcd))
+
+
+### 🛠 Builds
+
+* **deps-dev:** Bump [@wmfs](https://github.com/wmfs)/hl-pg-client from 1.21.0 to 1.22.0 ([3560075](https://github.com/wmfs/supercopy/commit/3560075fab1808c64744ee5f0d8fcb6534f6e8e6))
+* **deps-dev:** Bump [@wmfs](https://github.com/wmfs)/hl-pg-client from 1.22.0 to 1.23.0 ([8ce3239](https://github.com/wmfs/supercopy/commit/8ce32393eb175ff82cd77ba153bb0212f5e6cfbf))
+* **deps-dev:** Bump codecov from 3.7.0 to 3.7.1 ([f94d9d8](https://github.com/wmfs/supercopy/commit/f94d9d81faff15ec10066c5d71b4f665c4ce7961))
+* **deps-dev:** Bump codecov from 3.7.1 to 3.7.2 ([eba119f](https://github.com/wmfs/supercopy/commit/eba119f4df44f7c0a9b4fbb594053a631457e5ae))
+* **deps-dev:** Bump cz-conventional-changelog from 3.2.0 to 3.2.1 ([5479925](https://github.com/wmfs/supercopy/commit/54799258097f732f4e80ee76f285a1b433e1a2e0))
+* **deps-dev:** Bump cz-conventional-changelog from 3.2.1 to 3.3.0 ([ff7d8c8](https://github.com/wmfs/supercopy/commit/ff7d8c88dc556c0077495e1e7b628e3b91e25752))
+* **deps-dev:** Bump mocha from 7.2.0 to 8.0.1 ([46f459b](https://github.com/wmfs/supercopy/commit/46f459be544ebefe6c6463315c7abb96cea04c87))
+* **deps-dev:** Bump mocha from 8.0.1 to 8.1.0 ([55f93a7](https://github.com/wmfs/supercopy/commit/55f93a79beb30e53fd89b833e4039e7a444e9776))
+* **deps-dev:** Bump mocha from 8.1.0 to 8.1.1 ([a12c1b5](https://github.com/wmfs/supercopy/commit/a12c1b5ec0cc7ff3413cd95c0b5e3cd42fad5736))
+* **deps-dev:** Bump mocha from 8.1.1 to 8.1.2 ([ddf7ddc](https://github.com/wmfs/supercopy/commit/ddf7ddc4cde81e28bb1a06e7ac610774746bdbb9))
+* **deps-dev:** Bump mocha from 8.1.2 to 8.1.3 ([8243fdb](https://github.com/wmfs/supercopy/commit/8243fdb54d90b3e9a067432ebec5984b06b02e67))
+* **deps-dev:** Bump nyc from 15.0.1 to 15.1.0 ([4b15ebd](https://github.com/wmfs/supercopy/commit/4b15ebd598b07110b6031c872bb95f3703878587))
+* **deps-dev:** Bump semantic-release from 17.0.8 to 17.1.0 ([b3d5543](https://github.com/wmfs/supercopy/commit/b3d554301a3148139452f4039ab55a02734b2a5a))
+* **deps-dev:** Bump semantic-release from 17.1.0 to 17.1.1 ([60519ea](https://github.com/wmfs/supercopy/commit/60519ea9268b1c6e337c2076584cbaf262ecc96a))
+
+
+### 📚 Documentation
+
+* Update README with quote option. ([d86f7ec](https://github.com/wmfs/supercopy/commit/d86f7ec9afa4b9913ef30b0cf8e5710d618ca0a8))
+
+
+### 🚨 Tests
+
+* Split up tests using describe blocks ([11ab3d3](https://github.com/wmfs/supercopy/commit/11ab3d3f819c8242f465eadb5769eb8fb5542a17))
+* Use promises rather than callbacks ([447bed5](https://github.com/wmfs/supercopy/commit/447bed5b7551f789f5d537a71f4a39daaa6524f9))
+
+
+### ⚙️ Continuous Integrations
+
+* **circle:** separate lint job [ch1009] ([15edcdc](https://github.com/wmfs/supercopy/commit/15edcdcb614897445bb867e99138ca0a2ebbc541))
+* **circle:** use updated circle node image [skip ci] ([ab27971](https://github.com/wmfs/supercopy/commit/ab27971f53f15b37ded15c8e6ffd05db40899646))
+
 # [1.38.0](https://github.com/wmfs/supercopy/compare/v1.37.0...v1.38.0) (2020-05-31)
@@ -2,0 +50,0 @@

lib/collect-file-info.js

@@ -55,2 +55,10 @@ 'use strict'
 
+  if (options.directoryNames) {
+    for (const [action, name] of Object.entries(options.directoryNames)) {
+      if (info[name]) {
+        info[action] = info[name]
+      }
+    }
+  }
+
   return info
@@ -57,0 +65,0 @@ } // collect

lib/generate-script-statements/add-insert-statements.js

@@ -11,3 +11,3 @@ 'use strict'
         scriptStatements.push(
-          `COPY ${_.snakeCase(options.schemaName)}.${_.snakeCase(info.tableName)}(${info.columnNames.all.join(',')}) FROM '${filePath}' CSV HEADER;`
+          `COPY ${_.snakeCase(options.schemaName)}.${_.snakeCase(info.tableName)}(${info.columnNames.all.join(',')}) FROM '${filePath}' CSV HEADER ${options.sqlQuote};`
         )
@@ -14,0 +14,0 @@ }

lib/generate-script-statements/add-update-statements.js

@@ -13,3 +13,3 @@ 'use strict'
         scriptStatements.push(
-          `COPY ${tempTableName}(${info.columnNames.all.join(',')}) FROM '${filePath}' CSV HEADER;`
+          `COPY ${tempTableName}(${info.columnNames.all.join(',')}) FROM '${filePath}' CSV HEADER ${options.sqlQuote};`
         )
@@ -16,0 +16,0 @@

lib/generate-script-statements/add-upsert-statements.js

@@ -13,3 +13,3 @@ 'use strict'
         scriptStatements.push(
-          `COPY ${tempTableName}(${info.columnNames.all.join(',')}) FROM '${filePath}' CSV HEADER;`
+          `COPY ${tempTableName}(${info.columnNames.all.join(',')}) FROM '${filePath}' CSV HEADER ${options.sqlQuote};`
         )
@@ -16,0 +16,0 @@

lib/generate-script-statements/index.js

@@ -12,2 +12,4 @@ 'use strict'
 module.exports = function generateScriptStatements (fileInfo, options) {
+  options.sqlQuote = expandQuote(options.quote)
+
   const scriptStatements = ['BEGIN;']
@@ -34,1 +36,7 @@ addTruncateStatements(scriptStatements, fileInfo, options)
 }
+
+function expandQuote (quoteChar) {
+  if (!quoteChar) return ''
+  if (quoteChar === '\'') return "QUOTE '''' ESCAPE ''''"
+  return `QUOTE '${quoteChar}' ESCAPE '${quoteChar}'`
+}

lib/index.js

@@ -11,3 +11,3 @@ 'use strict'
 function preprocess (options) {
-  const outputPath = path.resolve(options.sourceDir, 'inserts')
+  const outputPath = path.resolve(options.sourceDir)
   createFolders(outputPath)
@@ -14,0 +14,0 @@ }

package.json

 {
   "name": "@wmfs/supercopy",
-  "version": "1.38.0",
+  "version": "1.39.0",
   "description": "Takes a specifically-named directory structure of CSV files and conjures bulk insert, update and delete statements and applies them to a PostgreSQL database.",
@@ -33,17 +33,17 @@ "author": "West Midlands Fire Service",
     "chai": "4.2.0",
-    "codecov": "3.7.0",
+    "codecov": "3.7.2",
     "conventional-changelog-metahub": "4.0.1",
-    "cz-conventional-changelog": "3.2.0",
-    "mocha": "7.2.0",
-    "nyc": "15.0.1",
+    "cz-conventional-changelog": "3.3.0",
+    "mocha": "8.1.3",
+    "nyc": "15.1.0",
     "rimraf": "3.0.2",
-    "semantic-release": "17.0.8",
+    "semantic-release": "17.1.1",
     "standard": "14.3.4",
     "@semantic-release/changelog": "5.0.1",
     "@semantic-release/git": "9.0.0",
-    "@wmfs/hl-pg-client": "1.21.0"
+    "@wmfs/hl-pg-client": "1.23.0"
   },
   "scripts": {
     "lint": "standard",
-    "test": "nyc mocha && standard",
+    "test": "nyc mocha",
     "coverage": "nyc report --reporter=text-lcov > coverage.lcov && codecov",
@@ -50,0 +50,0 @@ "semantic-release": "semantic-release"

README.md

@@ -40,4 +40,5 @@ # supercopy
     truncateTables: true,
-    debug: true
-    multicopy: false
+    debug: true,
+    multicopy: false,
+    directoryNames: { ... }
   },
@@ -64,3 +65,5 @@ function (err) {
 | `debug`               | `boolean`  | Show debugging information on the console
-| `multicopy`               | `boolean`  | Enables 'sourceDir' to house many typical Supercopy 'sourceDir' shaped directories. Defaults to false.
+| `multicopy`           | `boolean`  | Enables 'sourceDir' to house many typical Supercopy 'sourceDir' shaped directories. Defaults to false.
+| `quote`               | `string`   | Override the the default quote character, ". It isn't necessary to quote fields but occasionally (especially when importing JSON fields) you need to, and this option will help.  
+| `directoryNames`      | `object`   | Overrides the default directory names - see below. 
 
@@ -102,2 +105,5 @@ ### <a name="structure"></a>File structure
 * The sub-directories here refer to the type of action that should be performed using CSV data files contained in it. Supported directory names are `insert`, `update`, `upsert` (try to update, failing that insert) and `delete`.
+* Directories are optional. A directory maybe missing or empty.
+* The `directoryNames` option can be used to apply actions to directories if the names don't meet the above structure. Eg 
+`directoryName : { 'inserts': 'new', 'deletes': 'old' }` would insert the contents of the directory named `new` and remove the contents of the `old` directory.
 * The filename of each file should refer to a table name in the schema identified by the `schemaName` option. 
@@ -104,0 +110,0 @@ * The expected format of the .csv files is:

test/supercopy-csv-tests.js

 /* eslint-env mocha */
 
-'use strict'
-
 const process = require('process')
@@ -13,3 +11,3 @@ const expect = require('chai').expect
 
-describe('Run some basic tests', function () {
+describe('Supercopy tests', function () {
   this.timeout(process.env.TIMEOUT || 5000)
@@ -27,7 +25,7 @@
 
-  it('Should create a new pg client', function () {
+  before('Create a new pg client', function () {
     client = new HlPgClient(connectionString)
   })
 
-  it('Should remove output directory ahead of csv tests, if it exists already', function (done) {
+  before('Remove output directory ahead of csv tests, if it exists already', function (done) {
     const outputPath = path.resolve(__dirname, './output')
@@ -41,26 +39,34 @@ if (fs.existsSync(outputPath)) {
 
-  it('Should load some test data', async () => {
+  before('Load test data', async () => {
     for (const filename of ['uninstall.sql', 'install.sql']) { await client.runFile(path.resolve(__dirname, path.join('fixtures', 'scripts', filename))) }
   })
 
-  it('Should promise to supercopy some people', function () {
-    return supercopy(
-      {
-        sourceDir: path.resolve(__dirname, './fixtures/input-data/people'),
-        topDownTableOrder: ['adults', 'children'],
-        headerColumnNamePkPrefix: '.',
-        client: client,
-        schemaName: 'supercopy_test',
-        debug: true
-      }
-    )
-  })
+  const goodOptions = {
+    topDownTableOrder: ['adults', 'children'],
+    headerColumnNamePkPrefix: '.',
+    schemaName: 'supercopy_test',
+    debug: true
+  }
 
-  it('Should return correctly modified adult rows', function (done) {
-    client.query(
-      'select adult_no,first_name,last_name from supercopy_test.adults order by adult_no',
-      function (err, result) {
-        if (err) {
-          return done(err)
-        }
+  const testConfigs = [
+    ['people'],
+    ['people-quote', '\'']
+  ]
+
+  for (const [goodFixture, quoted] of testConfigs) {
+    describe(`${goodFixture} data`, () => {
+      it('reset test data', async () => {
+        for (const filename of ['uninstall.sql', 'install.sql']) { await client.runFile(path.resolve(__dirname, path.join('fixtures', 'scripts', filename))) }
+      })
+
+      it('Supercopy some people', function () {
+        goodOptions.client = client
+        goodOptions.sourceDir = path.resolve(__dirname, './fixtures/input-data', goodFixture)
+        goodOptions.quote = quoted
+
+        return supercopy(goodOptions)
+      })
+
+      it('Verify adult rows', async () => {
+        const result = await client.query('select adult_no,first_name,last_name from supercopy_test.adults order by adult_no')
         expect(result.rows).to.eql(
@@ -79,14 +85,6 @@ [
         )
-        done()
-      }
-    )
-  })
+      })
 
-  it('Should return correctly modified children rows', function (done) {
-    client.query(
-      'select child_no,first_name,last_name from supercopy_test.children order by child_no',
-      function (err, result) {
-        if (err) {
-          return done(err)
-        }
+      it('Verify modified children rows', async () => {
+        const result = await client.query('select child_no,first_name,last_name from supercopy_test.children order by child_no')
         expect(result.rows).to.eql(
@@ -103,106 +101,89 @@ [
         )
-        done()
-      }
-    )
-  })
+      })
+    })
+  } // for ...
 
-  it('Should fail supercopy-if some bad-people files', function (done) {
-    supercopy(
-      {
-        sourceDir: path.resolve(__dirname, './fixtures/input-data/people-with-an-error'),
-        topDownTableOrder: ['adults', 'children'],
-        headerColumnNamePkPrefix: '.',
-        client: client,
-        schemaName: 'supercopy_test',
-        debug: true
-      },
-      function (err) {
-        expect(err).to.not.equal(null)
-        done()
-      }
-    )
-  })
+  describe('bad data', () => {
+    it('Fail on bad data', function (done) {
+      supercopy(
+        {
+          sourceDir: path.resolve(__dirname, './fixtures/input-data/people-with-an-error'),
+          topDownTableOrder: ['adults', 'children'],
+          headerColumnNamePkPrefix: '.',
+          client: client,
+          schemaName: 'supercopy_test',
+          debug: true
+        },
+        function (err) {
+          expect(err).to.not.equal(null)
+          done()
+        }
+      )
+    })
 
-  it('Should error on mis-shapen data', function (done) {
-    supercopy(
-      {
-        sourceDir: path.resolve(__dirname, './fixtures/input-data/bad-people'),
-        topDownTableOrder: ['adults'],
-        headerColumnNamePkPrefix: '.',
-        client: client,
-        schemaName: 'supercopy_test',
-        truncateTables: true,
-        debug: true
-      },
-      function (err) {
-        expect(err).to.not.equal(null)
-        done()
-      }
-    )
+    it('Fail on mis-shapen data', function (done) {
+      supercopy(
+        {
+          sourceDir: path.resolve(__dirname, './fixtures/input-data/bad-people'),
+          topDownTableOrder: ['adults'],
+          headerColumnNamePkPrefix: '.',
+          client: client,
+          schemaName: 'supercopy_test',
+          truncateTables: true,
+          debug: true
+        },
+        function (err) {
+          expect(err).to.not.equal(null)
+          done()
+        }
+      )
+    })
   })
 
-  it('Should supercopy some people, truncating the tables first', function (done) {
-    supercopy(
-      {
-        sourceDir: path.resolve(__dirname, './fixtures/input-data/people'),
-        topDownTableOrder: ['adults', 'children'],
-        headerColumnNamePkPrefix: '.',
-        client: client,
-        schemaName: 'supercopy_test',
-        truncateTables: true,
-        debug: true
-      },
-      function (err) {
-        done(err)
-      }
-    )
-  })
-
-  it('Should return correctly modified adult rows (truncated)', function (done) {
-    client.query(
-      'select adult_no,first_name,last_name from supercopy_test.adults order by adult_no',
-      function (err, result) {
-        if (err) {
-          return done(err)
+  describe('truncate first', () => {
+    it('supercopy some people, truncating the tables first', function () {
+      return supercopy(
+        {
+          sourceDir: path.resolve(__dirname, './fixtures/input-data/people'),
+          topDownTableOrder: ['adults', 'children'],
+          headerColumnNamePkPrefix: '.',
+          client: client,
+          schemaName: 'supercopy_test',
+          truncateTables: true,
+          debug: true
         }
-        expect(result.rows).to.eql(
-          [
-            { adult_no: 30, first_name: 'Maud', last_name: 'Flanders' },
-            { adult_no: 40, first_name: 'Ned', last_name: 'Flanders' },
-            { adult_no: 80, first_name: 'Clancy', last_name: 'Wiggum' },
-            { adult_no: 90, first_name: 'Abraham', last_name: 'Simpson' },
-            { adult_no: 100, first_name: 'Mona', last_name: 'Simpson' }
-          ]
-        )
-        done()
-      }
-    )
-  })
+      )
+    })
 
-  it('Should return correctly modified children rows (truncated)', function (done) {
-    client.query(
-      'select child_no,first_name,last_name from supercopy_test.children order by child_no',
-      function (err, result) {
-        if (err) {
-          return done(err)
-        }
-        expect(result.rows).to.eql(
-          [
-            { child_no: 50, first_name: 'Todd', last_name: 'Flanders' },
-            { child_no: 70, first_name: 'Milhouse', last_name: 'Van Houten' }
-          ]
-        )
-        done()
-      }
-    )
+    it('Verify adult rows, truncated', async () => {
+      const result = await client.query('select adult_no,first_name,last_name from supercopy_test.adults order by adult_no')
+      expect(result.rows).to.eql(
+        [
+          { adult_no: 30, first_name: 'Maud', last_name: 'Flanders' },
+          { adult_no: 40, first_name: 'Ned', last_name: 'Flanders' },
+          { adult_no: 80, first_name: 'Clancy', last_name: 'Wiggum' },
+          { adult_no: 90, first_name: 'Abraham', last_name: 'Simpson' },
+          { adult_no: 100, first_name: 'Mona', last_name: 'Simpson' }
+        ]
+      )
+    })
+
+    it('Verify children rows (truncated)', async () => {
+      const result = await client.query('select child_no,first_name,last_name from supercopy_test.children order by child_no')
+      expect(result.rows).to.eql(
+        [
+          { child_no: 50, first_name: 'Todd', last_name: 'Flanders' },
+          { child_no: 70, first_name: 'Milhouse', last_name: 'Van Houten' }
+        ]
+      )
+    })
   })
 
-  it('Should cleanup the test data', async () => {
+  after('Cleanup the test data', async () => {
     await client.runFile(path.resolve(__dirname, path.join('fixtures', 'scripts', 'uninstall.sql')))
   })
 
-  it('Should close database connections', function (done) {
+  after('Close database connections', async () => {
     client.end()
-    done()
   })
 })

No alert changes

Improved metrics

Total package byte prevSize

68155

increased by14.28%

Number of package files

45

increased by32.35%

Lines of code

703

increased by15.44%

Number of lines in readme file

128

increased by4.92%

Worsened metrics

Number of low supply chain risk alerts

16

increased by77.78%

No dependency changes