ddf-validation - npm Package Compare versions

Comparing version 0.1.1 to 0.2.0

ddf-schema/ddf-measure-values.schema.js

@@ -1,2 +0,4 @@
-module.exports = {
+'use strict';
+
+let schema = {
   // 0 - file name
@@ -7,4 +9,11 @@ // 1 - measure gid
   fileExp: /^ddf--data_for--([\w]+)(?:--by((?:--(?:[\w]+))+)){0,1}\.csv$/i,
-  dimensions: (fileName) => (this.fileExp.exec(fileName)[2] || '').split('--').splice(1),
-  measure: (fileName) => (this.fileExp.exec(fileName)[1] || '').split('--').splice(1)
+  dimensions: function (fileName) {
+    return (this.fileExp.exec(fileName)[2] || '').split('--').splice(1);
+  },
+  measure: function (fileName) {
+    return (this.fileExp.exec(fileName)[1] || '');
+  }
 };
+
+
+module.exports = schema;

index.js

@@ -8,6 +8,6 @@ #! /usr/bin/env node
 
+const _ = require('lodash');
 const fs = require('fs');
 const path = require('path');
 const rx = require('rxjs');
-require('console.table');
 
@@ -20,3 +20,3 @@ const errorCodes = require('./lib/ddf-error-codes');
 if (!require('./utils/path-exists-sync')(normalizedPath)) {
-  logger.log(errorCodes.err_folder_not_found.message(folder));
+  logger.error(errorCodes.err_folder_not_found.message(folder));
   return;
@@ -50,4 +50,2 @@ }
     logger.log(res.ddfFolders);
-
-    return validateDdfFolder(ddfFolders$);
   }, err => console.error(err));
@@ -58,4 +56,4 @@
   ddfFolders$
-    .do(x=>console.log('Validating ddf folder', x))
-    .map(folderPath => {
+    .do(x=>logger.log(`Validating ddf folder ${x}`))
+    .mergeMap(folderPath => {
       // validate dimensions file
@@ -75,9 +73,154 @@ const dimensionsFile$ = require('./ddf-utils/rx-read-dimension')(folderPath);
         .combineLatest([isMeasuresValid, isIdUnique], (a, b, c)=> {
-          const d = a.concat(b).concat(c);
-          console.table(d);
-          return d;
+          return a.concat(b).concat(c);
         });
     })
-    //.do(x=>console.log(x))
-    .subscribe(x=>x.subscribe(), x=>console.error(x.stack));
-}
+    .subscribe(x=>logger.results(x), x=>console.error(x.stack));
+}
+
+function validateMeasureValues(ddfFolders$) {
+  // 1.
+  // gather measures
+  // check is they are present in measures.csv
+  // 2.
+  // gather dimensions
+  // check is they are present in dimensions.csv
+  // 3.
+  // gather dimension values from measure values
+  // gather dimension values from dimensions values files
+  // check is all (dv from mv) are present in dimensions values files
+  // check is all (dv from dv) are present in measure files
+  // 4.
+  // check data points consistency
+  const rxReadCsv = require('./utils/rx-read-csv-file-to-json');
+  const measuresSchema = require('./ddf-schema/ddf-measures.schema');
+  const measureValuesSchema = require('./ddf-schema/ddf-measure-values.schema');
+  const dimensionsSchema = require('./ddf-schema/ddf-dimensions.schema');
+  const dimensionValuesSchema = require('./ddf-schema/ddf-dimension-values.schema');
+
+  ddfFolders$
+    .do(x=>logger.log(`Validating ddf folder ${x}`))
+    .map(folderPath => {
+      const filesInFolder$ = require('./utils/rx-read-files-in-folder')(folderPath)
+        .map(fileName => path.basename(fileName));
+      // filter measure values files
+      const measureValuesFiles$ = filesInFolder$
+        .filter(fileName=>measureValuesSchema.fileExp.test(fileName));
+      // filter dimensions values files
+      const dimensionValuesFiles$ = filesInFolder$
+        .filter(fileName => dimensionValuesSchema.fileExp.test(fileName));
+
+      // measure.csv file stream
+      const measureFile$ = require('./ddf-utils/rx-read-measures')(folderPath);
+      // dimensions.csv file stream
+      const dimensionsFile$ = require('./ddf-utils/rx-read-dimension')(folderPath);
+
+      // read measure IDs from measure.csv
+      const measures$ = measureFile$
+        .first()
+        .mergeMapTo(measureFile$.skip(1), _.zipObject)
+        .pluck(measuresSchema.gid)
+        .toArray();
+
+      // read dimension IDs from dimensions.csv
+      const dimensions$ = dimensionsFile$
+        .first()
+        .mergeMapTo(measureFile$.skip(1), _.zipObject)
+        .pluck(measuresSchema.gid)
+        .toArray();
+
+      // read measure IDs from dimension values fileNames
+      const measuresFromDvFiles$ = measureValuesFiles$
+        .map(fileName=> measureValuesSchema.measure(fileName))
+        // todo: replace toArray.map with .distinct when implemented in rxjs
+        //.distinct() // todo: do nor remove this line
+        .toArray().map(x => require('lodash').uniq(x));
+
+      // read dimension IDs from measure values fileNames
+      const dimensionsFromMvFiles$ = measureValuesFiles$
+        .map(fileName=> measureValuesSchema.dimensions(fileName))
+        // todo: replace toArray.map with .distinct when implemented in rxjs
+        //.distinct() // todo: do nor remove this line
+        .toArray().mergeMap(x => require('lodash').uniq(x));
+
+      // compare and validate measure IDs
+      measures$.combineLatest(measuresFromDvFiles$, (measuresFromCsv, measuresFromFiles) => {
+        // check missing measure in measures.csv
+        const diff1 = _.difference(measuresFromFiles, measuresFromCsv);
+        if (diff1.length) {
+          logger.error(`Error! Please add measures to ${measuresSchema.fileName} ${diff1}`);
+        }
+        // check missing measure values in folder
+        const diff2 = _.difference(measuresFromCsv, measuresFromFiles);
+        if (diff2.length) {
+          logger.warning(`Warning! Values for measures described in ${measuresSchema.fileName} are missing: ${diff2}`);
+        }
+        return 0;
+      })
+      //.do(x=>console.log(x))
+      //.subscribe();
+
+      // compare and validate dimension IDs
+      dimensions$.combineLatest(dimensionsFromMvFiles$, (dimensionsFromCsv, dimensionsFromFiles) => {
+        // check missing measure in measures.csv
+        const diff1 = _.difference(dimensionsFromFiles, dimensionsFromCsv);
+        if (diff1.length) {
+          logger.warning(`Warning! Please add dimensions to ${dimensionsSchema.fileName} ${diff1}`);
+        }
+        // check missing measure values in folder
+        const diff2 = _.difference(dimensionsFromCsv, dimensionsFromFiles);
+        if (diff2.length) {
+          logger.warning(`Warning! Values for dimensions described in ${dimensionsSchema.fileName} are missing: ${diff2}`);
+        }
+        return 0;
+      })
+      //.do(x=>console.log(x))
+      //.subscribe();
+
+      // build dimension values hash map
+      // return { dimension_id: {dimension_value_id: true} }
+      const dimensionsValuesIndex = dimensionValuesFiles$
+        .mergeMap(fileName => {
+          const rows$ = rxReadCsv(path.join(folderPath, fileName));
+          const rowsObj$ = rows$.first().mergeMapTo(rows$.skip(1), _.zipObject);
+          const dimensions = dimensionValuesSchema.dimensions(fileName);
+          // TODO: fix geo - country issue
+          return rowsObj$.reduce((memo, entry) => {
+            _.each(dimensions, dim => {
+              if (entry[dim]) {
+                memo[dim] = memo[dim] || {};
+                memo[dim][entry[dim]] = true;
+              }
+            });
+            return memo;
+          }, {});
+        })
+        .reduce((memo, entry)=> {
+          _.each(Object.keys(entry), key => {
+            memo[key] = Object.assign({}, memo[key], entry[key]);
+          });
+          return memo;
+        }, {});
+      //.do(x=>console.log(x))
+      //.subscribe();
+      measureValuesFiles$
+        .first()
+        .map(fileName => {
+          const fullFileName = path.join(folderPath, fileName);
+          const file$ = rxReadCsv(fullFileName);
+          const rows$ = file$.first().mergeMapTo(file$.skip(1), _.zipObject);
+          // todo: measure values validation
+          // 1. check dimension ids
+          // unknown dimension ids
+          // missing dimension ids
+          // 2. check data points
+          // report missing measure values data points
+          // like abkh 1983
+        })
+        //.do(x=>console.log(x))
+        .subscribe();
+    })
+    .subscribe();
+}
+
+validateDdfFolder(ddfFolders$);
+validateMeasureValues(ddfFolders$);

package.json

 {
   "name": "ddf-validation",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "DDF validion tool",
@@ -10,3 +10,3 @@ "main": "index.js",
   "scripts": {
-    "test": "./node_modules/.bin/mocha \"lib/**/*.spec.js\""
+    "test": "./node_modules/.bin/mocha \"test/**/*.spec.js\""
   },
@@ -18,6 +18,7 @@ "author": "Dmitriy Shekhovtsov<[e-mail]>",
     "async": "1.5.2",
-    "console.table": "0.4.0",
+    "blessed": "0.1.81",
     "csv": "0.4.6",
     "csvtojson": "0.5.1",
     "lodash": "4.0.0",
+    "prettyjson": "1.1.3",
     "rxjs": "5.0.0-beta.1"
@@ -24,0 +25,0 @@ },

test/ddf-folder-validation.spec.js

@@ -10,3 +10,3 @@ 'use strict';
 
-let ddfFolderValidation = require('./ddf-folder-validation');
+let ddfFolderValidation = require('../lib/ddf-root-folder.validator');
 let logger = require('../utils/logger');
@@ -13,0 +13,0 @@

utils/logger.js

@@ -1,1 +0,30 @@
-module.exports.log = console.log.bind(console);
+'use strict';
+
+//module.exports.log = console.log.bind(console);
+
+const ui = require('../ui');
+const globals = require('../ui/globals');
+
+function log(out, level) {
+  if (typeof out !== 'string' && out.toString) {
+    out = out.toString();
+  }
+
+  ui.addMessage(out, level || globals.NOTICE);
+}
+
+module.exports.log = (out) => {
+  log(out);
+};
+
+module.exports.error = (out) => {
+  log(out, globals.ERROR)
+};
+
+module.exports.warning = (out) => {
+  log(out, globals.WARNING);
+};
+
+module.exports.results = (data) => {
+  ui.addResults(data);
+};

utils/rx-read-csv-file-to-json.js

@@ -15,2 +15,3 @@ 'use strict';
     while (record = parser.read()) {
+      // [val1, val2, ...]
       subject.next(record);
@@ -17,0 +18,0 @@ }

wallaby.js

@@ -13,3 +13,3 @@ module.exports = function () {
     tests: [
-      'lib/*spec.js'
+      'test/*spec.js'
     ],
@@ -16,0 +16,0 @@ testFramework: "mocha",

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

New author

Supply chain risk

A new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

75791

increased by15.74%

Number of package files

49

increased by11.36%

Lines of code

1194

increased by38.52%

Worsened metrics

Dependency count

8

increased by14.29%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addedblessed@0.1.81

• + Addedprettyjson@1.1.3

• + Addedblessed@0.1.81(transitive)

• + Addedcolors@1.4.0(transitive)

• + Addedminimist@1.2.8(transitive)

• + Addedprettyjson@1.1.3(transitive)

• - Removedconsole.table@0.4.0

• - Removedconsole.table@0.4.0(transitive)

• - Removedeasy-table@0.3.0(transitive)