git-csv-diff - npm Package Compare versions

Comparing version 1.1.15 to 1.2.0

index.js

 'use strict';
 
-const gitCsvDiff = require('./src/git-csv-diff');
-module.exports = gitCsvDiff;
+const diffByFile = require('./src/diff-by-file');
+
+/* Declaration */
+
+function gitCsvDiff() {}
+
+/* API */
+
+gitCsvDiff.prototype.process = function (metaData, dataDiff, streams, callback) {
+  return callback(diffByFile.process(metaData, dataDiff, streams));
+};
+
+// Alias :: backward compatibility
+
+gitCsvDiff.prototype.processUpdated = gitCsvDiff.prototype.process;
+
+/* Export */
+
+module.exports = new gitCsvDiff();

package.json

 {
   "name": "git-csv-diff",
   "author": "Valor-Software",
-  "version": "1.1.15",
+  "version": "1.2.0",
   "license": "GPL-3.0",
@@ -6,0 +6,0 @@ "description": "Library generate difference between csv-files based on Git commit hash",

README.md

@@ -13,29 +13,2 @@ # Git CSV Diff
 
-## Usage, Diff generated in Memory
-
-```bash
-
-const gitCsvDiff = require('git-csv-diff');
-
-/*
-    fileName: 'ddf--datapoints--forest_products_removal_total_dollar--by--geo--time.csv';
-    diff: {
-        from: 'state of the base file'
-        to: 'state of the target file'
-    };
-    fileModifier: 'M'/'D'/'A'
-*/
-
-let diffResult = {};
-
-gitCsvDiff.process(fileName, diff, fileModifier, function(error, result) {
-  //console.log("Files:", result.file);
-  //console.log("Changes:", result.diff);
-  diffResult[result.file] = result.diff;
-
-  gitCsvDiff.translations(diffResult);
-});
-
-```
-
 ## Usage, generate Diff partially and write result by chunk into streams
@@ -42,0 +15,0 @@

src/diff-by-file.js

@@ -5,6 +5,17 @@ 'use strict';
 const daff = require('daff');
+const path = require('path');
 const async = require("async");
 
-const ModelDiff = require('./model-diff');
+/* Models */
 
+const ModelDiff = require('./model/diff');
+const ModelResponse = require('./model/response');
+
+/* Diff generators */
+
+const diffModifiers = require('./diff/modifiers');
+const diffColumns = require('./diff/columns');
+const diffStrategy = require('./diff/strategy');
+const diffHelpers = require('./diff/helpers');
+
 function diffByFile() {
@@ -14,300 +25,155 @@ return {
   };
-};
+}
 
-/* protected */
+// metaData.fileName;
+// metaData.fileModifier;
 
-function _process(fileName, dataDiff, fileModifier) {
+// dataDiff.from;
+// dataDiff.to;
 
-  let diffResult = [];
+// streams.diff;
+// streams.lang;
 
-  const tableFrom = new daff.Csv().makeTable(dataDiff.from);
-  const tableTo = new daff.Csv().makeTable(dataDiff.to);
+function _process(metaData, dataDiff, streams) {
 
-  let filesDiff = daff.compareTables(tableFrom, tableTo).align();
+  const isTranslations = diffHelpers.isLanguageFile(metaData.fileName);
+  const baseStream = isTranslations ? streams.lang : streams.diff;
 
-  let flags = new daff.CompareFlags();
-  flags.show_unchanged = true;
-  flags.show_unchanged_columns = true;
-  flags.always_show_header = true;
+  /* Prepare Data Structure */
 
-  let highlighter = new daff.TableDiff(filesDiff, flags);
-  highlighter.hilite(diffResult);
+  let modelDiff = ModelDiff.init();
+  let modelResponse = ModelResponse.init();
 
-  /* Prepare Data Structure */
-  let fileDiffData = ModelDiff.init();
+  setMetaDataFile(modelResponse.metadata.file, metaData);
 
-  /* Slice Groupd of Changes */
+  // validate input data
+  if(!isSchemaExists(modelResponse.metadata) || !isValidFilePath(metaData.fileName)) {
+    return;
+  }
 
-  let firsDiffRow = diffResult.shift();
-  let diffResultHeader = [];
-  let diffResultColumns = [];
+  // setup response meta data
 
-  if (firsDiffRow[0] == '!') {
+  setMetaDataType(modelResponse.metadata);
+  setMetaDataLanguage(modelResponse.metadata, metaData.fileName);
 
-    // [ '!', '', '(old_column)', '+++', '---' ],
-    diffResultHeader = firsDiffRow;
-    // [ '@@', 'city', 'name', 'country' ],
-    diffResultColumns = diffResult.shift();
+  /* Process Diff by Daff */
 
-    if (diffResultHeader[0] == "!") {
+  const diffResult = initDaffDiff(dataDiff);
 
-      diffResultHeader.shift();
-      diffResultHeader.forEach(function (value, index) {
 
-        if (value != '') {
+  /* Main flow */
 
-          if (value == '+++') {
-            // added
-            fileDiffData.header.create.push(diffResultColumns[index + 1]);
-          } else if (value == '---') {
-            // removed
-            fileDiffData.header.remove.push(diffResultColumns[index + 1]);
-          } else {
-            // modified
-            let oldColumn = value.substring(1, value.length - 1);
-            let diffColumns = {};
-            diffColumns[diffResultColumns[index + 1]] = oldColumn;
-            fileDiffData.header.update.push(diffColumns);
-            fileDiffData.header.remove.push(oldColumn);
-          }
-        }
 
-      });
-    }
+  /* Head, File Columns */
+  // [ 'city', 'name', 'country' ]
+  const diffResultColumns = diffColumns.process(diffResult, modelDiff);
 
-  } else {
-    // [ '@@', 'city', 'name', 'country' ],
-    diffResultColumns = firsDiffRow;
+  // setup `removedColumns` for files that are not removed
+  if (metaData.fileModifier != "D") {
+    modelResponse.metadata.removedColumns = _.clone(modelDiff.header.remove);
   }
 
-  let diffResultGidField;
-  if (diffResultColumns[0] == "@@") {
-    diffResultColumns.shift();
-    diffResultGidField = diffResultColumns[0];
-  }
+  // process each row if any changes detected
 
-  let isDataPointsFile = isDatapointFile(fileName);
-
   if (diffResult.length) {
+    diffResult.forEach(function (rowValue) {
 
-    diffResult.forEach(function (value, index) {
+      const modificationType = rowValue.shift();
 
-      // simple-way, collect all data (mean full row) for update
-
-      let modificationType = value.shift();
-
-      if (modificationType != '') {
-
-        if (modificationType == '+++') {
-
-          // added
-          let dataRow = {};
-          diffResultColumns.forEach(function (columnValue, columnIndex) {
-            if (fileDiffData.header.remove.indexOf(columnValue) == -1) {
-              // ready columns
-              dataRow[columnValue] = value[columnIndex];
-            }
-          });
-
-          if (dataRow) {
-            fileDiffData.body.create.push(dataRow);
-          }
-
-        } else if (modificationType == '---') {
-
-          // removed
-          let dataRowRemoved = {};
-
-          // check that file with datapoints
-          if (isDataPointsFile) {
-            diffResultColumns.forEach(function (columnValue, columnIndex) {
-              if (
-                // disable changes for removed files
-              // fileDiffData.header.remove.indexOf(columnValue) == -1 &&
-              fileDiffData.header.create.indexOf(columnValue) == -1
-              ) {
-                // ready columns
-                dataRowRemoved[columnValue] = value[columnIndex];
-              }
-            });
-          } else {
-            dataRowRemoved['gid'] = diffResultGidField;
-            dataRowRemoved[diffResultGidField] = value[0];
-          }
-
-          fileDiffData.body.remove.push(dataRowRemoved);
-
-        } else if (modificationType == '+') {
-
-          // updated, only added columns
-          let dataRow = {};
-          let dataRowOrigin = {};
-          diffResultHeader.forEach(function (columnValue, columnIndex) {
-            let columnKey = diffResultColumns[columnIndex];
-            if (fileDiffData.header.create.indexOf(columnKey) != -1) {
-              dataRow[columnKey] = value[columnIndex];
-            } else {
-              dataRowOrigin[columnKey] = value[columnIndex];
-            }
-          });
-
-          let dataRowUpdated = {};
-          dataRowUpdated["gid"] = diffResultGidField;
-          dataRowUpdated[diffResultGidField] = value[0];
-          dataRowUpdated["data-update"] = dataRow;
-
-          if (isDataPointsFile) {
-            dataRowUpdated["data-origin"] = dataRowOrigin;
-          }
-
-          fileDiffData.body.update.push(dataRowUpdated);
-
-        } else if (modificationType == '->') {
-
-          // updated, only changed cell
-          let dataRow = {};
-          let dataRowOrigin = {};
-
-          value.forEach(function (valueCell, indexCell) {
-            let modificationSeparatorPosition = valueCell.indexOf('->');
-            let columnKey = diffResultColumns[indexCell];
-
-            // cell modified
-            if (modificationSeparatorPosition != -1) {
-
-              let readyValueCell = valueCell.substring(modificationSeparatorPosition + 2);
-              let readyValueCellOrigin = valueCell.substring(0, modificationSeparatorPosition);
-
-              dataRow[columnKey] = readyValueCell;
-              dataRowOrigin[columnKey] = readyValueCellOrigin;
-
-            } else if (isDataPointsFile) {
-              dataRow[columnKey] = valueCell;
-              if (fileDiffData.header.create.indexOf(columnKey) == -1) {
-                dataRowOrigin[columnKey] = valueCell;
-              }
-              // check that it's not new column
-            } else if (fileDiffData.header.create.indexOf(columnKey) != -1) {
-              dataRow[columnKey] = valueCell;
-            }
-          });
-
-          // fix first column changes
-
-          let conceptValueSearchFor = value[0];
-          let conceptValueTypeIndex = conceptValueSearchFor.indexOf('->');
-
-          if (conceptValueTypeIndex != -1) {
-            conceptValueSearchFor = value[0].substring(0, conceptValueTypeIndex)
-          }
-
-          let dataRowUpdated = {};
-          dataRowUpdated["gid"] = diffResultGidField;
-          dataRowUpdated[diffResultGidField] = conceptValueSearchFor;
-          dataRowUpdated["data-update"] = dataRow;
-
-          if (isDataPointsFile) {
-            dataRowUpdated["data-origin"] = dataRowOrigin;
-          }
-
-          fileDiffData.body.change.push(dataRowUpdated);
+      // check that something was changed
+      if (modificationType !== diffModifiers.BLANK) {
+        if(diffStrategy.has(modificationType)){
+          const diffInstance = diffStrategy.get(modificationType);
+          diffInstance.process(baseStream, metaData, modelResponse, modelDiff, diffResultColumns, rowValue);
         }
-        // empty modifier symbol
+      // if nothing changed
       } else {
-        // check that there is no new columns were added
-        if (fileDiffData.header.create.length) {
+        // Case: new columns were added
+        if (modelDiff.header.create.length) {
+          const diffInstance = diffStrategy.get(diffModifiers.COLUMN_CREATE);
+          diffInstance.process(baseStream, metaData, modelResponse, modelDiff, diffResultColumns, rowValue);
+        }
+        // Case: columns were renamed
+        if (modelDiff.header.update.length) {
+          const diffInstance = diffStrategy.get(diffModifiers.COLUMN_UPDATE);
+          diffInstance.process(baseStream, metaData, modelResponse, modelDiff, diffResultColumns, rowValue);
+        }
+        // Case: columns were removed
+        if (modelDiff.header.remove.length) {
+          const diffInstance = diffStrategy.get(diffModifiers.COLUMN_REMOVE);
+          diffInstance.process(baseStream, metaData, modelResponse, modelDiff, diffResultColumns, rowValue);
+        }
+      }
+    });
+  }
 
-          let dataRow = {};
-          let dataRowOrigin = {};
+  return;
+}
 
-          value.forEach(function (valueCell, indexCell) {
-            let columnKey = diffResultColumns[indexCell];
-            if (fileDiffData.header.create.indexOf(columnKey) == -1) {
-              // check that file with datapoints
-              if (isDataPointsFile) {
-                // collect original values for datapoints
-                dataRowOrigin[columnKey] = valueCell;
-              }
-            } else {
-              // new values for added columns
-              dataRow[columnKey] = valueCell;
-            }
-          });
+function isValidFilePath(filename) {
+  return (_.includes(filename, "/") && !diffHelpers.isLanguageFile(filename)) ? false : true;
+}
 
-          let dataRowChanged = {};
-          dataRowChanged["gid"] = diffResultGidField;
-          dataRowChanged[diffResultGidField] = value[0];
-          dataRowChanged["data-update"] = dataRow;
+function setMetaDataLanguage(metaData, fileName) {
+  let lang = 'default';
+  if(diffHelpers.isLanguageFile(fileName)) {
+    const regexpRes = /lang\/(.+)\//.exec(fileName);
+    lang = regexpRes[1] || lang;
+  }
+  metaData.lang = lang;
+}
 
-          if (isDataPointsFile) {
-            dataRowChanged["data-origin"] = dataRowOrigin;
-          }
+function setMetaDataFile(file, metaData) {
+  const fileName = path.parse(metaData.fileName).base;
+  const resourcesByPathOld = _.keyBy(metaData.datapackage.old.resources, 'path');
+  file.old = resourcesByPathOld[fileName];
 
-          fileDiffData.body.change.push(dataRowChanged);
-        }
+  // info is not available if file was removed
+  if(metaData.fileModifier != "D") {
+    const resourcesByPathNew = _.keyBy(metaData.datapackage.new.resources, 'path');
+    file.new = resourcesByPathNew[fileName];
+  }
+}
 
-        // check that there is no renamed columns
-        if (fileDiffData.header.update.length) {
+function isSchemaExists(metadata) {
+  const schemaSource = metadata.file.new ? metadata.file.new : metadata.file.old;
+  return schemaSource && schemaSource.hasOwnProperty('schema');
+}
 
-          let dataRow = {};
-          let dataRowOrigin = {};
+function setMetaDataType(metadata) {
+  const constants = {
+    DATAPOINTS: 'datapoints',
+    CONCEPTS: 'concepts',
+    ENTITIES: 'entities'
+  };
+  const primaryKeys = diffHelpers.getPrimaryKeys(metadata);
 
-          // check that file with datapoints
-          value.forEach(function (valueCell, indexCell) {
-            let columnKey = diffResultColumns[indexCell];
+  if (primaryKeys.length > 1)
+    return metadata.type = constants.DATAPOINTS;
 
-            // not new column
-            if (fileDiffData.header.create.indexOf(columnKey) == -1) {
-              // is this changed column
-              let columnKeyOld = columnKey;
-              let oldColumnIndex = fileDiffData.header.update.findIndex(function (updateElement) {
-                return !!updateElement[columnKey];
-              });
-              if (oldColumnIndex !== -1) {
-                // get old column name
-                columnKeyOld = fileDiffData.header.update[oldColumnIndex][columnKey];
-              }
+  if (_.includes(constants.CONCEPTS, _.first(primaryKeys)))
+    return metadata.type = constants.CONCEPTS;
 
-              dataRow[columnKey] = valueCell;
-              if (isDataPointsFile) {
-                dataRowOrigin[columnKeyOld] = valueCell;
-              }
-            } else {
-              // new column
-              dataRow[columnKey] = valueCell;
-            }
-          });
+  return metadata.type = constants.ENTITIES;
+}
 
-          let dataRowChanged = {};
-          dataRowChanged["gid"] = diffResultGidField;
-          dataRowChanged[diffResultGidField] = value[0];
-          dataRowChanged["data-update"] = dataRow;
+function initDaffDiff(dataDiff) {
+  const diffResult = [];
 
-          if (isDataPointsFile) {
-            dataRowChanged["data-origin"] = dataRowOrigin;
-          }
+  const tableFrom = new daff.Csv().makeTable(dataDiff.from);
+  const tableTo = new daff.Csv().makeTable(dataDiff.to);
 
-          fileDiffData.body.change.push(dataRowChanged);
-        }
-      }
-    });
-  }
+  const filesDiff = daff.compareTables(tableFrom, tableTo).align();
 
-  // clear remove header section for removed files
-  if (fileModifier == "D") {
-    fileDiffData.header.remove = [];
-  }
+  const flags = new daff.CompareFlags();
+  flags.show_unchanged = true;
+  flags.show_unchanged_columns = true;
+  flags.always_show_header = true;
 
-  // Structure :: Create
+  const highlighter = new daff.TableDiff(filesDiff, flags);
+  highlighter.hilite(diffResult);
 
-  return {
-    file: fileName,
-    diff: fileDiffData
-  };
+  return diffResult;
 }
 
-function isDatapointFile(filename) {
-  return filename.indexOf("--datapoints--") != -1 ? true : false;
-}
-
 module.exports = new diffByFile();

test-diff.js

 'use strict';
 
-const util = require('util');
 const fs = require('fs');
-
 const gitCsvDiff = require('./index');
@@ -12,2 +10,3 @@
 MockFrom += "gap,Gapminder,Sweden" + "\r\n";
+MockFrom += "zsoft,ZSoft,Ukraine" + "\r\n";
 MockFrom += "valor,Valor Sotware,Ukraine";
@@ -18,14 +17,35 @@
 MockTo += "mcrsft_updated,United States of America" + "\r\n";
+MockTo += "gap,Gapminder_updated" + "\r\n";
 MockTo += "valor_updated,Ukraine";
 
-const fileName = "ddf--entities--company.csv";
-const fileModifier = "M";
-const resultDiff = {
+let MockDatapackageOld = {"name":"ddf--ws-testing","title":"ddf--ws-testing","description":"","version":"0.0.1","language":{"id":"en","name":"English"},"translations":[{"id":"nl-nl","name":"Dutch"}],"license":"","author":"","resources":[{"path":"ddf--concepts.csv","name":"ddf--concepts","schema":{"fields":[{"name":"concept"},{"name":"concept_type"},{"name":"domain"},{"name":"additional_column"}],"primaryKey":"concept"}},{"path":"ddf--entities--region.csv","name":"ddf--entities--region","schema":{"fields":[{"name":"region"},{"name":"full_name_changed"}],"primaryKey":"company"}}]};
+let MockDatapackageNew = {"name":"ddf--ws-testing","title":"ddf--ws-testing","description":"","version":"0.0.1","language":{"id":"en","name":"English"},"translations":[{"id":"nl-nl","name":"Dutch"}],"license":"","author":"","resources":[{"path":"ddf--entities--region.csv","name":"ddf--entities--region","schema":{"fields":[{"name":"company"},{"name":"full_name_changed"}],"primaryKey":"company"}},{"path":"ddf--concepts.csv","name":"ddf--concepts","schema":{"fields":[{"name":"concept"},{"name":"concept_type"},{"name":"domain"},{"name":"additional_column"}],"primaryKey":"company"}},{"path":"ddf--entities--region.csv","name":"ddf--entities--region","schema":{"fields":[{"name":"region"},{"name":"full_name_changed"}],"primaryKey":"company"}}]};
+
+/* params */
+
+const metaData = {
+  //fileName: "lang/nl-nl/ddf--entities--region.csv",
+  fileName: "ddf--entities--region.csv",
+  fileModifier: "M",
+  datapackage: {
+    old: MockDatapackageOld,
+    new: MockDatapackageNew
+  }
+};
+const dataDiff = {
   from: MockFrom,
   to: MockTo
 };
+const streams = {
+  diff: fs.createWriteStream('result-diff.txt'),
+  lang: fs.createWriteStream('result-diff-lang.txt')
+};
 
-gitCsvDiff.process(fileName, resultDiff, fileModifier, function(error, result){
+/* usage */
+
+gitCsvDiff.process(metaData, dataDiff, streams, function(){
   console.log("Done!");
-  console.log(util.inspect(result, false, null));
+
+  streams.diff.end();
+  streams.lang.end();
 });

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Number of package files

20

increased by53.85%

Number of low supply chain risk alerts

1

decreased by-50%

Worsened metrics

Total package byte prevSize

63786

decreased by-13.66%

Lines of code

690

decreased by-27.14%

Number of lines in readme file

46

decreased by-36.99%

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes