bank-csv-importer
Advanced tools
Comparing version 0.0.3 to 0.0.4
@@ -8,4 +8,7 @@ { | ||
}, | ||
"scripts": { | ||
"test": "node test.js" | ||
}, | ||
"type": "module", | ||
"version": "0.0.3" | ||
"version": "0.0.4" | ||
} |
@@ -31,6 +31,6 @@ import {findTypes} from "./parseTypes.js"; | ||
// Strip anything in quotes | ||
lines = lines.map(line => { | ||
return line.replace(/"[^"]+"/g, "") | ||
.replace(/'[^']+'/g, ""); | ||
}); | ||
lines = lines.map( | ||
line => line.replace(/"[^"]+"/g, "") | ||
.replace(/'[^']+'/g, "") | ||
); | ||
@@ -169,14 +169,11 @@ // Count which separator char appears the most | ||
// Check all records have the same number of fields | ||
for (let i = 1; i < records.length; i ++) { | ||
if (records[i-1].length != records[i].length) { | ||
throw { | ||
name : "FieldCountMismatch", | ||
message : `Not all records have the same number of fields:\n${records[i-1]}\n${records[i]}` | ||
}; | ||
} | ||
} | ||
// Find the row with the most fields | ||
const longestFieldCount = records.map(row => row.length) | ||
.reduce((acc, cur) => cur > acc ? cur : acc, 0); | ||
// Field counts are not zero... | ||
if (records[0].length == 0) { | ||
// Remove any zero length rows (probably shouldn't ever be required...) | ||
records = records.filter(row => row.length > 0); | ||
// This probably won't ever occur either, but best to check... | ||
if (longestFieldCount == 0 || records.length == 0) { | ||
throw { | ||
@@ -188,2 +185,11 @@ name : "NoFields", | ||
// Pad all other rows to the longest length | ||
records = records.map(row => { | ||
if (row.length < longestFieldCount) { | ||
row = row.concat(new Array(longestFieldCount - row.length).fill("")); | ||
} | ||
return row; | ||
}); | ||
// Attempt to coerce fields to various types (Date, Number) | ||
@@ -190,0 +196,0 @@ let typedRecords = findTypes(records); |
91
test.js
@@ -9,51 +9,67 @@ import fs from "fs"; | ||
let fileNames = fs.readdirSync("testdata"); | ||
let errors = []; | ||
for (let fileName of fileNames) { | ||
if (path.extname(fileName) == ".csv") { | ||
let data = fs.readFileSync("testdata/" + fileName, "utf-8"); | ||
function testdir(dirPath) { | ||
let fileNames; | ||
try { | ||
let result = bankImport(data); | ||
let resultsPath = "testdata/" + path.basename(fileName, ".csv") + ".output.json"; | ||
let typesPath = "testdata/" + path.basename(fileName, ".csv") + ".output.types.json"; | ||
try { | ||
fileNames = fs.readdirSync(dirPath); | ||
} catch (error) { | ||
if (error.code == "ENOENT") { | ||
console.log(`Directory "${dirPath}" not found, skipping...`); | ||
return; | ||
} else { | ||
throw error; | ||
} | ||
} | ||
// Get types as strings. Deep copy! | ||
let types = [...result.typedRecords]; | ||
console.log(`Testing csv files in directory "${dirPath}"...`); | ||
for (let i = 0; i < types.length; i ++) { | ||
types[i] = [...types[i]]; | ||
for (let fileName of fileNames) { | ||
if (path.extname(fileName) == ".csv") { | ||
let data = fs.readFileSync(path.join(dirPath, fileName), "utf-8"); | ||
for (let j = 0; j < types[i].length; j ++) { | ||
types[i][j] = types[i][j].constructor.name; | ||
try { | ||
let result = bankImport(data); | ||
let resultsPath = path.join(dirPath, "output", path.basename(fileName, ".csv") + ".output.json"); | ||
let typesPath = path.join(dirPath, "output", path.basename(fileName, ".csv") + ".output.types.json"); | ||
// Get types as strings. Deep copy! | ||
let types = [...result.typedRecords]; | ||
for (let i = 0; i < types.length; i ++) { | ||
types[i] = [...types[i]]; | ||
for (let j = 0; j < types[i].length; j ++) { | ||
types[i][j] = types[i][j].constructor.name; | ||
} | ||
} | ||
} | ||
// Stringify | ||
let resultString = JSON.stringify(result, null, "\t"); | ||
let typesString = JSON.stringify(types, null, "\t"); | ||
// Stringify | ||
let resultString = JSON.stringify(result, null, "\t"); | ||
let typesString = JSON.stringify(types, null, "\t"); | ||
// Write complete results (if didn't previously) | ||
if (!fs.existsSync(resultsPath)) { | ||
fs.writeFileSync(resultsPath, resultString); | ||
console.log("Warning, wrote new results as didn't exist", resultsPath); | ||
} | ||
// Write complete results (if didn't previously) | ||
if (!fs.existsSync(resultsPath)) { | ||
fs.writeFileSync(resultsPath, resultString); | ||
console.log("Warning, wrote new results as didn't exist", resultsPath); | ||
} | ||
// Write only the types (if didn't previously) | ||
if (!fs.existsSync(typesPath)) { | ||
fs.writeFileSync(typesPath, typesString); | ||
console.log("Warning, wrote new types as didn't exist", typesPath); | ||
} | ||
// Write only the types (if didn't previously) | ||
if (!fs.existsSync(typesPath)) { | ||
fs.writeFileSync(typesPath, typesString); | ||
console.log("Warning, wrote new types as didn't exist", typesPath); | ||
} | ||
// Compare new results to old, cached results! | ||
// Compare new results to old, cached results! | ||
let prevResult = fs.readFileSync(resultsPath, {encoding : "utf8"}); | ||
let prevTypes = fs.readFileSync(typesPath, {encoding : "utf8"}); | ||
let prevResult = fs.readFileSync(resultsPath, {encoding : "utf8"}); | ||
let prevTypes = fs.readFileSync(typesPath, {encoding : "utf8"}); | ||
if (resultString !== prevResult || typesString !== prevTypes) { | ||
errors.push("Mismatch with data file " + fileName); | ||
if (resultString !== prevResult || typesString !== prevTypes) { | ||
errors.push("Mismatch with data file " + fileName); | ||
} | ||
} catch (error) { | ||
console.log("Error", error); | ||
} | ||
} catch (error) { | ||
console.log("Error", error); | ||
} | ||
@@ -63,2 +79,5 @@ } | ||
testdir("testdata"); | ||
testdir("testdata-private"); | ||
if (errors.length == 0) { | ||
@@ -72,2 +91,4 @@ console.log("Success, no errors."); | ||
} | ||
process.exit(1); | ||
} |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
53392
31
2850