Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More
Socket
Sign inDemoInstall
Socket

csvnorm

Package Overview
Dependencies
Maintainers
1
Versions
16
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

csvnorm - npm Package Compare versions

Comparing version 0.7.0 to 0.8.0

binary/tests/datetimes.d.ts

24

binary/source/cli.js
#! /usr/bin/env node
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const path = require("path");
const yargs = require("yargs");
const index_1 = require("./index");
const path_1 = __importDefault(require("path"));
const yargs_1 = __importDefault(require("yargs"));
const index_1 = __importDefault(require("./index"));
const { stdin, stdout, argv } = process;

@@ -13,3 +16,3 @@ function logMetaInfos() {

function main(args) {
const options = yargs
const options = yargs_1.default
.usage([

@@ -22,3 +25,3 @@ 'Usage:',

'date-format': {
describe: 'Overwrite detected date format',
describe: 'Specify an additional prioritized input date format',
type: 'string',

@@ -35,2 +38,7 @@ },

},
'iso-datetime': {
default: false,
describe: 'Output datetimes with format YYYY-MM-DD[T]HH:mm:ss.SSS[Z]',
type: 'boolean',
},
'skip-start': {

@@ -53,3 +61,3 @@ default: 0,

if (stdin.isTTY) {
yargs.showHelp();
yargs_1.default.showHelp();
return;

@@ -63,2 +71,3 @@ }

encoding: options.encoding,
isoDatetime: options['iso-datetime'],
readableStream: stdin,

@@ -78,4 +87,5 @@ // skipLinesEnd: options['skip-end'],

encoding: options.encoding,
filePath: path.resolve(csvFilePath),
filePath: path_1.default.resolve(csvFilePath),
inPlace: options['in-place'],
isoDatetime: options['iso-datetime'],
// skipLinesEnd: options['skip-end'],

@@ -82,0 +92,0 @@ skipLinesStart: options['skip-start'],

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const formatNumber_1 = require("./formatNumber");
const formatNumber_1 = __importDefault(require("./formatNumber"));
exports.default = (value) => {

@@ -5,0 +8,0 @@ value = value.trim();

@@ -1,2 +0,2 @@

declare const _default: (value: string, dateFormat?: string | undefined) => string | undefined;
declare const _default: (value: string, dateFormat?: string | undefined, isoDatetime?: boolean) => string | undefined;
export default _default;

@@ -29,3 +29,3 @@ "use strict";

]);
exports.default = (value, dateFormat) => {
exports.default = (value, dateFormat, isoDatetime = false) => {
if (typeof value !== 'string') {

@@ -57,3 +57,19 @@ return undefined;

}
if (isoDatetime) {
const isoRegex = new RegExp([
/^([0-9]{4})-([01][0-9])-([0-3][0-9])/,
/[ t]/,
/[0-2][0-9]:[0-5][0-9](:[0-5][0-9])?/,
/(.[0-9]{3})?(|z|\+[0-9][0-9]:?[0-9][0-9])$/,
]
.map((regex) => regex.source)
.join(''), 'i');
if (isoRegex.test(value)) {
const parsedDate = new Date(value);
if (parsedDate) {
return parsedDate.toISOString();
}
}
}
return undefined;
};
/// <reference types="node" />
import * as stream from 'stream';
import stream from 'stream';
interface FormatterOptions {
objectMode?: boolean;
dateFormat?: string;
isoDatetime?: boolean;
}
export default class Formatter extends stream.Transform {
private dateFormat?;
private isoDatetime?;
constructor(opts: FormatterOptions);

@@ -10,0 +12,0 @@ _transform(row: string[], _1: string, chunkIsProcessedCb: () => void): void;

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const stream = require("stream");
const formatCurrency_1 = require("./formatCurrency");
const formatDate_1 = require("./formatDate");
const formatNumber_1 = require("./formatNumber");
class Formatter extends stream.Transform {
const stream_1 = __importDefault(require("stream"));
const formatCurrency_1 = __importDefault(require("./formatCurrency"));
const formatDate_1 = __importDefault(require("./formatDate"));
const formatNumber_1 = __importDefault(require("./formatNumber"));
class Formatter extends stream_1.default.Transform {
constructor(opts) {

@@ -12,2 +15,3 @@ opts.objectMode = true;

this.dateFormat = opts.dateFormat;
this.isoDatetime = opts.isoDatetime;
}

@@ -17,3 +21,3 @@ _transform(row, _1, chunkIsProcessedCb) {

.map((cell) => {
const date = formatDate_1.default(cell, this.dateFormat);
const date = formatDate_1.default(cell, this.dateFormat, this.isoDatetime);
if (date) {

@@ -20,0 +24,0 @@ return date;

/// <reference types="node" />
import * as stream from 'stream';
import stream from 'stream';
interface MainOptions {

@@ -8,2 +8,3 @@ dateFormat?: string;

inPlace?: boolean;
isoDatetime?: boolean;
readableStream?: stream;

@@ -10,0 +11,0 @@ skipLinesStart?: number;

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const assert = require("assert");
const fs = require("fs");
const path = require("path");
const stream = require("stream");
const csvParse = require("csv-parse");
const csvStringify = require("csv-stringify");
const tempfile = require("tempfile");
const toUtf8 = require("to-utf-8");
const Formatter_1 = require("./Formatter");
const assert_1 = __importDefault(require("assert"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const stream_1 = __importDefault(require("stream"));
const csv_parse_1 = __importDefault(require("csv-parse"));
const csv_stringify_1 = __importDefault(require("csv-stringify"));
const tempfile_1 = __importDefault(require("tempfile"));
const to_utf_8_1 = __importDefault(require("to-utf-8"));
const Formatter_1 = __importDefault(require("./Formatter"));
function printCsv(options) {
const { configGenerator, dateFormat, encoding, skipLinesStart = 0,
const { configGenerator, dateFormat, encoding, isoDatetime, skipLinesStart = 0,
// skipLinesEnd = 0,
inputFilePath, writableStream, } = options;
const formatter = new Formatter_1.default({ dateFormat });
const parser = csvParse({
const formatter = new Formatter_1.default({ dateFormat, isoDatetime });
const parser = csv_parse_1.default({
delimiter: configGenerator.mostFrequentDelimter,

@@ -22,7 +25,14 @@ from_line: skipLinesStart + 1,

parser.on('error', console.error);
const stringifier = csvStringify();
const stringifier = csv_stringify_1.default({
cast: {
date: (date) => {
console.error(date);
return date;
},
},
});
stringifier.on('error', console.error);
fs
fs_1.default
.createReadStream(inputFilePath)
.pipe(toUtf8({ encoding }))
.pipe(to_utf_8_1.default({ encoding }))
.pipe(parser)

@@ -33,3 +43,3 @@ .pipe(formatter)

}
class ConfigGenerator extends stream.Writable {
class ConfigGenerator extends stream_1.default.Writable {
constructor(opts) {

@@ -63,3 +73,3 @@ super(opts);

exports.default = (options) => {
const { dateFormat, encoding, filePath, inPlace, readableStream,
const { dateFormat, encoding, filePath, inPlace, isoDatetime = false, readableStream,
// skipLinesEnd = 0,

@@ -70,11 +80,11 @@ skipLinesStart = 0, } = options;

if (filePath) {
assert(path.isAbsolute(filePath));
fs
assert_1.default(path_1.default.isAbsolute(filePath));
fs_1.default
.createReadStream(filePath)
.pipe(configGenerator);
if (inPlace) {
const tempFilePath = tempfile('.csv');
writableStream = fs.createWriteStream(tempFilePath);
const tempFilePath = tempfile_1.default('.csv');
writableStream = fs_1.default.createWriteStream(tempFilePath);
writableStream.on('finish', () => {
fs.rename(tempFilePath, filePath, console.error);
fs_1.default.rename(tempFilePath, filePath, console.error);
});

@@ -88,2 +98,3 @@ }

inputFilePath: filePath,
isoDatetime,
// skipLinesEnd,

@@ -96,4 +107,4 @@ skipLinesStart,

}
const temporaryFilePath = tempfile('.csv');
const writableTempFile = fs.createWriteStream(temporaryFilePath);
const temporaryFilePath = tempfile_1.default('.csv');
const writableTempFile = fs_1.default.createWriteStream(temporaryFilePath);
let firstStreamFinished = false;

@@ -110,2 +121,3 @@ function syncStreams() {

inputFilePath: temporaryFilePath,
isoDatetime,
// skipLinesEnd,

@@ -112,0 +124,0 @@ skipLinesStart,

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const fs = require("fs");
const path = require("path");
const StreamTester = require("streamtester");
const index_1 = require("../source/index");
const streamTester = new StreamTester();
const testsDir = path.resolve(__dirname, '../../tests');
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const streamtester_1 = __importDefault(require("streamtester"));
const index_1 = __importDefault(require("../source/index"));
const streamTester = new streamtester_1.default();
const testsDir = path_1.default.resolve(__dirname, '../../tests');
streamTester.on('finish', () => {

@@ -14,4 +17,4 @@ console.info(' ✔︎');

index_1.default({
readableStream: fs.createReadStream(path.join(testsDir, 'banking/input-utf-16-le-empty-line.csv')),
readableStream: fs_1.default.createReadStream(path_1.default.join(testsDir, 'banking/input-utf-16-le-empty-line.csv')),
writableStream: streamTester,
});
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const assert = require("assert");
const fs = require("fs");
const path = require("path");
const StreamTester = require("streamtester");
const index_1 = require("../source/index");
const testsDir = path.resolve(__dirname, '../../tests');
const assert_1 = __importDefault(require("assert"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const streamtester_1 = __importDefault(require("streamtester"));
const index_1 = __importDefault(require("../source/index"));
const testsDir = path_1.default.resolve(__dirname, '../../tests');
let buffer = Buffer.alloc(0);
const streamTester = new StreamTester({
const streamTester = new streamtester_1.default({
test: (chunk) => {

@@ -23,3 +26,3 @@ buffer = Buffer.concat([buffer, chunk]);

].join('\n'));
assert.equal(buffer.toString(), expected.toString());
assert_1.default.equal(buffer.toString(), expected.toString());
});

@@ -29,4 +32,4 @@ process.stdout.write('Force data format for parsing CSV');

dateFormat: 'dd/mm/yyyy',
readableStream: fs.createReadStream(path.join(testsDir, 'banking/unusual-date-format.csv')),
readableStream: fs_1.default.createReadStream(path_1.default.join(testsDir, 'banking/unusual-date-format.csv')),
writableStream: streamTester,
});
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const assert = require("assert");
const path = require("path");
const execa = require("execa");
const fse = require("fs-extra");
const testsDir = path.resolve(__dirname, '../../tests');
const assert_1 = __importDefault(require("assert"));
const path_1 = __importDefault(require("path"));
const execa_1 = __importDefault(require("execa"));
const fs_extra_1 = __importDefault(require("fs-extra"));
const testsDir = path_1.default.resolve(__dirname, '../../tests');
async function main() {
process.stdout.write('Format banking CSV file via CLI in place');
const filePathAbsolute = path.join(testsDir, 'banking/input-latin1.csv');
const tempPathAbsolute = path.join(testsDir, 'banking/input-latin1.temp.csv');
const expectedOutput = await fse.readFile(path.join(testsDir, 'banking/expected-output.csv'), 'utf-8');
await fse.remove(tempPathAbsolute);
await fse.copy(filePathAbsolute, tempPathAbsolute);
await execa('./binary/source/cli.js', ['--in-place', tempPathAbsolute]);
const tempContent = await fse.readFile(tempPathAbsolute);
assert.equal(tempContent, expectedOutput);
await fse.remove(tempPathAbsolute);
const filePathAbsolute = path_1.default.join(testsDir, 'banking/input-latin1.csv');
const tempPathAbsolute = path_1.default.join(testsDir, 'banking/input-latin1.temp.csv');
const expectedOutput = await fs_extra_1.default.readFile(path_1.default.join(testsDir, 'banking/expected-output.csv'), 'utf-8');
await fs_extra_1.default.remove(tempPathAbsolute);
await fs_extra_1.default.copy(filePathAbsolute, tempPathAbsolute);
await execa_1.default('./binary/source/cli.js', ['--in-place', tempPathAbsolute]);
const tempContent = await fs_extra_1.default.readFile(tempPathAbsolute);
assert_1.default.equal(tempContent, expectedOutput);
await fs_extra_1.default.remove(tempPathAbsolute);
console.info(' ✔︎');
}
main();
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const assert = require("assert");
const assert_1 = __importDefault(require("assert"));
const child_process_1 = require("child_process");
const fs = require("fs");
const path = require("path");
const rootDir = path.resolve(__dirname, '../..');
const testsDir = path.resolve(rootDir, 'tests');
const expectedOutput = fs.readFileSync(path.join(testsDir, 'banking/expected-output.csv'), 'utf-8');
const inputFile = path.join(testsDir, 'banking/input-latin1.csv');
const executable = path.join(rootDir, 'binary/source/cli.js');
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const rootDir = path_1.default.resolve(__dirname, '../..');
const testsDir = path_1.default.resolve(rootDir, 'tests');
const expectedOutput = fs_1.default.readFileSync(path_1.default.join(testsDir, 'banking/expected-output.csv'), 'utf-8');
const inputFile = path_1.default.join(testsDir, 'banking/input-latin1.csv');
const executable = path_1.default.join(rootDir, 'binary/source/cli.js');
process.stdout.write('Format banking CSV from stdin via CLI');

@@ -20,4 +23,4 @@ child_process_1.exec(`cat ${inputFile} | ${executable}`, { cwd: testsDir }, (error, stdout, stderr) => {

}
assert.deepEqual(stdout, expectedOutput);
assert_1.default.deepEqual(stdout, expectedOutput);
console.info(' ✔︎');
});
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const assert = require("assert");
const fs = require("fs");
const path = require("path");
const StreamTester = require("streamtester");
const index_1 = require("../source/index");
const testsDir = path.resolve(__dirname, '../../tests');
const assert_1 = __importDefault(require("assert"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const streamtester_1 = __importDefault(require("streamtester"));
const index_1 = __importDefault(require("../source/index"));
const testsDir = path_1.default.resolve(__dirname, '../../tests');
let buffer = Buffer.alloc(0);
const streamTester = new StreamTester({
const streamTester = new streamtester_1.default({
test: (chunk) => {

@@ -24,3 +27,3 @@ buffer = Buffer.concat([buffer, chunk]);

].join('\n'));
assert.equal(buffer.toString(), expected.toString());
assert_1.default.equal(buffer.toString(), expected.toString());
});

@@ -30,3 +33,3 @@ process.stdout.write('Parse CSV file with preceding lines');

encoding: 'utf-8',
readableStream: fs.createReadStream(path.join(testsDir, 'banking/leading-lines.csv')),
readableStream: fs_1.default.createReadStream(path_1.default.join(testsDir, 'banking/leading-lines.csv')),
// skipLinesEnd: 4,

@@ -33,0 +36,0 @@ skipLinesStart: 5,

"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const assert = require("assert");
const assert_1 = __importDefault(require("assert"));
const child_process_1 = require("child_process");
const fs = require("fs");
const path = require("path");
const rootDir = path.resolve(__dirname, '../..');
const testsDir = path.resolve(rootDir, 'tests');
const expectedOutput = fs.readFileSync(path.join(testsDir, 'banking/expected-output.csv'), 'utf-8');
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const rootDir = path_1.default.resolve(__dirname, '../..');
const testsDir = path_1.default.resolve(rootDir, 'tests');
const expectedOutput = fs_1.default.readFileSync(path_1.default.join(testsDir, 'banking/expected-output.csv'), 'utf-8');
process.stdout.write('Format banking CSV file via CLI');
child_process_1.execFile(path.join(rootDir, 'binary/source/cli.js'), [path.join(testsDir, 'banking/input-latin1.csv')], { cwd: testsDir }, (error, stdout, stderr) => {
child_process_1.execFile(path_1.default.join(rootDir, 'binary/source/cli.js'), [path_1.default.join(testsDir, 'banking/input-latin1.csv')], { cwd: testsDir }, (error, stdout, stderr) => {
if (stderr) {

@@ -18,4 +21,4 @@ console.error(stderr);

}
assert.deepEqual(stdout, expectedOutput);
assert_1.default.deepEqual(stdout, expectedOutput);
console.info(' ✔︎');
});
"use strict";
var __importDefault = (this && this.__importDefault) || function (mod) {
return (mod && mod.__esModule) ? mod : { "default": mod };
};
Object.defineProperty(exports, "__esModule", { value: true });
const assert = require("assert");
const fs = require("fs");
const path = require("path");
const StreamTester = require("streamtester");
const index_1 = require("../source/index");
const testsDir = path.resolve(__dirname, '../../tests');
const streamTester = new StreamTester({
const assert_1 = __importDefault(require("assert"));
const fs_1 = __importDefault(require("fs"));
const path_1 = __importDefault(require("path"));
const streamtester_1 = __importDefault(require("streamtester"));
const index_1 = __importDefault(require("../source/index"));
const testsDir = path_1.default.resolve(__dirname, '../../tests');
const streamTester = new streamtester_1.default({
test: (csvChunk) => {

@@ -17,3 +20,3 @@ const match = csvChunk

: 0;
assert.equal(numberOfCommas, 6);
assert_1.default.equal(numberOfCommas, 6);
},

@@ -26,4 +29,4 @@ });

index_1.default({
readableStream: fs.createReadStream(path.join(testsDir, 'banking/input-latin1.csv')),
readableStream: fs_1.default.createReadStream(path_1.default.join(testsDir, 'banking/input-latin1.csv')),
writableStream: streamTester,
});
{
"name": "csvnorm",
"version": "0.7.0",
"version": "0.8.0",
"description": "Command line tool to normalize CSV and *SV files.",

@@ -17,20 +17,20 @@ "main": "binary/source/index.js",

"dependencies": {
"@types/execa": "^0.9.0",
"@types/fs-extra": "^5.0.4",
"@types/node": "^10.12.18",
"@types/tempfile": "^2.0.0",
"@types/yargs": "^12.0.5",
"csv-parse": "^4.3.0",
"csv-stringify": "^5.1.2",
"execa": "^1.0.0",
"fs-extra": "^7.0.1",
"tempfile": "^2.0.0",
"@types/execa": "^2.0.0",
"@types/fs-extra": "^8.0.1",
"@types/node": "^13.1.2",
"@types/tempfile": "^3.0.0",
"@types/yargs": "^13.0.4",
"csv-parse": "^4.8.3",
"csv-stringify": "^5.3.6",
"execa": "^4.0.0",
"fs-extra": "^8.1.0",
"tempfile": "^3.0.0",
"to-utf-8": "^1.3.0",
"yargs": "^12.0.5"
"yargs": "^15.1.0"
},
"devDependencies": {
"streamtester": "^0.2.0",
"ts-node": "^7.0.1",
"tslint": "^5.12.0",
"typescript": "^3.2.2"
"ts-node": "^8.5.4",
"tslint": "^5.20.1",
"typescript": "^3.7.4"
},

@@ -37,0 +37,0 @@ "author": "Adrian Sieber <mail@adriansieber.com>",

@@ -36,7 +36,10 @@ # Csvnorm

Options:
--encoding Overwrite detected input encoding [string]
--in-place Normalize CSV file in place [boolean] [default: false]
--skip-start Skip lines at the start of the input [number] [default: 0]
--version Show version number [boolean]
--help Show help [boolean]
--date-format Specify an additional prioritized input date format [string]
--encoding Overwrite detected input encoding [string]
--in-place Normalize CSV file in place [boolean] [default: false]
--iso-datetime Output datetimes with format YYYY-MM-DD[T]HH:mm:ss.SSS[Z]
[boolean] [default: false]
--skip-start Skip lines at the start of the input [number] [default: 0]
--version Show version number [boolean]
--help Show help [boolean]

@@ -46,2 +49,4 @@ Examples:

cat input.csv | csvnorm > normalized.csv Pipe and normalize a CSV file
csvnorm --date-format "dd/mm/yyyy" i.csv Normalize a CSV file with an unusual
date format
```

@@ -48,0 +53,0 @@

@@ -5,2 +5,3 @@ {

"declaration": true,
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,

@@ -18,4 +19,4 @@ "module": "commonjs",

"strictFunctionTypes": true,
"strictNullChecks" : true,
"strictPropertyInitialization": true,
"strictNullChecks" : true,
"target": "es2017"

@@ -22,0 +23,0 @@ },

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc