csvtojson - npm Package Compare versions

libs/core/filterRow.js

test/testCSVConverter3.js

58

bin/benchmark.js

		#!/usr/bin/env node
		var path=process.argv[2];
		var fs=require('fs');
		if (!fs.existsSync(path)){
		var path = process.argv[2];
		var fs = require('fs');
		if (!fs.existsSync(path)) {
		console.log("Run: benchmark [csvpath]");
		console.log("File not found: ",path);
		console.log("File not found: ", path);
		process.exit(1);
		}
		function testCSVLine(coreNum,checkType,cb){
		console.log("WorkerNum:",coreNum,"Check Type: ",checkType);
		var Converter=require("../libs/core/Converter");
		var rs=fs.createReadStream(path);
		var converter=new Converter({
		workerNum:coreNum,
		checkType:checkType,
		constructResult:false
		function testCSVLine(coreNum, checkType, cb){
		console.log("WorkerNum:", coreNum, "Check Type: ", checkType);
		var Converter = require("../libs/core/Converter");
		var rs = fs.createReadStream(path);
		var converter = new Converter({
		workerNum: coreNum,
		checkType: checkType,
		constructResult: false
		// fork:true
		});
		var totalLines=0;
		var secLines=0;
		converter.on("data",function(){
		totalLines++;
		secLines++;
		var totalLines = 0;
		var secLines = 0;
		converter.on("data", function() {
		totalLines++;
		secLines++;
		});
		converter.on("end",function(){
		converter.on("end", function() {
		clearInterval(timer);
		console.log("");
		var t=new Date()-start;
		console.log("Time elapsed: ",t, " ms");
		console.log("Total lines: "+totalLines);
		console.log("Average Speed: "+Math.round(totalLines/t*1000)+" Lines / Sec");
		var t = new Date() - start;
		console.log("Time elapsed: ", t, " ms");
		console.log("Total lines: " + totalLines);
		console.log("Average Speed: " + Math.round(totalLines / t * 1000) + " Lines / Sec");
		cb();
		})
		var timer=setInterval(function(){
		process.stdout.write("\r"+secLines+" CSV Lines/Sec");
		secLines=0;
		},1000);
		var start=new Date();
		});
		var timer = setInterval(function() {
		process.stdout.write("\r" + secLines + " CSV Lines/Sec");
		secLines = 0;
		}, 1000);
		var start = new Date();

		var stream=rs.pipe(converter);
		var stream = rs.pipe(converter);
		}
		@@ -42,0 +42,0 @@

2

bin/csvtojson.js

		@@ -71,3 +71,3 @@ function csvtojson() {
		process.exit(1);
		})
		});
		is.pipe(conv).pipe(process.stdout);
		@@ -74,0 +74,0 @@ // is.pipe(conv);

37

bin/genCsv.js

		#!/usr/bin/env node
		var minimist=require("minimist");
		var argv=process.argv;
		var minimist = require("minimist");
		var argv = process.argv;
		argv.shift();
		argv.shift();
		var args=minimist(argv);
		var headers=["name","header1","file2","description","header2","field2","header3"];
		var args = minimist(argv);
		var headers = ["name", "header1", "file2", "description", "header2", "field2", "header3"];

		if (args.headers){
		headers=JSON.parse(args.headers);
		if (args.headers) {
		headers = JSON.parse(args.headers);
		}
		var rowNum=args.row?args.row:10000;
		var chars=args.chars?args.chars:"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
		var maxLength=parseInt(args.max?args.max:"15");
		var rowNum = args.row ? args.row : 10000;
		var chars = args.chars ? args.chars : "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
		var maxLength = parseInt(args.max ? args.max : "15");
		console.log(headers.join(","));
		for (var i=0;i<rowNum;i++){
		var row=[];
		for (var j=0;j<headers.length;j++){
		for (var i = 0; i < rowNum; i++) {
		var row = [];
		for (var j = 0; j < headers.length; j++) {
		row.push(genWord());
		@@ -24,11 +24,10 @@ }


		function genWord(){
		var len=Math.round(Math.random()*maxLength);
		var rtn="";
		for (var i=0;i<len;i++){
		var pos=Math.round(Math.random()*chars.length);
		rtn+=chars[pos];
		function genWord() {
		var len = Math.round(Math.random() * maxLength);
		var rtn = "";
		for (var i = 0; i < len; i++) {
		var pos = Math.round(Math.random() * chars.length);
		rtn += chars[pos];
		}
		return rtn;
		}

6

bin/options.json

		@@ -20,3 +20,3 @@ {
		"--checkType": {
		"desc": "This parameter turns on and off whether check field type. default is true.",
		"desc": "This parameter turns on and off whether check field type. default is false.",
		"type": "boolean"
		@@ -70,7 +70,7 @@
		"--ignoreColumns": {
		"desc": "Columns to ignore on input. e.g. --ignoreColumns=# --ignoreColumns='[0,4,5]' ",
		"desc": "Columns to ignore on input. e.g. --ignoreColumns='[0,4,5, \"name\"]' ",
		"type": "~object"
		},
		"--includeColumns": {
		"desc": "Columns to include on input. e.g. --includeColumns=# --includeColumns='[0,4,5]' ",
		"desc": "Columns to include on input. e.g. --includeColumns='[0,4,5, \"title\"]' ",
		"type": "~object"
		@@ -77,0 +77,0 @@ }

2

Gruntfile.js

		@@ -27,3 +27,3 @@ /*
		mangle: true,
		banner:"/Automatically Generated. Do not modify./\n"
		banner: "/Automatically Generated. Do not modify./\n"
		},
		@@ -30,0 +30,0 @@ src: "./dist/csvtojson.js",

288

libs/core/Converter.js

		@@ -7,14 +7,13 @@ var util = require("util");
		var defParam = require("./defParam");
		var csvline = require("./csvline");
		var fileline = require("./fileline");
		var dataToCSVLine = require("./dataToCSVLine");
		var fileLineToCSVLine = require("./fileLineToCSVLine");
		var linesToJson = require("./linesToJson");
		var CSVError = require("./CSVError");
		var workerMgr = require("./workerMgr");
		var workerMgr = null;
		var _ = require('lodash');
		var rowSplit = require("./rowSplit");
		function Converter(params, options) {
		Transform.call(this, options);
		_param = defParam(params);
		this._options = options \|\| {};
		this.param = _param;
		this.param = defParam(params);
		this.param._options = this._options;
		@@ -51,12 +50,12 @@ // this.resultObject = new Result(this);
		if (this._needEmitFinalResult === null) {
		this._needEmitFinalResult = this.listeners("end_parsed").length > 0
		this._needEmitFinalResult = this.listeners("end_parsed").length > 0;
		}
		if (this._needEmitResult === null) {
		this._needEmitResult = this.listeners("record_parsed").length > 0
		this._needEmitResult = this.listeners("record_parsed").length > 0;
		}
		if (this._needEmitJson === null) {
		this._needEmitJson = this.listeners("json").length > 0
		this._needEmitJson = this.listeners("json").length > 0;
		}
		if (this._needEmitCsv === null) {
		this._needEmitCsv = this.listeners("csv").length > 0
		this._needEmitCsv = this.listeners("csv").length > 0;
		}
		@@ -67,19 +66,23 @@ if (this._needJson === null) {
		if (this._needPush === null) {
		this._needPush = this.listeners("data").length > 0 \|\| this.listeners("readable").length > 0
		this._needPush = this.listeners("data").length > 0 \|\| this.listeners("readable").length > 0;
		// this._needPush=false;
		}
		this.param._needParseJson = this._needJson \|\| this._needPush;
		}.bind(this));


		}.bind(this))
		return this;
		}

		util.inherits(Converter, Transform);
		function emitDone(conv) {
		return function (err) {
		process.nextTick(function () {
		conv.emit('done', err)
		})
		if (!conv._hasDone) {
		conv._hasDone = true;
		process.nextTick(function () {
		conv.emit('done', err);
		});
		};
		}
		}

		Converter.prototype._transform = function (data, encoding, cb) {
		@@ -100,10 +103,13 @@ if (this.param.toArrayString && this.started === false) {
		}
		})
		});
		};

		Converter.prototype.prepareData = function (data) {
		return this._csvLineBuffer + data;
		}
		};

		Converter.prototype.setPartialData = function (d) {
		this._csvLineBuffer = d;
		}
		};

		Converter.prototype.processData = function (data, cb) {
		@@ -114,6 +120,6 @@ var params = this.param;
		}
		var fileLines = fileline(data, this.param)
		var fileLines = fileline(data, this.param);
		if (fileLines.lines.length > 0) {
		if (this.preProcessLine && typeof this.preProcessLine === "function") {
		fileLines.lines = this._preProcessLines(fileLines.lines, this.lastIndex)
		fileLines.lines = this._preProcessLines(fileLines.lines, this.lastIndex);
		}
		@@ -127,3 +133,3 @@ if (!params._headers) { //header is not inited. init header
		var jsonArr = linesToJson(lines.lines, params, this.recordNum);
		this.processResult(jsonArr)
		this.processResult(jsonArr);
		this.lastIndex += jsonArr.length;
		@@ -137,44 +143,50 @@ this.recordNum += jsonArr.length;
		} else {
		this.setPartialData(fileLines.partial)
		this.setPartialData(fileLines.partial);
		cb();
		}
		}
		};

		Converter.prototype._preProcessLines = function (lines, startIdx) {
		var rtn = []
		for (var i = 0; i < lines.length; i++) {
		var result = this.preProcessLine(lines[i], startIdx + i + 1)
		var rtn = [];
		for (var i = 0, len = lines.length; i < len; i++) {
		var result = this.preProcessLine(lines[i], startIdx + i + 1);
		if (typeof result === "string") {
		rtn.push(result)
		rtn.push(result);
		} else {
		rtn.push(lines[i])
		this.emit("error", new Error("preProcessLine should return a string but got: " + JSON.stringify(result)))
		rtn.push(lines[i]);
		this.emit("error", new Error("preProcessLine should return a string but got: " + JSON.stringify(result)));
		}
		}
		return rtn
		}
		return rtn;
		};

		Converter.prototype.initWorker = function () {
		var workerNum = this.param.workerNum - 1;
		if (workerNum > 0) {
		workerMgr = require("./workerMgr");
		this.workerMgr = workerMgr();
		this.workerMgr.initWorker(workerNum, this.param);
		}
		}
		};

		Converter.prototype.preRawData = function (func) {
		this.preProcessRaw = func;
		return this;
		}
		};

		Converter.prototype.preFileLine = function (func) {
		this.preProcessLine = func;
		return this;
		}
		};

		/**
		* workerpRocess does not support embeded multiple lines.
		* workerpRocess does not support embeded multiple lines.
		*/

		Converter.prototype.workerProcess = function (fileLine, cb) {
		var self = this;
		var line = fileLine
		var eol = this.getEol()
		this.setPartialData(line.partial)
		var line = fileLine;
		var eol = this.getEol();
		this.setPartialData(line.partial);
		this.workerMgr.sendWorker(line.lines.join(eol) + eol, this.lastIndex, cb, function (results, lastIndex) {
		var buf;
		var cur = self.sequenceBuffer[0];
		@@ -185,10 +197,10 @@ if (cur.idx === lastIndex) {
		while (self.sequenceBuffer[0] && self.sequenceBuffer[0].result) {
		var buf = self.sequenceBuffer.shift();
		records = records.concat(buf.result)
		buf = self.sequenceBuffer.shift();
		records = records.concat(buf.result);
		}
		self.processResult(records)
		self.processResult(records);
		self.recordNum += records.length;
		} else {
		for (var i = 0; i < self.sequenceBuffer.length; i++) {
		var buf = self.sequenceBuffer[i];
		for (var i = 0, len = self.sequenceBuffer.length; i < len; i++) {
		buf = self.sequenceBuffer[i];
		if (buf.idx === lastIndex) {
		@@ -200,4 +212,3 @@ buf.result = results;
		}
		// self.processResult(JSON.parse(results),function(){},true);
		})
		});
		this.sequenceBuffer.push({
		@@ -208,37 +219,94 @@ idx: this.lastIndex,
		this.lastIndex += line.lines.length;
		}
		};

		Converter.prototype.processHead = function (fileLine, cb) {
		var params = this.param;
		if (!params._headers) { //header is not inited. init header
		var lines = fileLineToCSVLine(fileLine, params);
		this.setPartialData(lines.partial);
		if (params.noheader) {
		if (params.headers) {
		params._headers = params.headers;
		if (params._headers) {
		return cb();
		}
		//dirty hack
		params._needFilterRow = false;
		// if header is not inited. init header
		var lines = fileLine.lines;
		var left = "";
		var headerRow = [];
		if (!params.noheader) {
		while (lines.length) {
		var line = left + lines.shift();
		var row = rowSplit(line, params);
		if (row.closed) {
		headerRow = row.cols;
		left = "";
		break;
		} else {
		params._headers = [];
		left = line + this.getEol();
		}
		}
		}
		params._needFilterRow = true;
		if (!params.noheader && headerRow.length === 0) { //if one chunk of data does not complete header row.
		this.setPartialData(left);
		return cb();
		}
		if (params.noheader) {
		if (params.headers) {
		params._headers = params.headers;
		} else {
		var headerRow = lines.lines.shift();
		if (params.headers) {
		params._headers = params.headers;
		} else {
		params._headers = headerRow;
		params._headers = [];
		}
		} else {
		if (params.headers) {
		params._headers = params.headers;
		} else {
		params._headers = headerRow;
		}
		}
		configIgnoreIncludeColumns(params);
		params._headers = require("./filterRow")(params._headers, params);
		var lines = fileLineToCSVLine(fileLine, params);
		this.setPartialData(lines.partial);
		if (this.param.workerNum > 1) {
		this.workerMgr.setParams(params);
		}
		var res = linesToJson(lines.lines, params, 0);
		this.processResult(res);
		this.lastIndex += res.length;
		this.recordNum += res.length;

		cb();
		};
		function configIgnoreIncludeColumns(params) {
		if (params._postIgnoreColumns) {
		for (var i = 0; i < params.ignoreColumns.length; i++) {
		var ignoreCol = params.ignoreColumns[i];
		if (typeof ignoreCol === "string") {
		var idx = params._headers.indexOf(ignoreCol);
		if (idx > -1) {
		params.ignoreColumns[i] = idx;
		} else {
		params.ignoreColumns[i] = -1;
		}
		}
		}
		if (this.param.workerNum > 1) {
		this.workerMgr.setParams(params);
		params.ignoreColumns.sort(function (a, b) { return b - a; });
		}
		if (params._postIncludeColumns) {
		for (var i = 0; i < params.includeColumns.length; i++) {
		var includeCol = params.includeColumns[i];
		if (typeof includeCol === "string") {
		var idx = params._headers.indexOf(includeCol);
		if (idx > -1) {
		params.includeColumns[i] = idx;
		} else {
		params.includeColumns[i] = -1;
		}
		}
		}
		var res = linesToJson(lines.lines, params, 0);
		this.processResult(res);
		this.lastIndex += res.length;
		this.recordNum += res.length;
		cb();
		} else {
		cb();
		}
		params.ignoreColumns = _.uniq(params.ignoreColumns);
		params.includeColumns = _.uniq(params.includeColumns);
		}

		Converter.prototype.processResult = function (result) {

		for (var i = 0; i < result.length; i++) {
		for (var i = 0, len = result.length; i < len; i++) {
		var r = result[i];
		@@ -251,5 +319,3 @@ if (r.err) {
		}
		// this.lastIndex+=result.length;
		// cb();
		}
		};

		@@ -268,5 +334,5 @@ Converter.prototype.emitResult = function (r) {
		if (resultJson === null && this._needJson) {
		resultJson = JSON.parse(resultStr)
		resultJson = JSON.parse(resultStr);
		if (typeof row === "string") {
		row = JSON.parse(row)
		row = JSON.parse(row);
		}
		@@ -279,12 +345,12 @@ }
		if (this._needEmitJson) {
		this.emit("json", resultJson, index)
		this.emit("json", resultJson, index);
		}
		if (this._needEmitCsv) {
		if (typeof row === "string") {
		row = JSON.parse(row)
		row = JSON.parse(row);
		}
		this.emit("csv", row, index)
		this.emit("csv", row, index);
		}
		if (this.param.constructResult && this._needEmitFinalResult) {
		this.finalResult.push(resultJson)
		this.finalResult.push(resultJson);
		}
		@@ -302,3 +368,3 @@ if (this._needEmitResult) {
		if (resultStr === null) {
		resultStr = JSON.stringify(resultJson)
		resultStr = JSON.stringify(resultJson);
		}
		@@ -308,11 +374,13 @@ this.push(!this.param.toArrayString ? resultStr + eol : resultStr, "utf8");
		}
		}
		};

		Converter.prototype.preProcessRaw = function (data, cb) {
		cb(data);
		}
		};

		// FIXME: lineNumber is not used.
		Converter.prototype.preProcessLine = function (line, lineNumber) {
		return line;
		}
		};

		Converter.prototype._flush = function (cb) {
		@@ -325,10 +393,11 @@ var self = this;
		}
		cb()
		cb();
		if (!self._needPush) {
		self.emit("end")
		self.emit("end");
		}
		};
		if (this._csvLineBuffer.length > 0) {
		if (this._csvLineBuffer[this._csvLineBuffer.length - 1] != this.getEol()) {
		this._csvLineBuffer += this.getEol();
		var eol = this.getEol();
		if (this._csvLineBuffer[this._csvLineBuffer.length - 1] !== eol) {
		this._csvLineBuffer += eol;
		}
		@@ -343,9 +412,3 @@ this.processData(this._csvLineBuffer, function () {
		};
		// Converter.prototype._transformFork = function(data, encoding, cb) {
		// this.child.stdin.write(data, encoding, cb);
		// }
		// Converter.prototype._flushFork = function(cb) {
		// this.child.stdin.end();
		// this.child.on("exit", cb);
		// }

		Converter.prototype.checkAndFlush = function () {
		@@ -365,6 +428,7 @@ if (this._csvLineBuffer.length !== 0) {
		}
		}
		};

		Converter.prototype.getEol = function (data) {
		if (!this.param.eol && data) {
		for (var i = 0; i < data.length; i++) {
		for (var i = 0, len = data.length; i < len; i++) {
		if (data[i] === "\r") {
		@@ -387,5 +451,10 @@ if (data[i + 1] === "\n") {
		};
		Converter.prototype.fromFile = function (filePath, cb) {

		Converter.prototype.fromFile = function (filePath, cb, options) {
		var fs = require('fs');
		var rs = null;
		if (typeof cb ==="object" && typeof options === "undefined"){
		options=cb;
		cb=null;
		}
		this.wrapCallback(cb, function () {
		@@ -398,10 +467,11 @@ if (rs && rs.destroy) {
		if (exist) {
		rs = fs.createReadStream(filePath);
		rs = fs.createReadStream(filePath,options);
		rs.pipe(this);
		} else {
		this.emit('error', new Error("File not exist"))
		this.emit('error', new Error("File not exists"));
		}
		}.bind(this));
		return this;
		}
		};

		Converter.prototype.fromStream = function (readStream, cb) {
		@@ -413,9 +483,11 @@ if (cb && typeof cb === "function") {
		return this;
		}
		};

		Converter.prototype.transf = function (func) {
		this.transform = func;
		return this;
		}
		};

		Converter.prototype.fromString = function (csvString, cb) {
		if (typeof csvString != "string") {
		if (typeof csvString !== "string") {
		return cb(new Error("Passed CSV Data is not a string."));
		@@ -428,10 +500,10 @@ }
		process.nextTick(function () {
		this.end(csvString)
		}.bind(this))
		this.end(csvString);
		}.bind(this));
		return this;
		};

		Converter.prototype.wrapCallback = function (cb, clean) {

		if (clean === undefined) {
		clean = function () { }
		clean = function () { };
		}
		@@ -452,4 +524,4 @@ if (cb && typeof cb === "function") {
		}.bind(this));
		}
		};

		module.exports = Converter;

44

libs/core/CSVError.js

		@@ -1,27 +0,27 @@
		var util=require("util");
		module.exports=CSVError;
		function CSVError(err,index,extra){
		Error.call(this,"");
		this.err=err;
		this.line=index;
		this.extra=extra;
		this.message="Error: "+err+". JSON Line number: "+index+ (extra?" near: "+extra:"");
		this.name="CSV Error";
		var util = require("util");
		module.exports = CSVError;
		function CSVError(err, index, extra) {
		Error.call(this, "");
		this.err = err;
		this.line = index;
		this.extra = extra;
		this.message = "Error: " + err + ". JSON Line number: " + index + (extra ? " near: " + extra : "");
		this.name = "CSV Error";
		}
		util.inherits(CSVError,Error);
		util.inherits(CSVError, Error);

		CSVError.prototype.toString=function(){
		return JSON.stringify([this.err,this.line,this.extra]);
		}
		CSVError.prototype.toString = function() {
		return JSON.stringify([this.err, this.line, this.extra]);
		};

		CSVError.column_mismatched=function(index,extra){
		return new CSVError("column_mismatched",index,extra);
		}
		CSVError.column_mismatched = function(index, extra) {
		return new CSVError("column_mismatched", index, extra);
		};

		CSVError.unclosed_quote=function(index,extra){
		return new CSVError("unclosed_quote",index,extra);
		}
		CSVError.unclosed_quote = function(index, extra) {
		return new CSVError("unclosed_quote", index, extra);
		};

		CSVError.fromArray=function(arr){
		return new CSVError(arr[0],arr[1],arr[2]);
		}
		CSVError.fromArray = function(arr) {
		return new CSVError(arr[0], arr[1], arr[2]);
		};

30

libs/core/csvline.js

		@@ -1,5 +0,3 @@
		var getEol=require("./getEol");
		var getDelimiter=require("./getDelimiter");
		var toLines=require("./fileline");
		var rowSplit=require("./rowSplit");
		var getEol = require("./getEol");
		var rowSplit = require("./rowSplit");
		/**
		@@ -11,16 +9,16 @@ * Convert lines to csv columns
		*/
		module.exports=function(lines,param){
		var csvLines=[];
		var left="";
		while (lines.length){
		var line=left+lines.shift();
		var row=rowSplit(line,param);
		if (row.closed){
		module.exports = function(lines, param) {
		var csvLines = [];
		var left = "";
		while (lines.length) {
		var line = left + lines.shift();
		var row = rowSplit(line, param);
		if (row.closed) {
		csvLines.push(row.cols);
		left="";
		}else{
		left=line+getEol(line,param);
		left = "";
		} else {
		left = line + getEol(line, param);
		}
		}
		return {lines:csvLines,partial:left};
		}
		return {lines: csvLines, partial: left};
		};

18

libs/core/dataToCSVLine.js

		@@ -9,10 +9,10 @@ var fileline=require("./fileline");
		*/
		module.exports=function(data,params){
		var line=fileline(data,params);
		var lines=line.lines;
		var csvLines=csvline(lines,params);
		return {
		lines:csvLines.lines,
		partial:csvLines.partial+line.partial
		}
		}
		module.exports = function(data, params) {
		var line = fileline(data, params);
		var lines = line.lines;
		var csvLines = csvline(lines, params);
		return {
		lines: csvLines.lines,
		partial: csvLines.partial + line.partial
		};
		};

8

libs/core/defaultParsers/parser_flat.js

		module.exports = {
		"name": "flat",
		"processSafe":true,
		"processSafe": true,
		"regExp": /^\flat\/,
		"parserFunc": function parser_flat (params) {
		var key=this.getHeadStr();
		var val=params.item;
		params.resultRow[key]=val;
		var key = this.getHeadStr();
		var val = params.item;
		params.resultRow[key] = val;
		}
		};

4

libs/core/defaultParsers/parser_json.js

		@@ -39,3 +39,3 @@ var arrReg = /\[([0-9]*)\]/;
		var headArr = (params.config && params.config.flatKeys) ? [fieldStr] : fieldStr.split('.');
		var match, index, key, pointer;
		var match, index, key;
		//now the pointer is pointing the position to add a key/value pair.
		@@ -64,3 +64,3 @@ var pointer = processHead(params.resultRow, headArr, arrReg, params.config && params.config.flatKeys);
		} else {
		if (typeof pointer=== "string"){
		if (typeof pointer === "string"){
		params.resultRow[fieldStr] = params.item;
		@@ -67,0 +67,0 @@ }else{

26

libs/core/defParam.js

		@@ -0,1 +1,2 @@
		var numExp = /^[0-9]+$/;
		module.exports = function (params) {
		@@ -26,3 +27,3 @@ var _param = {
		_headers: null,
		_needFilterRow:false
		_needFilterRow: false
		};
		@@ -36,12 +37,21 @@ if (!params) {
		}
		};
		if (_param.ignoreColumns.length \|\| _param.includeColumns.length){
		_param._needFilterRow=true;
		_param.ignoreColumns.sort(function (a, b) { return b - a; });
		}

		if (_param.ignoreColumns.length > 0 && !numExp.test(_param.ignoreColumns.join(""))) {
		_param._postIgnoreColumns = true;
		}
		if (_param.includeColumns.length > 0 && !numExp.test(_param.includeColumns.join(""))) {
		_param._postIncludeColumns = true;
		}

		if (_param.ignoreColumns.length \|\| _param.includeColumns.length) {
		_param._needFilterRow = true;
		if (!_param._postIgnoreColumns){
		_param.ignoreColumns.sort(function (a, b) { return b-a;});
		}
		}


		return _param;
		}
		};


		function getEnv(key, def) {
		@@ -48,0 +58,0 @@ if (process.env[key]) {

22

libs/core/fileline.js

		@@ -1,2 +0,2 @@
		var getEol=require("./getEol");
		var getEol = require("./getEol");
		/**
		@@ -8,15 +8,7 @@ * convert data chunk to file lines array
		*/
		module.exports=function(data,param){
		var eol=getEol(data,param);
		var lines= data.split(eol);
		var partial=lines.pop();
		// if (param.ignoreEmpty){
		// var trimmedLines=[];
		// for (var i=0;i<lines.length;i++){
		// trimmedLines.push(lines[i].trim())
		// }
		// return {lines:trimmedLines,partial:partial};
		// }else{
		return {lines:lines,partial:partial};
		// }
		}
		module.exports = function(data, param) {
		var eol = getEol(data,param);
		var lines = data.split(eol);
		var partial = lines.pop();
		return {lines: lines, partial: partial};
		};

16

libs/core/fileLineToCSVLine.js

		@@ -8,9 +8,9 @@ var csvline=require("./csvline");
		*/
		module.exports=function(fileLine,params){
		var lines=fileLine.lines;
		var csvLines=csvline(lines,params);
		return {
		lines:csvLines.lines,
		partial:csvLines.partial+fileLine.partial
		}
		}
		module.exports = function(fileLine, params) {
		var lines = fileLine.lines;
		var csvLines = csvline(lines,params);
		return {
		lines: csvLines.lines,
		partial: csvLines.partial + fileLine.partial
		};
		};

30

libs/core/getDelimiter.js

		@@ -1,22 +0,22 @@
		module.exports=getDelimiter;
		var defaulDelimiters=[",","\|","\t",";",":"];
		module.exports = getDelimiter;
		var defaulDelimiters = [",", "\|", "\t", ";", ":"];
		function getDelimiter(rowStr,param) {
		var checker;
		if (param.delimiter==="auto"){
		checker=defaulDelimiters;
		}else if (param.delimiter instanceof Array){
		checker=param.delimiter;
		}else{
		if (param.delimiter === "auto"){
		checker = defaulDelimiters;
		} else if (param.delimiter instanceof Array) {
		checker = param.delimiter;
		} else {
		return param.delimiter;
		}
		var count=0;
		var rtn=",";
		checker.forEach(function(delim){
		var delimCount=rowStr.split(delim).length;
		if (delimCount>count){
		rtn=delim;
		count=delimCount;
		var count = 0;
		var rtn = ",";
		checker.forEach(function(delim) {
		var delimCount = rowStr.split(delim).length;
		if (delimCount > count) {
		rtn = delim;
		count = delimCount;
		}
		});
		return rtn;
		}
		}

24

libs/core/getEol.js

		//return eol from a data chunk.
		var eol=require("os").EOL;
		module.exports=function(data,param){
		var eol = require("os").EOL;
		module.exports = function(data, param) {
		if (!param.eol && data) {
		for (var i=0;i<data.length;i++){
		if (data[i]==="\r"){
		if (data[i+1] === "\n"){
		param.eol="\r\n";
		}else{
		param.eol="\r";
		for (var i = 0, len = data.length; i < len; i++) {
		if (data[i] === "\r") {
		if (data[i + 1] === "\n") {
		param.eol = "\r\n";
		} else {
		param.eol = "\r";
		}
		return param.eol;
		}else if (data[i]==="\n"){
		param.eol="\n";
		} else if (data[i] === "\n") {
		param.eol = "\n";
		return param.eol;
		}
		}
		param.eol=eol;
		param.eol = eol;
		}
		return param.eol;
		}
		};

11

libs/core/index.js

		@@ -1,9 +0,6 @@
		module.exports=constructor;
		module.exports = constructor;
		module.exports.Converter = require("./Converter.js");
		// module.exports.Parser = require("./parser.js");
		// module.exports.parserMgr = require("./parserMgr.js");


		function constructor(param,options){
		return new module.exports.Converter(param,options)
		}
		function constructor(param,options) {
		return new module.exports.Converter(param, options);
		}

142

libs/core/linesToJson.js

		var parserMgr = require("./parserMgr.js");
		var Parser = require("./parser");
		var CSVError = require("./CSVError");
		@@ -25,7 +24,7 @@ var numReg = /^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$/;
		}
		}
		};

		function justReturnRows(lines, params, idx) {
		var rtn = [];
		for (var i = 0; i < lines.length; i++) {
		for (var i = 0, len = lines.length; i < len; i++) {
		rtn.push({
		@@ -36,10 +35,10 @@ err: null,
		row: lines[i]
		})
		});
		}
		return rtn;
		}

		function processRows(csvRows, params, startIndex) {
		var count = csvRows.length;
		var res = [];
		for (var i = 0; i < csvRows.length; i++) {
		for (var i = 0, len = csvRows.length; i < len; i++) {
		var r = processRow(csvRows[i], params, startIndex++);
		@@ -52,20 +51,11 @@ if (r) {
		}
		function getConstParser(number, param) {
		var inst = new Parser("field" + number, /.*/, function (params) {
		var name = this.getName();
		params.resultRow[name] = params.item;
		}, true);
		inst.setParam(param);
		return inst;
		}

		function processRow(row, param, index) {
		var i, item, parser, head;
		var parseRules = param.parseRules;
		if (param.checkColumn && row.length != parseRules.length) {
		if (param.checkColumn && row.length !== parseRules.length) {
		return {
		err: CSVError.column_mismatched(index)
		}
		};
		}


		var headRow = param._headers;
		@@ -83,7 +73,9 @@ var resultRow = convertRowToJson(row, headRow, param);
		}

		function convertRowToJson(row, headRow, param) {
		var hasValue = false;
		var resultRow = {};
		for (i = 0; i < row.length; i++) {

		for (var i = 0, len = row.length; i < len; i++) {
		var convertFunc, head, item;
		item = row[i];
		@@ -96,40 +88,25 @@

		// parser = parseRules[i];
		// if (!parser) {
		// parser = parseRules[i] = getConstParser(i + 1, param);
		// }
		head = headRow[i];
		if (!head \|\| head === "") {
		head = headRow[i] = "field" + (i + 1);
		// parser.initHead(head);
		}
		var flag = getFlag(head, i, param)
		var flag = getFlag(head, i, param);
		if (flag === 'omit') {
		continue
		continue;
		}
		if (param.checkType) {
		convertFunc = checkType(item, head, i, param)
		item = convertFunc(item)
		convertFunc = checkType(item, head, i, param);
		item = convertFunc(item);
		}
		var title = getTitle(head, i, param)
		var title = getTitle(head, i, param);
		if (flag === 'flat' \|\| param.flatKeys) {
		resultRow[title] = item
		resultRow[title] = item;
		} else {
		setPath(resultRow, title, item)
		setPath(resultRow, title, item);
		}
		// _.set(resultRow,head,item)
		// parser.parse({
		// head: head,
		// item: item,
		// itemIndex: i,
		// rawRow: row,
		// resultRow: resultRow,
		// rowIndex: index,
		// config: param \|\| {}
		// });
		}
		if (hasValue) {
		return resultRow
		return resultRow;
		} else {
		return false
		return false;
		}
		@@ -139,32 +116,32 @@ }
		function setPath(json, path, value) {
		var _set = require('lodash/set')
		var pathArr = path.split('.')
		var _set = require('lodash/set');
		var pathArr = path.split('.');
		if (pathArr.length === 1) {
		json[path] = value;
		} else {
		_set(json, path, value)
		_set(json, path, value);
		}
		}

		function getFlag(head, i, param) {
		if (typeof param._headerFlag[i] === "string") {
		return param._headerFlag[i]
		return param._headerFlag[i];
		} else if (head.indexOf('omit') > -1) {
		return param._headerFlag[i] = 'omit';
		} else if (head.indexOf('flat') > -1) {
		return param._headerFlag[i] = 'flat';
		} else {
		if (head.indexOf('omit') > -1) {
		return param._headerFlag[i] = 'omit'
		} else if (head.indexOf('flat') > -1) {
		return param._headerFlag[i] = 'flat'
		} else {
		return param._headerFlag[i] = ''
		}
		return param._headerFlag[i] = '';
		}
		}

		function getTitle(head, i, param) {
		if (param._headerTitle[i]) {
		return param._headerTitle[i]
		} else {
		var flag = getFlag(head, i, param)
		var str = head.replace(flag, '')
		str = str.replace('string#!', '').replace('number#!', '')
		return param._headerTitle[i] = str
		return param._headerTitle[i];
		}

		var flag = getFlag(head, i, param);
		var str = head.replace(flag, '');
		str = str.replace('string#!', '').replace('number#!', '');
		return param._headerTitle[i] = str;
		}
		@@ -174,13 +151,11 @@
		if (param._headerType[headIdx]) {
		return param._headerType[headIdx]
		return param._headerType[headIdx];
		} else if (head.indexOf('number#!') > -1) {
		return param._headerType[headIdx] = numberType;
		} else if (head.indexOf('string#!') > -1) {
		return param._headerType[headIdx] = stringType;
		} else if (param.checkType) {
		return param._headerType[headIdx] = dynamicType;
		} else {
		if (head.indexOf('number#!') > -1) {
		return param._headerType[headIdx] = numberType
		} else if (head.indexOf('string#!') > -1) {
		return param._headerType[headIdx] = stringType
		} else if (param.checkType) {
		return param._headerType[headIdx] = dynamicType
		} else {
		return param._headerType[headIdx] = stringType
		}
		return param._headerType[headIdx] = stringType;
		}
		@@ -190,3 +165,3 @@ }
		function numberType(item) {
		var rtn = parseFloat(item)
		var rtn = parseFloat(item);
		if (isNaN(rtn)) {
		@@ -208,3 +183,3 @@ return item;
		if (numReg.test(trimed)) {
		return numberType(item)
		return numberType(item);
		} else if (trimed.length === 5 && trimed.toLowerCase() === "false" \|\| trimed.length === 4 && trimed.toLowerCase() === "true") {
		@@ -214,3 +189,2 @@ return booleanType(item);
		return jsonType(item);

		} else {
		@@ -237,23 +211,1 @@ return stringType(item);
		}
		// function dynamicType(item) {
		// var trimed = item.trim();
		// if (trimed === "") {
		// return trimed;
		// }
		// if (!isNaN(trimed)) {
		// return parseFloat(trimed);
		// } else if (trimed.length === 5 && trimed.toLowerCase() === "false") {
		// return false;
		// } else if (trimed.length === 4 && trimed.toLowerCase() === "true") {
		// return true;
		// } else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}" \|\| trimed[0] === "[" && trimed[trimed.length - 1] === "]") {
		// try {
		// return JSON.parse(trimed);
		// } catch (e) {
		// return item;
		// }
		// } else {
		// return item;

		// }
		// }

14

libs/core/parser.js

		@@ -51,13 +51,16 @@ var explicitTypes = ["number", "string"];
		return item;
		};

		}
		Parser.prototype.setParam = function(param) {
		this.param = param;
		}
		};

		Parser.prototype.test = function(str) {
		return this.regExp && this.regExp.test(str);
		};

		Parser.prototype.parse = function(params) {
		params.resultRow[params.head] = params.item;
		};

		Parser.prototype.getHeadStr = function() {
		@@ -75,5 +78,7 @@ if (this.headStr) {
		};

		Parser.prototype.getHead = function() {
		return this.head;
		};

		Parser.prototype.initHead = function(columnTitle) {
		@@ -98,4 +103,4 @@ this.head = columnTitle;
		}
		};

		}
		Parser.prototype.clone = function() {
		@@ -108,7 +113,8 @@ var obj = Object.create(this);
		return newParser;
		//return new Parser(this.name, this.regExp, this.parse, this.processSafe);
		};

		Parser.prototype.getName = function() {
		return this.name;
		};

		module.exports = Parser;

22

libs/core/parserMgr.js

		@@ -11,9 +11,10 @@ //implementation
		}
		function getParser (columnTitle, param) {

		function getParser(columnTitle, param) {
		var inst, parser;
		function getParserByName (parserName) {
		function getParserByName(parserName) {
		var parser;
		registeredParsers.forEach(function(p){
		if (p.getName() === parserName){
		parser=p;
		parser = p;
		}
		@@ -28,4 +29,4 @@ });
		columnTitle = columnTitle ? columnTitle : '';
		registeredParsers.forEach(function(p){
		if (p.test(columnTitle)){
		registeredParsers.forEach(function(p) {
		if (p.test(columnTitle)) {
		parser=p;
		@@ -44,11 +45,13 @@ }
		}
		function addParser (name, regExp, parseFunc) {

		function addParser(name, regExp, parseFunc) {
		var parser = new Parser(name, regExp, parseFunc,false); //TODO remove new
		registerParser(parser);
		}
		function addSafeParser(parserPath){

		function addSafeParser(parserPath) {
		//TODO impl
		}

		function initParsers (row, param) {
		function initParsers(row, param) {
		var parsers = [];
		@@ -60,5 +63,6 @@ row.forEach(function (columnTitle) {
		}

		defaultParser.forEach(function (parserCfg){
		//TODO refactor this
		addParser(parserCfg.name, parserCfg.regExp, parserCfg.parserFunc,parserCfg.processSafe);
		addParser(parserCfg.name, parserCfg.regExp, parserCfg.parserFunc, parserCfg.processSafe);
		});
		@@ -65,0 +69,0 @@

34

libs/core/rowSplit.js

		var getDelimiter = require("./getDelimiter");
		var filterRow=require("./filterRow");
		/**
		* Convert a line of string to csv columns according to its delimiter
		* the param._header may not be ready when this is called.
		* @param {[type]} rowStr [description]
		@@ -26,3 +28,3 @@ * @param {[type]} param [Converter param]
		var quoteBuff = '';
		for (var i = 0; i < rowArr.length; i++) {
		for (var i = 0, rowLen = rowArr.length; i < rowLen; i++) {
		var e = rowArr[i];
		@@ -38,3 +40,3 @@ if (!inquote && trim) {
		e = e.substring(0, e.length - 1);
		e = _escapeQuote(e, quote, escape);;
		e = _escapeQuote(e, quote, escape);
		row.push(e);
		@@ -85,22 +87,6 @@ continue;

		}
		function filterRow(row, param) {
		if (param.ignoreColumns instanceof Array && param.ignoreColumns.length > 0) {
		for (var irow = 0; irow < param.ignoreColumns.length; irow++) {
		if (param.ignoreColumns[irow] >= 0) {
		row.splice(param.ignoreColumns[irow], 1);
		}
		}
		}
		if (param.includeColumns instanceof Array && param.includeColumns.length > 0) {
		var cleanRowArr = [];
		for (var irow = 0; irow < param.includeColumns.length; irow++) {
		if (param.includeColumns[irow] >= 0) {
		cleanRowArr.push(row[param.includeColumns[irow]]);
		}
		}
		row = cleanRowArr;
		}
		return row;
		}
		};



		function isQuoteOpen(str, param) {
		@@ -124,2 +110,3 @@ var quote = param.quote;
		}

		function twoDoubleQuote(str, quote) {
		@@ -133,3 +120,4 @@ var twoQuote = quote + quote;
		}
		var cachedRegExp = {}

		var cachedRegExp = {};
		function _escapeQuote(segment, quote, escape) {
		@@ -136,0 +124,0 @@

90

libs/core/worker.js

		@@ -1,13 +0,13 @@
		var param=null;
		var fileLine=require("./fileline");
		var csvline=require("./csvline");
		var linesToJson=require("./linesToJson");
		var CSVError=require('./CSVError')
		var eom="\x03";
		var eom1="\x0e"
		var eom2="\x0f"
		var param = null;
		var fileLine = require("./fileline");
		var csvline = require("./csvline");
		var linesToJson = require("./linesToJson");
		var CSVError = require('./CSVError');
		var eom = "\x03";
		var eom1 = "\x0e";
		var eom2 = "\x0f";
		/**
		* message is like :
		* 0{"a":"b"}
		* 13345\|a,b,c
		* 13345\|a,b,c
		* <cmd><data>
		@@ -17,18 +17,18 @@ * <cmd> is 0-9
		var buffer="";
		process.stdin.on("data",function(d){
		// console.error(typeof d)
		var str=d.toString("utf8");
		var all=buffer+str;
		var cmdArr=all.split(eom)
		while (cmdArr.length >1){
		process.stdin.on("data", function(d) {
		var str = d.toString("utf8");
		var all = buffer + str;
		var cmdArr = all.split(eom);
		while (cmdArr.length > 1) {
		processMsg(cmdArr.shift());
		}
		buffer=cmdArr[0];
		buffer = cmdArr[0];
		});
		process.on("message",processMsg)
		function processMsg(msg){
		if (msg){
		var cmd=msg[0];
		var data=msg.substr(1);
		switch (cmd){

		process.on("message", processMsg);
		function processMsg(msg) {
		if (msg) {
		var cmd = msg[0];
		var data = msg.substr(1);
		switch (cmd) {
		case "0":
		@@ -41,3 +41,3 @@ initParams(data);
		default:
		console.error("Unknown command: "+msg);
		console.error("Unknown command: " + msg);
		}
		@@ -47,11 +47,12 @@ }

		function initParams(data){
		param=JSON.parse(data);
		function initParams(data) {
		param = JSON.parse(data);
		}

		/**
		* e.g.
		* 1023\|a,b,c,d\ne,f,g,h\n
		* 1023\|a,b,c,d\ne,f,g,h\n
		* <start line number>\|<raw csv data>
		*/
		function processData(data){
		function processData(data) {
		if (!param){
		@@ -61,14 +62,14 @@ console.error("Parameter not initialised when processing data.");
		}
		var sepIdx=data.indexOf("\|");
		var startIdx=parseInt(data.substr(0,sepIdx));
		var csvData=data.substr(sepIdx+1);
		var lines=fileLine(csvData,param);//convert to file lines.
		var sepIdx = data.indexOf("\|");
		var startIdx = parseInt(data.substr(0, sepIdx));
		var csvData = data.substr(sepIdx + 1);
		var lines = fileLine(csvData,param); //convert to file lines.
		// process.send("0"+lines.lines.length+"\|"+lines.partial);
		var csvLines=csvline(lines.lines,param);
		var res=linesToJson(csvLines.lines,param,startIdx);
		if (csvLines.partial){
		var lastIdx=res.length>0? res[res.length-1].index+1 : startIdx
		var csvLines = csvline(lines.lines,param);
		var res = linesToJson(csvLines.lines,param,startIdx);
		if (csvLines.partial) {
		var lastIdx = res.length > 0 ? res[res.length - 1].index + 1 : startIdx;
		res.push({
		err:CSVError.unclosed_quote(lastIdx,csvLines.partial)
		})
		err:CSVError.unclosed_quote(lastIdx, csvLines.partial)
		});
		}
		@@ -78,12 +79,11 @@ // console.error(res)
		var str="1";
		res.forEach(function(item){
		var errStr=item.err?item.err.toString():"";
		str+=item.index+eom2+JSON.stringify(item.row)+eom2+errStr+eom2+JSON.stringify(item.json)+eom1
		})
		res.forEach(function(item) {
		var errStr = item.err ? item.err.toString() : "";
		str += item.index + eom2 + JSON.stringify(item.row) + eom2 + errStr + eom2 + JSON.stringify(item.json) + eom1;
		});
		sendData(str);
		}

		function sendData(str){
		process.stdout.write(str+eom);
		// process.send(str)
		}
		function sendData(str) {
		process.stdout.write(str + eom);
		}

211

libs/core/workerMgr.js

		@@ -1,35 +0,34 @@
		module.exports=workerMgr;
		var spawn=require("child_process").spawn;
		var eom="\x03"
		var eom1="\x0e"
		var eom2="\x0f"
		var CSVError=require('./CSVError')
		function workerMgr(){
		module.exports = workerMgr;
		var spawn = require("child_process").spawn;
		var eom = "\x03";
		var eom1 = "\x0e";
		var eom2 = "\x0f";
		var CSVError = require('./CSVError');
		function workerMgr() {

		var exports={
		initWorker:initWorker,
		sendWorker:sendWorker,
		setParams:setParams,
		drain:function(){},
		isRunning:isRunning,
		destroyWorker:destroyWorker

		}
		var workers=[];
		var running=0;
		var waiting=null;
		function initWorker(num,params){
		workers=[];
		running=0;
		waiting=null;
		for (var i=0;i<num;i++){
		var exports = {
		initWorker: initWorker,
		sendWorker: sendWorker,
		setParams: setParams,
		drain: function(){},
		isRunning: isRunning,
		destroyWorker: destroyWorker
		};
		var workers = [];
		var running = 0;
		var waiting = null;
		function initWorker(num, params) {
		workers = [];
		running = 0;
		waiting = null;
		for (var i = 0; i < num; i++) {
		workers.push(new Worker(params));
		}


		}
		function isRunning(){
		return running>0;
		function isRunning() {
		return running > 0;
		}
		function destroyWorker(){
		workers.forEach(function(w){
		function destroyWorker() {
		workers.forEach(function(w) {
		w.destroy();
		@@ -39,7 +38,7 @@ });

		function sendWorker(data,startIdx,transformCb,cbResult){
		if (workers.length>0){
		var worker=workers.shift();
		function sendWorker(data, startIdx, transformCb, cbResult) {
		if (workers.length > 0) {
		var worker = workers.shift();
		running++;
		worker.parse(data,startIdx,function(result){
		worker.parse(data, startIdx, function(result) {
		// var arr=JSON.parse(result);
		@@ -49,20 +48,20 @@ // arr.forEach(function(item){
		// })
		workers.push(worker)
		cbResult(result,startIdx);
		workers.push(worker);
		cbResult(result, startIdx);
		running--;
		if (waiting === null && running===0){
		if (waiting === null && running === 0) {
		exports.drain();
		}else if (waiting){
		sendWorker.apply(this,waiting)
		waiting=null;
		} else if (waiting) {
		sendWorker.apply(this, waiting);
		waiting = null;
		}
		});
		process.nextTick(transformCb)
		}else{
		waiting=[data,startIdx,transformCb,cbResult];
		process.nextTick(transformCb);
		} else {
		waiting = [data, startIdx, transformCb, cbResult];
		}
		}

		function setParams(params){
		workers.forEach(function(w){
		function setParams(params) {
		workers.forEach(function(w) {
		w.setParams(params);
		@@ -74,29 +73,30 @@ });

		function Worker(params){
		this.cp=spawn(process.execPath,[__dirname+"/worker.js"],{
		env:{
		function Worker(params) {
		this.cp = spawn(process.execPath, [__dirname + "/worker.js"], {
		env: {
		child:true
		},
		stdio:['pipe','pipe',2,'ipc']
		stdio:['pipe', 'pipe', 2, 'ipc']
		// stdio:[0,1,2,'ipc']
		});
		this.setParams(params);
		this.cp.on("message",this.onChildMsg.bind(this));
		this.buffer="";
		var self=this;
		this.cp.stdout.on("data",function(d){
		var str=d.toString("utf8");
		var all=self.buffer+str;
		var cmdArr=all.split(eom)
		while (cmdArr.length >1){
		self.onChildMsg(cmdArr.shift());
		}
		self.buffer=cmdArr[0];
		})
		this.cp.on("message", this.onChildMsg.bind(this));
		this.buffer = "";
		var self = this;
		this.cp.stdout.on("data", function(d) {
		var str = d.toString("utf8");
		var all = self.buffer + str;
		var cmdArr = all.split(eom);
		while (cmdArr.length > 1) {
		self.onChildMsg(cmdArr.shift());
		}
		self.buffer = cmdArr[0];
		});
		}

		Worker.prototype.setParams=function(params){
		var msg="0"+JSON.stringify(params);
		Worker.prototype.setParams = function(params) {
		var msg = "0" + JSON.stringify(params);
		this.sendMsg(msg);
		}
		};

		/**
		@@ -107,46 +107,49 @@ * msg is like:
		*/
		Worker.prototype.onChildMsg=function(msg){
		if (msg){
		var cmd=msg[0];
		var data=msg.substr(1);
		switch (cmd){
		case "0": //total line number of current chunk
		if (this.cbLine){
		var sp=data.split("\|");
		var len=parseInt(sp[0]);
		var partial=sp[1];
		this.cbLine(len,partial);
		}
		break;
		case "1": // json array of current chunk
		if (this.cbResult){
		var rows=data.split(eom1);
		rows.pop();
		var res=[];
		rows.forEach(function(row){
		var sp=row.split(eom2);
		res.push({
		index:sp[0],
		row:sp[1],
		err:sp[2]?CSVError.fromArray(JSON.parse(sp[2])):null,
		json:sp[3]
		})
		})
		this.cbResult(res);
		}
		break;
		Worker.prototype.onChildMsg = function(msg) {
		if (msg) {
		var cmd = msg[0];
		var data = msg.substr(1);
		switch (cmd) {
		case "0": //total line number of current chunk
		if (this.cbLine) {
		var sp = data.split("\|");
		var len = parseInt(sp[0]);
		var partial = sp[1];
		this.cbLine(len, partial);
		}
		break;
		case "1": // json array of current chunk
		if (this.cbResult) {
		var rows = data.split(eom1);
		rows.pop();
		var res = [];
		rows.forEach(function(row) {
		var sp = row.split(eom2);
		res.push({
		index: sp[0],
		row: sp[1],
		err: sp[2] ? CSVError.fromArray(JSON.parse(sp[2])) : null,
		json: sp[3]
		});
		});
		this.cbResult(res);
		}
		break;
		}
		}
		}
		}
		Worker.prototype.parse=function(data,startIdx,cbResult){
		this.cbResult=cbResult;
		var msg="1"+startIdx+"\|"+data;
		};

		Worker.prototype.parse = function(data, startIdx, cbResult) {
		this.cbResult = cbResult;
		var msg = "1" + startIdx + "\|" + data;
		this.sendMsg(msg);
		}
		Worker.prototype.destroy=function(){
		};

		Worker.prototype.destroy = function() {
		this.cp.kill();
		}
		Worker.prototype.sendMsg=function(msg){
		this.cp.stdin.write(msg+eom,"utf8")
		};

		Worker.prototype.sendMsg = function(msg) {
		this.cp.stdin.write(msg + eom, "utf8");
		// this.cp.send(msg)
		}
		};

6

package.json

		@@ -5,3 +5,3 @@ {
		"author": "Keyang Xiang <keyang.xiang@gmail.com>",
		"homepage": "http://keyangxiang.com/blog/csv2json/",
		"homepage": "https://github.com/Keyang/node-csvtojson",
		"bugs": "https://github.com/Keyang/node-csvtojson/issues",
		@@ -194,3 +194,3 @@ "repository": {
		],
		"version": "1.1.4",
		"version": "1.1.5",
		"keywords": [
		@@ -238,2 +238,2 @@ "csv",
		}
		}
		}

32

readme.md

		@@ -97,2 +97,4 @@ [![Build Status](https://travis-ci.org/Keyang/node-csvtojson.svg?branch=master)](https://travis-ci.org/Keyang/node-csvtojson)

		Note that `.fromFile(filePath[ ,cb ,options])` takes an `options` parameter which will be passed to `fs.createReadStream()`. See [here](https://nodejs.org/dist/latest-v6.x/docs/api/fs.html#fs_fs_createreadstream_path_options) for docs.

		### From CSV Stream
		@@ -115,2 +117,19 @@

		### From CSV Url

		```js
		const request=require('request')
		const csv=require('csvtojson')

		csv()
		.fromStream(request.get('http://mywebsite.com/mycsvfile.csv'))
		.on('csv',(csvRow)=>{
		// csvRow is an array
		})
		.on('done',(error)=>{

		})

		```

		### Convert to CSV row arrays with csv header row
		@@ -247,5 +266,5 @@
		* trim: Indicate if parser trim off spaces surrounding column content. e.g. " content " will be trimmed to "content". Default: true
		* checkType: This parameter turns on and off whether check field type. default is true.
		* checkType: This parameter turns on and off whether check field type. Default is false.
		* toArrayString: Stringify the stream output to JSON array. This is useful when pipe output to a file which expects stringified JSON array. default is false and only stringified JSON (without []) will be pushed to downstream.
		* ignoreEmpty: Ignore the empty value in CSV columns. If a column value is not giving, set this to true to skip them. Defalut: false.
		* ignoreEmpty: Ignore the empty value in CSV columns. If a column value is not given, set this to true to skip them. Default: false.
		* workerNum: Number of worker processes. The worker process will use multi-cores to help process CSV data. Set to number of Core to improve the performance of processing large csv file. Keep 1 for small csv files. Default 1.
		@@ -259,4 +278,4 @@ * noheader:Indicating csv data has no header row and first row is data row. Default is false. See [header row](#header-row)
		* escape: escape character used in quoted column. Default is double quote (") according to RFC4108. Change to back slash (\\) or other chars for your own case.
		* includeColumns: This parameter instructs the parser to include only those columns as specified by an array of column indexes. Example: [0,2,3] will parse and include only columns 0, 2, and 3 in the JSON output.
		* ignoreColumns: This parameter instructs the parser to ignore columns as specified by an array of column indexes. Example: [1,3,5] will ignore columns 1, 3, and 5 and will not return them in the JSON output.
		* includeColumns: This parameter instructs the parser to include only those columns as specified by an array of column indexes or header names. Example: [0,2,3,"name"] will parse and include only columns 0, 2, 3, and column with header "name" in the JSON output.
		* ignoreColumns: This parameter instructs the parser to ignore columns as specified by an array of column indexes or header names. Example: [1,3,5,"title","age"] will ignore columns 1, 3, 5, title column and age column and will not return them in the JSON output.

		@@ -575,3 +594,8 @@ All parameters can be used in Command Line tool.

		## 1.1.5

		* `ignoreColumns` and `includeColumns` now allow put in header names and indecies.
		* only include `child_process` when multi worker is needed.
		* allow `fs.createReadStream` options being passed in through `fromFile` function

		## 1.1.4
		@@ -578,0 +602,0 @@

12

test/errorHandle.js

		@@ -12,3 +12,3 @@ var Converter = require("../libs/core/Converter.js");
		done();
		})
		});
		rs.pipe(conv);
		@@ -60,3 +60,3 @@ });

		it ("should handle column number mismatched error",function(done){
		it ("should handle column number mismatched error", function(done) {
		var rs = fs.createReadStream(__dirname + "/data/dataWithMismatchedColumn");
		@@ -75,9 +75,9 @@ var conv = new Converter({
		});
		conv.on("json",function(){})
		conv.on('done',function(){
		assert(tested)
		conv.on("json",function() {});
		conv.on('done',function() {
		assert(tested);
		done();
		})
		});
		rs.pipe(conv);
		});
		});

33

test/testCSVConverter.js

		var Converter = require("../libs/core/Converter.js");
		var csv = require('../')
		var csv = require('../');
		var assert = require("assert");
		@@ -62,2 +62,3 @@ var fs = require("fs");
		});

		it("should handle comma in column which is surrounded by qoutes", function (done) {
		@@ -102,2 +103,3 @@ var testData = __dirname + "/data/dataWithComma";
		});

		it("should be able to convert csv string directly", function (done) {
		@@ -116,2 +118,3 @@ var testData = __dirname + "/data/testData";
		});

		it("should be able to convert csv string with error", function (done) {
		@@ -127,2 +130,3 @@ var testData = __dirname + "/data/dataWithUnclosedQuotes";
		});

		it("should be able to convert csv string without callback provided", function (done) {
		@@ -139,2 +143,3 @@ var testData = __dirname + "/data/testData";
		});

		it("should be able to handle columns with double quotes", function (done) {
		@@ -167,2 +172,3 @@ var testData = __dirname + "/data/dataWithQoutes";
		});

		it("should handle empty csv file", function (done) {
		@@ -178,2 +184,3 @@ var testData = __dirname + "/data/emptyFile";
		});

		it("should parse large csv file", function (done) {
		@@ -196,2 +203,3 @@ var testData = __dirname + "/data/large-csv-sample.csv";
		});

		it("should parse data and covert to specific types", function (done) {
		@@ -220,2 +228,3 @@ var testData = __dirname + "/data/dataWithType";
		});

		it("should turn off field type check", function (done) {
		@@ -246,2 +255,3 @@ var testData = __dirname + "/data/dataWithType";
		});

		it("should emit data event correctly", function (done) {
		@@ -263,2 +273,3 @@ var testData = __dirname + "/data/large-csv-sample.csv";
		});

		it("should process column with linebreaks", function (done) {
		@@ -278,2 +289,3 @@ var testData = __dirname + "/data/lineBreak";
		});

		it("should stream to array string", function (done) {
		@@ -283,4 +295,4 @@ var testData = __dirname + "/data/dataDiffDelimiter";
		var data = "";
		var st = rs.pipe(new Converter({
		constructResult: false, delimiter: ';', trim: true, toArrayString: true, checkType:true
		var st = rs.pipe(new Converter({
		constructResult: false, delimiter: ';', trim: true, toArrayString: true, checkType:true
		}));
		@@ -297,4 +309,4 @@ st.on("data", function (d) {
		});
		});

		});
		it("be able to ignore empty columns", function (done) {
		@@ -315,4 +327,4 @@ var testData = __dirname + "/data/dataIgnoreEmpty";
		});
		});

		});
		it("should allow no header", function (done) {
		@@ -329,3 +341,4 @@ var testData = __dirname + "/data/noheadercsv";
		});
		})
		});

		it("should allow customised header", function (done) {
		@@ -346,3 +359,4 @@ var testData = __dirname + "/data/noheadercsv";
		});
		})
		});

		it("should allow customised header to override existing header", function (done) {
		@@ -361,3 +375,4 @@ var testData = __dirname + "/data/complexJSONCSV";
		});
		})
		});

		it("should handle when there is an empty string", function (done) {
		@@ -380,3 +395,3 @@ var testData = __dirname + "/data/dataWithEmptyString";
		});
		})
		});
		});

230

test/testCSVConverter2.js

		@@ -19,3 +19,2 @@ var Converter = require("../libs/core/Converter.js");
		it("should set eol ", function (done) {

		var rs = fs.createReadStream(__dirname + "/data/large-csv-sample.csv");
		@@ -28,4 +27,7 @@ var conv = new Converter({
		var count = 0;
		conv.on("record_parsed", function (rec) {
		conv.on("record_parsed", function (resultJson, row, index) {
		count++;
		assert(resultJson);
		assert(row.length === 2);
		assert(index >= 0);
		});
		@@ -35,3 +37,4 @@ conv.on("error", function () {
		});
		conv.on("end_parsed", function () {
		conv.on("end_parsed", function (result) {
		assert(result);
		assert(count === 5290);
		@@ -42,2 +45,3 @@ done();
		});

		it("should convert tsv String", function (done) {
		@@ -52,2 +56,3 @@ var tsv = __dirname + "/data/dataTsv";
		});

		it("should allow customised header with nohead csv string.", function (done) {
		@@ -66,2 +71,3 @@ var testData = __dirname + "/data/noheadercsv";
		});

		it("should parse fromFile", function (done) {
		@@ -78,2 +84,3 @@ var csvFile = __dirname + "/data/large-csv-sample.csv";
		});

		it("should fromFile should emit error", function (done) {
		@@ -89,2 +96,3 @@ var csvFile = __dirname + "/data/dataWithUnclosedQuotes";
		});

		it("should parse no header with dynamic column number", function (done) {
		@@ -102,2 +110,3 @@ var testData = __dirname + "/data/noheaderWithVaryColumnNum";
		});

		it("should parse tabsv data with dynamic columns", function (done) {
		@@ -114,2 +123,3 @@ var testData = __dirname + "/data/tabsv";
		});

		it("should use first line break as eol", function (done) {
		@@ -124,3 +134,4 @@ var testData = __dirname + "/data/testEol";
		});
		})
		});

		it("should use sync transform", function (done) {
		@@ -132,7 +143,10 @@ var testData = __dirname + "/data/complexJSONCSV";
		json.rowNum = index;
		}
		conv.on("record_parsed", function (j) {
		assert(j.rowNum >= 0);
		};
		conv.on("record_parsed", function (resultJson, row, index) {
		assert(resultJson.rowNum >= 0);
		assert(row.length === 9);
		assert(index >= 0);
		});
		conv.on("end_parsed", function (res) {
		assert(res.length === 2);
		assert(res[0].rowNum === 0);
		@@ -144,2 +158,3 @@ assert(res[1].rowNum === 1);
		});

		it("should detect delimiter ", function (done) {
		@@ -155,4 +170,4 @@ var testData = __dirname + "/data/dataWithAutoDelimiter";
		rs.pipe(conv);
		});

		});
		it("should stripe out whitespaces if trim is true", function (done) {
		@@ -163,3 +178,2 @@ var testData = __dirname + "/data/dataWithWhiteSpace";
		conv.on("end_parsed", function (res) {
		// console.log(res);
		assert.equal(res[0]["Column 1"], "Column1Row1");
		@@ -170,4 +184,4 @@ assert.equal(res[0]["Column 2"], "Column2Row1");
		rs.pipe(conv);
		});

		});
		it("should convert triple quotes correctly", function (done) {
		@@ -184,4 +198,4 @@ var testData = __dirname + "/data/dataWithTripleQoutes";
		rs.pipe(conv);
		});

		});
		// it ("should auto flat header if header is not valid nested json keys",function(done){
		@@ -199,2 +213,3 @@ // var testData = __dirname + "/data/invalidHeader";
		// })

		it("should pre process raw data in the line", function (done) {
		@@ -207,3 +222,3 @@ var testData = __dirname + "/data/quoteTolerant";
		cb(d);
		})
		});
		conv.on("end_parsed", function (res) {
		@@ -214,3 +229,4 @@ assert(res[0].Description.indexOf('THIN') > -1);
		rs.pipe(conv);
		})
		});

		it("should pre process by line in the line", function (done) {
		@@ -224,4 +240,4 @@ var testData = __dirname + "/data/quoteTolerant";
		}
		return line
		})
		return line;
		});

		@@ -233,3 +249,4 @@ conv.on("end_parsed", function (res) {
		rs.pipe(conv);
		})
		});

		it("should support object mode", function (done) {
		@@ -248,5 +265,6 @@ var testData = __dirname + "/data/complexJSONCSV";
		done();
		})
		});
		rs.pipe(conv);
		})
		});

		it("should get delimiter automatically if there is no header", function (done) {
		@@ -269,2 +287,3 @@ var test_converter = new Converter({
		});

		it("should process escape chars", function (done) {
		@@ -285,2 +304,3 @@ var test_converter = new Converter({
		});

		it("should output ndjson format", function (done) {
		@@ -290,6 +310,7 @@ var conv = new Converter();
		d = d.toString();
		assert.equal(d[d.length - 1], "\n")
		assert.equal(d[d.length - 1], "\n");
		})
		.on("end", done)
		})
		.on("end", done);
		});

		it("should parse from stream", function (done) {
		@@ -303,30 +324,35 @@ var testData = __dirname + "/data/complexJSONCSV";
		done();
		})
		})
		});
		});

		it("should emit json and csv and finish event", function (done) {
		var testData = __dirname + "/data/complexJSONCSV";
		var rs = fs.createReadStream(testData);
		var numofrow = 0;
		var numofjson = 0;
		var numOfRow = 0;
		var numOfJson = 0;
		csv()
		.fromStream(rs)
		.on('csv', function (row) {
		numofrow++;
		.on('csv', function (row, idx) {
		numOfRow++;
		assert(row);
		assert(idx >= 0);
		})
		.on("json", function (res) {
		numofjson++;
		assert.equal(typeof res, "object")
		.on("json", function (res, idx) {
		numOfJson++;
		assert.equal(typeof res, "object");
		assert(idx >= 0);
		})
		.on("done", function (error) {
		assert(!error)
		assert.equal(numofjson, numofrow)
		assert(numofrow != 0)
		assert(!error);
		assert.equal(numOfJson, numOfRow);
		assert(numOfRow !== 0);
		done();
		})
		})
		});
		});

		it("should transform with transf function", function (done) {
		var testData = __dirname + "/data/complexJSONCSV";
		var rs = fs.createReadStream(testData);
		var numofrow = 0;
		var numofjson = 0;
		var numOfRow = 0;
		var numOfJson = 0;
		csv()
		@@ -336,17 +362,23 @@ .fromStream(rs)
		json.a = "test";
		assert(row);
		assert(idx >= 0);
		})
		.on('csv', function (row) {
		numofrow++;
		.on('csv', function (row, idx) {
		numOfRow++;
		assert(row);
		assert(idx >= 0);
		})
		.on("json", function (res) {
		numofjson++;
		assert.equal(typeof res, "object")
		assert.equal(res.a, "test")
		.on("json", function (res, idx) {
		numOfJson++;
		assert.equal(typeof res, "object");
		assert.equal(res.a, "test");
		assert(idx >= 0);
		})
		.on("end", function () {
		assert.equal(numofjson, numofrow)
		assert(numofrow != 0)
		assert.equal(numOfJson, numOfRow);
		assert(numOfRow !== 0);
		done();
		})
		})
		});
		});

		it("should parse a complex JSON", function (done) {
		@@ -372,3 +404,4 @@ var converter = new Converter({checkType:true});
		r.pipe(converter);
		})
		});

		it("should allow flatKey to change parse behaviour", function (done) {
		@@ -379,33 +412,44 @@ var conv = new Converter({
		conv.fromString("a.b,b.d,c.a\n1,2,3\n4,5,6").on("json", function (d) {
		assert(d["a.b"])
		assert(d["b.d"])
		assert(d["c.a"])
		assert(d["a.b"]);
		assert(d["b.d"]);
		assert(d["c.a"]);
		})
		.on("end", done)
		})
		it("should process long header", function (done) {
		.on("end", done);
		});

		it("should process long header", function (done) {
		var testData = __dirname + "/data/longHeader";
		var rs = fs.createReadStream(testData,{highWaterMark:100});
		var numofrow = 0;
		var numofjson = 0;
		var rs = fs.createReadStream(testData,{highWaterMark: 100});
		var numOfRow = 0;
		var numOfJson = 0;
		csv({},{highWaterMark:100})
		.fromStream(rs)
		.on("json", function (res) {
		assert.equal(res.Date,'8/26/16')
		.on('csv', function (row, idx) {
		numOfRow++;
		assert(idx >= 0);
		})
		.on("json", function (res, idx) {
		numOfJson++;
		assert.equal(res.Date, '8/26/16');
		assert(idx >= 0);
		})
		.on("end", function () {
		assert.equal(numOfJson, numOfRow);
		assert(numOfJson === 1);
		done();
		})
		})
		it("should parse #139",function(done){
		var rs=fs.createReadStream(__dirname+"/data/data#139")
		});
		});

		it("should parse #139", function(done) {
		var rs = fs.createReadStream(__dirname + "/data/data#139");
		csv()
		.fromStream(rs)
		.on("end_parsed",function(res){
		assert.equal(res[1].field3,"9001009395 9001009990")
		done();
		})
		})
		it("should ignore column",function(done){
		var rs=fs.createReadStream(__dirname+"/data/dataWithQoutes")
		.fromStream(rs)
		.on("end_parsed", function(res) {
		assert.equal(res[1].field3, "9001009395 9001009990");
		done();
		});
		});

		it("should ignore column", function(done) {
		var rs = fs.createReadStream(__dirname + "/data/dataWithQoutes");
		csv({
		@@ -415,17 +459,19 @@ ignoreColumns:[0]
		.fromStream(rs)
		.on("csv",function(row,idx){
		.on("csv", function(row, idx) {
		assert(idx >= 0);
		if (idx ===1){
		assert.equal(row[0],"n")
		assert.equal(row[0],"n");
		}
		})
		.on("json",function(j,idx){
		assert(!j.TIMESTAMP)
		.on("json", function(j, idx) {
		assert(!j.TIMESTAMP);
		assert(idx >= 0);
		})
		.on("end",function(){
		.on("end", function() {
		done();
		})
		})

		it("should include column",function(done){
		var rs=fs.createReadStream(__dirname+"/data/dataWithQoutes")
		});
		});

		it("should include column",function(done) {
		var rs = fs.createReadStream(__dirname + "/data/dataWithQoutes");
		csv({
		@@ -435,6 +481,8 @@ includeColumns:[0]
		.fromStream(rs)
		.on("csv",function(row,idx){
		.on("csv", function(row, idx) {
		assert(idx >= 0);
		assert.equal(row.length, 1);
		})
		.on("json",function(j,idx){
		.on("json", function(j, idx) {
		assert(idx >= 0);
		if (idx === 1){
		@@ -444,6 +492,18 @@ assert.equal(j.TIMESTAMP, "abc, def, ccc");
		})
		.on("end",function(){
		.on("end", function() {
		done();
		})
		});
		});
		it ("should only call done once",function(done){
		var counter=0;
		csv()
		.fromString('"a","b", "c""')
		.on('done',function(){
		counter++;
		});
		setTimeout(function(){
		assert.equal(counter,1);
		done();
		},100);
		})
		});

56

test/testCSVLine.js

		@@ -1,18 +0,19 @@
		var assert=require("assert");
		var func=require("../libs/core/csvline");
		var defParam=require("../libs/core/defParam");
		var assert = require("assert");
		var func = require("../libs/core/csvline");
		var defParam = require("../libs/core/defParam");

		describe("CSVLine function",function(){
		it ("should convert lines to csv lines",function(){
		var lines=[
		describe("CSVLine function", function() {
		it ("should convert lines to csv lines", function() {
		var lines = [
		"a,b,c,d",
		"hello,world,csvtojson,abc",
		"1,2,3,4"
		]
		var res=func(lines,defParam({}));
		assert.equal(res.lines.length,3);
		assert.equal(res.partial,"");
		})
		it ("should process line breaks",function(){
		var lines=[
		];
		var res = func(lines, defParam({}));
		assert.equal(res.lines.length, 3);
		assert.equal(res.partial, "");
		});

		it ("should process line breaks", function() {
		var lines = [
		"a,b,c",
		@@ -23,11 +24,12 @@ '15",hello,"ab',
		];
		var res=func(lines,defParam({}));
		assert.equal(res.lines.length,3);
		assert.equal(res.lines[1][0],"15\"");
		assert.equal(res.lines[1][2],"ab\ncde");
		assert.equal(res.lines[2][0],"b\"b");
		assert.equal(res.partial,"");
		})
		it ("should return partial if line not closed",function(){
		var lines=[
		var res=func(lines, defParam({}));
		assert.equal(res.lines.length, 3);
		assert.equal(res.lines[1][0], "15\"");
		assert.equal(res.lines[1][2], "ab\ncde");
		assert.equal(res.lines[2][0], "b\"b");
		assert.equal(res.partial, "");
		});

		it ("should return partial if line not closed", function() {
		var lines = [
		"a,b,c",
		@@ -37,6 +39,6 @@ '15",hello,"ab',
		];
		var res=func(lines,defParam({}));
		assert.equal(res.lines.length,1);
		assert.equal(res.partial,"15\",hello,\"ab\nd,e,f\n");
		})
		})
		var res = func(lines, defParam({}));
		assert.equal(res.lines.length, 1);
		assert.equal(res.partial, "15\",hello,\"ab\nd,e,f\n");
		});
		});

18

test/testDataToCSV.js

		@@ -1,10 +0,10 @@
		var func=require("../libs/core/dataToCSVLine");
		var defParam=require("../libs/core/defParam");
		var assert=require("assert");
		describe("dataToCSVLine function",function(){
		it ("parse data correctly",function(){
		var data="fieldA.title, fieldA.children[0].name, fieldA.children[0].id,fieldA.children[1].name, fieldA.children[1].employee[].name,fieldA.children[1].employee[].name, fieldA.address[],fieldA.address[], description\n";
		var res=func(data,defParam({}));
		var func = require("../libs/core/dataToCSVLine");
		var defParam = require("../libs/core/defParam");
		var assert = require("assert");
		describe("dataToCSVLine function", function() {
		it ("parse data correctly", function() {
		var data = "fieldA.title, fieldA.children[0].name, fieldA.children[0].id,fieldA.children[1].name, fieldA.children[1].employee[].name,fieldA.children[1].employee[].name, fieldA.address[],fieldA.address[], description\n";
		var res = func(data,defParam({}));
		// console.log(res);
		})
		})
		});
		});

27

test/testDelimiter.js

		@@ -6,26 +6,25 @@ var getDelimiter = require("../libs/core/getDelimiter.js");
		it("should return the explicitly specified delimiter", function() {
		delimiter = ';'
		rowStr = 'a;b;c';
		returnedDelimiter = getDelimiter(rowStr, {delimiter:";"});
		assert.equal(returnedDelimiter , delimiter);
		var delimiter = ";";
		var rowStr = "a;b;c";
		var returnedDelimiter = getDelimiter(rowStr, {delimiter:";"});
		assert.equal(returnedDelimiter, delimiter);
		});

		it("should return the autodetected delimiter if 'auto' specified", function() {
		delimiter = 'auto'
		rowStr = 'a;b;c';
		returnedDelimiter = getDelimiter(rowStr, {delimiter:"auto"});
		assert(returnedDelimiter === ';');
		var rowStr = "a;b;c";
		var returnedDelimiter = getDelimiter(rowStr, {delimiter: "auto"});
		assert(returnedDelimiter === ";");
		});

		it("should return the ',' delimiter if delimiter cannot be specified, in case of 'auto'", function() {
		delimiter = 'auto'
		rowStr = 'abc';
		returnedDelimiter = getDelimiter(rowStr, {delimiter:"auto"});
		assert(returnedDelimiter === ',');
		var rowStr = "abc";
		var returnedDelimiter = getDelimiter(rowStr, {delimiter: "auto"});
		assert(returnedDelimiter === ",");
		});

		it("should accetp an array with potential delimiters", function() {
		rowStr = 'a$b$c';
		returnedDelimiter = getDelimiter(rowStr, {delimiter:[",",";","$"]});
		var rowStr = "a$b$c";
		var returnedDelimiter = getDelimiter(rowStr, {delimiter: [",", ";", "$"]});
		assert(returnedDelimiter === '$');
		});
		});

22

test/testFileLine.js


		var func=require("../libs/core/fileline");
		var assert=require("assert");
		describe("fileline function",function(){
		it ("should convert data to multiple lines ",function(){
		var data="abcde\nefef";
		var result=func(data,{});
		assert.equal(result.lines.length ,1);
		assert.equal(result.partial,"efef");
		assert.equal(result.lines[0],"abcde");
		})
		})
		var func = require("../libs/core/fileline");
		var assert = require("assert");
		describe("fileline function", function() {
		it ("should convert data to multiple lines ", function() {
		var data = "abcde\nefef";
		var result = func(data, {});
		assert.equal(result.lines.length, 1);
		assert.equal(result.partial, "efef");
		assert.equal(result.lines[0], "abcde");
		});
		});

16

test/testGetDelimiter.js

		@@ -1,9 +0,9 @@
		var func=require("../libs/core/getDelimiter");
		var assert=require("assert");
		describe("getDelimiter function",function(){
		it ("should retrieve delimiter from data",function(){
		var data="abcde,efef";
		var delimiter=func(data,{delimiter:"auto"});
		var func = require("../libs/core/getDelimiter");
		var assert = require("assert");
		describe("getDelimiter function", function() {
		it ("should retrieve delimiter from data", function() {
		var data = "abcde,efef";
		var delimiter = func(data, {delimiter: "auto"});
		assert(delimiter === ",");
		})
		})
		});
		});

16

test/testGetEol.js

		@@ -1,9 +0,9 @@
		var func=require("../libs/core/getEol");
		var assert=require("assert");
		describe("getEol function",function(){
		it ("should retrieve eol from data",function(){
		var data="abcde\nefef";
		var eol=func(data,{});
		var func = require("../libs/core/getEol");
		var assert = require("assert");
		describe("getEol function", function() {
		it ("should retrieve eol from data", function() {
		var data = "abcde\nefef";
		var eol = func(data, {});
		assert(eol === "\n");
		})
		})
		});
		});

8

test/testMultiCore.js

		var assert = require("assert");
		var fs=require("fs");
		var Converter=require("../libs/core/Converter");
		var fs = require("fs");
		var Converter = require("../libs/core/Converter");
		describe("csvtojson multi core", function () {
		@@ -15,3 +15,3 @@ it("should run on test data", function (done) {
		stream.pipe(obj);
		})
		})
		});
		});

11

test/testParserMgr.js

		var assert = require("assert");
		var parserMgr = require("../libs/core/parserMgr.js");
		var Converter = require("../libs/core/Converter.js");
		var fs = require("fs");
		describe("ParserMgr", function() {
		@@ -9,2 +7,3 @@ it("should add a correct parser", function() {
		});

		it("should add a parser if regular expression is a string", function() {
		@@ -37,2 +36,3 @@ parserMgr.addParser("myparserName", "hello regexp", function() {});
		});

		describe("json parser", function() {
		@@ -59,2 +59,3 @@ it("should return an json parser with specific column title", function() {
		});

		it("should parse a json containing array", function() {
		@@ -75,2 +76,3 @@ var parser1 = parserMgr.getParser("jsonmyJSON.item1[0]");
		});

		it("should parse a json containing child json with array", function() {
		@@ -97,2 +99,3 @@ var parser1 = parserMgr.getParser("jsonmyJSON.item1.arr[0]");
		});

		it("should parse a json containing child json with array containing child json", function() {
		@@ -119,2 +122,3 @@ var parser1 = parserMgr.getParser("jsonmyJSON.item1.arr[0].title");
		});

		it("should parse a json containing child json with dynamic array containing child json", function() {
		@@ -159,2 +163,3 @@ var parser1 = parserMgr.getParser("jsonmyJSON.item1.arr[].title");
		});

		it("should parse as flat json if a column is markd as 'flat'", function() {
		@@ -176,2 +181,3 @@ var parser1 = parserMgr.getParser("flatmyJSON1.item[0].foo");
		});

		describe("json array parser", function () {
		@@ -201,2 +207,3 @@ it("should return an json array parser with specific column title", function () {
		});

		describe("omit parser", function() {
		@@ -203,0 +210,0 @@ it("should return an omit parser with specific column title", function() {

70

test/testRowSplit.js

		@@ -1,39 +0,43 @@
		var assert=require("assert");
		var func=require("../libs/core/rowSplit");
		var defParam=require("../libs/core/defParam");
		var assert = require("assert");
		var func = require("../libs/core/rowSplit");
		var defParam = require("../libs/core/defParam");

		describe("RowSplit function",function(){
		describe("RowSplit function",function() {
		it ("should split complete csv line",function(){
		var str="hello,world,csvtojson,awesome";
		var res=func(str,defParam({}));
		assert.equal(res.cols.length,4);
		assert.equal(res.closed,true);
		})
		it ("should split incomplete csv line",function(){
		var str="hello,world,\"csvtojson,awesome";
		var res=func(str,defParam({}));
		assert.equal(res.closed,false);
		})
		it ("should allow multiple line",function(){
		var str="\"he\"llo\",world,\"csvtojson,a\"\nwesome\"";
		var res=func(str,defParam({}));
		assert.equal(res.closed,true);
		assert.equal(res.cols[2],'csvtojson,a"\nwesome')
		})
		it ("should allow columns to be ignored on csv line",function(){
		var str="hello,world,csvtojson,awesome,great";
		var res=func(str,defParam({ignoreColumns:[0,3,2]}));
		assert.equal(res.cols.length,2);
		var str = "hello,world,csvtojson,awesome";
		var res = func(str, defParam({}));
		assert.equal(res.cols.length, 4);
		assert.equal(res.closed, true);
		});

		it ("should split incomplete csv line", function(){
		var str = "hello,world,\"csvtojson,awesome";
		var res = func(str, defParam({}));
		assert.equal(res.closed, false);
		});

		it ("should allow multiple line", function(){
		var str = "\"he\"llo\",world,\"csvtojson,a\"\nwesome\"";
		var res = func(str, defParam({}));
		assert.equal(res.closed, true);
		assert.equal(res.cols[2], 'csvtojson,a"\nwesome');
		});

		it ("should allow columns to be ignored on csv line", function() {
		var str = "hello,world,csvtojson,awesome,great";
		var res = func(str,defParam({ignoreColumns: [0,3,2]}));
		assert.equal(res.cols.length, 2);
		assert.equal(res.cols[0], "world");
		assert.equal(res.cols[1], "great");
		assert.equal(res.closed,true);
		})
		it ("should include only requested columns on csv line",function(){
		var str="hello,world,csvtojson,awesome,great";
		var res=func(str,defParam({includeColumns:[0,3,2]}));
		assert.equal(res.cols.length,3);
		assert.equal(res.closed, true);
		});

		it ("should include only requested columns on csv line", function() {
		var str = "hello,world,csvtojson,awesome,great";
		var res = func(str,defParam({includeColumns: [0,3,2]}));
		assert.equal(res.cols.length, 3);
		assert.equal(res.cols[0], "hello");
		assert.equal(res.cols[1], "awesome");
		assert.equal(res.closed,true);
		})
		})
		assert.equal(res.closed, true);
		});
		});

csvtojson - npm Package Compare versions

New alerts

Fixed alerts

Improved metrics