Comparing version 0.4.6 to 0.4.7
@@ -23,7 +23,7 @@ var util = require("util"); | ||
fork: false, //use another CPU core to convert the csv stream | ||
noheader:false, //indicate if first line of CSV file is header or not. | ||
headers:null, //an array of header strings. If noheader is false and headers is array, csv header will be ignored. | ||
noheader: false, //indicate if first line of CSV file is header or not. | ||
headers: null, //an array of header strings. If noheader is false and headers is array, csv header will be ignored. | ||
flatKeys: false, // Don't interpret dots and square brackets in header fields as nested object or array identifiers at all. | ||
maxRowLength:0, //the max character a csv row could have. 0 means infinite. If max number exceeded, parser will emit "error" of "row_exceed". if a possibly corrupted csv data provided, give it a number like 65535 so the parser wont consume memory. default: 0 | ||
checkColumn:false //whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false | ||
maxRowLength: 0, //the max character a csv row could have. 0 means infinite. If max number exceeded, parser will emit "error" of "row_exceed". if a possibly corrupted csv data provided, give it a number like 65535 so the parser wont consume memory. default: 0 | ||
checkColumn: false //whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false | ||
}; | ||
@@ -44,3 +44,2 @@ if (params && typeof params === "object") { | ||
this.started = false; | ||
this._callback = null; | ||
this.recordNum = 0; | ||
@@ -59,9 +58,4 @@ //this._pipe(this.lineParser).pipe(this.processor); | ||
this.emit("end_parsed", finalResult); | ||
if (typeof this._callback === "function") { | ||
var func = this._callback; | ||
this._callback = null; | ||
func(null, finalResult); | ||
} | ||
}.bind(this)); | ||
this.on("error",function(){}); | ||
this.on("error", function() {}); | ||
return this; | ||
@@ -78,4 +72,4 @@ } | ||
this.child.stdout.on("data", function(d, e) { | ||
this.push(d,e); | ||
// this.emit("record_parsed"); | ||
this.push(d, e); | ||
// this.emit("record_parsed"); | ||
}.bind(this)); | ||
@@ -86,7 +80,7 @@ this.child.on("message", function(msg) { | ||
//var recs = msg.arguments; | ||
var args=msg.arguments; | ||
var args = msg.arguments; | ||
//console.log(recs); | ||
//var recs=args[0]; | ||
//for (var i=0;i<recs.length;i++){ | ||
//this.emit("record_parsed", recs[i][0], recs[i][1], recs[i][2]); | ||
//this.emit("record_parsed", recs[i][0], recs[i][1], recs[i][2]); | ||
//} | ||
@@ -97,6 +91,6 @@ this.emit("record_parsed", args[0], args[1], args[2]); | ||
this.push(new Buffer(args[0]), args[1]); | ||
}else if (msg.action==="error"){ | ||
var args=msg.arguments; | ||
} else if (msg.action === "error") { | ||
var args = msg.arguments; | ||
args.unshift("error"); | ||
this.emit.apply(this,args); | ||
this.emit.apply(this, args); | ||
} | ||
@@ -120,17 +114,12 @@ }.bind(this)); | ||
Converter.prototype.initNoFork = function() { | ||
function onError(){ | ||
var args=Array.prototype.slice.call(arguments,0); | ||
args.unshift("error"); | ||
this.hasError=true; | ||
this.emit.apply(this,args); | ||
if (typeof this._callback === "function") { | ||
var func = this._callback; | ||
this._callback = null; | ||
func(args, []); | ||
} | ||
function onError() { | ||
var args = Array.prototype.slice.call(arguments, 0); | ||
args.unshift("error"); | ||
this.hasError = true; | ||
this.emit.apply(this, args); | ||
}; | ||
this.lineParser = new CSVLine(this.param); | ||
this.lineParser.on("error",onError.bind(this)); | ||
this.lineParser.on("error", onError.bind(this)); | ||
this.processor = new Processor(this.param); | ||
this.processor.on("error",onError.bind(this)); | ||
this.processor.on("error", onError.bind(this)); | ||
var syncWorker = new Worker(this.param, true); | ||
@@ -141,3 +130,3 @@ // syncWorker.on("error",onError); | ||
for (var i = 1; i < this.param.workerNum; i++) { | ||
var worker=new Worker(this.param, false); | ||
var worker = new Worker(this.param, false); | ||
// worker.on("error",onError); | ||
@@ -197,3 +186,3 @@ this.processor.addWorker(worker); | ||
} | ||
this.lineParser.write(data, encoding,cb); | ||
this.lineParser.write(data, encoding, cb); | ||
//this.push(data,encoding); | ||
@@ -209,3 +198,3 @@ // cb(); | ||
Converter.prototype._transformFork = function(data, encoding, cb) { | ||
this.child.stdin.write(data, encoding,cb); | ||
this.child.stdin.write(data, encoding, cb); | ||
} | ||
@@ -227,6 +216,21 @@ Converter.prototype._flushFork = function(cb) { | ||
}; | ||
Converter.prototype.fromFile = function(filePath, cb) { | ||
var fs = require('fs'); | ||
fs.exists(filePath, function(exist) { | ||
if (exist) { | ||
var rs = fs.createReadStream(filePath); | ||
rs.pipe(this); | ||
this.wrapCallback(cb,function(){ | ||
fs.destroy(); | ||
}); | ||
} else { | ||
cb(new Error(filePath + " cannot be found.")); | ||
} | ||
}.bind(this)); | ||
return this; | ||
} | ||
Converter.prototype.fromString = function(csvString, cb) { | ||
var rs = new Readable(); | ||
var offset=0; | ||
if (typeof csvString !="string"){ | ||
var offset = 0; | ||
if (typeof csvString != "string") { | ||
return cb(new Error("Passed CSV Data is not a string.")); | ||
@@ -236,6 +240,6 @@ } | ||
// console.log(offset,len,csvString.length); | ||
var sub=csvString.substr(offset,len); | ||
var sub = csvString.substr(offset, len); | ||
this.push(sub); | ||
offset+=len; | ||
if (offset>=csvString.length){ | ||
offset += len; | ||
if (offset >= csvString.length) { | ||
this.push(null); | ||
@@ -246,7 +250,20 @@ } | ||
if (cb && typeof cb === "function") { | ||
this._callback = cb; | ||
this.wrapCallback(cb,function(){ | ||
rs.pause(); | ||
}); | ||
} | ||
return this; | ||
}; | ||
Converter.prototype.wrapCallback = function(cb,clean) { | ||
this.once("end_parsed", function(res) { | ||
if (!this.hasError) { | ||
cb(null, res); | ||
} | ||
}.bind(this)); | ||
this.once("error", function(err) { | ||
cb(Array.prototype.join.call(arguments, ", ")); | ||
clean(); | ||
}); | ||
} | ||
module.exports = Converter; |
@@ -76,3 +76,3 @@ /** | ||
this._recordBuffer = ''; | ||
this.push(data, "utf8"); | ||
this.push(data.trim(), "utf8"); | ||
this.rowIndex++; | ||
@@ -79,0 +79,0 @@ // if (this.rowIndex % 10000 ===0){ |
@@ -76,3 +76,7 @@ /** | ||
} else { | ||
this.emit("record_parsed", resultRow, row, index - 1); | ||
if (resultRow){ | ||
this.emit("record_parsed", resultRow, row, index - 1); | ||
}else{ | ||
//Empty row detedted. skip | ||
} | ||
//this.push(JSON.stringify([resultRow,row,obj.rowIndex]),"utf8"); | ||
@@ -79,0 +83,0 @@ } |
@@ -42,3 +42,7 @@ /** | ||
}else{ | ||
cb(null, res.resultRow, res.row, res.index); | ||
if (res){ | ||
cb(null, res.resultRow, res.row, res.index); | ||
}else{ | ||
cb(null); | ||
} | ||
} | ||
@@ -45,0 +49,0 @@ }); |
@@ -8,7 +8,7 @@ var parserMgr = require("./parserMgr.js"); | ||
inst[action](m, function(err, res) { | ||
if (!res){ | ||
res={}; | ||
if (!res) { | ||
res = {}; | ||
} | ||
if (err) { | ||
res.error=err; | ||
res.error = err; | ||
} | ||
@@ -28,18 +28,19 @@ res.action = m.action; | ||
function genConstHeadRow(msg,cb){ | ||
var Parser=require("./parser"); | ||
var number=msg.number; | ||
parseRules=[]; | ||
headRow=[]; | ||
while (number>0){ | ||
var p=new Parser("field"+number,/.*/,function(params){ | ||
var name=this.getName(); | ||
params.resultRow[name]=params.item; | ||
},true); | ||
parseRules.unshift(p); | ||
headRow.unshift(p.getName()); | ||
number--; | ||
} | ||
cb(); | ||
function genConstHeadRow(msg, cb) { | ||
var Parser = require("./parser"); | ||
var number = msg.number; | ||
parseRules = []; | ||
headRow = []; | ||
while (number > 0) { | ||
var p = new Parser("field" + number, /.*/, function(params) { | ||
var name = this.getName(); | ||
params.resultRow[name] = params.item; | ||
}, true); | ||
parseRules.unshift(p); | ||
headRow.unshift(p.getName()); | ||
number--; | ||
} | ||
cb(); | ||
} | ||
function processHeadRow(msg, cb) { | ||
@@ -58,6 +59,7 @@ headRow = msg.row; | ||
var row = utils.rowSplit(data, param.delimiter, param.quote, param.trim); | ||
if (param.checkColumn && row.length !=parseRules.length ){ | ||
return cb("Error: column_mismatched. Data: "+data+". Row index: "+index); | ||
if (param.checkColumn && row.length != parseRules.length) { | ||
return cb("Error: column_mismatched. Data: " + data + ". Row index: " + index); | ||
} | ||
var resultRow = {}; | ||
var hasValue = false; | ||
for (i = 0; i < parseRules.length; i++) { | ||
@@ -68,2 +70,3 @@ item = row[i]; | ||
} | ||
hasValue = true; | ||
parser = parseRules[i]; | ||
@@ -81,7 +84,11 @@ head = headRow[i]; | ||
} | ||
cb(null, { | ||
resultRow: resultRow, | ||
row: row, | ||
index: index | ||
}); | ||
if (hasValue) { | ||
cb(null, { | ||
resultRow: resultRow, | ||
row: row, | ||
index: index | ||
}); | ||
} else { | ||
cb(); | ||
} | ||
@@ -92,5 +99,5 @@ } | ||
processRow: processRow, | ||
genConstHeadRow:genConstHeadRow | ||
genConstHeadRow: genConstHeadRow | ||
} | ||
} | ||
module.exports=init; | ||
module.exports = init; |
@@ -21,3 +21,3 @@ { | ||
], | ||
"version": "0.4.6", | ||
"version": "0.4.7", | ||
"keywords": [ | ||
@@ -24,0 +24,0 @@ "csv", |
@@ -48,3 +48,5 @@ # CSVTOJSON | ||
var Converter = require("csvtojson").Converter; | ||
var converter = new Converter({}); | ||
var converter = new Converter({ | ||
checkType:false //turn off auto type check to increase performance | ||
}); | ||
@@ -614,2 +616,7 @@ //end_parsed will be emitted once parsing finished | ||
## 0.4.7 | ||
* ignoreEmpty now ignores empty rows as well | ||
* optimised performance | ||
* added fromFile method | ||
## 0.4.4 | ||
@@ -616,0 +623,0 @@ * Add error handling for corrupted CSV data |
@@ -113,2 +113,11 @@ var Converter = require("../libs/core/Converter.js"); | ||
}); | ||
it("should be able to convert csv string with error", function (done) { | ||
var testData = __dirname + "/data/dataWithUnclosedQuotes"; | ||
var data = fs.readFileSync(testData).toString(); | ||
var csvConverter = new Converter(); | ||
csvConverter.fromString(data, function (err, jsonObj) { | ||
assert(err); | ||
done(); | ||
}); | ||
}); | ||
it("should be able to convert csv string without callback provided", function (done) { | ||
@@ -270,2 +279,3 @@ var testData = __dirname + "/data/testData"; | ||
var j = res[0]; | ||
assert(res.length === 2); | ||
assert (j.col2.length === 1); | ||
@@ -275,2 +285,3 @@ assert(j.col2[0] === "d3"); | ||
assert(j.col4.col5 === "world"); | ||
assert(res[1].col1==="d2"); | ||
done(); | ||
@@ -277,0 +288,0 @@ }); |
@@ -61,2 +61,13 @@ var Converter = require("../libs/core/Converter.js"); | ||
}); | ||
it ("should parse fromFile",function(done){ | ||
var csvFile = __dirname + "/data/large-csv-sample.csv"; | ||
var conv = new Converter({ | ||
workerNum: 3 | ||
}); | ||
conv.fromFile(csvFile, function(err, res) { | ||
assert(!err); | ||
assert(res.length === 5290); | ||
done(); | ||
}); | ||
}); | ||
// it ("should convert big csv",function(done){ | ||
@@ -63,0 +74,0 @@ // // var rs=fs.createReadStream(__dirname+"/data/large-csv-sample.csv"); |
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
362814
2248
696
12