Comparing version 1.1.2 to 1.1.3
@@ -6,16 +6,16 @@ var util = require("util"); | ||
// var Processor = require("./Processor.js"); | ||
var defParam=require("./defParam"); | ||
var csvline=require("./csvline"); | ||
var fileline=require("./fileline"); | ||
var dataToCSVLine=require("./dataToCSVLine"); | ||
var fileLineToCSVLine=require("./fileLineToCSVLine"); | ||
var linesToJson=require("./linesToJson"); | ||
var CSVError=require("./CSVError"); | ||
var workerMgr=require("./workerMgr"); | ||
function Converter(params,options) { | ||
Transform.call(this,options); | ||
_param=defParam(params); | ||
this._options=options || {}; | ||
var defParam = require("./defParam"); | ||
var csvline = require("./csvline"); | ||
var fileline = require("./fileline"); | ||
var dataToCSVLine = require("./dataToCSVLine"); | ||
var fileLineToCSVLine = require("./fileLineToCSVLine"); | ||
var linesToJson = require("./linesToJson"); | ||
var CSVError = require("./CSVError"); | ||
var workerMgr = require("./workerMgr"); | ||
function Converter(params, options) { | ||
Transform.call(this, options); | ||
_param = defParam(params); | ||
this._options = options || {}; | ||
this.param = _param; | ||
this.param._options=this._options; | ||
this.param._options = this._options; | ||
// this.resultObject = new Result(this); | ||
@@ -25,22 +25,22 @@ // this.pipe(this.resultObject); // it is important to have downstream for a transform otherwise it will stuck | ||
this.recordNum = 0; | ||
this.lineNumber=0; //file line number | ||
this._csvLineBuffer=""; | ||
this.lastIndex=0; // index in result json array | ||
this.lineNumber = 0; //file line number | ||
this._csvLineBuffer = ""; | ||
this.lastIndex = 0; // index in result json array | ||
//this._pipe(this.lineParser).pipe(this.processor); | ||
// this.initNoFork(); | ||
if (this.param.forked){ | ||
this.param.forked=false; | ||
this.workerNum=2; | ||
} | ||
if (this.param.forked) { | ||
this.param.forked = false; | ||
this.workerNum = 2; | ||
} | ||
this.flushCb = null; | ||
this.processEnd = false; | ||
this.sequenceBuffer = []; | ||
this._needJson=null; | ||
this._needEmitResult=null; | ||
this._needEmitFinalResult=null; | ||
this._needEmitJson=null; | ||
this._needPush=null; | ||
this._needEmitCsv=null; | ||
this._csvTransf=null; | ||
this.finalResult=[]; | ||
this._needJson = null; | ||
this._needEmitResult = null; | ||
this._needEmitFinalResult = null; | ||
this._needEmitJson = null; | ||
this._needPush = null; | ||
this._needEmitCsv = null; | ||
this._csvTransf = null; | ||
this.finalResult = []; | ||
// this.on("data", function() {}); | ||
@@ -50,23 +50,23 @@ this.on("error", emitDone(this)); | ||
this.initWorker(); | ||
process.nextTick(function(){ | ||
if (this._needEmitFinalResult === null){ | ||
this._needEmitFinalResult=this.listeners("end_parsed").length > 0 | ||
process.nextTick(function () { | ||
if (this._needEmitFinalResult === null) { | ||
this._needEmitFinalResult = this.listeners("end_parsed").length > 0 | ||
} | ||
if (this._needEmitResult===null){ | ||
this._needEmitResult=this.listeners("record_parsed").length>0 | ||
if (this._needEmitResult === null) { | ||
this._needEmitResult = this.listeners("record_parsed").length > 0 | ||
} | ||
if (this._needEmitJson === null){ | ||
this._needEmitJson=this.listeners("json").length>0 | ||
if (this._needEmitJson === null) { | ||
this._needEmitJson = this.listeners("json").length > 0 | ||
} | ||
if (this._needEmitCsv === null){ | ||
this._needEmitCsv=this.listeners("csv").length>0 | ||
if (this._needEmitCsv === null) { | ||
this._needEmitCsv = this.listeners("csv").length > 0 | ||
} | ||
if (this._needJson === null){ | ||
this._needJson=this._needEmitJson || this._needEmitFinalResult || this._needEmitResult || this.transform || this._options.objectMode; | ||
if (this._needJson === null) { | ||
this._needJson = this._needEmitJson || this._needEmitFinalResult || this._needEmitResult || this.transform || this._options.objectMode; | ||
} | ||
if (this._needPush === null){ | ||
this._needPush = this.listeners("data").length > 0 || this.listeners("readable").length>0 | ||
if (this._needPush === null) { | ||
this._needPush = this.listeners("data").length > 0 || this.listeners("readable").length > 0 | ||
// this._needPush=false; | ||
} | ||
this.param._needParseJson=this._needJson || this._needPush; | ||
this.param._needParseJson = this._needJson || this._needPush; | ||
@@ -77,22 +77,22 @@ }.bind(this)) | ||
util.inherits(Converter, Transform); | ||
function emitDone(conv){ | ||
return function(err){ | ||
process.nextTick(function(){ | ||
conv.emit('done',err) | ||
function emitDone(conv) { | ||
return function (err) { | ||
process.nextTick(function () { | ||
conv.emit('done', err) | ||
}) | ||
} | ||
} | ||
Converter.prototype._transform = function(data, encoding, cb) { | ||
Converter.prototype._transform = function (data, encoding, cb) { | ||
if (this.param.toArrayString && this.started === false) { | ||
this.started = true; | ||
if (this._needPush){ | ||
if (this._needPush) { | ||
this.push("[" + eol, "utf8"); | ||
} | ||
} | ||
data=data.toString("utf8"); | ||
var self=this; | ||
this.preProcessRaw(data,function(d){ | ||
if (d && d.length>0){ | ||
data = data.toString("utf8"); | ||
var self = this; | ||
this.preProcessRaw(data, function (d) { | ||
if (d && d.length > 0) { | ||
self.processData(self.prepareData(d), cb); | ||
}else{ | ||
} else { | ||
cb(); | ||
@@ -102,39 +102,47 @@ } | ||
}; | ||
Converter.prototype.prepareData=function(data){ | ||
return this._csvLineBuffer+data; | ||
Converter.prototype.prepareData = function (data) { | ||
return this._csvLineBuffer + data; | ||
} | ||
Converter.prototype.setPartialData=function(d){ | ||
this._csvLineBuffer=d; | ||
Converter.prototype.setPartialData = function (d) { | ||
this._csvLineBuffer = d; | ||
} | ||
Converter.prototype.processData=function(data,cb){ | ||
var params=this.param; | ||
var fileLines=fileline(data,this.param) | ||
if (this.preProcessLine && typeof this.preProcessLine === "function"){ | ||
fileLines.lines=this._preProcessLines(fileLines.lines,this.lastIndex) | ||
Converter.prototype.processData = function (data, cb) { | ||
var params = this.param; | ||
if (params.ignoreEmpty && !params._headers){ | ||
data=data.trimLeft(); | ||
} | ||
if (!params._headers){ //header is not inited. init header | ||
this.processHead(fileLines,cb); | ||
var fileLines = fileline(data, this.param) | ||
if (fileLines.lines.length > 0) { | ||
if (this.preProcessLine && typeof this.preProcessLine === "function") { | ||
fileLines.lines = this._preProcessLines(fileLines.lines, this.lastIndex) | ||
} | ||
if (!params._headers) { //header is not inited. init header | ||
this.processHead(fileLines, cb); | ||
} else { | ||
if (params.workerNum <= 1) { | ||
var lines = fileLineToCSVLine(fileLines, params); | ||
this.setPartialData(lines.partial); | ||
var jsonArr = linesToJson(lines.lines, params, this.recordNum); | ||
this.processResult(jsonArr) | ||
this.lastIndex += jsonArr.length; | ||
this.recordNum += jsonArr.length; | ||
cb(); | ||
} else { | ||
this.workerProcess(fileLines, cb); | ||
} | ||
} | ||
}else{ | ||
if (params.workerNum<=1){ | ||
var lines=fileLineToCSVLine(fileLines,params); | ||
this.setPartialData(lines.partial); | ||
var jsonArr=linesToJson(lines.lines,params,this.recordNum); | ||
this.processResult(jsonArr) | ||
this.lastIndex+=jsonArr.length; | ||
this.recordNum+=jsonArr.length; | ||
cb(); | ||
}else{ | ||
this.workerProcess(fileLines,cb); | ||
} | ||
this.setPartialData(fileLines.partial) | ||
cb(); | ||
} | ||
} | ||
Converter.prototype._preProcessLines=function(lines,startIdx){ | ||
var rtn=[] | ||
for (var i=0;i<lines.length;i++){ | ||
var result=this.preProcessLine(lines[i],startIdx+i+1) | ||
if (typeof result ==="string"){ | ||
Converter.prototype._preProcessLines = function (lines, startIdx) { | ||
var rtn = [] | ||
for (var i = 0; i < lines.length; i++) { | ||
var result = this.preProcessLine(lines[i], startIdx + i + 1) | ||
if (typeof result === "string") { | ||
rtn.push(result) | ||
}else{ | ||
} else { | ||
rtn.push(lines[i]) | ||
this.emit("error",new Error("preProcessLine should return a string but got: "+JSON.stringify(result))) | ||
this.emit("error", new Error("preProcessLine should return a string but got: " + JSON.stringify(result))) | ||
} | ||
@@ -144,15 +152,15 @@ } | ||
} | ||
Converter.prototype.initWorker=function(){ | ||
var workerNum=this.param.workerNum-1; | ||
if (workerNum>0){ | ||
this.workerMgr=workerMgr(); | ||
this.workerMgr.initWorker(workerNum,this.param); | ||
Converter.prototype.initWorker = function () { | ||
var workerNum = this.param.workerNum - 1; | ||
if (workerNum > 0) { | ||
this.workerMgr = workerMgr(); | ||
this.workerMgr.initWorker(workerNum, this.param); | ||
} | ||
} | ||
Converter.prototype.preRawData=function(func){ | ||
this.preProcessRaw=func; | ||
Converter.prototype.preRawData = function (func) { | ||
this.preProcessRaw = func; | ||
return this; | ||
} | ||
Converter.prototype.preFileLine=function(func){ | ||
this.preProcessLine=func; | ||
Converter.prototype.preFileLine = function (func) { | ||
this.preProcessLine = func; | ||
return this; | ||
@@ -164,114 +172,114 @@ } | ||
Converter.prototype.workerProcess=function(fileLine,cb){ | ||
var self=this; | ||
var line=fileLine | ||
var eol=this.getEol() | ||
Converter.prototype.workerProcess = function (fileLine, cb) { | ||
var self = this; | ||
var line = fileLine | ||
var eol = this.getEol() | ||
this.setPartialData(line.partial) | ||
this.workerMgr.sendWorker(line.lines.join(eol)+eol,this.lastIndex,cb,function(results,lastIndex){ | ||
var cur=self.sequenceBuffer[0]; | ||
if (cur.idx === lastIndex){ | ||
cur.result=results; | ||
var records=[]; | ||
while (self.sequenceBuffer[0] && self.sequenceBuffer[0].result){ | ||
var buf=self.sequenceBuffer.shift(); | ||
records=records.concat(buf.result) | ||
this.workerMgr.sendWorker(line.lines.join(eol) + eol, this.lastIndex, cb, function (results, lastIndex) { | ||
var cur = self.sequenceBuffer[0]; | ||
if (cur.idx === lastIndex) { | ||
cur.result = results; | ||
var records = []; | ||
while (self.sequenceBuffer[0] && self.sequenceBuffer[0].result) { | ||
var buf = self.sequenceBuffer.shift(); | ||
records = records.concat(buf.result) | ||
} | ||
self.processResult(records) | ||
self.recordNum += records.length; | ||
} else { | ||
for (var i = 0; i < self.sequenceBuffer.length; i++) { | ||
var buf = self.sequenceBuffer[i]; | ||
if (buf.idx === lastIndex) { | ||
buf.result = results; | ||
break; | ||
} | ||
self.processResult(records) | ||
self.recordNum+=records.length; | ||
}else{ | ||
for (var i=0;i<self.sequenceBuffer.length;i++){ | ||
var buf=self.sequenceBuffer[i]; | ||
if (buf.idx === lastIndex){ | ||
buf.result=results; | ||
break; | ||
} | ||
} | ||
} | ||
// self.processResult(JSON.parse(results),function(){},true); | ||
} | ||
// self.processResult(JSON.parse(results),function(){},true); | ||
}) | ||
this.sequenceBuffer.push({ | ||
idx:this.lastIndex, | ||
result:null | ||
idx: this.lastIndex, | ||
result: null | ||
}); | ||
this.lastIndex+=line.lines.length; | ||
this.lastIndex += line.lines.length; | ||
} | ||
Converter.prototype.processHead=function(fileLine,cb){ | ||
var params=this.param; | ||
if (!params._headers){ //header is not inited. init header | ||
var lines=fileLineToCSVLine(fileLine,params); | ||
Converter.prototype.processHead = function (fileLine, cb) { | ||
var params = this.param; | ||
if (!params._headers) { //header is not inited. init header | ||
var lines = fileLineToCSVLine(fileLine, params); | ||
this.setPartialData(lines.partial); | ||
if (params.noheader){ | ||
if (params.headers){ | ||
params._headers=params.headers; | ||
}else{ | ||
params._headers=[]; | ||
if (params.noheader) { | ||
if (params.headers) { | ||
params._headers = params.headers; | ||
} else { | ||
params._headers = []; | ||
} | ||
}else{ | ||
var headerRow=lines.lines.shift(); | ||
if (params.headers){ | ||
params._headers=params.headers; | ||
}else{ | ||
params._headers=headerRow; | ||
} else { | ||
var headerRow = lines.lines.shift(); | ||
if (params.headers) { | ||
params._headers = params.headers; | ||
} else { | ||
params._headers = headerRow; | ||
} | ||
} | ||
if (this.param.workerNum>1){ | ||
if (this.param.workerNum > 1) { | ||
this.workerMgr.setParams(params); | ||
} | ||
var res=linesToJson(lines.lines,params,0); | ||
var res = linesToJson(lines.lines, params, 0); | ||
this.processResult(res); | ||
this.lastIndex+=res.length; | ||
this.recordNum+=res.length; | ||
this.lastIndex += res.length; | ||
this.recordNum += res.length; | ||
cb(); | ||
}else{ | ||
} else { | ||
cb(); | ||
} | ||
} | ||
Converter.prototype.processResult=function(result){ | ||
for (var i=0;i<result.length;i++){ | ||
var r=result[i]; | ||
if (r.err){ | ||
this.emit("error",r.err); | ||
}else{ | ||
this.emitResult(r); | ||
} | ||
Converter.prototype.processResult = function (result) { | ||
for (var i = 0; i < result.length; i++) { | ||
var r = result[i]; | ||
if (r.err) { | ||
this.emit("error", r.err); | ||
} else { | ||
this.emitResult(r); | ||
} | ||
// this.lastIndex+=result.length; | ||
// cb(); | ||
} | ||
// this.lastIndex+=result.length; | ||
// cb(); | ||
} | ||
Converter.prototype.emitResult=function(r){ | ||
var index=r.index; | ||
var row=r.row; | ||
var result=r.json; | ||
var resultJson=null; | ||
var resultStr=null; | ||
if (typeof result === "string"){ | ||
resultStr=result; | ||
}else{ | ||
resultJson=result; | ||
Converter.prototype.emitResult = function (r) { | ||
var index = r.index; | ||
var row = r.row; | ||
var result = r.json; | ||
var resultJson = null; | ||
var resultStr = null; | ||
if (typeof result === "string") { | ||
resultStr = result; | ||
} else { | ||
resultJson = result; | ||
} | ||
if (resultJson===null && this._needJson){ | ||
resultJson=JSON.parse(resultStr) | ||
if (typeof row ==="string"){ | ||
row=JSON.parse(row) | ||
if (resultJson === null && this._needJson) { | ||
resultJson = JSON.parse(resultStr) | ||
if (typeof row === "string") { | ||
row = JSON.parse(row) | ||
} | ||
} | ||
if (this.transform && typeof this.transform==="function"){ | ||
this.transform(resultJson,row,index); | ||
resultStr=null; | ||
if (this.transform && typeof this.transform === "function") { | ||
this.transform(resultJson, row, index); | ||
resultStr = null; | ||
} | ||
if (this._needEmitJson){ | ||
this.emit("json",resultJson,index) | ||
if (this._needEmitJson) { | ||
this.emit("json", resultJson, index) | ||
} | ||
if (this._needEmitCsv){ | ||
if (typeof row ==="string"){ | ||
row=JSON.parse(row) | ||
if (this._needEmitCsv) { | ||
if (typeof row === "string") { | ||
row = JSON.parse(row) | ||
} | ||
this.emit("csv",row,index) | ||
this.emit("csv", row, index) | ||
} | ||
if (this.param.constructResult && this._needEmitFinalResult){ | ||
if (this.param.constructResult && this._needEmitFinalResult) { | ||
this.finalResult.push(resultJson) | ||
} | ||
if (this._needEmitResult){ | ||
if (this._needEmitResult) { | ||
this.emit("record_parsed", resultJson, row, index); | ||
@@ -282,10 +290,10 @@ } | ||
} | ||
if (this._options && this._options.objectMode){ | ||
if (this._options && this._options.objectMode) { | ||
this.push(resultJson); | ||
}else{ | ||
if (this._needPush){ | ||
if (resultStr===null){ | ||
resultStr=JSON.stringify(resultJson) | ||
} else { | ||
if (this._needPush) { | ||
if (resultStr === null) { | ||
resultStr = JSON.stringify(resultJson) | ||
} | ||
this.push(!this.param.toArrayString?resultStr+eol:resultStr, "utf8"); | ||
this.push(!this.param.toArrayString ? resultStr + eol : resultStr, "utf8"); | ||
} | ||
@@ -295,18 +303,18 @@ } | ||
Converter.prototype.preProcessRaw=function(data,cb){ | ||
Converter.prototype.preProcessRaw = function (data, cb) { | ||
cb(data); | ||
} | ||
Converter.prototype.preProcessLine=function(line,lineNumber){ | ||
return line; | ||
Converter.prototype.preProcessLine = function (line, lineNumber) { | ||
return line; | ||
} | ||
Converter.prototype._flush = function(cb) { | ||
Converter.prototype._flush = function (cb) { | ||
var self = this; | ||
this.flushCb=function(){ | ||
self.emit("end_parsed",self.finalResult); | ||
if (self.workerMgr){ | ||
this.flushCb = function () { | ||
self.emit("end_parsed", self.finalResult); | ||
if (self.workerMgr) { | ||
self.workerMgr.destroyWorker(); | ||
} | ||
cb() | ||
if (!self._needPush){ | ||
if (!self._needPush) { | ||
self.emit("end") | ||
@@ -316,6 +324,6 @@ } | ||
if (this._csvLineBuffer.length > 0) { | ||
if (this._csvLineBuffer[this._csvLineBuffer.length-1] != this.getEol()){ | ||
this._csvLineBuffer+=this.getEol(); | ||
if (this._csvLineBuffer[this._csvLineBuffer.length - 1] != this.getEol()) { | ||
this._csvLineBuffer += this.getEol(); | ||
} | ||
this.processData(this._csvLineBuffer,function(){ | ||
this.processData(this._csvLineBuffer, function () { | ||
this.checkAndFlush(); | ||
@@ -335,33 +343,33 @@ }.bind(this)); | ||
// } | ||
Converter.prototype.checkAndFlush = function() { | ||
if (this._csvLineBuffer.length !== 0) { | ||
this.emit("error", CSVError.unclosed_quote(this.recordNum,this._csvLineBuffer), this._csvLineBuffer); | ||
} | ||
if (this.param.toArrayString && this._needPush) { | ||
this.push(eol + "]", "utf8"); | ||
} | ||
if (this.workerMgr && this.workerMgr.isRunning()){ | ||
this.workerMgr.drain=function(){ | ||
this.flushCb(); | ||
}.bind(this); | ||
}else{ | ||
Converter.prototype.checkAndFlush = function () { | ||
if (this._csvLineBuffer.length !== 0) { | ||
this.emit("error", CSVError.unclosed_quote(this.recordNum, this._csvLineBuffer), this._csvLineBuffer); | ||
} | ||
if (this.param.toArrayString && this._needPush) { | ||
this.push(eol + "]", "utf8"); | ||
} | ||
if (this.workerMgr && this.workerMgr.isRunning()) { | ||
this.workerMgr.drain = function () { | ||
this.flushCb(); | ||
} | ||
}.bind(this); | ||
} else { | ||
this.flushCb(); | ||
} | ||
} | ||
Converter.prototype.getEol = function(data) { | ||
Converter.prototype.getEol = function (data) { | ||
if (!this.param.eol && data) { | ||
for (var i=0;i<data.length;i++){ | ||
if (data[i]==="\r"){ | ||
if (data[i+1] === "\n"){ | ||
this.param.eol="\r\n"; | ||
}else{ | ||
this.param.eol="\r"; | ||
for (var i = 0; i < data.length; i++) { | ||
if (data[i] === "\r") { | ||
if (data[i + 1] === "\n") { | ||
this.param.eol = "\r\n"; | ||
} else { | ||
this.param.eol = "\r"; | ||
} | ||
return this.param.eol; | ||
}else if (data[i]==="\n"){ | ||
this.param.eol="\n"; | ||
} else if (data[i] === "\n") { | ||
this.param.eol = "\n"; | ||
return this.param.eol; | ||
} | ||
} | ||
this.param.eol=eol; | ||
this.param.eol = eol; | ||
} | ||
@@ -371,11 +379,11 @@ | ||
}; | ||
Converter.prototype.fromFile = function(filePath, cb) { | ||
Converter.prototype.fromFile = function (filePath, cb) { | ||
var fs = require('fs'); | ||
var rs=null; | ||
this.wrapCallback(cb, function() { | ||
if (rs && rs.destroy){ | ||
var rs = null; | ||
this.wrapCallback(cb, function () { | ||
if (rs && rs.destroy) { | ||
rs.destroy(); | ||
} | ||
}); | ||
fs.exists(filePath, function(exist) { | ||
fs.exists(filePath, function (exist) { | ||
if (exist) { | ||
@@ -385,3 +393,3 @@ rs = fs.createReadStream(filePath); | ||
} else { | ||
this.emit('error',new Error("File not exist")) | ||
this.emit('error', new Error("File not exist")) | ||
} | ||
@@ -391,16 +399,14 @@ }.bind(this)); | ||
} | ||
Converter.prototype.fromStream=function(readStream,cb){ | ||
if (cb && typeof cb ==="function"){ | ||
Converter.prototype.fromStream = function (readStream, cb) { | ||
if (cb && typeof cb === "function") { | ||
this.wrapCallback(cb); | ||
} | ||
process.nextTick(function(){ | ||
readStream.pipe(this); | ||
}.bind(this)) | ||
readStream.pipe(this); | ||
return this; | ||
} | ||
Converter.prototype.transf=function(func){ | ||
this.transform=func; | ||
Converter.prototype.transf = function (func) { | ||
this.transform = func; | ||
return this; | ||
} | ||
Converter.prototype.fromString = function(csvString, cb) { | ||
Converter.prototype.fromString = function (csvString, cb) { | ||
if (typeof csvString != "string") { | ||
@@ -410,6 +416,6 @@ return cb(new Error("Passed CSV Data is not a string.")); | ||
if (cb && typeof cb === "function") { | ||
this.wrapCallback(cb, function() { | ||
this.wrapCallback(cb, function () { | ||
}); | ||
} | ||
process.nextTick(function(){ | ||
process.nextTick(function () { | ||
this.end(csvString) | ||
@@ -419,9 +425,9 @@ }.bind(this)) | ||
}; | ||
Converter.prototype.wrapCallback = function(cb, clean) { | ||
Converter.prototype.wrapCallback = function (cb, clean) { | ||
if (clean === undefined){ | ||
clean=function(){} | ||
if (clean === undefined) { | ||
clean = function () { } | ||
} | ||
if (cb && typeof cb ==="function"){ | ||
this.once("end_parsed", function(res) { | ||
if (cb && typeof cb === "function") { | ||
this.once("end_parsed", function (res) { | ||
if (!this.hasError) { | ||
@@ -432,5 +438,5 @@ cb(null, res); | ||
} | ||
this.once("error", function(err) { | ||
this.hasError=true; | ||
if (cb && typeof cb ==="function"){ | ||
this.once("error", function (err) { | ||
this.hasError = true; | ||
if (cb && typeof cb === "function") { | ||
cb(err); | ||
@@ -437,0 +443,0 @@ } |
@@ -12,3 +12,11 @@ var getEol=require("./getEol"); | ||
var partial=lines.pop(); | ||
return {lines:lines,partial:partial}; | ||
// if (param.ignoreEmpty){ | ||
// var trimmedLines=[]; | ||
// for (var i=0;i<lines.length;i++){ | ||
// trimmedLines.push(lines[i].trim()) | ||
// } | ||
// return {lines:trimmedLines,partial:partial}; | ||
// }else{ | ||
return {lines:lines,partial:partial}; | ||
// } | ||
} |
@@ -80,3 +80,2 @@ var parserMgr = require("./parserMgr.js"); | ||
} | ||
var _ = require('lodash') | ||
function convertRowToJson(row, headRow, param) { | ||
@@ -136,2 +135,3 @@ var hasValue = false; | ||
function setPath(json, path, value) { | ||
var _set = require('lodash/set') | ||
var pathArr = path.split('.') | ||
@@ -141,3 +141,3 @@ if (pathArr.length === 1) { | ||
} else { | ||
_.set(json, path, value) | ||
_set(json, path, value) | ||
} | ||
@@ -188,3 +188,3 @@ } | ||
if (isNaN(rtn)) { | ||
return 0; | ||
return item; | ||
} | ||
@@ -191,0 +191,0 @@ return rtn; |
@@ -21,3 +21,3 @@ { | ||
], | ||
"version": "1.1.2", | ||
"version": "1.1.3", | ||
"keywords": [ | ||
@@ -24,0 +24,0 @@ "csv", |
var Converter = require("../libs/core/Converter.js"); | ||
var csv=require('../') | ||
var assert = require("assert"); | ||
@@ -198,3 +199,3 @@ var fs = require("fs"); | ||
assert(d.column7 === "1234"); | ||
assert(d.column8 === 0); | ||
assert(d.column8 === "abcd"); | ||
assert(d.column9 === true); | ||
@@ -285,3 +286,3 @@ assert(d.column10[0]===23); | ||
var rs = fs.createReadStream(testData); | ||
var st = rs.pipe(new Converter({ignoreEmpty:true})); | ||
var st = rs.pipe(csv({ignoreEmpty:true})); | ||
st.on("end_parsed",function (res){ | ||
@@ -288,0 +289,0 @@ var j = res[0]; |
@@ -355,2 +355,17 @@ var Converter = require("../libs/core/Converter.js"); | ||
}) | ||
it("should process long header", function (done) { | ||
var testData = __dirname + "/data/longHeader"; | ||
var rs = fs.createReadStream(testData,{highWaterMark:100}); | ||
var numofrow = 0; | ||
var numofjson = 0; | ||
csv({},{highWaterMark:100}) | ||
.fromStream(rs) | ||
.on("json", function (res) { | ||
console.log(res) | ||
assert.equal(res.Date,'8/26/16') | ||
}) | ||
.on("end", function () { | ||
done(); | ||
}) | ||
}) | ||
}); |
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is too big to display
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Uses eval
Supply chain riskPackage uses dynamic code execution (e.g., eval()), which is a dangerous practice. This can prevent the code from running in certain environments and increases the risk that the code may contain exploits or malicious behavior.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
828159
89
12650
6