Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More
Socket
Sign inDemoInstall
Socket

csvtojson

Package Overview
Dependencies
Maintainers
1
Versions
82
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

csvtojson - npm Package Compare versions

Comparing version 1.1.2 to 1.1.3

test/data/longHeader

480

libs/core/Converter.js

@@ -6,16 +6,16 @@ var util = require("util");

// var Processor = require("./Processor.js");
var defParam=require("./defParam");
var csvline=require("./csvline");
var fileline=require("./fileline");
var dataToCSVLine=require("./dataToCSVLine");
var fileLineToCSVLine=require("./fileLineToCSVLine");
var linesToJson=require("./linesToJson");
var CSVError=require("./CSVError");
var workerMgr=require("./workerMgr");
function Converter(params,options) {
Transform.call(this,options);
_param=defParam(params);
this._options=options || {};
var defParam = require("./defParam");
var csvline = require("./csvline");
var fileline = require("./fileline");
var dataToCSVLine = require("./dataToCSVLine");
var fileLineToCSVLine = require("./fileLineToCSVLine");
var linesToJson = require("./linesToJson");
var CSVError = require("./CSVError");
var workerMgr = require("./workerMgr");
function Converter(params, options) {
Transform.call(this, options);
_param = defParam(params);
this._options = options || {};
this.param = _param;
this.param._options=this._options;
this.param._options = this._options;
// this.resultObject = new Result(this);

@@ -25,22 +25,22 @@ // this.pipe(this.resultObject); // it is important to have downstream for a transform otherwise it will stuck

this.recordNum = 0;
this.lineNumber=0; //file line number
this._csvLineBuffer="";
this.lastIndex=0; // index in result json array
this.lineNumber = 0; //file line number
this._csvLineBuffer = "";
this.lastIndex = 0; // index in result json array
//this._pipe(this.lineParser).pipe(this.processor);
// this.initNoFork();
if (this.param.forked){
this.param.forked=false;
this.workerNum=2;
}
if (this.param.forked) {
this.param.forked = false;
this.workerNum = 2;
}
this.flushCb = null;
this.processEnd = false;
this.sequenceBuffer = [];
this._needJson=null;
this._needEmitResult=null;
this._needEmitFinalResult=null;
this._needEmitJson=null;
this._needPush=null;
this._needEmitCsv=null;
this._csvTransf=null;
this.finalResult=[];
this._needJson = null;
this._needEmitResult = null;
this._needEmitFinalResult = null;
this._needEmitJson = null;
this._needPush = null;
this._needEmitCsv = null;
this._csvTransf = null;
this.finalResult = [];
// this.on("data", function() {});

@@ -50,23 +50,23 @@ this.on("error", emitDone(this));

this.initWorker();
process.nextTick(function(){
if (this._needEmitFinalResult === null){
this._needEmitFinalResult=this.listeners("end_parsed").length > 0
process.nextTick(function () {
if (this._needEmitFinalResult === null) {
this._needEmitFinalResult = this.listeners("end_parsed").length > 0
}
if (this._needEmitResult===null){
this._needEmitResult=this.listeners("record_parsed").length>0
if (this._needEmitResult === null) {
this._needEmitResult = this.listeners("record_parsed").length > 0
}
if (this._needEmitJson === null){
this._needEmitJson=this.listeners("json").length>0
if (this._needEmitJson === null) {
this._needEmitJson = this.listeners("json").length > 0
}
if (this._needEmitCsv === null){
this._needEmitCsv=this.listeners("csv").length>0
if (this._needEmitCsv === null) {
this._needEmitCsv = this.listeners("csv").length > 0
}
if (this._needJson === null){
this._needJson=this._needEmitJson || this._needEmitFinalResult || this._needEmitResult || this.transform || this._options.objectMode;
if (this._needJson === null) {
this._needJson = this._needEmitJson || this._needEmitFinalResult || this._needEmitResult || this.transform || this._options.objectMode;
}
if (this._needPush === null){
this._needPush = this.listeners("data").length > 0 || this.listeners("readable").length>0
if (this._needPush === null) {
this._needPush = this.listeners("data").length > 0 || this.listeners("readable").length > 0
// this._needPush=false;
}
this.param._needParseJson=this._needJson || this._needPush;
this.param._needParseJson = this._needJson || this._needPush;

@@ -77,22 +77,22 @@ }.bind(this))

util.inherits(Converter, Transform);
function emitDone(conv){
return function(err){
process.nextTick(function(){
conv.emit('done',err)
function emitDone(conv) {
return function (err) {
process.nextTick(function () {
conv.emit('done', err)
})
}
}
Converter.prototype._transform = function(data, encoding, cb) {
Converter.prototype._transform = function (data, encoding, cb) {
if (this.param.toArrayString && this.started === false) {
this.started = true;
if (this._needPush){
if (this._needPush) {
this.push("[" + eol, "utf8");
}
}
data=data.toString("utf8");
var self=this;
this.preProcessRaw(data,function(d){
if (d && d.length>0){
data = data.toString("utf8");
var self = this;
this.preProcessRaw(data, function (d) {
if (d && d.length > 0) {
self.processData(self.prepareData(d), cb);
}else{
} else {
cb();

@@ -102,39 +102,47 @@ }

};
Converter.prototype.prepareData=function(data){
return this._csvLineBuffer+data;
Converter.prototype.prepareData = function (data) {
return this._csvLineBuffer + data;
}
Converter.prototype.setPartialData=function(d){
this._csvLineBuffer=d;
Converter.prototype.setPartialData = function (d) {
this._csvLineBuffer = d;
}
Converter.prototype.processData=function(data,cb){
var params=this.param;
var fileLines=fileline(data,this.param)
if (this.preProcessLine && typeof this.preProcessLine === "function"){
fileLines.lines=this._preProcessLines(fileLines.lines,this.lastIndex)
Converter.prototype.processData = function (data, cb) {
var params = this.param;
if (params.ignoreEmpty && !params._headers){
data=data.trimLeft();
}
if (!params._headers){ //header is not inited. init header
this.processHead(fileLines,cb);
var fileLines = fileline(data, this.param)
if (fileLines.lines.length > 0) {
if (this.preProcessLine && typeof this.preProcessLine === "function") {
fileLines.lines = this._preProcessLines(fileLines.lines, this.lastIndex)
}
if (!params._headers) { //header is not inited. init header
this.processHead(fileLines, cb);
} else {
if (params.workerNum <= 1) {
var lines = fileLineToCSVLine(fileLines, params);
this.setPartialData(lines.partial);
var jsonArr = linesToJson(lines.lines, params, this.recordNum);
this.processResult(jsonArr)
this.lastIndex += jsonArr.length;
this.recordNum += jsonArr.length;
cb();
} else {
this.workerProcess(fileLines, cb);
}
}
}else{
if (params.workerNum<=1){
var lines=fileLineToCSVLine(fileLines,params);
this.setPartialData(lines.partial);
var jsonArr=linesToJson(lines.lines,params,this.recordNum);
this.processResult(jsonArr)
this.lastIndex+=jsonArr.length;
this.recordNum+=jsonArr.length;
cb();
}else{
this.workerProcess(fileLines,cb);
}
this.setPartialData(fileLines.partial)
cb();
}
}
Converter.prototype._preProcessLines=function(lines,startIdx){
var rtn=[]
for (var i=0;i<lines.length;i++){
var result=this.preProcessLine(lines[i],startIdx+i+1)
if (typeof result ==="string"){
Converter.prototype._preProcessLines = function (lines, startIdx) {
var rtn = []
for (var i = 0; i < lines.length; i++) {
var result = this.preProcessLine(lines[i], startIdx + i + 1)
if (typeof result === "string") {
rtn.push(result)
}else{
} else {
rtn.push(lines[i])
this.emit("error",new Error("preProcessLine should return a string but got: "+JSON.stringify(result)))
this.emit("error", new Error("preProcessLine should return a string but got: " + JSON.stringify(result)))
}

@@ -144,15 +152,15 @@ }

}
Converter.prototype.initWorker=function(){
var workerNum=this.param.workerNum-1;
if (workerNum>0){
this.workerMgr=workerMgr();
this.workerMgr.initWorker(workerNum,this.param);
Converter.prototype.initWorker = function () {
var workerNum = this.param.workerNum - 1;
if (workerNum > 0) {
this.workerMgr = workerMgr();
this.workerMgr.initWorker(workerNum, this.param);
}
}
Converter.prototype.preRawData=function(func){
this.preProcessRaw=func;
Converter.prototype.preRawData = function (func) {
this.preProcessRaw = func;
return this;
}
Converter.prototype.preFileLine=function(func){
this.preProcessLine=func;
Converter.prototype.preFileLine = function (func) {
this.preProcessLine = func;
return this;

@@ -164,114 +172,114 @@ }

Converter.prototype.workerProcess=function(fileLine,cb){
var self=this;
var line=fileLine
var eol=this.getEol()
Converter.prototype.workerProcess = function (fileLine, cb) {
var self = this;
var line = fileLine
var eol = this.getEol()
this.setPartialData(line.partial)
this.workerMgr.sendWorker(line.lines.join(eol)+eol,this.lastIndex,cb,function(results,lastIndex){
var cur=self.sequenceBuffer[0];
if (cur.idx === lastIndex){
cur.result=results;
var records=[];
while (self.sequenceBuffer[0] && self.sequenceBuffer[0].result){
var buf=self.sequenceBuffer.shift();
records=records.concat(buf.result)
this.workerMgr.sendWorker(line.lines.join(eol) + eol, this.lastIndex, cb, function (results, lastIndex) {
var cur = self.sequenceBuffer[0];
if (cur.idx === lastIndex) {
cur.result = results;
var records = [];
while (self.sequenceBuffer[0] && self.sequenceBuffer[0].result) {
var buf = self.sequenceBuffer.shift();
records = records.concat(buf.result)
}
self.processResult(records)
self.recordNum += records.length;
} else {
for (var i = 0; i < self.sequenceBuffer.length; i++) {
var buf = self.sequenceBuffer[i];
if (buf.idx === lastIndex) {
buf.result = results;
break;
}
self.processResult(records)
self.recordNum+=records.length;
}else{
for (var i=0;i<self.sequenceBuffer.length;i++){
var buf=self.sequenceBuffer[i];
if (buf.idx === lastIndex){
buf.result=results;
break;
}
}
}
// self.processResult(JSON.parse(results),function(){},true);
}
// self.processResult(JSON.parse(results),function(){},true);
})
this.sequenceBuffer.push({
idx:this.lastIndex,
result:null
idx: this.lastIndex,
result: null
});
this.lastIndex+=line.lines.length;
this.lastIndex += line.lines.length;
}
Converter.prototype.processHead=function(fileLine,cb){
var params=this.param;
if (!params._headers){ //header is not inited. init header
var lines=fileLineToCSVLine(fileLine,params);
Converter.prototype.processHead = function (fileLine, cb) {
var params = this.param;
if (!params._headers) { //header is not inited. init header
var lines = fileLineToCSVLine(fileLine, params);
this.setPartialData(lines.partial);
if (params.noheader){
if (params.headers){
params._headers=params.headers;
}else{
params._headers=[];
if (params.noheader) {
if (params.headers) {
params._headers = params.headers;
} else {
params._headers = [];
}
}else{
var headerRow=lines.lines.shift();
if (params.headers){
params._headers=params.headers;
}else{
params._headers=headerRow;
} else {
var headerRow = lines.lines.shift();
if (params.headers) {
params._headers = params.headers;
} else {
params._headers = headerRow;
}
}
if (this.param.workerNum>1){
if (this.param.workerNum > 1) {
this.workerMgr.setParams(params);
}
var res=linesToJson(lines.lines,params,0);
var res = linesToJson(lines.lines, params, 0);
this.processResult(res);
this.lastIndex+=res.length;
this.recordNum+=res.length;
this.lastIndex += res.length;
this.recordNum += res.length;
cb();
}else{
} else {
cb();
}
}
Converter.prototype.processResult=function(result){
for (var i=0;i<result.length;i++){
var r=result[i];
if (r.err){
this.emit("error",r.err);
}else{
this.emitResult(r);
}
Converter.prototype.processResult = function (result) {
for (var i = 0; i < result.length; i++) {
var r = result[i];
if (r.err) {
this.emit("error", r.err);
} else {
this.emitResult(r);
}
// this.lastIndex+=result.length;
// cb();
}
// this.lastIndex+=result.length;
// cb();
}
Converter.prototype.emitResult=function(r){
var index=r.index;
var row=r.row;
var result=r.json;
var resultJson=null;
var resultStr=null;
if (typeof result === "string"){
resultStr=result;
}else{
resultJson=result;
Converter.prototype.emitResult = function (r) {
var index = r.index;
var row = r.row;
var result = r.json;
var resultJson = null;
var resultStr = null;
if (typeof result === "string") {
resultStr = result;
} else {
resultJson = result;
}
if (resultJson===null && this._needJson){
resultJson=JSON.parse(resultStr)
if (typeof row ==="string"){
row=JSON.parse(row)
if (resultJson === null && this._needJson) {
resultJson = JSON.parse(resultStr)
if (typeof row === "string") {
row = JSON.parse(row)
}
}
if (this.transform && typeof this.transform==="function"){
this.transform(resultJson,row,index);
resultStr=null;
if (this.transform && typeof this.transform === "function") {
this.transform(resultJson, row, index);
resultStr = null;
}
if (this._needEmitJson){
this.emit("json",resultJson,index)
if (this._needEmitJson) {
this.emit("json", resultJson, index)
}
if (this._needEmitCsv){
if (typeof row ==="string"){
row=JSON.parse(row)
if (this._needEmitCsv) {
if (typeof row === "string") {
row = JSON.parse(row)
}
this.emit("csv",row,index)
this.emit("csv", row, index)
}
if (this.param.constructResult && this._needEmitFinalResult){
if (this.param.constructResult && this._needEmitFinalResult) {
this.finalResult.push(resultJson)
}
if (this._needEmitResult){
if (this._needEmitResult) {
this.emit("record_parsed", resultJson, row, index);

@@ -282,10 +290,10 @@ }

}
if (this._options && this._options.objectMode){
if (this._options && this._options.objectMode) {
this.push(resultJson);
}else{
if (this._needPush){
if (resultStr===null){
resultStr=JSON.stringify(resultJson)
} else {
if (this._needPush) {
if (resultStr === null) {
resultStr = JSON.stringify(resultJson)
}
this.push(!this.param.toArrayString?resultStr+eol:resultStr, "utf8");
this.push(!this.param.toArrayString ? resultStr + eol : resultStr, "utf8");
}

@@ -295,18 +303,18 @@ }

Converter.prototype.preProcessRaw=function(data,cb){
Converter.prototype.preProcessRaw = function (data, cb) {
cb(data);
}
Converter.prototype.preProcessLine=function(line,lineNumber){
return line;
Converter.prototype.preProcessLine = function (line, lineNumber) {
return line;
}
Converter.prototype._flush = function(cb) {
Converter.prototype._flush = function (cb) {
var self = this;
this.flushCb=function(){
self.emit("end_parsed",self.finalResult);
if (self.workerMgr){
this.flushCb = function () {
self.emit("end_parsed", self.finalResult);
if (self.workerMgr) {
self.workerMgr.destroyWorker();
}
cb()
if (!self._needPush){
if (!self._needPush) {
self.emit("end")

@@ -316,6 +324,6 @@ }

if (this._csvLineBuffer.length > 0) {
if (this._csvLineBuffer[this._csvLineBuffer.length-1] != this.getEol()){
this._csvLineBuffer+=this.getEol();
if (this._csvLineBuffer[this._csvLineBuffer.length - 1] != this.getEol()) {
this._csvLineBuffer += this.getEol();
}
this.processData(this._csvLineBuffer,function(){
this.processData(this._csvLineBuffer, function () {
this.checkAndFlush();

@@ -335,33 +343,33 @@ }.bind(this));

// }
Converter.prototype.checkAndFlush = function() {
if (this._csvLineBuffer.length !== 0) {
this.emit("error", CSVError.unclosed_quote(this.recordNum,this._csvLineBuffer), this._csvLineBuffer);
}
if (this.param.toArrayString && this._needPush) {
this.push(eol + "]", "utf8");
}
if (this.workerMgr && this.workerMgr.isRunning()){
this.workerMgr.drain=function(){
this.flushCb();
}.bind(this);
}else{
Converter.prototype.checkAndFlush = function () {
if (this._csvLineBuffer.length !== 0) {
this.emit("error", CSVError.unclosed_quote(this.recordNum, this._csvLineBuffer), this._csvLineBuffer);
}
if (this.param.toArrayString && this._needPush) {
this.push(eol + "]", "utf8");
}
if (this.workerMgr && this.workerMgr.isRunning()) {
this.workerMgr.drain = function () {
this.flushCb();
}
}.bind(this);
} else {
this.flushCb();
}
}
Converter.prototype.getEol = function(data) {
Converter.prototype.getEol = function (data) {
if (!this.param.eol && data) {
for (var i=0;i<data.length;i++){
if (data[i]==="\r"){
if (data[i+1] === "\n"){
this.param.eol="\r\n";
}else{
this.param.eol="\r";
for (var i = 0; i < data.length; i++) {
if (data[i] === "\r") {
if (data[i + 1] === "\n") {
this.param.eol = "\r\n";
} else {
this.param.eol = "\r";
}
return this.param.eol;
}else if (data[i]==="\n"){
this.param.eol="\n";
} else if (data[i] === "\n") {
this.param.eol = "\n";
return this.param.eol;
}
}
this.param.eol=eol;
this.param.eol = eol;
}

@@ -371,11 +379,11 @@

};
Converter.prototype.fromFile = function(filePath, cb) {
Converter.prototype.fromFile = function (filePath, cb) {
var fs = require('fs');
var rs=null;
this.wrapCallback(cb, function() {
if (rs && rs.destroy){
var rs = null;
this.wrapCallback(cb, function () {
if (rs && rs.destroy) {
rs.destroy();
}
});
fs.exists(filePath, function(exist) {
fs.exists(filePath, function (exist) {
if (exist) {

@@ -385,3 +393,3 @@ rs = fs.createReadStream(filePath);

} else {
this.emit('error',new Error("File not exist"))
this.emit('error', new Error("File not exist"))
}

@@ -391,16 +399,14 @@ }.bind(this));

}
Converter.prototype.fromStream=function(readStream,cb){
if (cb && typeof cb ==="function"){
Converter.prototype.fromStream = function (readStream, cb) {
if (cb && typeof cb === "function") {
this.wrapCallback(cb);
}
process.nextTick(function(){
readStream.pipe(this);
}.bind(this))
readStream.pipe(this);
return this;
}
Converter.prototype.transf=function(func){
this.transform=func;
Converter.prototype.transf = function (func) {
this.transform = func;
return this;
}
Converter.prototype.fromString = function(csvString, cb) {
Converter.prototype.fromString = function (csvString, cb) {
if (typeof csvString != "string") {

@@ -410,6 +416,6 @@ return cb(new Error("Passed CSV Data is not a string."));

if (cb && typeof cb === "function") {
this.wrapCallback(cb, function() {
this.wrapCallback(cb, function () {
});
}
process.nextTick(function(){
process.nextTick(function () {
this.end(csvString)

@@ -419,9 +425,9 @@ }.bind(this))

};
Converter.prototype.wrapCallback = function(cb, clean) {
Converter.prototype.wrapCallback = function (cb, clean) {
if (clean === undefined){
clean=function(){}
if (clean === undefined) {
clean = function () { }
}
if (cb && typeof cb ==="function"){
this.once("end_parsed", function(res) {
if (cb && typeof cb === "function") {
this.once("end_parsed", function (res) {
if (!this.hasError) {

@@ -432,5 +438,5 @@ cb(null, res);

}
this.once("error", function(err) {
this.hasError=true;
if (cb && typeof cb ==="function"){
this.once("error", function (err) {
this.hasError = true;
if (cb && typeof cb === "function") {
cb(err);

@@ -437,0 +443,0 @@ }

@@ -12,3 +12,11 @@ var getEol=require("./getEol");

var partial=lines.pop();
return {lines:lines,partial:partial};
// if (param.ignoreEmpty){
// var trimmedLines=[];
// for (var i=0;i<lines.length;i++){
// trimmedLines.push(lines[i].trim())
// }
// return {lines:trimmedLines,partial:partial};
// }else{
return {lines:lines,partial:partial};
// }
}

@@ -80,3 +80,2 @@ var parserMgr = require("./parserMgr.js");

}
var _ = require('lodash')
function convertRowToJson(row, headRow, param) {

@@ -136,2 +135,3 @@ var hasValue = false;

function setPath(json, path, value) {
var _set = require('lodash/set')
var pathArr = path.split('.')

@@ -141,3 +141,3 @@ if (pathArr.length === 1) {

} else {
_.set(json, path, value)
_set(json, path, value)
}

@@ -188,3 +188,3 @@ }

if (isNaN(rtn)) {
return 0;
return item;
}

@@ -191,0 +191,0 @@ return rtn;

@@ -21,3 +21,3 @@ {

],
"version": "1.1.2",
"version": "1.1.3",
"keywords": [

@@ -24,0 +24,0 @@ "csv",

var Converter = require("../libs/core/Converter.js");
var csv=require('../')
var assert = require("assert");

@@ -198,3 +199,3 @@ var fs = require("fs");

assert(d.column7 === "1234");
assert(d.column8 === 0);
assert(d.column8 === "abcd");
assert(d.column9 === true);

@@ -285,3 +286,3 @@ assert(d.column10[0]===23);

var rs = fs.createReadStream(testData);
var st = rs.pipe(new Converter({ignoreEmpty:true}));
var st = rs.pipe(csv({ignoreEmpty:true}));
st.on("end_parsed",function (res){

@@ -288,0 +289,0 @@ var j = res[0];

@@ -355,2 +355,17 @@ var Converter = require("../libs/core/Converter.js");

})
it("should process long header", function (done) {
var testData = __dirname + "/data/longHeader";
var rs = fs.createReadStream(testData,{highWaterMark:100});
var numofrow = 0;
var numofjson = 0;
csv({},{highWaterMark:100})
.fromStream(rs)
.on("json", function (res) {
console.log(res)
assert.equal(res.Date,'8/26/16')
})
.on("end", function () {
done();
})
})
});

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is too big to display

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc