Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

csvtojson

Package Overview
Dependencies
Maintainers
1
Versions
82
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

csvtojson - npm Package Compare versions

Comparing version 0.2.2 to 0.3.0

libs/core/Result.js

89

libs/core/csvConverter.js
module.exports=csvAdv;
//implementation
var csv=require("csv");//see http://www.adaltas.com/projects/node-csv/from.html
var parserMgr=require("./parserMgr.js");
var utils=require("util");
//it is a bridge from csv component to our parsers
function csvAdv(constructResult){
if (constructResult !== false){
constructResult=true;
var Transform=require("stream").Transform;
var Result=require("./Result");
function csvAdv(params){
Transform.call(this);
var _param={
"constructResult":true,
"delimiter":","
}
var instance= csv.apply(this);
if (params && typeof params =="object"){
for (var key in params){
_param[key]=params[key];
}
}else if (typeof params =="boolean"){ //backcompatible with older version
console.warn("Parameter should be a JSON object like {'constructResult':false}");
_param.constructResult=params;
}
this.param=_param;
this.parseRules=[];
this.resultObject={csvRows:[]};
this.resultObject=new Result();
if (this.param.constructResult){
this.pipe(this.resultObject);
}
this.headRow=[];
var that=this;
instance.on("record",function(row,index){
this._buffer="";
this.rowIndex=0;
var self=this;
var started=false;
self.on("record",function(rowStr,index,lastLine){
var row=rowStr.split(self.param.delimiter);
if (index ==0){
that._headRowProcess(row);
}else{
self._headRowProcess(row);
self.push("[\n");
}else if(rowStr.length>0){
var resultRow={};
that._rowProcess(row,index,resultRow);
if (constructResult){
that.resultObject.csvRows.push(resultRow);
self._rowProcess(row,index,resultRow);
self.emit("record_parsed",resultRow,row,index);
if (started===true ){
self.push(",\n");
}
instance.emit("record_parsed",resultRow,row,index);
self.push(JSON.stringify(resultRow));
started=true;
}
});
instance.on("end",function(){
instance.emit("end_parsed",that.resultObject,that.resultObject.csvRows);
self.on("end",function(){
self.emit("end_parsed",self.param.constructResult?self.resultObject.getBuffer():{});
});
return instance;
return this;
};
utils.inherits(csvAdv,Transform);
csvAdv.prototype._transform=function(data,encoding,cb){
var self=this;
if (encoding=="buffer"){
encoding="utf8";
}
this._buffer+=data.toString(encoding);
if (this._buffer.indexOf("\n")>-1){
var arr=this._buffer.split("\n");
while(arr.length>1){
var data=arr.shift();
if (data.length>0){
this.emit("record",data,this.rowIndex++);
}
}
this._buffer=arr[0];
}
cb();
};
csvAdv.prototype._flush=function(cb){
if (this._buffer.length!=0){ //emit last line
this.emit("record",this._buffer,this.rowIndex++,true);
}
this.push("\n]");
cb();
};
csvAdv.prototype._headRowProcess=function(headRow){
this.headRow=headRow;
this.parseRules=parserMgr.initParsers(headRow);
}
};
csvAdv.prototype._rowProcess=function(row,index,resultRow){

@@ -57,4 +102,4 @@ for (var i=0;i<this.parseRules.length;i++){

}
}
};

62

libs/interfaces/web/webServer.js

@@ -5,4 +5,5 @@ //module interfaces

//implementation
var express=require("express");
var expressApp=express();
// var express=require("express");
// var expressApp=express();
var http=require("http");
var CSVConverter=require("../../core").Converter;

@@ -13,5 +14,6 @@ var defaultArgs={

}
var server=null;
function applyWebServer(app,url){
app.post(url,_POSTData);
console.error("applyWebServer is deprecated. Use core you create your own handler.");
}

@@ -30,43 +32,27 @@ function startWebServer(args){

}
server=http.createServer();
server.on("request",function(req,res){
if (req.url==serverArgs.urlPath && req.method =="POST"){
_POSTData(req,res);
}else{
res.end("Please post data to: "+serverArgs.urlPath);
}
});
server.listen(serverArgs.port);
//expressApp.use(express.bodyParser());
expressApp.post(serverArgs.urlPath,_POSTData);
expressApp.get("/",function(req,res){
res.end("POST to "+serverArgs.urlPath+" with CSV data to get parsed.");
});
expressApp.listen(serverArgs.port);
// expressApp.post(serverArgs.urlPath,_POSTData);
// expressApp.get("/",function(req,res){
// res.end("POST to "+serverArgs.urlPath+" with CSV data to get parsed.");
// });
// expressApp.listen(serverArgs.port);
console.log("CSV Web Server Listen On:"+serverArgs.port);
console.log("POST to "+serverArgs.urlPath+" with CSV data to get parsed.");
return expressApp;
return server;
}
function _POSTData(req,res){
var csvString="";
req.setEncoding('utf8');
req.on("data",function(chunk){
csvString+=chunk;
});
req.on("end",function(){
_ParseString(csvString,function(err,JSONData){
if (err){
console.error(err)
}else{
res.json(JSONData);
}
});
});
var converter=new CSVConverter({constructResult:false});
req.pipe(converter).pipe(res);
}
function _ParseString(csvString,cb){
var converter=new CSVConverter();
converter.on("end_parsed",function(JSONData){
cb(null,JSONData);
});
converter.on("error",function(err){
cb(err);
});
converter.from(csvString);
}
function _ParseFile(filePath,cb){
}

6

package.json

@@ -17,7 +17,3 @@ {

],
"version":"0.2.2",
"dependencies":{
"express":"3.4.4",
"csv":"0.3.6"
},
"version":"0.3.0",
"keywords":[

@@ -24,0 +20,0 @@ "csv",

131

readme.md
#CSV2JSON
All you need nodejs csv to json converter. Support big json data, CLI, server. can be easily plugged and used in other nodejs app.
All you need nodejs csv to json converter. Support big json data, CLI, server. Easily convert your CSV data to nested JSON object.
#IMPORTANT!!
Since version 0.3, the core class of csvtojson has been inheriting from stream.Transform class. Therefore, it will behave like a normal Stream object and CSV features will not be available any more. Now the usage is like:
```js
//Converter Class
var Converter=require("csvtojson").core.Converter;
var fs=require("fs");
var csvFileName="./myCSVFile";
var fileStream=fs.createReadStream(csvFileName);
//new converter instance
var csvConverter=new Converter({constructResult:true});
//end_parsed will be emitted once parsing finished
csvConverter.on("end_parsed",function(jsonObj){
console.log(jsonObj); //here is your result json object
});
//read from file
fileStream.pipe(csvConverter);
```
#Change Log
##version 0.3
* Removed all dependencies
* Deprecated applyWebServer
* Added construct parameter for Converter Class
* Converter Class now works as a proper stream object
##Menu

@@ -18,2 +48,3 @@ * [Installation](#installation)

* [Big CSV File Streaming](#big-csv-file)
* [Process Big CSV File in CLI](#convert-big-csv-file-with-command-line-tool)

@@ -33,4 +64,4 @@ GitHub: https://github.com/Keyang/node-csvtojson

* Multiple interfaces (webservice, command line)
##Usage

@@ -74,3 +105,3 @@

#### Quick Start
The core of the tool is Converter class. It is based on node-csv library (version 0.3.6). Therefore it has all features of [node-csv](http://www.adaltas.com/projects/node-csv/). To start a parse, simply use following code:
~~The core of the tool is Converter class. It is based on node-csv library (version 0.3.6). Therefore it has all features of [node-csv](http://www.adaltas.com/projects/node-csv/).~~ To start a parse, simply use following code:

@@ -80,9 +111,10 @@ ```js

var Converter=require("csvtojson").core.Converter;
//CSV File Path or CSV String or Readable Stream Object
var fs=require("fs");
var csvFileName="./myCSVFile";
var fileStream=fs.createReadStream(csvFileName);
//new converter instance
var csvConverter=new Converter();
var param={};
var csvConverter=new Converter(param);
//end_parsed will be emitted once parsing finished

@@ -92,7 +124,12 @@ csvConverter.on("end_parsed",function(jsonObj){

});
//read from file
csvConverter.from(csvFileName);
fileStream.pipe(csvConverter);
```
#### Params
The parameters for Converter constructor are:
* constructResult: true/false. Whether to constrcut final json object in memory which will be populated in "end_parsed" event. Set to false if deal with huge csv data. default: true.
* delimiter: delimiter used for seperating columns. default: ","
#### Parser

@@ -137,5 +174,5 @@ CSVTOJSON allows adding customised parsers which concentrating on what to parse and how to parse.

**resultObject**: the reference of result object in JSON format. It always has a field called csvRows which is in Array format. It changes as parsing going on. e.g.
**resultObject**: the reference of result object in JSON format. It always has a field called csvRows which is in Array format. It changes as parsing going on. e.g.
```json
```json
{

@@ -161,3 +198,3 @@ "csvRows":[

var expressApp=webServer.startWebServer({
var server=webServer.startWebServer({
"port":"8801",

@@ -168,14 +205,4 @@ "urlPath":"/parseCSV"

It will return an [expressjs](http://expressjs.com/) Application. You can add your own web app content there.
~~It will return an [expressjs](http://expressjs.com/) Application. You can add your own web app content there.~~ It will return http.Server object.
If you already have an express application, simply use following code to extend your current application
```js
var webServer=require("csvtojson").interfaces.web;
//..your code to setup the application object.
webServer.applyWebServer(app, postURL); //postURL can be omitted by using default one.
```
#### Events

@@ -185,3 +212,3 @@

* end_parsed: It is emitted when parsing finished. the callback function will contain the JSON object
* end_parsed: It is emitted when parsing finished. the callback function will contain the JSON object if constructResult is set to true.
* record_parsed: it is emitted each time a row has been parsed. The callback function has following parameters: result row JSON object reference, Original row array object reference, row index

@@ -194,3 +221,3 @@

var Converter=require("csvtojson").core.Converter;
//end_parsed will be emitted once parsing finished

@@ -200,3 +227,3 @@ csvConverter.on("end_parsed",function(jsonObj){

});
//record_parsed will be emitted each time a row has been parsed.

@@ -211,9 +238,9 @@ csvConverter.on("record_parsed",function(resultRow,rawRow,rowIndex){

**Array**: For columns head start with "*array*" e.g. "*array*fieldName", this parser will combine cells data with same fieldName to one Array.
**Array**: For columns head start with "\*array\*" e.g. "\*array\*fieldName", this parser will combine cells data with same fieldName to one Array.
**Nested JSON**: For columns head start with "*json*" e.g. "*json*my.nested.json.structure", this parser will create nested nested JSON structure: my.nested.json
**Nested JSON**: For columns head start with "\*json\*" e.g. "\*json\*my.nested.json.structure", this parser will create nested nested JSON structure: my.nested.json
**Nested JSON Array**: For columns head start with "*jsonarray*" e.g. "*jsonarray*my.items", this parser will create structure like my.items[].
**Nested JSON Array**: For columns head start with "\*jsonarray\*" e.g. "\*jsonarray\*my.items", this parser will create structure like my.items[].
**Omitted column**: For columns head start with "*omit*" e.g. "*omit*id", the parser will omit the column's data.
**Omitted column**: For columns head start with "\*omit\*" e.g. "\*omit\*id", the parser will omit the column's data.

@@ -269,32 +296,34 @@ ####Example:

#### Big CSV File
csvtojson library was designed to accept big csv file converting. To avoid memory consumption, it is recommending to use read stream and write stream.
csvtojson library was designed to accept big csv file converting. To avoid memory consumption, it is recommending to use read stream and write stream.
```js
var Converter=require("csvtojson").core.Converter;
var csvConverter=new Converter(false); // The parameter false will turn off final result construction. It can avoid huge memory consumption while parsing. The trade off is final result will not be populated to end_parsed event.
var csvConverter=new Converter({constructResult:false}); // The parameter false will turn off final result construction. It can avoid huge memory consumption while parsing. The trade off is final result will not be populated to end_parsed event.
var readStream=require("fs").createReadStream("inputData.csv");
var readStream=require("fs").createReadStream("inputData.csv");
var writeStream=require("fs").createWriteStream("outpuData.json");
var started=false;
csvConverter.on("record_parsed",function(rowJSON){
if (started){
writeStream.write(",\n");
}
writeStream.write(JSON.stringify(rowJSON)); //write parsed JSON object one by one.
if (started==false){
started=true;
}
});
readStream.pipe(csvConverter).pipe(writeStream);
```
writeStream.write("[\n"); //write array symbol
The constructResult:false will tell the constructor not to combine the final result which would drain the memory as progressing. The output is piped directly to writeStream.
csvConverter.on("end_parsed",function(){
writeStream.write("\n]"); //end array symbol
});
csvConverter.from(readStream);
#### Convert Big CSV File with Command line tool
csvtojson command line tool supports streaming in big csv file and stream out json file.
It is very convenient to process any kind of big csv file. It's proved having no issue to proceed csv files over 3,000,000 lines (over 500MB) with memory usage under 30MB.
Once you have installed [csvtojson](#installation), you could use the tool with command:
```
csvtojson [path to bigcsvdata] > converted.json
```
The Converter constructor was passed in a "false" parameter which will tell the constructor not to combine the final result which would take simlar memory as the file size. The output is constructed line by line through writable stream object.
Or if you prefer streaming data in from another application:
```
cat [path to bigcsvdata] | csvtojson > converted.json
```
They will do the same job.

30

test/testCSVConverter.js

@@ -15,7 +15,7 @@ var CSVAdv=require("../libs/core/csvConverter.js");

var stream=fs.createReadStream(file);
obj.on("end",function(count){
assert(count==3);
obj.on("end_parsed",function(obj){
assert(obj.length===2);
done();
});
obj.from(stream);
stream.pipe(obj);
});

@@ -33,3 +33,3 @@

});
obj.from(stream);
stream.pipe(obj);
});

@@ -42,14 +42,14 @@

assert(result);
assert(result.csvRows.length==2);
assert(result.csvRows[0].date);
assert(result.csvRows[0].employee);
assert(result.csvRows[0].employee.name);
assert(result.csvRows[0].employee.age);
assert(result.csvRows[0].employee.number);
assert(result.csvRows[0].employee.key.length===2);
assert(result.csvRows[0].address.length===2);
assert(result.length==2);
assert(result[0].date);
assert(result[0].employee);
assert(result[0].employee.name);
assert(result[0].employee.age);
assert(result[0].employee.number);
assert(result[0].employee.key.length===2);
assert(result[0].address.length===2);
//console.log(JSON.stringify(result));
done();
});
obj.from(stream);
stream.pipe(obj);
});

@@ -62,8 +62,8 @@

assert(result);
assert(result.csvRows.length>0);
assert(result.length>0);
//console.log(JSON.stringify(result));
done();
});
obj.from(stream);
stream.pipe(obj);
});
});
.npmignore

Sorry, the diff of this file is not supported yet

bin/csvtojson

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc