csvtojson - npm Package Compare versions

Comparing version 0.4.6 to 0.4.7

libs/core/Converter.js

@@ -23,7 +23,7 @@ var util = require("util");
     fork: false, //use another CPU core to convert the csv stream
-    noheader:false, //indicate if first line of CSV file is header or not.
-    headers:null, //an array of header strings. If noheader is false and headers is array, csv header will be ignored.
+    noheader: false, //indicate if first line of CSV file is header or not.
+    headers: null, //an array of header strings. If noheader is false and headers is array, csv header will be ignored.
     flatKeys: false, // Don't interpret dots and square brackets in header fields as nested object or array identifiers at all.
-    maxRowLength:0, //the max character a csv row could have. 0 means infinite. If max number exceeded, parser will emit "error" of "row_exceed". if a possibly corrupted csv data provided, give it a number like 65535 so the parser wont consume memory. default: 0
-    checkColumn:false //whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false
+    maxRowLength: 0, //the max character a csv row could have. 0 means infinite. If max number exceeded, parser will emit "error" of "row_exceed". if a possibly corrupted csv data provided, give it a number like 65535 so the parser wont consume memory. default: 0
+    checkColumn: false //whether check column number of a row is the same as headers. If column number mismatched headers number, an error of "mismatched_column" will be emitted.. default: false
   };
@@ -44,3 +44,2 @@ if (params && typeof params === "object") {
   this.started = false;
-  this._callback = null;
   this.recordNum = 0;
@@ -59,9 +58,4 @@ //this._pipe(this.lineParser).pipe(this.processor);
     this.emit("end_parsed", finalResult);
-    if (typeof this._callback === "function") {
-      var func = this._callback;
-      this._callback = null;
-      func(null, finalResult);
-    }
   }.bind(this));
-  this.on("error",function(){});
+  this.on("error", function() {});
   return this;
@@ -78,4 +72,4 @@ }
   this.child.stdout.on("data", function(d, e) {
-    this.push(d,e);
-      // this.emit("record_parsed");
+    this.push(d, e);
+    // this.emit("record_parsed");
   }.bind(this));
@@ -86,7 +80,7 @@ this.child.on("message", function(msg) {
       //var recs = msg.arguments;
-      var args=msg.arguments;
+      var args = msg.arguments;
       //console.log(recs);
       //var recs=args[0];
       //for (var i=0;i<recs.length;i++){
-        //this.emit("record_parsed", recs[i][0], recs[i][1], recs[i][2]);
+      //this.emit("record_parsed", recs[i][0], recs[i][1], recs[i][2]);
       //}
@@ -97,6 +91,6 @@ this.emit("record_parsed", args[0], args[1], args[2]);
       this.push(new Buffer(args[0]), args[1]);
-    }else if (msg.action==="error"){
-      var args=msg.arguments;
+    } else if (msg.action === "error") {
+      var args = msg.arguments;
       args.unshift("error");
-      this.emit.apply(this,args);
+      this.emit.apply(this, args);
     }
@@ -120,17 +114,12 @@ }.bind(this));
 Converter.prototype.initNoFork = function() {
-  function onError(){
-      var args=Array.prototype.slice.call(arguments,0);
-      args.unshift("error");
-      this.hasError=true;
-      this.emit.apply(this,args);
-      if (typeof this._callback === "function") {
-        var func = this._callback;
-        this._callback = null;
-        func(args, []);
-      }
+  function onError() {
+    var args = Array.prototype.slice.call(arguments, 0);
+    args.unshift("error");
+    this.hasError = true;
+    this.emit.apply(this, args);
   };
   this.lineParser = new CSVLine(this.param);
-  this.lineParser.on("error",onError.bind(this));
+  this.lineParser.on("error", onError.bind(this));
   this.processor = new Processor(this.param);
-  this.processor.on("error",onError.bind(this));
+  this.processor.on("error", onError.bind(this));
   var syncWorker = new Worker(this.param, true);
@@ -141,3 +130,3 @@ // syncWorker.on("error",onError);
     for (var i = 1; i < this.param.workerNum; i++) {
-      var worker=new Worker(this.param, false);
+      var worker = new Worker(this.param, false);
       // worker.on("error",onError);
@@ -197,3 +186,3 @@ this.processor.addWorker(worker);
   }
-  this.lineParser.write(data, encoding,cb);
+  this.lineParser.write(data, encoding, cb);
   //this.push(data,encoding);
@@ -209,3 +198,3 @@ // cb();
 Converter.prototype._transformFork = function(data, encoding, cb) {
-  this.child.stdin.write(data, encoding,cb);
+  this.child.stdin.write(data, encoding, cb);
 }
@@ -227,6 +216,21 @@ Converter.prototype._flushFork = function(cb) {
 };
+Converter.prototype.fromFile = function(filePath, cb) {
+  var fs = require('fs');
+  fs.exists(filePath, function(exist) {
+    if (exist) {
+      var rs = fs.createReadStream(filePath);
+      rs.pipe(this);
+      this.wrapCallback(cb,function(){
+        fs.destroy();
+      });
+    } else {
+      cb(new Error(filePath + " cannot be found."));
+    }
+  }.bind(this));
+  return this;
+}
 Converter.prototype.fromString = function(csvString, cb) {
   var rs = new Readable();
-  var offset=0;
-  if (typeof csvString !="string"){
+  var offset = 0;
+  if (typeof csvString != "string") {
     return cb(new Error("Passed CSV Data is not a string."));
@@ -236,6 +240,6 @@ }
     // console.log(offset,len,csvString.length);
-    var sub=csvString.substr(offset,len);
+    var sub = csvString.substr(offset, len);
     this.push(sub);
-    offset+=len;
-    if (offset>=csvString.length){
+    offset += len;
+    if (offset >= csvString.length) {
       this.push(null);
@@ -246,7 +250,20 @@ }
   if (cb && typeof cb === "function") {
-    this._callback = cb;
+    this.wrapCallback(cb,function(){
+      rs.pause();
+    });
   }
-
+  return this;
 };
+Converter.prototype.wrapCallback = function(cb,clean) {
+  this.once("end_parsed", function(res) {
+    if (!this.hasError) {
+      cb(null, res);
+    }
+  }.bind(this));
+  this.once("error", function(err) {
+    cb(Array.prototype.join.call(arguments, ", "));
+    clean();
+  });
+}
 
 module.exports = Converter;

libs/core/CSVLine.js

@@ -76,3 +76,3 @@ /**
     this._recordBuffer = '';
-    this.push(data, "utf8");
+    this.push(data.trim(), "utf8");
     this.rowIndex++;
@@ -79,0 +79,0 @@ // if (this.rowIndex % 10000 ===0){

libs/core/Processor.js

@@ -76,3 +76,7 @@ /**
       } else {
-        this.emit("record_parsed", resultRow, row, index - 1);
+        if (resultRow){
+          this.emit("record_parsed", resultRow, row, index - 1);
+        }else{
+          //Empty row detedted. skip
+        }
         //this.push(JSON.stringify([resultRow,row,obj.rowIndex]),"utf8");
@@ -79,0 +83,0 @@ }

libs/core/Worker.js

@@ -42,3 +42,7 @@ /**
     }else{
-      cb(null, res.resultRow, res.row, res.index);
+      if (res){
+        cb(null, res.resultRow, res.row, res.index);
+      }else{
+        cb(null);
+      }
     }
@@ -45,0 +49,0 @@ });

libs/core/workerRunner.js

@@ -8,7 +8,7 @@ var parserMgr = require("./parserMgr.js");
     inst[action](m, function(err, res) {
-      if (!res){
-        res={};
+      if (!res) {
+        res = {};
       }
       if (err) {
-        res.error=err;
+        res.error = err;
       }
@@ -28,18 +28,19 @@ res.action = m.action;
 
-  function genConstHeadRow(msg,cb){
-      var Parser=require("./parser");
-      var number=msg.number;
-      parseRules=[];
-      headRow=[];
-      while (number>0){
-        var p=new Parser("field"+number,/.*/,function(params){
-          var name=this.getName();
-          params.resultRow[name]=params.item;
-        },true);
-        parseRules.unshift(p);
-        headRow.unshift(p.getName());
-        number--;
-      }
-      cb();
+  function genConstHeadRow(msg, cb) {
+    var Parser = require("./parser");
+    var number = msg.number;
+    parseRules = [];
+    headRow = [];
+    while (number > 0) {
+      var p = new Parser("field" + number, /.*/, function(params) {
+        var name = this.getName();
+        params.resultRow[name] = params.item;
+      }, true);
+      parseRules.unshift(p);
+      headRow.unshift(p.getName());
+      number--;
+    }
+    cb();
   }
+
   function processHeadRow(msg, cb) {
@@ -58,6 +59,7 @@ headRow = msg.row;
     var row = utils.rowSplit(data, param.delimiter, param.quote, param.trim);
-    if (param.checkColumn && row.length !=parseRules.length ){
-      return cb("Error: column_mismatched. Data: "+data+". Row index: "+index);
+    if (param.checkColumn && row.length != parseRules.length) {
+      return cb("Error: column_mismatched. Data: " + data + ". Row index: " + index);
     }
     var resultRow = {};
+    var hasValue = false;
     for (i = 0; i < parseRules.length; i++) {
@@ -68,2 +70,3 @@ item = row[i];
       }
+      hasValue = true;
       parser = parseRules[i];
@@ -81,7 +84,11 @@ head = headRow[i];
     }
-    cb(null, {
-      resultRow: resultRow,
-      row: row,
-      index: index
-    });
+    if (hasValue) {
+      cb(null, {
+        resultRow: resultRow,
+        row: row,
+        index: index
+      });
+    } else {
+      cb();
+    }
 
@@ -92,5 +99,5 @@ }
     processRow: processRow,
-    genConstHeadRow:genConstHeadRow
+    genConstHeadRow: genConstHeadRow
   }
 }
-module.exports=init;
+module.exports = init;

package.json

@@ -21,3 +21,3 @@ {
   ],
-  "version": "0.4.6",
+  "version": "0.4.7",
   "keywords": [
@@ -24,0 +24,0 @@ "csv",

readme.md

@@ -48,3 +48,5 @@ # CSVTOJSON
 var Converter = require("csvtojson").Converter;
-var converter = new Converter({});
+var converter = new Converter({
+  checkType:false //turn off auto type check to increase performance
+});
 
@@ -614,2 +616,7 @@ //end_parsed will be emitted once parsing finished
 
+## 0.4.7
+* ignoreEmpty now ignores empty rows as well
+* optimised performance
+* added fromFile method
+
 ## 0.4.4
@@ -616,0 +623,0 @@ * Add error handling for corrupted CSV data

test/testCSVConverter.js

@@ -113,2 +113,11 @@ var Converter = require("../libs/core/Converter.js");
   });
+  it("should be able to convert csv string with error", function (done) {
+    var testData = __dirname + "/data/dataWithUnclosedQuotes";
+    var data = fs.readFileSync(testData).toString();
+    var csvConverter = new Converter();
+    csvConverter.fromString(data, function (err, jsonObj) {
+      assert(err);
+      done();
+    });
+  });
   it("should be able to convert csv string without callback provided", function (done) {
@@ -270,2 +279,3 @@ var testData = __dirname + "/data/testData";
       var j = res[0];
+      assert(res.length  === 2);
       assert (j.col2.length === 1);
@@ -275,2 +285,3 @@ assert(j.col2[0] === "d3");
       assert(j.col4.col5 === "world");
+      assert(res[1].col1==="d2");
       done();
@@ -277,0 +288,0 @@ });

test/testCSVConverter2.js

@@ -61,2 +61,13 @@ var Converter = require("../libs/core/Converter.js");
   });
+  it ("should parse fromFile",function(done){
+    var csvFile = __dirname + "/data/large-csv-sample.csv";
+    var conv = new Converter({
+      workerNum: 3
+    });
+    conv.fromFile(csvFile, function(err, res) {
+      assert(!err);
+      assert(res.length === 5290);
+      done();
+    });
+  });
   // it ("should convert big csv",function(done){
@@ -63,0 +74,0 @@ //   // var rs=fs.createReadStream(__dirname+"/data/large-csv-sample.csv");

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

362814

increased by0.44%

Lines of code

2248

increased by2.46%

Number of lines in readme file

696

increased by1.02%

Worsened metrics

Number of low supply chain risk alerts

12

increased by9.09%

No dependency changes