csv-array - npm Package Compare versions

Comparing version 0.0.13 to 0.0.14

csv-array.js

 module.exports = {
+	maxSizeForDirectRead : 10000000,
 	parseCSV : function(fileName, callBack) {
@@ -7,3 +8,9 @@ var presentInstance = this;
 			if(exists) {
-				presentInstance.parseFile(fileName, callBack);
+				fs.stat(fileName, function(err, stats){
+					if(stats.size > presentInstance.maxSizeForDirectRead) {
+						presentInstance.parseBigFile(fileName, callBack);
+					} else {
+						presentInstance.parseFile(fileName, callBack);	
+					}
+				});
 			} else {
@@ -30,2 +37,3 @@ console.log("The provided file " + fileName + " doesn't exists or inaccessible");
 
+	// returns total data
 	getDataArray : function(dataArray) {
@@ -35,6 +43,8 @@ presentInstance = this;
 		var finalArray = [];
-		for(var index=1; index<dataArray.length; index++) {
+		var length = dataArray.length;
+		for(var index=1; index<length; index++) {
 			var tempArray = {};
 			var dataList = presentInstance.getDataFromLine(dataArray[index]);
-			for(var index2=0; index2<attributeNameArray.length; index2++) {
+			var atrributeNameArrayLength = attributeNameArray.length;
+			for(var index2=0; index2<atrributeNameArrayLength; index2++) {
 				tempArray[attributeNameArray[index2]] = dataList[index2];
@@ -45,9 +55,10 @@ }
 		return finalArray;
-
 	},
 
+	// returns data from a single line 
 	getDataFromLine : function(line) {
 		var dataArray = [];
 		var tempString="";
-		for(var index=0; index<line.length; index++) {
+		var lineLength = line.length;
+		for(var index=0; index<lineLength; index++) {
 			if(line[index]=='"') {
@@ -70,4 +81,47 @@ var index2=index+1;
 		return dataArray;
+	},
+	tempLineCounter : 0,// only used for large files
+	tempDataArray : [],// only used for large files
+	tempAttributeNameArray : [],// only used for large files
+	parseBigFile : function(fileName, callBack) {
+		var presentObject = module.exports;
+		var lblReader = require('line-by-line');
+		var readStream = new lblReader(fileName);
+
+		presentObject.tempDataArray = [];
+		presentObject.tempAttributeNameArray = [];
+		presentObject.tempLineCounter = 0;
+		
+		readStream.on('error', function(){
+			console.log("cannot read the file any more.");
+		});
+		readStream.on('line', function(line) {
+			readStream.pause();
+			presentObject.buildOutputData(line);
+			setTimeout(function() {
+				readStream.resume();
+			},50);
+			
+		});
+		readStream.on('end', function() {
+			callBack(presentObject.tempDataArray);
+		});
+	},
+
+	buildOutputData : function(line) {
+		var presentObject = module.exports;
+		if(presentObject.tempLineCounter == 0) {
+			presentObject.tempAttributeNameArray = line.split(",");
+		} else {
+			var dataArray = presentObject.getDataFromLine(line);
+			var tempObject = {};
+			var tempAttributeNameArrayLength = presentObject.tempAttributeNameArray.length;
+			for(var index=0; index<tempAttributeNameArrayLength; index++) {
+				tempObject[presentObject.tempAttributeNameArray[index]] = dataArray[index];
+			}
+			presentObject.tempDataArray.push(tempObject);
+		}
+		presentObject.tempLineCounter += 1;
 	}
 
 }

package.json

 {
   "name": "csv-array",
-  "version": "0.0.13",
-  "description": "This is a CSV parser created and made for nodeJS. Which takes a csv file and produce an array from it",
+  "version": "0.0.14",
+  "description": "Intelligent CSV parser created and made for nodeJS. Which takes a csv file and produce an array from it.",
   "main": "csv-array.js",
@@ -19,3 +19,4 @@ "scripts": {},
 	"json",
-	"parser"
+	"parser",
+	"streaming"
   ],
@@ -26,4 +27,5 @@ "author": "Sahasrangshu Guha",
   "dependencies": {
-    "fs": "0.0.2"
+    "fs": "0.0.2",
+    "line-by-line": "~0.1.3"
   }
 }

README.md

@@ -5,4 +5,10 @@ #csv-array
 ## Dependencies
-This package got only one dependency of "fs". You may install that first using following command
+This package got only two dependencies of "fs", and "line-by-line".
 
+## Change log
+* Performence improvement
+* Better runtime memory management
+  * For small files memory usage improved
+  * For large files the option of streaming rather than reading all at once, added
+
 ## Usage Guide
@@ -9,0 +15,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

7351

increased by39.65%

Lines of code

113

increased by82.26%

Number of lines in readme file

70

increased by9.38%

Worsened metrics

Dependency count

2

increased by100%

Dependency changes

• + Addedline-by-line@~0.1.3

• + Addedline-by-line@0.1.6(transitive)