Comparing version 0.5.1 to 0.5.2
@@ -17,6 +17,3 @@ /* | ||
module.exports = function(grunt) { | ||
grunt.loadNpmTasks('grunt-contrib-jshint'); | ||
grunt.loadNpmTasks('grunt-mocha-test'); | ||
grunt.loadNpmTasks('grunt-contrib-watch'); | ||
grunt.loadNpmTasks('grunt-newer'); | ||
require('load-grunt-tasks')(grunt); | ||
grunt.loadTasks('./devops/grunt-wiki'); | ||
@@ -27,2 +24,18 @@ var files = ['Gruntfile.js', 'libs/**/*.js', 'libs/**/**/*.js', 'test/*.js', 'bin/csvtojson', | ||
grunt.initConfig({ | ||
uglify: { | ||
client: { | ||
options: { | ||
mangle: true, | ||
banner:"/*Automatically Generated. Do not modify.*/\n" | ||
}, | ||
src: "./dist/csvtojson.js", | ||
dest: "./dist/csvtojson.min.js", | ||
} | ||
}, | ||
browserify: { | ||
dist: { | ||
src: "./browser_index.js", | ||
dest: "./dist/csvtojson.js" | ||
} | ||
}, | ||
jshint: { | ||
@@ -29,0 +42,0 @@ all: { |
@@ -325,3 +325,3 @@ var util = require("util"); | ||
this.wrapCallback(cb, function() { | ||
fs.destroy(); | ||
rs.destroy(); | ||
}); | ||
@@ -328,0 +328,0 @@ } else { |
@@ -0,1 +1,3 @@ | ||
var explicitTypes = ["number", "string"]; | ||
function Parser(name, regExp, parser, processSafe) { | ||
@@ -17,2 +19,36 @@ this.name = typeof name === "undefined" ? "Default" : name; | ||
} | ||
var numReg = /^[-+]?[0-9]*\.?[0-9]+$/; | ||
Parser.prototype.convertType = function(item) { | ||
var type=this.type; | ||
if (type === 'number') { | ||
var rtn = parseFloat(item); | ||
if (isNaN(rtn)) { | ||
return 0; | ||
} else { | ||
return rtn; | ||
} | ||
} else if (this.param && this.param.checkType && type === '') { | ||
var trimed = item.trim(); | ||
if (numReg.test(trimed)) { | ||
return parseFloat(trimed); | ||
} else if (trimed.length === 5 && trimed.toLowerCase() === "false") { | ||
return false; | ||
} else if (trimed.length === 4 && trimed.toLowerCase() === "true") { | ||
return true; | ||
} else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}") { | ||
try { | ||
return JSON.parse(trimed); | ||
} catch (e) { | ||
return item; | ||
} | ||
} else { | ||
return item; | ||
} | ||
} | ||
return item; | ||
} | ||
Parser.prototype.setParam = function(param) { | ||
this.param = param; | ||
} | ||
Parser.prototype.test = function(str) { | ||
@@ -30,2 +66,5 @@ return this.regExp && this.regExp.test(str); | ||
this.headStr = head.replace(this.regExp, ''); | ||
if (!this.headStr) { | ||
this.headStr = "Unknown Header"; | ||
} | ||
return this.getHeadStr(); | ||
@@ -37,2 +76,23 @@ } | ||
}; | ||
Parser.prototype.initHead = function(columnTitle) { | ||
this.head = columnTitle; | ||
var wholeHead = columnTitle.replace(this.regExp, ''); | ||
//init type && headStr | ||
var splitArr = wholeHead.split("#!"); | ||
if (splitArr.length === 1) { //no explicit type | ||
this.headStr = splitArr[0]; | ||
} else { | ||
var type = splitArr.shift(); | ||
if (explicitTypes.indexOf(type.toLowerCase()) > -1) { | ||
this.type = type; | ||
this.headStr = splitArr.join("#!"); | ||
} else { //no explicit type | ||
this.headStr = wholeHead; | ||
} | ||
} | ||
if (!this.headStr) { | ||
this.headStr = wholeHead ? wholeHead : "Unknown Head"; | ||
} | ||
} | ||
Parser.prototype.clone = function() { | ||
@@ -39,0 +99,0 @@ var obj = Object.create(this); |
@@ -11,20 +11,5 @@ //implementation | ||
} | ||
function splitTitle (columnTitle){ | ||
var splitArr = columnTitle.split("#"); | ||
var rtn; | ||
if (splitArr.length === 1){ | ||
splitArr.unshift(""); | ||
return splitArr; | ||
} else if (splitArr.length > 2) { | ||
rtn = []; | ||
rtn.push(splitArr.shift()); | ||
rtn.push(splitArr.join("#")); | ||
return rtn; | ||
} | ||
return splitArr; | ||
} | ||
function getParser (columnTitle, checkType) { | ||
function getParser (columnTitle, param) { | ||
var inst, parser; | ||
var type = ""; | ||
function getParserByName (parserName, columnTitle) { | ||
function getParserByName (parserName) { | ||
var parser; | ||
@@ -38,3 +23,2 @@ registeredParsers.forEach(function(p){ | ||
var inst = parser.clone(); | ||
inst.head = columnTitle; | ||
return inst; | ||
@@ -45,7 +29,2 @@ } | ||
columnTitle = columnTitle ? columnTitle : ''; | ||
if (checkType){ | ||
var split = splitTitle(columnTitle); | ||
type = split[0]; | ||
columnTitle = split[1]; | ||
} | ||
registeredParsers.forEach(function(p){ | ||
@@ -62,3 +41,4 @@ if (p.test(columnTitle)){ | ||
} | ||
inst.type = type; | ||
inst.setParam(param); | ||
inst.initHead(columnTitle); | ||
return inst; | ||
@@ -74,6 +54,6 @@ } | ||
function initParsers (row, checkType) { | ||
function initParsers (row, param) { | ||
var parsers = []; | ||
row.forEach(function (columnTitle) { | ||
parsers.push(getParser(columnTitle, checkType)); | ||
parsers.push(getParser(columnTitle, param)); | ||
}); | ||
@@ -80,0 +60,0 @@ return parsers; |
@@ -27,7 +27,9 @@ var parserMgr = require("./parserMgr.js"); | ||
function getConstParser(number) { | ||
return new Parser("field" + number, /.*/, function(params) { | ||
function getConstParser(number,param) { | ||
var inst= new Parser("field" + number, /.*/, function(params) { | ||
var name = this.getName(); | ||
params.resultRow[name] = params.item; | ||
}, true); | ||
inst.setParam(param); | ||
return inst; | ||
} | ||
@@ -44,3 +46,3 @@ | ||
while (number > 0) { | ||
var p = getConstParser(number); | ||
var p = getConstParser(number,param); | ||
parseRules.unshift(p); | ||
@@ -63,3 +65,3 @@ headRow.unshift(p.getName()); | ||
if (row.length > 0) { | ||
parseRules = parserMgr.initParsers(row, param.checkType); | ||
parseRules = parserMgr.initParsers(row, param); | ||
} | ||
@@ -116,3 +118,3 @@ cb(null, {}); | ||
if (!parser) { | ||
parser = parseRules[i] = getConstParser(i + 1); | ||
parser = parseRules[i] = getConstParser(i + 1,param); | ||
} | ||
@@ -122,6 +124,6 @@ head = headRow[i]; | ||
head = headRow[i] = "field" + (i + 1); | ||
parser.head = head; | ||
parser.initHead(head); | ||
} | ||
if (param.checkType){ | ||
item=parseParamType(parser.type,item,param); | ||
item=parser.convertType(item); | ||
} | ||
@@ -150,31 +152,2 @@ parser.parse({ | ||
var numReg=/^[-+]?[0-9]*\.?[0-9]+$/; | ||
function parseParamType(type, item) { | ||
if (type === 'number') { | ||
var rtn = parseFloat(item); | ||
if (isNaN(rtn)) { | ||
return 0; | ||
} else { | ||
return rtn; | ||
} | ||
} else if (type === '') { | ||
var trimed = item.trim(); | ||
if (numReg.test(trimed)) { | ||
return parseFloat(trimed); | ||
} else if (trimed.length === 5 && trimed.toLowerCase() === "false") { | ||
return false; | ||
} else if (trimed.length === 4 && trimed.toLowerCase() === "true") { | ||
return true; | ||
} else if (trimed[0] === "{" && trimed[trimed.length - 1] === "}") { | ||
try { | ||
return JSON.parse(trimed); | ||
} catch (e) { | ||
return item; | ||
} | ||
} else { | ||
return item; | ||
} | ||
} | ||
return item; | ||
} | ||
return { | ||
@@ -181,0 +154,0 @@ processHeadRow: processHeadRow, |
@@ -21,3 +21,3 @@ { | ||
], | ||
"version": "0.5.1", | ||
"version": "0.5.2", | ||
"keywords": [ | ||
@@ -46,4 +46,5 @@ "csv", | ||
"grunt": "^0.4.5", | ||
"grunt-browserify": "^4.0.1", | ||
"grunt-contrib-jshint": "^0.11.2", | ||
"grunt-contrib-uglify": "^0.9.1", | ||
"grunt-contrib-uglify": "^0.11.0", | ||
"grunt-contrib-watch": "^0.6.1", | ||
@@ -55,2 +56,3 @@ "grunt-git": "^0.3.5", | ||
"imgur": "^0.1.5", | ||
"load-grunt-tasks": "^3.4.0", | ||
"mocha": "^2.2.5" | ||
@@ -57,0 +59,0 @@ }, |
@@ -12,5 +12,5 @@ # CSVTOJSON | ||
# Version 0.5 | ||
# Demo | ||
Version 0.5 contains big refactor especially for performance. The parser is like **7 times** faster than version 0.4. | ||
[Here](http://keyangxiang.com/csvtojson/) is a free online csv to json service ultilising latest csvtojson module. | ||
@@ -529,3 +529,3 @@ ## Menu | ||
CSV header column can explicitly define the type of the field. | ||
Simply add type before column name with a hash symbol (#). | ||
Simply add type before column name with a hash and exclaimation (#!). | ||
@@ -535,3 +535,2 @@ ### Supported types: | ||
* number | ||
* date (Not supported since 0.3.21) | ||
@@ -542,4 +541,4 @@ ### Define Type | ||
```csv | ||
string#appNumber, string#finished, string#msg | ||
201401010002, true, {"hello":"world","total":23} | ||
string#!appNumber, string#!finished, *flat*string#!user.msg, unknown#!msg | ||
201401010002, true, {"hello":"world","total":23},a message | ||
``` | ||
@@ -552,3 +551,3 @@ The data will be converted to: | ||
"finished":"true", | ||
"msg":"{\"hello\":\"world\",\"total\":23}" | ||
"user.msg":"{\"hello\":\"world\",\"total\":23}" | ||
} | ||
@@ -732,2 +731,7 @@ ``` | ||
## 0.5.2 | ||
* Changed type separator from # to #! | ||
* Fixed bugs | ||
## 0.5.0 | ||
@@ -734,0 +738,0 @@ |
@@ -192,3 +192,3 @@ var Converter = require("../libs/core/Converter.js"); | ||
assert(typeof d.column2 === "string"); | ||
assert(d.colume4 === "someinvaliddate"); | ||
assert(d["date#!colume4"] === "someinvaliddate"); | ||
assert(d.column5.hello === "world"); | ||
@@ -201,2 +201,3 @@ assert(d.column6 === '{"hello":"world"}'); | ||
assert(d.column10[1]===31); | ||
assert(d["name#!"]==="sss"); | ||
}); | ||
@@ -217,11 +218,12 @@ csvConverter.on("end_parsed",function (){ | ||
assert(typeof d.column2 === "string"); | ||
assert( d["date#column3"] === "2012-01-01"); | ||
assert(d["date#colume4"] === "someinvaliddate"); | ||
assert( d["date#!column3"] === "2012-01-01"); | ||
assert(d["date#!colume4"] === "someinvaliddate"); | ||
assert(d.column5 === '{"hello":"world"}'); | ||
assert(d["string#column6"] === '{"hello":"world"}'); | ||
assert(d["string#column7"] === "1234"); | ||
assert(d["number#column8"] === "abcd"); | ||
assert(d["column6"] === '{"hello":"world"}'); | ||
assert(d["column7"] === "1234"); | ||
assert(d["column8"] === "abcd"); | ||
assert(d.column9 === "true"); | ||
assert(d.column10[0]==="23"); | ||
assert(d.column10[1]==="31"); | ||
assert(d["name#!"]==="sss"); | ||
}); | ||
@@ -228,0 +230,0 @@ csvConverter.on("end_parsed",function (){ |
@@ -71,2 +71,12 @@ var Converter = require("../libs/core/Converter.js"); | ||
}); | ||
it ("should fromFile should emit error",function(done){ | ||
var csvFile = __dirname + "/data/dataWithUnclosedQuotes"; | ||
var conv = new Converter({ | ||
workerNum: 1 | ||
}); | ||
conv.fromFile(csvFile, function(err, res) { | ||
assert(err); | ||
done(); | ||
}); | ||
}); | ||
it ("should parse no header with dynamic column number",function(done){ | ||
@@ -73,0 +83,0 @@ var testData = __dirname + "/data/noheaderWithVaryColumnNum"; |
Sorry, the diff of this file is not supported yet
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Dynamic require
Supply chain riskDynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.
Found 1 instance in 1 package
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 2 instances in 1 package
Minified code
QualityThis package contains minified code. This may be harmless in some cases where minified code is included in packaged libraries, however packages on npm should not minify code.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
774024
71
11730
819
12
3
23
6