csv-parse
Advanced tools
Comparing version 1.2.4 to 1.3.0
# Changelog | ||
## v1.3.0 | ||
* options: auto_parse as a user function | ||
* options: auto_parse_date as a user function | ||
* test: should require handled by mocha | ||
* package: coffeescript 2 and use semver tilde | ||
* options: ensure objectMode is cloned | ||
## v1.2.4 | ||
@@ -5,0 +13,0 @@ |
229
lib/index.js
@@ -1,2 +0,9 @@ | ||
// Generated by CoffeeScript 1.12.7 | ||
// Generated by CoffeeScript 2.0.1 | ||
// # CSV Parser | ||
// This module provides a CSV parser tested and used against large datasets. Over | ||
// the year, it has been enhance and is now full of useful options. | ||
// Please look at the [README], the [project website][site] the [samples] and the | ||
// [tests] for additional information. | ||
var Parser, StringDecoder, isObjLiteral, stream, util; | ||
@@ -8,4 +15,13 @@ | ||
StringDecoder = require('string_decoder').StringDecoder; | ||
({StringDecoder} = require('string_decoder')); | ||
// ## Usage | ||
// Callback approach, for ease of use: | ||
// `parse(data, [options], callback)` | ||
// [Node.js Stream API][stream], for maximum of power: | ||
// `parse([options], [callback])` | ||
module.exports = function() { | ||
@@ -18,8 +34,9 @@ var callback, called, chunks, data, err, options, parser; | ||
if (typeof callback !== 'function') { | ||
throw Error("Invalid callback argument: " + (JSON.stringify(callback))); | ||
throw Error(`Invalid callback argument: ${JSON.stringify(callback)}`); | ||
} | ||
if (!(typeof data === 'string' || Buffer.isBuffer(arguments[0]))) { | ||
return callback(Error("Invalid data argument: " + (JSON.stringify(data)))); | ||
return callback(Error(`Invalid data argument: ${JSON.stringify(data)}`)); | ||
} | ||
} else if (arguments.length === 2) { | ||
// 1st arg is data:string or options:object | ||
if (typeof arguments[0] === 'string' || Buffer.isBuffer(arguments[0])) { | ||
@@ -30,4 +47,5 @@ data = arguments[0]; | ||
} else { | ||
err = "Invalid first argument: " + (JSON.stringify(arguments[0])); | ||
err = `Invalid first argument: ${JSON.stringify(arguments[0])}`; | ||
} | ||
// 2nd arg is options:object or callback:function | ||
if (typeof arguments[1] === 'function') { | ||
@@ -42,3 +60,3 @@ callback = arguments[1]; | ||
} else { | ||
err = "Invalid first argument: " + (JSON.stringify(arguments[1])); | ||
err = `Invalid first argument: ${JSON.stringify(arguments[1])}`; | ||
} | ||
@@ -97,8 +115,8 @@ if (err) { | ||
Parser = function(options) { | ||
// ## `Parser([options])` | ||
// Options are documented [here](http://csv.adaltas.com/parse/). | ||
Parser = function(options = {}) { | ||
var base, base1, base10, base11, base12, base13, base14, base15, base16, base2, base3, base4, base5, base6, base7, base8, base9, k, v; | ||
if (options == null) { | ||
options = {}; | ||
} | ||
options.objectMode = true; | ||
// @options = options | ||
this.options = {}; | ||
@@ -109,2 +127,3 @@ for (k in options) { | ||
} | ||
this.options.objectMode = true; | ||
stream.Transform.call(this, this.options); | ||
@@ -153,2 +172,12 @@ if ((base = this.options).rowDelimiter == null) { | ||
} | ||
if (this.options.auto_parse_date === true) { | ||
this.options.auto_parse_date = function(value) { | ||
var m; | ||
m = Date.parse(value); | ||
if (!isNaN(m)) { | ||
value = new Date(m); | ||
} | ||
return value; | ||
}; | ||
} | ||
if ((base12 = this.options).relax == null) { | ||
@@ -169,10 +198,16 @@ base12.relax = false; | ||
} | ||
this.lines = 0; | ||
this.count = 0; | ||
this.skipped_line_count = 0; | ||
this.empty_line_count = 0; | ||
// Counters | ||
// lines = count + skipped_line_count + empty_line_count | ||
this.lines = 0; // Number of lines encountered in the source dataset | ||
this.count = 0; // Number of records being processed | ||
this.skipped_line_count = 0; // Number of records skipped due to errors | ||
this.empty_line_count = 0; // Number of empty lines | ||
// Constants | ||
this.is_int = /^(\-|\+)?([1-9]+[0-9]*)$/; | ||
// @is_float = /^(\-|\+)?([0-9]+(\.[0-9]+)([eE][0-9]+)?|Infinity)$/ | ||
// @is_float = /^(\-|\+)?((([0-9])|([1-9]+[0-9]*))(\.[0-9]+)([eE][0-9]+)?|Infinity)$/ | ||
this.is_float = function(value) { | ||
return (value - parseFloat(value) + 1) >= 0; | ||
return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery | ||
}; | ||
// Internal state | ||
this._ = { | ||
@@ -189,3 +224,3 @@ decoder: new StringDecoder(), | ||
buf: '', | ||
rowDelimiterLength: this.options.rowDelimiter ? Math.max.apply(Math, this.options.rowDelimiter.map(function(v) { | ||
rowDelimiterLength: this.options.rowDelimiter ? Math.max(...this.options.rowDelimiter.map(function(v) { | ||
return v.length; | ||
@@ -197,6 +232,26 @@ })) : void 0 | ||
// ## Internal API | ||
// The Parser implement a [`stream.Transform` class][transform]. | ||
// ### Events | ||
// The library extends Node [EventEmitter][event] class and emit all | ||
// the events of the Writable and Readable [Stream API][stream]. | ||
util.inherits(Parser, stream.Transform); | ||
// For extra flexibility, you can get access to the original Parser | ||
// class: `require('csv-parse').Parser`. | ||
module.exports.Parser = Parser; | ||
// ### `_transform(chunk, encoding, callback)` | ||
// * `chunk` Buffer | String | ||
// The chunk to be transformed. Will always be a buffer unless the decodeStrings option was set to false. | ||
// * `encoding` String | ||
// If the chunk is a string, then this is the encoding type. (Ignore if decodeStrings chunk is a buffer.) | ||
// * `callback` Function | ||
// Call this function (optionally with an error argument) when you are done processing the supplied chunk. | ||
// Implementation of the [`stream.Transform` API][transform] | ||
Parser.prototype._transform = function(chunk, encoding, callback) { | ||
@@ -221,3 +276,3 @@ var err; | ||
if (this._.quoting) { | ||
this.emit('error', new Error("Quoted field not terminated at line " + (this.lines + 1))); | ||
this.emit('error', new Error(`Quoted field not terminated at line ${this.lines + 1}`)); | ||
return; | ||
@@ -235,3 +290,3 @@ } | ||
Parser.prototype.__push = function(line) { | ||
var call_column_udf, columns, err, field, i, j, len, lineAsColumns, rawBuf, ref, row; | ||
var call_column_udf, columns, err, field, i, j, len, lineAsColumns, rawBuf, row; | ||
if (this.options.skip_lines_with_empty_values && line.join('').trim() === '') { | ||
@@ -256,3 +311,3 @@ return; | ||
}; | ||
ref = call_column_udf(this.options.columns, line), err = ref[0], columns = ref[1]; | ||
[err, columns] = call_column_udf(this.options.columns, line); | ||
if (err) { | ||
@@ -268,5 +323,7 @@ return err; | ||
} | ||
// Dont check column count on empty lines | ||
if (line.length === 1 && line[0] === '') { | ||
this.empty_line_count++; | ||
} else if (line.length !== this._.line_length) { | ||
// Dont check column count with relax_column_count | ||
if (this.options.relax_column_count) { | ||
@@ -276,5 +333,5 @@ this.count++; | ||
} else if (this.options.columns != null) { | ||
return Error("Number of columns on line " + this.lines + " does not match header"); | ||
return Error(`Number of columns on line ${this.lines} does not match header`); | ||
} else { | ||
return Error("Number of columns is inconsistent on line " + this.lines); | ||
return Error(`Number of columns is inconsistent on line ${this.lines}`); | ||
} | ||
@@ -321,39 +378,33 @@ } else { | ||
var areNextCharsDelimiter, areNextCharsRowDelimiters, auto_parse, char, err, escapeIsQuote, i, isDelimiter, isEscape, isNextCharAComment, isQuote, isRowDelimiter, isRowDelimiterLength, is_float, is_int, l, ltrim, nextCharPos, ref, ref1, ref2, ref3, ref4, ref5, remainingBuffer, rowDelimiter, rtrim, wasCommenting; | ||
is_int = (function(_this) { | ||
return function(value) { | ||
if (typeof _this.is_int === 'function') { | ||
return _this.is_int(value); | ||
} else { | ||
return _this.is_int.test(value); | ||
} | ||
}; | ||
})(this); | ||
is_float = (function(_this) { | ||
return function(value) { | ||
if (typeof _this.is_float === 'function') { | ||
return _this.is_float(value); | ||
} else { | ||
return _this.is_float.test(value); | ||
} | ||
}; | ||
})(this); | ||
auto_parse = (function(_this) { | ||
return function(value) { | ||
var m; | ||
if (!_this.options.auto_parse) { | ||
return value; | ||
} | ||
if (is_int(value)) { | ||
value = parseInt(value); | ||
} else if (is_float(value)) { | ||
value = parseFloat(value); | ||
} else if (_this.options.auto_parse_date) { | ||
m = Date.parse(value); | ||
if (!isNaN(m)) { | ||
value = new Date(m); | ||
} | ||
} | ||
is_int = (value) => { | ||
if (typeof this.is_int === 'function') { | ||
return this.is_int(value); | ||
} else { | ||
return this.is_int.test(value); | ||
} | ||
}; | ||
is_float = (value) => { | ||
if (typeof this.is_float === 'function') { | ||
return this.is_float(value); | ||
} else { | ||
return this.is_float.test(value); | ||
} | ||
}; | ||
auto_parse = (value) => { | ||
if (!this.options.auto_parse) { | ||
return value; | ||
}; | ||
})(this); | ||
} | ||
if (typeof this.options.auto_parse === 'function') { | ||
return this.options.auto_parse(value); | ||
} | ||
// auto_parse == true | ||
if (is_int(value)) { | ||
value = parseInt(value); | ||
} else if (is_float(value)) { | ||
value = parseFloat(value); | ||
} else if (this.options.auto_parse_date) { | ||
value = this.options.auto_parse_date(value); | ||
} | ||
return value; | ||
}; | ||
ltrim = this.options.trim || this.options.ltrim; | ||
@@ -365,14 +416,20 @@ rtrim = this.options.trim || this.options.rtrim; | ||
if (this.lines === 0 && 0xFEFF === chars.charCodeAt(0)) { | ||
// Strip BOM header | ||
i++; | ||
} | ||
while (i < l) { | ||
// Ensure we get enough space to look ahead | ||
if (!end) { | ||
remainingBuffer = chars.substr(i, l - i); | ||
// (i+1000 >= l) or | ||
// Skip if the remaining buffer can be comment | ||
// Skip if the remaining buffer can be row delimiter | ||
if ((!this.options.rowDelimiter && i + 3 > l) || (!this._.commenting && l - i < this.options.comment.length && this.options.comment.substr(0, l - i) === remainingBuffer) || (this.options.rowDelimiter && l - i < this._.rowDelimiterLength && this.options.rowDelimiter.some(function(rd) { | ||
return rd.substr(0, l - i) === remainingBuffer; | ||
})) || (this.options.rowDelimiter && this._.quoting && l - i < (this.options.quote.length + this._.rowDelimiterLength) && this.options.rowDelimiter.some((function(_this) { | ||
return function(rd) { | ||
return (_this.options.quote + rd).substr(0, l - i) === remainingBuffer; | ||
}; | ||
})(this))) || (l - i <= this.options.delimiter.length && this.options.delimiter.substr(0, l - i) === remainingBuffer) || (l - i <= this.options.escape.length && this.options.escape.substr(0, l - i) === remainingBuffer)) { | ||
// Skip if the remaining buffer can be row delimiter following the closing quote | ||
})) || (this.options.rowDelimiter && this._.quoting && l - i < (this.options.quote.length + this._.rowDelimiterLength) && this.options.rowDelimiter.some((rd) => { | ||
return (this.options.quote + rd).substr(0, l - i) === remainingBuffer; | ||
// Skip if the remaining buffer can be delimiter | ||
// Skip if the remaining buffer can be escape sequence | ||
})) || (l - i <= this.options.delimiter.length && this.options.delimiter.substr(0, l - i) === remainingBuffer) || (l - i <= this.options.escape.length && this.options.escape.substr(0, l - i) === remainingBuffer)) { | ||
break; | ||
@@ -386,5 +443,7 @@ } | ||
} | ||
// Auto discovery of rowDelimiter, unix, mac and windows supported | ||
if (this.options.rowDelimiter == null) { | ||
nextCharPos = i; | ||
rowDelimiter = null; | ||
// First empty line | ||
if (!this._.quoting && (char === '\n' || char === '\r')) { | ||
@@ -408,3 +467,9 @@ rowDelimiter = char; | ||
} | ||
// Parse that damn char | ||
// Note, shouldn't we have sth like chars.substr(i, @options.escape.length) | ||
if (!this._.commenting && char === this.options.escape) { | ||
// Make sure the escape is really here for escaping: | ||
// If escape is same as quote, and escape is first char of a field | ||
// and it's not quoted, then it is a quote | ||
// Next char should be an escape or a quote | ||
escapeIsQuote = this.options.escape === this.options.quote; | ||
@@ -421,2 +486,3 @@ isEscape = this._.nextChar === this.options.escape; | ||
this._.field += char; | ||
// Since we're skipping the next one, better add it now if in raw mode. | ||
if (this.options.raw) { | ||
@@ -431,2 +497,8 @@ this._.rawBuf += char; | ||
if (this._.quoting) { | ||
// Make sure a closing quote is followed by a delimiter | ||
// If we have a next character and | ||
// it isnt a rowDelimiter and | ||
// it isnt an column delimiter and | ||
// it isnt the begining of a comment | ||
// Otherwise, if this is not "relax" mode, throw an error | ||
areNextCharsRowDelimiters = this.options.rowDelimiter && this.options.rowDelimiter.some(function(rd) { | ||
@@ -441,6 +513,6 @@ return chars.substr(i + 1, rd.length) === rd; | ||
if (this._.field) { | ||
this._.field = "" + this.options.quote + this._.field; | ||
this._.field = `${this.options.quote}${this._.field}`; | ||
} | ||
} else { | ||
return Error("Invalid closing quote at line " + (this.lines + 1) + "; found " + (JSON.stringify(this._.nextChar)) + " instead of delimiter " + (JSON.stringify(this.options.delimiter))); | ||
return Error(`Invalid closing quote at line ${this.lines + 1}; found ${JSON.stringify(this._.nextChar)} instead of delimiter ${JSON.stringify(this.options.delimiter)}`); | ||
} | ||
@@ -462,5 +534,6 @@ } else { | ||
} else if ((this._.field != null) && !this.options.relax) { | ||
return Error("Invalid opening quote at line " + (this.lines + 1)); | ||
return Error(`Invalid opening quote at line ${this.lines + 1}`); | ||
} | ||
} | ||
// Otherwise, treat quote as a regular character | ||
isRowDelimiter = this.options.rowDelimiter && this.options.rowDelimiter.some(function(rd) { | ||
@@ -472,2 +545,3 @@ return chars.substr(i, rd.length) === rd; | ||
} | ||
// Set the commenting flag | ||
wasCommenting = false; | ||
@@ -487,2 +561,3 @@ if (!this._.commenting && !this._.quoting && this.options.comment && chars.substr(i, this.options.comment.length) === this.options.comment) { | ||
} | ||
// Empty lines | ||
if (isRowDelimiter && this._.line.length === 0 && (this._.field == null)) { | ||
@@ -503,3 +578,3 @@ if (wasCommenting || this.options.skip_empty_lines) { | ||
this._.field = null; | ||
if (isDelimiter) { | ||
if (isDelimiter) { // End of field | ||
i += this.options.delimiter.length; | ||
@@ -517,2 +592,3 @@ this._.nextChar = chars.charAt(i); | ||
} | ||
// Some cleanup for the next row | ||
this._.line = []; | ||
@@ -525,2 +601,3 @@ i += isRowDelimiterLength; | ||
if (this._.field == null) { | ||
// Left trim unless we are quoting or field already filled | ||
this._.field = ''; | ||
@@ -542,8 +619,9 @@ } | ||
if (!this._.commenting && ((ref2 = this._.field) != null ? ref2.length : void 0) > this.options.max_limit_on_data_read) { | ||
return Error("Field exceeds max_limit_on_data_read setting (" + this.options.max_limit_on_data_read + ") " + (JSON.stringify(this.options.delimiter))); | ||
return Error(`Field exceeds max_limit_on_data_read setting (${this.options.max_limit_on_data_read}) ${JSON.stringify(this.options.delimiter)}`); | ||
} | ||
if (!this._.commenting && ((ref3 = this._.line) != null ? ref3.length : void 0) > this.options.max_limit_on_data_read) { | ||
return Error("Row delimiter not found in the file " + (JSON.stringify(this.options.rowDelimiter))); | ||
return Error(`Row delimiter not found in the file ${JSON.stringify(this.options.rowDelimiter)}`); | ||
} | ||
} | ||
// Flush remaining fields and lines | ||
if (end) { | ||
@@ -560,3 +638,3 @@ if (this._.field != null) { | ||
if (((ref5 = this._.field) != null ? ref5.length : void 0) > this.options.max_limit_on_data_read) { | ||
return Error("Delimiter not found in the file " + (JSON.stringify(this.options.delimiter))); | ||
return Error(`Delimiter not found in the file ${JSON.stringify(this.options.delimiter)}`); | ||
} | ||
@@ -567,5 +645,6 @@ if (l === 0) { | ||
if (this._.line.length > this.options.max_limit_on_data_read) { | ||
return Error("Row delimiter not found in the file " + (JSON.stringify(this.options.rowDelimiter))); | ||
return Error(`Row delimiter not found in the file ${JSON.stringify(this.options.rowDelimiter)}`); | ||
} | ||
} | ||
// Store un-parsed chars for next call | ||
this._.buf = chars.substr(i); | ||
@@ -575,2 +654,3 @@ return null; | ||
// ## Utils | ||
isObjLiteral = function(_obj) { | ||
@@ -592,1 +672,8 @@ var _test; | ||
}; | ||
// [readme]: https://github.com/wdavidw/node-csv-parse | ||
// [site]: http://csv.adaltas.com/parse/ | ||
// [samples]: https://github.com/wdavidw/node-csv-parse/tree/master/samples | ||
// [tests]: https://github.com/wdavidw/node-csv-parse/tree/master/test | ||
// [stream]: (http://nodejs.org/api/stream.html | ||
// [transform]: (http://nodejs.org/api/stream.html#stream_class_stream_transform_1) |
@@ -1,13 +0,15 @@ | ||
// Generated by CoffeeScript 1.12.7 | ||
// Generated by CoffeeScript 2.0.1 | ||
// # CSV Parse Sync | ||
// Provides a synchronous alternative to the CSV parser. | ||
// Usage: `records = parse(data, [options]` | ||
var StringDecoder, parse; | ||
StringDecoder = require('string_decoder').StringDecoder; | ||
({StringDecoder} = require('string_decoder')); | ||
parse = require('./index'); | ||
module.exports = function(data, options) { | ||
module.exports = function(data, options = {}) { | ||
var decoder, err, parser, records; | ||
if (options == null) { | ||
options = {}; | ||
} | ||
records = options.objname ? {} : []; | ||
@@ -14,0 +16,0 @@ if (data instanceof Buffer) { |
{ | ||
"version": "1.2.4", | ||
"version": "1.3.0", | ||
"name": "csv-parse", | ||
@@ -36,8 +36,8 @@ "description": "CSV parsing implementing the Node.js `stream.Transform` API", | ||
"devDependencies": { | ||
"each": "0.6.1", | ||
"coffee-script": "1.12.7", | ||
"csv-generate": "1.0.0", | ||
"csv-spectrum": "1.0.0", | ||
"mocha": "3.5.3", | ||
"should": "13.0.1" | ||
"each": "~0.6.1", | ||
"coffeescript": "~2.0.1", | ||
"csv-generate": "~1.0.0", | ||
"csv-spectrum": "~1.0.0", | ||
"mocha": "~4.0.1", | ||
"should": "~13.1.2" | ||
}, | ||
@@ -49,4 +49,4 @@ "optionalDependencies": {}, | ||
"pretest": "./node_modules/.bin/coffee -b -o lib src", | ||
"test": "NODE_ENV=test ./node_modules/.bin/mocha --compilers coffee:coffee-script/register --reporter dot" | ||
"test": "NODE_ENV=test ./node_modules/.bin/mocha test/**/*.coffee" | ||
} | ||
} |
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
51192
23
1254
4