Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

babyparse

Package Overview
Dependencies
Maintainers
2
Versions
10
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

babyparse - npm Package Compare versions

Comparing version 0.2.1 to 0.4.0

621

babyparse.js
/*
Baby Parse
v0.2.1
v0.4.0
https://github.com/Rich-Harris/BabyParse
based on Papa Parse v3.0.1
Created by Rich Harris
Maintained by Matt Holt
Based on Papa Parse v4.0.7 by Matt Holt
https://github.com/mholt/PapaParse
*/
(function(global)
{
(function ( global ) {
// A configuration object from which to draw default settings
var DEFAULTS = {
delimiter: "", // empty: auto-detect
newline: "", // empty: auto-detect
header: false,

@@ -22,3 +25,4 @@ dynamicTyping: false,

complete: undefined,
keepEmptyRows: false
skipEmptyLines: false,
fastMode: false
};

@@ -33,2 +37,5 @@

Baby.BAD_DELIMITERS = ["\r", "\n", "\"", Baby.BYTE_ORDER_MARK];
Baby.DefaultDelimiter = ","; // Used if not specified and detection fails
Baby.Parser = Parser; // For testing/dev only
Baby.ParserHandle = ParserHandle; // For testing/dev only

@@ -175,3 +182,3 @@

{
if (typeof str === "undefined")
if (typeof str === "undefined" || str === null)
return "";

@@ -211,2 +218,7 @@

var self = this;
var _stepCounter = 0; // Number of times step was called (number of rows parsed)
var _input; // The input being parsed
var _parser; // The core parser being used
var _paused = false; // Whether we are paused or not
var _delimiterError; // Temporary state between delimiter detection and processing results

@@ -219,6 +231,34 @@ var _fields = []; // Fields are from the header row of the input, if there is one

};
_config = copy(_config);
if (isFunction(_config.step))
{
var userStep = _config.step;
_config.step = function(results)
{
_results = results;
if (needsHeaderRow())
processResults();
else // only call user's step function after header row
{
processResults();
// It's possbile that this line was empty and there's no row here after all
if (_results.data.length == 0)
return;
_stepCounter += results.data.length;
if (_config.preview && _stepCounter > _config.preview)
_parser.abort();
else
userStep(_results, self);
}
};
}
this.parse = function(input)
{
if (!_config.newline)
_config.newline = guessLineEndings(input);
_delimiterError = false;

@@ -233,3 +273,3 @@ if (!_config.delimiter)

_delimiterError = true; // add error after parsing (otherwise it would be overwritten)
_config.delimiter = ",";
_config.delimiter = Baby.DefaultDelimiter;
}

@@ -239,17 +279,42 @@ _results.meta.delimiter = _config.delimiter;

if (isFunction(_config.step))
var parserConfig = copy(_config);
if (_config.preview && _config.header)
parserConfig.preview++; // to compensate for header row
_input = input;
_parser = new Parser(parserConfig);
_results = _parser.parse(_input);
processResults();
if (isFunction(_config.complete) && !_paused && (!self.streamer || self.streamer.finished()))
_config.complete(_results);
return _paused ? { meta: { paused: true } } : (_results || { meta: { paused: false } });
};
this.pause = function()
{
_paused = true;
_parser.abort();
_input = _input.substr(_parser.getCharIndex());
};
this.resume = function()
{
_paused = false;
_parser = new Parser(_config);
_parser.parse(_input);
if (!_paused)
{
var userStep = _config.step;
_config.step = function(results, parser)
{
_results = results;
if (needsHeaderRow())
processResults();
else
userStep(processResults(), parser);
};
if (self.streamer && !self.streamer.finished())
self.streamer.resume(); // more of the file yet to come
else if (isFunction(_config.complete))
_config.complete(_results);
}
};
_results = new Parser(_config).parse(input);
return processResults();
this.abort = function()
{
_parser.abort();
if (isFunction(_config.complete))
_config.complete(_results);
_input = "";
};

@@ -261,6 +326,13 @@

{
addError("Delimiter", "UndetectableDelimiter", "Unable to auto-detect delimiting character; defaulted to comma");
addError("Delimiter", "UndetectableDelimiter", "Unable to auto-detect delimiting character; defaulted to '"+Baby.DefaultDelimiter+"'");
_delimiterError = false;
}
if (_config.skipEmptyLines)
{
for (var i = 0; i < _results.data.length; i++)
if (_results.data[i].length == 1 && _results.data[i][0] == "")
_results.data.splice(i--, 1);
}
if (needsHeaderRow())

@@ -295,2 +367,3 @@ fillHeaderFields();

var row = {};
for (var j = 0; j < _results.data[i].length; j++)

@@ -317,3 +390,4 @@ {

}
row[_fields[j]] = _results.data[i][j];
else
row[_fields[j]] = _results.data[i][j];
}

@@ -332,5 +406,4 @@ }

if (_config.header && _results.meta);
if (_config.header && _results.meta)
_results.meta.fields = _fields;
return _results;

@@ -390,2 +463,21 @@ }

function guessLineEndings(input)
{
input = input.substr(0, 1024*1024); // max length 1 MB
var r = input.split('\r');
if (r.length == 1)
return '\n';
var numWithN = 0;
for (var i = 0; i < r.length; i++)
{
if (r[i][0] == '\n')
numWithN++;
}
return numWithN >= r.length / 2 ? '\r\n' : '\r';
}
function tryParseFloat(val)

@@ -413,284 +505,277 @@ {

// The core parser implements speedy and correct CSV parsing
function Parser(config)
{
var self = this;
var EMPTY = /^\s*$/;
var _input; // The input text being parsed
var _delimiter; // The delimiting character
var _comments; // Comment character (default '#') or boolean
var _step; // The step (streaming) function
var _callback; // The callback to invoke when finished
var _preview; // Maximum number of lines (not rows) to parse
var _ch; // Current character
var _i; // Current character's positional index
var _inQuotes; // Whether in quotes or not
var _lineNum; // Current line number (1-based indexing)
var _data; // Parsed data (results)
var _errors; // Parse errors
var _rowIdx; // Current row index within results (0-based)
var _colIdx; // Current col index within result row (0-based)
var _runningRowIdx; // Cumulative row index, used by the preview feature
var _aborted = false; // Abort flag
var _paused = false; // Pause flag
// Unpack the config object
config = config || {};
_delimiter = config.delimiter;
_comments = config.comments;
_step = config.step;
_preview = config.preview;
var delim = config.delimiter;
var newline = config.newline;
var comments = config.comments;
var step = config.step;
var preview = config.preview;
var fastMode = config.fastMode;
// Delimiter integrity check
if (typeof _delimiter !== 'string'
|| _delimiter.length != 1
|| Baby.BAD_DELIMITERS.indexOf(_delimiter) > -1)
_delimiter = ",";
// Delimiter must be valid
if (typeof delim !== 'string'
|| delim.length != 1
|| Baby.BAD_DELIMITERS.indexOf(delim) > -1)
delim = ",";
// Comment character integrity check
if (_comments === true)
_comments = "#";
else if (typeof _comments !== 'string'
|| _comments.length != 1
|| Baby.BAD_DELIMITERS.indexOf(_comments) > -1
|| _comments == _delimiter)
_comments = false;
// Comment character must be valid
if (comments === delim)
throw "Comment character same as delimiter";
else if (comments === true)
comments = "#";
else if (typeof comments !== 'string'
|| Baby.BAD_DELIMITERS.indexOf(comments) > -1)
comments = false;
// Newline must be valid: \r, \n, or \r\n
if (newline != '\n' && newline != '\r' && newline != '\r\n')
newline = '\n';
// We're gonna need these at the Parser scope
var cursor = 0;
var aborted = false;
this.parse = function(input)
{
// For some reason, in Chrome, this speeds things up (!?)
if (typeof input !== 'string')
throw "Input must be a string";
reset(input);
return parserLoop();
};
this.abort = function()
{
_aborted = true;
};
// We don't need to compute some of these every time parse() is called,
// but having them in a more local scope seems to perform better
var inputLen = input.length,
delimLen = delim.length,
newlineLen = newline.length,
commentsLen = comments.length;
var stepIsFunction = typeof step === 'function';
function parserLoop()
{
while (_i < _input.length)
{
if (_aborted) break;
if (_preview > 0 && _runningRowIdx >= _preview) break;
if (_paused) return finishParsing();
// Establish starting state
cursor = 0;
var data = [], errors = [], row = [];
if (_ch == '"')
parseQuotes();
else if (_inQuotes)
parseInQuotes();
else
parseNotInQuotes();
if (!input)
return returnable();
nextChar();
if (fastMode)
{
// Fast mode assumes there are no quoted fields in the input
var rows = input.split(newline);
for (var i = 0; i < rows.length; i++)
{
if (comments && rows[i].substr(0, commentsLen) == comments)
continue;
if (stepIsFunction)
{
data = [ rows[i].split(delim) ];
doStep();
if (aborted)
return returnable();
}
else
data.push(rows[i].split(delim));
if (preview && i >= preview)
{
data = data.slice(0, preview);
return returnable(true);
}
}
return returnable();
}
return finishParsing();
}
var nextDelim = input.indexOf(delim, cursor);
var nextNewline = input.indexOf(newline, cursor);
function nextChar()
{
_i++;
_ch = _input[_i];
}
// Parser loop
for (;;)
{
// Field has opening quote
if (input[cursor] == '"')
{
// Start our search for the closing quote where the cursor is
var quoteSearch = cursor;
function finishParsing()
{
if (_aborted)
addError("Abort", "ParseAbort", "Parsing was aborted by the user's step function");
if (_inQuotes)
addError("Quotes", "MissingQuotes", "Unescaped or mismatched quotes");
endRow(); // End of input is also end of the last row
if (!isFunction(_step))
return returnable();
}
// Skip the opening quote
cursor++;
function parseQuotes()
{
if (quotesOnBoundary() && !quotesEscaped())
_inQuotes = !_inQuotes;
else
{
saveChar();
if (_inQuotes && quotesEscaped())
_i++
else
addError("Quotes", "UnexpectedQuotes", "Unexpected quotes");
}
}
for (;;)
{
// Find closing quote
var quoteSearch = input.indexOf('"', quoteSearch+1);
function parseInQuotes()
{
if (twoCharLineBreak(_i) || oneCharLineBreak(_i))
_lineNum++;
saveChar();
}
if (quoteSearch === -1)
{
// No closing quote... what a pity
errors.push({
type: "Quotes",
code: "MissingQuotes",
message: "Quoted field unterminated",
row: data.length, // row has yet to be inserted
index: cursor
});
return finish();
}
function parseNotInQuotes()
{
if (_ch == _delimiter)
newField();
else if (twoCharLineBreak(_i))
{
newRow();
nextChar();
}
else if (oneCharLineBreak(_i))
newRow();
else if (isCommentStart())
skipLine();
else
saveChar();
}
if (quoteSearch === inputLen-1)
{
// Closing quote at EOF
row.push(input.substring(cursor, quoteSearch).replace(/""/g, '"'));
data.push(row);
if (stepIsFunction)
doStep();
return returnable();
}
function isCommentStart()
{
if (!_comments)
return false;
// If this quote is escaped, it's part of the data; skip it
if (input[quoteSearch+1] == '"')
{
quoteSearch++;
continue;
}
var firstCharOfLine = _i == 0
|| oneCharLineBreak(_i-1)
|| twoCharLineBreak(_i-2);
return firstCharOfLine && _input[_i] === _comments;
}
if (input[quoteSearch+1] == delim)
{
// Closing quote followed by delimiter
row.push(input.substring(cursor, quoteSearch).replace(/""/g, '"'));
cursor = quoteSearch + 1 + delimLen;
nextDelim = input.indexOf(delim, cursor);
nextNewline = input.indexOf(newline, cursor);
break;
}
function skipLine()
{
while (!twoCharLineBreak(_i)
&& !oneCharLineBreak(_i)
&& _i < _input.length)
{
nextChar();
}
}
if (input.substr(quoteSearch+1, newlineLen) === newline)
{
// Closing quote followed by newline
row.push(input.substring(cursor, quoteSearch).replace(/""/g, '"'));
saveRow(quoteSearch + 1 + newlineLen);
nextDelim = input.indexOf(delim, cursor); // because we may have skipped the nextDelim in the quoted field
function saveChar()
{
_data[_rowIdx][_colIdx] += _ch;
}
if (stepIsFunction)
{
doStep();
if (aborted)
return returnable();
}
if (preview && data.length >= preview)
return returnable(true);
function newField()
{
_data[_rowIdx].push("");
_colIdx = _data[_rowIdx].length - 1;
}
break;
}
}
function newRow()
{
endRow();
continue;
}
_lineNum++;
_runningRowIdx++;
_data.push([]);
_rowIdx = _data.length - 1;
newField();
}
// Comment found at start of new line
if (comments && row.length === 0 && input.substr(cursor, commentsLen) === comments)
{
if (nextNewline == -1) // Comment ends at EOF
return returnable();
cursor = nextNewline + newlineLen;
nextNewline = input.indexOf(newline, cursor);
nextDelim = input.indexOf(delim, cursor);
continue;
}
function endRow()
{
trimEmptyLastRow();
if (isFunction(_step))
{
if (_data[_rowIdx])
_step(returnable(), self);
clearErrorsAndData();
}
}
// Next delimiter comes before next newline, so we've reached end of field
if (nextDelim !== -1 && (nextDelim < nextNewline || nextNewline === -1))
{
row.push(input.substring(cursor, nextDelim));
cursor = nextDelim + delimLen;
nextDelim = input.indexOf(delim, cursor);
continue;
}
function trimEmptyLastRow()
{
if (_data[_rowIdx].length == 1 && EMPTY.test(_data[_rowIdx][0]))
{
if (config.keepEmptyRows)
_data[_rowIdx].splice(0, 1); // leave row, but no fields
else
_data.splice(_rowIdx, 1); // cut out row entirely
_rowIdx = _data.length - 1;
}
}
// End of row
if (nextNewline !== -1)
{
row.push(input.substring(cursor, nextNewline));
saveRow(nextNewline + newlineLen);
function twoCharLineBreak(i)
{
return i < _input.length - 1 &&
((_input[i] == "\r" && _input[i+1] == "\n")
|| (_input[i] == "\n" && _input[i+1] == "\r"))
}
if (stepIsFunction)
{
doStep();
if (aborted)
return returnable();
}
function oneCharLineBreak(i)
{
return _input[i] == "\r" || _input[i] == "\n";
}
if (preview && data.length >= preview)
return returnable(true);
function quotesEscaped()
{
// Quotes as data cannot be on boundary, for example: ,"", are not escaped quotes
return !quotesOnBoundary() && _i < _input.length - 1 && _input[_i+1] == '"';
}
continue;
}
function quotesOnBoundary()
{
return (!_inQuotes && isBoundary(_i-1)) || isBoundary(_i+1);
}
break;
}
function isBoundary(i)
{
if (typeof i != 'number')
i = _i;
var ch = _input[i];
return finish();
return (i <= -1 || i >= _input.length)
|| (ch == _delimiter
|| ch == "\r"
|| ch == "\n");
}
function addError(type, code, msg)
{
_errors.push({
type: type,
code: code,
message: msg,
line: _lineNum,
row: _rowIdx,
index: _i
});
}
// Appends the remaining input from cursor to the end into
// row, saves the row, calls step, and returns the results.
function finish()
{
row.push(input.substr(cursor));
data.push(row);
cursor = inputLen; // important in case parsing is paused
if (stepIsFunction)
doStep();
return returnable();
}
function reset(input)
{
_input = input;
_inQuotes = false;
_i = 0, _runningRowIdx = 0, _lineNum = 1;
clearErrorsAndData();
_data = [ [""] ]; // starting parsing requires an empty field
_ch = _input[_i];
}
// Appends the current row to the results. It sets the cursor
// to newCursor and finds the nextNewline. The caller should
// take care to execute user's step function and check for
// preview and end parsing if necessary.
function saveRow(newCursor)
{
data.push(row);
row = [];
cursor = newCursor;
nextNewline = input.indexOf(newline, cursor);
}
function clearErrorsAndData()
// Returns an object with the results, errors, and meta.
function returnable(stopped)
{
return {
data: data,
errors: errors,
meta: {
delimiter: delim,
linebreak: newline,
aborted: aborted,
truncated: !!stopped
}
};
}
// Executes the user's step function and resets data & errors.
function doStep()
{
step(returnable());
data = [], errors = [];
}
};
// Sets the abort flag
this.abort = function()
{
_data = [];
_errors = [];
_rowIdx = 0;
_colIdx = 0;
}
aborted = true;
};
function returnable()
// Gets the cursor position
this.getCharIndex = function()
{
return {
data: _data,
errors: _errors,
meta: {
lines: _lineNum,
delimiter: _delimiter,
aborted: _aborted
}
};
}
return cursor;
};
}
// Replaces bad config values with good, default ones

@@ -709,2 +794,7 @@ function copyAndValidateConfig(origConfig)

if (config.newline != '\n'
&& config.newline != '\r'
&& config.newline != '\r\n')
config.newline = DEFAULTS.newline;
if (typeof config.header !== 'boolean')

@@ -725,5 +815,8 @@ config.header = DEFAULTS.header;

if (typeof config.keepEmptyRows !== 'boolean')
config.keepEmptyRows = DEFAULTS.keepEmptyRows;
if (typeof config.skipEmptyLines !== 'boolean')
config.skipEmptyLines = DEFAULTS.skipEmptyLines;
if (typeof config.fastMode !== 'boolean')
config.fastMode = DEFAULTS.fastMode;
return config;

@@ -767,4 +860,2 @@ }

}( typeof window !== 'undefined' ? window : this ));
})(typeof window !== 'undefined' ? window : this);

9

package.json
{
"name": "babyparse",
"version": "0.2.1",
"version": "0.4.0",
"title": "BabyParse",
"repository": {
"type": "git",
"url": "git@github.com:Rich-Harris/BabyParse.git"
},
"description": "Fast and reliable CSV parser based on PapaParse",

@@ -17,5 +21,2 @@ "keywords": [

"tab",
"pipe",
"file",
"filereader",
"stream"

@@ -22,0 +23,0 @@ ],

589

tests/test-cases.js
var RECORD_SEP = String.fromCharCode(30);
var UNIT_SEP = String.fromCharCode(31);
// Tests for Papa.parse() function (CSV to JSON)
var PARSE_TESTS = [
// Tests for the core parser using new Baby.Parser().parse() (CSV to JSON)
var CORE_PARSER_TESTS = [
{

@@ -17,3 +16,3 @@ description: "One row",

description: "Two rows",
input: 'A,b,c\r\nd,E,f',
input: 'A,b,c\nd,E,f',
expected: {

@@ -25,6 +24,6 @@ data: [['A', 'b', 'c'], ['d', 'E', 'f']],

{
description: "Two rows, just \\r",
input: 'A,b,c\rd,E,f',
description: "Three rows",
input: 'A,b,c\nd,E,f\nG,h,i',
expected: {
data: [['A', 'b', 'c'], ['d', 'E', 'f']],
data: [['A', 'b', 'c'], ['d', 'E', 'f'], ['G', 'h', 'i']],
errors: []

@@ -34,10 +33,2 @@ }

{
description: "Two rows, just \\n",
input: 'A,b,c\nd,E,f',
expected: {
data: [['A', 'b', 'c'], ['d', 'E', 'f']],
errors: []
}
},
{
description: "Whitespace at edges of unquoted field",

@@ -76,6 +67,6 @@ input: 'a, b ,c',

{
description: "Quoted field with \\r\\n",
input: 'A,"B\r\nB",C',
description: "Quoted field with line break",
input: 'A,"B\nB",C',
expected: {
data: [['A', 'B\r\nB', 'C']],
data: [['A', 'B\nB', 'C']],
errors: []

@@ -85,6 +76,6 @@ }

{
description: "Quoted field with \\r",
input: 'A,"B\rB",C',
description: "Quoted fields with line breaks",
input: 'A,"B\nB","C\nC\nC"',
expected: {
data: [['A', 'B\rB', 'C']],
data: [['A', 'B\nB', 'C\nC\nC']],
errors: []

@@ -94,6 +85,6 @@ }

{
description: "Quoted field with \\n",
input: 'A,"B\nB",C',
description: "Quoted fields at end of row with delimiter and line break",
input: 'a,b,"c,c\nc"\nd,e,f',
expected: {
data: [['A', 'B\nB', 'C']],
data: [['a', 'b', 'c,c\nc'], ['d', 'e', 'f']],
errors: []

@@ -119,2 +110,11 @@ }

{
description: "Unquoted field with quotes at end of field",
notes: "The quotes character is misplaced, but shouldn't generate an error or break the parser",
input: 'A,B",C',
expected: {
data: [['A', 'B"', 'C']],
errors: []
}
},
{
description: "Quoted field with quotes around delimiter",

@@ -129,5 +129,5 @@ input: 'A,""",""",C',

{
description: "Quoted field with quotes on one side of delimiter",
description: "Quoted field with quotes on right side of delimiter",
input: 'A,",""",C',
notes: "Similar to the test above but with quotes only after the delimiter",
notes: "Similar to the test above but with quotes only after the comma",
expected: {

@@ -139,19 +139,34 @@ data: [['A', ',"', 'C']],

{
description: "Quoted field with quotes on left side of delimiter",
input: 'A,""",",C',
notes: "Similar to the test above but with quotes only before the comma",
expected: {
data: [['A', '",', 'C']],
errors: []
}
},
{
description: "Quoted field with 5 quotes in a row and a delimiter in there, too",
input: '"1","cnonce="""",nc=""""","2"',
notes: "Actual input reported in issue #121",
expected: {
data: [['1', 'cnonce="",nc=""', '2']],
errors: []
}
},
{
description: "Quoted field with whitespace around quotes",
input: 'A, "B" ,C',
notes: "This is malformed input, but it should be parsed gracefully (with errors)",
notes: "The quotes must be immediately adjacent to the delimiter to indicate a quoted field",
expected: {
data: [['A', ' "B" ', 'C']],
errors: [
{"type": "Quotes", "code": "UnexpectedQuotes", "message": "Unexpected quotes", "line": 1, "row": 0, "index": 3},
{"type": "Quotes", "code": "UnexpectedQuotes", "message": "Unexpected quotes", "line": 1, "row": 0, "index": 5}
]
errors: []
}
},
{
description: "Tab delimiter",
input: 'a\tb\tc\r\nd\te\tf',
config: { delimiter: "\t" },
description: "Misplaced quotes in data, not as opening quotes",
input: 'A,B "B",C',
notes: "The input is technically malformed, but this syntax should not cause an error",
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
data: [['A', 'B "B"', 'C']],
errors: []

@@ -161,6 +176,19 @@ }

{
description: "Pipe delimiter",
input: 'a|b|c\r\nd|e|f',
config: { delimiter: "|" },
description: "Quoted field has no closing quote",
input: 'a,"b,c\nd,e,f',
expected: {
data: [['a', 'b,c\nd,e,f']],
errors: [{
"type": "Quotes",
"code": "MissingQuotes",
"message": "Quoted field unterminated",
"row": 0,
"index": 3
}]
}
},
{
description: "Line starts with quoted field",
input: 'a,b,c\n"d",e,f',
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],

@@ -171,7 +199,6 @@ errors: []

{
description: "ASCII 30 delimiter",
input: 'a'+RECORD_SEP+'b'+RECORD_SEP+'c\r\nd'+RECORD_SEP+'e'+RECORD_SEP+'f',
config: { delimiter: RECORD_SEP },
description: "Line ends with quoted field",
input: 'a,b,c\nd,e,f\n"g","h","i"\n"j","k","l"',
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
data: [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i'], ['j', 'k', 'l']],
errors: []

@@ -181,7 +208,6 @@ }

{
description: "ASCII 31 delimiter",
input: 'a'+UNIT_SEP+'b'+UNIT_SEP+'c\r\nd'+UNIT_SEP+'e'+UNIT_SEP+'f',
config: { delimiter: UNIT_SEP },
description: "Quoted field at end of row (but not at EOF) has quotes",
input: 'a,b,"c""c"""\nd,e,f',
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
data: [['a', 'b', 'c"c"'], ['d', 'e', 'f']],
errors: []

@@ -191,8 +217,6 @@ }

{
description: "Bad delimiter",
input: 'a,b,c',
config: { delimiter: "DELIM" },
notes: "Should silently default to comma",
description: "Multiple consecutive empty fields",
input: 'a,b,,,c,d\n,,e,,,f',
expected: {
data: [['a', 'b', 'c']],
data: [['a', 'b', '', '', 'c', 'd'], ['', '', 'e', '', '', 'f']],
errors: []

@@ -202,4 +226,36 @@ }

{
description: "Commented line at beginning (comments: true)",
input: '# Comment!\r\na,b,c',
description: "Empty input string",
input: '',
expected: {
data: [],
errors: []
}
},
{
description: "Input is just the delimiter (2 empty fields)",
input: ',',
expected: {
data: [['', '']],
errors: []
}
},
{
description: "Input is just empty fields",
input: ',,\n,,,',
expected: {
data: [['', '', ''], ['', '', '', '']],
errors: []
}
},
{
description: "Input is just a string (a single field)",
input: 'Abc def',
expected: {
data: [['Abc def']],
errors: []
}
},
{
description: "Commented line at beginning",
input: '# Comment!\na,b,c',
config: { comments: true },

@@ -212,4 +268,4 @@ expected: {

{
description: "Commented line in middle (comments: true)",
input: 'a,b,c\r\n# Comment\r\nd,e,f',
description: "Commented line in middle",
input: 'a,b,c\n# Comment\nd,e,f',
config: { comments: true },

@@ -222,6 +278,24 @@ expected: {

{
description: "Commented line at end (comments: true)",
input: 'a,b,c\r\n# Comment',
description: "Commented line at end",
input: 'a,true,false\n# Comment',
config: { comments: true },
expected: {
data: [['a', 'true', 'false']],
errors: []
}
},
{
description: "Two comment lines consecutively",
input: 'a,b,c\n#comment1\n#comment2\nd,e,f',
config: { comments: true },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "Two comment lines consecutively at end of file",
input: 'a,b,c\n#comment1\n#comment2',
config: { comments: true },
expected: {
data: [['a', 'b', 'c']],

@@ -232,4 +306,22 @@ errors: []

{
description: "Comment with non-default character (comments: '!')",
input: 'a,b,c\r\n!Comment goes here\r\nd,e,f',
description: "Three comment lines consecutively at beginning of file",
input: '#comment1\n#comment2\n#comment3\na,b,c',
config: { comments: true },
expected: {
data: [['a', 'b', 'c']],
errors: []
}
},
{
description: "Entire file is comment lines",
input: '#comment1\n#comment2\n#comment3',
config: { comments: true },
expected: {
data: [],
errors: []
}
},
{
description: "Comment with non-default character",
input: 'a,b,c\n!Comment goes here\nd,e,f',
config: { comments: '!' },

@@ -242,8 +334,8 @@ expected: {

{
description: "Comment, but bad char specified (comments: \"=N(\")",
input: 'a,b,c\r\n=N(Comment)\r\nd,e,f',
config: { comments: '=N(' },
description: "Bad comments value specified",
notes: "Should silently disable comment parsing",
input: 'a,b,c\n5comment\nd,e,f',
config: { comments: 5 },
expected: {
data: [['a', 'b', 'c'], ['=N(Comment)'], ['d', 'e', 'f']],
data: [['a', 'b', 'c'], ['5comment'], ['d', 'e', 'f']],
errors: []

@@ -253,4 +345,13 @@ }

{
description: "Input with only a commented line (comments: true)",
input: '#commented line\r\n',
description: "Multi-character comment string",
input: 'a,b,c\n=N(Comment)\nd,e,f',
config: { comments: "=N(" },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "Input with only a commented line",
input: '#commented line',
config: { comments: true, delimiter: ',' },

@@ -263,3 +364,12 @@ expected: {

{
description: "Input with comment without comments enabled",
description: "Input with only a commented line and blank line after",
input: '#commented line\n',
config: { comments: true, delimiter: ',' },
expected: {
data: [['']],
errors: []
}
},
{
description: "Input with only a commented line, without comments enabled",
input: '#commented line',

@@ -274,3 +384,3 @@ config: { delimiter: ',' },

description: "Input without comments with line starting with whitespace",
input: 'a\r\n b\r\nc',
input: 'a\n b\nc',
config: { delimiter: ',' },

@@ -284,8 +394,6 @@ notes: "\" \" == false, but \" \" !== false, so === comparison is required",

{
description: "Comment char same as delimiter",
input: 'a#b#c\r\n# Comment',
config: { delimiter: '#', comments: '#' },
notes: "Comment parsing should automatically be silently disabled in this case",
description: "Multiple rows, one column (no delimiter found)",
input: 'a\nb\nc\nd\ne',
expected: {
data: [['a', 'b', 'c'], ['', ' Comment']],
data: [['a'], ['b'], ['c'], ['d'], ['e']],
errors: []

@@ -295,6 +403,224 @@ }

{
description: "One column input with empty fields",
input: 'a\nb\n\n\nc\nd\ne\n',
expected: {
data: [['a'], ['b'], [''], [''], ['c'], ['d'], ['e'], ['']],
errors: []
}
},
{
description: "Fast mode, basic",
input: 'a,b,c\nd,e,f',
config: { fastMode: true },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "Fast mode with comments",
input: '// Commented line\na,b,c',
config: { fastMode: true, comments: "//" },
expected: {
data: [['a', 'b', 'c']],
errors: []
}
},
{
description: "Fast mode with preview",
input: 'a,b,c\nd,e,f\nh,j,i\n',
config: { fastMode: true, preview: 2 },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "Fast mode with blank line at end",
input: 'a,b,c\n',
config: { fastMode: true },
expected: {
data: [['a', 'b', 'c'], ['']],
errors: []
}
}
];
// Tests for Baby.parse() function -- high-level wrapped parser (CSV to JSON)
var PARSE_TESTS = [
{
description: "Two rows, just \\r",
input: 'A,b,c\rd,E,f',
expected: {
data: [['A', 'b', 'c'], ['d', 'E', 'f']],
errors: []
}
},
{
description: "Two rows, \\r\\n",
input: 'A,b,c\r\nd,E,f',
expected: {
data: [['A', 'b', 'c'], ['d', 'E', 'f']],
errors: []
}
},
{
description: "Quoted field with \\r\\n",
input: 'A,"B\r\nB",C',
expected: {
data: [['A', 'B\r\nB', 'C']],
errors: []
}
},
{
description: "Quoted field with \\r",
input: 'A,"B\rB",C',
expected: {
data: [['A', 'B\rB', 'C']],
errors: []
}
},
{
description: "Quoted field with \\n",
input: 'A,"B\nB",C',
expected: {
data: [['A', 'B\nB', 'C']],
errors: []
}
},
{
description: "Header row with one row of data",
input: 'A,B,C\r\na,b,c',
config: { header: true },
expected: {
data: [{"A": "a", "B": "b", "C": "c"}],
errors: []
}
},
{
description: "Header row only",
input: 'A,B,C',
config: { header: true },
expected: {
data: [],
errors: []
}
},
{
description: "Row with too few fields",
input: 'A,B,C\r\na,b',
config: { header: true },
expected: {
data: [{"A": "a", "B": "b"}],
errors: [{
"type": "FieldMismatch",
"code": "TooFewFields",
"message": "Too few fields: expected 3 fields but parsed 2",
"row": 0
}]
}
},
{
description: "Row with too many fields",
input: 'A,B,C\r\na,b,c,d,e\r\nf,g,h',
config: { header: true },
expected: {
data: [{"A": "a", "B": "b", "C": "c", "__parsed_extra": ["d", "e"]}, {"A": "f", "B": "g", "C": "h"}],
errors: [{
"type": "FieldMismatch",
"code": "TooManyFields",
"message": "Too many fields: expected 3 fields but parsed 5",
"row": 0
}]
}
},
{
description: "Row with enough fields but blank field at end",
input: 'A,B,C\r\na,b,',
config: { header: true },
expected: {
data: [{"A": "a", "B": "b", "C": ""}],
errors: []
}
},
{
description: "Tab delimiter",
input: 'a\tb\tc\r\nd\te\tf',
config: { delimiter: "\t" },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "Pipe delimiter",
input: 'a|b|c\r\nd|e|f',
config: { delimiter: "|" },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "ASCII 30 delimiter",
input: 'a'+RECORD_SEP+'b'+RECORD_SEP+'c\r\nd'+RECORD_SEP+'e'+RECORD_SEP+'f',
config: { delimiter: RECORD_SEP },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "ASCII 31 delimiter",
input: 'a'+UNIT_SEP+'b'+UNIT_SEP+'c\r\nd'+UNIT_SEP+'e'+UNIT_SEP+'f',
config: { delimiter: UNIT_SEP },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "Bad delimiter",
input: 'a,b,c',
config: { delimiter: "DELIM" },
notes: "Should silently default to comma",
expected: {
data: [['a', 'b', 'c']],
errors: []
}
},
{
description: "Dynamic typing converts numeric literals",
input: '1,2.2,1e3\r\n-4,-4.5,-4e-5\r\n-,5a,5-2',
config: { dynamicTyping: true },
expected: {
data: [[1, 2.2, 1000], [-4, -4.5, -0.00004], ["-", "5a", "5-2"]],
errors: []
}
},
{
description: "Dynamic typing converts boolean literals",
input: 'true,false,T,F,TRUE,False',
config: { dynamicTyping: true },
expected: {
data: [[true, false, "T", "F", "TRUE", "False"]],
errors: []
}
},
{
description: "Dynamic typing doesn't convert other types",
input: 'A,B,C\r\nundefined,null,[\r\nvar,float,if',
config: { dynamicTyping: true },
expected: {
data: [["A", "B", "C"], ["undefined", "null", "["], ["var", "float", "if"]],
errors: []
}
},
{
description: "Blank line at beginning",
input: '\r\na,b,c\r\nd,e,f',
config: { newline: '\r\n' },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
data: [[''], ['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []

@@ -306,4 +632,5 @@ }

input: 'a,b,c\r\n\r\nd,e,f',
config: { newline: '\r\n' },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
data: [['a', 'b', 'c'], [''], ['d', 'e', 'f']],
errors: []

@@ -314,5 +641,5 @@ }

description: "Blank lines at end",
input: 'a,b,c\r\nd,e,f\r\n\r\n',
input: 'a,b,c\nd,e,f\n\n',
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
data: [['a', 'b', 'c'], ['d', 'e', 'f'], [''], ['']],
errors: []

@@ -325,3 +652,3 @@ }

expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
data: [['a', 'b', 'c'], [" "], ['d', 'e', 'f']],
errors: []

@@ -362,3 +689,3 @@ }

"code": "UndetectableDelimiter",
"message": "Unable to auto-detect delimiting character; defaulted to comma"
"message": "Unable to auto-detect delimiting character; defaulted to ','"
}]

@@ -384,3 +711,3 @@ }

"code": "UndetectableDelimiter",
"message": "Unable to auto-detect delimiting character; defaulted to comma"
"message": "Unable to auto-detect delimiting character; defaulted to ','"
}

@@ -445,7 +772,8 @@ ]

{
description: "Keep empty rows",
input: 'a,b,c\r\n\r\nd,e,f',
config: { keepEmptyRows: true },
description: "Preview with header row",
notes: "Preview is defined to be number of rows of input not including header row",
input: 'a,b,c\r\nd,e,f\r\ng,h,i\r\nj,k,l',
config: { header: true, preview: 2 },
expected: {
data: [['a', 'b', 'c'], [], ['d', 'e', 'f']],
data: [{"a": "d", "b": "e", "c": "f"}, {"a": "g", "b": "h", "c": "i"}],
errors: []

@@ -455,7 +783,25 @@ }

{
description: "Keep empty rows, with newline at end of input",
description: "Empty lines",
input: '\na,b,c\n\nd,e,f\n\n',
config: { delimiter: ',' },
expected: {
data: [[''], ['a', 'b', 'c'], [''], ['d', 'e', 'f'], [''], ['']],
errors: []
}
},
{
description: "Skip empty lines",
input: 'a,b,c\n\nd,e,f',
config: { skipEmptyLines: true },
expected: {
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []
}
},
{
description: "Skip empty lines, with newline at end of input",
input: 'a,b,c\r\n\r\nd,e,f\r\n',
config: { keepEmptyRows: true },
config: { skipEmptyLines: true },
expected: {
data: [['a', 'b', 'c'], [], ['d', 'e', 'f'], []],
data: [['a', 'b', 'c'], ['d', 'e', 'f']],
errors: []

@@ -465,7 +811,7 @@ }

{
description: "Keep empty rows, with empty input",
description: "Skip empty lines, with empty input",
input: '',
config: { keepEmptyRows: true },
config: { skipEmptyLines: true },
expected: {
data: [[]],
data: [],
errors: [

@@ -475,3 +821,3 @@ {

"code": "UndetectableDelimiter",
"message": "Unable to auto-detect delimiting character; defaulted to comma"
"message": "Unable to auto-detect delimiting character; defaulted to ','"
}

@@ -482,8 +828,8 @@ ]

{
description: "Keep empty rows, with first line only whitespace empty",
notes: "Even with keepEmptyRows enabled, rows with just a single field,<br>being whitespace, should be stripped of that field",
input: ' \r\na,b,c',
config: { keepEmptyRows: true },
description: "Skip empty lines, with first line only whitespace",
notes: "A line must be absolutely empty to be considered empty",
input: ' \na,b,c',
config: { skipEmptyLines: true, delimiter: ',' },
expected: {
data: [[], ['a', 'b', 'c']],
data: [[" "], ['a', 'b', 'c']],
errors: []

@@ -494,41 +840,3 @@ }

var PARSE_ASYNC_TESTS = [
{
description: "Simple worker",
input: "A,B,C\nX,Y,Z",
config: {
worker: true,
},
expected: {
data: [['A','B','C'],['X','Y','Z']],
errors: []
}
}
// These tests aren't applicable to BabyParse
/*,
{
description: "Simple download",
input: "/tests/sample.csv",
config: {
download: true
},
expected: {
data: [['A','B','C'],['X','Y','Z']],
errors: []
}
},
{
description: "Simple download + worker",
input: "/tests/sample.csv",
config: {
worker: true,
download: true
},
expected: {
data: [['A','B','C'],['X','Y','Z']],
errors: []
}
}*/
];

@@ -542,3 +850,5 @@

// Tests for Papa.unparse() function (JSON to CSV)
// Tests for Baby.unparse() function (JSON to CSV)
var UNPARSE_TESTS = [

@@ -589,3 +899,3 @@ {

description: "Specifying column names only (no data)",
notes: "Papa should add a data property that is an empty array to prevent errors (no copy is made)",
notes: "Baby should add a data property that is an empty array to prevent errors (no copy is made)",
input: { fields: ["Col1", "Col2", "Col3"] },

@@ -596,3 +906,3 @@ expected: 'Col1,Col2,Col3'

description: "Specifying data only (no field names), improperly",
notes: "A single array for a single row is wrong, but it can be compensated.<br>Papa should add empty fields property to prevent errors.",
notes: "A single array for a single row is wrong, but it can be compensated.<br>Baby should add empty fields property to prevent errors.",
input: { data: ["abc", "d", "ef"] },

@@ -603,3 +913,3 @@ expected: 'abc,d,ef'

description: "Specifying data only (no field names), properly",
notes: "An array of arrays, even if just a single row.<br>Papa should add empty fields property to prevent errors.",
notes: "An array of arrays, even if just a single row.<br>Baby should add empty fields property to prevent errors.",
input: { data: [["a", "b", "c"]] },

@@ -684,3 +994,8 @@ expected: 'a,b,c'

expected: 'a,b,c\r\nd,e\r\nf'
},
{
description: "JSON null is treated as empty value",
input: [{ "Col1": "a", "Col2": null, "Col3": "c" }],
expected: 'Col1,Col2,Col3\r\na,,c'
}
];

206

tests/test-runner.js

@@ -38,3 +38,4 @@ var passCount = 0;

function asyncDone() {
function allDone()
{
// Finally, show the overall status.

@@ -48,3 +49,4 @@ if (failCount == 0)

// Next, run tests and render results!
runParseTests(asyncDone);
runCoreParserTests();
runParseTests(allDone);
runUnparseTests();

@@ -55,6 +57,29 @@

// Executes all tests in CORE_PARSER_TESTS from test-cases.js
// and renders results in the table.
function runCoreParserTests()
{
for (var i = 0; i < CORE_PARSER_TESTS.length; i++)
{
var test = CORE_PARSER_TESTS[i];
var passed = runTest(test);
if (passed)
passCount++;
else
failCount++;
}
function runTest(test)
{
var actual = new Baby.Parser(test.config).parse(test.input);
var results = compare(actual.data, actual.errors, test.expected);
displayResults('#tests-for-core-parser', test, actual, results);
return results.data.passed && results.errors.passed
}
}
// Executes all tests in PARSE_TESTS from test-cases.js
// and renders results in the table.
function runParseTests(asyncDone)
function runParseTests(allDone)
{

@@ -71,136 +96,108 @@ for (var i = 0; i < PARSE_TESTS.length; i++)

var asyncRemaining = PARSE_ASYNC_TESTS.length;
allDone();
PARSE_ASYNC_TESTS.forEach(function(test) {
var config = test.config;
config.complete = function(actual) {
var results = compare(actual.data, actual.errors, test.expected);
function runTest(test)
{
var actual = Baby.parse(test.input, test.config);
var results = compare(actual.data, actual.errors, test.expected);
displayResults('#tests-for-parse', test, actual, results);
return results.data.passed && results.errors.passed
}
}
displayResults(test, actual, results);
if (results.data.passed && results.errors.passed) {
passCount++;
} else {
failCount++;
}
if (--asyncRemaining === 0) {
asyncDone();
}
}
config.error = function(err) {
failCount++;
displayResults(test, {data:[],errors:err}, test.expected);
if (--asyncRemaining === 0) {
asyncDone();
}
}
Baby.parse(test.input, test.config);
});
function runTest(test) {
var actual;
function displayResults(tableId, test, actual, results)
{
var testId = testCount++;
try {
actual = Baby.parse(test.input, test.config);
} catch (e) {
if (e instanceof Error) {
throw e;
}
actual.data = [];
actual.errors = [e];
}
var testDescription = (test.description || "");
if (testDescription.length > 0)
testDescription += '<br>';
if (test.notes)
testDescription += '<span class="notes">' + test.notes + '</span>';
var results = compare(actual.data, actual.errors, test.expected);
var tr = '<tr class="collapsed" id="test-'+testId+'">'
+ '<td class="rvl">+</td>'
+ '<td>' + testDescription + '</td>'
+ passOrFailTd(results.data)
+ passOrFailTd(results.errors)
+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">' + JSON.stringify(test.config, null, 2) + '</div></td>'
+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">' + revealChars(test.input) + '</div></td>'
+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">data: ' + JSON.stringify(test.expected.data, null, 4) + '\r\nerrors: ' + JSON.stringify(test.expected.errors, null, 4) + '</div></td>'
+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">data: ' + JSON.stringify(actual.data, null, 4) + '\r\nerrors: ' + JSON.stringify(actual.errors, null, 4) + '</div></td>'
+ '</tr>';
displayResults(test, actual, results);
$(tableId+' .results').append(tr);
return results.data.passed && results.errors.passed
}
if (!results.data.passed || !results.errors.passed)
$('#test-'+testId+' td.rvl').click();
function displayResults(test, actual, results) {
var testId = testCount++;
}
var testDescription = (test.description || "");
if (testDescription.length > 0)
testDescription += '<br>';
if (test.notes)
testDescription += '<span class="notes">' + test.notes + '</span>';
var tr = '<tr class="collapsed" id="test-'+testId+'">'
+ '<td class="rvl">+</td>'
+ '<td>' + testDescription + '</td>'
+ passOrFailTd(results.data)
+ passOrFailTd(results.errors)
+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">' + JSON.stringify(test.config, null, 2) + '</div></td>'
+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">' + revealChars(test.input) + '</div></td>'
+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">data: ' + JSON.stringify(test.expected.data, null, 4) + '\r\nerrors: ' + JSON.stringify(test.expected.errors, null, 4) + '</div></td>'
+ '<td class="revealable pre"><div class="revealer">condensed</div><div class="hidden">data: ' + JSON.stringify(actual.data, null, 4) + '\r\nerrors: ' + JSON.stringify(actual.errors, null, 4) + '</div></td>'
+ '</tr>';
function compare(actualData, actualErrors, expected)
{
var data = compareData(actualData, expected.data);
var errors = compareErrors(actualErrors, expected.errors);
$('#tests-for-parse .results').append(tr);
if (!results.data.passed || !results.errors.passed)
$('#test-' + testId + ' td.rvl').click();
return {
data: data,
errors: errors
}
function compare(actualData, actualErrors, expected)
function compareData(actual, expected)
{
var data = compareData(actualData, expected.data);
var errors = compareErrors(actualErrors, expected.errors);
var passed = true;
return {
data: data,
errors: errors
}
function compareData(actual, expected)
if (actual.length != expected.length)
passed = false;
else
{
var passed = true;
// The order is important, so we go through manually before using stringify to check everything else
for (var row = 0; row < expected.length; row++)
{
if (actual[row].length != expected[row].length)
{
passed = false;
break;
}
if (actual.length != expected.length) {
passed = false;
} else {
for (var row = 0; row < expected.length; row++) {
if (actual[row].length != expected[row].length) {
passed = false;
break;
}
for (var col = 0; col < expected[row].length; col++)
{
var expectedVal = expected[row][col];
var actualVal = actual[row][col];
var expectedVal = expected[row][col];
var actualVal = actual[row][col];
if (actualVal !== expectedVal)
{
passed = false;
break;
}
passed = false;
break;
}
}
}
// We pass back an object right now, even though it only contains
// one value, because we might add details to the test results later
// (same with compareErrors below)
return {
passed: passed
};
}
if (passed) // final check will catch any other differences
passed = JSON.stringify(actual) == JSON.stringify(expected);
function compareErrors(actual, expected)
{
var passed = JSON.stringify(actual) == JSON.stringify(expected);
// We pass back an object right now, even though it only contains
// one value, because we might add details to the test results later
// (same with compareErrors below)
return {
passed: passed
};
}
return {
passed: passed
};
}
function compareErrors(actual, expected)
{
var passed = JSON.stringify(actual) == JSON.stringify(expected);
return {
passed: passed
};
}

@@ -213,3 +210,2 @@ }

// Executes all tests in UNPARSE_TESTS from test-cases.js

@@ -322,2 +318,2 @@ // and renders results in the table.

return txt;
}
}
tests/tests.html

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc