Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

csv-parse

Package Overview
Dependencies
Maintainers
1
Versions
142
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

csv-parse - npm Package Compare versions

Comparing version 5.5.6 to 5.6.0

dist/esm/stream.d.ts
lib/stream.d.ts

17

lib/api/CsvError.js

@@ -1,14 +0,17 @@

class CsvError extends Error {
constructor(code, message, options, ...contexts) {
if(Array.isArray(message)) message = message.join(' ').trim();
if (Array.isArray(message)) message = message.join(" ").trim();
super(message);
if(Error.captureStackTrace !== undefined){
if (Error.captureStackTrace !== undefined) {
Error.captureStackTrace(this, CsvError);
}
this.code = code;
for(const context of contexts){
for(const key in context){
for (const context of contexts) {
for (const key in context) {
const value = context[key];
this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value));
this[key] = Buffer.isBuffer(value)
? value.toString(options.encoding)
: value == null
? value
: JSON.parse(JSON.stringify(value));
}

@@ -19,2 +22,2 @@ }

export {CsvError};
export { CsvError };

777

lib/api/index.js

@@ -0,9 +1,11 @@

import { normalize_columns_array } from "./normalize_columns_array.js";
import { init_state } from "./init_state.js";
import { normalize_options } from "./normalize_options.js";
import { CsvError } from "./CsvError.js";
import {normalize_columns_array} from './normalize_columns_array.js';
import {init_state} from './init_state.js';
import {normalize_options} from './normalize_options.js';
import {CsvError} from './CsvError.js';
const isRecordEmpty = function(record){
return record.every((field) => field == null || field.toString && field.toString().trim() === '');
const isRecordEmpty = function (record) {
return record.every(
(field) =>
field == null || (field.toString && field.toString().trim() === ""),
);
};

@@ -19,10 +21,10 @@

// Buffer.from('EFBBBF', 'hex')
'utf8': Buffer.from([239, 187, 191]),
utf8: Buffer.from([239, 187, 191]),
// Note, the following are equals:
// Buffer.from "\ufeff", 'utf16le
// Buffer.from([255, 254])
'utf16le': Buffer.from([255, 254])
utf16le: Buffer.from([255, 254]),
};
const transform = function(original_options = {}) {
const transform = function (original_options = {}) {
const info = {

@@ -34,3 +36,3 @@ bytes: 0,

lines: 1,
records: 0
records: 0,
};

@@ -43,6 +45,7 @@ const options = normalize_options(original_options);

state: init_state(options),
__needMoreData: function(i, bufLen, end){
if(end) return false;
const {encoding, escape, quote} = this.options;
const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state;
__needMoreData: function (i, bufLen, end) {
if (end) return false;
const { encoding, escape, quote } = this.options;
const { quoting, needMoreDataSize, recordDelimiterMaxLength } =
this.state;
const numOfCharLeft = bufLen - i - 1;

@@ -57,7 +60,9 @@ const requiredLength = Math.max(

// recordDelimiterMaxLength,
recordDelimiterMaxLength === 0 ? Buffer.from('\r\n', encoding).length : recordDelimiterMaxLength,
recordDelimiterMaxLength === 0
? Buffer.from("\r\n", encoding).length
: recordDelimiterMaxLength,
// Skip if remaining buffer can be an escaped quote
quoting ? ((escape === null ? 0 : escape.length) + quote.length) : 0,
quoting ? (escape === null ? 0 : escape.length) + quote.length : 0,
// Skip if remaining buffer can be record delimiter following the closing quote
quoting ? (quote.length + recordDelimiterMaxLength) : 0,
quoting ? quote.length + recordDelimiterMaxLength : 0,
);

@@ -67,27 +72,40 @@ return numOfCharLeft < requiredLength;

// Central parser implementation
parse: function(nextBuf, end, push, close){
const {bom, comment_no_infix, encoding, from_line, ltrim, max_record_size,raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options;
let {comment, escape, quote, record_delimiter} = this.options;
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state;
parse: function (nextBuf, end, push, close) {
const {
bom,
comment_no_infix,
encoding,
from_line,
ltrim,
max_record_size,
raw,
relax_quotes,
rtrim,
skip_empty_lines,
to,
to_line,
} = this.options;
let { comment, escape, quote, record_delimiter } = this.options;
const { bomSkipped, previousBuf, rawBuffer, escapeIsQuote } = this.state;
let buf;
if(previousBuf === undefined){
if(nextBuf === undefined){
if (previousBuf === undefined) {
if (nextBuf === undefined) {
// Handle empty string
close();
return;
}else{
} else {
buf = nextBuf;
}
}else if(previousBuf !== undefined && nextBuf === undefined){
} else if (previousBuf !== undefined && nextBuf === undefined) {
buf = previousBuf;
}else{
} else {
buf = Buffer.concat([previousBuf, nextBuf]);
}
// Handle UTF BOM
if(bomSkipped === false){
if(bom === false){
if (bomSkipped === false) {
if (bom === false) {
this.state.bomSkipped = true;
}else if(buf.length < 3){
} else if (buf.length < 3) {
// No enough data
if(end === false){
if (end === false) {
// Wait for more data

@@ -97,5 +115,5 @@ this.state.previousBuf = buf;

}
}else{
for(const encoding in boms){
if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){
} else {
for (const encoding in boms) {
if (boms[encoding].compare(buf, 0, boms[encoding].length) === 0) {
// Skip BOM

@@ -106,5 +124,8 @@ const bomLength = boms[encoding].length;

// Renormalize original options with the new encoding
this.options = normalize_options({...this.original_options, encoding: encoding});
this.options = normalize_options({
...this.original_options,
encoding: encoding,
});
// Options will re-evaluate the Buffer with the new encoding
({comment, escape, quote } = this.options);
({ comment, escape, quote } = this.options);
break;

@@ -118,13 +139,13 @@ }

let pos;
for(pos = 0; pos < bufLen; pos++){
for (pos = 0; pos < bufLen; pos++) {
// Ensure we get enough space to look ahead
// There should be a way to move this out of the loop
if(this.__needMoreData(pos, bufLen, end)){
if (this.__needMoreData(pos, bufLen, end)) {
break;
}
if(this.state.wasRowDelimiter === true){
if (this.state.wasRowDelimiter === true) {
this.info.lines++;
this.state.wasRowDelimiter = false;
}
if(to_line !== -1 && this.info.lines > to_line){
if (to_line !== -1 && this.info.lines > to_line) {
this.state.stop = true;

@@ -135,5 +156,8 @@ close();

// Auto discovery of record_delimiter, unix, mac and windows supported
if(this.state.quoting === false && record_delimiter.length === 0){
const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos);
if(record_delimiterCount){
if (this.state.quoting === false && record_delimiter.length === 0) {
const record_delimiterCount = this.__autoDiscoverRecordDelimiter(
buf,
pos,
);
if (record_delimiterCount) {
record_delimiter = this.options.record_delimiter;

@@ -143,6 +167,9 @@ }

const chr = buf[pos];
if(raw === true){
if (raw === true) {
rawBuffer.append(chr);
}
if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){
if (
(chr === cr || chr === nl) &&
this.state.wasRowDelimiter === false
) {
this.state.wasRowDelimiter = true;

@@ -152,11 +179,16 @@ }

// treat the current char as a regular char
if(this.state.escaping === true){
if (this.state.escaping === true) {
this.state.escaping = false;
}else{
} else {
// Escape is only active inside quoted fields
// We are quoting, the char is an escape chr and there is a chr to escape
// if(escape !== null && this.state.quoting === true && chr === escape && pos + 1 < bufLen){
if(escape !== null && this.state.quoting === true && this.__isEscape(buf, pos, chr) && pos + escape.length < bufLen){
if(escapeIsQuote){
if(this.__isQuote(buf, pos+escape.length)){
if (
escape !== null &&
this.state.quoting === true &&
this.__isEscape(buf, pos, chr) &&
pos + escape.length < bufLen
) {
if (escapeIsQuote) {
if (this.__isQuote(buf, pos + escape.length)) {
this.state.escaping = true;

@@ -166,3 +198,3 @@ pos += escape.length - 1;

}
}else{
} else {
this.state.escaping = true;

@@ -175,14 +207,34 @@ pos += escape.length - 1;

// TODO: need to compare bytes instead of single char
if(this.state.commenting === false && this.__isQuote(buf, pos)){
if(this.state.quoting === true){
const nextChr = buf[pos+quote.length];
const isNextChrTrimable = rtrim && this.__isCharTrimable(buf, pos+quote.length);
const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr);
const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr);
const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length);
if (this.state.commenting === false && this.__isQuote(buf, pos)) {
if (this.state.quoting === true) {
const nextChr = buf[pos + quote.length];
const isNextChrTrimable =
rtrim && this.__isCharTrimable(buf, pos + quote.length);
const isNextChrComment =
comment !== null &&
this.__compareBytes(comment, buf, pos + quote.length, nextChr);
const isNextChrDelimiter = this.__isDelimiter(
buf,
pos + quote.length,
nextChr,
);
const isNextChrRecordDelimiter =
record_delimiter.length === 0
? this.__autoDiscoverRecordDelimiter(buf, pos + quote.length)
: this.__isRecordDelimiter(nextChr, buf, pos + quote.length);
// Escape a quote
// Treat next char as a regular character
if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){
if (
escape !== null &&
this.__isEscape(buf, pos, chr) &&
this.__isQuote(buf, pos + escape.length)
) {
pos += escape.length - 1;
}else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){
} else if (
!nextChr ||
isNextChrDelimiter ||
isNextChrRecordDelimiter ||
isNextChrComment ||
isNextChrTrimable
) {
this.state.quoting = false;

@@ -192,14 +244,19 @@ this.state.wasQuoting = true;

continue;
}else if(relax_quotes === false){
} else if (relax_quotes === false) {
const err = this.__error(
new CsvError('CSV_INVALID_CLOSING_QUOTE', [
'Invalid Closing Quote:',
`got "${String.fromCharCode(nextChr)}"`,
`at line ${this.info.lines}`,
'instead of delimiter, record delimiter, trimable character',
'(if activated) or comment',
], this.options, this.__infoField())
new CsvError(
"CSV_INVALID_CLOSING_QUOTE",
[
"Invalid Closing Quote:",
`got "${String.fromCharCode(nextChr)}"`,
`at line ${this.info.lines}`,
"instead of delimiter, record delimiter, trimable character",
"(if activated) or comment",
],
this.options,
this.__infoField(),
),
);
if(err !== undefined) return err;
}else{
if (err !== undefined) return err;
} else {
this.state.quoting = false;

@@ -210,20 +267,30 @@ this.state.wasQuoting = true;

}
}else{
if(this.state.field.length !== 0){
} else {
if (this.state.field.length !== 0) {
// In relax_quotes mode, treat opening quote preceded by chrs as regular
if(relax_quotes === false){
if (relax_quotes === false) {
const info = this.__infoField();
const bom = Object.keys(boms).map(b => boms[b].equals(this.state.field.toString()) ? b : false).filter(Boolean)[0];
const bom = Object.keys(boms)
.map((b) =>
boms[b].equals(this.state.field.toString()) ? b : false,
)
.filter(Boolean)[0];
const err = this.__error(
new CsvError('INVALID_OPENING_QUOTE', [
'Invalid Opening Quote:',
`a quote is found on field ${JSON.stringify(info.column)} at line ${info.lines}, value is ${JSON.stringify(this.state.field.toString(encoding))}`,
bom ? `(${bom} bom)` : undefined
], this.options, info, {
field: this.state.field,
})
new CsvError(
"INVALID_OPENING_QUOTE",
[
"Invalid Opening Quote:",
`a quote is found on field ${JSON.stringify(info.column)} at line ${info.lines}, value is ${JSON.stringify(this.state.field.toString(encoding))}`,
bom ? `(${bom} bom)` : undefined,
],
this.options,
info,
{
field: this.state.field,
},
),
);
if(err !== undefined) return err;
if (err !== undefined) return err;
}
}else{
} else {
this.state.quoting = true;

@@ -235,13 +302,26 @@ pos += quote.length - 1;

}
if(this.state.quoting === false){
const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos);
if(recordDelimiterLength !== 0){
if (this.state.quoting === false) {
const recordDelimiterLength = this.__isRecordDelimiter(
chr,
buf,
pos,
);
if (recordDelimiterLength !== 0) {
// Do not emit comments which take a full line
const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0);
if(skipCommentLine){
const skipCommentLine =
this.state.commenting &&
this.state.wasQuoting === false &&
this.state.record.length === 0 &&
this.state.field.length === 0;
if (skipCommentLine) {
this.info.comment_lines++;
// Skip full comment line
}else{
} else {
// Activate records emition if above from_line
if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){
if (
this.state.enabled === false &&
this.info.lines +
(this.state.wasRowDelimiter === true ? 1 : 0) >=
from_line
) {
this.state.enabled = true;

@@ -254,3 +334,8 @@ this.__resetField();

// Skip if line is empty and skip_empty_lines activated
if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){
if (
skip_empty_lines === true &&
this.state.wasQuoting === false &&
this.state.record.length === 0 &&
this.state.field.length === 0
) {
this.info.empty_lines++;

@@ -262,7 +347,8 @@ pos += recordDelimiterLength - 1;

const errField = this.__onField();
if(errField !== undefined) return errField;
this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength;
if (errField !== undefined) return errField;
this.info.bytes =
this.state.bufBytesStart + pos + recordDelimiterLength;
const errRecord = this.__onRecord(push);
if(errRecord !== undefined) return errRecord;
if(to !== -1 && this.info.records >= to){
if (errRecord !== undefined) return errRecord;
if (to !== -1 && this.info.records >= to) {
this.state.stop = true;

@@ -277,8 +363,13 @@ close();

}
if(this.state.commenting){
if (this.state.commenting) {
continue;
}
if(comment !== null && (comment_no_infix === false || (this.state.record.length === 0 && this.state.field.length === 0))) {
if (
comment !== null &&
(comment_no_infix === false ||
(this.state.record.length === 0 &&
this.state.field.length === 0))
) {
const commentCount = this.__compareBytes(comment, buf, pos, chr);
if(commentCount !== 0){
if (commentCount !== 0) {
this.state.commenting = true;

@@ -289,6 +380,6 @@ continue;

const delimiterLength = this.__isDelimiter(buf, pos, chr);
if(delimiterLength !== 0){
if (delimiterLength !== 0) {
this.info.bytes = this.state.bufBytesStart + pos;
const errField = this.__onField();
if(errField !== undefined) return errField;
if (errField !== undefined) return errField;
pos += delimiterLength - 1;

@@ -299,29 +390,46 @@ continue;

}
if(this.state.commenting === false){
if(max_record_size !== 0 && this.state.record_length + this.state.field.length > max_record_size){
if (this.state.commenting === false) {
if (
max_record_size !== 0 &&
this.state.record_length + this.state.field.length > max_record_size
) {
return this.__error(
new CsvError('CSV_MAX_RECORD_SIZE', [
'Max Record Size:',
'record exceed the maximum number of tolerated bytes',
`of ${max_record_size}`,
`at line ${this.info.lines}`,
], this.options, this.__infoField())
new CsvError(
"CSV_MAX_RECORD_SIZE",
[
"Max Record Size:",
"record exceed the maximum number of tolerated bytes",
`of ${max_record_size}`,
`at line ${this.info.lines}`,
],
this.options,
this.__infoField(),
),
);
}
}
const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(buf, pos);
const lappend =
ltrim === false ||
this.state.quoting === true ||
this.state.field.length !== 0 ||
!this.__isCharTrimable(buf, pos);
// rtrim in non quoting is handle in __onField
const rappend = rtrim === false || this.state.wasQuoting === false;
if(lappend === true && rappend === true){
if (lappend === true && rappend === true) {
this.state.field.append(chr);
}else if(rtrim === true && !this.__isCharTrimable(buf, pos)){
} else if (rtrim === true && !this.__isCharTrimable(buf, pos)) {
return this.__error(
new CsvError('CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE', [
'Invalid Closing Quote:',
'found non trimable byte after quote',
`at line ${this.info.lines}`,
], this.options, this.__infoField())
new CsvError(
"CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE",
[
"Invalid Closing Quote:",
"found non trimable byte after quote",
`at line ${this.info.lines}`,
],
this.options,
this.__infoField(),
),
);
}else{
if(lappend === false){
} else {
if (lappend === false) {
pos += this.__isCharTrimable(buf, pos) - 1;

@@ -332,31 +440,40 @@ }

}
if(end === true){
if (end === true) {
// Ensure we are not ending in a quoting state
if(this.state.quoting === true){
if (this.state.quoting === true) {
const err = this.__error(
new CsvError('CSV_QUOTE_NOT_CLOSED', [
'Quote Not Closed:',
`the parsing is finished with an opening quote at line ${this.info.lines}`,
], this.options, this.__infoField())
new CsvError(
"CSV_QUOTE_NOT_CLOSED",
[
"Quote Not Closed:",
`the parsing is finished with an opening quote at line ${this.info.lines}`,
],
this.options,
this.__infoField(),
),
);
if(err !== undefined) return err;
}else{
if (err !== undefined) return err;
} else {
// Skip last line if it has no characters
if(this.state.wasQuoting === true || this.state.record.length !== 0 || this.state.field.length !== 0){
if (
this.state.wasQuoting === true ||
this.state.record.length !== 0 ||
this.state.field.length !== 0
) {
this.info.bytes = this.state.bufBytesStart + pos;
const errField = this.__onField();
if(errField !== undefined) return errField;
if (errField !== undefined) return errField;
const errRecord = this.__onRecord(push);
if(errRecord !== undefined) return errRecord;
}else if(this.state.wasRowDelimiter === true){
if (errRecord !== undefined) return errRecord;
} else if (this.state.wasRowDelimiter === true) {
this.info.empty_lines++;
}else if(this.state.commenting === true){
} else if (this.state.commenting === true) {
this.info.comment_lines++;
}
}
}else{
} else {
this.state.bufBytesStart += pos;
this.state.previousBuf = buf.slice(pos);
}
if(this.state.wasRowDelimiter === true){
if (this.state.wasRowDelimiter === true) {
this.info.lines++;

@@ -366,6 +483,17 @@ this.state.wasRowDelimiter = false;

},
__onRecord: function(push){
const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options;
const {enabled, record} = this.state;
if(enabled === false){
__onRecord: function (push) {
const {
columns,
group_columns_by_name,
encoding,
info,
from,
relax_column_count,
relax_column_count_less,
relax_column_count_more,
raw,
skip_records_with_empty_values,
} = this.options;
const { enabled, record } = this.state;
if (enabled === false) {
return this.__resetRecord();

@@ -375,4 +503,4 @@ }

const recordLength = record.length;
if(columns === true){
if(skip_records_with_empty_values === true && isRecordEmpty(record)){
if (columns === true) {
if (skip_records_with_empty_values === true && isRecordEmpty(record)) {
this.__resetRecord();

@@ -383,38 +511,54 @@ return;

}
if(columns === false && this.info.records === 0){
if (columns === false && this.info.records === 0) {
this.state.expectedRecordLength = recordLength;
}
if(recordLength !== this.state.expectedRecordLength){
const err = columns === false ?
new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [
'Invalid Record Length:',
`expect ${this.state.expectedRecordLength},`,
`got ${recordLength} on line ${this.info.lines}`,
], this.options, this.__infoField(), {
record: record,
})
:
new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [
'Invalid Record Length:',
`columns length is ${columns.length},`, // rename columns
`got ${recordLength} on line ${this.info.lines}`,
], this.options, this.__infoField(), {
record: record,
});
if(relax_column_count === true ||
(relax_column_count_less === true && recordLength < this.state.expectedRecordLength) ||
(relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){
if (recordLength !== this.state.expectedRecordLength) {
const err =
columns === false
? new CsvError(
"CSV_RECORD_INCONSISTENT_FIELDS_LENGTH",
[
"Invalid Record Length:",
`expect ${this.state.expectedRecordLength},`,
`got ${recordLength} on line ${this.info.lines}`,
],
this.options,
this.__infoField(),
{
record: record,
},
)
: new CsvError(
"CSV_RECORD_INCONSISTENT_COLUMNS",
[
"Invalid Record Length:",
`columns length is ${columns.length},`, // rename columns
`got ${recordLength} on line ${this.info.lines}`,
],
this.options,
this.__infoField(),
{
record: record,
},
);
if (
relax_column_count === true ||
(relax_column_count_less === true &&
recordLength < this.state.expectedRecordLength) ||
(relax_column_count_more === true &&
recordLength > this.state.expectedRecordLength)
) {
this.info.invalid_field_length++;
this.state.error = err;
// Error is undefined with skip_records_with_error
}else{
// Error is undefined with skip_records_with_error
} else {
const finalErr = this.__error(err);
if(finalErr) return finalErr;
if (finalErr) return finalErr;
}
}
if(skip_records_with_empty_values === true && isRecordEmpty(record)){
if (skip_records_with_empty_values === true && isRecordEmpty(record)) {
this.__resetRecord();
return;
}
if(this.state.recordHasError === true){
if (this.state.recordHasError === true) {
this.__resetRecord();

@@ -425,12 +569,15 @@ this.state.recordHasError = false;

this.info.records++;
if(from === 1 || this.info.records >= from){
const {objname} = this.options;
if (from === 1 || this.info.records >= from) {
const { objname } = this.options;
// With columns, records are object
if(columns !== false){
if (columns !== false) {
const obj = {};
// Transform record array to an object
for(let i = 0, l = record.length; i < l; i++){
if(columns[i] === undefined || columns[i].disabled) continue;
for (let i = 0, l = record.length; i < l; i++) {
if (columns[i] === undefined || columns[i].disabled) continue;
// Turn duplicate columns into an array
if (group_columns_by_name === true && obj[columns[i].name] !== undefined) {
if (
group_columns_by_name === true &&
obj[columns[i].name] !== undefined
) {
if (Array.isArray(obj[columns[i].name])) {

@@ -446,41 +593,49 @@ obj[columns[i].name] = obj[columns[i].name].concat(record[i]);

// Without objname (default)
if(raw === true || info === true){
if (raw === true || info === true) {
const extRecord = Object.assign(
{record: obj},
(raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}),
(info === true ? {info: this.__infoRecord()}: {})
{ record: obj },
raw === true
? { raw: this.state.rawBuffer.toString(encoding) }
: {},
info === true ? { info: this.__infoRecord() } : {},
);
const err = this.__push(
objname === undefined ? extRecord : [obj[objname], extRecord]
, push);
if(err){
objname === undefined ? extRecord : [obj[objname], extRecord],
push,
);
if (err) {
return err;
}
}else{
} else {
const err = this.__push(
objname === undefined ? obj : [obj[objname], obj]
, push);
if(err){
objname === undefined ? obj : [obj[objname], obj],
push,
);
if (err) {
return err;
}
}
// Without columns, records are array
}else{
if(raw === true || info === true){
// Without columns, records are array
} else {
if (raw === true || info === true) {
const extRecord = Object.assign(
{record: record},
raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {},
info === true ? {info: this.__infoRecord()}: {}
{ record: record },
raw === true
? { raw: this.state.rawBuffer.toString(encoding) }
: {},
info === true ? { info: this.__infoRecord() } : {},
);
const err = this.__push(
objname === undefined ? extRecord : [record[objname], extRecord]
, push);
if(err){
objname === undefined ? extRecord : [record[objname], extRecord],
push,
);
if (err) {
return err;
}
}else{
} else {
const err = this.__push(
objname === undefined ? record : [record[objname], record]
, push);
if(err){
objname === undefined ? record : [record[objname], record],
push,
);
if (err) {
return err;

@@ -493,15 +648,24 @@ }

},
__firstLineToColumns: function(record){
const {firstLineToHeaders} = this.state;
try{
const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record);
if(!Array.isArray(headers)){
__firstLineToColumns: function (record) {
const { firstLineToHeaders } = this.state;
try {
const headers =
firstLineToHeaders === undefined
? record
: firstLineToHeaders.call(null, record);
if (!Array.isArray(headers)) {
return this.__error(
new CsvError('CSV_INVALID_COLUMN_MAPPING', [
'Invalid Column Mapping:',
'expect an array from column function,',
`got ${JSON.stringify(headers)}`
], this.options, this.__infoField(), {
headers: headers,
})
new CsvError(
"CSV_INVALID_COLUMN_MAPPING",
[
"Invalid Column Mapping:",
"expect an array from column function,",
`got ${JSON.stringify(headers)}`,
],
this.options,
this.__infoField(),
{
headers: headers,
},
),
);

@@ -514,8 +678,8 @@ }

return;
}catch(err){
} catch (err) {
return err;
}
},
__resetRecord: function(){
if(this.options.raw === true){
__resetRecord: function () {
if (this.options.raw === true) {
this.state.rawBuffer.reset();

@@ -527,16 +691,16 @@ }

},
__onField: function(){
const {cast, encoding, rtrim, max_record_size} = this.options;
const {enabled, wasQuoting} = this.state;
__onField: function () {
const { cast, encoding, rtrim, max_record_size } = this.options;
const { enabled, wasQuoting } = this.state;
// Short circuit for the from_line options
if(enabled === false){
if (enabled === false) {
return this.__resetField();
}
let field = this.state.field.toString(encoding);
if(rtrim === true && wasQuoting === false){
if (rtrim === true && wasQuoting === false) {
field = field.trimRight();
}
if(cast === true){
if (cast === true) {
const [err, f] = this.__cast(field);
if(err !== undefined) return err;
if (err !== undefined) return err;
field = f;

@@ -546,3 +710,3 @@ }

// Increment record length if record size must not exceed a limit
if(max_record_size !== 0 && typeof field === 'string'){
if (max_record_size !== 0 && typeof field === "string") {
this.state.record_length += field.length;

@@ -552,16 +716,18 @@ }

},
__resetField: function(){
__resetField: function () {
this.state.field.reset();
this.state.wasQuoting = false;
},
__push: function(record, push){
const {on_record} = this.options;
if(on_record !== undefined){
__push: function (record, push) {
const { on_record } = this.options;
if (on_record !== undefined) {
const info = this.__infoRecord();
try{
try {
record = on_record.call(null, record, info);
}catch(err){
} catch (err) {
return err;
}
if(record === undefined || record === null){ return; }
if (record === undefined || record === null) {
return;
}
}

@@ -571,4 +737,4 @@ push(record);

// Return a tuple with the error and the casted value
__cast: function(field){
const {columns, relax_column_count} = this.options;
__cast: function (field) {
const { columns, relax_column_count } = this.options;
const isColumns = Array.isArray(columns);

@@ -578,16 +744,20 @@ // Dont loose time calling cast

// and this field can't be associated to a key present in columns
if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){
if (
isColumns === true &&
relax_column_count &&
this.options.columns.length <= this.state.record.length
) {
return [undefined, undefined];
}
if(this.state.castField !== null){
try{
if (this.state.castField !== null) {
try {
const info = this.__infoField();
return [undefined, this.state.castField.call(null, field, info)];
}catch(err){
} catch (err) {
return [err];
}
}
if(this.__isFloat(field)){
if (this.__isFloat(field)) {
return [undefined, parseFloat(field)];
}else if(this.options.cast_date !== false){
} else if (this.options.cast_date !== false) {
const info = this.__infoField();

@@ -599,9 +769,9 @@ return [undefined, this.options.cast_date.call(null, field, info)];

// Helper to test if a character is a space or a line delimiter
__isCharTrimable: function(buf, pos){
__isCharTrimable: function (buf, pos) {
const isTrim = (buf, pos) => {
const {timchars} = this.state;
loop1: for(let i = 0; i < timchars.length; i++){
const { timchars } = this.state;
loop1: for (let i = 0; i < timchars.length; i++) {
const timchar = timchars[i];
for(let j = 0; j < timchar.length; j++){
if(timchar[j] !== buf[pos+j]) continue loop1;
for (let j = 0; j < timchar.length; j++) {
if (timchar[j] !== buf[pos + j]) continue loop1;
}

@@ -620,25 +790,32 @@ return timchar.length;

// }
__isFloat: function(value){
return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery
__isFloat: function (value) {
return value - parseFloat(value) + 1 >= 0; // Borrowed from jquery
},
__compareBytes: function(sourceBuf, targetBuf, targetPos, firstByte){
if(sourceBuf[0] !== firstByte) return 0;
__compareBytes: function (sourceBuf, targetBuf, targetPos, firstByte) {
if (sourceBuf[0] !== firstByte) return 0;
const sourceLength = sourceBuf.length;
for(let i = 1; i < sourceLength; i++){
if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0;
for (let i = 1; i < sourceLength; i++) {
if (sourceBuf[i] !== targetBuf[targetPos + i]) return 0;
}
return sourceLength;
},
__isDelimiter: function(buf, pos, chr){
const {delimiter, ignore_last_delimiters} = this.options;
if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){
__isDelimiter: function (buf, pos, chr) {
const { delimiter, ignore_last_delimiters } = this.options;
if (
ignore_last_delimiters === true &&
this.state.record.length === this.options.columns.length - 1
) {
return 0;
}else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){
} else if (
ignore_last_delimiters !== false &&
typeof ignore_last_delimiters === "number" &&
this.state.record.length === ignore_last_delimiters - 1
) {
return 0;
}
loop1: for(let i = 0; i < delimiter.length; i++){
loop1: for (let i = 0; i < delimiter.length; i++) {
const del = delimiter[i];
if(del[0] === chr){
for(let j = 1; j < del.length; j++){
if(del[j] !== buf[pos+j]) continue loop1;
if (del[0] === chr) {
for (let j = 1; j < del.length; j++) {
if (del[j] !== buf[pos + j]) continue loop1;
}

@@ -650,13 +827,13 @@ return del.length;

},
__isRecordDelimiter: function(chr, buf, pos){
const {record_delimiter} = this.options;
__isRecordDelimiter: function (chr, buf, pos) {
const { record_delimiter } = this.options;
const recordDelimiterLength = record_delimiter.length;
loop1: for(let i = 0; i < recordDelimiterLength; i++){
loop1: for (let i = 0; i < recordDelimiterLength; i++) {
const rd = record_delimiter[i];
const rdLength = rd.length;
if(rd[0] !== chr){
if (rd[0] !== chr) {
continue;
}
for(let j = 1; j < rdLength; j++){
if(rd[j] !== buf[pos+j]){
for (let j = 1; j < rdLength; j++) {
if (rd[j] !== buf[pos + j]) {
continue loop1;

@@ -669,9 +846,9 @@ }

},
__isEscape: function(buf, pos, chr){
const {escape} = this.options;
if(escape === null) return false;
__isEscape: function (buf, pos, chr) {
const { escape } = this.options;
if (escape === null) return false;
const l = escape.length;
if(escape[0] === chr){
for(let i = 0; i < l; i++){
if(escape[i] !== buf[pos+i]){
if (escape[0] === chr) {
for (let i = 0; i < l; i++) {
if (escape[i] !== buf[pos + i]) {
return false;

@@ -684,8 +861,8 @@ }

},
__isQuote: function(buf, pos){
const {quote} = this.options;
if(quote === null) return false;
__isQuote: function (buf, pos) {
const { quote } = this.options;
if (quote === null) return false;
const l = quote.length;
for(let i = 0; i < l; i++){
if(quote[i] !== buf[pos+i]){
for (let i = 0; i < l; i++) {
if (quote[i] !== buf[pos + i]) {
return false;

@@ -696,3 +873,3 @@ }

},
__autoDiscoverRecordDelimiter: function(buf, pos){
__autoDiscoverRecordDelimiter: function (buf, pos) {
const { encoding } = this.options;

@@ -704,10 +881,10 @@ // Note, we don't need to cache this information in state,

// Important, the windows line ending must be before mac os 9
Buffer.from('\r\n', encoding),
Buffer.from('\n', encoding),
Buffer.from('\r', encoding),
Buffer.from("\r\n", encoding),
Buffer.from("\n", encoding),
Buffer.from("\r", encoding),
];
loop: for(let i = 0; i < rds.length; i++){
loop: for (let i = 0; i < rds.length; i++) {
const l = rds[i].length;
for(let j = 0; j < l; j++){
if(rds[i][j] !== buf[pos + j]){
for (let j = 0; j < l; j++) {
if (rds[i][j] !== buf[pos + j]) {
continue loop;

@@ -722,24 +899,27 @@ }

},
__error: function(msg){
const {encoding, raw, skip_records_with_error} = this.options;
const err = typeof msg === 'string' ? new Error(msg) : msg;
if(skip_records_with_error){
__error: function (msg) {
const { encoding, raw, skip_records_with_error } = this.options;
const err = typeof msg === "string" ? new Error(msg) : msg;
if (skip_records_with_error) {
this.state.recordHasError = true;
if(this.options.on_skip !== undefined){
this.options.on_skip(err, raw ? this.state.rawBuffer.toString(encoding) : undefined);
if (this.options.on_skip !== undefined) {
this.options.on_skip(
err,
raw ? this.state.rawBuffer.toString(encoding) : undefined,
);
}
// this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined);
return undefined;
}else{
} else {
return err;
}
},
__infoDataSet: function(){
__infoDataSet: function () {
return {
...this.info,
columns: this.options.columns
columns: this.options.columns,
};
},
__infoRecord: function(){
const {columns, raw, encoding} = this.options;
__infoRecord: function () {
const { columns, raw, encoding } = this.options;
return {

@@ -750,23 +930,22 @@ ...this.__infoDataSet(),

index: this.state.record.length,
raw: raw ? this.state.rawBuffer.toString(encoding) : undefined
raw: raw ? this.state.rawBuffer.toString(encoding) : undefined,
};
},
__infoField: function(){
const {columns} = this.options;
__infoField: function () {
const { columns } = this.options;
const isColumns = Array.isArray(columns);
return {
...this.__infoRecord(),
column: isColumns === true ?
(columns.length > this.state.record.length ?
columns[this.state.record.length].name :
null
) :
this.state.record.length,
column:
isColumns === true
? columns.length > this.state.record.length
? columns[this.state.record.length].name
: null
: this.state.record.length,
quoting: this.state.wasQuoting,
};
}
},
};
};
export {transform, CsvError};
export { transform, CsvError };

38

lib/api/init_state.js

@@ -0,4 +1,3 @@

import ResizeableBuffer from "../utils/ResizeableBuffer.js";
import ResizeableBuffer from '../utils/ResizeableBuffer.js';
// white space characters

@@ -14,3 +13,3 @@ // https://en.wikipedia.org/wiki/Whitespace_character

const init_state = function(options){
const init_state = function (options) {
return {

@@ -25,5 +24,10 @@ bomSkipped: false,

escaping: false,
escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0,
escapeIsQuote:
Buffer.isBuffer(options.escape) &&
Buffer.isBuffer(options.quote) &&
Buffer.compare(options.escape, options.quote) === 0,
// columns can be `false`, `true`, `Array`
expectedRecordLength: Array.isArray(options.columns) ? options.columns.length : undefined,
expectedRecordLength: Array.isArray(options.columns)
? options.columns.length
: undefined,
field: new ResizeableBuffer(20),

@@ -46,16 +50,22 @@ firstLineToHeaders: options.cast_first_line_to_header,

record_length: 0,
recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 0 : Math.max(...options.record_delimiter.map((v) => v.length)),
trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]],
recordDelimiterMaxLength:
options.record_delimiter.length === 0
? 0
: Math.max(...options.record_delimiter.map((v) => v.length)),
trimChars: [
Buffer.from(" ", options.encoding)[0],
Buffer.from("\t", options.encoding)[0],
],
wasQuoting: false,
wasRowDelimiter: false,
timchars: [
Buffer.from(Buffer.from([cr], 'utf8').toString(), options.encoding),
Buffer.from(Buffer.from([nl], 'utf8').toString(), options.encoding),
Buffer.from(Buffer.from([np], 'utf8').toString(), options.encoding),
Buffer.from(Buffer.from([space], 'utf8').toString(), options.encoding),
Buffer.from(Buffer.from([tab], 'utf8').toString(), options.encoding),
]
Buffer.from(Buffer.from([cr], "utf8").toString(), options.encoding),
Buffer.from(Buffer.from([nl], "utf8").toString(), options.encoding),
Buffer.from(Buffer.from([np], "utf8").toString(), options.encoding),
Buffer.from(Buffer.from([space], "utf8").toString(), options.encoding),
Buffer.from(Buffer.from([tab], "utf8").toString(), options.encoding),
],
};
};
export {init_state};
export { init_state };

35

lib/api/normalize_columns_array.js

@@ -0,27 +1,26 @@

import { CsvError } from "./CsvError.js";
import { is_object } from "../utils/is_object.js";
import {CsvError} from './CsvError.js';
import {is_object} from '../utils/is_object.js';
const normalize_columns_array = function(columns){
const normalize_columns_array = function (columns) {
const normalizedColumns = [];
for(let i = 0, l = columns.length; i < l; i++){
for (let i = 0, l = columns.length; i < l; i++) {
const column = columns[i];
if(column === undefined || column === null || column === false){
if (column === undefined || column === null || column === false) {
normalizedColumns[i] = { disabled: true };
}else if(typeof column === 'string'){
} else if (typeof column === "string") {
normalizedColumns[i] = { name: column };
}else if(is_object(column)){
if(typeof column.name !== 'string'){
throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [
'Option columns missing name:',
} else if (is_object(column)) {
if (typeof column.name !== "string") {
throw new CsvError("CSV_OPTION_COLUMNS_MISSING_NAME", [
"Option columns missing name:",
`property "name" is required at position ${i}`,
'when column is an object literal'
"when column is an object literal",
]);
}
normalizedColumns[i] = column;
}else{
throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [
'Invalid column definition:',
'expect a string or a literal object,',
`got ${JSON.stringify(column)} at position ${i}`
} else {
throw new CsvError("CSV_INVALID_COLUMN_DEFINITION", [
"Invalid column definition:",
"expect a string or a literal object,",
`got ${JSON.stringify(column)} at position ${i}`,
]);

@@ -33,2 +32,2 @@ }

export {normalize_columns_array};
export { normalize_columns_array };

752

lib/api/normalize_options.js

@@ -0,10 +1,9 @@

import { normalize_columns_array } from "./normalize_columns_array.js";
import { CsvError } from "./CsvError.js";
import { underscore } from "../utils/underscore.js";
import {normalize_columns_array} from './normalize_columns_array.js';
import {CsvError} from './CsvError.js';
import {underscore} from '../utils/underscore.js';
const normalize_options = function(opts){
const normalize_options = function (opts) {
const options = {};
// Merge with user options
for(const opt in opts){
for (const opt in opts) {
options[underscore(opt)] = opts[opt];

@@ -15,131 +14,213 @@ }

// to convert chars/strings into buffers.
if(options.encoding === undefined || options.encoding === true){
options.encoding = 'utf8';
}else if(options.encoding === null || options.encoding === false){
if (options.encoding === undefined || options.encoding === true) {
options.encoding = "utf8";
} else if (options.encoding === null || options.encoding === false) {
options.encoding = null;
}else if(typeof options.encoding !== 'string' && options.encoding !== null){
throw new CsvError('CSV_INVALID_OPTION_ENCODING', [
'Invalid option encoding:',
'encoding must be a string or null to return a buffer,',
`got ${JSON.stringify(options.encoding)}`
], options);
} else if (
typeof options.encoding !== "string" &&
options.encoding !== null
) {
throw new CsvError(
"CSV_INVALID_OPTION_ENCODING",
[
"Invalid option encoding:",
"encoding must be a string or null to return a buffer,",
`got ${JSON.stringify(options.encoding)}`,
],
options,
);
}
// Normalize option `bom`
if(options.bom === undefined || options.bom === null || options.bom === false){
if (
options.bom === undefined ||
options.bom === null ||
options.bom === false
) {
options.bom = false;
}else if(options.bom !== true){
throw new CsvError('CSV_INVALID_OPTION_BOM', [
'Invalid option bom:', 'bom must be true,',
`got ${JSON.stringify(options.bom)}`
], options);
} else if (options.bom !== true) {
throw new CsvError(
"CSV_INVALID_OPTION_BOM",
[
"Invalid option bom:",
"bom must be true,",
`got ${JSON.stringify(options.bom)}`,
],
options,
);
}
// Normalize option `cast`
options.cast_function = null;
if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){
if (
options.cast === undefined ||
options.cast === null ||
options.cast === false ||
options.cast === ""
) {
options.cast = undefined;
}else if(typeof options.cast === 'function'){
} else if (typeof options.cast === "function") {
options.cast_function = options.cast;
options.cast = true;
}else if(options.cast !== true){
throw new CsvError('CSV_INVALID_OPTION_CAST', [
'Invalid option cast:', 'cast must be true or a function,',
`got ${JSON.stringify(options.cast)}`
], options);
} else if (options.cast !== true) {
throw new CsvError(
"CSV_INVALID_OPTION_CAST",
[
"Invalid option cast:",
"cast must be true or a function,",
`got ${JSON.stringify(options.cast)}`,
],
options,
);
}
// Normalize option `cast_date`
if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){
if (
options.cast_date === undefined ||
options.cast_date === null ||
options.cast_date === false ||
options.cast_date === ""
) {
options.cast_date = false;
}else if(options.cast_date === true){
options.cast_date = function(value){
} else if (options.cast_date === true) {
options.cast_date = function (value) {
const date = Date.parse(value);
return !isNaN(date) ? new Date(date) : value;
};
}else if (typeof options.cast_date !== 'function'){
throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [
'Invalid option cast_date:', 'cast_date must be true or a function,',
`got ${JSON.stringify(options.cast_date)}`
], options);
} else if (typeof options.cast_date !== "function") {
throw new CsvError(
"CSV_INVALID_OPTION_CAST_DATE",
[
"Invalid option cast_date:",
"cast_date must be true or a function,",
`got ${JSON.stringify(options.cast_date)}`,
],
options,
);
}
// Normalize option `columns`
options.cast_first_line_to_header = null;
if(options.columns === true){
if (options.columns === true) {
// Fields in the first line are converted as-is to columns
options.cast_first_line_to_header = undefined;
}else if(typeof options.columns === 'function'){
} else if (typeof options.columns === "function") {
options.cast_first_line_to_header = options.columns;
options.columns = true;
}else if(Array.isArray(options.columns)){
} else if (Array.isArray(options.columns)) {
options.columns = normalize_columns_array(options.columns);
}else if(options.columns === undefined || options.columns === null || options.columns === false){
} else if (
options.columns === undefined ||
options.columns === null ||
options.columns === false
) {
options.columns = false;
}else{
throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [
'Invalid option columns:',
'expect an array, a function or true,',
`got ${JSON.stringify(options.columns)}`
], options);
} else {
throw new CsvError(
"CSV_INVALID_OPTION_COLUMNS",
[
"Invalid option columns:",
"expect an array, a function or true,",
`got ${JSON.stringify(options.columns)}`,
],
options,
);
}
// Normalize option `group_columns_by_name`
if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){
if (
options.group_columns_by_name === undefined ||
options.group_columns_by_name === null ||
options.group_columns_by_name === false
) {
options.group_columns_by_name = false;
}else if(options.group_columns_by_name !== true){
throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [
'Invalid option group_columns_by_name:',
'expect an boolean,',
`got ${JSON.stringify(options.group_columns_by_name)}`
], options);
}else if(options.columns === false){
throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [
'Invalid option group_columns_by_name:',
'the `columns` mode must be activated.'
], options);
} else if (options.group_columns_by_name !== true) {
throw new CsvError(
"CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME",
[
"Invalid option group_columns_by_name:",
"expect an boolean,",
`got ${JSON.stringify(options.group_columns_by_name)}`,
],
options,
);
} else if (options.columns === false) {
throw new CsvError(
"CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME",
[
"Invalid option group_columns_by_name:",
"the `columns` mode must be activated.",
],
options,
);
}
// Normalize option `comment`
if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){
if (
options.comment === undefined ||
options.comment === null ||
options.comment === false ||
options.comment === ""
) {
options.comment = null;
}else{
if(typeof options.comment === 'string'){
} else {
if (typeof options.comment === "string") {
options.comment = Buffer.from(options.comment, options.encoding);
}
if(!Buffer.isBuffer(options.comment)){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment:',
'comment must be a buffer or a string,',
`got ${JSON.stringify(options.comment)}`
], options);
if (!Buffer.isBuffer(options.comment)) {
throw new CsvError(
"CSV_INVALID_OPTION_COMMENT",
[
"Invalid option comment:",
"comment must be a buffer or a string,",
`got ${JSON.stringify(options.comment)}`,
],
options,
);
}
}
// Normalize option `comment_no_infix`
if(options.comment_no_infix === undefined || options.comment_no_infix === null || options.comment_no_infix === false){
if (
options.comment_no_infix === undefined ||
options.comment_no_infix === null ||
options.comment_no_infix === false
) {
options.comment_no_infix = false;
}else if(options.comment_no_infix !== true){
throw new CsvError('CSV_INVALID_OPTION_COMMENT', [
'Invalid option comment_no_infix:',
'value must be a boolean,',
`got ${JSON.stringify(options.comment_no_infix)}`
], options);
} else if (options.comment_no_infix !== true) {
throw new CsvError(
"CSV_INVALID_OPTION_COMMENT",
[
"Invalid option comment_no_infix:",
"value must be a boolean,",
`got ${JSON.stringify(options.comment_no_infix)}`,
],
options,
);
}
// Normalize option `delimiter`
const delimiter_json = JSON.stringify(options.delimiter);
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter];
if(options.delimiter.length === 0){
throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [
'Invalid option delimiter:',
'delimiter must be a non empty string or buffer or array of string|buffer,',
`got ${delimiter_json}`
], options);
if (!Array.isArray(options.delimiter))
options.delimiter = [options.delimiter];
if (options.delimiter.length === 0) {
throw new CsvError(
"CSV_INVALID_OPTION_DELIMITER",
[
"Invalid option delimiter:",
"delimiter must be a non empty string or buffer or array of string|buffer,",
`got ${delimiter_json}`,
],
options,
);
}
options.delimiter = options.delimiter.map(function(delimiter){
if(delimiter === undefined || delimiter === null || delimiter === false){
return Buffer.from(',', options.encoding);
options.delimiter = options.delimiter.map(function (delimiter) {
if (delimiter === undefined || delimiter === null || delimiter === false) {
return Buffer.from(",", options.encoding);
}
if(typeof delimiter === 'string'){
if (typeof delimiter === "string") {
delimiter = Buffer.from(delimiter, options.encoding);
}
if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){
throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [
'Invalid option delimiter:',
'delimiter must be a non empty string or buffer or array of string|buffer,',
`got ${delimiter_json}`
], options);
if (!Buffer.isBuffer(delimiter) || delimiter.length === 0) {
throw new CsvError(
"CSV_INVALID_OPTION_DELIMITER",
[
"Invalid option delimiter:",
"delimiter must be a non empty string or buffer or array of string|buffer,",
`got ${delimiter_json}`,
],
options,
);
}

@@ -149,99 +230,145 @@ return delimiter;

// Normalize option `escape`
if(options.escape === undefined || options.escape === true){
if (options.escape === undefined || options.escape === true) {
options.escape = Buffer.from('"', options.encoding);
}else if(typeof options.escape === 'string'){
} else if (typeof options.escape === "string") {
options.escape = Buffer.from(options.escape, options.encoding);
}else if (options.escape === null || options.escape === false){
} else if (options.escape === null || options.escape === false) {
options.escape = null;
}
if(options.escape !== null){
if(!Buffer.isBuffer(options.escape)){
throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`);
if (options.escape !== null) {
if (!Buffer.isBuffer(options.escape)) {
throw new Error(
`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`,
);
}
}
// Normalize option `from`
if(options.from === undefined || options.from === null){
if (options.from === undefined || options.from === null) {
options.from = 1;
}else{
if(typeof options.from === 'string' && /\d+/.test(options.from)){
} else {
if (typeof options.from === "string" && /\d+/.test(options.from)) {
options.from = parseInt(options.from);
}
if(Number.isInteger(options.from)){
if(options.from < 0){
throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`);
if (Number.isInteger(options.from)) {
if (options.from < 0) {
throw new Error(
`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`,
);
}
}else{
throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`);
} else {
throw new Error(
`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`,
);
}
}
// Normalize option `from_line`
if(options.from_line === undefined || options.from_line === null){
if (options.from_line === undefined || options.from_line === null) {
options.from_line = 1;
}else{
if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){
} else {
if (
typeof options.from_line === "string" &&
/\d+/.test(options.from_line)
) {
options.from_line = parseInt(options.from_line);
}
if(Number.isInteger(options.from_line)){
if(options.from_line <= 0){
throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`);
if (Number.isInteger(options.from_line)) {
if (options.from_line <= 0) {
throw new Error(
`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`,
);
}
}else{
throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`);
} else {
throw new Error(
`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`,
);
}
}
// Normalize options `ignore_last_delimiters`
if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){
if (
options.ignore_last_delimiters === undefined ||
options.ignore_last_delimiters === null
) {
options.ignore_last_delimiters = false;
}else if(typeof options.ignore_last_delimiters === 'number'){
} else if (typeof options.ignore_last_delimiters === "number") {
options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters);
if(options.ignore_last_delimiters === 0){
if (options.ignore_last_delimiters === 0) {
options.ignore_last_delimiters = false;
}
}else if(typeof options.ignore_last_delimiters !== 'boolean'){
throw new CsvError('CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS', [
'Invalid option `ignore_last_delimiters`:',
'the value must be a boolean value or an integer,',
`got ${JSON.stringify(options.ignore_last_delimiters)}`
], options);
} else if (typeof options.ignore_last_delimiters !== "boolean") {
throw new CsvError(
"CSV_INVALID_OPTION_IGNORE_LAST_DELIMITERS",
[
"Invalid option `ignore_last_delimiters`:",
"the value must be a boolean value or an integer,",
`got ${JSON.stringify(options.ignore_last_delimiters)}`,
],
options,
);
}
if(options.ignore_last_delimiters === true && options.columns === false){
throw new CsvError('CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS', [
'The option `ignore_last_delimiters`',
'requires the activation of the `columns` option'
], options);
if (options.ignore_last_delimiters === true && options.columns === false) {
throw new CsvError(
"CSV_IGNORE_LAST_DELIMITERS_REQUIRES_COLUMNS",
[
"The option `ignore_last_delimiters`",
"requires the activation of the `columns` option",
],
options,
);
}
// Normalize option `info`
if(options.info === undefined || options.info === null || options.info === false){
if (
options.info === undefined ||
options.info === null ||
options.info === false
) {
options.info = false;
}else if(options.info !== true){
throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`);
} else if (options.info !== true) {
throw new Error(
`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`,
);
}
// Normalize option `max_record_size`
if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){
if (
options.max_record_size === undefined ||
options.max_record_size === null ||
options.max_record_size === false
) {
options.max_record_size = 0;
}else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0){
} else if (
Number.isInteger(options.max_record_size) &&
options.max_record_size >= 0
) {
// Great, nothing to do
}else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){
} else if (
typeof options.max_record_size === "string" &&
/\d+/.test(options.max_record_size)
) {
options.max_record_size = parseInt(options.max_record_size);
}else{
throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`);
} else {
throw new Error(
`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`,
);
}
// Normalize option `objname`
if(options.objname === undefined || options.objname === null || options.objname === false){
if (
options.objname === undefined ||
options.objname === null ||
options.objname === false
) {
options.objname = undefined;
}else if(Buffer.isBuffer(options.objname)){
if(options.objname.length === 0){
} else if (Buffer.isBuffer(options.objname)) {
if (options.objname.length === 0) {
throw new Error(`Invalid Option: objname must be a non empty buffer`);
}
if(options.encoding === null){
if (options.encoding === null) {
// Don't call `toString`, leave objname as a buffer
}else{
} else {
options.objname = options.objname.toString(options.encoding);
}
}else if(typeof options.objname === 'string'){
if(options.objname.length === 0){
} else if (typeof options.objname === "string") {
if (options.objname.length === 0) {
throw new Error(`Invalid Option: objname must be a non empty string`);
}
// Great, nothing to do
}else if(typeof options.objname === 'number'){
} else if (typeof options.objname === "number") {
// if(options.objname.length === 0){

@@ -251,13 +378,20 @@ // throw new Error(`Invalid Option: objname must be a non empty string`);

// Great, nothing to do
}else{
throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`);
} else {
throw new Error(
`Invalid Option: objname must be a string or a buffer, got ${options.objname}`,
);
}
if(options.objname !== undefined){
if(typeof options.objname === 'number'){
if(options.columns !== false){
throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field');
if (options.objname !== undefined) {
if (typeof options.objname === "number") {
if (options.columns !== false) {
throw Error(
"Invalid Option: objname index cannot be combined with columns or be defined as a field",
);
}
}else{ // A string or a buffer
if(options.columns === false){
throw Error('Invalid Option: objname field must be combined with columns or be defined as an index');
} else {
// A string or a buffer
if (options.columns === false) {
throw Error(
"Invalid Option: objname field must be combined with columns or be defined as an index",
);
}

@@ -267,10 +401,14 @@ }

// Normalize option `on_record`
if(options.on_record === undefined || options.on_record === null){
if (options.on_record === undefined || options.on_record === null) {
options.on_record = undefined;
}else if(typeof options.on_record !== 'function'){
throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [
'Invalid option `on_record`:',
'expect a function,',
`got ${JSON.stringify(options.on_record)}`
], options);
} else if (typeof options.on_record !== "function") {
throw new CsvError(
"CSV_INVALID_OPTION_ON_RECORD",
[
"Invalid option `on_record`:",
"expect a function,",
`got ${JSON.stringify(options.on_record)}`,
],
options,
);
}

@@ -281,60 +419,97 @@ // Normalize option `on_skip`

// };
if(options.on_skip !== undefined && options.on_skip !== null && typeof options.on_skip !== 'function'){
throw new Error(`Invalid Option: on_skip must be a function, got ${JSON.stringify(options.on_skip)}`);
if (
options.on_skip !== undefined &&
options.on_skip !== null &&
typeof options.on_skip !== "function"
) {
throw new Error(
`Invalid Option: on_skip must be a function, got ${JSON.stringify(options.on_skip)}`,
);
}
// Normalize option `quote`
if(options.quote === null || options.quote === false || options.quote === ''){
if (
options.quote === null ||
options.quote === false ||
options.quote === ""
) {
options.quote = null;
}else{
if(options.quote === undefined || options.quote === true){
} else {
if (options.quote === undefined || options.quote === true) {
options.quote = Buffer.from('"', options.encoding);
}else if(typeof options.quote === 'string'){
} else if (typeof options.quote === "string") {
options.quote = Buffer.from(options.quote, options.encoding);
}
if(!Buffer.isBuffer(options.quote)){
throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`);
if (!Buffer.isBuffer(options.quote)) {
throw new Error(
`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`,
);
}
}
// Normalize option `raw`
if(options.raw === undefined || options.raw === null || options.raw === false){
if (
options.raw === undefined ||
options.raw === null ||
options.raw === false
) {
options.raw = false;
}else if(options.raw !== true){
throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`);
} else if (options.raw !== true) {
throw new Error(
`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`,
);
}
// Normalize option `record_delimiter`
if(options.record_delimiter === undefined){
if (options.record_delimiter === undefined) {
options.record_delimiter = [];
}else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){
if(options.record_delimiter.length === 0){
throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [
'Invalid option `record_delimiter`:',
'value must be a non empty string or buffer,',
`got ${JSON.stringify(options.record_delimiter)}`
], options);
} else if (
typeof options.record_delimiter === "string" ||
Buffer.isBuffer(options.record_delimiter)
) {
if (options.record_delimiter.length === 0) {
throw new CsvError(
"CSV_INVALID_OPTION_RECORD_DELIMITER",
[
"Invalid option `record_delimiter`:",
"value must be a non empty string or buffer,",
`got ${JSON.stringify(options.record_delimiter)}`,
],
options,
);
}
options.record_delimiter = [options.record_delimiter];
}else if(!Array.isArray(options.record_delimiter)){
throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [
'Invalid option `record_delimiter`:',
'value must be a string, a buffer or array of string|buffer,',
`got ${JSON.stringify(options.record_delimiter)}`
], options);
} else if (!Array.isArray(options.record_delimiter)) {
throw new CsvError(
"CSV_INVALID_OPTION_RECORD_DELIMITER",
[
"Invalid option `record_delimiter`:",
"value must be a string, a buffer or array of string|buffer,",
`got ${JSON.stringify(options.record_delimiter)}`,
],
options,
);
}
options.record_delimiter = options.record_delimiter.map(function(rd, i){
if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){
throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [
'Invalid option `record_delimiter`:',
'value must be a string, a buffer or array of string|buffer',
`at index ${i},`,
`got ${JSON.stringify(rd)}`
], options);
}else if(rd.length === 0){
throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [
'Invalid option `record_delimiter`:',
'value must be a non empty string or buffer',
`at index ${i},`,
`got ${JSON.stringify(rd)}`
], options);
options.record_delimiter = options.record_delimiter.map(function (rd, i) {
if (typeof rd !== "string" && !Buffer.isBuffer(rd)) {
throw new CsvError(
"CSV_INVALID_OPTION_RECORD_DELIMITER",
[
"Invalid option `record_delimiter`:",
"value must be a string, a buffer or array of string|buffer",
`at index ${i},`,
`got ${JSON.stringify(rd)}`,
],
options,
);
} else if (rd.length === 0) {
throw new CsvError(
"CSV_INVALID_OPTION_RECORD_DELIMITER",
[
"Invalid option `record_delimiter`:",
"value must be a non empty string or buffer",
`at index ${i},`,
`got ${JSON.stringify(rd)}`,
],
options,
);
}
if(typeof rd === 'string'){
if (typeof rd === "string") {
rd = Buffer.from(rd, options.encoding);

@@ -345,112 +520,173 @@ }

// Normalize option `relax_column_count`
if(typeof options.relax_column_count === 'boolean'){
if (typeof options.relax_column_count === "boolean") {
// Great, nothing to do
}else if(options.relax_column_count === undefined || options.relax_column_count === null){
} else if (
options.relax_column_count === undefined ||
options.relax_column_count === null
) {
options.relax_column_count = false;
}else{
throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`);
} else {
throw new Error(
`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`,
);
}
if(typeof options.relax_column_count_less === 'boolean'){
if (typeof options.relax_column_count_less === "boolean") {
// Great, nothing to do
}else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){
} else if (
options.relax_column_count_less === undefined ||
options.relax_column_count_less === null
) {
options.relax_column_count_less = false;
}else{
throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`);
} else {
throw new Error(
`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`,
);
}
if(typeof options.relax_column_count_more === 'boolean'){
if (typeof options.relax_column_count_more === "boolean") {
// Great, nothing to do
}else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){
} else if (
options.relax_column_count_more === undefined ||
options.relax_column_count_more === null
) {
options.relax_column_count_more = false;
}else{
throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`);
} else {
throw new Error(
`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`,
);
}
// Normalize option `relax_quotes`
if(typeof options.relax_quotes === 'boolean'){
if (typeof options.relax_quotes === "boolean") {
// Great, nothing to do
}else if(options.relax_quotes === undefined || options.relax_quotes === null){
} else if (
options.relax_quotes === undefined ||
options.relax_quotes === null
) {
options.relax_quotes = false;
}else{
throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`);
} else {
throw new Error(
`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`,
);
}
// Normalize option `skip_empty_lines`
if(typeof options.skip_empty_lines === 'boolean'){
if (typeof options.skip_empty_lines === "boolean") {
// Great, nothing to do
}else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){
} else if (
options.skip_empty_lines === undefined ||
options.skip_empty_lines === null
) {
options.skip_empty_lines = false;
}else{
throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`);
} else {
throw new Error(
`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`,
);
}
// Normalize option `skip_records_with_empty_values`
if(typeof options.skip_records_with_empty_values === 'boolean'){
if (typeof options.skip_records_with_empty_values === "boolean") {
// Great, nothing to do
}else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){
} else if (
options.skip_records_with_empty_values === undefined ||
options.skip_records_with_empty_values === null
) {
options.skip_records_with_empty_values = false;
}else{
throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`);
} else {
throw new Error(
`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`,
);
}
// Normalize option `skip_records_with_error`
if(typeof options.skip_records_with_error === 'boolean'){
if (typeof options.skip_records_with_error === "boolean") {
// Great, nothing to do
}else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){
} else if (
options.skip_records_with_error === undefined ||
options.skip_records_with_error === null
) {
options.skip_records_with_error = false;
}else{
throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`);
} else {
throw new Error(
`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`,
);
}
// Normalize option `rtrim`
if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){
if (
options.rtrim === undefined ||
options.rtrim === null ||
options.rtrim === false
) {
options.rtrim = false;
}else if(options.rtrim !== true){
throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`);
} else if (options.rtrim !== true) {
throw new Error(
`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`,
);
}
// Normalize option `ltrim`
if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){
if (
options.ltrim === undefined ||
options.ltrim === null ||
options.ltrim === false
) {
options.ltrim = false;
}else if(options.ltrim !== true){
throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`);
} else if (options.ltrim !== true) {
throw new Error(
`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`,
);
}
// Normalize option `trim`
if(options.trim === undefined || options.trim === null || options.trim === false){
if (
options.trim === undefined ||
options.trim === null ||
options.trim === false
) {
options.trim = false;
}else if(options.trim !== true){
throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`);
} else if (options.trim !== true) {
throw new Error(
`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`,
);
}
// Normalize options `trim`, `ltrim` and `rtrim`
if(options.trim === true && opts.ltrim !== false){
if (options.trim === true && opts.ltrim !== false) {
options.ltrim = true;
}else if(options.ltrim !== true){
} else if (options.ltrim !== true) {
options.ltrim = false;
}
if(options.trim === true && opts.rtrim !== false){
if (options.trim === true && opts.rtrim !== false) {
options.rtrim = true;
}else if(options.rtrim !== true){
} else if (options.rtrim !== true) {
options.rtrim = false;
}
// Normalize option `to`
if(options.to === undefined || options.to === null){
if (options.to === undefined || options.to === null) {
options.to = -1;
}else{
if(typeof options.to === 'string' && /\d+/.test(options.to)){
} else {
if (typeof options.to === "string" && /\d+/.test(options.to)) {
options.to = parseInt(options.to);
}
if(Number.isInteger(options.to)){
if(options.to <= 0){
throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`);
if (Number.isInteger(options.to)) {
if (options.to <= 0) {
throw new Error(
`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`,
);
}
}else{
throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`);
} else {
throw new Error(
`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`,
);
}
}
// Normalize option `to_line`
if(options.to_line === undefined || options.to_line === null){
if (options.to_line === undefined || options.to_line === null) {
options.to_line = -1;
}else{
if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){
} else {
if (typeof options.to_line === "string" && /\d+/.test(options.to_line)) {
options.to_line = parseInt(options.to_line);
}
if(Number.isInteger(options.to_line)){
if(options.to_line <= 0){
throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`);
if (Number.isInteger(options.to_line)) {
if (options.to_line <= 0) {
throw new Error(
`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`,
);
}
}else{
throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`);
} else {
throw new Error(
`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`,
);
}

@@ -461,2 +697,2 @@ }

export {normalize_options};
export { normalize_options };

135

lib/index.js

@@ -1,2 +0,1 @@

/*

@@ -9,13 +8,16 @@ CSV Parse

import { Transform } from 'stream';
import {is_object} from './utils/is_object.js';
import {transform} from './api/index.js';
import {CsvError} from './api/CsvError.js';
import { Transform } from "stream";
import { is_object } from "./utils/is_object.js";
import { transform } from "./api/index.js";
import { CsvError } from "./api/CsvError.js";
class Parser extends Transform {
constructor(opts = {}){
super({...{readableObjectMode: true}, ...opts, encoding: null});
this.api = transform({on_skip: (err, chunk) => {
this.emit('skip', err, chunk);
}, ...opts});
constructor(opts = {}) {
super({ ...{ readableObjectMode: true }, ...opts, encoding: null });
this.api = transform({
on_skip: (err, chunk) => {
this.emit("skip", err, chunk);
},
...opts,
});
// Backward compatibility

@@ -27,23 +29,28 @@ this.state = this.api.state;

// Implementation of `Transform._transform`
_transform(buf, _, callback){
if(this.state.stop === true){
_transform(buf, _, callback) {
if (this.state.stop === true) {
return;
}
const err = this.api.parse(buf, false, (record) => {
this.push(record);
}, () => {
this.push(null);
this.end();
// Fix #333 and break #410
// ko: api.stream.iterator.coffee
// ko with v21.4.0, ok with node v20.5.1: api.stream.finished # aborted (with generate())
// ko: api.stream.finished # aborted (with Readable)
// this.destroy()
// Fix #410 and partially break #333
// ok: api.stream.iterator.coffee
// ok: api.stream.finished # aborted (with generate())
// broken: api.stream.finished # aborted (with Readable)
this.on('end', this.destroy);
});
if(err !== undefined){
const err = this.api.parse(
buf,
false,
(record) => {
this.push(record);
},
() => {
this.push(null);
this.end();
// Fix #333 and break #410
// ko: api.stream.iterator.coffee
// ko with v21.4.0, ok with node v20.5.1: api.stream.finished # aborted (with generate())
// ko: api.stream.finished # aborted (with Readable)
// this.destroy()
// Fix #410 and partially break #333
// ok: api.stream.iterator.coffee
// ok: api.stream.finished # aborted (with generate())
// broken: api.stream.finished # aborted (with Readable)
this.on("end", this.destroy);
},
);
if (err !== undefined) {
this.state.stop = true;

@@ -54,12 +61,17 @@ }

// Implementation of `Transform._flush`
_flush(callback){
if(this.state.stop === true){
_flush(callback) {
if (this.state.stop === true) {
return;
}
const err = this.api.parse(undefined, true, (record) => {
this.push(record);
}, () => {
this.push(null);
this.on('end', this.destroy);
});
const err = this.api.parse(
undefined,
true,
(record) => {
this.push(record);
},
() => {
this.push(null);
this.on("end", this.destroy);
},
);
callback(err);

@@ -69,29 +81,34 @@ }

const parse = function(){
const parse = function () {
let data, options, callback;
for(const i in arguments){
for (const i in arguments) {
const argument = arguments[i];
const type = typeof argument;
if(data === undefined && (typeof argument === 'string' || Buffer.isBuffer(argument))){
if (
data === undefined &&
(typeof argument === "string" || Buffer.isBuffer(argument))
) {
data = argument;
}else if(options === undefined && is_object(argument)){
} else if (options === undefined && is_object(argument)) {
options = argument;
}else if(callback === undefined && type === 'function'){
} else if (callback === undefined && type === "function") {
callback = argument;
}else{
throw new CsvError('CSV_INVALID_ARGUMENT', [
'Invalid argument:',
`got ${JSON.stringify(argument)} at index ${i}`
], options || {});
} else {
throw new CsvError(
"CSV_INVALID_ARGUMENT",
["Invalid argument:", `got ${JSON.stringify(argument)} at index ${i}`],
options || {},
);
}
}
const parser = new Parser(options);
if(callback){
const records = options === undefined || options.objname === undefined ? [] : {};
parser.on('readable', function(){
if (callback) {
const records =
options === undefined || options.objname === undefined ? [] : {};
parser.on("readable", function () {
let record;
while((record = this.read()) !== null){
if(options === undefined || options.objname === undefined){
while ((record = this.read()) !== null) {
if (options === undefined || options.objname === undefined) {
records.push(record);
}else{
} else {
records[record[0]] = record[1];

@@ -101,11 +118,11 @@ }

});
parser.on('error', function(err){
parser.on("error", function (err) {
callback(err, undefined, parser.api.__infoDataSet());
});
parser.on('end', function(){
parser.on("end", function () {
callback(undefined, records, parser.api.__infoDataSet());
});
}
if(data !== undefined){
const writer = function(){
if (data !== undefined) {
const writer = function () {
parser.write(data);

@@ -115,5 +132,5 @@ parser.end();

// Support Deno, Rollup doesnt provide a shim for setImmediate
if(typeof setImmediate === 'function'){
if (typeof setImmediate === "function") {
setImmediate(writer);
}else{
} else {
setTimeout(writer, 0);

@@ -120,0 +137,0 @@ }

51

lib/stream.js

@@ -0,27 +1,36 @@

import { TransformStream, CountQueuingStrategy } from "node:stream/web";
import { transform } from "./api/index.js";
import {
TransformStream,
} from 'node:stream/web';
import {transform} from './api/index.js';
const parse = (opts) => {
const api = transform(opts);
return new TransformStream({
async transform(chunk, controller) {
api.parse(chunk, false, (record) => {
controller.enqueue(record);
}, () => {
controller.close();
});
let controller;
const enqueue = (record) => {
controller.enqueue(record);
};
const terminate = () => {
controller.terminate();
};
return new TransformStream(
{
start(ctr) {
controller = ctr;
},
transform(chunk) {
const error = api.parse(chunk, false, enqueue, terminate);
if (error) {
controller.error(error);
}
},
flush() {
const error = api.parse(undefined, true, enqueue, terminate);
if (error) {
controller.error(error);
}
},
},
async flush(controller){
api.parse(undefined, true, (record) => {
controller.enqueue(record);
}, () => {
controller.close();
});
}
});
new CountQueuingStrategy({ highWaterMark: 1024 }),
new CountQueuingStrategy({ highWaterMark: 1024 }),
);
};
export {parse};
export { parse };

16

lib/sync.js

@@ -0,6 +1,5 @@

import { CsvError, transform } from "./api/index.js";
import {CsvError, transform} from './api/index.js';
const parse = function(data, opts={}){
if(typeof data === 'string'){
const parse = function (data, opts = {}) {
if (typeof data === "string") {
data = Buffer.from(data);

@@ -11,5 +10,4 @@ }

const push = (record) => {
if(parser.options.objname === undefined)
records.push(record);
else{
if (parser.options.objname === undefined) records.push(record);
else {
records[record[0]] = record[1];

@@ -20,5 +18,5 @@ }

const err1 = parser.parse(data, false, push, close);
if(err1 !== undefined) throw err1;
if (err1 !== undefined) throw err1;
const err2 = parser.parse(undefined, true, push, close);
if(err2 !== undefined) throw err2;
if (err2 !== undefined) throw err2;
return records;

@@ -25,0 +23,0 @@ };

7

lib/utils/is_object.js

@@ -1,6 +0,5 @@

const is_object = function(obj){
return (typeof obj === 'object' && obj !== null && !Array.isArray(obj));
const is_object = function (obj) {
return typeof obj === "object" && obj !== null && !Array.isArray(obj);
};
export {is_object};
export { is_object };

43

lib/utils/ResizeableBuffer.js

@@ -1,4 +0,3 @@

class ResizeableBuffer{
constructor(size=100){
class ResizeableBuffer {
constructor(size = 100) {
this.size = size;

@@ -8,9 +7,9 @@ this.length = 0;

}
prepend(val){
if(Buffer.isBuffer(val)){
prepend(val) {
if (Buffer.isBuffer(val)) {
const length = this.length + val.length;
if(length >= this.size){
if (length >= this.size) {
this.resize();
if(length >= this.size){
throw Error('INVALID_BUFFER_STATE');
if (length >= this.size) {
throw Error("INVALID_BUFFER_STATE");
}

@@ -23,5 +22,5 @@ }

this.length += val.length;
}else{
} else {
const length = this.length++;
if(length === this.size){
if (length === this.size) {
this.resize();

@@ -31,8 +30,8 @@ }

this.buf[0] = val;
buf.copy(this.buf,1, 0, length);
buf.copy(this.buf, 1, 0, length);
}
}
append(val){
append(val) {
const length = this.length++;
if(length === this.size){
if (length === this.size) {
this.resize();

@@ -42,23 +41,23 @@ }

}
clone(){
clone() {
return Buffer.from(this.buf.slice(0, this.length));
}
resize(){
resize() {
const length = this.length;
this.size = this.size * 2;
const buf = Buffer.allocUnsafe(this.size);
this.buf.copy(buf,0, 0, length);
this.buf.copy(buf, 0, 0, length);
this.buf = buf;
}
toString(encoding){
if(encoding){
toString(encoding) {
if (encoding) {
return this.buf.slice(0, this.length).toString(encoding);
}else{
} else {
return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length));
}
}
toJSON(){
return this.toString('utf8');
toJSON() {
return this.toString("utf8");
}
reset(){
reset() {
this.length = 0;

@@ -65,0 +64,0 @@ }

9

lib/utils/underscore.js

@@ -1,8 +0,7 @@

const underscore = function(str){
return str.replace(/([A-Z])/g, function(_, match){
return '_' + match.toLowerCase();
const underscore = function (str) {
return str.replace(/([A-Z])/g, function (_, match) {
return "_" + match.toLowerCase();
});
};
export {underscore};
export { underscore };

64

package.json
{
"version": "5.5.6",
"version": "5.6.0",
"name": "csv-parse",

@@ -55,2 +55,12 @@ "description": "CSV parsing implementing the Node.js `stream.Transform` API",

},
"./stream": {
"import": {
"types": "./lib/stream.d.ts",
"default": "./lib/stream.js"
},
"require": {
"types": "./dist/cjs/stream.d.cts",
"default": "./dist/cjs/stream.cjs"
}
},
"./browser/esm": {

@@ -66,21 +76,24 @@ "types": "./dist/esm/index.d.ts",

"devDependencies": {
"@rollup/plugin-eslint": "^9.0.4",
"@rollup/plugin-node-resolve": "^15.2.1",
"@types/mocha": "^10.0.1",
"@types/node": "^20.5.6",
"coffeelint": "^2.1.0",
"@eslint/js": "^9.15.0",
"@rollup/plugin-node-resolve": "^15.3.0",
"@types/mocha": "^10.0.9",
"@types/node": "^22.9.1",
"coffeescript": "^2.7.0",
"csv-generate": "^4.4.1",
"csv-spectrum": "^1.0.0",
"each": "^2.4.0",
"eslint": "^8.47.0",
"mocha": "^10.2.0",
"pad": "^3.2.0",
"rollup": "^3.28.1",
"csv-generate": "^4.4.2",
"csv-spectrum": "^2.0.0",
"each": "^2.7.2",
"eslint": "^9.15.0",
"eslint-config-prettier": "^9.1.0",
"eslint-plugin-mocha": "^10.5.0",
"eslint-plugin-prettier": "^5.2.1",
"mocha": "^10.8.2",
"pad": "^3.3.0",
"prettier": "^3.3.3",
"rollup": "^4.27.3",
"rollup-plugin-node-builtins": "^2.1.2",
"rollup-plugin-node-globals": "^1.4.0",
"should": "^13.2.3",
"stream-transform": "^3.3.2",
"ts-node": "^10.9.1",
"typescript": "^5.2.2"
"stream-transform": "^3.3.3",
"ts-node": "^10.9.2",
"typescript": "^5.6.3"
},

@@ -102,5 +115,9 @@ "files": [

],
"throw-deprecation": true,
"throw-deprecation": false,
"timeout": 40000
},
"lint-staged": {
"*.js": "npm run lint:fix",
"*.md": "prettier -w"
},
"repository": {

@@ -116,11 +133,8 @@ "type": "git",

"postbuild:ts": "find dist/cjs -name '*.d.cts' -exec sh -c \"sed -i \"s/\\.js'/\\.cjs'/g\" {} || sed -i '' \"s/\\.js'/\\.cjs'/g\" {}\" \\;",
"lint": "npm run lint:lib && npm run lint:samples && npm run lint:test",
"postlint": "tsc --noEmit true",
"lint:lib": "eslint --fix lib/*.js",
"lint:samples": "eslint --fix samples/*.js",
"lint:test": "coffeelint --fix test/*.coffee",
"lint:check": "eslint",
"lint:fix": "eslint --fix",
"lint:ts": "tsc --noEmit true",
"preversion": "npm run build && git add dist",
"pretest": "npm run build",
"test": "mocha 'test/**/*.{coffee,ts}'",
"test:legacy": "mocha --ignore test/api.web_stream.coffee --ignore test/api.stream.finished.coffee --ignore test/api.stream.iterator.coffee --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'"
"test:legacy": "mocha --ignore test/api.web_stream.coffee --ignore test/api.web_stream.ts --ignore test/api.stream.finished.coffee --ignore test/api.stream.iterator.coffee --loader=./test/loaders/legacy/all.js 'test/**/*.{coffee,ts}'"
},

@@ -145,3 +159,3 @@ "type": "module",

},
"gitHead": "6aadcace1a9e77ea308159733391e42ad019c5cb"
"gitHead": "cc1235a58de98dd9eab0665c7b1d03213e9633c7"
}

62

README.md

@@ -1,2 +0,1 @@

# CSV parser for Node.js and the web

@@ -7,3 +6,3 @@

[![NPM](https://img.shields.io/npm/v/csv-parse)](https://www.npmjs.com/package/csv-parse)
The [`csv-parse` package](https://csv.js.org/parse/) is a parser converting CSV text input into arrays or objects. It is part of the [CSV project](https://csv.js.org/).

@@ -15,22 +14,22 @@

* [Project homepage](https://csv.js.org/parse/)
* [API](https://csv.js.org/parse/api/)
* [Options](https://csv.js.org/parse/options/)
* [Info properties](https://csv.js.org/parse/info/)
* [Common errors](https://csv.js.org/parse/errors/)
* [Examples](https://csv.js.org/project/examples/)
- [Project homepage](https://csv.js.org/parse/)
- [API](https://csv.js.org/parse/api/)
- [Options](https://csv.js.org/parse/options/)
- [Info properties](https://csv.js.org/parse/info/)
- [Common errors](https://csv.js.org/parse/errors/)
- [Examples](https://csv.js.org/project/examples/)
## Main features
* Flexible with lot of [options](https://csv.js.org/parse/options/)
* Multiple [distributions](https://csv.js.org/parse/distributions/): Node.js, Web, ECMAScript modules and CommonJS
* Follow the Node.js streaming API
* Simplicity with the optional callback API
* Support delimiters, quotes, escape characters and comments
* Line breaks discovery
* Support big datasets
* Complete test coverage and lot of samples for inspiration
* No external dependencies
* Work nicely with the [csv-generate](https://csv.js.org/generate/), [stream-transform](https://csv.js.org/transform/) and [csv-stringify](https://csv.js.org/stringify/) packages
* MIT License
- Flexible with lot of [options](https://csv.js.org/parse/options/)
- Multiple [distributions](https://csv.js.org/parse/distributions/): Node.js, Web, ECMAScript modules and CommonJS
- Follow the Node.js streaming API
- Simplicity with the optional callback API
- Support delimiters, quotes, escape characters and comments
- Line breaks discovery
- Support big datasets
- Complete test coverage and lot of samples for inspiration
- No external dependencies
- Work nicely with the [csv-generate](https://csv.js.org/generate/), [stream-transform](https://csv.js.org/transform/) and [csv-stringify](https://csv.js.org/stringify/) packages
- MIT License

@@ -48,4 +47,4 @@ ## Usage

```js
import assert from 'assert';
import { parse } from 'csv-parse';
import assert from "assert";
import { parse } from "csv-parse";

@@ -55,6 +54,6 @@ const records = [];

const parser = parse({
delimiter: ':'
delimiter: ":",
});
// Use the readable stream api to consume records
parser.on('readable', function(){
parser.on("readable", function () {
let record;

@@ -66,14 +65,11 @@ while ((record = parser.read()) !== null) {

// Catch any error
parser.on('error', function(err){
parser.on("error", function (err) {
console.error(err.message);
});
// Test that the parsed records matched the expected records
parser.on('end', function(){
assert.deepStrictEqual(
records,
[
[ 'root','x','0','0','root','/root','/bin/bash' ],
[ 'someone','x','1022','1022','','/home/someone','/bin/bash' ]
]
);
parser.on("end", function () {
assert.deepStrictEqual(records, [
["root", "x", "0", "0", "root", "/root", "/bin/bash"],
["someone", "x", "1022", "1022", "", "/home/someone", "/bin/bash"],
]);
});

@@ -91,2 +87,2 @@ // Write data to the stream

* David Worms: <https://github.com/wdavidw>
- David Worms: <https://github.com/wdavidw>
./dist/cjs/index.cjs
dist/cjs/index.cjs

Sorry, the diff of this file is not supported yet

dist/cjs/sync.cjs

Sorry, the diff of this file is not supported yet

dist/esm/index.js

Sorry, the diff of this file is too big to display

dist/esm/sync.js

Sorry, the diff of this file is too big to display

dist/iife/index.js

Sorry, the diff of this file is too big to display

dist/iife/sync.js

Sorry, the diff of this file is too big to display

dist/umd/index.js

Sorry, the diff of this file is too big to display

dist/umd/sync.js

Sorry, the diff of this file is too big to display

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc