csv-parse
Advanced tools
Comparing version 4.16.3 to 5.0.0
@@ -7,263 +7,268 @@ // Original definitions in https://github.com/DefinitelyTyped/DefinitelyTyped by: David Muller <https://github.com/davidm77> | ||
export = parse; | ||
export type Callback = (err: CsvError | undefined, records: any | undefined, info: Info) => void; | ||
declare function parse(input: Buffer | string, options?: parse.Options, callback?: parse.Callback): parse.Parser; | ||
declare function parse(input: Buffer | string, callback?: parse.Callback): parse.Parser; | ||
declare function parse(options?: parse.Options, callback?: parse.Callback): parse.Parser; | ||
declare function parse(callback?: parse.Callback): parse.Parser; | ||
declare namespace parse { | ||
export interface Parser extends stream.Transform {} | ||
type Callback = (err: Error | undefined, records: any | undefined, info: Info) => void; | ||
export class Parser { | ||
constructor(options: Options); | ||
__push(line: any): any; | ||
__write(chars: any, end: any, callback: any): any; | ||
readonly options: Options | ||
readonly info: Info; | ||
} | ||
interface Parser extends stream.Transform {} | ||
export interface CastingContext { | ||
readonly column: number | string; | ||
readonly empty_lines: number; | ||
readonly error: CsvError; | ||
readonly header: boolean; | ||
readonly index: number; | ||
readonly quoting: boolean; | ||
readonly lines: number; | ||
readonly records: number; | ||
readonly invalid_field_length: number; | ||
} | ||
class Parser { | ||
constructor(options: Options); | ||
__push(line: any): any; | ||
__write(chars: any, end: any, callback: any): any; | ||
readonly options: Options | ||
readonly info: Info; | ||
} | ||
export type CastingFunction = (value: string, context: CastingContext) => any; | ||
interface CastingContext { | ||
readonly column: number | string; | ||
readonly empty_lines: number; | ||
readonly error: CsvError; | ||
readonly header: boolean; | ||
readonly index: number; | ||
readonly quoting: boolean; | ||
readonly lines: number; | ||
readonly records: number; | ||
readonly invalid_field_length: number; | ||
} | ||
export type CastingDateFunction = (value: string, context: CastingContext) => Date; | ||
type CastingFunction = (value: string, context: CastingContext) => any; | ||
export type ColumnOption = string | undefined | null | false | { name: string }; | ||
type CastingDateFunction = (value: string, context: CastingContext) => Date; | ||
/* | ||
Note, could not `extends stream.TransformOptions` because encoding can be | ||
BufferEncoding and undefined as well as null which is not defined in the | ||
extended type. | ||
*/ | ||
export interface Options { | ||
/** | ||
* If true, the parser will attempt to convert read data types to native types. | ||
* @deprecated Use {@link cast} | ||
*/ | ||
auto_parse?: boolean | CastingFunction; | ||
autoParse?: boolean | CastingFunction; | ||
/** | ||
* If true, the parser will attempt to convert read data types to dates. It requires the "auto_parse" option. | ||
* @deprecated Use {@link cast_date} | ||
*/ | ||
auto_parse_date?: boolean | CastingDateFunction; | ||
autoParseDate?: boolean | CastingDateFunction; | ||
/** | ||
* If true, detect and exclude the byte order mark (BOM) from the CSV input if present. | ||
*/ | ||
bom?: boolean; | ||
/** | ||
* If true, the parser will attempt to convert input string to native types. | ||
* If a function, receive the value as first argument, a context as second argument and return a new value. More information about the context properties is available below. | ||
*/ | ||
cast?: boolean | CastingFunction; | ||
/** | ||
* If true, the parser will attempt to convert input string to dates. | ||
* If a function, receive the value as argument and return a new value. It requires the "auto_parse" option. Be careful, it relies on Date.parse. | ||
*/ | ||
cast_date?: boolean | CastingDateFunction; | ||
castDate?: boolean | CastingDateFunction; | ||
/** | ||
* List of fields as an array, | ||
* a user defined callback accepting the first line and returning the column names or true if autodiscovered in the first CSV line, | ||
* default to null, | ||
* affect the result data set in the sense that records will be objects instead of arrays. | ||
*/ | ||
columns?: ColumnOption[] | boolean | ((record: any) => ColumnOption[]); | ||
/** | ||
* Convert values into an array of values when columns are activated and | ||
* when multiple columns of the same name are found. | ||
*/ | ||
group_columns_by_name?: boolean; | ||
groupColumnsByName?: boolean; | ||
/** | ||
* Treat all the characters after this one as a comment, default to '' (disabled). | ||
*/ | ||
comment?: string; | ||
/** | ||
* Set the field delimiter. One character only, defaults to comma. | ||
*/ | ||
delimiter?: string | string[] | Buffer; | ||
/** | ||
* Set the source and destination encoding, a value of `null` returns buffer instead of strings. | ||
*/ | ||
encoding?: BufferEncoding | undefined; | ||
/** | ||
* Set the escape character, one character only, defaults to double quotes. | ||
*/ | ||
escape?: string | null | false | Buffer; | ||
/** | ||
* Start handling records from the requested number of records. | ||
*/ | ||
from?: number; | ||
/** | ||
* Start handling records from the requested line number. | ||
*/ | ||
from_line?: number; | ||
fromLine?: number; | ||
/** | ||
* Don't interpret delimiters as such in the last field according to the number of fields calculated from the number of columns, the option require the presence of the `column` option when `true`. | ||
*/ | ||
ignore_last_delimiters?: boolean | number; | ||
/** | ||
* Generate two properties `info` and `record` where `info` is a snapshot of the info object at the time the record was created and `record` is the parsed array or object. | ||
*/ | ||
info?: boolean; | ||
/** | ||
* If true, ignore whitespace immediately following the delimiter (i.e. left-trim all fields), defaults to false. | ||
* Does not remove whitespace in a quoted field. | ||
*/ | ||
ltrim?: boolean; | ||
/** | ||
* Maximum numer of characters to be contained in the field and line buffers before an exception is raised, | ||
* used to guard against a wrong delimiter or record_delimiter, | ||
* default to 128000 characters. | ||
*/ | ||
max_record_size?: number; | ||
maxRecordSize?: number; | ||
/** | ||
* Name of header-record title to name objects by. | ||
*/ | ||
objname?: string; | ||
/** | ||
* Alter and filter records by executing a user defined function. | ||
*/ | ||
on_record?: (record: any, context: CastingContext) => any; | ||
onRecord?: (record: any, context: CastingContext) => any; | ||
/** | ||
* Optional character surrounding a field, one character only, defaults to double quotes. | ||
*/ | ||
quote?: string | boolean | Buffer | null; | ||
/** | ||
* Generate two properties raw and row where raw is the original CSV row content and row is the parsed array or object. | ||
*/ | ||
raw?: boolean; | ||
/** | ||
* Discard inconsistent columns count, default to false. | ||
*/ | ||
relax_column_count?: boolean; | ||
relaxColumnCount?: boolean; | ||
/** | ||
* Discard inconsistent columns count when the record contains less fields than expected, default to false. | ||
*/ | ||
relax_column_count_less?: boolean; | ||
relaxColumnCountLess?: boolean; | ||
/** | ||
* Discard inconsistent columns count when the record contains more fields than expected, default to false. | ||
*/ | ||
relax_column_count_more?: boolean; | ||
relaxColumnCountMore?: boolean; | ||
/** | ||
* Preserve quotes inside unquoted field. | ||
*/ | ||
relax_quotes?: boolean; | ||
relaxQuotes?: boolean; | ||
/** | ||
* One or multiple characters used to delimit record rows; defaults to auto discovery if not provided. | ||
* Supported auto discovery method are Linux ("\n"), Apple ("\r") and Windows ("\r\n") row delimiters. | ||
*/ | ||
record_delimiter?: string | string[] | Buffer | Buffer[]; | ||
recordDelimiter?: string | string[] | Buffer | Buffer[]; | ||
/** | ||
* If true, ignore whitespace immediately preceding the delimiter (i.e. right-trim all fields), defaults to false. | ||
* Does not remove whitespace in a quoted field. | ||
*/ | ||
rtrim?: boolean; | ||
/** | ||
* Dont generate empty values for empty lines. | ||
* Defaults to false | ||
*/ | ||
skip_empty_lines?: boolean; | ||
skipEmptyLines?: boolean; | ||
/** | ||
* Skip a line with error found inside and directly go process the next line. | ||
*/ | ||
skip_records_with_error?: boolean; | ||
skipRecordsWithError?: boolean; | ||
/** | ||
* Don't generate records for lines containing empty column values (column matching /\s*\/), defaults to false. | ||
*/ | ||
skip_records_with_empty_values?: boolean; | ||
skipRecordsWithEmptyValues?: boolean; | ||
/** | ||
* Stop handling records after the requested number of records. | ||
*/ | ||
to?: number; | ||
/** | ||
* Stop handling records after the requested line number. | ||
*/ | ||
to_line?: number; | ||
toLine?: number; | ||
/** | ||
* If true, ignore whitespace immediately around the delimiter, defaults to false. | ||
* Does not remove whitespace in a quoted field. | ||
*/ | ||
trim?: boolean; | ||
} | ||
type ColumnOption = string | undefined | null | false | { name: string }; | ||
export interface Info { | ||
/** | ||
* Count the number of lines being fully commented. | ||
*/ | ||
readonly comment_lines: number; | ||
/** | ||
* Count the number of processed empty lines. | ||
*/ | ||
readonly empty_lines: number; | ||
/** | ||
* The number of lines encountered in the source dataset, start at 1 for the first line. | ||
*/ | ||
readonly lines: number; | ||
/** | ||
* Count the number of processed records. | ||
*/ | ||
readonly records: number; | ||
/** | ||
* Count of the number of processed bytes. | ||
*/ | ||
readonly bytes: number; | ||
/** | ||
* Number of non uniform records when `relax_column_count` is true. | ||
*/ | ||
readonly invalid_field_length: number; | ||
} | ||
interface Options { | ||
/** | ||
* If true, the parser will attempt to convert read data types to native types. | ||
* @deprecated Use {@link cast} | ||
*/ | ||
auto_parse?: boolean | CastingFunction; | ||
autoParse?: boolean | CastingFunction; | ||
/** | ||
* If true, the parser will attempt to convert read data types to dates. It requires the "auto_parse" option. | ||
* @deprecated Use {@link cast_date} | ||
*/ | ||
auto_parse_date?: boolean | CastingDateFunction; | ||
autoParseDate?: boolean | CastingDateFunction; | ||
/** | ||
* If true, detect and exclude the byte order mark (BOM) from the CSV input if present. | ||
*/ | ||
bom?: boolean; | ||
/** | ||
* If true, the parser will attempt to convert input string to native types. | ||
* If a function, receive the value as first argument, a context as second argument and return a new value. More information about the context properties is available below. | ||
*/ | ||
cast?: boolean | CastingFunction; | ||
/** | ||
* If true, the parser will attempt to convert input string to dates. | ||
* If a function, receive the value as argument and return a new value. It requires the "auto_parse" option. Be careful, it relies on Date.parse. | ||
*/ | ||
cast_date?: boolean | CastingDateFunction; | ||
castDate?: boolean | CastingDateFunction; | ||
/** | ||
* List of fields as an array, | ||
* a user defined callback accepting the first line and returning the column names or true if autodiscovered in the first CSV line, | ||
* default to null, | ||
* affect the result data set in the sense that records will be objects instead of arrays. | ||
*/ | ||
columns?: ColumnOption[] | boolean | ((record: any) => ColumnOption[]); | ||
/** | ||
* Convert values into an array of values when columns are activated and | ||
* when multiple columns of the same name are found. | ||
*/ | ||
columns_duplicates_to_array?: boolean; | ||
columnsDuplicatesToArray?: boolean; | ||
/** | ||
* Treat all the characters after this one as a comment, default to '' (disabled). | ||
*/ | ||
comment?: string; | ||
/** | ||
* Set the field delimiter. One character only, defaults to comma. | ||
*/ | ||
delimiter?: string | string[] | Buffer; | ||
/** | ||
* Set the source and destination encoding, a value of `null` returns buffer instead of strings. | ||
*/ | ||
encoding?: string | null; | ||
/** | ||
* Set the escape character, one character only, defaults to double quotes. | ||
*/ | ||
escape?: string | null | false | Buffer; | ||
/** | ||
* Start handling records from the requested number of records. | ||
*/ | ||
from?: number; | ||
/** | ||
* Start handling records from the requested line number. | ||
*/ | ||
from_line?: number; | ||
fromLine?: number; | ||
/** | ||
* Don't interpret delimiters as such in the last field according to the number of fields calculated from the number of columns, the option require the presence of the `column` option when `true`. | ||
*/ | ||
ignore_last_delimiters?: boolean | number; | ||
/** | ||
* Generate two properties `info` and `record` where `info` is a snapshot of the info object at the time the record was created and `record` is the parsed array or object. | ||
*/ | ||
info?: boolean; | ||
/** | ||
* If true, ignore whitespace immediately following the delimiter (i.e. left-trim all fields), defaults to false. | ||
* Does not remove whitespace in a quoted field. | ||
*/ | ||
ltrim?: boolean; | ||
/** | ||
* Maximum numer of characters to be contained in the field and line buffers before an exception is raised, | ||
* used to guard against a wrong delimiter or record_delimiter, | ||
* default to 128000 characters. | ||
*/ | ||
max_record_size?: number; | ||
maxRecordSize?: number; | ||
/** | ||
* Name of header-record title to name objects by. | ||
*/ | ||
objname?: string; | ||
/** | ||
* Alter and filter records by executing a user defined function. | ||
*/ | ||
on_record?: (record: any, context: CastingContext) => any; | ||
onRecord?: (record: any, context: CastingContext) => any; | ||
/** | ||
* Optional character surrounding a field, one character only, defaults to double quotes. | ||
*/ | ||
quote?: string | boolean | Buffer | null; | ||
/** | ||
* Generate two properties raw and row where raw is the original CSV row content and row is the parsed array or object. | ||
*/ | ||
raw?: boolean; | ||
/** | ||
* Preserve quotes inside unquoted field. | ||
*/ | ||
relax?: boolean; | ||
/** | ||
* Discard inconsistent columns count, default to false. | ||
*/ | ||
relax_column_count?: boolean; | ||
relaxColumnCount?: boolean; | ||
/** | ||
* Discard inconsistent columns count when the record contains less fields than expected, default to false. | ||
*/ | ||
relax_column_count_less?: boolean; | ||
relaxColumnCountLess?: boolean; | ||
/** | ||
* Discard inconsistent columns count when the record contains more fields than expected, default to false. | ||
*/ | ||
relax_column_count_more?: boolean; | ||
relaxColumnCountMore?: boolean; | ||
/** | ||
* One or multiple characters used to delimit record rows; defaults to auto discovery if not provided. | ||
* Supported auto discovery method are Linux ("\n"), Apple ("\r") and Windows ("\r\n") row delimiters. | ||
*/ | ||
record_delimiter?: string | string[] | Buffer | Buffer[]; | ||
recordDelimiter?: string | string[] | Buffer | Buffer[]; | ||
/** | ||
* If true, ignore whitespace immediately preceding the delimiter (i.e. right-trim all fields), defaults to false. | ||
* Does not remove whitespace in a quoted field. | ||
*/ | ||
rtrim?: boolean; | ||
/** | ||
* Dont generate empty values for empty lines. | ||
* Defaults to false | ||
*/ | ||
skip_empty_lines?: boolean; | ||
skipEmptyLines?: boolean; | ||
/** | ||
* Skip a line with error found inside and directly go process the next line. | ||
*/ | ||
skip_lines_with_error?: boolean; | ||
skipLinesWithError?: boolean; | ||
/** | ||
* Don't generate records for lines containing empty column values (column matching /\s*\/), defaults to false. | ||
*/ | ||
skip_lines_with_empty_values?: boolean; | ||
skipLinesWithEmptyValues?: boolean; | ||
/** | ||
* Stop handling records after the requested number of records. | ||
*/ | ||
to?: number; | ||
/** | ||
* Stop handling records after the requested line number. | ||
*/ | ||
to_line?: number; | ||
toLine?: number; | ||
/** | ||
* If true, ignore whitespace immediately around the delimiter, defaults to false. | ||
* Does not remove whitespace in a quoted field. | ||
*/ | ||
trim?: boolean; | ||
} | ||
export type CsvErrorCode = | ||
'CSV_INVALID_OPTION_BOM' | ||
| 'CSV_INVALID_OPTION_CAST' | ||
| 'CSV_INVALID_OPTION_CAST_DATE' | ||
| 'CSV_INVALID_OPTION_COLUMNS' | ||
| 'CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME' | ||
| 'CSV_INVALID_OPTION_COMMENT' | ||
| 'CSV_INVALID_OPTION_DELIMITER' | ||
| 'CSV_INVALID_OPTION_ON_RECORD' | ||
| 'CSV_INVALID_CLOSING_QUOTE' | ||
| 'INVALID_OPENING_QUOTE' | ||
| 'CSV_INVALID_COLUMN_MAPPING' | ||
| 'CSV_INVALID_ARGUMENT' | ||
| 'CSV_INVALID_COLUMN_DEFINITION' | ||
| 'CSV_MAX_RECORD_SIZE' | ||
| 'CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE' | ||
| 'CSV_QUOTE_NOT_CLOSED' | ||
| 'CSV_RECORD_INCONSISTENT_FIELDS_LENGTH' | ||
| 'CSV_RECORD_INCONSISTENT_COLUMNS' | ||
| 'CSV_OPTION_COLUMNS_MISSING_NAME' | ||
interface Info { | ||
/** | ||
* Count the number of lines being fully commented. | ||
*/ | ||
readonly comment_lines: number; | ||
/** | ||
* Count the number of processed empty lines. | ||
*/ | ||
readonly empty_lines: number; | ||
/** | ||
* The number of lines encountered in the source dataset, start at 1 for the first line. | ||
*/ | ||
readonly lines: number; | ||
/** | ||
* Count the number of processed records. | ||
*/ | ||
readonly records: number; | ||
/** | ||
* Count of the number of processed bytes. | ||
*/ | ||
readonly bytes: number; | ||
/** | ||
* Number of non uniform records when `relax_column_count` is true. | ||
*/ | ||
readonly invalid_field_length: number; | ||
} | ||
class CsvError extends Error { | ||
readonly code: CsvErrorCode; | ||
[key: string]: any; | ||
constructor(code: CsvErrorCode, message: string | string[], options?: Options, ...contexts: any[]); | ||
} | ||
type CsvErrorCode = | ||
'CSV_INVALID_OPTION_BOM' | ||
| 'CSV_INVALID_OPTION_CAST' | ||
| 'CSV_INVALID_OPTION_CAST_DATE' | ||
| 'CSV_INVALID_OPTION_COLUMNS' | ||
| 'CSV_INVALID_OPTION_COLUMNS_DUPLICATES_TO_ARRAY' | ||
| 'CSV_INVALID_OPTION_COMMENT' | ||
| 'CSV_INVALID_OPTION_DELIMITER' | ||
| 'CSV_INVALID_OPTION_ON_RECORD' | ||
| 'CSV_INVALID_CLOSING_QUOTE' | ||
| 'INVALID_OPENING_QUOTE' | ||
| 'CSV_INVALID_COLUMN_MAPPING' | ||
| 'CSV_INVALID_ARGUMENT' | ||
| 'CSV_INVALID_COLUMN_DEFINITION' | ||
| 'CSV_MAX_RECORD_SIZE' | ||
| 'CSV_NON_TRIMABLE_CHAR_AFTER_CLOSING_QUOTE' | ||
| 'CSV_QUOTE_NOT_CLOSED' | ||
| 'CSV_INCONSISTENT_RECORD_LENGTH' | ||
| 'CSV_RECORD_DONT_MATCH_COLUMNS_LENGTH' | ||
| 'CSV_OPTION_COLUMNS_MISSING_NAME' | ||
export class CsvError extends Error { | ||
readonly code: CsvErrorCode; | ||
[key: string]: any; | ||
constructor(code: CsvErrorCode, message: string | string[], options?: Options, ...contexts: any[]); | ||
} | ||
declare function parse(input: Buffer | string, options?: Options, callback?: Callback): Parser; | ||
declare function parse(input: Buffer | string, callback?: Callback): Parser; | ||
declare function parse(options?: Options, callback?: Callback): Parser; | ||
declare function parse(callback?: Callback): Parser; | ||
// export default parse; | ||
export { parse } |
1112
lib/index.js
@@ -9,4 +9,4 @@ | ||
const { Transform } = require('stream') | ||
const ResizeableBuffer = require('./ResizeableBuffer') | ||
import { Transform } from 'stream'; | ||
import ResizeableBuffer from './ResizeableBuffer.js'; | ||
@@ -17,7 +17,7 @@ // white space characters | ||
// \f\n\r\t\v\u00a0\u1680\u2000-\u200a\u2028\u2029\u202f\u205f\u3000\ufeff | ||
const tab = 9 | ||
const nl = 10 // \n, 0x0A in hexadecimal, 10 in decimal | ||
const np = 12 | ||
const cr = 13 // \r, 0x0D in hexadécimal, 13 in decimal | ||
const space = 32 | ||
const tab = 9; | ||
const nl = 10; // \n, 0x0A in hexadecimal, 10 in decimal | ||
const np = 12; | ||
const cr = 13; // \r, 0x0D in hexadécimal, 13 in decimal | ||
const space = 32; | ||
const boms = { | ||
@@ -33,15 +33,74 @@ // Note, the following are equals: | ||
'utf16le': Buffer.from([255, 254]) | ||
}; | ||
class CsvError extends Error { | ||
constructor(code, message, options, ...contexts) { | ||
if(Array.isArray(message)) message = message.join(' '); | ||
super(message); | ||
if(Error.captureStackTrace !== undefined){ | ||
Error.captureStackTrace(this, CsvError); | ||
} | ||
this.code = code; | ||
for(const context of contexts){ | ||
for(const key in context){ | ||
const value = context[key]; | ||
this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)); | ||
} | ||
} | ||
} | ||
} | ||
const underscore = function(str){ | ||
return str.replace(/([A-Z])/g, function(_, match){ | ||
return '_' + match.toLowerCase(); | ||
}); | ||
}; | ||
const isObject = function(obj){ | ||
return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)); | ||
}; | ||
const isRecordEmpty = function(record){ | ||
return record.every((field) => field == null || field.toString && field.toString().trim() === ''); | ||
}; | ||
const normalizeColumnsArray = function(columns){ | ||
const normalizedColumns = []; | ||
for(let i = 0, l = columns.length; i < l; i++){ | ||
const column = columns[i]; | ||
if(column === undefined || column === null || column === false){ | ||
normalizedColumns[i] = { disabled: true }; | ||
}else if(typeof column === 'string'){ | ||
normalizedColumns[i] = { name: column }; | ||
}else if(isObject(column)){ | ||
if(typeof column.name !== 'string'){ | ||
throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ | ||
'Option columns missing name:', | ||
`property "name" is required at position ${i}`, | ||
'when column is an object literal' | ||
]); | ||
} | ||
normalizedColumns[i] = column; | ||
}else{ | ||
throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ | ||
'Invalid column definition:', | ||
'expect a string or a literal object,', | ||
`got ${JSON.stringify(column)} at position ${i}` | ||
]); | ||
} | ||
} | ||
return normalizedColumns; | ||
}; | ||
class Parser extends Transform { | ||
constructor(opts = {}){ | ||
super({...{readableObjectMode: true}, ...opts, encoding: null}) | ||
this.__originalOptions = opts | ||
this.__normalizeOptions(opts) | ||
super({...{readableObjectMode: true}, ...opts, encoding: null}); | ||
this.__originalOptions = opts; | ||
this.__normalizeOptions(opts); | ||
} | ||
__normalizeOptions(opts){ | ||
const options = {} | ||
const options = {}; | ||
// Merge with user options | ||
for(let opt in opts){ | ||
options[underscore(opt)] = opts[opt] | ||
for(const opt in opts){ | ||
options[underscore(opt)] = opts[opt]; | ||
} | ||
@@ -52,5 +111,5 @@ // Normalize option `encoding` | ||
if(options.encoding === undefined || options.encoding === true){ | ||
options.encoding = 'utf8' | ||
options.encoding = 'utf8'; | ||
}else if(options.encoding === null || options.encoding === false){ | ||
options.encoding = null | ||
options.encoding = null; | ||
}else if(typeof options.encoding !== 'string' && options.encoding !== null){ | ||
@@ -61,7 +120,7 @@ throw new CsvError('CSV_INVALID_OPTION_ENCODING', [ | ||
`got ${JSON.stringify(options.encoding)}` | ||
], options) | ||
], options); | ||
} | ||
// Normalize option `bom` | ||
if(options.bom === undefined || options.bom === null || options.bom === false){ | ||
options.bom = false | ||
options.bom = false; | ||
}else if(options.bom !== true){ | ||
@@ -71,11 +130,11 @@ throw new CsvError('CSV_INVALID_OPTION_BOM', [ | ||
`got ${JSON.stringify(options.bom)}` | ||
], options) | ||
], options); | ||
} | ||
// Normalize option `cast` | ||
let fnCastField = null | ||
let fnCastField = null; | ||
if(options.cast === undefined || options.cast === null || options.cast === false || options.cast === ''){ | ||
options.cast = undefined | ||
options.cast = undefined; | ||
}else if(typeof options.cast === 'function'){ | ||
fnCastField = options.cast | ||
options.cast = true | ||
fnCastField = options.cast; | ||
options.cast = true; | ||
}else if(options.cast !== true){ | ||
@@ -85,12 +144,12 @@ throw new CsvError('CSV_INVALID_OPTION_CAST', [ | ||
`got ${JSON.stringify(options.cast)}` | ||
], options) | ||
], options); | ||
} | ||
// Normalize option `cast_date` | ||
if(options.cast_date === undefined || options.cast_date === null || options.cast_date === false || options.cast_date === ''){ | ||
options.cast_date = false | ||
options.cast_date = false; | ||
}else if(options.cast_date === true){ | ||
options.cast_date = function(value){ | ||
const date = Date.parse(value) | ||
return !isNaN(date) ? new Date(date) : value | ||
} | ||
const date = Date.parse(value); | ||
return !isNaN(date) ? new Date(date) : value; | ||
}; | ||
}else{ | ||
@@ -100,16 +159,16 @@ throw new CsvError('CSV_INVALID_OPTION_CAST_DATE', [ | ||
`got ${JSON.stringify(options.cast_date)}` | ||
], options) | ||
], options); | ||
} | ||
// Normalize option `columns` | ||
let fnFirstLineToHeaders = null | ||
let fnFirstLineToHeaders = null; | ||
if(options.columns === true){ | ||
// Fields in the first line are converted as-is to columns | ||
fnFirstLineToHeaders = undefined | ||
fnFirstLineToHeaders = undefined; | ||
}else if(typeof options.columns === 'function'){ | ||
fnFirstLineToHeaders = options.columns | ||
options.columns = true | ||
fnFirstLineToHeaders = options.columns; | ||
options.columns = true; | ||
}else if(Array.isArray(options.columns)){ | ||
options.columns = normalizeColumnsArray(options.columns) | ||
options.columns = normalizeColumnsArray(options.columns); | ||
}else if(options.columns === undefined || options.columns === null || options.columns === false){ | ||
options.columns = false | ||
options.columns = false; | ||
}else{ | ||
@@ -120,25 +179,25 @@ throw new CsvError('CSV_INVALID_OPTION_COLUMNS', [ | ||
`got ${JSON.stringify(options.columns)}` | ||
], options) | ||
], options); | ||
} | ||
// Normalize option `columns_duplicates_to_array` | ||
if(options.columns_duplicates_to_array === undefined || options.columns_duplicates_to_array === null || options.columns_duplicates_to_array === false){ | ||
options.columns_duplicates_to_array = false | ||
}else if(options.columns_duplicates_to_array !== true){ | ||
throw new CsvError('CSV_INVALID_OPTION_COLUMNS_DUPLICATES_TO_ARRAY', [ | ||
'Invalid option columns_duplicates_to_array:', | ||
// Normalize option `group_columns_by_name` | ||
if(options.group_columns_by_name === undefined || options.group_columns_by_name === null || options.group_columns_by_name === false){ | ||
options.group_columns_by_name = false; | ||
}else if(options.group_columns_by_name !== true){ | ||
throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ | ||
'Invalid option group_columns_by_name:', | ||
'expect an boolean,', | ||
`got ${JSON.stringify(options.columns_duplicates_to_array)}` | ||
], options) | ||
`got ${JSON.stringify(options.group_columns_by_name)}` | ||
], options); | ||
}else if(options.columns === false){ | ||
throw new CsvError('CSV_INVALID_OPTION_COLUMNS_DUPLICATES_TO_ARRAY', [ | ||
'Invalid option columns_duplicates_to_array:', | ||
throw new CsvError('CSV_INVALID_OPTION_GROUP_COLUMNS_BY_NAME', [ | ||
'Invalid option group_columns_by_name:', | ||
'the `columns` mode must be activated.' | ||
], options) | ||
], options); | ||
} | ||
// Normalize option `comment` | ||
if(options.comment === undefined || options.comment === null || options.comment === false || options.comment === ''){ | ||
options.comment = null | ||
options.comment = null; | ||
}else{ | ||
if(typeof options.comment === 'string'){ | ||
options.comment = Buffer.from(options.comment, options.encoding) | ||
options.comment = Buffer.from(options.comment, options.encoding); | ||
} | ||
@@ -150,8 +209,8 @@ if(!Buffer.isBuffer(options.comment)){ | ||
`got ${JSON.stringify(options.comment)}` | ||
], options) | ||
], options); | ||
} | ||
} | ||
// Normalize option `delimiter` | ||
const delimiter_json = JSON.stringify(options.delimiter) | ||
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter] | ||
const delimiter_json = JSON.stringify(options.delimiter); | ||
if(!Array.isArray(options.delimiter)) options.delimiter = [options.delimiter]; | ||
if(options.delimiter.length === 0){ | ||
@@ -162,12 +221,12 @@ throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ | ||
`got ${delimiter_json}` | ||
], options) | ||
], options); | ||
} | ||
options.delimiter = options.delimiter.map(function(delimiter){ | ||
if(delimiter === undefined || delimiter === null || delimiter === false){ | ||
return Buffer.from(',', options.encoding) | ||
return Buffer.from(',', options.encoding); | ||
} | ||
if(typeof delimiter === 'string'){ | ||
delimiter = Buffer.from(delimiter, options.encoding) | ||
delimiter = Buffer.from(delimiter, options.encoding); | ||
} | ||
if( !Buffer.isBuffer(delimiter) || delimiter.length === 0){ | ||
if(!Buffer.isBuffer(delimiter) || delimiter.length === 0){ | ||
throw new CsvError('CSV_INVALID_OPTION_DELIMITER', [ | ||
@@ -177,17 +236,17 @@ 'Invalid option delimiter:', | ||
`got ${delimiter_json}` | ||
], options) | ||
], options); | ||
} | ||
return delimiter | ||
}) | ||
return delimiter; | ||
}); | ||
// Normalize option `escape` | ||
if(options.escape === undefined || options.escape === true){ | ||
options.escape = Buffer.from('"', options.encoding) | ||
options.escape = Buffer.from('"', options.encoding); | ||
}else if(typeof options.escape === 'string'){ | ||
options.escape = Buffer.from(options.escape, options.encoding) | ||
options.escape = Buffer.from(options.escape, options.encoding); | ||
}else if (options.escape === null || options.escape === false){ | ||
options.escape = null | ||
options.escape = null; | ||
} | ||
if(options.escape !== null){ | ||
if(!Buffer.isBuffer(options.escape)){ | ||
throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`) | ||
throw new Error(`Invalid Option: escape must be a buffer, a string or a boolean, got ${JSON.stringify(options.escape)}`); | ||
} | ||
@@ -197,13 +256,13 @@ } | ||
if(options.from === undefined || options.from === null){ | ||
options.from = 1 | ||
options.from = 1; | ||
}else{ | ||
if(typeof options.from === 'string' && /\d+/.test(options.from)){ | ||
options.from = parseInt(options.from) | ||
options.from = parseInt(options.from); | ||
} | ||
if(Number.isInteger(options.from)){ | ||
if(options.from < 0){ | ||
throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`) | ||
throw new Error(`Invalid Option: from must be a positive integer, got ${JSON.stringify(opts.from)}`); | ||
} | ||
}else{ | ||
throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`) | ||
throw new Error(`Invalid Option: from must be an integer, got ${JSON.stringify(options.from)}`); | ||
} | ||
@@ -213,13 +272,13 @@ } | ||
if(options.from_line === undefined || options.from_line === null){ | ||
options.from_line = 1 | ||
options.from_line = 1; | ||
}else{ | ||
if(typeof options.from_line === 'string' && /\d+/.test(options.from_line)){ | ||
options.from_line = parseInt(options.from_line) | ||
options.from_line = parseInt(options.from_line); | ||
} | ||
if(Number.isInteger(options.from_line)){ | ||
if(options.from_line <= 0){ | ||
throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`) | ||
throw new Error(`Invalid Option: from_line must be a positive integer greater than 0, got ${JSON.stringify(opts.from_line)}`); | ||
} | ||
}else{ | ||
throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`) | ||
throw new Error(`Invalid Option: from_line must be an integer, got ${JSON.stringify(opts.from_line)}`); | ||
} | ||
@@ -229,7 +288,7 @@ } | ||
if(options.ignore_last_delimiters === undefined || options.ignore_last_delimiters === null){ | ||
options.ignore_last_delimiters = false | ||
options.ignore_last_delimiters = false; | ||
}else if(typeof options.ignore_last_delimiters === 'number'){ | ||
options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters) | ||
options.ignore_last_delimiters = Math.floor(options.ignore_last_delimiters); | ||
if(options.ignore_last_delimiters === 0){ | ||
options.ignore_last_delimiters = false | ||
options.ignore_last_delimiters = false; | ||
} | ||
@@ -241,3 +300,3 @@ }else if(typeof options.ignore_last_delimiters !== 'boolean'){ | ||
`got ${JSON.stringify(options.ignore_last_delimiters)}` | ||
], options) | ||
], options); | ||
} | ||
@@ -248,26 +307,26 @@ if(options.ignore_last_delimiters === true && options.columns === false){ | ||
'requires the activation of the `columns` option' | ||
], options) | ||
], options); | ||
} | ||
// Normalize option `info` | ||
if(options.info === undefined || options.info === null || options.info === false){ | ||
options.info = false | ||
options.info = false; | ||
}else if(options.info !== true){ | ||
throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`) | ||
throw new Error(`Invalid Option: info must be true, got ${JSON.stringify(options.info)}`); | ||
} | ||
// Normalize option `max_record_size` | ||
if(options.max_record_size === undefined || options.max_record_size === null || options.max_record_size === false){ | ||
options.max_record_size = 0 | ||
options.max_record_size = 0; | ||
}else if(Number.isInteger(options.max_record_size) && options.max_record_size >= 0){ | ||
// Great, nothing to do | ||
}else if(typeof options.max_record_size === 'string' && /\d+/.test(options.max_record_size)){ | ||
options.max_record_size = parseInt(options.max_record_size) | ||
options.max_record_size = parseInt(options.max_record_size); | ||
}else{ | ||
throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`) | ||
throw new Error(`Invalid Option: max_record_size must be a positive integer, got ${JSON.stringify(options.max_record_size)}`); | ||
} | ||
// Normalize option `objname` | ||
if(options.objname === undefined || options.objname === null || options.objname === false){ | ||
options.objname = undefined | ||
options.objname = undefined; | ||
}else if(Buffer.isBuffer(options.objname)){ | ||
if(options.objname.length === 0){ | ||
throw new Error(`Invalid Option: objname must be a non empty buffer`) | ||
throw new Error(`Invalid Option: objname must be a non empty buffer`); | ||
} | ||
@@ -277,15 +336,31 @@ if(options.encoding === null){ | ||
}else{ | ||
options.objname = options.objname.toString(options.encoding) | ||
options.objname = options.objname.toString(options.encoding); | ||
} | ||
}else if(typeof options.objname === 'string'){ | ||
if(options.objname.length === 0){ | ||
throw new Error(`Invalid Option: objname must be a non empty string`) | ||
throw new Error(`Invalid Option: objname must be a non empty string`); | ||
} | ||
// Great, nothing to do | ||
}else if(typeof options.objname === 'number'){ | ||
// if(options.objname.length === 0){ | ||
// throw new Error(`Invalid Option: objname must be a non empty string`); | ||
// } | ||
// Great, nothing to do | ||
}else{ | ||
throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`) | ||
throw new Error(`Invalid Option: objname must be a string or a buffer, got ${options.objname}`); | ||
} | ||
if(options.objname !== undefined){ | ||
if(typeof options.objname === 'number'){ | ||
if(options.columns !== false){ | ||
throw Error('Invalid Option: objname index cannot be combined with columns or be defined as a field'); | ||
} | ||
}else{ // A string or a buffer | ||
if(options.columns === false){ | ||
throw Error('Invalid Option: objname field must be combined with columns or be defined as an index'); | ||
} | ||
} | ||
} | ||
// Normalize option `on_record` | ||
if(options.on_record === undefined || options.on_record === null){ | ||
options.on_record = undefined | ||
options.on_record = undefined; | ||
}else if(typeof options.on_record !== 'function'){ | ||
@@ -296,15 +371,15 @@ throw new CsvError('CSV_INVALID_OPTION_ON_RECORD', [ | ||
`got ${JSON.stringify(options.on_record)}` | ||
], options) | ||
], options); | ||
} | ||
// Normalize option `quote` | ||
if(options.quote === null || options.quote === false || options.quote === ''){ | ||
options.quote = null | ||
options.quote = null; | ||
}else{ | ||
if(options.quote === undefined || options.quote === true){ | ||
options.quote = Buffer.from('"', options.encoding) | ||
options.quote = Buffer.from('"', options.encoding); | ||
}else if(typeof options.quote === 'string'){ | ||
options.quote = Buffer.from(options.quote, options.encoding) | ||
options.quote = Buffer.from(options.quote, options.encoding); | ||
} | ||
if(!Buffer.isBuffer(options.quote)){ | ||
throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`) | ||
throw new Error(`Invalid Option: quote must be a buffer or a string, got ${JSON.stringify(options.quote)}`); | ||
} | ||
@@ -314,26 +389,46 @@ } | ||
if(options.raw === undefined || options.raw === null || options.raw === false){ | ||
options.raw = false | ||
options.raw = false; | ||
}else if(options.raw !== true){ | ||
throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`) | ||
throw new Error(`Invalid Option: raw must be true, got ${JSON.stringify(options.raw)}`); | ||
} | ||
// Normalize option `record_delimiter` | ||
if(!options.record_delimiter){ | ||
options.record_delimiter = [] | ||
if(options.record_delimiter === undefined){ | ||
options.record_delimiter = []; | ||
}else if(typeof options.record_delimiter === 'string' || Buffer.isBuffer(options.record_delimiter)){ | ||
if(options.record_delimiter.length === 0){ | ||
throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ | ||
'Invalid option `record_delimiter`:', | ||
'value must be a non empty string or buffer,', | ||
`got ${JSON.stringify(options.record_delimiter)}` | ||
], options); | ||
} | ||
options.record_delimiter = [options.record_delimiter]; | ||
}else if(!Array.isArray(options.record_delimiter)){ | ||
options.record_delimiter = [options.record_delimiter] | ||
throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ | ||
'Invalid option `record_delimiter`:', | ||
'value must be a string, a buffer or array of string|buffer,', | ||
`got ${JSON.stringify(options.record_delimiter)}` | ||
], options); | ||
} | ||
options.record_delimiter = options.record_delimiter.map( function(rd){ | ||
options.record_delimiter = options.record_delimiter.map(function(rd, i){ | ||
if(typeof rd !== 'string' && ! Buffer.isBuffer(rd)){ | ||
throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ | ||
'Invalid option `record_delimiter`:', | ||
'value must be a string, a buffer or array of string|buffer', | ||
`at index ${i},`, | ||
`got ${JSON.stringify(rd)}` | ||
], options); | ||
}else if(rd.length === 0){ | ||
throw new CsvError('CSV_INVALID_OPTION_RECORD_DELIMITER', [ | ||
'Invalid option `record_delimiter`:', | ||
'value must be a non empty string or buffer', | ||
`at index ${i},`, | ||
`got ${JSON.stringify(rd)}` | ||
], options); | ||
} | ||
if(typeof rd === 'string'){ | ||
rd = Buffer.from(rd, options.encoding) | ||
rd = Buffer.from(rd, options.encoding); | ||
} | ||
return rd | ||
}) | ||
// Normalize option `relax` | ||
if(typeof options.relax === 'boolean'){ | ||
// Great, nothing to do | ||
}else if(options.relax === undefined || options.relax === null){ | ||
options.relax = false | ||
}else{ | ||
throw new Error(`Invalid Option: relax must be a boolean, got ${JSON.stringify(options.relax)}`) | ||
} | ||
return rd; | ||
}); | ||
// Normalize option `relax_column_count` | ||
@@ -343,5 +438,5 @@ if(typeof options.relax_column_count === 'boolean'){ | ||
}else if(options.relax_column_count === undefined || options.relax_column_count === null){ | ||
options.relax_column_count = false | ||
options.relax_column_count = false; | ||
}else{ | ||
throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`) | ||
throw new Error(`Invalid Option: relax_column_count must be a boolean, got ${JSON.stringify(options.relax_column_count)}`); | ||
} | ||
@@ -351,5 +446,5 @@ if(typeof options.relax_column_count_less === 'boolean'){ | ||
}else if(options.relax_column_count_less === undefined || options.relax_column_count_less === null){ | ||
options.relax_column_count_less = false | ||
options.relax_column_count_less = false; | ||
}else{ | ||
throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`) | ||
throw new Error(`Invalid Option: relax_column_count_less must be a boolean, got ${JSON.stringify(options.relax_column_count_less)}`); | ||
} | ||
@@ -359,6 +454,14 @@ if(typeof options.relax_column_count_more === 'boolean'){ | ||
}else if(options.relax_column_count_more === undefined || options.relax_column_count_more === null){ | ||
options.relax_column_count_more = false | ||
options.relax_column_count_more = false; | ||
}else{ | ||
throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`) | ||
throw new Error(`Invalid Option: relax_column_count_more must be a boolean, got ${JSON.stringify(options.relax_column_count_more)}`); | ||
} | ||
// Normalize option `relax_quotes` | ||
if(typeof options.relax_quotes === 'boolean'){ | ||
// Great, nothing to do | ||
}else if(options.relax_quotes === undefined || options.relax_quotes === null){ | ||
options.relax_quotes = false; | ||
}else{ | ||
throw new Error(`Invalid Option: relax_quotes must be a boolean, got ${JSON.stringify(options.relax_quotes)}`); | ||
} | ||
// Normalize option `skip_empty_lines` | ||
@@ -368,64 +471,64 @@ if(typeof options.skip_empty_lines === 'boolean'){ | ||
}else if(options.skip_empty_lines === undefined || options.skip_empty_lines === null){ | ||
options.skip_empty_lines = false | ||
options.skip_empty_lines = false; | ||
}else{ | ||
throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`) | ||
throw new Error(`Invalid Option: skip_empty_lines must be a boolean, got ${JSON.stringify(options.skip_empty_lines)}`); | ||
} | ||
// Normalize option `skip_lines_with_empty_values` | ||
if(typeof options.skip_lines_with_empty_values === 'boolean'){ | ||
// Normalize option `skip_records_with_empty_values` | ||
if(typeof options.skip_records_with_empty_values === 'boolean'){ | ||
// Great, nothing to do | ||
}else if(options.skip_lines_with_empty_values === undefined || options.skip_lines_with_empty_values === null){ | ||
options.skip_lines_with_empty_values = false | ||
}else if(options.skip_records_with_empty_values === undefined || options.skip_records_with_empty_values === null){ | ||
options.skip_records_with_empty_values = false; | ||
}else{ | ||
throw new Error(`Invalid Option: skip_lines_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_lines_with_empty_values)}`) | ||
throw new Error(`Invalid Option: skip_records_with_empty_values must be a boolean, got ${JSON.stringify(options.skip_records_with_empty_values)}`); | ||
} | ||
// Normalize option `skip_lines_with_error` | ||
if(typeof options.skip_lines_with_error === 'boolean'){ | ||
// Normalize option `skip_records_with_error` | ||
if(typeof options.skip_records_with_error === 'boolean'){ | ||
// Great, nothing to do | ||
}else if(options.skip_lines_with_error === undefined || options.skip_lines_with_error === null){ | ||
options.skip_lines_with_error = false | ||
}else if(options.skip_records_with_error === undefined || options.skip_records_with_error === null){ | ||
options.skip_records_with_error = false; | ||
}else{ | ||
throw new Error(`Invalid Option: skip_lines_with_error must be a boolean, got ${JSON.stringify(options.skip_lines_with_error)}`) | ||
throw new Error(`Invalid Option: skip_records_with_error must be a boolean, got ${JSON.stringify(options.skip_records_with_error)}`); | ||
} | ||
// Normalize option `rtrim` | ||
if(options.rtrim === undefined || options.rtrim === null || options.rtrim === false){ | ||
options.rtrim = false | ||
options.rtrim = false; | ||
}else if(options.rtrim !== true){ | ||
throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`) | ||
throw new Error(`Invalid Option: rtrim must be a boolean, got ${JSON.stringify(options.rtrim)}`); | ||
} | ||
// Normalize option `ltrim` | ||
if(options.ltrim === undefined || options.ltrim === null || options.ltrim === false){ | ||
options.ltrim = false | ||
options.ltrim = false; | ||
}else if(options.ltrim !== true){ | ||
throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`) | ||
throw new Error(`Invalid Option: ltrim must be a boolean, got ${JSON.stringify(options.ltrim)}`); | ||
} | ||
// Normalize option `trim` | ||
if(options.trim === undefined || options.trim === null || options.trim === false){ | ||
options.trim = false | ||
options.trim = false; | ||
}else if(options.trim !== true){ | ||
throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`) | ||
throw new Error(`Invalid Option: trim must be a boolean, got ${JSON.stringify(options.trim)}`); | ||
} | ||
// Normalize options `trim`, `ltrim` and `rtrim` | ||
if(options.trim === true && opts.ltrim !== false){ | ||
options.ltrim = true | ||
options.ltrim = true; | ||
}else if(options.ltrim !== true){ | ||
options.ltrim = false | ||
options.ltrim = false; | ||
} | ||
if(options.trim === true && opts.rtrim !== false){ | ||
options.rtrim = true | ||
options.rtrim = true; | ||
}else if(options.rtrim !== true){ | ||
options.rtrim = false | ||
options.rtrim = false; | ||
} | ||
// Normalize option `to` | ||
if(options.to === undefined || options.to === null){ | ||
options.to = -1 | ||
options.to = -1; | ||
}else{ | ||
if(typeof options.to === 'string' && /\d+/.test(options.to)){ | ||
options.to = parseInt(options.to) | ||
options.to = parseInt(options.to); | ||
} | ||
if(Number.isInteger(options.to)){ | ||
if(options.to <= 0){ | ||
throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`) | ||
throw new Error(`Invalid Option: to must be a positive integer greater than 0, got ${JSON.stringify(opts.to)}`); | ||
} | ||
}else{ | ||
throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`) | ||
throw new Error(`Invalid Option: to must be an integer, got ${JSON.stringify(opts.to)}`); | ||
} | ||
@@ -435,13 +538,13 @@ } | ||
if(options.to_line === undefined || options.to_line === null){ | ||
options.to_line = -1 | ||
options.to_line = -1; | ||
}else{ | ||
if(typeof options.to_line === 'string' && /\d+/.test(options.to_line)){ | ||
options.to_line = parseInt(options.to_line) | ||
options.to_line = parseInt(options.to_line); | ||
} | ||
if(Number.isInteger(options.to_line)){ | ||
if(options.to_line <= 0){ | ||
throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`) | ||
throw new Error(`Invalid Option: to_line must be a positive integer greater than 0, got ${JSON.stringify(opts.to_line)}`); | ||
} | ||
}else{ | ||
throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`) | ||
throw new Error(`Invalid Option: to_line must be an integer, got ${JSON.stringify(opts.to_line)}`); | ||
} | ||
@@ -456,4 +559,4 @@ } | ||
records: 0 | ||
} | ||
this.options = options | ||
}; | ||
this.options = options; | ||
this.state = { | ||
@@ -468,3 +571,2 @@ bomSkipped: false, | ||
escaping: false, | ||
// escapeIsQuote: options.escape === options.quote, | ||
escapeIsQuote: Buffer.isBuffer(options.escape) && Buffer.isBuffer(options.quote) && Buffer.compare(options.escape, options.quote) === 0, | ||
@@ -479,3 +581,3 @@ // columns can be `false`, `true`, `Array` | ||
// Skip if the remaining buffer can be delimiter | ||
...options.delimiter.map( (delimiter) => delimiter.length), | ||
...options.delimiter.map((delimiter) => delimiter.length), | ||
// Skip if the remaining buffer can be escape sequence | ||
@@ -491,7 +593,7 @@ options.quote !== null ? options.quote.length : 0, | ||
record_length: 0, | ||
recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map( (v) => v.length)), | ||
recordDelimiterMaxLength: options.record_delimiter.length === 0 ? 2 : Math.max(...options.record_delimiter.map((v) => v.length)), | ||
trimChars: [Buffer.from(' ', options.encoding)[0], Buffer.from('\t', options.encoding)[0]], | ||
wasQuoting: false, | ||
wasRowDelimiter: false | ||
} | ||
}; | ||
} | ||
@@ -501,9 +603,9 @@ // Implementation of `Transform._transform` | ||
if(this.state.stop === true){ | ||
return | ||
return; | ||
} | ||
const err = this.__parse(buf, false) | ||
const err = this.__parse(buf, false); | ||
if(err !== undefined){ | ||
this.state.stop = true | ||
this.state.stop = true; | ||
} | ||
callback(err) | ||
callback(err); | ||
} | ||
@@ -513,25 +615,25 @@ // Implementation of `Transform._flush` | ||
if(this.state.stop === true){ | ||
return | ||
return; | ||
} | ||
const err = this.__parse(undefined, true) | ||
callback(err) | ||
const err = this.__parse(undefined, true); | ||
callback(err); | ||
} | ||
// Central parser implementation | ||
__parse(nextBuf, end){ | ||
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax, rtrim, skip_empty_lines, to, to_line} = this.options | ||
let {record_delimiter} = this.options | ||
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state | ||
let buf | ||
const {bom, comment, escape, from_line, ltrim, max_record_size, quote, raw, relax_quotes, rtrim, skip_empty_lines, to, to_line} = this.options; | ||
let {record_delimiter} = this.options; | ||
const {bomSkipped, previousBuf, rawBuffer, escapeIsQuote} = this.state; | ||
let buf; | ||
if(previousBuf === undefined){ | ||
if(nextBuf === undefined){ | ||
// Handle empty string | ||
this.push(null) | ||
return | ||
this.push(null); | ||
return; | ||
}else{ | ||
buf = nextBuf | ||
buf = nextBuf; | ||
} | ||
}else if(previousBuf !== undefined && nextBuf === undefined){ | ||
buf = previousBuf | ||
buf = previousBuf; | ||
}else{ | ||
buf = Buffer.concat([previousBuf, nextBuf]) | ||
buf = Buffer.concat([previousBuf, nextBuf]); | ||
} | ||
@@ -541,3 +643,3 @@ // Handle UTF BOM | ||
if(bom === false){ | ||
this.state.bomSkipped = true | ||
this.state.bomSkipped = true; | ||
}else if(buf.length < 3){ | ||
@@ -547,22 +649,22 @@ // No enough data | ||
// Wait for more data | ||
this.state.previousBuf = buf | ||
return | ||
this.state.previousBuf = buf; | ||
return; | ||
} | ||
}else{ | ||
for(let encoding in boms){ | ||
for(const encoding in boms){ | ||
if(boms[encoding].compare(buf, 0, boms[encoding].length) === 0){ | ||
// Skip BOM | ||
let bomLength = boms[encoding].length | ||
this.state.bufBytesStart += bomLength | ||
buf = buf.slice(bomLength) | ||
const bomLength = boms[encoding].length; | ||
this.state.bufBytesStart += bomLength; | ||
buf = buf.slice(bomLength); | ||
// Renormalize original options with the new encoding | ||
this.__normalizeOptions({...this.__originalOptions, encoding: encoding}) | ||
break | ||
this.__normalizeOptions({...this.__originalOptions, encoding: encoding}); | ||
break; | ||
} | ||
} | ||
this.state.bomSkipped = true | ||
this.state.bomSkipped = true; | ||
} | ||
} | ||
const bufLen = buf.length | ||
let pos | ||
const bufLen = buf.length; | ||
let pos; | ||
for(pos = 0; pos < bufLen; pos++){ | ||
@@ -572,26 +674,26 @@ // Ensure we get enough space to look ahead | ||
if(this.__needMoreData(pos, bufLen, end)){ | ||
break | ||
break; | ||
} | ||
if(this.state.wasRowDelimiter === true){ | ||
this.info.lines++ | ||
this.state.wasRowDelimiter = false | ||
this.info.lines++; | ||
this.state.wasRowDelimiter = false; | ||
} | ||
if(to_line !== -1 && this.info.lines > to_line){ | ||
this.state.stop = true | ||
this.push(null) | ||
return | ||
this.state.stop = true; | ||
this.push(null); | ||
return; | ||
} | ||
// Auto discovery of record_delimiter, unix, mac and windows supported | ||
if(this.state.quoting === false && record_delimiter.length === 0){ | ||
const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos) | ||
const record_delimiterCount = this.__autoDiscoverRecordDelimiter(buf, pos); | ||
if(record_delimiterCount){ | ||
record_delimiter = this.options.record_delimiter | ||
record_delimiter = this.options.record_delimiter; | ||
} | ||
} | ||
const chr = buf[pos] | ||
const chr = buf[pos]; | ||
if(raw === true){ | ||
rawBuffer.append(chr) | ||
rawBuffer.append(chr); | ||
} | ||
if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false ){ | ||
this.state.wasRowDelimiter = true | ||
if((chr === cr || chr === nl) && this.state.wasRowDelimiter === false){ | ||
this.state.wasRowDelimiter = true; | ||
} | ||
@@ -601,3 +703,3 @@ // Previous char was a valid escape char | ||
if(this.state.escaping === true){ | ||
this.state.escaping = false | ||
this.state.escaping = false; | ||
}else{ | ||
@@ -610,10 +712,10 @@ // Escape is only active inside quoted fields | ||
if(this.__isQuote(buf, pos+escape.length)){ | ||
this.state.escaping = true | ||
pos += escape.length - 1 | ||
continue | ||
this.state.escaping = true; | ||
pos += escape.length - 1; | ||
continue; | ||
} | ||
}else{ | ||
this.state.escaping = true | ||
pos += escape.length - 1 | ||
continue | ||
this.state.escaping = true; | ||
pos += escape.length - 1; | ||
continue; | ||
} | ||
@@ -625,17 +727,17 @@ } | ||
if(this.state.quoting === true){ | ||
const nextChr = buf[pos+quote.length] | ||
const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr) | ||
const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr) | ||
const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr) | ||
const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length) | ||
const nextChr = buf[pos+quote.length]; | ||
const isNextChrTrimable = rtrim && this.__isCharTrimable(nextChr); | ||
const isNextChrComment = comment !== null && this.__compareBytes(comment, buf, pos+quote.length, nextChr); | ||
const isNextChrDelimiter = this.__isDelimiter(buf, pos+quote.length, nextChr); | ||
const isNextChrRecordDelimiter = record_delimiter.length === 0 ? this.__autoDiscoverRecordDelimiter(buf, pos+quote.length) : this.__isRecordDelimiter(nextChr, buf, pos+quote.length); | ||
// Escape a quote | ||
// Treat next char as a regular character | ||
if(escape !== null && this.__isEscape(buf, pos, chr) && this.__isQuote(buf, pos + escape.length)){ | ||
pos += escape.length - 1 | ||
pos += escape.length - 1; | ||
}else if(!nextChr || isNextChrDelimiter || isNextChrRecordDelimiter || isNextChrComment || isNextChrTrimable){ | ||
this.state.quoting = false | ||
this.state.wasQuoting = true | ||
pos += quote.length - 1 | ||
continue | ||
}else if(relax === false){ | ||
this.state.quoting = false; | ||
this.state.wasQuoting = true; | ||
pos += quote.length - 1; | ||
continue; | ||
}else if(relax_quotes === false){ | ||
const err = this.__error( | ||
@@ -649,14 +751,14 @@ new CsvError('CSV_INVALID_CLOSING_QUOTE', [ | ||
], this.options, this.__infoField()) | ||
) | ||
if(err !== undefined) return err | ||
); | ||
if(err !== undefined) return err; | ||
}else{ | ||
this.state.quoting = false | ||
this.state.wasQuoting = true | ||
this.state.field.prepend(quote) | ||
pos += quote.length - 1 | ||
this.state.quoting = false; | ||
this.state.wasQuoting = true; | ||
this.state.field.prepend(quote); | ||
pos += quote.length - 1; | ||
} | ||
}else{ | ||
if(this.state.field.length !== 0){ | ||
// In relax mode, treat opening quote preceded by chrs as regular | ||
if( relax === false ){ | ||
// In relax_quotes mode, treat opening quote preceded by chrs as regular | ||
if(relax_quotes === false){ | ||
const err = this.__error( | ||
@@ -669,9 +771,9 @@ new CsvError('INVALID_OPENING_QUOTE', [ | ||
}) | ||
) | ||
if(err !== undefined) return err | ||
); | ||
if(err !== undefined) return err; | ||
} | ||
}else{ | ||
this.state.quoting = true | ||
pos += quote.length - 1 | ||
continue | ||
this.state.quoting = true; | ||
pos += quote.length - 1; | ||
continue; | ||
} | ||
@@ -681,8 +783,8 @@ } | ||
if(this.state.quoting === false){ | ||
let recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos) | ||
const recordDelimiterLength = this.__isRecordDelimiter(chr, buf, pos); | ||
if(recordDelimiterLength !== 0){ | ||
// Do not emit comments which take a full line | ||
const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0) | ||
const skipCommentLine = this.state.commenting && (this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0); | ||
if(skipCommentLine){ | ||
this.info.comment_lines++ | ||
this.info.comment_lines++; | ||
// Skip full comment line | ||
@@ -692,45 +794,45 @@ }else{ | ||
if(this.state.enabled === false && this.info.lines + (this.state.wasRowDelimiter === true ? 1: 0) >= from_line){ | ||
this.state.enabled = true | ||
this.__resetField() | ||
this.__resetRecord() | ||
pos += recordDelimiterLength - 1 | ||
continue | ||
this.state.enabled = true; | ||
this.__resetField(); | ||
this.__resetRecord(); | ||
pos += recordDelimiterLength - 1; | ||
continue; | ||
} | ||
// Skip if line is empty and skip_empty_lines activated | ||
if(skip_empty_lines === true && this.state.wasQuoting === false && this.state.record.length === 0 && this.state.field.length === 0){ | ||
this.info.empty_lines++ | ||
pos += recordDelimiterLength - 1 | ||
continue | ||
this.info.empty_lines++; | ||
pos += recordDelimiterLength - 1; | ||
continue; | ||
} | ||
this.info.bytes = this.state.bufBytesStart + pos; | ||
const errField = this.__onField() | ||
if(errField !== undefined) return errField | ||
const errField = this.__onField(); | ||
if(errField !== undefined) return errField; | ||
this.info.bytes = this.state.bufBytesStart + pos + recordDelimiterLength; | ||
const errRecord = this.__onRecord() | ||
if(errRecord !== undefined) return errRecord | ||
const errRecord = this.__onRecord(); | ||
if(errRecord !== undefined) return errRecord; | ||
if(to !== -1 && this.info.records >= to){ | ||
this.state.stop = true | ||
this.push(null) | ||
return | ||
this.state.stop = true; | ||
this.push(null); | ||
return; | ||
} | ||
} | ||
this.state.commenting = false | ||
pos += recordDelimiterLength - 1 | ||
continue | ||
this.state.commenting = false; | ||
pos += recordDelimiterLength - 1; | ||
continue; | ||
} | ||
if(this.state.commenting){ | ||
continue | ||
continue; | ||
} | ||
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr) | ||
const commentCount = comment === null ? 0 : this.__compareBytes(comment, buf, pos, chr); | ||
if(commentCount !== 0){ | ||
this.state.commenting = true | ||
continue | ||
this.state.commenting = true; | ||
continue; | ||
} | ||
let delimiterLength = this.__isDelimiter(buf, pos, chr) | ||
const delimiterLength = this.__isDelimiter(buf, pos, chr); | ||
if(delimiterLength !== 0){ | ||
this.info.bytes = this.state.bufBytesStart + pos; | ||
const errField = this.__onField() | ||
if(errField !== undefined) return errField | ||
pos += delimiterLength - 1 | ||
continue | ||
const errField = this.__onField(); | ||
if(errField !== undefined) return errField; | ||
pos += delimiterLength - 1; | ||
continue; | ||
} | ||
@@ -748,11 +850,11 @@ } | ||
], this.options, this.__infoField()) | ||
) | ||
if(err !== undefined) return err | ||
); | ||
if(err !== undefined) return err; | ||
} | ||
} | ||
const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr) | ||
const lappend = ltrim === false || this.state.quoting === true || this.state.field.length !== 0 || !this.__isCharTrimable(chr); | ||
// rtrim in non quoting is handle in __onField | ||
const rappend = rtrim === false || this.state.wasQuoting === false | ||
if( lappend === true && rappend === true ){ | ||
this.state.field.append(chr) | ||
const rappend = rtrim === false || this.state.wasQuoting === false; | ||
if(lappend === true && rappend === true){ | ||
this.state.field.append(chr); | ||
}else if(rtrim === true && !this.__isCharTrimable(chr)){ | ||
@@ -765,4 +867,4 @@ const err = this.__error( | ||
], this.options, this.__infoField()) | ||
) | ||
if(err !== undefined) return err | ||
); | ||
if(err !== undefined) return err; | ||
} | ||
@@ -778,4 +880,4 @@ } | ||
], this.options, this.__infoField()) | ||
) | ||
if(err !== undefined) return err | ||
); | ||
if(err !== undefined) return err; | ||
}else{ | ||
@@ -785,44 +887,42 @@ // Skip last line if it has no characters | ||
this.info.bytes = this.state.bufBytesStart + pos; | ||
const errField = this.__onField() | ||
if(errField !== undefined) return errField | ||
const errRecord = this.__onRecord() | ||
if(errRecord !== undefined) return errRecord | ||
const errField = this.__onField(); | ||
if(errField !== undefined) return errField; | ||
const errRecord = this.__onRecord(); | ||
if(errRecord !== undefined) return errRecord; | ||
}else if(this.state.wasRowDelimiter === true){ | ||
this.info.empty_lines++ | ||
this.info.empty_lines++; | ||
}else if(this.state.commenting === true){ | ||
this.info.comment_lines++ | ||
this.info.comment_lines++; | ||
} | ||
} | ||
}else{ | ||
this.state.bufBytesStart += pos | ||
this.state.previousBuf = buf.slice(pos) | ||
this.state.bufBytesStart += pos; | ||
this.state.previousBuf = buf.slice(pos); | ||
} | ||
if(this.state.wasRowDelimiter === true){ | ||
this.info.lines++ | ||
this.state.wasRowDelimiter = false | ||
this.info.lines++; | ||
this.state.wasRowDelimiter = false; | ||
} | ||
} | ||
__onRecord(){ | ||
const {columns, columns_duplicates_to_array, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_lines_with_empty_values} = this.options | ||
const {enabled, record} = this.state | ||
const {columns, group_columns_by_name, encoding, info, from, relax_column_count, relax_column_count_less, relax_column_count_more, raw, skip_records_with_empty_values} = this.options; | ||
const {enabled, record} = this.state; | ||
if(enabled === false){ | ||
return this.__resetRecord() | ||
return this.__resetRecord(); | ||
} | ||
// Convert the first line into column names | ||
const recordLength = record.length | ||
const recordLength = record.length; | ||
if(columns === true){ | ||
if(skip_lines_with_empty_values === true && isRecordEmpty(record)){ | ||
this.__resetRecord() | ||
return | ||
if(skip_records_with_empty_values === true && isRecordEmpty(record)){ | ||
this.__resetRecord(); | ||
return; | ||
} | ||
return this.__firstLineToColumns(record) | ||
return this.__firstLineToColumns(record); | ||
} | ||
if(columns === false && this.info.records === 0){ | ||
this.state.expectedRecordLength = recordLength | ||
this.state.expectedRecordLength = recordLength; | ||
} | ||
if(recordLength !== this.state.expectedRecordLength){ | ||
const err = columns === false ? | ||
// Todo: rename CSV_INCONSISTENT_RECORD_LENGTH to | ||
// CSV_RECORD_INCONSISTENT_FIELDS_LENGTH | ||
new CsvError('CSV_INCONSISTENT_RECORD_LENGTH', [ | ||
new CsvError('CSV_RECORD_INCONSISTENT_FIELDS_LENGTH', [ | ||
'Invalid Record Length:', | ||
@@ -834,6 +934,4 @@ `expect ${this.state.expectedRecordLength},`, | ||
}) | ||
: | ||
// Todo: rename CSV_RECORD_DONT_MATCH_COLUMNS_LENGTH to | ||
// CSV_RECORD_INCONSISTENT_COLUMNS | ||
new CsvError('CSV_RECORD_DONT_MATCH_COLUMNS_LENGTH', [ | ||
: | ||
new CsvError('CSV_RECORD_INCONSISTENT_COLUMNS', [ | ||
'Invalid Record Length:', | ||
@@ -844,76 +942,62 @@ `columns length is ${columns.length},`, // rename columns | ||
record: record, | ||
}) | ||
}); | ||
if(relax_column_count === true || | ||
(relax_column_count_less === true && recordLength < this.state.expectedRecordLength) || | ||
(relax_column_count_more === true && recordLength > this.state.expectedRecordLength) ){ | ||
this.info.invalid_field_length++ | ||
this.state.error = err | ||
// Error is undefined with skip_lines_with_error | ||
(relax_column_count_more === true && recordLength > this.state.expectedRecordLength)){ | ||
this.info.invalid_field_length++; | ||
this.state.error = err; | ||
// Error is undefined with skip_records_with_error | ||
}else{ | ||
const finalErr = this.__error(err) | ||
if(finalErr) return finalErr | ||
const finalErr = this.__error(err); | ||
if(finalErr) return finalErr; | ||
} | ||
} | ||
if(skip_lines_with_empty_values === true && isRecordEmpty(record)){ | ||
this.__resetRecord() | ||
return | ||
if(skip_records_with_empty_values === true && isRecordEmpty(record)){ | ||
this.__resetRecord(); | ||
return; | ||
} | ||
if(this.state.recordHasError === true){ | ||
this.__resetRecord() | ||
this.state.recordHasError = false | ||
return | ||
this.__resetRecord(); | ||
this.state.recordHasError = false; | ||
return; | ||
} | ||
this.info.records++ | ||
this.info.records++; | ||
if(from === 1 || this.info.records >= from){ | ||
const {objname} = this.options; | ||
// With columns, records are object | ||
if(columns !== false){ | ||
const obj = {} | ||
const obj = {}; | ||
// Transform record array to an object | ||
for(let i = 0, l = record.length; i < l; i++){ | ||
if(columns[i] === undefined || columns[i].disabled) continue | ||
if(columns[i] === undefined || columns[i].disabled) continue; | ||
// Turn duplicate columns into an array | ||
if (columns_duplicates_to_array === true && obj[columns[i].name] !== undefined) { | ||
if (group_columns_by_name === true && obj[columns[i].name] !== undefined) { | ||
if (Array.isArray(obj[columns[i].name])) { | ||
obj[columns[i].name] = obj[columns[i].name].concat(record[i]) | ||
obj[columns[i].name] = obj[columns[i].name].concat(record[i]); | ||
} else { | ||
obj[columns[i].name] = [obj[columns[i].name], record[i]] | ||
obj[columns[i].name] = [obj[columns[i].name], record[i]]; | ||
} | ||
} else { | ||
obj[columns[i].name] = record[i] | ||
obj[columns[i].name] = record[i]; | ||
} | ||
} | ||
const {objname} = this.options | ||
// Without objname (default) | ||
if(objname === undefined){ | ||
if(raw === true || info === true){ | ||
const err = this.__push(Object.assign( | ||
{record: obj}, | ||
(raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), | ||
(info === true ? {info: this.__infoRecord()}: {}) | ||
)) | ||
if(err){ | ||
return err | ||
} | ||
}else{ | ||
const err = this.__push(obj) | ||
if(err){ | ||
return err | ||
} | ||
if(raw === true || info === true){ | ||
const extRecord = Object.assign( | ||
{record: obj}, | ||
(raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}), | ||
(info === true ? {info: this.__infoRecord()}: {}) | ||
); | ||
const err = this.__push( | ||
objname === undefined ? extRecord : [obj[objname], extRecord] | ||
); | ||
if(err){ | ||
return err; | ||
} | ||
// With objname (default) | ||
}else{ | ||
if(raw === true || info === true){ | ||
const err = this.__push(Object.assign( | ||
{record: [obj[objname], obj]}, | ||
raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, | ||
info === true ? {info: this.__infoRecord()}: {} | ||
)) | ||
if(err){ | ||
return err | ||
} | ||
}else{ | ||
const err = this.__push([obj[objname], obj]) | ||
if(err){ | ||
return err | ||
} | ||
const err = this.__push( | ||
objname === undefined ? obj : [obj[objname], obj] | ||
); | ||
if(err){ | ||
return err; | ||
} | ||
@@ -924,14 +1008,19 @@ } | ||
if(raw === true || info === true){ | ||
const err = this.__push(Object.assign( | ||
const extRecord = Object.assign( | ||
{record: record}, | ||
raw === true ? {raw: this.state.rawBuffer.toString(encoding)}: {}, | ||
info === true ? {info: this.__infoRecord()}: {} | ||
)) | ||
); | ||
const err = this.__push( | ||
objname === undefined ? extRecord : [record[objname], extRecord] | ||
); | ||
if(err){ | ||
return err | ||
return err; | ||
} | ||
}else{ | ||
const err = this.__push(record) | ||
const err = this.__push( | ||
objname === undefined ? record : [record[objname], record] | ||
); | ||
if(err){ | ||
return err | ||
return err; | ||
} | ||
@@ -941,8 +1030,8 @@ } | ||
} | ||
this.__resetRecord() | ||
this.__resetRecord(); | ||
} | ||
__firstLineToColumns(record){ | ||
const {firstLineToHeaders} = this.state | ||
const {firstLineToHeaders} = this.state; | ||
try{ | ||
const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record) | ||
const headers = firstLineToHeaders === undefined ? record : firstLineToHeaders.call(null, record); | ||
if(!Array.isArray(headers)){ | ||
@@ -957,11 +1046,11 @@ return this.__error( | ||
}) | ||
) | ||
); | ||
} | ||
const normalizedHeaders = normalizeColumnsArray(headers) | ||
this.state.expectedRecordLength = normalizedHeaders.length | ||
this.options.columns = normalizedHeaders | ||
this.__resetRecord() | ||
return | ||
const normalizedHeaders = normalizeColumnsArray(headers); | ||
this.state.expectedRecordLength = normalizedHeaders.length; | ||
this.options.columns = normalizedHeaders; | ||
this.__resetRecord(); | ||
return; | ||
}catch(err){ | ||
return err | ||
return err; | ||
} | ||
@@ -971,77 +1060,77 @@ } | ||
if(this.options.raw === true){ | ||
this.state.rawBuffer.reset() | ||
this.state.rawBuffer.reset(); | ||
} | ||
this.state.error = undefined | ||
this.state.record = [] | ||
this.state.record_length = 0 | ||
this.state.error = undefined; | ||
this.state.record = []; | ||
this.state.record_length = 0; | ||
} | ||
__onField(){ | ||
const {cast, encoding, rtrim, max_record_size} = this.options | ||
const {enabled, wasQuoting} = this.state | ||
const {cast, encoding, rtrim, max_record_size} = this.options; | ||
const {enabled, wasQuoting} = this.state; | ||
// Short circuit for the from_line options | ||
if(enabled === false){ | ||
return this.__resetField() | ||
return this.__resetField(); | ||
} | ||
let field = this.state.field.toString(encoding) | ||
let field = this.state.field.toString(encoding); | ||
if(rtrim === true && wasQuoting === false){ | ||
field = field.trimRight() | ||
field = field.trimRight(); | ||
} | ||
if(cast === true){ | ||
const [err, f] = this.__cast(field) | ||
if(err !== undefined) return err | ||
field = f | ||
const [err, f] = this.__cast(field); | ||
if(err !== undefined) return err; | ||
field = f; | ||
} | ||
this.state.record.push(field) | ||
this.state.record.push(field); | ||
// Increment record length if record size must not exceed a limit | ||
if(max_record_size !== 0 && typeof field === 'string'){ | ||
this.state.record_length += field.length | ||
this.state.record_length += field.length; | ||
} | ||
this.__resetField() | ||
this.__resetField(); | ||
} | ||
__resetField(){ | ||
this.state.field.reset() | ||
this.state.wasQuoting = false | ||
this.state.field.reset(); | ||
this.state.wasQuoting = false; | ||
} | ||
__push(record){ | ||
const {on_record} = this.options | ||
const {on_record} = this.options; | ||
if(on_record !== undefined){ | ||
const info = this.__infoRecord() | ||
const info = this.__infoRecord(); | ||
try{ | ||
record = on_record.call(null, record, info) | ||
record = on_record.call(null, record, info); | ||
}catch(err){ | ||
return err | ||
return err; | ||
} | ||
if(record === undefined || record === null){ return } | ||
if(record === undefined || record === null){ return; } | ||
} | ||
this.push(record) | ||
this.push(record); | ||
} | ||
// Return a tuple with the error and the casted value | ||
__cast(field){ | ||
const {columns, relax_column_count} = this.options | ||
const isColumns = Array.isArray(columns) | ||
const {columns, relax_column_count} = this.options; | ||
const isColumns = Array.isArray(columns); | ||
// Dont loose time calling cast | ||
// because the final record is an object | ||
// and this field can't be associated to a key present in columns | ||
if( isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length ){ | ||
return [undefined, undefined] | ||
if(isColumns === true && relax_column_count && this.options.columns.length <= this.state.record.length){ | ||
return [undefined, undefined]; | ||
} | ||
if(this.state.castField !== null){ | ||
try{ | ||
const info = this.__infoField() | ||
return [undefined, this.state.castField.call(null, field, info)] | ||
const info = this.__infoField(); | ||
return [undefined, this.state.castField.call(null, field, info)]; | ||
}catch(err){ | ||
return [err] | ||
return [err]; | ||
} | ||
} | ||
if(this.__isFloat(field)){ | ||
return [undefined, parseFloat(field)] | ||
return [undefined, parseFloat(field)]; | ||
}else if(this.options.cast_date !== false){ | ||
const info = this.__infoField() | ||
return [undefined, this.options.cast_date.call(null, field, info)] | ||
const info = this.__infoField(); | ||
return [undefined, this.options.cast_date.call(null, field, info)]; | ||
} | ||
return [undefined, field] | ||
return [undefined, field]; | ||
} | ||
// Helper to test if a character is a space or a line delimiter | ||
__isCharTrimable(chr){ | ||
return chr === space || chr === tab || chr === cr || chr === nl || chr === np | ||
return chr === space || chr === tab || chr === cr || chr === nl || chr === np; | ||
} | ||
@@ -1055,17 +1144,17 @@ // Keep it in case we implement the `cast_int` option | ||
__isFloat(value){ | ||
return (value - parseFloat( value ) + 1) >= 0 // Borrowed from jquery | ||
return (value - parseFloat(value) + 1) >= 0; // Borrowed from jquery | ||
} | ||
__compareBytes(sourceBuf, targetBuf, targetPos, firstByte){ | ||
if(sourceBuf[0] !== firstByte) return 0 | ||
const sourceLength = sourceBuf.length | ||
if(sourceBuf[0] !== firstByte) return 0; | ||
const sourceLength = sourceBuf.length; | ||
for(let i = 1; i < sourceLength; i++){ | ||
if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0 | ||
if(sourceBuf[i] !== targetBuf[targetPos+i]) return 0; | ||
} | ||
return sourceLength | ||
return sourceLength; | ||
} | ||
__needMoreData(i, bufLen, end){ | ||
if(end) return false | ||
const {quote} = this.options | ||
const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state | ||
const numOfCharLeft = bufLen - i - 1 | ||
if(end) return false; | ||
const {quote} = this.options; | ||
const {quoting, needMoreDataSize, recordDelimiterMaxLength} = this.state; | ||
const numOfCharLeft = bufLen - i - 1; | ||
const requiredLength = Math.max( | ||
@@ -1078,95 +1167,95 @@ needMoreDataSize, | ||
quoting ? (quote.length + recordDelimiterMaxLength) : 0, | ||
) | ||
return numOfCharLeft < requiredLength | ||
); | ||
return numOfCharLeft < requiredLength; | ||
} | ||
__isDelimiter(buf, pos, chr){ | ||
const {delimiter, ignore_last_delimiters} = this.options | ||
const {delimiter, ignore_last_delimiters} = this.options; | ||
if(ignore_last_delimiters === true && this.state.record.length === this.options.columns.length - 1){ | ||
return 0 | ||
return 0; | ||
}else if(ignore_last_delimiters !== false && typeof ignore_last_delimiters === 'number' && this.state.record.length === ignore_last_delimiters - 1){ | ||
return 0 | ||
return 0; | ||
} | ||
loop1: for(let i = 0; i < delimiter.length; i++){ | ||
const del = delimiter[i] | ||
const del = delimiter[i]; | ||
if(del[0] === chr){ | ||
for(let j = 1; j < del.length; j++){ | ||
if(del[j] !== buf[pos+j]) continue loop1 | ||
if(del[j] !== buf[pos+j]) continue loop1; | ||
} | ||
return del.length | ||
return del.length; | ||
} | ||
} | ||
return 0 | ||
return 0; | ||
} | ||
__isRecordDelimiter(chr, buf, pos){ | ||
const {record_delimiter} = this.options | ||
const recordDelimiterLength = record_delimiter.length | ||
const {record_delimiter} = this.options; | ||
const recordDelimiterLength = record_delimiter.length; | ||
loop1: for(let i = 0; i < recordDelimiterLength; i++){ | ||
const rd = record_delimiter[i] | ||
const rdLength = rd.length | ||
const rd = record_delimiter[i]; | ||
const rdLength = rd.length; | ||
if(rd[0] !== chr){ | ||
continue | ||
continue; | ||
} | ||
for(let j = 1; j < rdLength; j++){ | ||
if(rd[j] !== buf[pos+j]){ | ||
continue loop1 | ||
continue loop1; | ||
} | ||
} | ||
return rd.length | ||
return rd.length; | ||
} | ||
return 0 | ||
return 0; | ||
} | ||
__isEscape(buf, pos, chr){ | ||
const {escape} = this.options | ||
if(escape === null) return false | ||
const l = escape.length | ||
const {escape} = this.options; | ||
if(escape === null) return false; | ||
const l = escape.length; | ||
if(escape[0] === chr){ | ||
for(let i = 0; i < l; i++){ | ||
if(escape[i] !== buf[pos+i]){ | ||
return false | ||
return false; | ||
} | ||
} | ||
return true | ||
return true; | ||
} | ||
return false | ||
return false; | ||
} | ||
__isQuote(buf, pos){ | ||
const {quote} = this.options | ||
if(quote === null) return false | ||
const l = quote.length | ||
const {quote} = this.options; | ||
if(quote === null) return false; | ||
const l = quote.length; | ||
for(let i = 0; i < l; i++){ | ||
if(quote[i] !== buf[pos+i]){ | ||
return false | ||
return false; | ||
} | ||
} | ||
return true | ||
return true; | ||
} | ||
__autoDiscoverRecordDelimiter(buf, pos){ | ||
const {encoding} = this.options | ||
const chr = buf[pos] | ||
const {encoding} = this.options; | ||
const chr = buf[pos]; | ||
if(chr === cr){ | ||
if(buf[pos+1] === nl){ | ||
this.options.record_delimiter.push(Buffer.from('\r\n', encoding)) | ||
this.state.recordDelimiterMaxLength = 2 | ||
return 2 | ||
this.options.record_delimiter.push(Buffer.from('\r\n', encoding)); | ||
this.state.recordDelimiterMaxLength = 2; | ||
return 2; | ||
}else{ | ||
this.options.record_delimiter.push(Buffer.from('\r', encoding)) | ||
this.state.recordDelimiterMaxLength = 1 | ||
return 1 | ||
this.options.record_delimiter.push(Buffer.from('\r', encoding)); | ||
this.state.recordDelimiterMaxLength = 1; | ||
return 1; | ||
} | ||
}else if(chr === nl){ | ||
this.options.record_delimiter.push(Buffer.from('\n', encoding)) | ||
this.state.recordDelimiterMaxLength = 1 | ||
return 1 | ||
this.options.record_delimiter.push(Buffer.from('\n', encoding)); | ||
this.state.recordDelimiterMaxLength = 1; | ||
return 1; | ||
} | ||
return 0 | ||
return 0; | ||
} | ||
__error(msg){ | ||
const {skip_lines_with_error} = this.options | ||
const err = typeof msg === 'string' ? new Error(msg) : msg | ||
if(skip_lines_with_error){ | ||
this.state.recordHasError = true | ||
this.emit('skip', err) | ||
return undefined | ||
const {encoding, raw, skip_records_with_error} = this.options; | ||
const err = typeof msg === 'string' ? new Error(msg) : msg; | ||
if(skip_records_with_error){ | ||
this.state.recordHasError = true; | ||
this.emit('skip', err, raw ? this.state.rawBuffer.toString(encoding) : undefined); | ||
return undefined; | ||
}else{ | ||
return err | ||
return err; | ||
} | ||
@@ -1178,6 +1267,6 @@ } | ||
columns: this.options.columns | ||
} | ||
}; | ||
} | ||
__infoRecord(){ | ||
const {columns} = this.options | ||
const {columns, raw, encoding} = this.options; | ||
return { | ||
@@ -1188,11 +1277,12 @@ ...this.__infoDataSet(), | ||
index: this.state.record.length, | ||
} | ||
raw: raw ? this.state.rawBuffer.toString(encoding) : undefined | ||
}; | ||
} | ||
__infoField(){ | ||
const {columns} = this.options | ||
const isColumns = Array.isArray(columns) | ||
const {columns} = this.options; | ||
const isColumns = Array.isArray(columns); | ||
return { | ||
...this.__infoRecord(), | ||
column: isColumns === true ? | ||
( columns.length > this.state.record.length ? | ||
(columns.length > this.state.record.length ? | ||
columns[this.state.record.length].name : | ||
@@ -1203,3 +1293,3 @@ null | ||
quoting: this.state.wasQuoting, | ||
} | ||
}; | ||
} | ||
@@ -1209,12 +1299,12 @@ } | ||
const parse = function(){ | ||
let data, options, callback | ||
for(let i in arguments){ | ||
const argument = arguments[i] | ||
const type = typeof argument | ||
let data, options, callback; | ||
for(const i in arguments){ | ||
const argument = arguments[i]; | ||
const type = typeof argument; | ||
if(data === undefined && (typeof argument === 'string' || Buffer.isBuffer(argument))){ | ||
data = argument | ||
data = argument; | ||
}else if(options === undefined && isObject(argument)){ | ||
options = argument | ||
options = argument; | ||
}else if(callback === undefined && type === 'function'){ | ||
callback = argument | ||
callback = argument; | ||
}else{ | ||
@@ -1224,103 +1314,41 @@ throw new CsvError('CSV_INVALID_ARGUMENT', [ | ||
`got ${JSON.stringify(argument)} at index ${i}` | ||
], options || {}) | ||
], options || {}); | ||
} | ||
} | ||
const parser = new Parser(options) | ||
const parser = new Parser(options); | ||
if(callback){ | ||
const records = options === undefined || options.objname === undefined ? [] : {} | ||
const records = options === undefined || options.objname === undefined ? [] : {}; | ||
parser.on('readable', function(){ | ||
let record | ||
let record; | ||
while((record = this.read()) !== null){ | ||
if(options === undefined || options.objname === undefined){ | ||
records.push(record) | ||
records.push(record); | ||
}else{ | ||
records[record[0]] = record[1] | ||
records[record[0]] = record[1]; | ||
} | ||
} | ||
}) | ||
}); | ||
parser.on('error', function(err){ | ||
callback(err, undefined, parser.__infoDataSet()) | ||
}) | ||
callback(err, undefined, parser.__infoDataSet()); | ||
}); | ||
parser.on('end', function(){ | ||
callback(undefined, records, parser.__infoDataSet()) | ||
}) | ||
callback(undefined, records, parser.__infoDataSet()); | ||
}); | ||
} | ||
if(data !== undefined){ | ||
// Give a chance for events to be registered later | ||
const writer = function(){ | ||
parser.write(data); | ||
parser.end(); | ||
}; | ||
// Support Deno, Rollup doesnt provide a shim for setImmediate | ||
if(typeof setImmediate === 'function'){ | ||
setImmediate(function(){ | ||
parser.write(data) | ||
parser.end() | ||
}) | ||
setImmediate(writer); | ||
}else{ | ||
parser.write(data) | ||
parser.end() | ||
setTimeout(writer, 0); | ||
} | ||
} | ||
return parser | ||
} | ||
return parser; | ||
}; | ||
class CsvError extends Error { | ||
constructor(code, message, options, ...contexts) { | ||
if(Array.isArray(message)) message = message.join(' ') | ||
super(message) | ||
if(Error.captureStackTrace !== undefined){ | ||
Error.captureStackTrace(this, CsvError) | ||
} | ||
this.code = code | ||
for(const context of contexts){ | ||
for(const key in context){ | ||
const value = context[key] | ||
this[key] = Buffer.isBuffer(value) ? value.toString(options.encoding) : value == null ? value : JSON.parse(JSON.stringify(value)) | ||
} | ||
} | ||
} | ||
} | ||
parse.Parser = Parser | ||
parse.CsvError = CsvError | ||
module.exports = parse | ||
const underscore = function(str){ | ||
return str.replace(/([A-Z])/g, function(_, match){ | ||
return '_' + match.toLowerCase() | ||
}) | ||
} | ||
const isObject = function(obj){ | ||
return (typeof obj === 'object' && obj !== null && !Array.isArray(obj)) | ||
} | ||
const isRecordEmpty = function(record){ | ||
return record.every( (field) => field == null || field.toString && field.toString().trim() === '' ) | ||
} | ||
const normalizeColumnsArray = function(columns){ | ||
const normalizedColumns = []; | ||
for(let i = 0, l = columns.length; i < l; i++){ | ||
const column = columns[i] | ||
if(column === undefined || column === null || column === false){ | ||
normalizedColumns[i] = { disabled: true } | ||
}else if(typeof column === 'string'){ | ||
normalizedColumns[i] = { name: column } | ||
}else if(isObject(column)){ | ||
if(typeof column.name !== 'string'){ | ||
throw new CsvError('CSV_OPTION_COLUMNS_MISSING_NAME', [ | ||
'Option columns missing name:', | ||
`property "name" is required at position ${i}`, | ||
'when column is an object literal' | ||
]) | ||
} | ||
normalizedColumns[i] = column | ||
}else{ | ||
throw new CsvError('CSV_INVALID_COLUMN_DEFINITION', [ | ||
'Invalid column definition:', | ||
'expect a string or a literal object,', | ||
`got ${JSON.stringify(column)} at position ${i}` | ||
]) | ||
} | ||
} | ||
return normalizedColumns; | ||
} | ||
// export default parse | ||
export { parse, Parser, CsvError }; |
@@ -5,62 +5,62 @@ | ||
constructor(size=100){ | ||
this.size = size | ||
this.length = 0 | ||
this.buf = Buffer.alloc(size) | ||
this.size = size; | ||
this.length = 0; | ||
this.buf = Buffer.alloc(size); | ||
} | ||
prepend(val){ | ||
if(Buffer.isBuffer(val)){ | ||
const length = this.length + val.length | ||
const length = this.length + val.length; | ||
if(length >= this.size){ | ||
this.resize() | ||
this.resize(); | ||
if(length >= this.size){ | ||
throw Error('INVALID_BUFFER_STATE') | ||
throw Error('INVALID_BUFFER_STATE'); | ||
} | ||
} | ||
const buf = this.buf | ||
this.buf = Buffer.alloc(this.size) | ||
val.copy(this.buf, 0) | ||
buf.copy(this.buf, val.length) | ||
this.length += val.length | ||
const buf = this.buf; | ||
this.buf = Buffer.alloc(this.size); | ||
val.copy(this.buf, 0); | ||
buf.copy(this.buf, val.length); | ||
this.length += val.length; | ||
}else{ | ||
const length = this.length++ | ||
const length = this.length++; | ||
if(length === this.size){ | ||
this.resize() | ||
this.resize(); | ||
} | ||
const buf = this.clone() | ||
this.buf[0] = val | ||
buf.copy(this.buf,1, 0, length) | ||
const buf = this.clone(); | ||
this.buf[0] = val; | ||
buf.copy(this.buf,1, 0, length); | ||
} | ||
} | ||
append(val){ | ||
const length = this.length++ | ||
const length = this.length++; | ||
if(length === this.size){ | ||
this.resize() | ||
this.resize(); | ||
} | ||
this.buf[length] = val | ||
this.buf[length] = val; | ||
} | ||
clone(){ | ||
return Buffer.from(this.buf.slice(0, this.length)) | ||
return Buffer.from(this.buf.slice(0, this.length)); | ||
} | ||
resize(){ | ||
const length = this.length | ||
this.size = this.size * 2 | ||
const buf = Buffer.alloc(this.size) | ||
this.buf.copy(buf,0, 0, length) | ||
this.buf = buf | ||
const length = this.length; | ||
this.size = this.size * 2; | ||
const buf = Buffer.alloc(this.size); | ||
this.buf.copy(buf,0, 0, length); | ||
this.buf = buf; | ||
} | ||
toString(encoding){ | ||
if(encoding){ | ||
return this.buf.slice(0, this.length).toString(encoding) | ||
return this.buf.slice(0, this.length).toString(encoding); | ||
}else{ | ||
return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length)) | ||
return Uint8Array.prototype.slice.call(this.buf.slice(0, this.length)); | ||
} | ||
} | ||
toJSON(){ | ||
return this.toString('utf8') | ||
return this.toString('utf8'); | ||
} | ||
reset(){ | ||
this.length = 0 | ||
this.length = 0; | ||
} | ||
} | ||
module.exports = ResizeableBuffer | ||
export default ResizeableBuffer; |
@@ -1,6 +0,11 @@ | ||
import * as csvParse from './index'; | ||
export = parse; | ||
import { Options } from './index'; | ||
declare function parse(input: Buffer | string, options?: csvParse.Options): any; | ||
declare namespace parse {} | ||
declare function parse(input: Buffer | string, options?: Options): any; | ||
// export default parse; | ||
export { parse }; | ||
export { | ||
CastingContext, CastingFunction, CastingDateFunction, | ||
ColumnOption, Options, Info, CsvErrorCode, CsvError | ||
} from './index'; |
const parse = require('.') | ||
import { Parser } from './index.js'; | ||
module.exports = function(data, options={}){ | ||
const parse = function(data, options={}){ | ||
if(typeof data === 'string'){ | ||
data = Buffer.from(data) | ||
data = Buffer.from(data); | ||
} | ||
const records = options && options.objname ? {} : [] | ||
const parser = new parse.Parser(options) | ||
const records = options && options.objname ? {} : []; | ||
const parser = new Parser(options); | ||
parser.push = function(record){ | ||
if(record === null){ | ||
return | ||
return; | ||
} | ||
if(options.objname === undefined) | ||
records.push(record) | ||
records.push(record); | ||
else{ | ||
records[record[0]] = record[1] | ||
records[record[0]] = record[1]; | ||
} | ||
} | ||
const err1 = parser.__parse(data, false) | ||
if(err1 !== undefined) throw err1 | ||
const err2 = parser.__parse(undefined, true) | ||
if(err2 !== undefined) throw err2 | ||
return records | ||
} | ||
}; | ||
const err1 = parser.__parse(data, false); | ||
if(err1 !== undefined) throw err1; | ||
const err2 = parser.__parse(undefined, true); | ||
if(err2 !== undefined) throw err2; | ||
return records; | ||
}; | ||
// export default parse | ||
export { parse }; | ||
export { CsvError } from './index.js'; |
116
package.json
{ | ||
"version": "4.16.3", | ||
"version": "5.0.0", | ||
"name": "csv-parse", | ||
@@ -13,15 +13,2 @@ "description": "CSV parsing implementing the Node.js `stream.Transform` API", | ||
], | ||
"coffeelintConfig": { | ||
"indentation": { | ||
"level": "error", | ||
"value": 2 | ||
}, | ||
"line_endings": { | ||
"level": "error", | ||
"value": "unix" | ||
}, | ||
"max_line_length": { | ||
"level": "ignore" | ||
} | ||
}, | ||
"author": "David Worms <david@adaltas.com> (https://www.adaltas.com)", | ||
@@ -46,56 +33,83 @@ "contributors": [ | ||
], | ||
"license": "MIT", | ||
"repository": { | ||
"type": "git", | ||
"url": "https://github.com/wdavidw/node-csv-parse" | ||
"exports": { | ||
".": { | ||
"import": "./lib/index.js", | ||
"require": "./dist/cjs/index.cjs" | ||
}, | ||
"./sync": { | ||
"import": "./lib/sync.js", | ||
"require": "./dist/cjs/sync.cjs" | ||
} | ||
}, | ||
"homepage": "https://csv.js.org/parse/", | ||
"devDependencies": { | ||
"@babel/cli": "^7.14.8", | ||
"@babel/core": "^7.15.0", | ||
"@babel/preset-env": "^7.15.0", | ||
"@rollup/plugin-eslint": "^8.0.1", | ||
"@rollup/plugin-node-resolve": "^13.0.6", | ||
"@types/mocha": "^9.0.0", | ||
"@types/node": "^16.7.8", | ||
"babelify": "^10.0.0", | ||
"browserify": "^17.0.0", | ||
"coffeescript": "^2.5.1", | ||
"csv-generate": "^3.4.3", | ||
"@types/node": "^16.11.7", | ||
"coffeelint": "^2.1.0", | ||
"coffeescript": "^2.6.1", | ||
"csv-generate": "^4.0.0", | ||
"csv-spectrum": "^1.0.0", | ||
"each": "^1.2.2", | ||
"eslint": "^7.32.0", | ||
"express": "^4.17.1", | ||
"mocha": "^9.1.1", | ||
"eslint": "^8.2.0", | ||
"mocha": "^9.1.3", | ||
"pad": "^3.2.0", | ||
"rollup": "^2.60.0", | ||
"rollup-plugin-node-builtins": "^2.1.2", | ||
"rollup-plugin-node-globals": "^1.4.0", | ||
"should": "^13.2.3", | ||
"stream-transform": "^2.1.3", | ||
"ts-node": "^10.2.1", | ||
"typescript": "^4.4.2" | ||
"stream-transform": "^3.0.0", | ||
"ts-node": "^10.4.0", | ||
"typescript": "^4.4.4" | ||
}, | ||
"files": [ | ||
"/lib" | ||
"dist", | ||
"lib", | ||
"samples" | ||
], | ||
"main": "./lib", | ||
"homepage": "https://csv.js.org/parse/", | ||
"license": "MIT", | ||
"main": "./dist/cjs/index.cjs", | ||
"mocha": { | ||
"throw-deprecation": true, | ||
"inline-diffs": true, | ||
"loader": "./test/loaders/all.mjs", | ||
"recursive": true, | ||
"reporter": "spec", | ||
"require": [ | ||
"should", | ||
"coffeescript/register", | ||
"ts-node/register" | ||
"should" | ||
], | ||
"inline-diffs": true, | ||
"timeout": 40000, | ||
"reporter": "spec", | ||
"recursive": true | ||
"throw-deprecation": true, | ||
"timeout": 40000 | ||
}, | ||
"repository": { | ||
"type": "git", | ||
"url": "https://github.com/adaltas/node-csv.git", | ||
"directory": "packages/csv-parse" | ||
}, | ||
"scripts": { | ||
"build:babel": "cd lib && babel *.js -d es5 && cd ..", | ||
"build:browserify": "browserify lib/index.js --transform babelify --standalone parse > lib/browser/index.js && browserify lib/sync.js --transform babelify --standalone parse > lib/browser/sync.js", | ||
"build": "npm run build:babel && npm run build:browserify", | ||
"preversion": "cp lib/*.ts lib/es5 && git add lib/es5/*.ts", | ||
"build": "npm run build:rollup && npm run build:ts", | ||
"build:rollup": "npx rollup -c", | ||
"build:ts": "cp lib/*.ts dist/cjs && cp lib/*.ts dist/esm", | ||
"lint": "npm run lint:lib && npm run lint:samples && npm run lint:test", | ||
"lint:lib": "eslint --fix lib/*.js", | ||
"lint:samples": "eslint --fix samples/*.js", | ||
"lint:test": "coffeelint --fix test/*.coffee", | ||
"preversion": "npm run build && git add dist", | ||
"pretest": "npm run build", | ||
"lint": "eslint lib/*.js", | ||
"test": "npm run lint && TS_NODE_COMPILER_OPTIONS='{\"strictNullChecks\":true}' mocha test/**/*.{coffee,ts}" | ||
"test": "mocha 'test/**/*.{coffee,ts}'", | ||
"test:legacy": "mocha --loader=./test/loaders/legacy/all.mjs 'test/**/*.{coffee,ts}'" | ||
}, | ||
"types": "./lib/index.d.ts", | ||
"gitHead": "0fd5209b6862655c384cda052abf38019b959e70" | ||
"type": "module", | ||
"types": "dist/esm/index.d.ts", | ||
"typesVersions": { | ||
"*": { | ||
".": [ | ||
"dist/esm/index.d.ts" | ||
], | ||
"sync": [ | ||
"dist/esm/sync.d.ts" | ||
] | ||
} | ||
}, | ||
"gitHead": "d95c35dc7fd698e8e3278f539c8f1d43fb77640b" | ||
} |
# CSV Parser for Node.js | ||
# CSV parser for Node.js and the web | ||
[![Build Status](https://api.travis-ci.org/adaltas/node-csv-parse.svg)](https://travis-ci.org/#!/adaltas/node-csv-parse) [![NPM](https://img.shields.io/npm/dm/csv-parse)](https://www.npmjs.com/package/csv-parse) [![NPM](https://img.shields.io/npm/v/csv-parse)](https://www.npmjs.com/package/csv-parse) | ||
[![Build Status](https://img.shields.io/github/workflow/status/adaltas/node-csv/Node.js)](https://github.com/adaltas/node-csv/actions) | ||
[![NPM](https://img.shields.io/npm/dm/csv-parse)](https://www.npmjs.com/package/csv-parse) | ||
[![NPM](https://img.shields.io/npm/v/csv-parse)](https://www.npmjs.com/package/csv-parse) | ||
The [`csv-parse` package](https://csv.js.org/parse/) is a parser converting CSV text input into arrays or objects. It is part of the [CSV project](https://csv.js.org/). | ||
Part of the [CSV module](https://csv.js.org/), this project is a parser converting CSV text input into arrays or objects. It implements the Node.js [`stream.Transform` API](http://nodejs.org/api/stream.html#stream_class_stream_transform). It also provides a simple callback-based API for convenience. It is both extremely easy to use and powerful. It was first released in 2010 and is used against big data sets by a large community. | ||
It implements the Node.js [`stream.Transform` API](http://nodejs.org/api/stream.html#stream_class_stream_transform). It also provides a simple callback-based API for convenience. It is both extremely easy to use and powerful. It was first released in 2010 and is used against big data sets by a large community. | ||
## Documentation | ||
* [Project homepage](http://csv.js.org/parse/) | ||
* [API](http://csv.js.org/parse/api/) | ||
* [Options](http://csv.js.org/parse/options/) | ||
* [Info properties](http://csv.js.org/parse/info/) | ||
* [Common errors](http://csv.js.org/parse/errors/) | ||
* [Examples](http://csv.js.org/project/examples/) | ||
* [Project homepage](https://csv.js.org/parse/) | ||
* [API](https://csv.js.org/parse/api/) | ||
* [Options](https://csv.js.org/parse/options/) | ||
* [Info properties](https://csv.js.org/parse/info/) | ||
* [Common errors](https://csv.js.org/parse/errors/) | ||
* [Examples](https://csv.js.org/project/examples/) | ||
## Features | ||
## Main features | ||
* Flexible with lot of [options](https://csv.js.org/parse/options/) | ||
* Multiple [distributions](https://csv.js.org/parse/distributions/): Node.js, Web, ECMAScript modules and CommonJS | ||
* Follow the Node.js streaming API | ||
@@ -24,5 +30,58 @@ * Simplicity with the optional callback API | ||
* Support big datasets | ||
* Complete test coverage and samples for inspiration | ||
* Complete test coverage and lot of samples for inspiration | ||
* No external dependencies | ||
* Work nicely with the [csv-generate](https://csv.js.org/generate/), [stream-transform](https://csv.js.org/transform/) and [csv-stringify](https://csv.js.org/stringify/) packages | ||
* MIT License | ||
## Usage | ||
Run `npm install csv` to install the full csv module or run `npm install csv-parse` if you are only interested by the CSV parser. | ||
Use the callback and sync APIs for simplicity or the stream based API for scalability. | ||
## Example | ||
The [API](https://csv.js.org/parse/api/) is available in multiple flavors. This example illustrates the stream API. | ||
```js | ||
import assert from 'assert'; | ||
import { parse } from 'csv-parse'; | ||
const records = []; | ||
// Initialize the parser | ||
const parser = parse({ | ||
delimiter: ':' | ||
}); | ||
// Use the readable stream api to consume records | ||
parser.on('readable', function(){ | ||
let record; | ||
while ((record = parser.read()) !== null) { | ||
records.push(record); | ||
} | ||
}); | ||
// Catch any error | ||
parser.on('error', function(err){ | ||
console.error(err.message); | ||
}); | ||
// Test that the parsed records matched the expected records | ||
parser.on('end', function(){ | ||
assert.deepStrictEqual( | ||
records, | ||
[ | ||
[ 'root','x','0','0','root','/root','/bin/bash' ], | ||
[ 'someone','x','1022','1022','','/home/someone','/bin/bash' ] | ||
] | ||
); | ||
}); | ||
// Write data to the stream | ||
parser.write("root:x:0:0:root:/root:/bin/bash\n"); | ||
parser.write("someone:x:1022:1022::/home/someone:/bin/bash\n"); | ||
// Close the readable stream | ||
parser.end(); | ||
``` | ||
## Contributors | ||
The project is sponsored by [Adaltas](https://www.adaltas.com), an Big Data consulting firm based in Paris, France. | ||
* David Worms: <https://github.com/wdavidw> |
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
Filesystem access
Supply chain riskAccesses the file system, and could potentially read sensitive data.
Found 1 instance in 1 package
Dynamic require
Supply chain riskDynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.
Found 1 instance in 1 package
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 1 instance in 1 package
1907150
89
47469
87
Yes
1