@cerios/csv-nested-json - npm Package Compare versions

Comparing version 1.2.1 to 1.3.0

dist/index.d.mts

 import * as node_stream from 'node:stream';
-import { Readable, TransformOptions, Transform, TransformCallback } from 'node:stream';
+import { Readable, Transform, TransformOptions, TransformCallback } from 'node:stream';
 
@@ -692,2 +692,3 @@ /**
      * @param content - The CSV content
+     * @param quote - The quote character (default: '"')
      * @returns Array of lines (quoted newlines are preserved within lines)
@@ -701,3 +702,3 @@ *
      */
-    static splitLines(content: string): string[];
+    static splitLines(content: string, quote?: string): string[];
     /**
@@ -751,9 +752,2 @@ * Parse a single CSV line into an array of values, respecting quotes and delimiters.
     /**
-     * Whether to emit nested objects (true) or flat records (false).
-     * When true, records are converted using NestedJsonConverter logic.
-     * When false, raw flat records are emitted.
-     * @default true
-     */
-    nested?: boolean;
-    /**
      * Callback function called periodically to report parsing progress.
@@ -799,2 +793,8 @@ * Can be synchronous or asynchronous.
     batchSize?: number;
+    /**
+     * Maximum number of raw rows buffered for one continuation group.
+     * Prevents unbounded memory growth when identifier values are missing.
+     * @default 10000
+     */
+    maxContinuationGroupSize?: number;
 }
@@ -808,5 +808,4 @@ /**
  *
- * Note: For true nested JSON conversion with continuation rows (rows that extend
- * previous records), consider using the standard CsvParser methods, as continuation
- * row handling requires buffering related rows together.
+ * By default, continuation rows (rows where the identifier column is empty) are
+ * grouped with the previous record to match `CsvParser` behavior.
  *
@@ -868,2 +867,5 @@ * @example
     private recordBatch;
+    private groupedRecords;
+    private groupingIdentifierColumn;
+    private maxContinuationGroupSize;
     /**
@@ -910,5 +912,5 @@ * Creates a new streaming CSV parser.
      * ```typescript
-     * // Get flat records instead of nested
-     * const flatRecords = await CsvStreamParser.parseStream(stream, {
-     *   nested: false
+     * // Parse with stream-specific options
+     * const records = await CsvStreamParser.parseStream(stream, {
+     *   batchSize: 100
      * });
@@ -946,2 +948,26 @@ * ```
     /**
+     * Whether continuation grouping should be applied for this parser instance.
+     */
+    private shouldGroupContinuationRows;
+    /**
+     * Determine the column used for continuation grouping.
+     */
+    private resolveGroupingIdentifierColumn;
+    /**
+     * Add a record to the active continuation group with memory guardrails.
+     */
+    private pushGroupedRecord;
+    /**
+     * Buffer records and flush groups when a new identifier value is encountered.
+     */
+    private bufferGroupedRecord;
+    /**
+     * Flush buffered grouped records through NestedJsonConverter.
+     */
+    private flushGroupedRecords;
+    /**
+     * Emit a single parsed output record while respecting batching, limits, and progress callbacks.
+     */
+    private emitRecord;
+    /**
      * Parse a single line into values.
@@ -1230,2 +1256,10 @@ */
     /**
+     * Resolve which array path (if any) owns a header.
+     */
+    private static findArrayPathForHeader;
+    /**
+     * Read a nested value from an object using dot-notation.
+     */
+    private static getNestedValueAtPath;
+    /**
      * Convert a value to string for CSV output.
@@ -1374,2 +1408,7 @@ */
     /**
+     * Resolve the last item for an array path using merge state first,
+     * then fallback to the current target structure.
+     */
+    private static getLastArrayItemAtPath;
+    /**
      * Ensure an array exists at the given path, creating intermediate objects as needed.
@@ -1376,0 +1415,0 @@ */

dist/index.d.ts

 import * as node_stream from 'node:stream';
-import { Readable, TransformOptions, Transform, TransformCallback } from 'node:stream';
+import { Readable, Transform, TransformOptions, TransformCallback } from 'node:stream';
 
@@ -692,2 +692,3 @@ /**
      * @param content - The CSV content
+     * @param quote - The quote character (default: '"')
      * @returns Array of lines (quoted newlines are preserved within lines)
@@ -701,3 +702,3 @@ *
      */
-    static splitLines(content: string): string[];
+    static splitLines(content: string, quote?: string): string[];
     /**
@@ -751,9 +752,2 @@ * Parse a single CSV line into an array of values, respecting quotes and delimiters.
     /**
-     * Whether to emit nested objects (true) or flat records (false).
-     * When true, records are converted using NestedJsonConverter logic.
-     * When false, raw flat records are emitted.
-     * @default true
-     */
-    nested?: boolean;
-    /**
      * Callback function called periodically to report parsing progress.
@@ -799,2 +793,8 @@ * Can be synchronous or asynchronous.
     batchSize?: number;
+    /**
+     * Maximum number of raw rows buffered for one continuation group.
+     * Prevents unbounded memory growth when identifier values are missing.
+     * @default 10000
+     */
+    maxContinuationGroupSize?: number;
 }
@@ -808,5 +808,4 @@ /**
  *
- * Note: For true nested JSON conversion with continuation rows (rows that extend
- * previous records), consider using the standard CsvParser methods, as continuation
- * row handling requires buffering related rows together.
+ * By default, continuation rows (rows where the identifier column is empty) are
+ * grouped with the previous record to match `CsvParser` behavior.
  *
@@ -868,2 +867,5 @@ * @example
     private recordBatch;
+    private groupedRecords;
+    private groupingIdentifierColumn;
+    private maxContinuationGroupSize;
     /**
@@ -910,5 +912,5 @@ * Creates a new streaming CSV parser.
      * ```typescript
-     * // Get flat records instead of nested
-     * const flatRecords = await CsvStreamParser.parseStream(stream, {
-     *   nested: false
+     * // Parse with stream-specific options
+     * const records = await CsvStreamParser.parseStream(stream, {
+     *   batchSize: 100
      * });
@@ -946,2 +948,26 @@ * ```
     /**
+     * Whether continuation grouping should be applied for this parser instance.
+     */
+    private shouldGroupContinuationRows;
+    /**
+     * Determine the column used for continuation grouping.
+     */
+    private resolveGroupingIdentifierColumn;
+    /**
+     * Add a record to the active continuation group with memory guardrails.
+     */
+    private pushGroupedRecord;
+    /**
+     * Buffer records and flush groups when a new identifier value is encountered.
+     */
+    private bufferGroupedRecord;
+    /**
+     * Flush buffered grouped records through NestedJsonConverter.
+     */
+    private flushGroupedRecords;
+    /**
+     * Emit a single parsed output record while respecting batching, limits, and progress callbacks.
+     */
+    private emitRecord;
+    /**
      * Parse a single line into values.
@@ -1230,2 +1256,10 @@ */
     /**
+     * Resolve which array path (if any) owns a header.
+     */
+    private static findArrayPathForHeader;
+    /**
+     * Read a nested value from an object using dot-notation.
+     */
+    private static getNestedValueAtPath;
+    /**
      * Convert a value to string for CSV output.
@@ -1374,2 +1408,7 @@ */
     /**
+     * Resolve the last item for an array path using merge state first,
+     * then fallback to the current target structure.
+     */
+    private static getLastArrayItemAtPath;
+    /**
      * Ensure an array exists at the given path, creating intermediate objects as needed.
@@ -1376,0 +1415,0 @@ */

package.json

 {
 	"name": "@cerios/csv-nested-json",
-	"version": "1.2.1",
+	"version": "1.3.0",
 	"author": "Ronald Veth - Cerios",
@@ -64,11 +64,11 @@ "description": "Parse CSV files into nested JSON objects with support for dot notation, arrays, and complex data structures",
 		"@arethetypeswrong/cli": "^0.18.2",
-		"@biomejs/biome": "2.3.6",
-		"@changesets/cli": "^2.29.7",
-		"@types/jest": "^29.5.5",
-		"@types/node": "^24.10.1",
+		"@biomejs/biome": "2.4.10",
+		"@changesets/cli": "^2.30.0",
+		"@types/jest": "^30.0.0",
+		"@types/node": "^25.5.2",
 		"husky": "^9.1.7",
-		"jest": "^29.7.0",
-		"lint-staged": "^16.2.6",
-		"npm-check-updates": "^19.1.2",
-		"ts-jest": "^29.4.5",
+		"jest": "^30.3.0",
+		"lint-staged": "^16.4.0",
+		"npm-check-updates": "^20.0.0",
+		"ts-jest": "^29.4.9",
 		"tsup": "^8.5.1",
@@ -75,0 +75,0 @@ "typescript": "^5.9.3"

README.md

@@ -67,4 +67,7 @@ # @cerios/csv-nested-json
 | `CsvParser.parseString()` | Parse CSV string content |
-| `CsvParser.parseStream()` | Parse CSV from readable stream |
+| `CsvParser.parseStream()` | Parse CSV from readable stream (buffers full content in memory) |
 | `CsvStreamParser` | True streaming parser for very large files |
+| `CsvReader.parse()` | Low-level CSV parser that returns flat records |
+| `CsvFileReader.readFile*()` | Low-level file/stream text reader |
+| `NestedJsonConverter.convert()` | Low-level flat-record to nested JSON converter |
 | `JsonToCsv.stringify()` | Convert JSON objects to CSV string |
@@ -120,7 +123,9 @@ | `JsonToCsv.writeFileSync()` | Write JSON objects to CSV file (sync) |
 
-**When to use:** Very large files (>100MB), memory-constrained environments, real-time processing.
+**When to use:** CSV input already comes from a readable stream and buffering the full result in memory is acceptable.
 
+**Note:** `CsvParser.parseStream()` buffers the entire stream before converting. For true incremental parsing, use `CsvStreamParser`.
+
 ### 5. True Streaming Parser (Memory Efficient)
 
-For very large files where you want to process records one at a time without loading everything into memory:
+For very large files where you want streaming processing without loading everything into memory (continuation rows are grouped by default):
 
@@ -545,3 +550,3 @@ ```typescript
 
-// Keep first occurrence (default)
+// Keep first occurrence
 const result1 = CsvParser.parseString(csvContent, {
@@ -665,2 +670,4 @@ duplicateHeaders: 'first'
 
+Null conversion is opt-in. Values are only treated as null when `nullValues` is provided.
+
 ### BOM Handling
@@ -717,6 +724,5 @@
     },
-    "addresses": {
-      "type": "home",
-      "city": "Chicago"
-    }
+    "addresses": [
+      { "type": "home", "city": "Chicago" }
+    ]
   }
@@ -726,3 +732,3 @@ ]
 
-**Note:** The first record has an array of addresses (multiple entries), while the second has a single address object.
+**Note:** Array shape is normalized across records. If one record needs an array for a path, other records with a single value for that path are represented as single-item arrays.
 
@@ -904,4 +910,4 @@ ### Custom Delimiters
   // Column selection
-  includeColumns?: string[];                      // Include only these columns
-  excludeColumns?: string[];                      // Exclude these columns
+  includeColumns?: string[];                      // Include only these columns (matched against original CSV headers)
+  excludeColumns?: string[];                      // Exclude columns after includeColumns is applied
 
@@ -939,4 +945,4 @@ // Duplicate header handling
 interface CsvStreamParserOptions extends CsvParserOptions {
-  nested?: boolean;                               // Emit nested objects (default: true)
   batchSize?: number;                             // Emit records in batches
+  maxContinuationGroupSize?: number;              // Max raw rows buffered per continuation group (default: 10000)
   progressCallback?: ProgressCallback;            // Progress tracking callback
@@ -1065,2 +1071,5 @@ progressInterval?: number;                      // Records between callbacks (default: 100)
 
+`includeColumns` and `excludeColumns` can be combined. Inclusion is applied first, then exclusion.
+Column filtering matches original CSV header names before `headerTransformer` and `columnMapping` are applied.
+
 #### `excludeColumns`
@@ -1092,2 +1101,6 @@
 
+The first data row in a group must contain an identifier value. If the first row is a continuation row (empty identifier), parsing throws `CsvParseError` to avoid ambiguous grouping.
+
+If `headerTransformer` or `columnMapping` is used, set `identifierColumn` to the transformed/mapped header name (not the original CSV header).
+
 ```typescript
@@ -1098,2 +1111,13 @@ // Use 'productId' instead of first column to group rows
 
+#### `maxContinuationGroupSize` (streaming only)
+
+Maximum number of raw rows buffered for a single continuation group in `CsvStreamParser`. This protects against unbounded memory growth when identifier values are missing for long stretches.
+
+- Default: `10000`
+- When exceeded, parsing throws `CsvParseError`
+
+```typescript
+maxContinuationGroupSize: 5000
+```
+
 #### `arraySuffixIndicator`
@@ -1111,3 +1135,3 @@
 
-Strings to interpret as null values. Default: `['null', 'NULL', 'nil', 'NIL', '']`
+Strings to interpret as null values. Null detection is disabled unless this option is provided.
 
@@ -1201,3 +1225,3 @@ #### `nullRepresentation`
 
-Parses CSV from a readable stream.
+Parses CSV from a readable stream and returns all records. This method buffers the full stream in memory before conversion.
 
@@ -1212,6 +1236,6 @@ ### CsvStreamParser Class
 const parser = new CsvStreamParser({
-  nested: true,           // Emit nested objects (default: true)
   autoParseNumbers: true,
   limit: 1000,            // Stop after 1000 records
   batchSize: 100,         // Emit in batches of 100
+  maxContinuationGroupSize: 10000, // Safety guard for continuation buffering
   progressCallback: (info: ProgressInfo) => {
@@ -1388,4 +1412,4 @@ console.log(`Progress: ${info.recordsEmitted} records, ${info.elapsedMs}ms`);
 | `parseString()` | API responses, testing | Any (in-memory) | Yes |
-| `parseStream()` | Large files, memory efficiency | >100MB | No |
-| `CsvStreamParser` | Very large files, ETL pipelines | Any size | No |
+| `parseStream()` | Readable stream inputs with buffered output | Any (buffered in memory) | No |
+| `CsvStreamParser` | Large/very large files, ETL pipelines | Any size | No |
 
@@ -1531,7 +1555,7 @@ ### Traditional CSV Parsing
 1. **Choose the Right Method:**
-   - Use `parseFileSync()` for small files in scripts
-   - Use `parseFile()` for web servers and async workflows
-   - Use `parseString()` for API responses and testing
-   - Use `parseStream()` for large files
-   - Use `CsvStreamParser` for very large files or when you need to process records one at a time
+  - Use `parseFileSync()` for small files in scripts
+  - Use `parseFile()` for web servers and async workflows
+  - Use `parseString()` for API responses and testing
+  - Use `parseStream()` when your source is a readable stream and buffering all content is acceptable
+  - Use `CsvStreamParser` for large/very large files or incremental processing; it groups continuation rows to match `CsvParser` continuation semantics
 
@@ -1538,0 +1562,0 @@ 2. **Use Appropriate Validation Mode:**

No alert changes

Improved metrics

Total package byte prevSize

627435

7.6%

Lines of code

6162

7.33%

Number of lines in readme file

1626

1.5%

No dependency changes