fast-csv - npm Package Compare versions

Comparing version 3.6.0 to 3.7.0

build/src/formatter/CsvFormatterStream.js

@@ -51,6 +51,13 @@ "use strict";
     _flush(cb) {
-        if (this.formatterOptions.includeEndRowDelimiter) {
-            this.push(this.formatterOptions.rowDelimiter);
-        }
-        cb();
+        this.rowFormatter.finish((err, rows) => {
+            if (err) {
+                return cb(err);
+            }
+            if (rows) {
+                rows.forEach((r) => {
+                    this.push(Buffer.from(r, 'utf8'));
+                });
+            }
+            return cb();
+        });
     }
@@ -57,0 +64,0 @@ }

build/src/formatter/formatter/RowFormatter.d.ts

@@ -18,2 +18,3 @@ import { FormatterOptions } from '../FormatterOptions';
     format(row: Row, cb: RowFormatterCallback): void;
+    finish(cb: RowFormatterCallback): void;
     private checkHeaders;
@@ -20,0 +21,0 @@ private gatherColumns;

build/src/formatter/formatter/RowFormatter.js

@@ -89,2 +89,16 @@ "use strict";
     }
+    finish(cb) {
+        const rows = [];
+        // check if we should write headers and we didnt get any rows
+        if (this.formatterOptions.alwaysWriteHeaders && this.rowCount === 0) {
+            if (!this.headers) {
+                return cb(new Error('`alwaysWriteHeaders` option is set to true but `headers` option not provided.'));
+            }
+            rows.push(this.formatColumns(this.headers, true));
+        }
+        if (this.formatterOptions.includeEndRowDelimiter) {
+            rows.push(this.formatterOptions.rowDelimiter);
+        }
+        return cb(null, rows);
+    }
     // check if we need to write header return true if we should also write a row
@@ -91,0 +105,0 @@ // could be false if headers is true and the header row(first item) is passed in

build/src/formatter/FormatterOptions.d.ts

@@ -18,2 +18,3 @@ import { RowTransformFunction } from './types';
     transform?: RowTransformFunction;
+    alwaysWriteHeaders?: boolean;
 }
@@ -35,4 +36,5 @@ export declare class FormatterOptions {
     readonly BOM: string;
+    readonly alwaysWriteHeaders: boolean;
     constructor(opts?: FormatterOptionsArgs);
 }
 export {};

build/src/formatter/FormatterOptions.js

@@ -18,2 +18,3 @@ "use strict";
         this.BOM = '\ufeff';
+        this.alwaysWriteHeaders = false;
         Object.assign(this, opts || {});
@@ -20,0 +21,0 @@ if (typeof ((_a = opts) === null || _a === void 0 ? void 0 : _a.quoteHeaders) === 'undefined') {

build/src/index.d.ts

@@ -9,5 +9,5 @@ /**
 export { format, write, writeToStream, writeToBuffer, writeToString, writeToPath, FormatterOptionsArgs, Row as FormatterRow, RowMap as FormatterRowMap, RowArray as FormatterRowArray, RowHashArray as FormatterRowHashArray, RowTransformCallback as FormatterRowTransformCallback, RowTransformFunction as FormatterRowTransformFunction, } from './formatter';
-export { parse, parseString, parseStream, parseFile, ParserOptionsArgs, Row as ParserRow, RowMap as ParserRowMap, RowArray as ParserRowArray, RowValidateCallback as ParserRowValidateCallback, SyncRowValidate as ParserSyncRowValidate, AsyncRowValidate as ParserAsyncRowValidate, RowValidate as ParserRowValidate, RowTransformCallback as ParserRowTransformCallback, SyncRowTransform as ParserSyncRowTransform, AsyncRowTransform as ParserAsyncRowTransform, RowTransformFunction as ParserRowTransformFunction, } from './parser';
+export { parse, parseString, parseStream, parseFile, ParserOptionsArgs, Row as ParserRow, RowMap as ParserRowMap, RowArray as ParserRowArray, RowValidateCallback as ParserRowValidateCallback, SyncRowValidate as ParserSyncRowValidate, AsyncRowValidate as ParserAsyncRowValidate, RowValidate as ParserRowValidate, RowTransformCallback as ParserRowTransformCallback, SyncRowTransform as ParserSyncRowTransform, AsyncRowTransform as ParserAsyncRowTransform, RowTransformFunction as ParserRowTransformFunction, HeaderArray as ParserHeaderArray, HeaderTransformFunction as ParserHeaderTransformFunction, } from './parser';
 export declare const fromString: (string: string, options?: import("./parser").ParserOptionsArgs | undefined) => import("./parser").CsvParserStream;
 export declare const fromStream: (stream: NodeJS.ReadableStream, options?: import("./parser").ParserOptionsArgs | undefined) => import("./parser").CsvParserStream;
 export declare const fromPath: (location: string, options?: import("./parser").ParserOptionsArgs) => import("./parser").CsvParserStream;

build/src/parser/CsvParserStream.js

@@ -61,3 +61,3 @@ "use strict";
         catch (e) {
-            return done(e);
+            return this.destroy(e);
         }
@@ -90,2 +90,13 @@ }
         const iterate = (i) => {
+            const callNext = (err) => {
+                if (err) {
+                    return this.destroy(err);
+                }
+                if (i % 100 === 0) {
+                    // incase the transform are sync insert a next tick to prevent stack overflow
+                    setImmediate(() => iterate(i + 1));
+                    return undefined;
+                }
+                return iterate(i + 1);
+            };
             // if we have emitted all rows or we have hit the maxRows limit option
@@ -98,3 +109,3 @@ // then end
             if (this.shouldSkipLine) {
-                return iterate(i + 1);
+                return callNext();
             }
@@ -108,6 +119,6 @@ const row = rows[i];
                     this.rowCount -= 1;
-                    return cb(err);
+                    return callNext(err);
                 }
                 if (!transformResult) {
-                    return cb(new Error('expected transform result'));
+                    return callNext(new Error('expected transform result'));
                 }
@@ -118,10 +129,5 @@ if (!transformResult.isValid) {
                 else if (transformResult.row) {
-                    this.pushRow(transformResult.row);
+                    return this.pushRow(transformResult.row, callNext);
                 }
-                if (i % 100 === 0) {
-                    // incase the transform are sync insert a next tick to prevent stack overflow
-                    setImmediate(() => iterate(i + 1));
-                    return undefined;
-                }
-                return iterate(i + 1);
+                return callNext();
             });
@@ -161,8 +167,14 @@ };
     }
-    pushRow(row) {
-        if (!this.parserOptions.objectMode) {
-            this.push(JSON.stringify(row));
+    pushRow(row, cb) {
+        try {
+            if (!this.parserOptions.objectMode) {
+                this.push(JSON.stringify(row));
+            }
+            else {
+                this.push(row);
+            }
+            cb();
         }
-        else {
-            this.push(row);
+        catch (e) {
+            cb(e);
         }
@@ -169,0 +181,0 @@ }

build/src/parser/ParserOptions.d.ts

@@ -0,1 +1,2 @@
+import { HeaderArray, HeaderTransformFunction } from './types';
 export interface ParserOptionsArgs {
@@ -6,3 +7,3 @@ objectMode?: boolean;
     escape?: string;
-    headers?: boolean | (string | undefined | null)[];
+    headers?: boolean | HeaderTransformFunction | HeaderArray;
     renameHeaders?: boolean;
@@ -34,3 +35,3 @@ ignoreEmpty?: boolean;
     readonly trim: boolean;
-    readonly headers: boolean | string[] | null;
+    readonly headers: boolean | HeaderTransformFunction | HeaderArray | null;
     readonly renameHeaders: boolean;
@@ -37,0 +38,0 @@ readonly strictColumnHandling: boolean;

build/src/parser/transforms/HeaderTransformer.d.ts

@@ -10,2 +10,3 @@ import { ParserOptions } from '../ParserOptions';
     private headersLength;
+    private readonly headersTransform?;
     constructor(parserOptions: ParserOptions);
@@ -16,2 +17,3 @@ transform(row: RowArray, cb: RowValidatorCallback): void;
     private mapHeaders;
+    private setHeaders;
 }

build/src/parser/transforms/HeaderTransformer.js

@@ -7,13 +7,22 @@ "use strict";
 const lodash_isundefined_1 = __importDefault(require("lodash.isundefined"));
+const lodash_isfunction_1 = __importDefault(require("lodash.isfunction"));
+const lodash_uniq_1 = __importDefault(require("lodash.uniq"));
+const lodash_groupby_1 = __importDefault(require("lodash.groupby"));
 class HeaderTransformer {
     constructor(parserOptions) {
+        this.headers = null;
+        this.receivedHeaders = false;
+        this.shouldUseFirstRow = false;
         this.processedFirstRow = false;
         this.headersLength = 0;
         this.parserOptions = parserOptions;
-        this.headers = Array.isArray(parserOptions.headers) ? parserOptions.headers : null;
-        this.receivedHeaders = Array.isArray(parserOptions.headers);
-        this.shouldUseFirstRow = parserOptions.headers === true;
-        if (this.receivedHeaders && this.headers) {
-            this.headersLength = this.headers.length;
+        if (parserOptions.headers === true) {
+            this.shouldUseFirstRow = true;
         }
+        else if (Array.isArray(parserOptions.headers)) {
+            this.setHeaders(parserOptions.headers);
+        }
+        else if (lodash_isfunction_1.default(parserOptions.headers)) {
+            this.headersTransform = parserOptions.headers;
+        }
     }
@@ -28,3 +37,3 @@ transform(row, cb) {
         const { parserOptions } = this;
-        if (parserOptions.renameHeaders && !this.processedFirstRow) {
+        if (!this.headersTransform && parserOptions.renameHeaders && !this.processedFirstRow) {
             if (!this.receivedHeaders) {
@@ -36,6 +45,13 @@ throw new Error('Error renaming headers: new headers must be provided in an array');
         }
-        if (!this.receivedHeaders && this.shouldUseFirstRow && Array.isArray(row)) {
-            this.headers = row;
-            this.receivedHeaders = true;
-            this.headersLength = row.length;
+        if (!this.receivedHeaders && Array.isArray(row)) {
+            if (this.headersTransform) {
+                this.setHeaders(this.headersTransform(row));
+            }
+            else if (this.shouldUseFirstRow) {
+                this.setHeaders(row);
+            }
+            else {
+                // dont do anything with the headers if we didnt receive a transform or shouldnt use the first row.
+                return true;
+            }
             return false;
@@ -87,4 +103,16 @@ }
     }
+    setHeaders(headers) {
+        var _a;
+        const filteredHeaders = headers.filter(h => !!h);
+        if (lodash_uniq_1.default(filteredHeaders).length !== filteredHeaders.length) {
+            const grouped = lodash_groupby_1.default(filteredHeaders);
+            const duplicates = Object.keys(grouped).filter(dup => grouped[dup].length > 1);
+            throw new Error(`Duplicate headers found ${JSON.stringify(duplicates)}`);
+        }
+        this.headers = headers;
+        this.receivedHeaders = true;
+        this.headersLength = ((_a = this.headers) === null || _a === void 0 ? void 0 : _a.length) || 0;
+    }
 }
 exports.default = HeaderTransformer;
 //# sourceMappingURL=HeaderTransformer.js.map

build/src/parser/types.d.ts

@@ -22,1 +22,3 @@ export interface RowMap {
 export declare const isSyncValidate: (validate: RowValidate) => validate is SyncRowValidate;
+export declare type HeaderArray = (string | undefined | null)[];
+export declare type HeaderTransformFunction = (headers: HeaderArray) => HeaderArray;

History.md

@@ -0,1 +1,11 @@
+# v3.7.0
+
+* [ADDED] Ability to Transform Header [#287](https://github.com/C2FO/fast-csv/issues/287)
+* [ADDED] Example require and import to README [#301](https://github.com/C2FO/fast-csv/issues/301)
+* [ADDED] Added new formatting option `alwaysWriteHeaders` to always write headers even if no rows are provided [#300](https://github.com/C2FO/fast-csv/issues/300)
+* [ADDED] Appending to csv example and docs [#272](https://github.com/C2FO/fast-csv/issues/300)
+* [FIXED] Issue with duplicate headers causing dataloss, duplicate headers will can an error to be emitted. [#276](https://github.com/C2FO/fast-csv/issues/272)
+* [FIXED] Issue where an error thrown while processing rows causes stream continue to parse, causing duplicate writes or swallowed exceptions.
+
+
 # v3.6.0
@@ -2,0 +12,0 @@

package.json

 {
   "name": "fast-csv",
-  "version": "3.6.0",
+  "version": "3.7.0",
   "description": "CSV parser and writer",
@@ -17,3 +17,5 @@ "main": "./build/src/index.js",
   },
-  "files": ["build/src/**"],
+  "files": [
+    "build/src/**"
+  ],
   "repository": {
@@ -44,2 +46,4 @@ "type": "git",
     "@types/lodash.partition": "^4.6.6",
+    "@types/lodash.uniq": "^4.5.6",
+    "@types/lodash.groupby": "^4.6.6",
     "@types/mocha": "^5.2.7",
@@ -71,2 +75,3 @@ "@types/sinon": "^7.5.1",
     "lodash.escaperegexp": "^4.1.2",
+    "lodash.groupby": "^4.6.0",
     "lodash.isboolean": "^3.0.3",
@@ -77,4 +82,5 @@ "lodash.isequal": "^4.5.0",
     "lodash.isstring": "^4.0.1",
-    "lodash.isundefined": "^3.0.1"
+    "lodash.isundefined": "^3.0.1",
+    "lodash.uniq": "^4.5.0"
   }
 }

README.md

@@ -14,2 +14,16 @@ [![npm version](https://img.shields.io/npm/v/fast-csv.svg)](https://www.npmjs.org/package/fast-csv)
 
+## Usage
+
+To use `fast-csv` in `javascript` you can require the module/
+
+```js
+const csv = require('fast-csv');
+```
+
+To import with typescript 
+
+```typescript
+import * as csv from 'fast-csv';
+```
+
 ## Documentation
@@ -59,1 +73,2 @@
 
+

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

124752

increased by4.85%

Lines of code

1710

increased by4.33%

Number of lines in readme file

73

increased by25.86%

Worsened metrics

Dependency count

10

increased by25%

Dev dependency count

30

increased by7.14%

Dependency changes

• + Addedlodash.groupby@^4.6.0

• + Addedlodash.uniq@^4.5.0

• + Addedlodash.groupby@4.6.0(transitive)

• + Addedlodash.uniq@4.5.0(transitive)