fast-csv - npm Package Compare versions

Comparing version 3.0.2 to 3.1.0

build/src/parser/CsvParserStream.js

@@ -10,3 +10,2 @@ "use strict";
         super({ objectMode: parserOptions.objectMode });
-        this.decoder = new string_decoder_1.StringDecoder();
         this.lines = '';
@@ -18,2 +17,3 @@ this.rowCount = 0;
         this.headerTransformer = new transforms_1.HeaderTransformer(parserOptions);
+        this.decoder = new string_decoder_1.StringDecoder(parserOptions.encoding);
         this.rowTransformerValidator = new transforms_1.RowTransformerValidator();
@@ -53,3 +53,4 @@ }
         try {
-            const rows = this.parse(this.lines, false);
+            const newLine = (this.lines + this.decoder.end());
+            const rows = this.parse(newLine, false);
             this.processRows(rows, done);
@@ -56,0 +57,0 @@ }

build/src/parser/parser/column/QuotedColumnParser.js

@@ -77,6 +77,6 @@ "use strict";
         const { parserOptions } = this;
-        const { nextCharacterToken } = scanner;
-        if (nextCharacterToken) {
-            const isNextTokenADelimiter = isTokenDelimiter(nextCharacterToken, parserOptions);
-            const isNextTokenARowDelimiter = isTokenRowDelimiter(nextCharacterToken);
+        const { nextNonSpaceToken } = scanner;
+        if (nextNonSpaceToken) {
+            const isNextTokenADelimiter = isTokenDelimiter(nextNonSpaceToken, parserOptions);
+            const isNextTokenARowDelimiter = isTokenRowDelimiter(nextNonSpaceToken);
             if (!(isNextTokenADelimiter || isNextTokenARowDelimiter)) {
@@ -86,5 +86,9 @@ // if the final quote was NOT followed by a column (,) or row(\n) delimiter then its a bad column
                 const linePreview = scanner.lineFromCursor.substr(0, 10).replace(/[\r\n]/g, "\\n'");
-                throw new Error(`Parse Error: expected: '${parserOptions.escapedDelimiter}' OR new line got: '${nextCharacterToken.token}'. at '${linePreview}`);
+                throw new Error(`Parse Error: expected: '${parserOptions.escapedDelimiter}' OR new line got: '${nextNonSpaceToken.token}'. at '${linePreview}`);
             }
+            scanner.advanceToToken(nextNonSpaceToken);
         }
+        else if (!scanner.hasMoreData) {
+            scanner.advancePastLine();
+        }
     }
@@ -91,0 +95,0 @@ }

build/src/parser/parser/Parser.js

@@ -59,3 +59,3 @@ "use strict";
     parseRow(scanner, rows) {
-        const nextToken = scanner.nextCharacterToken;
+        const nextToken = scanner.nextNonSpaceToken;
         if (!nextToken) {
@@ -62,0 +62,0 @@ return false;

build/src/parser/ParserOptions.d.ts

@@ -15,2 +15,3 @@ export interface ParserOptionsArgs {
     rtrim?: boolean;
+    encoding?: string;
 }
@@ -36,3 +37,4 @@ export declare class ParserOptions {
     readonly NEXT_TOKEN_REGEXP: RegExp;
+    readonly encoding: string;
     constructor(opts?: ParserOptionsArgs);
 }

build/src/parser/ParserOptions.js

@@ -22,2 +22,3 @@ "use strict";
         this.carriageReturn = '\r';
+        this.encoding = 'utf8';
         Object.assign(this, opts || {});
@@ -24,0 +25,0 @@ if (this.delimiter.length > 1) {

docs/parsing.md

@@ -48,2 +48,3 @@ # Parsing
 * `ltrim: {boolean} = false`: Set to `true` to left trim all fields.
+* `encoding: {string} = 'utf8'`: Passed to [StringDecoder](https://nodejs.org/api/string_decoder.html#string_decoder_new_stringdecoder_encoding) when decoding incoming buffers. Change if incoming content is not 'utf8' encoded.
 
@@ -50,0 +51,0 @@ <a name="parsing-events"></a>

History.md

@@ -0,1 +1,6 @@
+# v3.1.0
+
+* Skip trailing whitespace after a quoted field [#223](https://github.com/C2FO/fast-csv/pull/223) - [@peet](https://github.com/peet)
+* Add support for passing in encoding. [#185](https://github.com/C2FO/fast-csv/pull/185) - [@pierrelouisd4j](https://github.com/pierrelouisd4j)
+
 # v3.0.2
@@ -2,0 +7,0 @@

package.json

 {
   "name": "fast-csv",
-  "version": "3.0.2",
+  "version": "3.1.0",
   "description": "CSV parser and writer",
@@ -5,0 +5,0 @@ "main": "./build/src/index.js",

src/parser/CsvParserStream.ts

@@ -14,3 +14,3 @@ import { NodeStringDecoder, StringDecoder } from 'string_decoder';
 
-    private readonly decoder: NodeStringDecoder = new StringDecoder();
+    private readonly decoder: NodeStringDecoder;
 
@@ -35,2 +35,3 @@ private readonly parser: Parser;
         this.headerTransformer = new HeaderTransformer(parserOptions);
+        this.decoder = new StringDecoder(parserOptions.encoding);
         this.rowTransformerValidator = new RowTransformerValidator();
@@ -75,3 +76,4 @@ }
         try {
-            const rows = this.parse(this.lines, false);
+            const newLine = (this.lines + this.decoder.end());
+            const rows = this.parse(newLine, false);
             this.processRows(rows, done);
@@ -78,0 +80,0 @@ } catch (e) {

src/parser/parser/column/QuotedColumnParser.ts

@@ -88,6 +88,6 @@ import ColumnFormatter from './ColumnFormatter';
         const { parserOptions } = this;
-        const { nextCharacterToken } = scanner;
-        if (nextCharacterToken) {
-            const isNextTokenADelimiter = isTokenDelimiter(nextCharacterToken, parserOptions);
-            const isNextTokenARowDelimiter = isTokenRowDelimiter(nextCharacterToken);
+        const { nextNonSpaceToken } = scanner;
+        if (nextNonSpaceToken) {
+            const isNextTokenADelimiter = isTokenDelimiter(nextNonSpaceToken, parserOptions);
+            const isNextTokenARowDelimiter = isTokenRowDelimiter(nextNonSpaceToken);
             if (!(isNextTokenADelimiter || isNextTokenARowDelimiter)) {
@@ -97,6 +97,9 @@ // if the final quote was NOT followed by a column (,) or row(\n) delimiter then its a bad column
                 const linePreview = scanner.lineFromCursor.substr(0, 10).replace(/[\r\n]/g, "\\n'");
-                throw new Error(`Parse Error: expected: '${parserOptions.escapedDelimiter}' OR new line got: '${nextCharacterToken.token}'. at '${linePreview}`);
+                throw new Error(`Parse Error: expected: '${parserOptions.escapedDelimiter}' OR new line got: '${nextNonSpaceToken.token}'. at '${linePreview}`);
             }
+            scanner.advanceToToken(nextNonSpaceToken);
+        } else if (!scanner.hasMoreData) {
+            scanner.advancePastLine();
         }
     }
 }

src/parser/parser/Parser.ts

@@ -73,3 +73,3 @@ import { Scanner, Token } from './Scanner';
     private parseRow(scanner: Scanner, rows: RowArray[]): boolean {
-        const nextToken = scanner.nextCharacterToken;
+        const nextToken = scanner.nextNonSpaceToken;
         if (!nextToken) {
@@ -76,0 +76,0 @@ return false;

src/parser/ParserOptions.ts

@@ -17,2 +17,3 @@ import { escapeRegExp, isString, isNil } from 'lodash';
     rtrim?: boolean;
+    encoding?: string;
 }
@@ -57,2 +58,4 @@
 
+    public readonly encoding: string = 'utf8';
+
     public constructor(opts?: ParserOptionsArgs) {
@@ -59,0 +62,0 @@ Object.assign(this, opts || {});

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

299420

increased by4.44%

Lines of code

2781

increased by0.54%

No dependency changes