fast-csv - npm Package Compare versions

Comparing version 0.4.4 to 0.5.0

lib/extended.js

@@ -0,5 +1,61 @@
+var is = require("is-extended");
 module.exports = require("extended")()
-    .register(require("is-extended"))
+    .register(is)
     .register(require("object-extended"))
     .register(require("string-extended"))
-    .register("LINE_BREAK", (process.platform === 'win32' ? '\r\n' : '\n'));
+    .register("LINE_BREAK", require("os").EOL)
+    .register("asyncEach", function (arr, iter, cb) {
+        (function asyncIterator(i, l, rows, cb) {
+            try {
+                if (++i < l) {
+                    iter(rows[i], function (err) {
+                        if (err) {
+                            cb(err);
+                        } else {
+                            if ((i % 100) === 0) {
+                                //dont overflow the stack
+                                setImmediate(function () {
+                                    asyncIterator(i, l, rows, cb);
+                                });
+                            } else {
+                                asyncIterator(i, l, rows, cb);
+                            }
+                        }
+                    });
+                } else {
+                    cb(null, arr);
+                }
+            } catch (e) {
+                cb(e);
+            }
+        }(-1, arr.length, arr, cb));
+    })
+    .register("spreadArgs", function spreadArgs(f, args, scope) {
+        var ret;
+        switch ((args || []).length) {
+            case 0:
+                ret = f.call(scope);
+                break;
+            case 1:
+                ret = f.call(scope, args[0]);
+                break;
+            case 2:
+                ret = f.call(scope, args[0], args[1]);
+                break;
+            case 3:
+                ret = f.call(scope, args[0], args[1], args[2]);
+                break;
+            default:
+                ret = f.apply(scope, args);
+        }
+        return ret;
+    })
+    .register("keys", function (obj) {
+        var ret = [];
+        if (is.isObject(obj)) {
+            for (var i in obj) {
+                ret.push(i);
+            }
+        }
+        return ret;
+    });

lib/index.js

@@ -9,41 +9,22 @@ /**
 var fs = require("fs"),
-    extended = require("./extended"),
-    ParserStream = require("./parser_stream"),
-    stream = require("stream"),
+    parser = require("./parser"),
     formatter = require("./formatter");
 
-function parse(options) {
-    return new ParserStream(options);
+function csv() {
+    return parser.apply(void 0, arguments);
 }
 
-function toCsvStream(arr, options) {
-    return formatter.writeToStream(arr, options);
-}
+csv.parse = csv;
+csv.fromString = parser.fromString;
+csv.fromPath = parser.fromPath;
+csv.fromStream = parser.fromStream;
+csv.format = formatter;
+csv.write = formatter.write;
+csv.writeToStream = formatter.writeToStream;
+csv.writeToString = formatter.writeToString;
+csv.writeToBuffer = formatter.writeToBuffer;
+csv.writeToPath = formatter.writeToPath;
+csv.createWriteStream = formatter.createWriteStream;
+csv.createReadStream = formatter.createWriteStream;
 
-function fromStream(stream, options) {
-    return stream.pipe(new ParserStream(options));
-}
-
-function fromPath(location, options) {
-    return fs.createReadStream(location).pipe(new ParserStream(options));
-}
-
-function fromString(string, options) {
-    var rs = new stream.Readable();
-    rs.push(string);
-    rs.push(null);
-    return rs.pipe(new ParserStream(options));
-}
-
-
-parse.fromString = fromString;
-parse.toCsvStream = toCsvStream;
-parse.fromPath = fromPath;
-parse.fromStream = fromStream;
-parse.write = formatter.write;
-parse.writeToStream = formatter.writeToStream;
-parse.writeToString = formatter.writeToString;
-parse.writeToPath = formatter.writeToPath;
-parse.createWriteStream = formatter.createWriteStream;
-
-module.exports = parse;
+module.exports = csv;

package.json

 {
-  "name": "fast-csv",
-  "version": "0.4.4",
-  "description": "CSV parser and writer",
-  "main": "index.js",
-  "scripts": {
-    "test": "grunt jshint it"
-  },
-  "repository": {
-    "type": "git",
-    "url": "[e-mail]:C2FO/fast-csv.git"
-  },
-  "keywords": [
-    "csv",
-    "parser",
-    "fast",
-    "writer",
-    "csv writer",
-    "CSV"
-  ],
-  "homepage": "http://c2fo.github.com/fast-csv/index.html",
-  "author": "Doug Martin",
-  "license": "MIT",
-  "devDependencies": {
-    "it": "~0.2.6",
-    "grunt-it": "~0.3.1",
-    "grunt": "~0.4.1",
-    "grunt-contrib-jshint": "~0.10.0",
-    "grunt-exec": "^0.4.5"
-  },
-  "engines": {
-    "node": ">=0.10"
-  },
-  "dependencies": {
-    "is-extended": "0.0.10",
-    "object-extended": "0.0.7",
-    "extended": "0.0.6",
-    "string-extended": "0.0.8"
-  }
+    "name": "fast-csv",
+    "version": "0.5.0",
+    "description": "CSV parser and writer",
+    "main": "index.js",
+    "scripts": {
+        "test": "grunt jshint it"
+    },
+    "repository": {
+        "type": "git",
+        "url": "[e-mail]:C2FO/fast-csv.git"
+    },
+    "keywords": [
+        "csv",
+        "parser",
+        "fast",
+        "writer",
+        "csv writer",
+        "CSV"
+    ],
+    "homepage": "http://c2fo.github.com/fast-csv/index.html",
+    "author": "Doug Martin",
+    "license": "MIT",
+    "devDependencies": {
+        "it": "~0.2.6",
+        "grunt-it": "~0.3.1",
+        "grunt": "~0.4.1",
+        "grunt-contrib-jshint": "~0.10.0",
+        "grunt-exec": "^0.4.5"
+    },
+    "engines": {
+        "node": ">=0.10"
+    },
+    "dependencies": {
+        "is-extended": "0.0.10",
+        "object-extended": "0.0.7",
+        "extended": "0.0.6",
+        "string-extended": "0.0.8"
+    }
 }

README.md

@@ -36,7 +36,8 @@ [![build status](https://secure.travis-ci.org/C2FO/fast-csv.png)](http://travis-ci.org/C2FO/fast-csv)
 
-* `record`: Emitted when a record is parsed.
+* `data`: Emitted when a record is parsed.
+* `record`: Emitted when a record is parsed. **DEPRECATED**
 * `data-invalid`: Emitted if there was invalid row encounted, **only emitted if the `validate` function is used**.
 * `data`: Emitted with the object or `stringified` version if the `objectMode` is set to `false`.
 
-**([options])**
+**`([options])` or `.parse(options)`**
 
@@ -49,12 +50,64 @@ If you use `fast-csv` as a function it returns a transform stream that can be piped into.
 var csvStream = csv()
- .on("record", function(data){
-     console.log(data);
- })
- .on("end", function(){
-     console.log("done");
- });
+    .on("data", function(data){
+         console.log(data);
+    })
+    .on("end", function(){
+         console.log("done");
+    });
 
 stream.pipe(csvStream);
+
+//or
+
+var csvStream = csv
+    .parse()
+    .on("data", function(data){
+         console.log(data);
+    })
+    .on("end", function(){
+         console.log("done");
+    });
+
+stream.pipe(csvStream);
 ```
 
+```javascript
+fs.createReadStream("my.csv")
+    .pipe(csv())
+    .on("data", function(data){
+        console.log(data);
+    })
+    .on("end", function(){
+        console.log("done");
+    });
+```
+
+```
+var fileStream = fs.createReadStream("my.csv"),
+    parser = fastCsv();
+
+fileStream
+    .on("readable", function () {
+        var data;
+        while ((data = fileStream.read()) !== null) {
+            parser.write(data);
+        }
+    })
+    .on("end", function () {
+        parser.end();
+    });
+
+parser
+    .on("readable", function () {
+        var data;
+        while ((data = parser.read()) !== null) {
+            console.log(data);
+        }
+    })
+    .on("end", function () {
+        console.log("done");
+    });
+```
+
+
 **`.fromPath(path[, options])`**
@@ -69,3 +122,3 @@
  .fromPath("my.csv")
- .on("record", function(data){
+ .on("data", function(data){
      console.log(data);
@@ -91,3 +144,3 @@ })
  .fromString(CSV_STRING, {headers: true})
- .on("record", function(data){
+ .on("data", function(data){
      console.log(data);
@@ -109,3 +162,3 @@ })
  .fromStream(stream)
- .on("record", function(data){
+ .on("data", function(data){
      console.log(data);
@@ -126,3 +179,3 @@ })
  .fromStream(stream, {headers : true})
- .on("record", function(data){
+ .on("data", function(data){
      console.log(data);
@@ -144,3 +197,3 @@ })
  .fromStream(stream, {headers : ["firstName", "lastName", "address"]})
- .on("record", function(data){
+ .on("data", function(data){
      console.log(data);
@@ -164,3 +217,3 @@ })
  .fromStream(stream, {ignoreEmpty: true})
- .on("record", function(data){
+ .on("data", function(data){
      console.log(data);
@@ -190,3 +243,3 @@ })
  })
- .on("record", function(data){
+ .on("data", function(data){
      console.log(data);
@@ -200,5 +253,29 @@ })
 
+If your validation is `async` then your validation function
+
+```javascript
+var stream = fs.createReadStream("my.csv");
+
+csv
+ .fromStream(stream)
+ .validate(function(data, next){
+     MyModel.findById(data.id, function(err, model){
+        if(err){
+            next(err);
+        }else{
+            next(null, !model); //valid if the model does not exist
+        }
+     });
+ })
+ .on("data", function(data){
+     console.log(data);
+ })
+ .on("end", function(){
+     console.log("done");
+ });
+```
+
 ### Transforming
 
-You can transform data by providing in a transform function. What is returned from the transform function will
+You can transform data by providing a transform function. What is returned from the transform function will
 be provided to validate and emitted as a row.
@@ -214,3 +291,3 @@
  })
- .on("record", function(data){
+ .on("data", function(data){
      console.log(data);
@@ -224,2 +301,21 @@ })
 
+If your transform function expects two arguments then a callback function will be provided and should be called once the validation is complete. This is useful when doing async validation
+
+```javascript
+var stream = fs.createReadStream("my.csv");
+
+csv
+ .fromStream(stream)
+ .transform(function(data, next){
+     MyModel.findById(data.id, next);
+ })
+ .on("data", function(data){
+     console.log(data);
+ })
+ .on("end", function(){
+     console.log("done");
+ });
+
+```
+
 ### Formatting
@@ -231,8 +327,75 @@
 
-* `transform(row)`: A function that accepts a row and returns a transformed one to be written.
+* `transform(row[, cb])`: A function that accepts a row and returns a transformed one to be written, or your function can accept an optional callback to do async transformations.
 * `rowDelimiter='\n'`: Specify an alternate row delimiter (i.e `\r\n`)
 * `includeEndRowDelimiter=false`: Set to `true` to include a row delimiter at the end of the csv.
+* `quoteHeaders=false`
+   * If `true` then all headers will be quoted.
+   * If it is an object then each key that has a true value will be quoted (see example below)
+   * If it is an array then each item in the array that is true will have the header at the corresponding index quoted (see example below)
+* `quoteColumns=false`
+   * If `true` then columns and headers will be quoted (unless `quoteHeaders` is specified).
+   * If it is an object then each key that has a true value will be quoted ((unless `quoteHeaders` is specified)
+   * If it is an array then each item in the array that is true will have the column at the corresponding index quoted (unless `quoteHeaders` is specified)
 
-**`createWriteStream(options)`**
+### Data Types
 
+When creating a CSV `fast-csv` supports a few data formats.
+
+**`Objects`**
+
+You can pass in object to any formatter function if your csv requires headers the keys of the first object will be used as the header names.
+
+```
+[
+    {
+        a: "a1",
+        b: "b1",
+        c: "c1"
+    }
+]
+
+//Generated CSV
+//a,b,c
+//a1,b1,c1
+```
+
+**`Arrays`**
+
+You can also pass in your rows as arrays. If your csv requires headers the first row passed in will be the headers used.
+
+```
+[
+    ["a", "b", "c"],
+    ["a1", "b1", "c1"]
+]
+//Generated CSV
+//a,b,c
+//a1,b1,c1
+```
+
+**`Arrays of Array`**
+
+This is the least commonly used format but can be useful if you have requirements to generate a CSV with headers with the same column name (Crazy we know but we have seen it).
+
+```
+[
+    [
+        ["a", "a1"],
+        ["a", "a2"],
+        ["b", "b1"],
+        ["b", "b2"],
+        ["c", "c1"],
+        ["c", "c2"]
+    ]
+]
+
+//Generated CSV
+//a,a,b,b,c,c
+//a1,a2,b1,b2,c1,c2
+```
+
+### Formatting Functions
+
+**`createWriteStream(options)` or `.format(options)`**
+
 This is the lowest level of the write methods, it creates a stream that can be used to create a csv of unknown size and pipe to an output csv.
@@ -248,3 +411,3 @@
 
-csvSream.pipe(writableStream);
+csvStream.pipe(writableStream);
 csvStream.write({a: "a0", b: "b0"});
@@ -255,3 +418,20 @@ csvStream.write({a: "a1", b: "b1"});
 csvStream.write({a: "a3", b: "b4"});
-csvStream.write(null);
+csvStream.end();
+
+//or
+
+var csvStream = csv.format({headers: true}),
+    writableStream = fs.createWriteStream("my.csv");
+
+writableStream.on("finish", function(){
+  console.log("DONE!");
+});
+
+csvStream.pipe(writableStream);
+csvStream.write({a: "a0", b: "b0"});
+csvStream.write({a: "a1", b: "b1"});
+csvStream.write({a: "a2", b: "b2"});
+csvStream.write({a: "a3", b: "b4"});
+csvStream.write({a: "a3", b: "b4"});
+csvStream.end();
 ```
@@ -276,3 +456,3 @@
 
-csvSream.pipe(writableStream);
+csvStream.pipe(writableStream);
 csvStream.write({a: "a0", b: "b0"});
@@ -283,5 +463,77 @@ csvStream.write({a: "a1", b: "b1"});
 csvStream.write({a: "a3", b: "b4"});
-csvStream.write(null);
+csvStream.end();
+
+//or
+var csvStream = csv
+    .format({headers: true})
+    .transform(function(row){
+        return {
+           A: row.a,
+           B: row.b
+        };
+    }),
+    writableStream = fs.createWriteStream("my.csv");
+
+writableStream.on("finish", function(){
+  console.log("DONE!");
+});
+
+csvStream.pipe(writableStream);
+csvStream.write({a: "a0", b: "b0"});
+csvStream.write({a: "a1", b: "b1"});
+csvStream.write({a: "a2", b: "b2"});
+csvStream.write({a: "a3", b: "b4"});
+csvStream.write({a: "a3", b: "b4"});
+csvStream.end();
 ```
 
+Transform can also be async by accepting a callback.
+
+
+```javascript
+var csvStream = csv
+    .createWriteStream({headers: true})
+    .transform(function(row, next){
+        setImmediate(function(){
+            next(null, {A: row.a, B: row.b});
+        });;
+    }),
+    writableStream = fs.createWriteStream("my.csv");
+
+writableStream.on("finish", function(){
+  console.log("DONE!");
+});
+
+csvStream.pipe(writableStream);
+csvStream.write({a: "a0", b: "b0"});
+csvStream.write({a: "a1", b: "b1"});
+csvStream.write({a: "a2", b: "b2"});
+csvStream.write({a: "a3", b: "b4"});
+csvStream.write({a: "a3", b: "b4"});
+csvStream.end();
+
+//or
+
+var csvStream = csv
+    .format({headers: true})
+    .transform(function(row, next){
+        setImmediate(function(){
+            next(null, {A: row.a, B: row.b});
+        });;
+    }),
+    writableStream = fs.createWriteStream("my.csv");
+
+writableStream.on("finish", function(){
+  console.log("DONE!");
+});
+
+csvStream.pipe(writableStream);
+csvStream.write({a: "a0", b: "b0"});
+csvStream.write({a: "a1", b: "b1"});
+csvStream.write({a: "a2", b: "b2"});
+csvStream.write({a: "a3", b: "b4"});
+csvStream.write({a: "a3", b: "b4"});
+csvStream.end();
+```
+
 **Writing Data**
@@ -421,26 +673,40 @@
 
-**`writeToString(arr[, options])`**
+**`writeToString(arr[, options], cb)`**
 
 ```javascript
-csv.writeToString([
-   ["a", "b"],
-   ["a1", "b1"],
-   ["a2", "b2"]
-], {headers: true}); //"a,b\na1,b1\na2,b2\n"
+csv.writeToString(
+    [
+        ["a", "b"],
+        ["a1", "b1"],
+        ["a2", "b2"]
+    ],
+    {headers: true},
+    function(err, data){
+        console.log(data); //"a,b\na1,b1\na2,b2\n"
+    }
+);
 ```
 
 ```javascript
-csv.writeToString([
-   {a: "a1", b: "b1"},
-   {a: "a2", b: "b2"}
-], {headers: true}); //"a,b\na1,b1\na2,b2\n"
+csv.writeToString(
+    [
+        {a: "a1", b: "b1"},
+        {a: "a2", b: "b2"}
+    ],
+    {headers: true},
+    function(err, data){
+        console.log(data); //"a,b\na1,b1\na2,b2\n"
+    }
+);
 ```
 
 ```javascript
-csv.writeToString([
-   {a: "a1", b: "b1"},
-   {a: "a2", b: "b2"}
-], {
+csv.writeToString(
+    [
+        {a: "a1", b: "b1"},
+        {a: "a2", b: "b2"}
+    ],
+    {
         headers: true,
-        transform: function(row){
+        transform: function (row) {
             return {
@@ -451,3 +717,7 @@ A: row.a,
         }
-   }); //"a,b\na1,b1\na2,b2\n"
+    },
+    function (err, data) {
+        console.log(data); //"a,b\na1,b1\na2,b2\n"
+    }
+);
 ```
@@ -486,3 +756,3 @@
 
-If you want to tranform on the formatting side
+If you want to transform on the formatting side
 
@@ -493,3 +763,3 @@
         .createWriteStream({headers: true})
-        .transform(function(){
+        .transform(function(obj){
             return {
@@ -508,32 +778,71 @@ name: obj.Name,
 
+## Quoting Columns
 
-## Benchmarks
+Sometimes you may need to quote columns is certain ways in order meet certain requirements. `fast-csv` can quote columns and headers almost anyway you may need.
 
-`Parsing 20000 records AVG over 3 runs`
+**Note** in the following example we use `writeToString` but the options option are valid for any of the formatting methods.
 
+### `quoteColumns`
+
 ```
-fast-csv: 198.67ms
-csv:      525.33ms
-```
+//quote all columns including headers
+var objectData = [{a: "a1", b: "b1"}, {a: "a2", b: "b2"}],
+    arrayData = [["a", "b"], ["a1", "b1"], ["a2", "b2"]];
+csv.writeToString(objectData, {headers: true, quoteColumns: true}, function(err, data){
+    console.log(data); //"a","b"
+                       //"a1","b1"
+                       //"a2","b2"
+});
 
-`Parsing 50000 records AVG over 3 runs`
+//only quote the "a" column
+csv.writeToString(objectData, {headers: true, quoteColumns: {a: true}}, function(err, data){
+    console.log(data); //"a",b
+                       //"a1",b1
+                       //"a2",b2
+});
 
+//only quote the second column
+csv.writeToString(arrayData, {headers: true, quoteColumns: [false, true]}, function(err, data){
+    console.log(data); //a,"b"
+                       //a1,"b1"
+                       //a2,"b2"
+});
+
 ```
-fast-csv: 441.33ms
-csv:      1291ms
-```
 
-`Parsing 100000 records AVG over 3 runs`
+### `quoteHeaders`
 
 ```
-fast-csv: 866ms
-csv:      2773.33ms
-```
+//quote all columns including headers
+var objectData = [{a: "a1", b: "b1"}, {a: "a2", b: "b2"}],
+    arrayData = [["a", "b"], ["a1", "b1"], ["a2", "b2"]];
+csv.writeToString(objectData, {headers: true, quoteHeaders: true}, function(err, data){
+    console.log(data); //"a","b"
+                       //a1,b1
+                       //a2,b2
+});
 
-`Parsing 1000000 records AVG over 3 runs`
+//only quote the "a" column
+csv.writeToString(objectData, {headers: true, quoteHeaders: {a: true}}, function(err, data){
+    console.log(data); //"a",b
+                       //a1,b1
+                       //a2,b2
+});
 
+//only quote the second column
+csv.writeToString(arrayData, {headers: true, quoteHeaders: [false, true]}, function(err, data){
+    console.log(data); //a,"b"
+                       //a1,b1
+                       //a2,b2
+});
+
+//quoting columns and not headers
+
+//only quote the second column
+csv.writeToString(arrayData, {headers: true, quoteHeaders: false, quoteColumns: true}, function(err, data){
+    console.log(data); //a,b
+                       //"a1","b1"
+                       //"a2","b2"
+});
 ```
-fast-csv: 8562.67ms
-csv:      30030.67ms
-```
 
@@ -540,0 +849,0 @@ ## License

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

125490

increased by17.27%

Number of package files

32

increased by10.34%

Lines of code

924

increased by36.08%

Number of lines in readme file

835

increased by58.75%

Worsened metrics

Number of low supply chain risk alerts

6

increased by200%

Number of medium supply chain risk alerts

1

increased byInfinity%

No dependency changes