sv - npm Package Compare versions

Comparing version 0.2.4 to 0.2.5

merge.js

@@ -10,3 +10,2 @@ #!/usr/bin/env node
 
-
 function File(path) {
@@ -55,7 +54,8 @@ this.path = path;
     // uniquify all columns, starting with special field for original filename
-    var columns = ['original_file'];
+    var columns = ['pkid', 'original_file'];
     files.forEach(function(file) {
       file.columns.forEach(function(column) {
-        if (columns.indexOf(column) == -1)
+        if (columns.indexOf(column) == -1) {
           columns.push(column);
+        }
       });
@@ -88,3 +88,3 @@ });
 
-    var total_out = 0;
+    var pkid = 1;
     var prefix = inference.commonPrefix(files.map(function(file) { return file.path; }));
@@ -100,2 +100,3 @@ console.error('Removing common prefix from filenames: ' + prefix);
         // first row appearing means that parser.columns is now set.
+        row.pkid = pkid++;
         row.original_file = original_file;
@@ -109,3 +110,2 @@ file_out++;
           console.error(original_file + ' (wrote ' + file_out + ' rows, out of ' + file.number_of_lines + ' lines)');
-          total_out += file_out;
           callback();
@@ -115,3 +115,3 @@ }, 1000);
     }, function(err) {
-      console.error('Done. Wrote a total of ' + total_out + ' rows.');
+      console.error('Done. Wrote a total of ' + pkid + ' rows.');
     });
@@ -118,0 +118,0 @@ });

package.json

 {
   "name": "sv",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "description": "Any separated values.",
@@ -5,0 +5,0 @@ "keywords": [

parser.js

@@ -35,5 +35,8 @@ 'use strict'; /*jslint node: true, es5: true, indent: 2 */
   this.encoding = opts.encoding;
-  this.escapechar = (opts.escapechar || '\\').charCodeAt(0);
-  this.quotechar = (opts.quotechar || '"').charCodeAt(0);
-  this.double_quotechar_regex = new RegExp(String.fromCharCode(this.quotechar) + String.fromCharCode(this.quotechar), 'g');
+  this.escapechar = opts.escapechar || '\\';
+  this.escapebyte = this.escapechar.charCodeAt(0);
+  this.quotechar = opts.quotechar || '"';
+  this.quotecharquotechar_regex = new RegExp(this.quotechar + this.quotechar, 'g');
+  this.escapequotechar_regex = new RegExp('\\\\' + this.quotechar, 'g');
+  this.quotebyte = this.quotechar.charCodeAt(0);
 
@@ -78,15 +81,16 @@ this._bytes_buffer = new Buffer(0);
     // if we are on an escape char, simply skip over it (++) and the (default)
-    if (!eos && buffer[i] == this.escapechar) {
-      // excel is bizarre. An escape before a quotechar doesn't count,
-      // so we only increment if the next character is not a quotechar
-      if (buffer[i+1] != this.quotechar) {
+    if (!eos && buffer[i] == this.escapebyte) {
+      // excel is bizarre. An escape before a quotebyte doesn't count,
+      //   so we only increment if the next character is not a quotebyte
+      // unless we are not inside quotes, in which case we do skip over it.
+      if (!inside_quote || buffer[i+1] != this.quotebyte) {
         i++;
       }
     }
-    else if (!eos && buffer[i] == this.quotechar) {
+    else if (!eos && buffer[i] == this.quotebyte) {
       // if we are inside, and on a "
       if (inside_quote) {
-        // handle excel dialect: double quotechar => single literal quotechar
-        if (buffer[i+1] == this.quotechar) {
-          // double quotechar
+        // handle excel dialect: double quotebyte => single literal quotebyte
+        if (buffer[i+1] == this.quotebyte) {
+          // double quotebyte
           // we just advance over it for now, so that we can put this back on the buffer, if needed.
@@ -96,3 +100,3 @@ i++;
         else {
-          // lone quotechar -> don't assume that they're always followed by a delimiter.
+          // lone quotebyte -> don't assume that they're always followed by a delimiter.
           // they might be followed by a newline
@@ -116,14 +120,18 @@ // and we advance so that buffer[i] skips over the delimiter
       ) {
+
+      // this generally won't hurt, since it will only go to the end of the buffer anyway.
+      if (eos) i++;
+
       // add the unprocessed buffer to our cells
       // inside_quote might be true if the file ends on a quote
-      if (eos) i++;
-
       if (inside_quote || outside_quote) {
         var trimmed_cell = buffer.toString(this.encoding, start + 1, i - 1);
         // is this good enough?
-        cells.push(trimmed_cell.replace(this.double_quotechar_regex, String.fromCharCode(this.quotechar)));
+        cells.push(trimmed_cell.replace(this.quotecharquotechar_regex, this.quotechar));
         outside_quote = false;
       }
       else {
-        cells.push(buffer.toString(this.encoding, start, i));
+        var cell = buffer.toString(this.encoding, start, i);
+        var dequoted_cell = cell.replace(this.escapequotechar_regex, this.quotechar);
+        cells.push(dequoted_cell);
       }
@@ -130,0 +138,0 @@

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

33174

increased by1.2%

Lines of code

790

increased by1.02%

No dependency changes