clarinet - npm Package Compare versions

Comparing version 0.9.0 to 0.9.1

clarinet.js

@@ -199,5 +199,6 @@ ;(function (clarinet) {
   CStream.prototype.write = function (data) {
-    var data = new Buffer(data);
+    data = new Buffer(data);
     for (var i = 0; i < data.length; i++) {
       var n = data[i];
+
       // check for carry over of a multi byte char split between data chunks
@@ -211,8 +212,14 @@ // & fill temp buffer it with start of this data chunk up to the boundary limit set in the last iteration
         this.bytes_in_sequence = this.bytes_remaining = 0;
+
+        // move iterator forward by number of byte read during sequencing
         i = i + j - 1;
 
+        // pass data to parser and move forward to parse rest of data
         this._parser.write(this.string);
         this.emit("data", this.string);
-        return true;
-      } else if (this.bytes_remaining === 0 && n >= 128) { // else if no remainder bytes carried over, parse multi byte (>=128) chars one at a time
+        continue;
+      }
+
+      // if no remainder bytes carried over, parse multi byte (>=128) chars one at a time
+      if (this.bytes_remaining === 0 && n >= 128) {
         if ((n >= 194) && (n <= 223)) this.bytes_in_sequence = 2;
@@ -222,2 +229,3 @@ if ((n >= 224) && (n <= 239)) this.bytes_in_sequence = 3;
         if ((this.bytes_in_sequence + i) > data.length) { // if bytes needed to complete char fall outside data length, we have a boundary split
+
           for (var k = 0; k <= (data.length - 1 - i); k++) {
@@ -227,3 +235,5 @@ this.temp_buffs[this.bytes_in_sequence][k] = data[i + k]; // fill temp data of correct size with bytes available in this chunk
           this.bytes_remaining = (i + this.bytes_in_sequence) - data.length;
-          i = data.length - 1;
+
+          // immediately return as we need another chunk to sequence the character
+          return true;
         } else {
@@ -235,9 +245,17 @@ this.string = data.slice(i, (i + this.bytes_in_sequence)).toString();
           this.emit("data", this.string);
-          return true;
+          continue;
         }
-      } else {
-        this._parser.write(data.toString());
-        this.emit("data", data);
-        return true;
       }
+
+      // is there a range of characters that are immediately parsable?
+      for (var p = i; p < data.length; p++) {
+        if (data[p] >= 128) break;
+      }
+      this.string = data.slice(i, p).toString();
+      this._parser.write(this.string);
+      this.emit("data", this.string);
+      i = p - 1;
+
+      // handle any remaining characters using multibyte logic
+      continue;
     }
@@ -244,0 +262,0 @@ };

package.json

@@ -11,3 +11,3 @@ {
   ],
-  "version": "0.9.0",
+  "version": "0.9.1",
   "main": "./clarinet.js",
@@ -41,3 +41,3 @@ "homepage": "https://github.com/dscape/clarinet",
   "scripts": {
-    "test": "./node_modules/mocha/bin/mocha -r should -t 10000 -s 2000  test/clarinet.js test/npm.js"
+    "test": "./node_modules/mocha/bin/mocha -r should -t 10000 -s 2000  test/clarinet.js test/npm.js test/utf8-chunks.js"
   },
@@ -44,0 +44,0 @@ "engines": {

test/clarinet.js

@@ -493,4 +493,7 @@ if (!clarinet) { // node
     , low_overflow :
-      { text       : '[-9223372036854775808]'
-      , events     : 
+      { text       : '[-9223372036854775808]',
+        chunks: [
+          '[-92233720', '36854775808]'
+        ]
+      , events     :
         [ ['openarray'  , undefined]
@@ -607,3 +610,4 @@ , ["value"      , -9223372036854775808]
     , array_null :
-      { text     : '[null,false,true]'
+      { text     : '[null,false,true]',
+        chunks   : ['[nu', 'll,', 'fa', 'lse,', 'tr', 'ue]']
       , events   :
@@ -726,5 +730,21 @@ [ ["openarray"   , undefined]
       }
+    ,
+    string_chunk_span :
+      {
+        text: '["L\'Oréal", "Lé\'Oral", "éalL\'Or"]',
+        chunks: [
+          '["L\'OrÃ',
+          '©al", "Lé\'Oral", "éalL\'Or"]'
+        ],
+        events: [
+          ['openarray', undefined],
+          ['value', 'L\'Oréal'],
+          ['value', 'Lé\'Oral'],
+          ['value', 'éalL\'Or'],
+          ['closearray', undefined]
+        ]
+      }
     };
 
-function generic(key,sep) {
+function generic(key, prechunked, sep) {
   return function () {
@@ -734,3 +754,3 @@ var doc        = docs[key].text
       , l          = typeof FastList === 'function' ? new FastList() : []
-      , doc_chunks = doc.split(sep)
+      , doc_chunks = !prechunked ? doc.split(sep) : docs[key].chunks
       , parser     = clarinet.parser()
@@ -780,3 +800,3 @@ , i          = 0
           it('[' + key + '] should be able to parse -> ' + sep,
-            generic(key,sep));
+            generic(key, false, sep));
         }
@@ -786,2 +806,12 @@ }
   });
+
+  describe('#pre-chunked', function() {
+    for (var key in docs) {
+      if (docs.hasOwnProperty(key)) {
+        if (!docs[key].chunks) continue;
+
+        it('[' + key + '] should be able to parse pre-chunked', generic(key, true));
+      }
+    }
+  });
 });

test/utf8-chunks.js

@@ -7,8 +7,15 @@ var fs             = require('fs')
 
-var han_value = '我';
+var han_value = '我spl我it我';
 var han_obj = '{"thing":"' + han_value + '"}';
 var han_test_obj = '';
 var han_buffer_first = new Buffer([0xe6, 0x88]);
-var han_buffer_second = new Buffer([0x91]);
-var han_buffer_full = new Buffer([0xe6, 0x88, 0x91]);
+var han_buffer_second = new Buffer([0x91, 0x73, 0x70, 0x6c, 0xe6, 0x88, 0x91, 0x69, 0x74, 0xe6, 0x88]);
+var han_buffer_third = new Buffer([0x91]);
+var han_buffer_full = new Buffer([
+  0xe6, 0x88, 0x91,
+  0x73, 0x70, 0x6c,
+  0xe6, 0x88, 0x91,
+  0x69, 0x74,
+  0xe6, 0x88, 0x91
+]);
 
@@ -19,5 +26,7 @@ describe('clarinet', function(){
       chunks.on("error", function (err) { done(err); });
+
       chunks.on("data", function (data) {
         han_test_obj += data;
       });
+
       chunks.on("end", function () {
@@ -31,2 +40,3 @@ assert.equal(han_obj, han_test_obj);
       chunks.write(han_buffer_second);
+      chunks.write(han_buffer_third);
       //chunks.write(han_buffer_full);
@@ -33,0 +43,0 @@ chunks.write('"}');

Fixed alerts

Mixed license

License

(Experimental) Package contains multiple licenses.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

39448604

Number of low license alerts

0

decreased by-100%

Lines of code

7796

increased by0.61%

No dependency changes