jschardet - npm Package Compare versions

Comparing version 2.1.1 to 2.2.0

index.d.ts

@@ -5,8 +5,4 @@ export interface IDetectedMap {
 }
-export function detect(buffer: Buffer, options?: { minimumThreshold: number }): IDetectedMap;
+export function detect(buffer: Buffer | string, options?: { minimumThreshold: number }): IDetectedMap;
 
-export const Constants: {
-    MINIMUM_THRESHOLD: number,
-}
-
-export function enableDebug(): void;
+export function enableDebug(): void;

package.json

 {
     "name": "jschardet",
-    "version": "2.1.1",
+    "version": "2.2.0",
     "description": "Character encoding auto-detection in JavaScript (port of python's chardet)",
@@ -5,0 +5,0 @@ "author": "António Afonso",

src/utf8prober.js

@@ -50,2 +50,4 @@ /*
         this._mNumOfMBChar = 0;
+        this._mMBCharLen = 0;
+        this._mFullLen = 0;
     }
@@ -58,2 +60,3 @@
     this.feed = function(aBuf) {
+        this._mFullLen = aBuf.length;
         for( var i = 0, c; i < aBuf.length; i++ ) {
@@ -71,2 +74,3 @@ c = aBuf[i];
                     this._mNumOfMBChar++;
+                    this._mMBCharLen += this._mCodingSM.getCurrentCharLen();
                 }
@@ -87,5 +91,5 @@ }
         var unlike = 0.99;
-        if( this._mNumOfMBChar < 6 ) {
+        if( this._mNumOfMBChar < 6 && (this._mMBCharLen / this._mFullLen) <= 0.6 ) {
             for( var i = 0; i < this._mNumOfMBChar; i++ ) {
-                unlike *= ONE_CHAR_PROB;
+                unlike *= Math.pow(ONE_CHAR_PROB, this._mNumOfMBChar);
             }
@@ -92,0 +96,0 @@ return 1 - unlike;

tests/jschardet.js

@@ -25,2 +25,23 @@ /*
 
+test("UTF-8 w/o BOM (short)", function() {
+    // àí
+    var str = "\xc3\xa0\xc3\xad";
+    equals( jschardet.detect(str).encoding, "UTF-8" );
+});
+
+test("UTF-8 w/o BOM (short, stream)", function() {
+    var u = new jschardet.UniversalDetector();
+    u.reset();
+    u.feed("\xc3\xa0");
+    u.feed("\xc3\xadabc");
+    u.close();
+    equals( u.result.encoding, "UTF-8" );
+});
+
+test("UTF-8 w/o BOM (short, with some ascii)", function() {
+    // àíà
+    var str = "\xc3\xa0\xc3\xad\xc3\xa0abc";
+    equals( jschardet.detect(str).encoding, "UTF-8" );
+});
+
 test("UTF-8 w/o BOM", function() {
@@ -27,0 +48,0 @@ // àíàçã

No alert changes

Improved metrics

Total package byte prevSize

1361634

increased by0.09%

Lines of code

17338

increased by0.14%

No dependency changes