de-dupe - npm Package Compare versions

Comparing version 0.0.10 to 0.0.11

cli.js

@@ -16,6 +16,7 @@ 'use strict';
             addScope: false,
+            algorithm: 'gzip',
             cleanStrings: false,
             includeReplacements: false,
-            minInstances: 10,
-            minLength: 10
+            minInstances: -1,
+            minLength: -1
         };
@@ -28,3 +29,3 @@ if (typeof options !== 'undefined') {
         if (this.options.addScope) {
-            code = "!function(){" + code + "}";
+            code = "!function(){" + code + "}();";
         }
@@ -46,4 +47,10 @@ var replacements = [];
             var stringMap = this.getStringMap(topLevelScopes[i]);
-            var scopeReplacements = this.getStringReplacements(stringMap, startingPos, usedNames);
-            replacements = replacements.concat(scopeReplacements);
+            if (this.options.algorithm === 'gzip') {
+                var scopeReplacements = this.getGzipStringReplacements(stringMap, startingPos, usedNames);
+                replacements = replacements.concat(scopeReplacements);
+            }
+            else {
+                var scopeReplacements = this.getAllStringReplacements(stringMap, startingPos, usedNames);
+                replacements = replacements.concat(scopeReplacements);
+            }
         }
@@ -59,25 +66,80 @@ var sortedReplacements = this.sortReplacements(replacements);
     };
-    Dedupe.prototype.shouldStringBeReplaced = function (str, count) {
-        var minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
-        var minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
-        var matches = str.match(INFREQUENT_CHARS);
-        if (matches && matches.length > 1) {
-            return true;
+    Dedupe.prototype.getGzipStringReplacements = function (stringMap, startingPos, usedVariableNames) {
+        var variableDeclarationBuffer = [];
+        var replacements = [];
+        var gzipSlidingWindowSize = 32768;
+        var stringKeysToReplace = [];
+        stringMap.forEach(function (values, key) {
+            values = values.sort(function (a, b) {
+                var start1 = a.getStart();
+                var start2 = b.getStart();
+                if (start1 > start2) {
+                    return 1;
+                }
+                if (start1 < start2) {
+                    return -1;
+                }
+                return 0;
+            });
+            for (var i = 1; i < values.length; i++) {
+                var start1 = values[i - 1].getStart();
+                var start2 = values[i].getStart();
+                if ((start2 - start1) > gzipSlidingWindowSize) {
+                    stringKeysToReplace.push(key);
+                    return;
+                }
+            }
+        });
+        for (var _i = 0, stringKeysToReplace_1 = stringKeysToReplace; _i < stringKeysToReplace_1.length; _i++) {
+            var key = stringKeysToReplace_1[_i];
+            var values = stringMap.get(key);
+            if (!values) {
+                break;
+            }
+            var variableName = this.getUniqueVariableName(usedVariableNames);
+            variableDeclarationBuffer.push(variableName + "=" + JSON.stringify(key));
+            for (var j = 0, length = values.length; j < length; j++) {
+                var node = values[j];
+                var start = node.getStart();
+                var end = node.getEnd();
+                replacements.push({
+                    end: end,
+                    start: start,
+                    text: variableName
+                });
+            }
         }
-        if (str.length > minLength) {
-            return true;
+        var variableDeclaration = '';
+        if (variableDeclarationBuffer.length > 0) {
+            variableDeclaration = "var " + variableDeclarationBuffer.join(',') + ";";
+            replacements.push({
+                end: startingPos,
+                start: startingPos,
+                text: variableDeclaration
+            });
         }
-        if (count > minInstances) {
-            return true;
-        }
-        return false;
+        return replacements;
     };
-    Dedupe.prototype.getStringReplacements = function (stringMap, startingPos, usedVariableNames) {
-        var _this = this;
+    Dedupe.prototype.getAllStringReplacements = function (stringMap, startingPos, usedVariableNames) {
         var variableDeclarationBuffer = [];
         var replacements = [];
         var rankingMap = [];
+        var minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
+        var minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
+        var shouldStringBeReplaced = function (str, count) {
+            var matches = str.match(INFREQUENT_CHARS);
+            if (matches && matches.length > 1) {
+                return true;
+            }
+            if (str.length > minLength) {
+                return true;
+            }
+            if (count > minInstances) {
+                return true;
+            }
+            return false;
+        };
         stringMap.forEach(function (values, key) {
             var count = values.length;
-            if (_this.shouldStringBeReplaced(key, count)) {
+            if (shouldStringBeReplaced(key, count)) {
                 rankingMap.push([key, count]);
@@ -224,3 +286,5 @@ }
     Dedupe.prototype.getStartingPositionOfScope = function (scope) {
-        var startingPos = scope.getChildAt(1).pos;
+        var firstChild = scope.getChildAt(0);
+        var secondChild = scope.getChildAt(1);
+        var startingPos = secondChild ? secondChild.getFullStart() : firstChild.getFullStart();
         // check the top level of the block for "use strict"
@@ -227,0 +291,0 @@ typescript.forEachChild(scope, function (expressionNode) {

index.d.ts

 export interface DedupeOptions {
     /**
+     * Specify the deduplication algorithm.
+     *    gzip - Apply an algorithm that works best for files that will be gzipped
+     *    all -
+     */
+    algorithm?: 'gzip' | 'all';
+    /**
      * Adds an IIFE around the entire script, this broadens the possible instances
@@ -39,4 +45,4 @@ * where strings duplication could be consolidated
     dedupe(code: string): Result;
-    private shouldStringBeReplaced(str, count);
-    private getStringReplacements(stringMap, startingPos, usedVariableNames);
+    private getGzipStringReplacements(stringMap, startingPos, usedVariableNames);
+    private getAllStringReplacements(stringMap, startingPos, usedVariableNames);
     private getUsedVariableNames(identifiers, startingNode);
@@ -43,0 +49,0 @@ private getStringMap(startingNode);

index.js

@@ -14,6 +14,7 @@ 'use strict';
             addScope: false,
+            algorithm: 'gzip',
             cleanStrings: false,
             includeReplacements: false,
-            minInstances: 10,
-            minLength: 10
+            minInstances: -1,
+            minLength: -1
         };
@@ -26,3 +27,3 @@ if (typeof options !== 'undefined') {
         if (this.options.addScope) {
-            code = "!function(){" + code + "}";
+            code = "!function(){" + code + "}();";
         }
@@ -44,4 +45,10 @@ var replacements = [];
             var stringMap = this.getStringMap(topLevelScopes[i]);
-            var scopeReplacements = this.getStringReplacements(stringMap, startingPos, usedNames);
-            replacements = replacements.concat(scopeReplacements);
+            if (this.options.algorithm === 'gzip') {
+                var scopeReplacements = this.getGzipStringReplacements(stringMap, startingPos, usedNames);
+                replacements = replacements.concat(scopeReplacements);
+            }
+            else {
+                var scopeReplacements = this.getAllStringReplacements(stringMap, startingPos, usedNames);
+                replacements = replacements.concat(scopeReplacements);
+            }
         }
@@ -57,25 +64,80 @@ var sortedReplacements = this.sortReplacements(replacements);
     };
-    Dedupe.prototype.shouldStringBeReplaced = function (str, count) {
-        var minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
-        var minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
-        var matches = str.match(INFREQUENT_CHARS);
-        if (matches && matches.length > 1) {
-            return true;
+    Dedupe.prototype.getGzipStringReplacements = function (stringMap, startingPos, usedVariableNames) {
+        var variableDeclarationBuffer = [];
+        var replacements = [];
+        var gzipSlidingWindowSize = 32768;
+        var stringKeysToReplace = [];
+        stringMap.forEach(function (values, key) {
+            values = values.sort(function (a, b) {
+                var start1 = a.getStart();
+                var start2 = b.getStart();
+                if (start1 > start2) {
+                    return 1;
+                }
+                if (start1 < start2) {
+                    return -1;
+                }
+                return 0;
+            });
+            for (var i = 1; i < values.length; i++) {
+                var start1 = values[i - 1].getStart();
+                var start2 = values[i].getStart();
+                if ((start2 - start1) > gzipSlidingWindowSize) {
+                    stringKeysToReplace.push(key);
+                    return;
+                }
+            }
+        });
+        for (var _i = 0, stringKeysToReplace_1 = stringKeysToReplace; _i < stringKeysToReplace_1.length; _i++) {
+            var key = stringKeysToReplace_1[_i];
+            var values = stringMap.get(key);
+            if (!values) {
+                break;
+            }
+            var variableName = this.getUniqueVariableName(usedVariableNames);
+            variableDeclarationBuffer.push(variableName + "=" + JSON.stringify(key));
+            for (var j = 0, length = values.length; j < length; j++) {
+                var node = values[j];
+                var start = node.getStart();
+                var end = node.getEnd();
+                replacements.push({
+                    end: end,
+                    start: start,
+                    text: variableName
+                });
+            }
         }
-        if (str.length > minLength) {
-            return true;
+        var variableDeclaration = '';
+        if (variableDeclarationBuffer.length > 0) {
+            variableDeclaration = "var " + variableDeclarationBuffer.join(',') + ";";
+            replacements.push({
+                end: startingPos,
+                start: startingPos,
+                text: variableDeclaration
+            });
         }
-        if (count > minInstances) {
-            return true;
-        }
-        return false;
+        return replacements;
     };
-    Dedupe.prototype.getStringReplacements = function (stringMap, startingPos, usedVariableNames) {
-        var _this = this;
+    Dedupe.prototype.getAllStringReplacements = function (stringMap, startingPos, usedVariableNames) {
         var variableDeclarationBuffer = [];
         var replacements = [];
         var rankingMap = [];
+        var minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
+        var minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
+        var shouldStringBeReplaced = function (str, count) {
+            var matches = str.match(INFREQUENT_CHARS);
+            if (matches && matches.length > 1) {
+                return true;
+            }
+            if (str.length > minLength) {
+                return true;
+            }
+            if (count > minInstances) {
+                return true;
+            }
+            return false;
+        };
         stringMap.forEach(function (values, key) {
             var count = values.length;
-            if (_this.shouldStringBeReplaced(key, count)) {
+            if (shouldStringBeReplaced(key, count)) {
                 rankingMap.push([key, count]);
@@ -222,3 +284,5 @@ }
     Dedupe.prototype.getStartingPositionOfScope = function (scope) {
-        var startingPos = scope.getChildAt(1).pos;
+        var firstChild = scope.getChildAt(0);
+        var secondChild = scope.getChildAt(1);
+        var startingPos = secondChild ? secondChild.getFullStart() : firstChild.getFullStart();
         // check the top level of the block for "use strict"
@@ -225,0 +289,0 @@ typescript.forEachChild(scope, function (expressionNode) {

package.json

 {
   "name": "de-dupe",
   "bin": ".bin/de-dupe",
-  "version": "0.0.10",
+  "version": "0.0.11",
   "description": "Deduplicate strings from javascript assets",
@@ -6,0 +6,0 @@ "main": "index.js",

src/index.ts

@@ -15,2 +15,8 @@ import {
   /**
+   * Specify the deduplication algorithm.
+   *    gzip - Apply an algorithm that works best for files that will be gzipped
+   *    all -
+   */
+  algorithm?: 'gzip' | 'all';
+  /**
    * Adds an IIFE around the entire script, this broadens the possible instances
@@ -71,6 +77,7 @@ * where strings duplication could be consolidated
     addScope: false,
+    algorithm: 'gzip',
     cleanStrings: false,
     includeReplacements: false,
-    minInstances: 10,
-    minLength: 10
+    minInstances: -1,
+    minLength: -1
   };
@@ -86,3 +93,3 @@
     if (this.options.addScope) {
-      code = `!function(){${code}}`;
+      code = `!function(){${code}}();`;
     }
@@ -106,4 +113,10 @@
       const stringMap = this.getStringMap(topLevelScopes[i]);
-      const scopeReplacements = this.getStringReplacements(stringMap, startingPos, usedNames);
-      replacements = replacements.concat(scopeReplacements);
+
+      if (this.options.algorithm === 'gzip') {
+        const scopeReplacements = this.getGzipStringReplacements(stringMap, startingPos, usedNames);
+        replacements = replacements.concat(scopeReplacements);
+      } else {
+        const scopeReplacements = this.getAllStringReplacements(stringMap, startingPos, usedNames);
+        replacements = replacements.concat(scopeReplacements);
+      }
     }
@@ -120,26 +133,93 @@ const sortedReplacements = this.sortReplacements(replacements);
 
-  private shouldStringBeReplaced(str: string, count: number): boolean {
-    const minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
-    const minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
-    const matches = (str.match(INFREQUENT_CHARS) as any);
-    if (matches && matches.length > 1) {
-      return true;
+  private getGzipStringReplacements(stringMap: StringMap, startingPos: number,
+                                    usedVariableNames: Map<string, string>): StringReplacement[] {
+
+    const variableDeclarationBuffer: string[] = [];
+    const replacements: StringReplacement[] = [];
+    const gzipSlidingWindowSize = 32768;
+    const stringKeysToReplace: string[] = [];
+
+    stringMap.forEach((values, key) => {
+      values = values.sort((a, b) => {
+        const start1 = a.getStart();
+        const start2 = b.getStart();
+
+        if (start1 > start2) {
+          return 1;
+        }
+        if (start1 < start2) {
+          return -1;
+        }
+        return 0;
+      });
+
+      for (let i = 1; i < values.length; i++) {
+        const start1 = values[i - 1].getStart();
+        const start2 = values[i].getStart();
+        if ((start2 - start1) > gzipSlidingWindowSize) {
+          stringKeysToReplace.push(key);
+          return;
+        }
+      }
+    });
+
+    for (let key of stringKeysToReplace) {
+      const values = stringMap.get(key);
+      if (!values) {
+        break;
+      }
+      const variableName = this.getUniqueVariableName(usedVariableNames);
+      variableDeclarationBuffer.push(`${variableName}=${JSON.stringify(key)}`);
+
+      for (let j = 0, length = values.length; j < length; j++) {
+        let node = values[j];
+        let start = node.getStart();
+        let end = node.getEnd();
+
+        replacements.push({
+          end,
+          start,
+          text: variableName
+        });
+      }
     }
-    if (str.length > minLength) {
-      return true;
+
+    let variableDeclaration = '';
+    if (variableDeclarationBuffer.length > 0) {
+      variableDeclaration = `var ${variableDeclarationBuffer.join(',')};`;
+      replacements.push({
+        end: startingPos,
+        start: startingPos,
+        text: variableDeclaration
+      });
     }
-    if (count > minInstances) {
-      return true;
-    }
-    return false;
+    return replacements;
   }
 
-  private getStringReplacements(stringMap: StringMap, startingPos: number,
-                                usedVariableNames: Map<string, string>): StringReplacement[] {
+  private getAllStringReplacements(stringMap: StringMap, startingPos: number,
+                                   usedVariableNames: Map<string, string>): StringReplacement[] {
     const variableDeclarationBuffer: string[] = [];
     const replacements: StringReplacement[] = [];
-    const rankingMap: Array<Array<string | number>> = [];
+    let rankingMap: Array<Array<string | number>> = [];
+
+    const minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
+    const minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
+
+    const shouldStringBeReplaced = (str: string, count: number) => {
+      const matches = (str.match(INFREQUENT_CHARS) as any);
+      if (matches && matches.length > 1) {
+        return true;
+      }
+      if (str.length > minLength) {
+        return true;
+      }
+      if (count > minInstances) {
+        return true;
+      }
+      return false;
+    };
+
     stringMap.forEach((values, key) => {
       const count = values.length;
-      if (this.shouldStringBeReplaced(key, count)) {
+      if (shouldStringBeReplaced(key, count)) {
         rankingMap.push([key, count]);
@@ -300,3 +380,5 @@ }
   private getStartingPositionOfScope(scope: Block) {
-    let startingPos = scope.getChildAt(1).pos;
+    const firstChild = scope.getChildAt(0);
+    const secondChild = scope.getChildAt(1);
+    let startingPos = secondChild ? secondChild.getFullStart() : firstChild.getFullStart();
 
@@ -303,0 +385,0 @@ // check the top level of the block for "use strict"

tests.js

@@ -16,6 +16,7 @@ 'use strict';
             addScope: false,
+            algorithm: 'gzip',
             cleanStrings: false,
             includeReplacements: false,
-            minInstances: 10,
-            minLength: 10
+            minInstances: -1,
+            minLength: -1
         };
@@ -28,3 +29,3 @@ if (typeof options !== 'undefined') {
         if (this.options.addScope) {
-            code = "!function(){" + code + "}";
+            code = "!function(){" + code + "}();";
         }
@@ -46,4 +47,10 @@ var replacements = [];
             var stringMap = this.getStringMap(topLevelScopes[i]);
-            var scopeReplacements = this.getStringReplacements(stringMap, startingPos, usedNames);
-            replacements = replacements.concat(scopeReplacements);
+            if (this.options.algorithm === 'gzip') {
+                var scopeReplacements = this.getGzipStringReplacements(stringMap, startingPos, usedNames);
+                replacements = replacements.concat(scopeReplacements);
+            }
+            else {
+                var scopeReplacements = this.getAllStringReplacements(stringMap, startingPos, usedNames);
+                replacements = replacements.concat(scopeReplacements);
+            }
         }
@@ -59,25 +66,80 @@ var sortedReplacements = this.sortReplacements(replacements);
     };
-    Dedupe.prototype.shouldStringBeReplaced = function (str, count) {
-        var minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
-        var minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
-        var matches = str.match(INFREQUENT_CHARS);
-        if (matches && matches.length > 1) {
-            return true;
+    Dedupe.prototype.getGzipStringReplacements = function (stringMap, startingPos, usedVariableNames) {
+        var variableDeclarationBuffer = [];
+        var replacements = [];
+        var gzipSlidingWindowSize = 32768;
+        var stringKeysToReplace = [];
+        stringMap.forEach(function (values, key) {
+            values = values.sort(function (a, b) {
+                var start1 = a.getStart();
+                var start2 = b.getStart();
+                if (start1 > start2) {
+                    return 1;
+                }
+                if (start1 < start2) {
+                    return -1;
+                }
+                return 0;
+            });
+            for (var i = 1; i < values.length; i++) {
+                var start1 = values[i - 1].getStart();
+                var start2 = values[i].getStart();
+                if ((start2 - start1) > gzipSlidingWindowSize) {
+                    stringKeysToReplace.push(key);
+                    return;
+                }
+            }
+        });
+        for (var _i = 0, stringKeysToReplace_1 = stringKeysToReplace; _i < stringKeysToReplace_1.length; _i++) {
+            var key = stringKeysToReplace_1[_i];
+            var values = stringMap.get(key);
+            if (!values) {
+                break;
+            }
+            var variableName = this.getUniqueVariableName(usedVariableNames);
+            variableDeclarationBuffer.push(variableName + "=" + JSON.stringify(key));
+            for (var j = 0, length = values.length; j < length; j++) {
+                var node = values[j];
+                var start = node.getStart();
+                var end = node.getEnd();
+                replacements.push({
+                    end: end,
+                    start: start,
+                    text: variableName
+                });
+            }
         }
-        if (str.length > minLength) {
-            return true;
+        var variableDeclaration = '';
+        if (variableDeclarationBuffer.length > 0) {
+            variableDeclaration = "var " + variableDeclarationBuffer.join(',') + ";";
+            replacements.push({
+                end: startingPos,
+                start: startingPos,
+                text: variableDeclaration
+            });
         }
-        if (count > minInstances) {
-            return true;
-        }
-        return false;
+        return replacements;
     };
-    Dedupe.prototype.getStringReplacements = function (stringMap, startingPos, usedVariableNames) {
-        var _this = this;
+    Dedupe.prototype.getAllStringReplacements = function (stringMap, startingPos, usedVariableNames) {
         var variableDeclarationBuffer = [];
         var replacements = [];
         var rankingMap = [];
+        var minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
+        var minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
+        var shouldStringBeReplaced = function (str, count) {
+            var matches = str.match(INFREQUENT_CHARS);
+            if (matches && matches.length > 1) {
+                return true;
+            }
+            if (str.length > minLength) {
+                return true;
+            }
+            if (count > minInstances) {
+                return true;
+            }
+            return false;
+        };
         stringMap.forEach(function (values, key) {
             var count = values.length;
-            if (_this.shouldStringBeReplaced(key, count)) {
+            if (shouldStringBeReplaced(key, count)) {
                 rankingMap.push([key, count]);
@@ -224,3 +286,5 @@ }
     Dedupe.prototype.getStartingPositionOfScope = function (scope) {
-        var startingPos = scope.getChildAt(1).pos;
+        var firstChild = scope.getChildAt(0);
+        var secondChild = scope.getChildAt(1);
+        var startingPos = secondChild ? secondChild.getFullStart() : firstChild.getFullStart();
         // check the top level of the block for "use strict"
@@ -295,2 +359,3 @@ typescript.forEachChild(scope, function (expressionNode) {
 var dedupe = new Dedupe({
+    algorithm: 'gzip',
     minInstances: 2,
@@ -300,2 +365,3 @@ minLength: 2
 var stringCleaner = new Dedupe({
+    algorithm: 'all',
     cleanStrings: true,
@@ -307,8 +373,13 @@ minInstances: 2,
     addScope: true,
+    algorithm: 'all',
     minInstances: 2,
     minLength: 2
 });
+var gzipSpace = Array(32800).join(' ');
+function stripWhitespace(str) {
+    return str.replace(/[\s]+/g, ' ');
+}
 describe('de-dupe', function () {
     it('can handle large strings', function () {
-        var code = "!function() { console.log('zzzzzzzzzz', 'zzzzzzzzzz'); }()";
+        var code = "!function() { console.log('zzzzzzzzzz', " + gzipSpace + " 'zzzzzzzzzz'); }()";
         var result = dedupe.dedupe(code);
@@ -319,11 +390,11 @@ var markers = result.code.match(/zzzzzzzzzz/g);
     it('can handle many small strings', function () {
-        var code = "!function() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }()";
-        var expected = "!function() {var _=\"z\"; console.log(_, _, _, _, _, _); }()";
+        var code = "!function() { console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z'); }()";
+        var expected = "!function() {var _=\"z\"; console.log(_, " + gzipSpace + " _, _, _, _, _); }()";
         var result = dedupe.dedupe(code);
         var markers = result.code.match(/z/g);
-        chai.expect(result.code).equal(expected);
+        chai.expect(stripWhitespace(result.code)).equal(stripWhitespace(expected));
         chai.expect(markers.length).equal(1);
     });
     it('can handle multiple scopes', function () {
-        var code = "\n      !function() {\n        console.log('z', 'z', 'z', 'z', 'z', 'z');\n      }()\n\n      !function() {\n        console.log('z', 'z', 'z', 'z', 'z', 'z');\n      }()\n    ";
+        var code = "\n      !function() {\n        console.log('z'," + gzipSpace + " 'z', 'z', 'z', 'z', 'z');\n      }()\n\n      !function() {\n        console.log('z'," + gzipSpace + " 'z', 'z', 'z', 'z', 'z');\n      }()\n    ";
         var result = dedupe.dedupe(code);
@@ -333,3 +404,3 @@ chai.expect(result.code.match(/z/g).length).equal(2);
     it('can handle multiple scopes from one global scope', function () {
-        var code = "\n      !function() {\n        !function() {\n          console.log('z', 'z', 'z', 'z', 'z', 'z');\n        }()\n\n        !function() {\n          console.log('z', 'z', 'z', 'z', 'z', 'z');\n        }()\n      }()\n    ";
+        var code = "\n      !function() {\n        !function() {\n          console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z');\n        }()\n\n        !function() {\n          console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z');\n        }()\n      }()\n    ";
         var result = dedupe.dedupe(code);
@@ -340,4 +411,4 @@ var markers = result.code.match(/z/g);
     it('can handle named functions', function () {
-        var code = "function x() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }";
-        var expected = "function x() {var _=\"z\"; console.log(_, _, _, _, _, _); }";
+        var code = "function x() { console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z'); }";
+        var expected = "function x() {var _=\"z\"; console.log(_, " + gzipSpace + " _, _, _, _, _); }";
         var result = dedupe.dedupe(code);
@@ -348,4 +419,4 @@ chai.expect(result.code).equal(expected);
     it('can handle arrow functions', function () {
-        var code = "() => { console.log('z', 'z', 'z', 'z', 'z', 'z'); }";
-        var expected = "() => {var _=\"z\"; console.log(_, _, _, _, _, _); }";
+        var code = "() => { console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z'); }";
+        var expected = "() => {var _=\"z\"; console.log(_, " + gzipSpace + " _, _, _, _, _); }";
         var result = dedupe.dedupe(code);
@@ -356,4 +427,4 @@ chai.expect(result.code).equal(expected);
     it('can handle strict mode', function () {
-        var code = "function x() { \"use strict\"; console.log('z', 'z', 'z', 'z', 'z', 'z'); }";
-        var expected = "function x() { \"use strict\";var _=\"z\"; console.log(_, _, _, _, _, _); }";
+        var code = "function x() { \"use strict\"; console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z'); }";
+        var expected = "function x() { \"use strict\";var _=\"z\"; console.log(_, " + gzipSpace + " _, _, _, _, _); }";
         var result = dedupe.dedupe(code);
@@ -364,3 +435,3 @@ chai.expect(result.code).equal(expected);
     it('will add scope', function () {
-        var code = "console.log('z', 'z', 'z', 'z', 'z', 'z');";
+        var code = "console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z');";
         var result = scopeAdder.dedupe(code);
@@ -371,3 +442,3 @@ chai.expect(result.code).contain('!function');
     it('will clean strings', function () {
-        var code = "\n      !function() {\n        console.log('z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ');\n      }()\n    ";
+        var code = "\n      !function() {\n        console.log('z  ', " + gzipSpace + " 'z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ');\n      }()\n    ";
         var result = stringCleaner.dedupe(code);
@@ -377,3 +448,3 @@ chai.expect(result.code).contain("\"z \"");
     it('will not effect non-function blocks', function () {
-        var code = "\n      if (true) {\n        console.log('z', 'z', 'z', 'z', 'z', 'z');\n      }\n    ";
+        var code = "\n      if (true) {\n        console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z');\n      }\n    ";
         var result = dedupe.dedupe(code);
@@ -383,3 +454,3 @@ chai.expect(result.code).equal(code);
     it('will not treat "use strict" as a string', function () {
-        var code = "\n      () => {\n        'use strict';\n        console.log('use strict', 'use strict', 'use strict', 'use strict', 'use strict', 'use strict');\n      }\n    ";
+        var code = "\n      () => {\n        'use strict';\n        console.log('use strict', " + gzipSpace + " 'use strict', 'use strict', 'use strict', 'use strict', 'use strict');\n      }\n    ";
         var result = dedupe.dedupe(code);
@@ -390,3 +461,3 @@ chai.expect(code).equal(result.code);
         var propertyAssignment = "var stuff = { 'z' : 123 };";
-        var code = "\n      function x() {\n        " + propertyAssignment + "\n        console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z');\n      }\n    ";
+        var code = "\n      function x() {\n        " + propertyAssignment + "\n        console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z', 'z', 'z');\n      }\n    ";
         var result = dedupe.dedupe(code);
@@ -396,4 +467,4 @@ chai.expect(result.code).contains(propertyAssignment);
     it('will not treat the left side of a property assignment as a string', function () {
-        var code = "\n      function x() {\n        var stuff = { 'z' : 'z' };\n        console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z');\n      }\n    ";
-        var expected = "\n      function x() {var _=\"z\";\n        var stuff = { 'z' : _ };\n        console.log(_, _, _, _, _, _, _, _);\n      }\n    ";
+        var code = "\n      function x() {\n        var stuff = { 'z' : 'z' };\n        console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z', 'z', 'z');\n      }\n    ";
+        var expected = "\n      function x() {var _=\"z\";\n        var stuff = { 'z' : _ };\n        console.log(_, " + gzipSpace + " _, _, _, _, _, _, _);\n      }\n    ";
         var result = dedupe.dedupe(code);
@@ -404,4 +475,4 @@ chai.expect(result.code).equals(expected);
         // notice the expected is expecting dedupe to identify that "a" is referring to an outside variable
-        var code = "function x() { _.thing = 'a'; console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z'); }";
-        var expected = "function x() {var e=\"z\"; _.thing = 'a'; console.log(e, e, e, e, e, e, e, e); }";
+        var code = "function x() { _.thing = 'a'; console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z', 'z', 'z'); }";
+        var expected = "function x() {var e=\"z\"; _.thing = 'a'; console.log(e, " + gzipSpace + " e, e, e, e, e, e, e); }";
         var result = dedupe.dedupe(code);
@@ -411,4 +482,4 @@ chai.expect(result.code).equal(expected);
     it('deal with strings next to reserved words', function () {
-        var code = "function x() { if ('z'in x) { console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z'); } }";
-        var expected = "function x() {var _=\"z\"; if (_ in x) { console.log(_, _, _, _, _, _, _, _); } }";
+        var code = "function x() { if ('z'in x) { console.log('z', " + gzipSpace + " 'z', 'z', 'z', 'z', 'z', 'z', 'z'); } }";
+        var expected = "function x() {var _=\"z\"; if (_ in x) { console.log(_, " + gzipSpace + " _, _, _, _, _, _, _); } }";
         var result = dedupe.dedupe(code);
@@ -415,0 +486,0 @@ chai.expect(result.code).equal(expected);

tests/index.ts

@@ -7,2 +7,3 @@ import 'mocha';
 const dedupe = new Dedupe({
+  algorithm: 'gzip',
   minInstances: 2,
@@ -12,2 +13,3 @@ minLength: 2
 const stringCleaner = new Dedupe({
+  algorithm: 'all',
   cleanStrings: true,
@@ -19,2 +21,3 @@ minInstances: 2,
   addScope: true,
+  algorithm: 'all',
   minInstances: 2,
@@ -24,6 +27,11 @@ minLength: 2
 
+const gzipSpace = Array(32800).join(' ');
+function stripWhitespace(str: string) {
+  return str.replace(/[\s]+/g, ' ');
+}
+
 describe('de-dupe', () => {
 
   it('can handle large strings', () => {
-    const code = `!function() { console.log('zzzzzzzzzz', 'zzzzzzzzzz'); }()`;
+    const code = `!function() { console.log('zzzzzzzzzz', ${gzipSpace} 'zzzzzzzzzz'); }()`;
 
@@ -37,4 +45,4 @@ const result = dedupe.dedupe(code);
   it('can handle many small strings', () => {
-    const code = `!function() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }()`;
-    const expected = `!function() {var _="z"; console.log(_, _, _, _, _, _); }()`;
+    const code = `!function() { console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z'); }()`;
+    const expected = `!function() {var _="z"; console.log(_, ${gzipSpace} _, _, _, _, _); }()`;
 
@@ -44,3 +52,3 @@ const result = dedupe.dedupe(code);
 
-    expect(result.code).equal(expected);
+    expect(stripWhitespace(result.code)).equal(stripWhitespace(expected));
     expect(markers.length).equal(1);
@@ -52,7 +60,7 @@ });
       !function() {
-        console.log('z', 'z', 'z', 'z', 'z', 'z');
+        console.log('z',${gzipSpace} 'z', 'z', 'z', 'z', 'z');
       }()
 
       !function() {
-        console.log('z', 'z', 'z', 'z', 'z', 'z');
+        console.log('z',${gzipSpace} 'z', 'z', 'z', 'z', 'z');
       }()
@@ -70,7 +78,7 @@ `;
         !function() {
-          console.log('z', 'z', 'z', 'z', 'z', 'z');
+          console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z');
         }()
 
         !function() {
-          console.log('z', 'z', 'z', 'z', 'z', 'z');
+          console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z');
         }()
@@ -87,4 +95,4 @@ }()
   it('can handle named functions', () => {
-    const code = `function x() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }`;
-    const expected = `function x() {var _="z"; console.log(_, _, _, _, _, _); }`;
+    const code = `function x() { console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z'); }`;
+    const expected = `function x() {var _="z"; console.log(_, ${gzipSpace} _, _, _, _, _); }`;
 
@@ -98,4 +106,4 @@ const result = dedupe.dedupe(code);
   it('can handle arrow functions', () => {
-    const code = `() => { console.log('z', 'z', 'z', 'z', 'z', 'z'); }`;
-    const expected = `() => {var _="z"; console.log(_, _, _, _, _, _); }`;
+    const code = `() => { console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z'); }`;
+    const expected = `() => {var _="z"; console.log(_, ${gzipSpace} _, _, _, _, _); }`;
 
@@ -109,4 +117,4 @@ const result = dedupe.dedupe(code);
   it('can handle strict mode', () => {
-    const code = `function x() { "use strict"; console.log('z', 'z', 'z', 'z', 'z', 'z'); }`;
-    const expected = `function x() { "use strict";var _="z"; console.log(_, _, _, _, _, _); }`;
+    const code = `function x() { "use strict"; console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z'); }`;
+    const expected = `function x() { "use strict";var _="z"; console.log(_, ${gzipSpace} _, _, _, _, _); }`;
 
@@ -120,3 +128,3 @@ const result = dedupe.dedupe(code);
   it('will add scope', () => {
-    const code = `console.log('z', 'z', 'z', 'z', 'z', 'z');`;
+    const code = `console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z');`;
 
@@ -132,3 +140,3 @@ const result = scopeAdder.dedupe(code);
       !function() {
-        console.log('z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ');
+        console.log('z  ', ${gzipSpace} 'z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ', 'z  ');
       }()
@@ -145,3 +153,3 @@ `;
       if (true) {
-        console.log('z', 'z', 'z', 'z', 'z', 'z');
+        console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z');
       }
@@ -159,3 +167,3 @@ `;
         'use strict';
-        console.log('use strict', 'use strict', 'use strict', 'use strict', 'use strict', 'use strict');
+        console.log('use strict', ${gzipSpace} 'use strict', 'use strict', 'use strict', 'use strict', 'use strict');
       }
@@ -174,3 +182,3 @@ `;
         ${propertyAssignment}
-        console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z');
+        console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z', 'z', 'z');
       }
@@ -188,3 +196,3 @@ `;
         var stuff = { 'z' : 'z' };
-        console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z');
+        console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z', 'z', 'z');
       }
@@ -195,3 +203,3 @@ `;
         var stuff = { 'z' : _ };
-        console.log(_, _, _, _, _, _, _, _);
+        console.log(_, ${gzipSpace} _, _, _, _, _, _, _);
       }
@@ -207,4 +215,4 @@ `;
     // notice the expected is expecting dedupe to identify that "a" is referring to an outside variable
-    const code = `function x() { _.thing = 'a'; console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z'); }`;
-    const expected = `function x() {var e="z"; _.thing = 'a'; console.log(e, e, e, e, e, e, e, e); }`;
+    const code = `function x() { _.thing = 'a'; console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z', 'z', 'z'); }`;
+    const expected = `function x() {var e="z"; _.thing = 'a'; console.log(e, ${gzipSpace} e, e, e, e, e, e, e); }`;
 
@@ -217,4 +225,4 @@ const result = dedupe.dedupe(code);
   it('deal with strings next to reserved words', () => {
-    const code = `function x() { if ('z'in x) { console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z'); } }`;
-    const expected = `function x() {var _="z"; if (_ in x) { console.log(_, _, _, _, _, _, _, _); } }`;
+    const code = `function x() { if ('z'in x) { console.log('z', ${gzipSpace} 'z', 'z', 'z', 'z', 'z', 'z', 'z'); } }`;
+    const expected = `function x() {var _="z"; if (_ in x) { console.log(_, ${gzipSpace} _, _, _, _, _, _, _); } }`;
 
@@ -221,0 +229,0 @@ const result = dedupe.dedupe(code);

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

95752

increased by37.44%

Number of package files

16

increased by6.67%

Lines of code

2333

increased by36.91%

No dependency changes