Socket
Socket
Sign inDemoInstall

de-dupe

Package Overview
Dependencies
Maintainers
1
Versions
28
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

de-dupe - npm Package Compare versions

Comparing version 0.0.6 to 0.0.7

cli.js

225

index.js
'use strict';
var fs = require('fs');
var path = require('path');
var typescript = require('typescript');
var RESERVED_WORDS = "\n abstract arguments await boolean break byte case catch char class const continue debugger\n default delete do double else enum eval export extends false final finally float for\n function goto if implements import in instanceof int interface let long native new null\n package private protected public return short static super switch synchronized this throw\n throws transient true try typeof var void volatile while with yield".trim().split(/[\s]*/g);
var INFREQUENT_CHARS = /[\\\/kwzq]+/ig;
var BOUNDARY = /\b/;
var USE_STRICT = 'use strict';
var RESERVED_WORDS = "\n abstract arguments await boolean break byte case catch char class const continue debugger\n default delete do double else enum eval export extends false final finally float for\n function goto if implements import in instanceof int interface let long native new null\n package private protected public return short static super switch synchronized this throw\n throws transient true try typeof var void volatile while with yield".trim().split(/[\s]+/g);
var Dedupe = (function () {

@@ -13,3 +14,5 @@ function Dedupe(options) {

addScope: false,
cleanStrings: false
cleanStrings: false,
minInstances: 10,
minLength: 10
};

@@ -47,10 +50,14 @@ if (typeof options !== 'undefined') {

Dedupe.prototype.shouldStringBeReplaced = function (str, count) {
if (count > 1) {
if (str.length > 5) {
return true;
}
if (count > 5) {
return true;
}
var minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
var minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
var matches = str.match(INFREQUENT_CHARS);
if (matches && matches.length > 1) {
return true;
}
if (str.length > minLength) {
return true;
}
if (count > minInstances) {
return true;
}
return false;

@@ -62,10 +69,26 @@ };

var replacements = [];
var rankingMap = [];
stringMap.forEach(function (values, key) {
if (_this.shouldStringBeReplaced(key, values.length)) {
var variableName = _this.getUniqueVariableName(usedVariableNames);
var count = values.length;
if (_this.shouldStringBeReplaced(key, count)) {
rankingMap.push([key, count]);
}
});
rankingMap.sort(function (a, b) {
return a[1] > b[1] ? 1 : a[1] < b[1] ? -1 : 0;
});
for (var _i = 0, rankingMap_1 = rankingMap; _i < rankingMap_1.length; _i++) {
var ranking = rankingMap_1[_i];
var key = ranking[0];
var values = stringMap.get(key);
if (values) {
var variableName = this.getUniqueVariableName(usedVariableNames);
variableDeclarationBuffer.push(variableName + "=" + JSON.stringify(key));
for (var j = 0, length = values.length; j < length; j++) {
var node = values[j];
var start = node.getStart();
var end = node.getEnd();
replacements.push({
end: values[j].getEnd(),
start: values[j].getStart(),
end: end,
start: start,
text: variableName

@@ -75,6 +98,6 @@ });

}
});
}
var variableDeclaration = '';
if (variableDeclarationBuffer.length > 0) {
variableDeclaration = "var " + variableDeclarationBuffer.join(',\n') + ";";
variableDeclaration = "var " + variableDeclarationBuffer.join(',') + ";";
replacements.push({

@@ -106,3 +129,2 @@ end: startingPos,

Dedupe.prototype.getStringMap = function (startingNode) {
var _this = this;
var stringMap = new Map();

@@ -112,4 +134,13 @@ var walk = this.createWalker(function (node) {

case typescript.SyntaxKind.StringLiteral:
// make sure this string is not part of an property assignment { "a": 123 }
if (node.parent && node.parent.kind === typescript.SyntaxKind.PropertyAssignment) {
if (node.parent.getChildAt(0) === node) {
break;
}
}
var stringNode = node;
var text = _this.cleanString(stringNode.text);
var text = stringNode.text;
if (text === USE_STRICT) {
break;
}
var strings = stringMap.get(text);

@@ -143,3 +174,3 @@ if (strings) {

Dedupe.prototype.translateNumberToVariable = function (num) {
var letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
var letters = '_eariotnslcu';
var charLen = Math.floor(num / letters.length);

@@ -161,22 +192,22 @@ var base = letters.length;

Dedupe.prototype.sortReplacements = function (replacements) {
return replacements.sort(function (a, b) { return a.start > b.start ? -1 : a.start < b.start ? 1 : 0; });
return replacements.sort(function (a, b) { return a.start > b.start ? 1 : a.start < b.start ? -1 : 0; });
};
Dedupe.prototype.makeAllReplacements = function (code, sortedReplacements) {
var codeBuffer = [];
var curser = code.length;
var cursor = 0;
for (var i = 0, length = sortedReplacements.length; i < length; i++) {
var replacement = sortedReplacements[i];
codeBuffer.unshift(code.substring(replacement.end, curser));
codeBuffer.unshift(replacement.text);
curser = replacement.start;
codeBuffer.push(code.substring(cursor, replacement.start));
if (BOUNDARY.test(code.charAt(replacement.start - 1))) {
codeBuffer.push(' ');
}
codeBuffer.push(this.cleanString(replacement.text));
if (BOUNDARY.test(code.charAt(replacement.end))) {
codeBuffer.push(' ');
}
cursor = replacement.end;
}
codeBuffer.unshift(code.substring(0, curser));
codeBuffer.push(code.substring(cursor, code.length));
return codeBuffer.join('');
};
Dedupe.prototype.findInNodes = function (startingNode, expression) {
var walker = function (node) {
typescript.forEachChild(node, walker);
};
typescript.forEachChild(startingNode, expression);
};
Dedupe.prototype.createWalker = function (traverser) {

@@ -190,7 +221,18 @@ var walker = function (node) {

Dedupe.prototype.getStartingPositionOfScope = function (scope) {
return scope.getChildAt(1).pos;
var startingPos = scope.getChildAt(1).pos;
// check the top level of the block for "use strict"
typescript.forEachChild(scope, function (expressionNode) {
if (expressionNode.kind === typescript.SyntaxKind.ExpressionStatement) {
typescript.forEachChild(expressionNode, function (stringNode) {
if (stringNode.kind === typescript.SyntaxKind.StringLiteral && stringNode.text === USE_STRICT) {
startingPos = expressionNode.getEnd();
}
});
}
});
return startingPos;
};
Dedupe.prototype.cleanString = function (fatString) {
return this.options.cleanString
? this.cleanString(fatString)
return this.options.cleanStrings
? fatString.replace(/[\s]+/g, ' ')
: fatString;

@@ -202,5 +244,34 @@ };

}
var scopes = [];
var functions = this.getTopLevelFunctions(node);
for (var _i = 0, functions_1 = functions; _i < functions_1.length; _i++) {
var functionNode = functions_1[_i];
var blocks = this.getTopLevelFunctionBlocks(functionNode);
scopes = scopes.concat(blocks);
}
return scopes;
};
Dedupe.prototype.getTopLevelFunctions = function (node) {
var queue = [];
var found = [];
while (node) {
if (node.kind !== typescript.SyntaxKind.ArrowFunction &&
node.kind !== typescript.SyntaxKind.FunctionDeclaration &&
node.kind !== typescript.SyntaxKind.FunctionExpression &&
node.kind !== typescript.SyntaxKind.FunctionType) {
typescript.forEachChild(node, function (childNode) {
queue.push(childNode);
});
}
else {
found.push(node);
}
node = queue.shift();
}
return found;
};
Dedupe.prototype.getTopLevelFunctionBlocks = function (node) {
var queue = [];
var found = [];
while (node) {
if (node.kind !== typescript.SyntaxKind.Block) {

@@ -221,84 +292,2 @@ typescript.forEachChild(node, function (childNode) {

// tslint:disable:no-var-requires
var config = require('./package.json');
process.title = config.name;
var options = parseArguments(process);
if (!options.showHelp && options.files.length > 0) {
processFiles(options);
}
else {
printHelp();
}
function processFiles(opt) {
var _loop_1 = function (file) {
fs.readFile(file, 'utf-8', function (err, code) {
if (err) {
console.error(err);
}
else {
var dedupeOptions = {
addScope: !!opt.addScope,
cleanStrings: !!opt.cleanString,
minInstances: opt.minInstances || 0
};
var dedupe = new Dedupe(dedupeOptions);
var dedupedCode = dedupe.dedupe(code);
fs.writeFile(file.replace('.js', '') + '.min.js', dedupedCode);
}
});
};
for (var _i = 0, _a = opt.files; _i < _a.length; _i++) {
var file = _a[_i];
_loop_1(file);
}
}
function printHelp() {
process.stdout.write(config.name + ' ' + config.version + '\n' +
'\n' +
'Usage: de-dupe [options] -- <...files>' +
'\n' +
'Options:\n' +
'--addScope, -s Adds an IIFE around the entire script\n' +
'--cleanStrings, -c Clean out duplicate spaces from strings\n');
}
function parseArguments(process) {
var parsedOptions = {
addScope: false,
cleanString: false,
files: [],
minInstances: undefined,
showHelp: true
};
var parseFiles = false;
process.argv.forEach(function (val, idx) {
if (parseFiles) {
parsedOptions.files.push(path.resolve(val));
}
else {
switch (val) {
case '--addScope':
case '-s':
parsedOptions.addScope = true;
break;
case '--cleanStrings':
case '-c':
parsedOptions.cleanString = true;
break;
case '--minInstances':
case '-m':
var minInstances = parseInt(process.argv[idx + 1], 10);
if (minInstances && minInstances > 0) {
parsedOptions.minInstances = minInstances;
}
break;
case '--':
parsedOptions.showHelp = false;
parseFiles = true;
break;
default:
break;
}
}
});
return parsedOptions;
}
module.exports = Dedupe;
{
"name": "de-dupe",
"bin": ".bin/de-dupe",
"version": "0.0.6",
"version": "0.0.7",
"description": "Deduplicate strings from javascript assets",

@@ -11,3 +11,5 @@ "main": "index.js",

"test": "npm run lint && rollup -c -o tests.js -i tests/index.ts && mocha tests.js",
"build": "rollup -c"
"build": "npm run build:index && npm run build:cli",
"build:index": "rollup -c -o index.js -i src/index.ts",
"build:cli": "rollup -c -o cli.js -i src/cli.ts"
},

@@ -40,4 +42,5 @@ "repository": {

"rollup": "^0.41.5",
"rollup-plugin-typescript": "^0.8.1"
"rollup-plugin-typescript": "^0.8.1",
"tslint": "^4.4.2"
}
}
#De-dupe - a Javascript string minifier
[![Build Status](https://travis-ci.org/markis/de-dupe.svg?branch=master)](https://travis-ci.org/markis/de-dupe)
[![Build Status](https://travis-ci.org/markis/de-dupe.svg?branch=master)](https://travis-ci.org/markis/de-dupe) [![Greenkeeper badge](https://badges.greenkeeper.io/markis/de-dupe.svg)](https://greenkeeper.io/)

@@ -22,2 +22,67 @@ De-dupe is an asset minification process that will identify duplicate strings in all scopes of a javascript file and will introduce a variable instead of the string itself. It does not introduce variables on the global scope, it will keep the variables to the individual scopes that it identifies. It can also clean up strings so that they don't have large amounts of white space in them.

--cleanStrings, -c Removes duplicate spaces from strings, usually strings in javascript render to the DOM and more than one space in the DOM is ignored and just bloats scripts
```
```
### Example
The follow demonstrates what the de-dupe code will do.
Original code:
```
function x() { console.log('long string', 'long string', 'long string', 'long string', 'long string', 'long string'); }
```
De-duplicated code:
```
function x() { var a='long string'; console.log(a, a, a, a, a, a); }
```
### Why?
#### Doesn't gzip/deflate take care of this?
Yes, it does. But, this can help. Read on...
To put it in very simple terms, gzip and deflate work by identifying patterns that already exist in the string and then
place markers telling the decompressor where the pattern could be identified. First 8 bits identify how long the pattern is,
and the last 15 bits identify how far away the previous pattern could be identified.
Before gzip:
```
console.log('long string', 'long string');
```
After gzip:
```
console.log('long string', <13,15>);
```
#### Today's javascript >32KB
It works in theory, but sometimes falls short in practice.
Notice, the distance bits are only 15 (1-32768), which means the patterns have to be within 32KB of uncompressed data of each other.
Basically, all of the major javascript frameworks are larger than 32KB. And after taking a cruise around Alexa's Top 10 most
popular websites on the internet (google.com, amazon.com, facebook.com, twitter.com, etc.), every single site had at least one
javascript file over 32KB (mostly, I saw ~200KB range). Assuming your site has a javascript file of over 200KB and a pattern at the
beginning and somewhere in the middle; then gzip won't be able to help you. Compounding the size of your javascript file.
#### de-dupe's variable algorithm
De-dupe uses only the most popular programming characters for variables. There is a very high chance that the character 'e' will
have been used at least once in any chunk of 32KB. But so does the algorithm built into uglify, but the algorithm in uglify runs out
so I recommend combining de-dupe with google closure compiler and uglify to get the absolute smallest file.
#### Strings can be >256
The length bits are only 8 (1-256), meaning that any pattern larger than 256 bytes in length will have to have more than one marker.
It is a gentle balance, but if a string that is over 256 bytes were converted into a single letter variable, then it wouldn't take
any more size than a single marker.
Rendering templates (handlebars, mustache, etc...) can easily take over 256 bytes. And when you work on a large site, you would be
amazed that somehow those very large templates sometimes accidentally get duplicated during your asset building process.
#### Conclusion
If your javascript is tiny and emmaculately procured down to it's smallest form. Then quite possibly this tool will not help you.
It might actually make your gzip'd javascript larger. But I am guessing since you found this tool, you probably are looking for
something to help make your javascript smaller.
var typescript = require('rollup-plugin-typescript');
module.exports = {
entry: './src/cli.ts',
entry: './src/index.ts',
dest: 'index.js',

@@ -6,0 +6,0 @@ format: 'cjs',

import { readFile, writeFile } from 'fs';
import { resolve } from 'path';
import Dedupe from './index';
import Dedupe, { DedupeOptions } from './index';
// tslint:disable:no-var-requires

@@ -24,8 +24,3 @@ const config = require('./package.json');

} else {
const dedupeOptions = {
addScope: !!opt.addScope,
cleanStrings: !!opt.cleanString,
minInstances: opt.minInstances || 0
};
const dedupe = new Dedupe(dedupeOptions);
const dedupe = new Dedupe(opt as DedupeOptions);
const dedupedCode = dedupe.dedupe(code);

@@ -51,8 +46,4 @@ writeFile(file.replace('.js', '') + '.min.js', dedupedCode);

function parseArguments(process: Process) {
const parsedOptions = {
addScope: false,
cleanString: false,
files: [] as string[],
minInstances: undefined,
showHelp: true
const parsedOptions: any = {
files: []
};

@@ -71,8 +62,15 @@ let parseFiles = false;

case '-c':
parsedOptions.cleanString = true;
parsedOptions.cleanStrings = true;
break;
case '--minLength':
case '-ml':
const minLength = parseInt(process.argv[idx + 1], 10);
if (!isNaN(minLength)) {
parsedOptions.minLength = minLength;
}
break;
case '--minInstances':
case '-m':
case '-mi':
const minInstances = parseInt(process.argv[idx + 1], 10);
if (minInstances && minInstances > 0) {
if (!isNaN(minInstances)) {
parsedOptions.minInstances = minInstances;

@@ -79,0 +77,0 @@ }

@@ -19,3 +19,3 @@ import {

*/
addScope: boolean;
addScope?: boolean;
/**

@@ -25,3 +25,11 @@ * Removes duplicate spaces from strings, usually strings in javascript render

*/
cleanStrings: boolean;
cleanStrings?: boolean;
/**
* Minimum number of string instances before de-dupe will replace the string.
*/
minInstances?: number;
/**
* Minimum length of the string before de-dupe will replace the string.
*/
minLength?: number;
}

@@ -41,2 +49,5 @@

const INFREQUENT_CHARS = /[\\\/kwzq]+/ig;
const BOUNDARY = /\b/;
const USE_STRICT = 'use strict';
const RESERVED_WORDS = `

@@ -47,3 +58,3 @@ abstract arguments await boolean break byte case catch char class const continue debugger

package private protected public return short static super switch synchronized this throw
throws transient true try typeof var void volatile while with yield`.trim().split(/[\s]*/g);
throws transient true try typeof var void volatile while with yield`.trim().split(/[\s]+/g);

@@ -55,3 +66,5 @@ export default class Dedupe {

addScope: false,
cleanStrings: false
cleanStrings: false,
minInstances: 10,
minLength: 10
};

@@ -96,10 +109,14 @@

private shouldStringBeReplaced(str: string, count: number): boolean {
if (count > 1) {
if (str.length > 5) {
return true;
}
if (count > 5) {
return true;
}
const minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
const minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
const matches = (str.match(INFREQUENT_CHARS) as any);
if (matches && matches.length > 1) {
return true;
}
if (str.length > minLength) {
return true;
}
if (count > minInstances) {
return true;
}
return false;

@@ -110,6 +127,20 @@ }

usedVariableNames: Map<string, string>): StringReplacement[] {
let variableDeclarationBuffer: string[] = [];
const variableDeclarationBuffer: string[] = [];
const replacements: StringReplacement[] = [];
const rankingMap: Array<Array<string | number>> = [];
stringMap.forEach((values, key) => {
if (this.shouldStringBeReplaced(key, values.length)) {
const count = values.length;
if (this.shouldStringBeReplaced(key, count)) {
rankingMap.push([key, count]);
}
});
rankingMap.sort((a, b) => {
return a[1] > b[1] ? 1 : a[1] < b[1] ? -1 : 0;
});
for (let ranking of rankingMap) {
let key = ranking[0] as string;
let values = stringMap.get(key);
if (values) {
const variableName = this.getUniqueVariableName(usedVariableNames);

@@ -119,5 +150,9 @@ variableDeclarationBuffer.push(`${variableName}=${JSON.stringify(key)}`);

for (let j = 0, length = values.length; j < length; j++) {
let node = values[j];
let start = node.getStart();
let end = node.getEnd();
replacements.push({
end: values[j].getEnd(),
start: values[j].getStart(),
end,
start,
text: variableName

@@ -127,7 +162,7 @@ });

}
});
}
let variableDeclaration = '';
if (variableDeclarationBuffer.length > 0) {
variableDeclaration = `var ${variableDeclarationBuffer.join(',\n')};`;
variableDeclaration = `var ${variableDeclarationBuffer.join(',')};`;
replacements.push({

@@ -165,4 +200,15 @@ end: startingPos,

case SyntaxKind.StringLiteral:
// make sure this string is not part of an property assignment { "a": 123 }
if (node.parent && node.parent.kind === SyntaxKind.PropertyAssignment) {
if (node.parent.getChildAt(0) === node) {
break;
}
}
const stringNode = node as StringLiteral;
const text = this.cleanString(stringNode.text);
const text = stringNode.text;
if (text === USE_STRICT) {
break;
}
const strings = stringMap.get(text);

@@ -196,3 +242,3 @@ if (strings) {

private translateNumberToVariable(num: number): string {
const letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
const letters = '_eariotnslcu';
const charLen = Math.floor(num / letters.length);

@@ -216,3 +262,3 @@

private sortReplacements(replacements: StringReplacement[]) {
return replacements.sort((a, b) => a.start > b.start ? -1 : a.start < b.start ? 1 : 0);
return replacements.sort((a, b) => a.start > b.start ? 1 : a.start < b.start ? -1 : 0);
}

@@ -222,10 +268,17 @@

const codeBuffer = [];
let curser = code.length;
let cursor = 0;
for (let i = 0, length = sortedReplacements.length; i < length; i++) {
let replacement = sortedReplacements[i];
codeBuffer.unshift(code.substring(replacement.end, curser));
codeBuffer.unshift(replacement.text);
curser = replacement.start;
codeBuffer.push(code.substring(cursor, replacement.start));
if (BOUNDARY.test(code.charAt(replacement.start - 1))) {
codeBuffer.push(' ');
}
codeBuffer.push(this.cleanString(replacement.text));
if (BOUNDARY.test(code.charAt(replacement.end))) {
codeBuffer.push(' ');
}
cursor = replacement.end;
}
codeBuffer.unshift(code.substring(0, curser));
codeBuffer.push(code.substring(cursor, code.length));

@@ -235,9 +288,2 @@ return codeBuffer.join('');

private findInNodes(startingNode: Node, expression: (node: Node) => boolean) {
const walker = (node: Node) => {
forEachChild(node, walker);
};
forEachChild(startingNode, expression);
}
private createWalker(traverser: StatementAction) {

@@ -252,8 +298,21 @@ const walker = (node: Node) => {

private getStartingPositionOfScope(scope: Block) {
return scope.getChildAt(1).pos;
let startingPos = scope.getChildAt(1).pos;
// check the top level of the block for "use strict"
forEachChild(scope, (expressionNode) => {
if (expressionNode.kind === SyntaxKind.ExpressionStatement) {
forEachChild(expressionNode, (stringNode: StringLiteral) => {
if (stringNode.kind === SyntaxKind.StringLiteral && stringNode.text === USE_STRICT) {
startingPos = expressionNode.getEnd();
}
});
}
});
return startingPos;
}
private cleanString(fatString: string): string {
return this.options.cleanString
? this.cleanString(fatString)
return this.options.cleanStrings
? fatString.replace(/[\s]+/g, ' ')
: fatString;

@@ -267,5 +326,36 @@ }

let scopes: Block[] = [];
const functions = this.getTopLevelFunctions(node);
for (let functionNode of functions) {
const blocks = this.getTopLevelFunctionBlocks(functionNode);
scopes = scopes.concat(blocks);
}
return scopes;
}
private getTopLevelFunctions(node?: Node) {
const queue: Node[] = [];
const found: Block[] = [];
while (node) {
if (
node.kind !== SyntaxKind.ArrowFunction &&
node.kind !== SyntaxKind.FunctionDeclaration &&
node.kind !== SyntaxKind.FunctionExpression &&
node.kind !== SyntaxKind.FunctionType
) {
forEachChild(node, childNode => {
queue.push(childNode);
});
} else {
found.push(node as Block);
}
node = queue.shift();
}
return found;
}
private getTopLevelFunctionBlocks(node?: Node) {
const queue: Node[] = [];
const found: Block[] = [];
while (node) {
if (node.kind !== SyntaxKind.Block) {

@@ -282,2 +372,3 @@ forEachChild(node, childNode => {

}
}

@@ -7,3 +7,6 @@ 'use strict';

var RESERVED_WORDS = "\n abstract arguments await boolean break byte case catch char class const continue debugger\n default delete do double else enum eval export extends false final finally float for\n function goto if implements import in instanceof int interface let long native new null\n package private protected public return short static super switch synchronized this throw\n throws transient true try typeof var void volatile while with yield".trim().split(/[\s]*/g);
var INFREQUENT_CHARS = /[\\\/kwzq]+/ig;
var BOUNDARY = /\b/;
var USE_STRICT = 'use strict';
var RESERVED_WORDS = "\n abstract arguments await boolean break byte case catch char class const continue debugger\n default delete do double else enum eval export extends false final finally float for\n function goto if implements import in instanceof int interface let long native new null\n package private protected public return short static super switch synchronized this throw\n throws transient true try typeof var void volatile while with yield".trim().split(/[\s]+/g);
var Dedupe = (function () {

@@ -14,3 +17,5 @@ function Dedupe(options) {

addScope: false,
cleanStrings: false
cleanStrings: false,
minInstances: 10,
minLength: 10
};

@@ -48,10 +53,14 @@ if (typeof options !== 'undefined') {

Dedupe.prototype.shouldStringBeReplaced = function (str, count) {
if (count > 1) {
if (str.length > 5) {
return true;
}
if (count > 5) {
return true;
}
var minLength = typeof this.options.minLength === 'undefined' ? -1 : this.options.minLength;
var minInstances = typeof this.options.minInstances === 'undefined' ? -1 : this.options.minInstances;
var matches = str.match(INFREQUENT_CHARS);
if (matches && matches.length > 1) {
return true;
}
if (str.length > minLength) {
return true;
}
if (count > minInstances) {
return true;
}
return false;

@@ -63,10 +72,26 @@ };

var replacements = [];
var rankingMap = [];
stringMap.forEach(function (values, key) {
if (_this.shouldStringBeReplaced(key, values.length)) {
var variableName = _this.getUniqueVariableName(usedVariableNames);
var count = values.length;
if (_this.shouldStringBeReplaced(key, count)) {
rankingMap.push([key, count]);
}
});
rankingMap.sort(function (a, b) {
return a[1] > b[1] ? 1 : a[1] < b[1] ? -1 : 0;
});
for (var _i = 0, rankingMap_1 = rankingMap; _i < rankingMap_1.length; _i++) {
var ranking = rankingMap_1[_i];
var key = ranking[0];
var values = stringMap.get(key);
if (values) {
var variableName = this.getUniqueVariableName(usedVariableNames);
variableDeclarationBuffer.push(variableName + "=" + JSON.stringify(key));
for (var j = 0, length = values.length; j < length; j++) {
var node = values[j];
var start = node.getStart();
var end = node.getEnd();
replacements.push({
end: values[j].getEnd(),
start: values[j].getStart(),
end: end,
start: start,
text: variableName

@@ -76,6 +101,6 @@ });

}
});
}
var variableDeclaration = '';
if (variableDeclarationBuffer.length > 0) {
variableDeclaration = "var " + variableDeclarationBuffer.join(',\n') + ";";
variableDeclaration = "var " + variableDeclarationBuffer.join(',') + ";";
replacements.push({

@@ -107,3 +132,2 @@ end: startingPos,

Dedupe.prototype.getStringMap = function (startingNode) {
var _this = this;
var stringMap = new Map();

@@ -113,4 +137,13 @@ var walk = this.createWalker(function (node) {

case typescript.SyntaxKind.StringLiteral:
// make sure this string is not part of an property assignment { "a": 123 }
if (node.parent && node.parent.kind === typescript.SyntaxKind.PropertyAssignment) {
if (node.parent.getChildAt(0) === node) {
break;
}
}
var stringNode = node;
var text = _this.cleanString(stringNode.text);
var text = stringNode.text;
if (text === USE_STRICT) {
break;
}
var strings = stringMap.get(text);

@@ -144,3 +177,3 @@ if (strings) {

Dedupe.prototype.translateNumberToVariable = function (num) {
var letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
var letters = '_eariotnslcu';
var charLen = Math.floor(num / letters.length);

@@ -162,22 +195,22 @@ var base = letters.length;

Dedupe.prototype.sortReplacements = function (replacements) {
return replacements.sort(function (a, b) { return a.start > b.start ? -1 : a.start < b.start ? 1 : 0; });
return replacements.sort(function (a, b) { return a.start > b.start ? 1 : a.start < b.start ? -1 : 0; });
};
Dedupe.prototype.makeAllReplacements = function (code, sortedReplacements) {
var codeBuffer = [];
var curser = code.length;
var cursor = 0;
for (var i = 0, length = sortedReplacements.length; i < length; i++) {
var replacement = sortedReplacements[i];
codeBuffer.unshift(code.substring(replacement.end, curser));
codeBuffer.unshift(replacement.text);
curser = replacement.start;
codeBuffer.push(code.substring(cursor, replacement.start));
if (BOUNDARY.test(code.charAt(replacement.start - 1))) {
codeBuffer.push(' ');
}
codeBuffer.push(this.cleanString(replacement.text));
if (BOUNDARY.test(code.charAt(replacement.end))) {
codeBuffer.push(' ');
}
cursor = replacement.end;
}
codeBuffer.unshift(code.substring(0, curser));
codeBuffer.push(code.substring(cursor, code.length));
return codeBuffer.join('');
};
Dedupe.prototype.findInNodes = function (startingNode, expression) {
var walker = function (node) {
typescript.forEachChild(node, walker);
};
typescript.forEachChild(startingNode, expression);
};
Dedupe.prototype.createWalker = function (traverser) {

@@ -191,7 +224,18 @@ var walker = function (node) {

Dedupe.prototype.getStartingPositionOfScope = function (scope) {
return scope.getChildAt(1).pos;
var startingPos = scope.getChildAt(1).pos;
// check the top level of the block for "use strict"
typescript.forEachChild(scope, function (expressionNode) {
if (expressionNode.kind === typescript.SyntaxKind.ExpressionStatement) {
typescript.forEachChild(expressionNode, function (stringNode) {
if (stringNode.kind === typescript.SyntaxKind.StringLiteral && stringNode.text === USE_STRICT) {
startingPos = expressionNode.getEnd();
}
});
}
});
return startingPos;
};
Dedupe.prototype.cleanString = function (fatString) {
return this.options.cleanString
? this.cleanString(fatString)
return this.options.cleanStrings
? fatString.replace(/[\s]+/g, ' ')
: fatString;

@@ -203,5 +247,34 @@ };

}
var scopes = [];
var functions = this.getTopLevelFunctions(node);
for (var _i = 0, functions_1 = functions; _i < functions_1.length; _i++) {
var functionNode = functions_1[_i];
var blocks = this.getTopLevelFunctionBlocks(functionNode);
scopes = scopes.concat(blocks);
}
return scopes;
};
Dedupe.prototype.getTopLevelFunctions = function (node) {
var queue = [];
var found = [];
while (node) {
if (node.kind !== typescript.SyntaxKind.ArrowFunction &&
node.kind !== typescript.SyntaxKind.FunctionDeclaration &&
node.kind !== typescript.SyntaxKind.FunctionExpression &&
node.kind !== typescript.SyntaxKind.FunctionType) {
typescript.forEachChild(node, function (childNode) {
queue.push(childNode);
});
}
else {
found.push(node);
}
node = queue.shift();
}
return found;
};
Dedupe.prototype.getTopLevelFunctionBlocks = function (node) {
var queue = [];
var found = [];
while (node) {
if (node.kind !== typescript.SyntaxKind.Block) {

@@ -222,14 +295,19 @@ typescript.forEachChild(node, function (childNode) {

var dedupe = new Dedupe();
var dedupe = new Dedupe({
minInstances: 2,
minLength: 2
});
var stringCleaner = new Dedupe({
addScope: false,
cleanStrings: true
cleanStrings: true,
minInstances: 2,
minLength: 2
});
var scopeAdder = new Dedupe({
addScope: true,
cleanStrings: false
minInstances: 2,
minLength: 2
});
describe('de-dupe', function () {
it('can dedupe large strings', function () {
var code = "!function() { console.log('zzzzzzzzzz', 'zzzzzzzzzz'); }";
it('can handle large strings', function () {
var code = "!function() { console.log('zzzzzzzzzz', 'zzzzzzzzzz'); }()";
var result = dedupe.dedupe(code);

@@ -239,40 +317,88 @@ var markers = result.match(/zzzzzzzzzz/g);

});
it('can dedupe many small strings', function () {
var code = "!function() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }";
it('can handle many small strings', function () {
var code = "!function() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }()";
var expected = "!function() {var _=\"z\"; console.log(_, _, _, _, _, _); }()";
var result = dedupe.dedupe(code);
var markers = result.match(/z/g);
chai.expect(markers.length).to.be.equal(1);
chai.expect(result).equal(expected);
chai.expect(markers.length).equal(1);
});
it('can dedupe multiple scopes', function () {
var code = "\n !function() {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }\n\n !function() {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }\n ";
it('can handle multiple scopes', function () {
var code = "\n !function() {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }()\n\n !function() {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }()\n ";
var result = dedupe.dedupe(code);
var markers = result.match(/z/g);
chai.expect(markers.length).to.be.equal(2);
chai.expect(result.match(/z/g).length).equal(2);
});
it('can dedupe multiple scopes from one global scope', function () {
var code = "\n !function() {\n !function() {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }\n\n !function() {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }\n }\n ";
it('can handle multiple scopes from one global scope', function () {
var code = "\n !function() {\n !function() {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }()\n\n !function() {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }()\n }()\n ";
var result = dedupe.dedupe(code);
var markers = result.match(/z/g);
chai.expect(markers.length).to.be.equal(1);
chai.expect(markers.length).equal(1);
});
it('can handle named functions', function () {
var code = "function x() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }";
var expected = "function x() {var _=\"z\"; console.log(_, _, _, _, _, _); }";
var result = dedupe.dedupe(code);
chai.expect(result).equal(expected);
chai.expect(result.match(/z/g).length).equal(1);
});
it('can handle arrow functions', function () {
var code = "() => { console.log('z', 'z', 'z', 'z', 'z', 'z'); }";
var expected = "() => {var _=\"z\"; console.log(_, _, _, _, _, _); }";
var result = dedupe.dedupe(code);
chai.expect(result).equal(expected);
chai.expect(result.match(/z/g).length).equal(1);
});
it('can handle strict mode', function () {
var code = "function x() { \"use strict\"; console.log('z', 'z', 'z', 'z', 'z', 'z'); }";
var expected = "function x() { \"use strict\";var _=\"z\"; console.log(_, _, _, _, _, _); }";
var result = dedupe.dedupe(code);
chai.expect(result).equal(expected);
chai.expect(result.match(/z/g).length).equal(1);
});
it('will add scope', function () {
var code = "\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n ";
var code = "console.log('z', 'z', 'z', 'z', 'z', 'z');";
var result = scopeAdder.dedupe(code);
var markers = result.match(/z/g);
chai.expect(result).to.contain('!function');
chai.expect(markers.length).to.be.equal(1);
chai.expect(result).contain('!function');
chai.expect(result.match(/z/g).length).equal(1);
});
it('will clean strings', function () {
var code = "\n !function() {\n console.log('z ', 'z ', 'z ', 'z ', 'z ', 'z ');\n }\n ";
var code = "\n !function() {\n console.log('z ', 'z ', 'z ', 'z ', 'z ', 'z ', 'z ', 'z ');\n }()\n ";
var result = stringCleaner.dedupe(code);
var markers = result.match(/z\s{1,1}/g);
chai.expect(result).to.contain('!function');
chai.expect(markers.length).to.be.equal(1);
chai.expect(result).contain("\"z \"");
});
xit('not effect global scope', function () {
it('will not effect non-function blocks', function () {
var code = "\n if (true) {\n console.log('z', 'z', 'z', 'z', 'z', 'z');\n }\n ";
var result = dedupe.dedupe(code);
var markers = result.match(/z/g);
chai.expect(markers.length).to.be.equal(6);
chai.expect(result).equal(code);
});
it('will not treat "use strict" as a string', function () {
var code = "\n () => {\n 'use strict';\n console.log('use strict', 'use strict', 'use strict', 'use strict', 'use strict', 'use strict');\n }\n ";
var result = dedupe.dedupe(code);
chai.expect(code).equal(result);
});
it('will not treat property assignment as a string', function () {
var propertyAssignment = "var stuff = { 'z' : 123 };";
var code = "\n function x() {\n " + propertyAssignment + "\n console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z');\n }\n ";
var result = dedupe.dedupe(code);
chai.expect(result).contains(propertyAssignment);
});
it('will not treat the left side of a property assignment as a string', function () {
var code = "\n function x() {\n var stuff = { 'z' : 'z' };\n console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z');\n }\n ";
var expected = "\n function x() {var _=\"z\";\n var stuff = { 'z' : _ };\n console.log(_, _, _, _, _, _, _, _);\n }\n ";
var result = dedupe.dedupe(code);
chai.expect(result).equals(expected);
});
it('will handle magical globals', function () {
// notice the expected is expecting dedupe to identify that "a" is referring to an outside variable
var code = "function x() { _.thing = 'a'; console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z'); }";
var expected = "function x() {var e=\"z\"; _.thing = 'a'; console.log(e, e, e, e, e, e, e, e); }";
var result = dedupe.dedupe(code);
chai.expect(result).equal(expected);
});
it('deal with strings next to reserved words', function () {
var code = "function x() { if ('z'in x) { console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z'); } }";
var expected = "function x() {var _=\"z\"; if (_ in x) { console.log(_, _, _, _, _, _, _, _); } }";
var result = dedupe.dedupe(code);
chai.expect(result).equal(expected);
});
});

@@ -6,10 +6,15 @@ import 'mocha';

const dedupe = new Dedupe();
const dedupe = new Dedupe({
minInstances: 2,
minLength: 2
});
const stringCleaner = new Dedupe({
addScope: false,
cleanStrings: true
cleanStrings: true,
minInstances: 2,
minLength: 2
});
const scopeAdder = new Dedupe({
addScope: true,
cleanStrings: false
minInstances: 2,
minLength: 2
});

@@ -19,4 +24,4 @@

it('can dedupe large strings', () => {
const code = `!function() { console.log('zzzzzzzzzz', 'zzzzzzzzzz'); }`;
it('can handle large strings', () => {
const code = `!function() { console.log('zzzzzzzzzz', 'zzzzzzzzzz'); }()`;

@@ -29,4 +34,5 @@ const result = dedupe.dedupe(code);

it('can dedupe many small strings', () => {
const code = `!function() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }`;
it('can handle many small strings', () => {
const code = `!function() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }()`;
const expected = `!function() {var _="z"; console.log(_, _, _, _, _, _); }()`;

@@ -36,23 +42,23 @@ const result = dedupe.dedupe(code);

expect(markers.length).to.be.equal(1);
expect(result).equal(expected);
expect(markers.length).equal(1);
});
it('can dedupe multiple scopes', () => {
it('can handle multiple scopes', () => {
const code = `
!function() {
console.log('z', 'z', 'z', 'z', 'z', 'z');
}
}()
!function() {
console.log('z', 'z', 'z', 'z', 'z', 'z');
}
}()
`;
const result = dedupe.dedupe(code);
const markers = result.match(/z/g) as any[];
expect(markers.length).to.be.equal(2);
expect((result.match(/z/g) as any[]).length).equal(2);
});
it('can dedupe multiple scopes from one global scope', () => {
it('can handle multiple scopes from one global scope', () => {
const code = `

@@ -62,8 +68,8 @@ !function() {

console.log('z', 'z', 'z', 'z', 'z', 'z');
}
}()
!function() {
console.log('z', 'z', 'z', 'z', 'z', 'z');
}
}
}()
}()
`;

@@ -74,15 +80,42 @@

expect(markers.length).to.be.equal(1);
expect(markers.length).equal(1);
});
it('can handle named functions', () => {
const code = `function x() { console.log('z', 'z', 'z', 'z', 'z', 'z'); }`;
const expected = `function x() {var _="z"; console.log(_, _, _, _, _, _); }`;
const result = dedupe.dedupe(code);
expect(result).equal(expected);
expect((result.match(/z/g) as any[]).length).equal(1);
});
it('can handle arrow functions', () => {
const code = `() => { console.log('z', 'z', 'z', 'z', 'z', 'z'); }`;
const expected = `() => {var _="z"; console.log(_, _, _, _, _, _); }`;
const result = dedupe.dedupe(code);
expect(result).equal(expected);
expect((result.match(/z/g) as any[]).length).equal(1);
});
it('can handle strict mode', () => {
const code = `function x() { "use strict"; console.log('z', 'z', 'z', 'z', 'z', 'z'); }`;
const expected = `function x() { "use strict";var _="z"; console.log(_, _, _, _, _, _); }`;
const result = dedupe.dedupe(code);
expect(result).equal(expected);
expect((result.match(/z/g) as any[]).length).equal(1);
});
it('will add scope', () => {
const code = `
console.log('z', 'z', 'z', 'z', 'z', 'z');
`;
const code = `console.log('z', 'z', 'z', 'z', 'z', 'z');`;
const result = scopeAdder.dedupe(code);
const markers = result.match(/z/g) as any[];
expect(result).to.contain('!function');
expect(markers.length).to.be.equal(1);
expect(result).contain('!function');
expect((result.match(/z/g) as any[]).length).equal(1);
});

@@ -93,14 +126,12 @@

!function() {
console.log('z ', 'z ', 'z ', 'z ', 'z ', 'z ');
}
console.log('z ', 'z ', 'z ', 'z ', 'z ', 'z ', 'z ', 'z ');
}()
`;
const result = stringCleaner.dedupe(code);
const markers = result.match(/z\s{1,1}/g) as any[];
expect(result).to.contain('!function');
expect(markers.length).to.be.equal(1);
expect(result).contain(`"z "`);
});
xit('not effect global scope', () => {
it('will not effect non-function blocks', () => {
const code = `

@@ -113,8 +144,72 @@ if (true) {

const result = dedupe.dedupe(code);
const markers = result.match(/z/g) as any[];
expect(markers.length).to.be.equal(6);
expect(result).equal(code);
});
it('will not treat "use strict" as a string', () => {
const code = `
() => {
'use strict';
console.log('use strict', 'use strict', 'use strict', 'use strict', 'use strict', 'use strict');
}
`;
const result = dedupe.dedupe(code);
expect(code).equal(result);
});
it('will not treat property assignment as a string', () => {
const propertyAssignment = `var stuff = { 'z' : 123 };`;
const code = `
function x() {
${propertyAssignment}
console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z');
}
`;
const result = dedupe.dedupe(code);
expect(result).contains(propertyAssignment);
});
it('will not treat the left side of a property assignment as a string', () => {
const code = `
function x() {
var stuff = { 'z' : 'z' };
console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z');
}
`;
const expected = `
function x() {var _="z";
var stuff = { 'z' : _ };
console.log(_, _, _, _, _, _, _, _);
}
`;
const result = dedupe.dedupe(code);
expect(result).equals(expected);
});
it('will handle magical globals', () => {
// notice the expected is expecting dedupe to identify that "a" is referring to an outside variable
const code = `function x() { _.thing = 'a'; console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z'); }`;
const expected = `function x() {var e="z"; _.thing = 'a'; console.log(e, e, e, e, e, e, e, e); }`;
const result = dedupe.dedupe(code);
expect(result).equal(expected);
});
it('deal with strings next to reserved words', () => {
const code = `function x() { if ('z'in x) { console.log('z', 'z', 'z', 'z', 'z', 'z', 'z', 'z'); } }`;
const expected = `function x() {var _="z"; if (_ in x) { console.log(_, _, _, _, _, _, _, _); } }`;
const result = dedupe.dedupe(code);
expect(result).equal(expected);
});
});

@@ -7,4 +7,5 @@ {

"arrow-parens": false,
"no-console": [false],
"trailing-comma": [false]
}
}

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc