Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
Socket
Sign inDemoInstall
Socket
Sign inDemoInstall

de-dupe

Package Overview
Dependencies
Maintainers
1
Versions
28
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

de-dupe - npm Package Compare versions

Comparing version 0.0.5 to 0.0.6

.travis.yml
rollup.config.js
src/cli.ts
src/index.ts
tests.js
tests/index.ts
tslint.json

478

index.js

@@ -1,227 +0,293 @@

"use strict";
/// <reference path="interfaces/_global.d.ts" />
'use strict';
var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();
var fs = require('fs');
var path = require('path');
var typescript = require('typescript');
var _createClass = function () { function defineProperties(target, props) { for (var i = 0; i < props.length; i++) { var descriptor = props[i]; descriptor.enumerable = descriptor.enumerable || false; descriptor.configurable = true; if ("value" in descriptor) descriptor.writable = true; Object.defineProperty(target, descriptor.key, descriptor); } } return function (Constructor, protoProps, staticProps) { if (protoProps) defineProperties(Constructor.prototype, protoProps); if (staticProps) defineProperties(Constructor, staticProps); return Constructor; }; }();
function _classCallCheck(instance, Constructor) { if (!(instance instanceof Constructor)) { throw new TypeError("Cannot call a class as a function"); } }
var parser = require('esprima');
var scopeFinder = require('escope');
var codeGenerator = require('escodegen');
var walker = require('estraverse');
var mangler = require('esshorten');
var uglifyjs = require('uglify-js');
var _defaultOptions = {
addScope: false,
cleanStrings: false
};
var Dedupe = function () {
function Dedupe() {
var options = arguments.length <= 0 || arguments[0] === undefined ? _defaultOptions : arguments[0];
_classCallCheck(this, Dedupe);
this.__generatedCount = 0;
this.__variablesDeclarations = {
type: 'VariableDeclaration',
declarations: [],
kind: 'var'
var RESERVED_WORDS = "\n abstract arguments await boolean break byte case catch char class const continue debugger\n default delete do double else enum eval export extends false final finally float for\n function goto if implements import in instanceof int interface let long native new null\n package private protected public return short static super switch synchronized this throw\n throws transient true try typeof var void volatile while with yield".trim().split(/[\s]*/g);
var Dedupe = (function () {
function Dedupe(options) {
this.attempt = 0;
this.options = {
addScope: false,
cleanStrings: false
};
this.options = _defaultOptions;
Object.assign(this.options, options);
if (typeof options !== 'undefined') {
Object.assign(this.options, options);
}
}
_createClass(Dedupe, [{
key: 'dedupe',
value: function dedupe(code) {
if (this.options.addScope === true) {
code = '!function(){' + code + '}()';
Dedupe.prototype.dedupe = function (code) {
if (this.options.addScope) {
code = "!function(){" + code + "}";
}
var replacements = [];
var sourceFile = typescript.createSourceFile('', code, typescript.ScriptTarget.Latest, true);
var topLevelScopes = this.getTopLevelScopes(sourceFile);
var identifiers = sourceFile.identifiers;
var usedIdentifiers = new Map(identifiers);
var reservedWords = RESERVED_WORDS;
for (var _i = 0, reservedWords_1 = reservedWords; _i < reservedWords_1.length; _i++) {
var word = reservedWords_1[_i];
usedIdentifiers.set(word, word);
}
for (var i = 0, length = topLevelScopes.length; i < length; i++) {
this.attempt = 0;
var startingPos = this.getStartingPositionOfScope(topLevelScopes[i]);
var usedNames = this.getUsedVariableNames(usedIdentifiers, topLevelScopes[i]);
var stringMap = this.getStringMap(topLevelScopes[i]);
var scopeReplacements = this.getStringReplacements(stringMap, startingPos, usedNames);
replacements = replacements.concat(scopeReplacements);
}
var sortedReplacements = this.sortReplacements(replacements);
code = this.makeAllReplacements(code, sortedReplacements);
return code;
};
Dedupe.prototype.shouldStringBeReplaced = function (str, count) {
if (count > 1) {
if (str.length > 5) {
return true;
}
var ast = parser.parse(code);
var scopeManager = scopeFinder.analyze(ast);
var globalScope = scopeManager.acquire(ast); // global scope
var scopes = globalScope.childScopes;
var strings = void 0;
for (var i = 0; i < scopes.length; i++) {
this.__variablesDeclarations = {
type: 'VariableDeclaration',
declarations: [],
kind: 'var'
};
var scope = scopes[i];
var block = scope.block;
var body = block.body;
strings = this.findStrings(scope);
this.change(strings);
if (this.__variablesDeclarations.declarations.length > 0) {
body.body.splice(0, 0, this.__variablesDeclarations);
}
if (count > 5) {
return true;
}
mangler.mangle(ast);
var mangledAst = this.uglify(ast);
return codeGenerator.generate(mangledAst, {
format: {
escapeless: true,
quotes: 'auto',
compact: true,
semicolons: false,
parentheses: false
}
});
}
}, {
key: 'uglify',
value: function uglify(ast) {
var uglify = uglifyjs;
// Conversion from SpiderMonkey AST to internal format
var uAST = uglify.AST_Node.from_mozilla_ast(ast);
// Compression
uAST.figure_out_scope();
uAST = uAST.transform(uglify.Compressor({ warnings: false }));
// Mangling (optional)
uAST.figure_out_scope();
uAST.compute_char_frequency();
uAST.mangle_names();
// Back-conversion to SpiderMonkey AST
return uAST.to_mozilla_ast();
}
}, {
key: 'findStrings',
value: function findStrings(scope) {
var block = void 0,
strings = void 0,
instances = void 0;
strings = new Map();
block = scope.block;
walker.traverse(block, {
enter: function enter(node, parent) {
var literal = node,
value = literal.value;
if (literal.type === 'Literal' && typeof value === 'string' && parent.type !== 'Property') {
instances = strings.get(value);
if (!instances) {
instances = [];
strings.set(value, instances);
}
instances.push({ node: node, parent: parent });
}
return false;
};
Dedupe.prototype.getStringReplacements = function (stringMap, startingPos, usedVariableNames) {
var _this = this;
var variableDeclarationBuffer = [];
var replacements = [];
stringMap.forEach(function (values, key) {
if (_this.shouldStringBeReplaced(key, values.length)) {
var variableName = _this.getUniqueVariableName(usedVariableNames);
variableDeclarationBuffer.push(variableName + "=" + JSON.stringify(key));
for (var j = 0, length = values.length; j < length; j++) {
replacements.push({
end: values[j].getEnd(),
start: values[j].getStart(),
text: variableName
});
}
}
});
var variableDeclaration = '';
if (variableDeclarationBuffer.length > 0) {
variableDeclaration = "var " + variableDeclarationBuffer.join(',\n') + ";";
replacements.push({
end: startingPos,
start: startingPos,
text: variableDeclaration
});
return strings;
}
}, {
key: 'change',
value: function change(strings) {
var _iteratorNormalCompletion = true;
var _didIteratorError = false;
var _iteratorError = undefined;
try {
for (var _iterator = strings.entries()[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true) {
var _step$value = _slicedToArray(_step.value, 2);
var key = _step$value[0];
var instances = _step$value[1];
if (key && instances && Array.isArray(instances)) {
this.cleanStrings(instances);
//if (key.length > 10 && instances.length > 5) {
// this.replaceInstances(instances[0], key);
//} else
if (instances.length > 5) {
this.replaceInstances(instances, key);
}
return replacements;
};
Dedupe.prototype.getUsedVariableNames = function (identifiers, startingNode) {
var names = new Map(identifiers);
var walk = this.createWalker(function (node) {
switch (node.kind) {
case typescript.SyntaxKind.VariableDeclaration:
var variableNode = node;
var id = variableNode.name;
if (!names.has(id.text)) {
names.set(id.text, id.text);
}
}
} catch (err) {
_didIteratorError = true;
_iteratorError = err;
} finally {
try {
if (!_iteratorNormalCompletion && _iterator.return) {
_iterator.return();
default:
break;
}
});
walk(startingNode);
return names;
};
Dedupe.prototype.getStringMap = function (startingNode) {
var _this = this;
var stringMap = new Map();
var walk = this.createWalker(function (node) {
switch (node.kind) {
case typescript.SyntaxKind.StringLiteral:
var stringNode = node;
var text = _this.cleanString(stringNode.text);
var strings = stringMap.get(text);
if (strings) {
strings.push(stringNode);
}
} finally {
if (_didIteratorError) {
throw _iteratorError;
else {
stringMap.set(stringNode.text, [stringNode]);
}
}
default:
break;
}
});
walk(startingNode);
return stringMap;
};
Dedupe.prototype.getUniqueVariableName = function (usedVariableNames) {
var _this = this;
var namer = function () {
var newVariable = _this.translateNumberToVariable(_this.attempt);
if (!usedVariableNames.has(newVariable)) {
usedVariableNames.set(newVariable, newVariable);
return newVariable;
}
_this.attempt++;
return namer();
};
return namer();
};
Dedupe.prototype.translateNumberToVariable = function (num) {
var letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
var charLen = Math.floor(num / letters.length);
var base = letters.length;
var rixit; // like 'digit', only in some non-decimal radix
var residual = Math.floor(num);
var result = '';
while (true) {
rixit = residual % base;
result = letters.charAt(rixit) + result;
residual = Math.floor(residual / base);
if (residual === 0) {
break;
}
}
}, {
key: 'replaceInstances',
value: function replaceInstances(instances, key) {
var instance = void 0,
newVariable = this.generateNewVariable(key),
name = newVariable.id['name'];
for (var i = 0; i < instances.length; i++) {
instance = instances[i];
walker.replace(instance.parent, {
enter: function enter(node) {
if (node === instance.node) {
return {
type: "Identifier",
name: name
};
}
}
return result;
};
Dedupe.prototype.sortReplacements = function (replacements) {
return replacements.sort(function (a, b) { return a.start > b.start ? -1 : a.start < b.start ? 1 : 0; });
};
Dedupe.prototype.makeAllReplacements = function (code, sortedReplacements) {
var codeBuffer = [];
var curser = code.length;
for (var i = 0, length = sortedReplacements.length; i < length; i++) {
var replacement = sortedReplacements[i];
codeBuffer.unshift(code.substring(replacement.end, curser));
codeBuffer.unshift(replacement.text);
curser = replacement.start;
}
codeBuffer.unshift(code.substring(0, curser));
return codeBuffer.join('');
};
Dedupe.prototype.findInNodes = function (startingNode, expression) {
var walker = function (node) {
typescript.forEachChild(node, walker);
};
typescript.forEachChild(startingNode, expression);
};
Dedupe.prototype.createWalker = function (traverser) {
var walker = function (node) {
traverser(node);
typescript.forEachChild(node, walker);
};
return walker;
};
Dedupe.prototype.getStartingPositionOfScope = function (scope) {
return scope.getChildAt(1).pos;
};
Dedupe.prototype.cleanString = function (fatString) {
return this.options.cleanString
? this.cleanString(fatString)
: fatString;
};
Dedupe.prototype.getTopLevelScopes = function (node) {
if (typeof node === 'undefined') {
return [];
}
var queue = [];
var found = [];
while (node) {
if (node.kind !== typescript.SyntaxKind.Block) {
typescript.forEachChild(node, function (childNode) {
queue.push(childNode);
});
}
else {
found.push(node);
}
node = queue.shift();
}
}, {
key: 'cleanStrings',
value: function cleanStrings(instances) {
if (this.options.cleanStrings) {
(function () {
var instance = void 0;
for (var i = 0; i < instances.length; i++) {
instance = instances[i];
walker.replace(instance.parent, {
enter: function enter(node) {
if (node === instance.node) {
var value = node.value,
quote = value.charAt(0);
node.value = value = Dedupe.cleanString(value);
node.raw = quote + value + quote;
return node;
}
}
});
return found;
};
return Dedupe;
}());
// tslint:disable:no-var-requires
var config = require('./package.json');
process.title = config.name;
var options = parseArguments(process);
if (!options.showHelp && options.files.length > 0) {
processFiles(options);
}
else {
printHelp();
}
function processFiles(opt) {
var _loop_1 = function (file) {
fs.readFile(file, 'utf-8', function (err, code) {
if (err) {
console.error(err);
}
else {
var dedupeOptions = {
addScope: !!opt.addScope,
cleanStrings: !!opt.cleanString,
minInstances: opt.minInstances || 0
};
var dedupe = new Dedupe(dedupeOptions);
var dedupedCode = dedupe.dedupe(code);
fs.writeFile(file.replace('.js', '') + '.min.js', dedupedCode);
}
});
};
for (var _i = 0, _a = opt.files; _i < _a.length; _i++) {
var file = _a[_i];
_loop_1(file);
}
}
function printHelp() {
process.stdout.write(config.name + ' ' + config.version + '\n' +
'\n' +
'Usage: de-dupe [options] -- <...files>' +
'\n' +
'Options:\n' +
'--addScope, -s Adds an IIFE around the entire script\n' +
'--cleanStrings, -c Clean out duplicate spaces from strings\n');
}
function parseArguments(process) {
var parsedOptions = {
addScope: false,
cleanString: false,
files: [],
minInstances: undefined,
showHelp: true
};
var parseFiles = false;
process.argv.forEach(function (val, idx) {
if (parseFiles) {
parsedOptions.files.push(path.resolve(val));
}
else {
switch (val) {
case '--addScope':
case '-s':
parsedOptions.addScope = true;
break;
case '--cleanStrings':
case '-c':
parsedOptions.cleanString = true;
break;
case '--minInstances':
case '-m':
var minInstances = parseInt(process.argv[idx + 1], 10);
if (minInstances && minInstances > 0) {
parsedOptions.minInstances = minInstances;
}
})();
break;
case '--':
parsedOptions.showHelp = false;
parseFiles = true;
break;
default:
break;
}
}
}, {
key: 'generateNewVariable',
value: function generateNewVariable(value) {
var newDeclarator = {
type: 'VariableDeclarator',
id: {
type: 'Identifier',
name: 'dedupe' + this.__generatedCount++
},
init: {
type: 'Literal',
value: value,
raw: '"' + value + '"'
}
};
this.__variablesDeclarations.declarations.push(newDeclarator);
return newDeclarator;
}
}], [{
key: 'cleanString',
value: function cleanString(value) {
return value.replace(/\s+/g, ' ');
}
}]);
return Dedupe;
}();
Object.defineProperty(exports, "__esModule", { value: true });
exports.default = Dedupe;
});
return parsedOptions;
}

35

package.json
{
"name": "de-dupe",
"bin": ".bin/de-dupe",
"version": "0.0.5",
"version": "0.0.6",
"description": "Deduplicate strings from javascript assets",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"build": "tsc && npm run build:index && npm run build:cli",
"build:cli": "babel cli.js > cli-tmp.js && mv -f cli-tmp.js cli.js",
"build:index": "babel index.js > index-tmp.js && mv -f index-tmp.js index.js"
"prepublish": "npm test && npm run build",
"lint": "tslint -c tslint.json ./src/**/*.ts",
"test": "npm run lint && rollup -c -o tests.js -i tests/index.ts && mocha tests.js",
"build": "rollup -c"
},
"repository": {
"type": "git",
"url": "git+ssh://git@github.com/markistaylor/de-dupe.git"
"url": "git+ssh://git@github.com/markis/de-dupe.git"
},

@@ -26,20 +26,17 @@ "keywords": [

"bugs": {
"url": "https://github.com/markistaylor/de-dupe/issues"
"url": "https://github.com/markis/de-dupe/issues"
},
"homepage": "https://github.com/markistaylor/de-dupe#readme",
"homepage": "https://github.com/markis/de-dupe#readme",
"dependencies": {
"acorn": "^3.0.4",
"escodegen": "^1.8.0",
"escope": "^3.6.0",
"esmangle": "^1.0.1",
"esprima": "^2.7.2",
"esshorten": "^1.1.1",
"estraverse": "^4.2.0",
"uglify-js": "^2.6.2"
"typescript": "^2.2.1"
},
"devDependencies": {
"babel-cli": "^6.6.5",
"babel-preset-es2015": "^6.6.0",
"typescript": "^1.8.7"
"@types/chai": "^3.4.35",
"@types/mocha": "^2.2.40",
"@types/node": "^7.0.8",
"chai": "^3.5.0",
"mocha": "^3.2.0",
"rollup": "^0.41.5",
"rollup-plugin-typescript": "^0.8.1"
}
}

2

README.md
#De-dupe - a Javascript string minifier
[![Build Status](https://travis-ci.org/markis/de-dupe.svg?branch=master)](https://travis-ci.org/markis/de-dupe)
De-dupe is an asset minification process that will identify duplicate strings in all scopes of a javascript file and will introduce a variable instead of the string itself. It does not introduce variables on the global scope, it will keep the variables to the individual scopes that it identifies. It can also clean up strings so that they don't have large amounts of white space in them.

@@ -4,0 +6,0 @@

25

tsconfig.json
{
"compilerOptions": {
"module": "commonjs",
"target": "es6",
"noImplicitAny": false,
"sourceMap": false,
"declaration": true
},
"files": [
"cli.ts", "index.ts"
"compilerOptions": {
"module": "commonjs",
"target": "es5",
"outDir": "tmp",
"lib": [ "es6" ],
"typeRoots": [
"./node_modules/@types/",
"./types/"
],
"exclude": [
"node_modules"
]
"noImplicitAny": true,
"sourceMap": false,
"declaration": true,
"strictNullChecks": true
}
}
.babelrc
cli.d.ts
cli.js
cli.ts
index.d.ts
index.ts
interfaces/_global.d.ts
interfaces/acorn.d.ts
interfaces/escodegen.d.ts
interfaces/escope.d.ts
interfaces/esmangle.d.ts
interfaces/esprima.d.ts
interfaces/estraverse.d.ts
interfaces/estree.d.ts
interfaces/node.d.ts
interfaces/uglify.d.ts
.bin/de-dupe

Sorry, the diff of this file is not supported yet

.npmignore

Sorry, the diff of this file is not supported yet

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc