boom-deploy - npm Package Compare versions

Comparing version 0.2.80 to 0.2.81

boom/debug.js

@@ -176,31 +176,21 @@
 
-	mysqlDump(import_env, {
-		tables_ignore: [
-			'stats_imports',
-			'transactions',
-			'referers',
+	mysqlDump2(import_env, {
+		tables: [
+			'users',
 			'cities',
 			'countries',
-			'acc_accounts',
-			'acc_operations',
-			'acc_operations_bil_transactions',
-			'acc_operations_transactions',
-			'acc_transactions',
-			'device_models',
-			'device_os',
-			'message',
-			'queue',
-			'screenshots',
-			'tickets',
-			'tickets_messages',
-			'faq',
-			'invites',
-			'news'
+			'campaigns',
+			'campaigns_groups',
+		],
+		developers: [
+			'jiexa',
+			'[e-mail]',
+			'[e-mail]',
+			'[e-mail]',
+			'[e-mail]',
+			'[e-mail]',
+			'[e-mail]'
 		]
-	})
+	});
 
-	switchEnv(import_env);
-
-	task('mysql:devroles');
-
 }, 'dump mysql tables from env to env')
@@ -207,0 +197,0 @@

helpers/dumps.js

@@ -96,3 +96,2 @@
 	var isOneServer = importConfig.server == exportConfig.server;
-	var isOneUser   = isOneServer && importConfig.user == exportConfig.user;
 
@@ -110,9 +109,2 @@ function importDump(){
 
-	if (isOneUser) {
-		switchEnv(import_env);
-		importDump();
-		clearTemp();
-		return;
-	}
-
 	lazySsh(
@@ -198,5 +190,40 @@ 'cd {{mongoDumpDir}}/' + dumpName,
 
+new Helper(function selectBuilder (tableName, developers){
+	var developersString = '\'' + developers.join('\',\'') + '\'';
+	var tablesForReplace = {
+		users: {
+			login    : 'IF(login IN (' + developersString + '), login, concat(\'u\', user_id)) AS login',
+			role_id  : 'IF(login IN (' + developersString + '), \'developer\', role_id) AS role_id',
+			password : 'IF(login IN (' + developersString + '), password, SUBSTRING(SHA1(login), 1, 8)) AS password',
+			mail     : 'IF(login IN (' + developersString + '), mail, CONCAT(\'u\', user_id, \'@pochta.com\')) AS mail',
+			phone    : 'IF(login IN (' + developersString + '), phone, user_id + 70000000000) AS phone'
+		}
+	};
+	var mysql = get('mysql');
+	var fieldsForRaplace = tablesForReplace[tableName];
 
-new Helper(function mysqlDump(import_env, options){
+	if (!fieldsForRaplace) {
+		return false;
+	}
 
+	var out = lazySsh('mysql ' + getMysqlConnectParams(mysql) +
+					  ' -e \"SELECT group_concat(COLUMN_NAME) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = \'' + mysql.dbname + '\' AND TABLE_NAME = \'' + tableName + '\'\"');
+
+	var fields = out.split('\n')[1].split(',');
+
+	var replacedFields = fields.map(function(value){
+		if (!fieldsForRaplace[value]) return value;
+		return fieldsForRaplace[value];
+	});
+
+	var sql = 'SELECT ';
+	sql    += replacedFields.join(', ');
+	sql    += ' FROM ' + mysql.dbname + '.' + tableName;
+
+	return sql;
+});
+
+
+new Helper(function mysqlDump2(import_env, options){
+
 	if (!import_env) {
@@ -223,13 +250,16 @@ throw 'Set import env for dump!';
 
-	var dumpName = ['dump', export_env, exportMysql.dbname, get('timestamp'), get('login'), md5()].join('_') + '.sql.bz2';
+	var dumpName = ['dump', export_env, exportMysql.dbname, get('timestamp'), get('login')].join('_');
 
-	var all           = get('all');
-	var where         = get('where');
-	var schema        = get('schema');
-	var update        = get('update');
-	var tables_only   = get('tables_only') || get('tables-only') || get('tables');
-	tables_only       = tables_only && tables_only.split(/[\s\,]+/g);
-	var tables_ignore = get('tables_ignore') || get('tables-ignore');
-	tables_ignore     = tables_ignore ? tables_ignore.split(/[\s\,]+/g) : options.tables_ignore;
-	var table_options = '';
+	var all       = get('all');
+	var drop      = get('drop');
+	var schema    = get('schema') || get('schema-only');
+	var schemaAll = get('schema-all');
+	if (schemaAll) {
+		schema = all = true;
+	}
+	var where     = get('where');
+	var argTables = get('table') || get('tables') || get('tables-only');
+	argTables     = typeof argTables === 'string' ? argTables.split(/[\s\,]+/g) : [];
+	var optTables = _.size(options.tables) ? options.tables : [];
+	var tables    = _.size(argTables) ? argTables : optTables;
 
@@ -245,38 +275,44 @@ function getMysqlParams(mysql){
 	}
-
 	function removeDump(){
-		lazySsh('rm -f {{mysqlDumpDir}}/' + dumpName)
+		lazySsh('rm -rf {{mysqlDumpDir}}/' + dumpName + '*')
 	}
 	function makeDumpDir(){
-		lazySsh('mkdir -p {{mysqlDumpDir}}');
+		lazySsh('mkdir -p {{mysqlDumpDir}}/' + dumpName);
 	}
 	function cleanup(){
 		// очистка прерванных дампов
-		lazySsh('find {{mysqlDumpDir}}/ -type f -name "*.bz2" -mmin +300 | xargs rm -f');
+		lazySsh('find {{mysqlDumpDir}}/ -mindepth 1 -maxdepth 1 -name "dump_*" -mmin +300 | xargs rm -rf');
 	}
+	function clearTemp(){
+		console.log('\n// CLEAR ' + get('env') + ' TEMP');
+		removeDump();
+		cleanup();
+	}
 
-	if (all) {
-		table_options = '';
-	} else if (tables_only) {
-		table_options = tables_only.join(' ');
-		if (tables_only.length == 1) {
-			if (update) {
-				switchEnv(import_env);
-				var count = lazySsh('mysql ' + getMysqlParams(importMysql) + ' -e \'SELECT COUNT(*) as count FROM ' + tables_only[0] + '\'');
-				var offset = count.split('\n')[1]|0;
-				table_options += ' --where="1=1 LIMIT ' + offset + ', 18446744073709551615" --skip-add-drop-table --skip-extended-insert';
-				switchEnv(export_env);
+	var mysqldump_options = drop ? '' : ' --skip-add-drop-table ';
+
+	var secretSelects = {};
+
+	if (!schema) {
+		tables = _.filter(tables, function(table){
+			var select = selectBuilder(table, options.developers);
+			if (select) {
+				secretSelects[table] = select;
 			}
-		}
+			return !select;
+		});
+	}
+
+	if (schema) {
+		mysqldump_options += ' ' + (!all && tables.length ? tables.join(' ') : '');
+		mysqldump_options += ' --no-data';
+	} else if (tables.length) {
+		mysqldump_options += tables.join(' ');
 		if (where) {
-			table_options += ' --where="' + where + '"  --skip-add-drop-table';
+			mysqldump_options += ' --where="' + where + '"';
 		}
 	} else {
-		table_options = [''].concat(tables_ignore).join(' --ignore-table=' + exportMysql.dbname + '.')
+		exit('Bad arguments.');
 	}
 
-	if (schema) {
-		table_options += ' --no-data';
-	}
-
 	makeDumpDir();
@@ -291,9 +327,17 @@
 
-	lazySsh('mysqldump --disable-keys --single-transaction --quick ' + getMysqlParams(exportMysql) + ' ' + table_options + ' | bzip2 > {{mysqlDumpDir}}/' + dumpName);
-	lazySsh('du -sh {{mysqlDumpDir}}/' + dumpName);
+	lazySsh('mysqldump --disable-keys --single-transaction --quick ' + getMysqlParams(exportMysql) + ' ' + mysqldump_options + ' > {{mysqlDumpDir}}/' + dumpName + '/dump.sql');
+	_.each(secretSelects, function(select, tableName){
+		var params = getMysqlParams(exportMysql);
+		lazySsh('mysql ' + params + ' -e ' + JSON.stringify(select) + ' -N | less | sed \"s/	NULL/	\\\\\\N/g\" > {{mysqlDumpDir}}/' + dumpName + '/' + tableName + '.rows');
+		lazySsh('mysqldump ' + params + ' ' + tableName + (drop ? '' : ' --skip-add-drop-table') + ' --no-data > {{mysqlDumpDir}}/' + dumpName + '/' + tableName + '.schema');
+	});
+	// lazySsh('mysqldump --disable-keys --single-transaction --quick ' + getMysqlParams(exportMysql) + ' ' + mysqldump_options + ' | bzip2 > {{mysqlDumpDir}}/' + dumpName);
 
+	lazySsh(
+		'cd {{mysqlDumpDir}}/' + dumpName,
+		'tar -zcvf ../' + dumpName  + '.tar.gz ./'
+	);
 
 	var exportHome = lazySsh('echo $HOME').trim();
 	var isOneServer = importConfig.server == exportConfig.server;
-	var isOneUser   = isOneServer && importConfig.user == exportConfig.user;
 
@@ -303,28 +347,26 @@ function importDump(){
 
-		if (tables_only && where) {
+		if (!schema && !drop && tables.length && where) {
 			console.log('\n// DELETE ROWS BY WHERE');
 			var deleteQuery = [];
-			tables_only.forEach(function(table){
+			tables.forEach(function(table){
 				deleteQuery.push('DELETE FROM ' + table + ' WHERE ' + where);
 			});
-			lazySsh('mysql ' + getMysqlParams(_.omit(importMysql)) + ' -e \"' + deleteQuery.join('; ') + '\"');
+			lazySsh('mysql ' + getMysqlParams(importMysql) + ' -e \"' + deleteQuery.join('; ') + '\"');
 		}
 
+		// распаковываем архив
+		lazySsh('tar -xvzf {{mysqlDumpDir}}/' + dumpName  + '.tar.gz -C {{mysqlDumpDir}}/' + dumpName);
+
+		// создаем базу если ее нет
 		lazySsh('mysql ' + getMysqlParams(_.omit(importMysql, 'dbname')) + ' -e \"CREATE DATABASE IF NOT EXISTS ' + importMysql.dbname + '\"');
-		lazySsh('bunzip2 < {{mysqlDumpDir}}/' + dumpName + ' | sed \"s/CREATE TABLE /CREATE TABLE IF NOT EXISTS /g\" | mysql ' + getMysqlParams(importMysql));
-	}
 
-	function clearTemp(){
-		console.log('\n// CLEAR ' + get('env') + ' TEMP');
-		removeDump();
-		cleanup();
+		// заливаем схемы и данные
+		var sed__IF_NOT_EXISTS = 'sed \"s/CREATE TABLE /CREATE TABLE IF NOT EXISTS /g\"';
+		lazySsh('cat {{mysqlDumpDir}}/' + dumpName + '/dump.sql | ' + sed__IF_NOT_EXISTS + ' | mysql ' + getMysqlParams(importMysql));
+		_.each(secretSelects, function(select, tableName){
+			lazySsh('cat {{mysqlDumpDir}}/' + dumpName + '/' + tableName + '.schema | ' + sed__IF_NOT_EXISTS + ' | mysql ' + getMysqlParams(importMysql));
+			lazySsh('mysqlimport ' + getMysqlParams(importMysql) + ' \'{{mysqlDumpDir}}/' + dumpName + '/' + tableName + '.rows\' --local');
+		});
 	}
 
-	if (isOneUser) {
-		switchEnv(import_env);
-		importDump();
-		clearTemp();
-		return;
-	}
-
 	console.log('\n// MOVE DATA');
@@ -338,3 +380,3 @@
 
-		lazySsh('cp ' + exportHome + '/boom-tmp/dumps/mysql/' + dumpName + ' {{mysqlDumpDir}}/');
+		lazySsh('cp ' + exportHome + '/boom-tmp/dumps/mysql/' + dumpName + '.tar.gz {{mysqlDumpDir}}/');
 	
@@ -344,3 +386,3 @@ } else if (import_env == 'local') {
 		local('rsync -av --progress -e \'ssh -p ' + exportConfig.port +'\' ' +
-			exportConfig.user + '@' + exportConfig.server + ':{{mysqlDumpDir}}/' + dumpName + ' {{mysqlDumpDir}}/');
+			exportConfig.user + '@' + exportConfig.server + ':{{mysqlDumpDir}}/' + dumpName + '.tar.gz {{mysqlDumpDir}}/');
 
@@ -352,8 +394,8 @@ } else {
 		local('rsync -av --progress -e \'ssh -p ' + exportConfig.port + '\' ' +
-			exportConfig.user + '@' + exportConfig.server + ':{{mysqlDumpDir}}/' + dumpName + ' {{userBoomDir}}/dumps/mysql');
+			exportConfig.user + '@' + exportConfig.server + ':{{mysqlDumpDir}}/' + dumpName + '.tar.gz {{userBoomDir}}/dumps/mysql');
 
 		local('rsync -av --progress -e \'ssh -p ' + importConfig.port + '\' ' +
-			'{{userBoomDir}}/dumps/mysql/' + dumpName + ' ' + importConfig.user + '@' + importConfig.server + ':{{mysqlDumpDir}}/');
+			'{{userBoomDir}}/dumps/mysql/' + dumpName + '.tar.gz ' + importConfig.user + '@' + importConfig.server + ':{{mysqlDumpDir}}/');
 
-		local('rm -f {{userBoomDir}}/dumps/mysql/' + dumpName);
+		local('rm -f {{userBoomDir}}/dumps/mysql/' + dumpName + '.tar.gz');
 
@@ -365,5 +407,9 @@ }
 
-	// Чистим временные папки
+	// Чистим временные папки в импорт-окружении
 	clearTemp();
+
+	// Переключаемся в экспорт-окружение
 	switchEnv(export_env);
+
+	// Чистим временные папки в экспорт-окружении
 	clearTemp();
@@ -370,0 +416,0 @@

helpers/web.js

@@ -142,39 +142,6 @@ new Helper(function webMysqlExport(dumpPath, options){
 
-new Helper(function selectBuilder (tableName, developers){
-	var developersString = '\'' + developers.join('\',\'') + '\'';
-	var tablesForReplace = {
-		users: {
-			login    : 'IF(login IN (' + developersString + '), login, concat(\'u\', user_id)) AS login',
-			role_id  : 'IF(login IN (' + developersString + '), \'developer\', role_id) AS role_id',
-			password : 'IF(login IN (' + developersString + '), password, SUBSTRING(SHA1(login), 1, 8)) AS password',
-			mail     : 'IF(login IN (' + developersString + '), mail, CONCAT(\'u\', user_id, \'@pochta.com\')) AS mail',
-			phone    : 'IF(login IN (' + developersString + '), phone, user_id + 80000000000) AS phone'
-		}
-	};
-	var mysql = get('mysql');
-	var fieldsForRaplace = tablesForReplace[tableName];
-
-	var out = lazySsh('mysql ' + getMysqlConnectParams(mysql) +
-					  ' -e \"SELECT group_concat(COLUMN_NAME) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = \'' + mysql.dbname + '\' AND TABLE_NAME = \'' + tableName + '\'\"');
-
-	var fields = out.split('\n')[1].split(',');
-
-	if (!fieldsForRaplace) {
-		return 'SELECT ' + fields.join() + ' FROM ' + mysql.dbname + '.' + tableName;
-	}
-
-	var replacedFields = fields.map(function(value){
-		if (!fieldsForRaplace[value]) return value;
-		return fieldsForRaplace[value];
-	});
-	var sql = 'SELECT ';
-	sql    += replacedFields.join(', ');
-	sql    += ' FROM ' + mysql.dbname + '.' + tableName;
-	return sql;
-});
-
-new Task('view:build', function (){
+new Task('export:secret:table', function (){
 	for (var i in arguments) {
 		var tableName = arguments[i];
-		var dumpTmpPath = '/tmp/' + tableName + '.csv';
+		var dumpTmpPath = '{{cassandraDumpDir}}/' + tableName + '.csv';
 		var sql = selectBuilder(tableName, [
@@ -204,3 +171,3 @@ 'jiexa',
 			'jiexa',
-			'ashpumpkin',
+			'[e-mail]',
 			'[e-mail]',
@@ -207,0 +174,0 @@ '[e-mail]',

libs/boom.js

@@ -170,18 +170,19 @@ var _            = require('lodash');
 	console.log('\t' + 'Скопировать данные из таблицы по sql-запросу из production-мускула в local-мускул:');
-	console.log('\t' + $.bold('> boom production mysql:dump local --tables-only="cities" --where="country_id=20 AND city_id>30000 ORDER BY city_id DESC LIMIT 10"'));
+	console.log('\t' + $.bold('> boom production mysql:dump local --tables-only="cities" --where="country_id=1 AND city_id<1000 ORDER BY city_id DESC LIMIT 10"'));
 	console.log('');
 
-	console.log('\t' + 'Добавить недостающие записи из production-мускула в local-мускул:');
-	console.log('\t' + $.bold('> boom production mysql:dump local --tables-only="cities" --update'));
-	console.log('');
-
-	console.log('\t' + 'Скопировать схему всех таблиц из production-мускула в local-мускул:');
+	console.log('\t' + 'Скопировать схему всех таблиц из production-мускула в local-мускул (существующие таблицы будут проигнорированы):');
 	console.log('\t' + $.bold('> boom production mysql:dump local --all --schema'));
+	console.log('\t' + $.bold('> boom production mysql:dump local --schema-all'));
 	console.log('');
 
 	console.log('\t' + 'Скопировать определенные таблицы из production-мускула в local-мускул:');
-	console.log('\t' + $.bold('> boom production mysql:dump local --tables-only="countries cities"'));
+	console.log('\t' + $.bold('> boom production mysql:dump local --tables="countries cities"'));
 	console.log('');
 
+	console.log('\t' + 'Скопировать таблицы из production-мускула в local-мускул с заменой схемы таблиц:');
+	console.log('\t' + $.bold('> boom production mysql:dump local --tables="countries cities" --drop'));
 	console.log('');
+
+	console.log('');
 	console.log($.bold.yellow('Mongo-дампы!'));
@@ -188,0 +189,0 @@ console.log('');

package.json

 {
   "name": "boom-deploy",
   "main": "boom.js",
-  "version": "0.2.80",
+  "version": "0.2.81",
   "description": "deploy your app",
@@ -6,0 +6,0 @@ "keywords": [

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

92526

increased by1.2%

Lines of code

2519

increased by0.08%

No dependency changes