elasticsearch-tools - npm Package Compare versions

Comparing version 2.0.0 to 2.1.0

lib/es-export-bulk.js

@@ -6,2 +6,3 @@ #!/usr/bin/env node
 var fs = require('fs');
+var vm = require('vm');
 var program = require('commander');
@@ -11,3 +12,3 @@ var elasticsearch = require('elasticsearch');
 var ProgressBar = require('progress');
-var bar;
+var bar, key;
 
@@ -25,3 +26,3 @@ var esOptions = {
 	scroll: 'specify how long a consistent view of the index should be maintained for scrolled search (default: 1m)',
-	size: 'number of hits to return',
+	size: 'number of hits to return during each scan',
 	sort: 'a comma-separated list of <field>:<direction> pairs',
@@ -35,5 +36,8 @@ timeout: 'explicit operation timeout'
 	.option('-u, --url <url>', 'the elasticsearch url to connect to')
-	.option('-f, --file <file>', 'the file to write data to');
+	.option('-f, --file <file>', 'the file to write data to')
+	.option('-m, --max <number>', 'the maximum number of items to export. different than the scroll size', parseInt)
+	.option('--transformMeta <js>', 'a javascript function that returns an object')
+	.option('--transformSource <js>', 'a javascript function that returns an object');
 // add es options
-for (var key in esOptions) {
+for (key in esOptions) {
 	if (esOptions.hasOwnProperty(key)) {
@@ -63,3 +67,3 @@ program.option('--' + key + ' <' + key + '>', 'ES OPTION: ' + esOptions[key]);
 };
-for (var key in esOptions) {
+for (key in esOptions) {
 	if (esOptions.hasOwnProperty(key) && program.hasOwnProperty(key)) {
@@ -73,6 +77,15 @@ search[key] = program[key];
 var processResults = function (error, response) {
-	var content = '', index, scrollOptions;
-	if (error) {
+	var content = '', hitMax = false, index, scrollOptions;
+	if (error && typeof response === 'string') {
+		console.log('\nattempting to parse invalid json returned from elasticsearch server');
+		response = vm.runInThisContext('(function () {return ' + response + ';}());');
+		if (typeof response !== 'object') {
+			helpers.exit('attempt to parse invalid json as javascript failed.');
+		}
+	} else if (error) {
 		helpers.exit(error);
 	}
+	if (response.hits.total === 0) {
+		helpers.exit('no results were returned, so exiting.');
+	}
 	// init progress bar if needed
@@ -89,4 +102,9 @@ if (!bar) {
 		var meta = {index:{}};
-		var doc = hit._source || {};
+		var source = hit._source || {};
 		var fields = hit.fields || {};
+		// if we passed in a max, stop processing
+		if (typeof program.max === 'number' && processed >= program.max) {
+			hitMax = true;
+			return;
+		}
 		// build meta
@@ -98,3 +116,3 @@ for (var key in hit) {
 		}
-		for (var key in fields) {
+		for (key in fields) {
 			if (fields.hasOwnProperty(key)) {
@@ -104,3 +122,20 @@ meta.index[key] = fields[key];
 		}
-		content += JSON.stringify(meta) + '\n' + JSON.stringify(doc) + '\n';
+		// transform meta
+		if (program.transformMeta) {
+			meta = vm.runInNewContext('(function () {' + program.transformMeta + ';return data;}());', {data: meta});
+		}
+		
+		// transform source
+		if (program.transformSource) {
+			source = vm.runInNewContext('(function () {' + program.transformSource + ';return data;}());', {data: source});
+		}
+
+		if (typeof meta !== 'object' || typeof source !== 'object') {
+			helpers.exit({
+				message: 'an invalid bulk item was created after transforming data',
+				meta: meta,
+				source: source
+			});
+		}
+		content += JSON.stringify(meta) + '\n' + JSON.stringify(source) + '\n';
 		processed++;
@@ -111,3 +146,3 @@ bar.tick();
 	// continue to scroll
-	if (response.hits.total !== processed) {
+	if (response.hits.total !== processed && !hitMax) {
 		scrollOptions = {scrollId: response._scroll_id};
@@ -114,0 +149,0 @@ if (program.scroll) {

lib/exporter.js

@@ -10,2 +10,3 @@ 'use strict';
 exports.indicesExport = function (fnName, exportKey, esOptions) {
+	var key;
 	// setup command line options
@@ -17,3 +18,3 @@ program
 	// add es options
-	for (var key in esOptions) {
+	for (key in esOptions) {
 		if (esOptions.hasOwnProperty(key)) {
@@ -34,3 +35,3 @@ program.option('--' + key + ' <' + key + '>', 'ES OPTION: ' + esOptions[key]);
 	var params = {};
-	for (var key in esOptions) {
+	for (key in esOptions) {
 		if (esOptions.hasOwnProperty(key) && program.hasOwnProperty(key)) {
@@ -37,0 +38,0 @@ params[key] = program[key];

lib/helpers.js

 // a few helper functions
+'use strict';
 
-exports.exit = function (str) {
-	str = str.toString().replace(/^error:/i, '');
-	console.error('  error: ' + str);
+exports.exit = function (err) {
+	var str = JSON.stringify(err).replace(/^error:/i, '');
+	console.error('\n  error: ' + str);
 	process.exit(1);
@@ -7,0 +8,0 @@ };

lib/importer.js

@@ -10,4 +10,4 @@ 'use strict';
 
-
 exports.indicesImport = function (fnName, importKey, alsoInclude, esOptions) {
+	var key;
 	// setup command line options
@@ -19,3 +19,3 @@ program
 	// add es options
-	for (var key in esOptions) {
+	for (key in esOptions) {
 		if (esOptions.hasOwnProperty(key)) {
@@ -36,3 +36,3 @@ program.option('--' + key + ' <' + key + '>', 'ES OPTION: ' + esOptions[key]);
 	var params = {};
-	for (var key in esOptions) {
+	for (key in esOptions) {
 		if (esOptions.hasOwnProperty(key) && program.hasOwnProperty(key)) {
@@ -45,2 +45,3 @@ params[key] = program[key];
 	fs.readFile(program.file, 'utf-8', function (err, contents) {
+		var parsedContent;
 		if (err) {
@@ -50,3 +51,3 @@ helpers.exit(err);
 		try {
-			var parsedContent = JSON.parse(contents);
+			parsedContent = JSON.parse(contents);
 		} catch (err) {
@@ -53,0 +54,0 @@ helpers.exit('Cannot parse file: ' + err.toString());

package.json

 {
   "name": "elasticsearch-tools",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Elasticsearch command line tools for importing, exporting, etc",
@@ -48,3 +48,8 @@ "homepage": "http://github.com/skratchdot/elasticsearch-tools",
     "schema"
-  ]
-}
+  ],
+  "devDependencies": {
+    "async": "^0.9.0",
+    "gulp": "^3.8.7",
+    "gulp-jshint": "^1.8.4"
+  }
+}

README.md

@@ -32,2 +32,288 @@ # elasticsearch-tools
 
+## Usage: es-export-bulk
+
+### Options
+
+```bash
+es-export-bulk --help
+
+Usage: es-export-bulk [options]
+
+  Options:
+
+    -h, --help                           output usage information
+    -v, --version                        output the version number
+    -u, --url <url>                      the elasticsearch url to connect to
+    -f, --file <file>                    the file to write data to
+    -m, --max <number>                   the maximum number of items to export. different than the scroll size
+    --transformMeta <js>                 a javascript function that returns an object
+    --transformSource <js>               a javascript function that returns an object
+    --index <index>                      ES OPTION: a comma-separated list of index names to search; use _all or empty string to perform the operation on all indices
+    --type <type>                        ES OPTION: a comma-separated list of document types to search; leave empty to perform the operation on all types
+    --body <body>                        ES OPTION: the body to send along with this request.
+    --analyzer <analyzer>                ES OPTION: The analyzer to use for the query string
+    --analyzeWildcard <analyzeWildcard>  ES OPTION: specify whether wildcard and prefix queries should be analyzed (default: false)
+    --fields <fields>                    ES OPTION: a comma-separated list of fields to return as part of a hit (default: "*")
+    --from <from>                        ES OPTION: starting offset (default: 0)
+    --q <q>                              ES OPTION: query in the Lucene query string syntax
+    --routing <routing>                  ES OPTION: a comma-separated list of specific routing values
+    --scroll <scroll>                    ES OPTION: specify how long a consistent view of the index should be maintained for scrolled search (default: 1m)
+    --size <size>                        ES OPTION: number of hits to return during each scan
+    --sort <sort>                        ES OPTION: a comma-separated list of <field>:<direction> pairs
+    --timeout <timeout>                  ES OPTION: explicit operation timeout
+```
+
+### Examples
+
+#### export 1 hour of data from local db
+```bash
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --body '
+{"query":{"range":{"timestamp":{"gte":"2014-08-13T11:00:00.000Z","lte":"2014-08-13T12:00:00.000Z"}}}}
+'
+```
+
+#### export "myIndex" from local db
+```bash
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --index myIndex
+```
+
+#### only export 42 documents
+```bash
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --max 42
+```
+
+#### only export 42 documents, grabbing/scanning 5 documents at a time
+```bash
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --max 42 --size 5
+```
+
+#### add a key/value to all exported documents
+```bash
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --transformSource 'data.foo = "neat"'
+# the return statement is optional
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --transformSource 'data.foo = "neat";return data;'
+```
+
+#### delete the key "foo" from all exported documents
+```bash
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --transformSource 'delete data.foo'
+```
+
+#### don't include _parent in meta data
+```bash
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --transformMeta 'delete data.index._parent'
+```
+
+#### change the index name and type that we export, so we can import into a different index/type
+```bash
+es-export-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json --transformMeta '
+data.index._index = "newIndex";
+data.index._type = "newType";
+'
+```
+
+
+## Usage: es-export-mappings
+
+### Options
+
+```bash
+es-export-mappings --help
+
+Usage: es-export-mappings [options]
+
+  Options:
+
+    -h, --help                               output usage information
+    -v, --version                            output the version number
+    -u, --url <url>                          the elasticsearch url to connect to
+    -f, --file <file>                        the file to write data to
+    --index <index>                          ES OPTION: String, String[], Boolean — A comma-separated list of index names
+    --type <type>                            ES OPTION: String, String[], Boolean — A comma-separated list of document types
+    --ignoreUnavailable <ignoreUnavailable>  ES OPTION: Boolean — Whether specified concrete indices should be ignored when unavailable (missing or closed)
+    --allowNoIndices <allowNoIndices>        ES OPTION: Boolean — Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes _all string or when no indices have been specified)
+    --expandWildcards <expandWildcards>      ES OPTION: String — Whether to expand wildcard expression to concrete indices that are open, closed or both.
+    --local <local>                          ES OPTION: Boolean — Return local information, do not retrieve the state from master node (default: false)
+```
+
+### Examples
+
+#### export mappings from local db
+```bash
+es-export-mappings --url http://localhost:9200 --file ~/backups/elasticsearch/prod/prod.mappings.json
+```
+
+
+## Usage: es-export-settings
+
+### Options
+
+```bash
+es-export-settings --help
+
+Usage: es-export-settings [options]
+
+  Options:
+
+    -h, --help                               output usage information
+    -v, --version                            output the version number
+    -u, --url <url>                          the elasticsearch url to connect to
+    -f, --file <file>                        the file to write data to
+    --index <index>                          ES OPTION: String, String[], Boolean — A comma-separated list of index names
+    --ignoreUnavailable <ignoreUnavailable>  ES OPTION: Boolean — Whether specified concrete indices should be ignored when unavailable (missing or closed)
+    --allowNoIndices <allowNoIndices>        ES OPTION: Boolean — Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes _all string or when no indices have been specified)
+    --expandWildcards <expandWildcards>      ES OPTION: String — Whether to expand wildcard expression to concrete indices that are open, closed or both.
+    --local <local>                          ES OPTION: Boolean — Return local information, do not retrieve the state from master node (default: false)
+    --name <name>                            ES OPTION: String, String[], Boolean — The name of the settings that should be included
+```
+
+### Examples
+
+#### export settings from local db
+```bash
+es-export-settings --url http://localhost:9200 --file ~/backups/elasticsearch/prod/prod.settings.json
+```
+
+
+## Usage: es-export-aliases
+
+### Options
+
+```bash
+es-export-aliases --help
+
+Usage: es-export-aliases [options]
+
+  Options:
+
+    -h, --help         output usage information
+    -v, --version      output the version number
+    -u, --url <url>    the elasticsearch url to connect to
+    -f, --file <file>  the file to write data to
+    --index <index>    ES OPTION: String, String[], Boolean — A comma-separated list of index names
+    --local <local>    ES OPTION: Boolean — Return local information, do not retrieve the state from master node (default: false)
+    --name <name>      ES OPTION: String, String[], Boolean — The name of the settings that should be included
+```
+
+### Examples
+
+#### export aliases from local db
+```bash
+es-export-aliases --url http://localhost:9200 --file ~/backups/elasticsearch/prod/prod.aliases.json
+```
+
+
+## Usage: es-import-bulk
+
+### Options
+
+```bash
+es-import-bulk --help
+
+Usage: es-import-bulk [options]
+
+  Options:
+
+    -h, --help         output usage information
+    -v, --version      output the version number
+    -u, --url <url>    the elasticsearch url to connect to
+    -f, --file <file>  the file to write data to
+    -m, --max <items>  the max number of lines to process per batch
+```
+
+### Examples
+
+#### import data to local db from file
+```bash
+es-import-bulk --url http://localhost:9200 --file ~/backups/elasticsearch/prod/data.json
+```
+
+
+## Usage: es-import-mappings
+
+### Options
+
+```bash
+es-import-mappings --help
+
+Usage: es-import-mappings [options]
+
+  Options:
+
+    -h, --help                               output usage information
+    -v, --version                            output the version number
+    -u, --url <url>                          the elasticsearch url to connect to
+    -f, --file <file>                        the file to write data to
+    --ignoreConflicts <ignoreConflicts>      ES OPTION: Boolean — Specify whether to ignore conflicts while updating the mapping (default: false)
+    --timeout <timeout>                      ES OPTION: Date, Number — Explicit operation timeout
+    --masterTimeout <masterTimeout>          ES OPTION: Date, Number — Specify timeout for connection to master
+    --ignoreUnavailable <ignoreUnavailable>  ES OPTION: Boolean — Whether specified concrete indices should be ignored when unavailable (missing or closed)
+    --allowNoIndices <allowNoIndices>        ES OPTION: Boolean — Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes _all string or when no indices have been specified)
+    --expandWildcards <expandWildcards>      ES OPTION: String — Whether to expand wildcard expression to concrete indices that are open, closed or both.
+```
+
+### Examples
+
+#### import mappings to local db
+```bash
+es-import-mappings --url http://localhost:9200 --file ~/backups/elasticsearch/prod/prod.mappings.json
+```
+
+
+## Usage: es-import-settings
+
+### Options
+
+```bash
+es-import-settings --help
+
+Usage: es-import-settings [options]
+
+  Options:
+
+    -h, --help                               output usage information
+    -v, --version                            output the version number
+    -u, --url <url>                          the elasticsearch url to connect to
+    -f, --file <file>                        the file to write data to
+    --masterTimeout <masterTimeout>          ES OPTION: Date, Number — Specify timeout for connection to master
+    --ignoreUnavailable <ignoreUnavailable>  ES OPTION: Boolean — Whether specified concrete indices should be ignored when unavailable (missing or closed)
+    --allowNoIndices <allowNoIndices>        ES OPTION: Boolean — Whether to ignore if a wildcard indices expression resolves into no concrete indices. (This includes _all string or when no indices have been specified)
+    --expandWildcards <expandWildcards>      ES OPTION: String — Whether to expand wildcard expression to concrete indices that are open, closed or both.
+```
+
+### Examples
+
+#### import settings to local db
+```bash
+es-import-settings --url http://localhost:9200 --file ~/backups/elasticsearch/prod/prod.settings.json
+```
+
+
+## Usage: es-import-aliases
+
+### Options
+
+```bash
+es-import-aliases --help
+
+Usage: es-import-aliases [options]
+
+  Options:
+
+    -h, --help                       output usage information
+    -v, --version                    output the version number
+    -u, --url <url>                  the elasticsearch url to connect to
+    -f, --file <file>                the file to write data to
+    --timeout <timeout>              ES OPTION: Date, Number — Explicit operation timeout
+    --masterTimeout <masterTimeout>  ES OPTION: Date, Number — Specify timeout for connection to master
+```
+
+### Examples
+
+#### import aliases to local db
+```bash
+es-import-aliases --url http://localhost:9200 --file ~/backups/elasticsearch/prod/prod.aliases.json
+```
+
+
 ## Other Elasticsearch Tools
@@ -34,0 +320,0 @@

New alerts

Shell access

Supply chain risk

This module accesses the system shell. Accessing the system shell increases the risk of executing arbitrary code.

Found 1 instance in 1 package

Debug access

Supply chain risk

Uses debug, reflection and dynamic code execution features.

Found 1 instance in 1 package

Fixed alerts

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

33519

increased by83.48%

Number of package files

17

increased by13.33%

Lines of code

492

increased by20.59%

Number of lines in readme file

329

increased by665.12%

Worsened metrics

Dev dependency count

3

increased byInfinity%

Number of low supply chain risk alerts

6

increased by50%

No dependency changes