git2consul - npm Package Compare versions

Comparing version 0.8.1 to 0.9.0

consul_data/raft/peers.json

@@ -1,1 +0,1 @@
-["172.17.6.27:8300"]
+["172.17.1.129:8300"]

lib/config_reader.js

@@ -1,2 +0,2 @@
-var consul = require('consul')({'host': global.endpoint}); 
+var consul = require('consul')({'host': global.endpoint});
 
@@ -3,0 +3,0 @@ var _ = require('underscore');

lib/consul/index.js

@@ -0,1 +1,2 @@
+var _ = require('underscore');
 var fs = require('fs');
@@ -10,2 +11,8 @@ var path = require('path');
 
+// This makes life a bit easier for expand_keys mode, allowing us to check for a .json
+// extension with less code per line.
+String.prototype.endsWith = function(suffix) {
+  return this.indexOf(suffix, this.length - suffix.length) !== -1;
+};
+
 /* istanbul ignore next */
@@ -36,2 +43,3 @@ exports.setToken = function(tok) {
   }
+
   // Finish with the file path
@@ -42,3 +50,42 @@ key_parts.push(file);
 
+
 /**
+ * Given an obj, recurse into it, populating the parts array with all of the key->value
+ * relationships, prefixed by parent objs.
+ * 
+ * For example, the obj { 'first': { 'second': { 'third' : 'whee' }}} should yield a
+ * parts array with a single entry: 'first/second/third' with value 'whee'.
+ */
+var render_obj = function(parts, prefix, obj) {
+
+  _.mapObject(obj, function(val, key) {
+    if (_.isArray(val)) return;
+
+    if (_.isObject(val)) return render_obj(parts, prefix + '/' + encodeURIComponent(key), val)
+
+    parts.push({'key': prefix + '/' + encodeURIComponent(key), 'value': val});
+  });
+}
+
+/**
+ * Walk through obj, creating a Consul write operation for each kv pair.  Objects are treated
+ * as parents of subtrees.  Arrays are ignored since they add annoying problems (multiple array
+ * entries can have the same value, so what do we do then?).
+ */
+var populate_kvs_from_json = function(branch, prefix, obj, cb) {
+
+  var writes = [];
+
+  render_obj(writes, prefix, obj);
+
+  logger.debug('JSON file %s yielded %s keys', prefix, writes.length);
+
+  cb = _.after(writes.length, cb);
+
+  writes.forEach(function(write) {
+    write_content_to_consul(write.key, write.value, cb);
+  });
+};
+
+/**
  * If a file was modified, read its new value and update consul's KV store.
@@ -52,8 +99,29 @@ */
 
-  fs.readFile(fqf, {encoding:'utf8'}, function(err, body) {
-    /* istanbul ignore if */
-    if (err) return cb('Failed to read key ' + fqf + ' due to ' + err);
-    var body = body ? body.trim() : '';
-    write_content_to_consul(create_key_name(branch, file), body, cb);
-  });
+  if (branch.expand_keys && file.endsWith('.json')) {
+    // Delete current tree.  Yeah, I know this is kinda the coward's way out, but it's a hell of a lot
+    // easier to get provably correct than diffing the file against the contents of the KV store.
+    file_deleted(branch, file, function(err) {
+      if (err) return cb('Failed to delete key ' + key_name + ' due to ' + err);
+
+      fs.readFile(fqf, {encoding:'utf8'}, function(err, body) {
+        /* istanbul ignore if */
+        if (err) return cb('Failed to read key ' + fqf + ' due to ' + err);
+        var body = body ? body.trim() : '';
+        try {
+          var obj = JSON.parse(body);
+          populate_kvs_from_json(branch, create_key_name(branch, file), obj, cb);
+        } catch(e) {
+          logger.warn("Failed to parse .json file.  Using body string as a KV.");
+          write_content_to_consul(create_key_name(branch, file), body, cb);
+        }
+      });
+    });
+  } else {
+    fs.readFile(fqf, {encoding:'utf8'}, function(err, body) {
+      /* istanbul ignore if */
+      if (err) return cb('Failed to read key ' + fqf + ' due to ' + err);
+      var body = body ? body.trim() : '';
+      write_content_to_consul(create_key_name(branch, file), body, cb);
+    });
+  }
 };
@@ -70,3 +138,4 @@
 
-  consul.kv.del({'key': key_name, token: token}, function(err) {
+  // Delete this key.  Or, if mode is branch.expand_keys, delete all files underneath this key.
+  consul.kv.del({'key': key_name, token: token, recurse: branch.expand_keys}, function(err) {
     /* istanbul ignore if */
@@ -208,1 +277,2 @@ if (err) return cb('Failed to delete key ' + key_name + ' due to ' + err);
 };
+

lib/git/branch.js

@@ -18,2 +18,3 @@ var fs = require('fs');
   Object.defineProperty(this, 'branch_directory', {value: this.branch_parent + path.sep + name});
+  Object.defineProperty(this, 'expand_keys', { value: repo_config['expand_keys'] === true });
   Object.defineProperty(this, 'include_branch_name', {
@@ -20,0 +21,0 @@ // If include_branch_name is not set, assume true.  Otherwise, identity check the value against true.

lib/index.js

@@ -63,2 +63,3 @@ var logging = require('./logging.js');
       config['local_store'] = process.argv[i+1];
+      ++i;
     }
@@ -65,0 +66,0 @@ }

package.json

 {
   "name": "git2consul",
   "description": "System for moving data from git to consul",
-  "version": "0.8.1",
+  "version": "0.9.0",
   "contributors": [
@@ -11,3 +11,3 @@ {
   ],
-  "repository" : {
+  "repository": {
     "type": "git",
@@ -20,7 +20,7 @@ "url": "git://github.com/Cimpress-MCP/git2consul.git"
     "consul": "^0.10.0",
+    "coveralls": "^2.11.2",
     "express": "~4.6.1",
     "mkdirp": "0.5.0",
-    "coveralls": "^2.11.2",
     "rimraf": "2.2.8",
-    "underscore": "1.6.0"
+    "underscore": "^1.8.0"
   },
@@ -27,0 +27,0 @@ "devDependencies": {

readme.md

@@ -9,2 +9,6 @@ #### git2consul
 
+##### Mailing List
+
+[Google Groups](https://groups.google.com/group/git2consul-tool/)
+
 ##### Requirements / Caveats
@@ -17,48 +21,98 @@
 
+##### Quick Start Guide
+
+Let's start off with a simple example to show you how it works.  You can use this as a starting point and then tailor it to your use-case.
+
+I've created a [simple repo with a few sample configuration files of different types](https://github.com/ryanbreen/git2consul_data/tree/dev).  Of course, I could have used thousands of files with arbitrarily nested directories, but this is a quick start guide.
+
+The most minimalistic viable git2consul configuration mirrors a single git repo into the KV store with a given prefix.  Here's how that would look mirroring the dev branch at `https://github.com/ryanbreen/git2consul_data.git` into the Consul K/V store with prefix `sample_configuration`:
+
+```javascript
+{
+  "version": "1.0",
+  "repos" : [{
+    "name" : "sample_configuration",
+    "url" : "https://github.com/ryanbreen/git2consul_data.git",
+    "branches" : ["dev"],
+    "hooks": [{
+      "type" : "polling",
+      "interval" : "1"
+    }]
+  }]
+}
+```
+
+Put that configuration in a file called `/tmp/git2consul.json`.  From the git2consul directory, upload that JSON file into the KV as your git2consul config:
+
+```
+node utils/config_seeder.js /tmp/git2consul.json
+```
+
+Start git2consul:
+
+```
+node .
+```
+
+git2consul will now poll the "dev" branch of the "git2consul_data.git" repo once per minute.  On first run, it will mirror the 3 files into your Consul K/V with keys:
+
+```
+/sample_configuration/dev/sample.conf
+/sample_configuration/dev/sample.json
+/sample_configuration/dev/sample.yaml
+```
+
+The Values of those Keys are the contents of the respective files.  Changing the contents of that git branch will change the corresponding KVs within 1 minute.
+
+Once you are happy with your configuration, you can run git2consul as a daemon either in a `screen` session or via an init script of whatever type is appropriate on your platform.
+
 ##### Configuration
 
-git2consul expects to be run on the same node as a Consul agent.  git2consul expects its own configuration to be stored as a JSON object in '/git2consul/config' in your Consul KV.  The utility `utils/config_seeder.js` will take a JSON file and place it in the correct location for you.
+git2consul expects to be run on the same node as a Consul agent.  git2consul expects its own configuration to be stored as a JSON object in '/git2consul/config' in your Consul KV.  The utility `utils/config_seeder.js` will take a JSON file and set `/git2consul/config` to contain its contents.
 
 ###### Configuration Format
 
+```javascript
+{
+  "version": "1.0",
+  "local_store": "/var/lib/git2consul_cache",
+  "logger" : {
+    "name" : "git2consul",
+    "streams" : [{
+      "level": "trace",
+      "stream": "process.stdout"
+    },
     {
-      "version": "1.0",
-      "local_store": "/var/lib/git2consul_cache",
-      "logger" : {
-        "name" : "git2consul",
-        "streams" : [{
-          "level": "trace",
-          "stream": "process.stdout"
-        },
-        {
-          "level": "debug",
-          "type": "rotating-file",
-          "path": "/var/log/git2consul/git2consul.log"
-        }]
-      },
-      "repos" : [{
-        "name" : "vp_config",
-        "url" : "ssh://stash.mydomain.com/team_configuration_data.git",
-        "include_branch_name" : false,
-        "branches" : ["development", "staging", "production"],
-        "hooks": [{
-          "type" : "stash",
-          "port" : "5050",
-          "url" : "/gitpoke"
-        },
-        {
-          "type" : "polling",
-          "interval" : "1"
-        }]
-      },{
-        "name" : "github_data",
-        "url" : "[e-mail]:ryanbreen/git2consul_data.git",
-        "branches" : [ "master" ],
-        "hooks": [{
-          "type" : "github",
-          "port" : "5151",
-          "url" : "/gitpoke"
-        }]
-      }]
-    }
+      "level": "debug",
+      "type": "rotating-file",
+      "path": "/var/log/git2consul/git2consul.log"
+    }]
+  },
+  "repos" : [{
+    "name" : "vp_config",
+    "url" : "ssh://stash.mydomain.com/team_configuration_data.git",
+    "include_branch_name" : false,
+    "branches" : ["development", "staging", "production"],
+    "hooks": [{
+      "type" : "stash",
+      "port" : "5050",
+      "url" : "/gitpoke"
+    },
+    {
+      "type" : "polling",
+      "interval" : "1"
+    }]
+  },{
+    "name" : "github_data",
+    "mode" : "expand_keys",
+    "url" : "[e-mail]:ryanbreen/git2consul_data.git",
+    "branches" : [ "master" ],
+    "hooks": [{
+      "type" : "github",
+      "port" : "5151",
+      "url" : "/gitpoke"
+    }]
+  }]
+}
+```
 
@@ -71,18 +125,52 @@ The above example illustrates a 2 repo git2consul setup: one repo lives in an on-premises Git solution and the other is hosted at github.  The hooks array under each repository defines how git2consul will be notified of changes.  git2consul supports [Atlassian Stash](https://confluence.atlassian.com/display/STASH/POST+service+webhook+for+Stash), [Atlassian Bitbucket](https://confluence.atlassian.com/display/BITBUCKET/POST+hook+management), [GitHub](https://developer.github.com/v3/repos/hooks/), and [Gitlab](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/web_hooks/web_hooks.md) webhooks as well as a basic polling model.
 
-###### No Daemon Mode
+##### How it works
 
+git2consul uses the name and branches of configured repos to namespace the created KVs.  The goal is to allow multiple teams to use the same Consul agents and KV store to migrate configuration data around a network without needing to worry about data conflicts.  In the above example, a settings file stored at `foo_service/settings.json` in the `development` branch of the repo `vp_config` would be persisted in Consul as `vp_config/development/foo_service/settings.json`.
+
+If you are using a more [Twelve-Factor](http://12factor.net/) approach, where you wish to configure your applications via environment variables, you would store these settings as files in Git whose name is the key and whose body is the value.  For example, we could create the file `foo_service/log_level` with the body `trace` in the `development` branch of the `foo_service` repo and git2consul will create the KV `vp_config/development/foo_service/log_level` with the value `trace`.
+
+As changes are detected in the specified Git repos, git2consul determines which files have been added, updated, or deleted and replicates those changes to the KV.  Because only changed branches and files are analyzed, git2consul should have a very slim profile on hosting systems.
+
+##### Alternative Modes of Operation
+
+###### No Daemon
+
 If there are no webhooks or polling watchers configured, git2consul will terminate as soon as all tracked repos and branches have been synced with Consul.  If you would like to force git2consul not to attach any webhooks or polling watchers, you can either pass the command-line switch `-n` or include the field `"no_daemon": true` at the top level of your config JSON.
 
-###### Halt-on-change Mode
+###### Halt-on-change
 
-If you would like git2consul to shutdown every time its configuration changes, you can enable halt-on-change mode with the command-line switch `-h` or inclusion of the field `"halt_on_change": true` at the top level of your config JSON.  If this mode is enabled, git2consul will wait for changes in the config (which is itself stored in Consul) and gracefully halt when a change is detected.  It is expected that your git2consul process is configured to run as a service, so restarting git2consul is the responsibility of your service manager.
+If you would like git2consul to shutdown every time its configuration changes, you can enable halt-on-change with the command-line switch `-h` or inclusion of the field `"halt_on_change": true` at the top level of your config JSON.  If this switch is enabled, git2consul will wait for changes in the config (which is itself stored in Consul) and gracefully halt when a change is detected.  It is expected that your git2consul process is configured to run as a service, so restarting git2consul is the responsibility of your service manager.
 
-##### How it works
+###### expand_keys
 
-git2consul uses the name and branches of configured repos to namespace the created KVs.  The goal is to allow multiple teams to use the same Consul agents and KV store to migrate configuration data around a network without needing to worry about data conflicts.  In the above example, a settings file stored at `foo_service/settings.json` in the `development` branch of the repo `vp_config` would be persisted in Consul as `vp_config/development/foo_service/settings.json`.
+If you would like git2consul to treat JSON documents in your repo as fully formed subtrees, you can enable expand_keys mode via inclusion of the field `"expand_keys": true` at the top level of the repo's configuration.  If this mode is enabled, git2consul will treat any valid JSON file (that is, any file with extension ".json" that parses to an object) as if it contains a subtree of Consul KVs.  For example, if you have the file `root.json` in repo `expando_keys` with the following contents:
 
-If you are using a more [Twelve-Factor](http://12factor.net/) approach, where you wish to configure your applications via environment variables, you would store these settings as files in Git whose name is the key and whose body is the value.  For example, we could create the file `foo_service/log_level` with the body `trace` in the `development` branch of the `foo_service` repo and git2consul will create the KV `vp_config/development/foo_service/log_level` with the value `trace`.
+```javascript
+{
+  'first_level' : {
+    'second_level' : {
+      'third_level' : {
+        'you get the picture' : 'right?'
+      }
+    }
+  }
+}
+```
 
-As changes are detected in the specified Git repos, git2consul determines which files have been added, updated, or deleted and replicates those changes to the KV.  Because only changed branches and files are analyzed, git2consul should have a very slim profile on hosting systems.
+git2consul in expand_keys mode will generate the following KV:
 
+```
+/expando_keys/root.json/first_level/second_level/third_level/you%20get%20the%20picture
+```
+
+The value in that KV pair will be `right?`.
+
+A few notes on how this behaves:
+
+* Any arrays in your JSON file are ignored.  Only objects and primitives are transformed into keys.
+
+* Expanded keys are URI-encoded.  The spaces in "you get the picture" are thus converted into `%20`.
+
+* Any non-JSON files, including files with the extension ".json" that contain invalid JSON, are stored in your KV as if expand_keys mode was not enabled.
+
 ##### Clients
@@ -89,0 +177,0 @@

No alert changes

Improved metrics

Total package byte prevSize

20985055

increased by0.1%

Number of package files

52

increased by1.96%

Lines of code

2620

increased by1.12%

Number of lines in readme file

194

increased by83.02%

Dependency changes

• + Addedunderscore@1.13.7(transitive)

• - Removedunderscore@1.6.0(transitive)

• Updatedunderscore@^1.8.0