builder - npm Package Compare versions

Comparing version 2.6.0 to 2.7.0

HISTORY.md

 History
 =======
 
+## 2.7.0
+
+* Add support for `package.json:config` analogous to `npm`.
+  [#89](https://github.com/FormidableLabs/builder/issues/89)
+
 ## 2.6.0
@@ -5,0 +10,0 @@

lib/config.js

@@ -37,4 +37,8 @@ "use strict";
 
-  // Array of [name, scripts array] pairs.
-  this.scripts = this._loadScripts(this.archetypes);
+  // Array of [name, package.json object] pairs.
+  var pkgs = this._loadPkgs(this.archetypes);
+
+  // Array of [name, scripts|config array] pairs.
+  this.scripts = this._loadScripts(pkgs);
+  this.configs = this._loadConfigs(pkgs);
 };
@@ -85,8 +89,8 @@
 /**
- * Archetype scripts.
+ * Archetype package.json.
  *
  * @param   {String} name   Archetype name
- * @returns {Object}        Package.json scripts object
+ * @returns {Object}        Package.json object
  */
-Config.prototype._loadArchetypeScripts = function (name) {
+Config.prototype._loadArchetypePkg = function (name) {
   var pkg;
@@ -114,15 +118,9 @@
 
-  var scripts = (pkg || {}).scripts || {};
-  return _(scripts)
-    .pairs()
-    // Remove `builder:` internal tasks.
-    .reject(function (pair) { return pair[0].indexOf("builder:") === 0; })
-    .object()
-    .value();
+  return pkg || {};
 };
 
 /**
- * Load archetype scripts.
+ * Load packages.
  *
- * **Note**: We load scripts into an _array_ because order of operation matters
+ * **Note**: We load packages into an _array_ because order of operation matters
  * and is as follows:
@@ -133,12 +131,11 @@ * - CWD
  * @param   {Array} archetypes  Archetype names
- * @returns {Array}             Array of script objects
+ * @returns {Array}             Array of [name, package.json object] pairs
  */
-Config.prototype._loadScripts = function (archetypes) {
+Config.prototype._loadPkgs = function (archetypes) {
   var CWD_PKG = this._lazyRequire(path.join(process.cwd(), "package.json")) || {};
-  var CWD_SCRIPTS = CWD_PKG.scripts || {};
 
-  return [["ROOT", CWD_SCRIPTS]].concat(_(archetypes)
+  return [["ROOT", CWD_PKG]].concat(_(archetypes)
     .map(function (name) {
       /*eslint-disable no-invalid-this*/
-      return [name, this._loadArchetypeScripts(name)];
+      return [name, this._loadArchetypePkg(name)];
     }, this)
@@ -150,12 +147,47 @@ .reverse()
 /**
- * Return display-friendly list of script commands.
+ * Archetype package scripts.
  *
+ * _Note_: We filter out any `builder:`-prefaced commands.
+ *
+ * @param   {Object} pkg    Archetype package.json object
+ * @returns {Object}        Package.json scripts object
+ */
+Config.prototype._loadArchetypeScripts = function (pkg) {
+  var scripts = pkg.scripts || {};
+  return _(scripts)
+    .pairs()
+    // Remove `builder:` internal tasks.
+    .reject(function (pair) { return pair[0].indexOf("builder:") === 0; })
+    .object()
+    .value();
+};
+
+/**
+ * Load archetype scripts.
+ *
+ * @param   {Array} pkgs  Array of [name, package.json object] pairs.
+ * @returns {Array}       Array of script objects
+ */
+Config.prototype._loadScripts = function (pkgs) {
+  return _.map(pkgs, function (pair) {
+    /*eslint-disable no-invalid-this*/
+    var name = pair[0];
+    var pkg = pair[1];
+    var scripts = name === "ROOT" ? pkg.scripts || {} : this._loadArchetypeScripts(pkg);
+
+    return [name, scripts];
+  }, this);
+};
+
+/**
+ * Return display-friendly list of package.json fields commands.
+ *
+ * @param   {Array} objs        Array of package.json data.
  * @param   {Array} archetypes  Archetype names to filter to (Default: all)
  * @returns {String}            Display string.
  */
-Config.prototype.displayScripts = function (archetypes) {
-  // Get filtered list of scripts.
-  var scripts = this.scripts;
+Config.prototype._displayFields = function (objs, archetypes) {
+  // Get filtered list of fields.
   if ((archetypes || []).length) {
-    scripts = _.filter(scripts, function (pair) {
+    objs = _.filter(objs, function (pair) {
       return _.contains(archetypes, pair[0]);
@@ -166,3 +198,3 @@ });
   // First, get all keys.
-  var keys = _(scripts)
+  var keys = _(objs)
     .map(function (pair) {
@@ -180,5 +212,5 @@ return _.keys(pair[1]);
 
-  // Then, map in order to scripts.
+  // Then, map in order.
   return _.map(keys, function (key) {
-    var tasks = _(scripts)
+    var tasks = _(objs)
       .filter(function (pair) { return pair[1][key]; })
@@ -196,2 +228,34 @@ .map(function (pair) {
 /**
+ * Return display-friendly list of script commands.
+ *
+ * @param   {Array} archetypes  Archetype names to filter to (Default: all)
+ * @returns {String}            Display string.
+ */
+Config.prototype.displayScripts = function (archetypes) {
+  return this._displayFields(this.scripts, archetypes);
+};
+
+/**
+ * Load archetype configs.
+ *
+ * @param   {Array} pkgs  Array of [name, package.json object] pairs.
+ * @returns {Array}       Array of config objects
+ */
+Config.prototype._loadConfigs = function (pkgs) {
+  return _.map(pkgs, function (pair) {
+    return [pair[0], pair[1].config || {}];
+  });
+};
+
+/**
+ * Return display-friendly list of configs.
+ *
+ * @param   {Array} archetypes  Archetype names to filter to (Default: all)
+ * @returns {String}            Display string.
+ */
+Config.prototype.displayConfigs = function (archetypes) {
+  return this._displayFields(this.configs, archetypes);
+};
+
+/**
  * Get list of tasks in preferred execution order.
@@ -235,3 +299,31 @@ *
     }
+  },
+
+  "pkgConfigs": {
+    /**
+     * Return object of resolved package.json config fields.
+     *
+     * Resolves in order of "root wins", then in reverse archetype order.
+     *
+     * @returns {Object} environment object.
+     */
+    get: function () {
+      var configs = this.configs;
+      var configNames = _(configs)
+        .map(function (pair) { return _.keys(pair[1]); })
+        .flatten()
+        .uniq()
+        .value();
+
+      // Take "first" config value in arrays as "winning" value.
+      return _(configNames)
+        .map(function (name) {
+          return [name, _.find(configs, function (pair) {
+            return _.has(pair[1], name);
+          })[1][name]];
+        })
+        .object()
+        .value();
+    }
   }
 });

lib/environment.js

@@ -9,2 +9,3 @@ "use strict";
  */
+var _ = require("lodash");
 var path = require("path");
@@ -42,5 +43,8 @@
 
-  // Mutate environment.
+  // Mutate environment paths.
   this.env.PATH = this.updatePath(this.config.archetypePaths);
   this.env.NODE_PATH = this.updateNodePath(this.config.archetypeNodePaths);
+
+  // Add in npm config environment variables.
+  this.updateConfigVars(this.config.pkgConfigs);
 };
@@ -92,1 +96,26 @@
 };
+
+/**
+ * Update environment with configuration variables from package.json
+ *
+ * **Note**: We only go _one level deep_ for process.env mutations.
+ *
+ * Resolution order:
+ * 1. Existing environment
+ * 2. `ROOT/package.json:config`
+ * 3. `ROOT/node_modules/ARCHETYPE[1-n]/package.json:config`
+ *
+ * @param   {Object}   pkgConfigs  Resolved config variables.
+ * @returns {Object}               Mutated environment variable.
+ */
+Environment.prototype.updateConfigVars = function (pkgConfigs) {
+  _.each(pkgConfigs, function (val, name) {
+    /*eslint-disable no-invalid-this*/
+    var fullName = "npm_package_config_" + name;
+    if (!_.has(this.env, fullName)) {
+      this.env[fullName] = val;
+    }
+  }, this);
+
+  return this.env;
+};

lib/task.js

@@ -170,2 +170,5 @@ "use strict";
 
+  // Display task configs if we have _some_.
+  var taskConfigs = this._config.displayConfigs(archetypes);
+
   log.info("help",
@@ -176,2 +179,3 @@ "\n\n" + chalk.green.bold("Usage") + ": \n\n  builder " + actionDisplay + " <task(s)>" +
     actionFlags +
+    (taskConfigs ? "\n\n" + chalk.green.bold("Task Configs") + ": \n" + taskConfigs : "") +
     "\n\n" + chalk.green.bold("Tasks") + ": \n" + this._config.displayScripts(archetypes));
@@ -178,0 +182,0 @@

package.json

 {
   "name": "builder",
-  "version": "2.6.0",
+  "version": "2.7.0",
   "description": "An NPM-based task runner",
@@ -5,0 +5,0 @@ "repository": {

README.md

@@ -10,11 +10,11 @@ [![Travis Status][trav_img]][trav_site]
 
-`npm` is fantastic for controlling dependencies, tasks (via `scripts`) and
-general project workflows. But a project-specific `package.json` simply doesn't
-scale when you're managing many (say 5-50) very similar repositories.
+`npm` is fantastic for controlling tasks (via `scripts`) and general project
+workflows. But a project-specific `package.json` simply doesn't scale when
+you're managing many (say 5-50) very similar repositories.
 
 _Enter Builder._ Builder is "almost" `npm`, but provides for off-the-shelf
-"archetypes" to provide central sets of `package.json` `scripts`,
-`dependencies` and `devDependencies`. The rest of this page will dive into
-the details and machinations of the tool, but first here are a few of the
-rough goals and motivations behind the project.
+"archetypes" to provide central sets of `package.json` `scripts` tasks, and
+`dependencies` and `devDependencies` for those tasks. The rest of this page will
+dive into the details and machinations of the tool, but first here are a few of
+the rough goals and motivations behind the project.
 
@@ -38,2 +38,8 @@ * **Single Point of Control**: A way to define a specific set of tasks /
   and `devDependencies`.
+* **A Few "Nice to Haves" Over `npm run <task>`**: Setting aside archetypes and
+  multi-project management, `builder` provides cross-OS compatible helpers for
+  common task running scenarios like concurrent execution (`concurrent`) and
+  spawning the _same_ tasks in parallel with different environment variables
+  (`env`). It also provides useful controls for task retries, buffered output,
+  setup tasks, etc.
 
@@ -51,3 +57,3 @@ ## Overview
 * `NODE_PATH`, `PATH` enhancements to run, build, import from archetypes so
-  dependencies and configurations don't have to be installed directly in a
+  task dependencies and configurations don't have to be installed directly in a
   root project.
@@ -226,3 +232,3 @@ * A task runner capable of single tasks (`run`) or multiple concurrent tasks
 Run multiple tasks from `script` concurrently. Roughly analogous to
-`npm run <task1> | npm run <task2> | npm run <task3>`, but kills all processes on
+`npm run <task1> & npm run <task2> & npm run <task3>`, but kills all processes on
 first non-zero exit (which makes it suitable for test tasks), unless `--no-bail`
@@ -255,3 +261,3 @@ is provided.
 ```sh
-$ FOO=VAL1 npm run <task> | FOO=VAL2 npm run <task> | FOO=VAL3 npm run <task>
+$ FOO=VAL1 npm run <task> & FOO=VAL2 npm run <task> & FOO=VAL3 npm run <task>
 ```
@@ -284,2 +290,5 @@
 
+_Note_: The environments JSON array will overwrite **existing** values in the
+environment.
+
 ###### Custom Flags
@@ -326,2 +335,3 @@
 
+
 ## Tasks
@@ -368,2 +378,189 @@
 
+
+## npm Config
+
+`builder` supports `package.json` `config` properties the same way that `npm`
+does, with slight enhancements in consideration of multiple `package.json`'s
+in play.
+
+### `npm` Config Overview
+
+As a refresher, `npm` utilizes the `config` field of `package.json` to make
+"per-package" environment variables to `scripts` tasks. For example, if you
+have:
+
+```js
+{
+  "config": {
+    "my_name": "Bob"
+  },
+  "scripts": {
+    "get-name": "echo Hello, ${npm_package_config_my_name}."
+  }
+}
+```
+
+and ran:
+
+```sh
+$ npm run get-name
+Hello, Bob.
+```
+
+More documentation about how `npm` does per-package configuration is at:
+
+* https://docs.npmjs.com/files/package.json#config
+* https://docs.npmjs.com/misc/config#per-package-config-settings
+
+
+### Builder Configs
+
+In `builder`, for a single `package.json` this works essentially the same in
+the above example.
+
+```sh
+$ builder run get-name
+Hello, Bob.
+```
+
+However, `builder` has the added complexity of adding in `config` variables
+from archetypes and the environment. So the basic resolution order for a
+config environment variable is:
+
+1. Look to `npm_package_config_<VAR_NAME>=<VAR_VAL>` on command line.
+2. If not set, then use `<root>/package.json:config:<VAR_NAME>` value.
+3. If not set, then use `<archetype>/package.json:config:<VAR_NAME>` value.
+
+So, let's dive in to a slightly more complex example:
+
+```js
+// <archetype>/package.json
+{
+  "config": {
+    "my_name": "ARCH BOB"
+  },
+  "scripts": {
+    "get-name": "echo Hello, ${npm_package_config_my_name}."
+  }
+}
+
+// <root>/package.json
+{
+  "config": {
+    "my_name": "ROOT JANE"
+  }
+}
+```
+
+When we run the `builder` command, the `<root>` value overrides:
+
+```sh
+$ builder run get-name
+Hello, ROOT JANE.
+```
+
+We can inject a command line flag to override even this value:
+
+```sh
+$ npm_package_config_my_name="CLI JOE" builder run get-name
+Hello, CLI JOE.
+```
+
+_Note_ that the ability to override via the process environment is unique
+to `builder` and not available in real `npm`.
+
+### Config Notes
+
+#### Tip - Use String Values
+
+Although `config` properties can be something like:
+
+```js
+"config": {
+  "enabled": true
+}
+```
+
+We strongly recommend that you always set _strings_ like:
+
+```js
+"config": {
+  "enabled": "true"
+}
+```
+
+And deal just with _string values_ in your tasks, and files. The reasoning here
+is that when overriding values from the command line, the values will always
+be strings, which has a potential for messy, hard-to-diagnose bugs if the
+overridden value is not also a string.
+
+#### npmrc Configuration
+
+`npm` has additional functionality for `config` values that are **not**
+presently supported, such as issuing commands like
+`npm config set <pkg-name>:my_name Bill` that store values in `~/.npmrc` and
+then override the `package.json` values at execution time. We _may_ extend
+support for this as well, but not at the present.
+
+#### Command Line Environment Variables
+
+`npm` does **not** support overriding `config` environment variables from the
+actual environment. So doing something in our original example like:
+
+```sh
+$ npm_package_config_my_name=George npm run get-name
+Hello, Bob.
+```
+
+In fact, npm will refuse to even add environment variables starting with
+`npm_package_config` to the `npm run` environment. E.g.
+
+```js
+{
+  "config": {},
+  "scripts": {
+    "get-npm-val": "echo NPM VAR: ${npm_package_config_var}",
+    "get-env-val": "echo ENV VAR: ${env_var}"
+  }
+}
+```
+
+The `npm` config variable doesn't make it through:
+
+```sh
+$ npm_package_config_var=SET npm run get-npm-val
+NPM VAR:
+```
+
+While a normal environment variable will:
+
+```sh
+$ env_var=SET npm run get-env-val
+ENV VAR: SET
+```
+
+By contrast, `builder` _does_ pass through environment variables already
+existing on the command line, and moreover those overrides takes precedence over
+the root and archetype package.json values. Those same examples with `builder`
+show that the environment variables _do_ make it through:
+
+```sh
+$ npm_package_config_var=SET builder run get-npm-val
+NPM VAR: SET
+
+$ env_var=SET builder run get-env-val
+ENV VAR: SET
+```
+
+Things are a little more complex when using with `builder envs`, but the
+rough rule is that the environment JSON array wins when specified, otherwise
+the existing environment is used:
+
+```sh
+$ npm_package_config_var=CLI builder envs get-npm-val --queue=1 \
+  '[{}, {"npm_package_config_var":"This Overrides"}]'
+NPM VAR: CLI
+NPM VAR: This Overrides
+```
+
 ## Archetypes
@@ -380,3 +577,3 @@
 * A `package.json` with `builder`-friendly `script` tasks.
-* Dependencies and dev dependencies to build, test, etc.
+* Dependencies and dev dependencies for all of the archetype `script` tasks.
 * Configuration files for all `script` tasks.
@@ -433,16 +630,30 @@
 
-As an **additional restriction**, non-`npm:FOO`-prefixed tasks with the same
-name (e.g., `FOO`) _may_ call then `npm:`-prefixed task, but _not_ the other
-way around. So
+We strongly recommend entirely
+[avoiding npm lifecycle task names](#avoid-npm-lifecycle-commands)
+in your archetype `package.json` files. So, instead of having:
 
 ```js
+// <archetype>/package.json
+// Bad
+"test": "builder concurrent --buffer test-frontend test-backend"
+```
+
+We recommend something like:
+
+```js
+// <archetype>/package.json
 // Good / OK
-"npm:test": "builder run test-frontend",
-"test": "builder run npm:test",
+"npm:test": "builder run test-all",
+"test-all": "builder concurrent --buffer test-frontend test-backend"
 
-// Bad
-"npm:test": "builder run test",
-"test": "builder run test-frontend",
+// Also OK
+"npm:test": "builder concurrent --buffer test-frontend test-backend"
 ```
 
+and then in your `<root>/package.json` using the _real_ lifecycle task name.
+
+```js
+"test": "builder run npm:test"
+```
+
 ### Creating an Archetype
@@ -458,3 +669,3 @@
 up a new archetype from scratch, make a directory for your new archetype,
-initialize NPM and link it for ease of development.
+initialize `npm` and link it for ease of development.
 
@@ -476,3 +687,3 @@ ```sh
 
-#### Managing the `dev` archetype
+#### Managing the `dev` Archetype
 
@@ -528,8 +739,44 @@ Because `builder` archetypes are included as simple npm modules, two separate
 
-#### Workflow for moving dependencies and scripts to your new archetype
+#### NOTE: Application vs Archetype Dependencies
 
+While we would love to have `builder` manage _all_ the dependencies of an
+application, the practical realities of how npm works is that archetypes can
+only manage dependencies for `scripts` commands **run by a `builder` command**.
+`builder` mutates `PATH` and `NODE_PATH` to include archetype dependencies, but
+without this `builder` magic, ordinary code won't otherwise be able to use
+archetype dependencies.
+
+Most notably, this means that if your _application_ code includes a dependency
+like `lodash`:
+
+```js
+// <root>/src/index.js
+var _ = require("lodash");
+
+module.exports = _.camelCase("Hi There");
+```
+
+and the root project is consumed in _anything besides a `builder` command_,
+then it **must** have a dependency like:
+
+```js
+// <root>/package.json
+"dependencies": {
+  "lodash": "^4.2.1"
+}
+```
+
+And, _even if_ your `<archetype>/package.json` also includes the exact same
+dependency.
+
+This rule applies to even simple scenarios such as the root project being
+published to npm, after which other users will rely on the code outside of
+`builder` processes.
+
+#### Moving `dependencies` and `scripts` to a New Archetype
+
 Once everything is configured and `npm link`'d, it should be easy to move
 scripts to your archetype and quickly test them out from a consuming project.
 
-##### Moving `dependencies` and `devDependencies` from an existing `package.json`
+##### Moving `dependencies` and `devDependencies` from an Existing `package.json`
 
@@ -539,4 +786,15 @@ * copy `dependencies` to `package.json` `dependencies`.
 
-##### Moving scripts and config files
+_Note_ that you should only copy `dependencies` from `<root>/package.json` to
+`<archetype>/package.json` that are needed within the archetype itself for:
 
+* Execution of a script. (E.g., the `istanbul` script).
+* Required by a configuration file in the archetype. (E.g., `webpack` if a
+  webpack configuration calls `require("webpack")`).
+
+You can then remove any dependencies _only_ used by the `scripts` tasks that
+you have moved to the archetype. However, take care to
+[not remove real application dependencies](#note-application-vs-archetype-dependencies).
+
+##### Moving `scripts` and Config Files
+
 All scripts defined in archetypes will be run from the root of the project
@@ -565,3 +823,3 @@ consuming the archetype. This means you have to change all paths in your scripts
 
-##### Updating path and module references within your script configs
+##### Updating Path and Module References in Config Files
 
@@ -619,3 +877,3 @@ Any JavaScript files run from within an archetype (such as config files) require
 
-#### Example `builder` archetype project structure
+#### Example `builder` Archetype Project Structure
 
@@ -676,2 +934,23 @@ ```
 
+### Avoid npm Lifecycle Commands
+
+We recommend _not_ using any of the special `npm` `scripts` commands listed in
+https://docs.npmjs.com/misc/scripts such as:
+
+* prepublish, postinstall
+* test
+* stop, start
+
+in your archetype `scripts`. This is due to the fact that the archetype
+`package.json` files are themselves consumed by `npm` for publishing (which
+can lead to tasks executing for the _archetype_ instead of the project _using_
+the archetype) and potentially lead to awkward recursive composed task
+scenarios.
+
+Instead, we recommend adding an `npm:<task>` prefix to your tasks to identify
+them as usable in root projects for real `npm` lifecycle tasks.
+
+We plan on issuing warnings for archetypes that do implement lifecycle tasks
+in: https://github.com/FormidableLabs/builder/issues/81
+
 ### Other Process Execution
@@ -736,2 +1015,4 @@
   your project.
+* Copy all `ARCHETYPE/package.json:config` variables to your
+  `PROJECT/package.json:config`.
 * Copy all `ARCHETYPE-dev/package.json:dependencies` to your
@@ -738,0 +1019,0 @@ `PROJECT/package.json:devDependencies`

No alert changes

Improved metrics

Total package byte prevSize

78526

increased by17.19%

Lines of code

1262

increased by9.74%

Number of lines in readme file

1037

increased by37.17%

No dependency changes