builder - npm Package Compare versions

Comparing version 2.7.1 to 2.8.0

HISTORY.md

 History
 =======
 
+## 2.8.0
+
+* Revises `PATH`, `NODE_PATH` ordering to place archetype first, then root
+  project.
+* Add `--expand-archetype` flag to expand `node_modules/<archetype` tokens in
+  task strings.
+  [builder-victory-component#23](https://github.com/FormidableLabs/builder-victory-component/issues/23)
+
 ## 2.7.1
@@ -5,0 +13,0 @@

lib/args.js

@@ -38,2 +38,7 @@ "use strict";
 };
+var FLAG_EXPAND_ARCHETYPE = {
+  desc: "Expand occurences of `node_modules/<archetype>` with full path (default: `false`)",
+  types: [Boolean],
+  default: false
+};
 var FLAG_HELP = {
@@ -64,2 +69,3 @@ desc: "Display help and exit",
   run: {
+    "expand-archetype": FLAG_EXPAND_ARCHETYPE,
     tries: FLAG_TRIES,
@@ -70,2 +76,3 @@ setup: FLAG_SETUP
   concurrent: {
+    "expand-archetype": FLAG_EXPAND_ARCHETYPE,
     tries: FLAG_TRIES,
@@ -79,2 +86,3 @@ setup: FLAG_SETUP,
   envs: {
+    "expand-archetype": FLAG_EXPAND_ARCHETYPE,
     tries: FLAG_TRIES,
@@ -81,0 +89,0 @@ setup: FLAG_SETUP,

lib/config.js

@@ -37,16 +37,22 @@ "use strict";
 
-  // Array of [name, package.json object] pairs.
-  var pkgs = this._loadPkgs(this.archetypes);
-
-  // Array of [name, scripts|config array] pairs.
-  this.scripts = this._loadScripts(pkgs);
-  this.configs = this._loadConfigs(pkgs);
+  // Array of `{ name, mod, path, scripts, config }`
+  this.pkgs = this._loadPkgs(this.archetypes);
 };
 
-// Expose `require()` for testing.
-//
-// Tests often have needs to mock `fs` which Node 4+ `require`-ing won't work
-// with, defeat the internal `require` cache, etc.
+/**
+ * Return imported module and full path to installed directory.
+ *
+ * Also to expose `require()` for testing.
+ *
+ * Tests often have needs to mock `fs` which Node 4+ `require`-ing won't work
+ * with, defeat the internal `require` cache, etc.
+ *
+ * @param {String} mod  Module name or path.
+ * @returns {Object}    `{ mod: MODULE, path: FULL_PATH_TO_MODULE }` object
+ */
 Config.prototype._lazyRequire = function (mod) {
-  return require(mod); // eslint-disable-line global-require
+  return {
+    mod: require(mod), // eslint-disable-line global-require
+    path: path.dirname(require.resolve(mod))
+  };
 };
@@ -92,3 +98,3 @@
  * @param   {String} name   Archetype name
- * @returns {Object}        Package.json object
+ * @returns {Object}        Object of `{ mod: Package, path: FULL_PATH_TO_MOD }`
  */
@@ -118,3 +124,3 @@ Config.prototype._loadArchetypePkg = function (name) {
 
-  return pkg || {};
+  return pkg;
 };
@@ -131,3 +137,3 @@
  * @param   {Array} archetypes  Archetype names
- * @returns {Array}             Array of [name, package.json object] pairs
+ * @returns {Array}             Array of `{ name, mod, path, scripts, config }`
  */
@@ -137,9 +143,23 @@ Config.prototype._loadPkgs = function (archetypes) {
 
-  return [["ROOT", CWD_PKG]].concat(_(archetypes)
+  // Load base packages.
+  var pkgs = [_.extend({ name: "ROOT" }, CWD_PKG)].concat(_(archetypes)
     .map(function (name) {
       /*eslint-disable no-invalid-this*/
-      return [name, this._loadArchetypePkg(name)];
+      return _.extend({ name: name }, this._loadArchetypePkg(name));
     }, this)
     .reverse()
     .value());
+
+  // Add scripts, config.
+  pkgs = _.mapValues(pkgs, function (pkg) {
+    /*eslint-disable no-invalid-this*/
+    var mod = pkg.mod || {};
+
+    return _.extend({
+      config: mod.config,
+      scripts: pkg.name === "ROOT" ? mod.scripts || {} : this._loadArchetypeScripts(pkg)
+    }, pkg);
+  }, this);
+
+  return pkgs;
 };
@@ -152,7 +172,7 @@
  *
- * @param   {Object} pkg    Archetype package.json object
+ * @param   {Object} pkg    Archetype `{ name, mod, path }` object
  * @returns {Object}        Package.json scripts object
  */
 Config.prototype._loadArchetypeScripts = function (pkg) {
-  var scripts = pkg.scripts || {};
+  var scripts = (pkg.mod || {}).scripts || {};
   return _(scripts)
@@ -167,30 +187,15 @@ .pairs()
 /**
- * Load archetype scripts.
+ * Return display-friendly list of package.json fields commands.
  *
- * @param   {Array} pkgs  Array of [name, package.json object] pairs.
- * @returns {Array}       Array of script objects
+ * @param   {String}  field       Field name to extract
+ * @param   {Array}   archetypes  Archetype names to filter to (Default: all)
+ * @returns {String}              Display string.
  */
-Config.prototype._loadScripts = function (pkgs) {
-  return _.map(pkgs, function (pair) {
-    /*eslint-disable no-invalid-this*/
-    var name = pair[0];
-    var pkg = pair[1];
-    var scripts = name === "ROOT" ? pkg.scripts || {} : this._loadArchetypeScripts(pkg);
+Config.prototype._displayFields = function (field, archetypes) {
+  var pkgs = this.pkgs;
 
-    return [name, scripts];
-  }, this);
-};
-
-/**
- * Return display-friendly list of package.json fields commands.
- *
- * @param   {Array} objs        Array of package.json data.
- * @param   {Array} archetypes  Archetype names to filter to (Default: all)
- * @returns {String}            Display string.
- */
-Config.prototype._displayFields = function (objs, archetypes) {
   // Get filtered list of fields.
   if ((archetypes || []).length) {
-    objs = _.filter(objs, function (pair) {
-      return _.contains(archetypes, pair[0]);
+    pkgs = _.filter(pkgs, function (pkg) {
+      return _.contains(archetypes, pkg.name);
     });
@@ -200,5 +205,5 @@ }
   // First, get all keys.
-  var keys = _(objs)
-    .map(function (pair) {
-      return _.keys(pair[1]);
+  var keys = _(pkgs)
+    .map(function (pkg) {
+      return _.keys(pkg[field]);
     })
@@ -216,6 +221,6 @@ .flatten()
   return _.map(keys, function (key) {
-    var tasks = _(objs)
-      .filter(function (pair) { return pair[1][key]; })
-      .map(function (pair) {
-        return "\n    " + chalk.gray("[" + pair[0] + "]") + " " + pair[1][key];
+    var tasks = _(pkgs)
+      .filter(function (pkg) { return pkg[field][key]; })
+      .map(function (pkg) {
+        return "\n    " + chalk.gray("[" + pkg.name + "]") + " " + pkg[field][key];
       })
@@ -236,18 +241,6 @@ .value()
 Config.prototype.displayScripts = function (archetypes) {
-  return this._displayFields(this.scripts, archetypes);
+  return this._displayFields("scripts", archetypes);
 };
 
 /**
- * Load archetype configs.
- *
- * @param   {Array} pkgs  Array of [name, package.json object] pairs.
- * @returns {Array}       Array of config objects
- */
-Config.prototype._loadConfigs = function (pkgs) {
-  return _.map(pkgs, function (pair) {
-    return [pair[0], pair[1].config || {}];
-  });
-};
-
-/**
  * Return display-friendly list of configs.
@@ -259,3 +252,3 @@ *
 Config.prototype.displayConfigs = function (archetypes) {
-  return this._displayFields(this.configs, archetypes);
+  return this._displayFields("config", archetypes);
 };
@@ -267,9 +260,10 @@
  * @param   {String} cmd  Script command
- * @returns {Array}       List of ordered matches
+ * @returns {Array}       List of ordered matches of `{ archetypeName, archetypePath, cmd }`
  */
 Config.prototype.getCommands = function (cmd) {
-  return _(this.scripts)
-    .map(function (pair) { return pair[1]; })
-    .pluck(cmd)
-    .filter(_.identity)
+  return _(this.pkgs)
+    .map(function (pkg) {
+      return { archetypeName: pkg.name, archetypePath: pkg.path, cmd: pkg.scripts[cmd] };
+    })
+    .filter(function (obj) { return obj.cmd; })
     .value();
@@ -314,5 +308,5 @@ };
     get: function () {
-      var configs = this.configs;
-      var configNames = _(configs)
-        .map(function (pair) { return _.keys(pair[1]); })
+      var pkgs = this.pkgs;
+      var configNames = _(pkgs)
+        .map(function (pkg) { return _.keys(pkg.config); })
         .flatten()
@@ -325,5 +319,5 @@ .uniq()
         .map(function (name) {
-          return [name, _.find(configs, function (pair) {
-            return _.has(pair[1], name);
-          })[1][name]];
+          return [name, _.find(pkgs, function (pkg) {
+            return _.has(pkg.config, name);
+          }).config[name]];
         })
@@ -330,0 +324,0 @@ .object()

lib/environment.js

@@ -66,4 +66,5 @@ "use strict";
 
-  return [CWD_BIN]
+  return []
     .concat(archetypePaths || [])
+    .concat([CWD_BIN])
     .concat(basePath)
@@ -90,4 +91,5 @@ .join(DELIM);
 
-  return [CWD_NODE_PATH]
+  return []
     .concat(archetypeNodePaths || [])
+    .concat([CWD_NODE_PATH])
     .concat(baseNodePath)
@@ -94,0 +96,0 @@ .join(DELIM);

lib/runner.js

@@ -5,2 +5,3 @@ "use strict";
 var exec = require("child_process").exec;
+var path = require("path");
 var _ = require("lodash");
@@ -69,8 +70,84 @@ var async = require("async");
   // Only add the custom flags to non-builder tasks.
-  return opts._isBuilderTask === true ?
-    cmd :
-    cmd + " " + customFlags.join(" ");
+  return cmd + (opts._isBuilderTask === true ? "" : " " + customFlags.join(" "));
 };
 
 /**
+ * Replace all instances of a token.
+ *
+ * _Note_: Only replaces in the following cases:
+ *
+ * - `^<token>`: Token is very first string.
+ * - `[\s\t]<token>`: Whitespace before token.
+ * - `['"]<token>`: Quotes before token.
+ *
+ * @param {String} str    String to parse
+ * @param {String} token  Token to replace
+ * @param {String} sub    Replacement
+ * @returns {String}      Mutated string.
+ */
+var replaceToken = function (str, token, sub) {
+  var tokenRe = new RegExp("(^|\\s|\\'|\\\")(" + _.escapeRegExp(token) + ")", "g");
+
+  return str.replace(tokenRe, function (match, prelimMatch, tokenMatch/* offset, origStr*/) {
+    // Sanity check.
+    if (tokenMatch !== token) {
+      throw new Error("Bad match " + match + " for token " + token);
+    }
+
+    return prelimMatch + sub;
+  });
+};
+
+/**
+ * Expand file paths for archetype within chosen script command.
+ *
+ * @param {String} cmd  Command
+ * @param {Object} opts Options object
+ * @param {Object} env  Environment object
+ * @returns {String}    Updated command
+ */
+var expandArchetype = function (cmd, opts, env) {
+  opts = opts || {};
+  env = env || {};
+
+  // Short-circuit if no expansion.
+  var expand = opts.expandArchetype || env._BUILDER_ARGS_EXPAND_ARCHETYPE === "true";
+  if (expand !== true) {
+    return cmd;
+  }
+
+  // Mark environment.
+  env._BUILDER_ARGS_EXPAND_ARCHETYPE = "true";
+
+  // Create regex around archetype controlling this command.
+  var archetypeName = opts._archetypeName;
+  if (!archetypeName) {
+    // This would be a programming error in builder itself.
+    // Should have been validated out to never happen.
+    throw new Error("Have --expand-archetype but no archetype name");
+  } else if (archetypeName === "ROOT") {
+    // Skip expanding the "ROOT" archetype.
+    //
+    // The root project should have a predictable install level for an archetype
+    // so we do this for safety.
+    //
+    // We _could_ reconsider this and pass in _all_ archetypes and expand them
+    // all everywhere.
+    return cmd;
+  }
+
+  // Infer full path to archetype.
+  var archetypePath = opts._archetypePath;
+  if (!archetypePath) {
+    // Sanity check for programming error.
+    throw new Error("Have --expand-archetype but no archetype path");
+  }
+
+  // Create final token for replacing.
+  var archetypeToken = path.join("node_modules", archetypeName);
+
+  return replaceToken(cmd, archetypeToken, archetypePath);
+};
+
+/**
  * Run a single task.
@@ -91,8 +168,13 @@ *
 
-  // Mutate environment and return new command with `--` custom flags.
-  cmd = cmdWithCustom(cmd, opts, shOpts.env);
-
-  // Check if buffered output or piped.
   var buffer = opts.buffer;
+  var env = shOpts.env;
 
+  // Mutation steps for command. Separated for easier ordering / testing.
+  //
+  // Mutate env and return new command w/ `--` custom flags.
+  cmd = cmdWithCustom(cmd, opts, env);
+  // Mutate env and return new command w/ file paths from the archetype itself.
+  cmd = expandArchetype(cmd, opts, env);
+
+
   log.info("proc:start", cmdStr(cmd, opts));
@@ -270,2 +352,4 @@ var proc = exec(cmd, shOpts, function (err, stdout, stderr) {
   _cmdWithCustom: cmdWithCustom,
+  _expandArchetype: expandArchetype,
+  _replaceToken: replaceToken,
 
@@ -272,0 +356,0 @@ /**

lib/task.js

@@ -112,9 +112,9 @@ "use strict";
  * @param   {String} cmd  Script command
- * @returns {String}      String to execute
+ * @returns {Object}      Command object `{ archetype, cmd }`
  */
 Task.prototype.getCommand = function (cmd) {
   // Select first non-passthrough command.
-  var task = _.find(this._config.getCommands(cmd), function (curCmd) {
+  var task = _.find(this._config.getCommands(cmd), function (obj) {
     /*eslint-disable no-invalid-this*/
-    return !this.isPassthrough(curCmd);
+    return !this.isPassthrough(obj.cmd);
   }, this);
@@ -133,3 +133,3 @@
  *
- * @param   {String} task   Task
+ * @param   {Object} task   Task object `{ archetype, cmd }`
  * @param   {Object} opts   Custom options
@@ -140,3 +140,5 @@ * @returns {Object}        Combined options
   return _.extend({
-    _isBuilderTask: this.isBuilderTask(task)
+    _isBuilderTask: this.isBuilderTask(task.cmd),
+    _archetypeName: task.archetypeName,
+    _archetypePath: task.archetypePath
   }, opts);
@@ -223,5 +225,5 @@ };
 
-  log.info(this._action, this._command + chalk.gray(" - " + task));
+  log.info(this._action, this._command + chalk.gray(" - " + task.cmd));
 
-  return this._runner.run(task, { env: env }, opts, callback);
+  return this._runner.run(task.cmd, { env: env }, opts, callback);
 };
@@ -240,9 +242,11 @@
   var flags = args.concurrent(this.argv);
-  var opts = this.getOpts(tasks[0] || "", flags);
+  var opts = this.getOpts(tasks[0] || {}, flags);
 
   log.info(this._action, cmds.join(", ") + tasks.map(function (t, i) {
-    return "\n * " + cmds[i] + chalk.gray(" - " + t);
+    return "\n * " + cmds[i] + chalk.gray(" - " + t.cmd);
   }).join(""));
 
-  this._runner.concurrent(tasks, { env: env }, opts, callback);
+  this._runner.concurrent(
+    tasks.map(function (task) { return task.cmd; }),
+    { env: env }, opts, callback);
 };
@@ -311,3 +315,3 @@
 
-  this._runner.envs(task, { env: env }, opts, callback);
+  this._runner.envs(task.cmd, { env: env }, opts, callback);
 };
@@ -314,0 +318,0 @@

package.json

 {
   "name": "builder",
-  "version": "2.7.1",
+  "version": "2.8.0",
   "description": "An NPM-based task runner",
@@ -5,0 +5,0 @@ "repository": {

README.md

@@ -223,2 +223,3 @@ [![Travis Status][trav_img]][trav_site]
 * `--builderrc`: Path to builder config file (default: `.builderrc`)
+* `--expand-archetype`: Expand `node_modules/<archetype>` with full path (default: `false`)
 * `--tries`: Number of times to attempt a task (default: `1`)
@@ -241,2 +242,3 @@ * `--setup`: Single task to run for the entirety of `<action>`.
 * `--builderrc`: Path to builder config file (default: `.builderrc`)
+* `--expand-archetype`: Expand `node_modules/<archetype>` with full path (default: `false`)
 * `--tries`: Number of times to attempt a task (default: `1`)
@@ -280,2 +282,3 @@ * `--setup`: Single task to run for the entirety of `<action>`.
 * `--builderrc`: Path to builder config file (default: `.builderrc`)
+* `--expand-archetype`: Expand `node_modules/<archetype>` with full path (default: `false`)
 * `--tries`: Number of times to attempt a task (default: `1`)
@@ -333,3 +336,86 @@ * `--setup`: Single task to run for the entirety of `<action>`.
 
+###### Expanding the Archetype Path
 
+Builder tasks often refer to configuration files in the archetype itself like:
+
+```js
+"postinstall": "webpack --bail --config node_modules/<archetype>/config/webpack/webpack.config.js",
+```
+
+In npm v2 this wasn't a problem because dependencies were usually nested. In
+npm v3, this all changes with aggressive
+[flattening](https://docs.npmjs.com/cli/dedupe) of dependencies. With flattened
+dependencies, the chance that the archetype and its dependencies no longer have
+a predictable contained structure increases.
+
+Thus, commands like the above succeed if the installation ends up like:
+
+```
+node_modules/
+  <a module>/
+    node_modules/
+      <archetype>/
+        node_modules/
+          webpack/
+```
+
+If npm flattens the tree like:
+
+```
+node_modules/
+  <a module>/
+  <archetype>/
+  webpack/
+```
+
+Then `builder` can still find `webpack` due to its `PATH` and `NODE_PATH`
+mutations. But an issue arises with something like a `postinstall` step after
+this flattening in that the current working directory of the process will be
+`PATH/TO/node_modules/<a module>/`, which in this flattened scenario would
+**not** find the file:
+
+```
+node_modules/<archetype>/config/webpack/webpack.config.js
+```
+
+because relative to `node_modules/<a module>/` it is now at:
+
+```
+../<archetype>/config/webpack/webpack.config.js
+```
+
+To address this problem `builder` has an `--expand-archetype` flag that will
+replace an occurrence of the specific `node_modules/<archetype>` in one of the
+archetype commands with the _full path_ to the archetype, to guarantee
+referenced files are correctly available.
+
+The basic heuristic of things to replace is:
+
+* `^node_modules/<archetype>`: Token is very first string.
+* `[\s\t]node_modules/<archetype>`: Whitespace before token.
+* `['"]node_modules/<archetype>`: Quotes before token.
+    * _Note_ that the path coming back from the underlying
+     `require.resolve(module)` will likely be escaped, so things like
+     whitespace in a path + quotes around it may not expand correctly.
+
+Some notes:
+
+* The only real scenario you'll need this is for a module that needs to run
+  a `postinstall` or something as part of an install in a larger project.
+  Root git clone projects controlled by an archetype should work just fine
+  because the archetype will be predictably located at:
+  `node_modules/<archetype>`
+* The `--expand-archetype` flag gets propagated down to all composed `builder`
+  commands internally.
+* The `--expand-archetype` only expands the specific archetype string for its
+  **own** commands and not those in the root projects or other archetypes.
+* The replacement assumes you are using `/` forward slash characters which
+  are the recommended cross-platform way to construct file paths (even on
+  windows).
+* The replacement only replaces at the _start_ of a command string or after
+  whitespace. This means it _won't_ replace `../node_modules/<archetype>` or
+  even `./node_modules/<archetype>`. (In the last case, just omit the `./`
+  in front of a path -- it's a great habit to pick up as `./` breaks on Windows
+  and omitting `./` works on all platforms!)
+
 ## Tasks
@@ -608,3 +694,3 @@
   a matching task `foo`
-* If found `foo`, check if it is a "passthrough" task, which means it delegates
+* If found `foo`, check if it is a "pass-through" task, which means it delegates
   to a later instance -- basically `"foo": "builder run foo"`. If so, then look
@@ -733,3 +819,3 @@ to next instance of task found in order above.
 
-#### NOTE: Application vs Archetype Dependencies
+#### NOTE: Application vs. Archetype Dependencies
 
@@ -899,9 +985,31 @@ While we would love to have `builder` manage _all_ the dependencies of an
 
+### PATH, NODE_PATH Resolution
+
+Builder uses some magic to enhance `PATH` and `NODE_PATH` to look in the
+installed modules of builder archetypes and in the root of your project (per
+normal). We mutate both of these environment variables to resolve in the
+following order:
+
+`PATH`:
+
+1. `<cwd>/node_modules/<archetype>/.bin`
+2. `<cwd>/node_modules/.bin`
+3. Existing `PATH`
+
+`NODE_PATH`:
+
+1. `<cwd>/node_modules/<archetype>/node_modules`
+2. `<cwd>/node_modules`
+3. Existing `NODE_PATH`
+
+The order of resolution doesn't often come up, but can sometimes be a factor
+in diagnosing archetype issues and script / file paths, especially when using
+`npm` v3.
+
 ### Project Root
 
-Builder uses some magic to enhance `NODE_PATH` to look in the root of your
-project (normal) and in the installed modules of builder archetypes. This
-latter path enhancement sometimes throws tools / libraries for a loop. We
-recommend using `require.resolve("LIBRARY_OR_REQUIRE_PATH")` to get the
-appropriate installed file path to a dependency.
+The enhancements to `NODE_PATH` that `builder` performs can throw tools /
+libraries for a loop. Generally speaking, we recommend using
+`require.resolve("LIBRARY_OR_REQUIRE_PATH")` to get the appropriate installed
+file path to a dependency.
 
@@ -1029,3 +1137,3 @@ This comes up in situations including:
 The `builder` project effectively starts at `v2.x.x`. Prior to that Builder was
-a small DOM utility that fell into disuse, so we repurposed it for a new
+a small DOM utility that fell into disuse, so we re-purposed it for a new
 wonderful destiny! But, because we follow semver, that means everything starts
@@ -1032,0 +1140,0 @@ at `v2` and as a helpful tip / warning:

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

86017

increased by9.4%

Lines of code

1349

increased by6.39%

Number of lines in readme file

1146

increased by10.4%

Worsened metrics

Number of low supply chain risk alerts

8

increased by14.29%

No dependency changes