broccoli-merge-trees - npm Package Compare versions

Comparing version 0.1.4 to 0.2.0

CHANGELOG.md

 # master
 
+# 0.2.0
+
+* Use symlinks as a performance optimization (see
+  [symlink-change.md](https://github.com/broccolijs/broccoli/blob/master/docs/symlink-change.md))
+
 # 0.1.4
@@ -4,0 +9,0 @@

index.js

 var fs = require('fs')
-var walkSync = require('walk-sync')
+var path = require('path')
 var Writer = require('broccoli-writer')
-var helpers = require('broccoli-kitchen-sink-helpers')
+var symlinkOrCopySync = require('symlink-or-copy').sync
 var mapSeries = require('promise-map-series')
 
+var isWindows = process.platform === 'win32'
 
@@ -22,52 +23,141 @@ module.exports = TreeMerger
   var self = this
-  var files = {}
-  var directories = {}
+
   return mapSeries(this.inputTrees, readTree).then(function (treePaths) {
-    for (var i = treePaths.length - 1; i >= 0; i--) {
-      var treeContents = walkSync(treePaths[i])
-      var fileIndex
-      for (var j = 0; j < treeContents.length; j++) {
-        var relativePath = treeContents[j]
-        var destPath = destDir + '/' + relativePath
-        if (relativePath.slice(-1) === '/') { // is directory
-          relativePath = relativePath.slice(0, -1) // chomp "/"
-          fileIndex = files[relativePath]
-          if (fileIndex != null) {
-            throwFileAndDirectoryCollision(relativePath, fileIndex, i)
+    mergeRelativePath('')
+
+    function mergeRelativePath (baseDir, possibleIndices) {
+      // baseDir has a trailing path.sep if non-empty
+      var i, j, fileName, fullPath
+
+      // Array of readdir arrays
+      var names = treePaths.map(function (treePath, i) {
+        if (possibleIndices == null || possibleIndices.indexOf(i) !== -1) {
+          return fs.readdirSync(treePath + path.sep + baseDir).sort()
+        } else {
+          return []
+        }
+      })
+
+      // Guard against conflicting capitalizations
+      var lowerCaseNames = {}
+      for (i = 0; i < treePaths.length; i++) {
+        for (j = 0; j < names[i].length; j++) {
+          fileName = names[i][j]
+          var lowerCaseName = fileName.toLowerCase()
+          if (lowerCaseNames[lowerCaseName] == null) {
+            lowerCaseNames[lowerCaseName] = {
+              index: i,
+              originalName: fileName
+            }
+          } else {
+            var originalIndex = lowerCaseNames[lowerCaseName].index
+            var originalName = lowerCaseNames[lowerCaseName].originalName
+            if (originalName !== fileName) {
+              throw new Error('Merge error: conflicting capitalizations:\n'
+                + baseDir + originalName + ' in ' + treePaths[originalIndex] + '\n'
+                + baseDir + fileName + ' in ' + treePaths[i] + '\n'
+                + 'Remove one of the files and re-add it with matching capitalization.\n'
+                + 'We are strict about this to avoid divergent behavior '
+                + 'between case-insensitive Mac/Windows and case-sensitive Linux.'
+              )
+            }
           }
-          if (directories[relativePath] == null) {
-            fs.mkdirSync(destPath)
-            directories[relativePath] = i
-          }
-        } else { // is file
-          var directoryIndex = directories[relativePath]
-          if (directoryIndex != null) {
-            throwFileAndDirectoryCollision(relativePath, i, directoryIndex)
-          }
-          fileIndex = files[relativePath.toLowerCase()]
-          if (fileIndex != null) {
-            if (!self.options.overwrite) {
-              throw new Error('Merge error: ' +
-                'file "' + relativePath + '" exists in ' +
-                treePaths[i] + ' and ' + treePaths[fileIndex] + ' - ' +
-                'pass option { overwrite: true } to mergeTrees in order ' +
-                'to have the latter file win')
+        }
+      }
+      // From here on out, no files and directories exist with conflicting
+      // capitalizations, which means we can use `===` without .toLowerCase
+      // normalization.
+
+      // Accumulate fileInfo hashes of { isDirectory, indices }.
+      // Also guard against conflicting file types and overwriting.
+      var fileInfo = {}
+      for (i = 0; i < treePaths.length; i++) {
+        for (j = 0; j < names[i].length; j++) {
+          fileName = names[i][j]
+          fullPath = treePaths[i] + path.sep + baseDir + fileName
+          var isDirectory = checkIsDirectory(fullPath)
+          if (fileInfo[fileName] == null) {
+            fileInfo[fileName] = {
+              isDirectory: isDirectory,
+              indices: [i] // indices into treePaths in which this file exists
             }
-            // Else, ignore this file. It is "overwritten" by a file we copied
-            // earlier, thanks to reverse iteration over trees
           } else {
-            helpers.copyPreserveSync(
-              treePaths[i] + '/' + relativePath, destPath)
-            files[relativePath.toLowerCase()] = i
+            fileInfo[fileName].indices.push(i)
+
+            // Guard against conflicting file types
+            var originallyDirectory = fileInfo[fileName].isDirectory
+            if (originallyDirectory !== isDirectory) {
+              throw new Error('Merge error: conflicting file types: ' + baseDir + fileName
+                + ' is a ' + (originallyDirectory ? 'directory' : 'file')
+                  + ' in ' + treePaths[fileInfo[fileName].indices[0]]
+                + ' but a ' + (isDirectory ? 'directory' : 'file')
+                  + ' in ' + treePaths[i] + '\n'
+                + 'Remove or rename either of those.'
+              )
+            }
+
+            // Guard against overwriting when disabled
+            if (!isDirectory && !self.options.overwrite) {
+              throw new Error('Merge error: '
+                + 'file ' + baseDir + fileName + ' exists in '
+                + treePaths[fileInfo[fileName].indices[0]] + ' and ' + treePaths[i] + '\n'
+                + 'Pass option { overwrite: true } to mergeTrees in order '
+                + 'to have the latter file win.'
+              )
+            }
           }
         }
       }
-    }
 
-    function throwFileAndDirectoryCollision (relativePath, fileIndex, directoryIndex) {
-      throw new Error('Merge error: "' + relativePath +
-        '" exists as a file in ' + treePaths[fileIndex] +
-        ' but as a directory in ' + treePaths[directoryIndex])
+      // Done guarding against all error conditions. Actually merge now.
+      for (i = 0; i < treePaths.length; i++) {
+        for (j = 0; j < names[i].length; j++) {
+          fileName = names[i][j]
+          fullPath = treePaths[i] + path.sep + baseDir + fileName
+          var destPath = destDir + path.sep + baseDir + fileName
+          var infoHash = fileInfo[fileName]
+
+          if (infoHash.isDirectory) {
+            if (isWindows || infoHash.indices.length > 1) {
+              // Copy/merge subdirectory
+              if (infoHash.indices[0] === i) { // avoid duplicate recursion
+                fs.mkdirSync(destPath)
+                mergeRelativePath(baseDir + fileName + path.sep, infoHash.indices)
+              }
+            } else {
+              // Symlink entire subdirectory
+              if (fs.lstatSync(fullPath).isSymbolicLink()) {
+                // When we encounter symlinks, follow them. This prevents indirection
+                // from growing out of control. Note: At the moment `realpath` on Node
+                // is 70x slower than native: https://github.com/joyent/node/issues/7902
+                fullPath = fs.realpathSync(fullPath)
+              } else if (fullPath[0] !== path.sep) {
+                fullPath = process.cwd() + path.sep + fullPath
+              }
+              fs.symlinkSync(fullPath, destPath)
+            }
+          } else { // isFile
+            if (infoHash.indices[infoHash.indices.length-1] === i) {
+              symlinkOrCopySync(fullPath, destPath)
+            } else {
+              // This file exists in a later tree. Do nothing here to have the
+              // later file win out and thus "overwrite" the earlier file.
+            }
+          }
+        }
+      }
     }
   })
 }
+
+// True if directory, false if file, exception otherwise
+function checkIsDirectory (fullPath) {
+  var stat = fs.statSync(fullPath) // may throw ENOENT on broken symlink
+  if (stat.isDirectory()) {
+    return true
+  } else if (stat.isFile()) {
+    return false
+  } else {
+    throw new Error('Unexpected file type for ' + fullPath)
+  }
+}

package.json

 {
   "name": "broccoli-merge-trees",
   "description": "Broccoli plugin to merge multiple trees into one",
-  "version": "0.1.4",
+  "version": "0.2.0",
   "author": "Jo Liss <[e-mail]>",
@@ -19,12 +19,12 @@ "main": "index.js",
     "promise-map-series": "^0.2.0",
-    "walk-sync": "^0.1.2",
     "broccoli-writer": "^0.1.1",
-    "broccoli-kitchen-sink-helpers": "^0.2.0"
+    "symlink-or-copy": "^1.0.0"
   },
   "devDependencies": {
-    "tap": "^0.4.8",
-    "rsvp": "^3.0.6",
+    "coffee-script": "~1.7.1",
     "fixturify": "^0.1.1",
     "jshint": "~2.5.0",
-    "coffee-script": "~1.7.1"
+    "ncp": "^0.6.0",
+    "rsvp": "^3.0.6",
+    "tap": "^0.4.8"
   },
@@ -31,0 +31,0 @@ "scripts": {

README.md

 # broccoli-merge-trees
 
-[![Build Status](https://travis-ci.org/joliss/broccoli-merge-trees.png?branch=master)](https://travis-ci.org/joliss/broccoli-merge-trees)
+[![Build Status](https://travis-ci.org/broccolijs/broccoli-merge-trees.png?branch=master)](https://travis-ci.org/broccolijs/broccoli-merge-trees)
 
@@ -5,0 +5,0 @@ Copy multiple trees on top of each other, resulting in a single merged tree.

test/broccoli_test_helpers.js

 var Writer = require('broccoli-writer')
+var RSVP = require('rsvp')
 var Promise = require('rsvp').Promise
+var ncp = require('ncp')
 var fixturify = require('fixturify')
@@ -61,3 +63,3 @@
 function FixtureTree (fixtureObject) {
-  if (!(this instanceof FixtureTree)) return new FixtureTree(fixtureObject);
+  if (!(this instanceof FixtureTree)) return new FixtureTree(fixtureObject)
   this.fixtureObject = fixtureObject
@@ -69,1 +71,19 @@ }
 }
+
+
+exports.dereferenceSymlinks = SymlinkDereferencer
+SymlinkDereferencer.prototype = Object.create(Writer.prototype)
+SymlinkDereferencer.prototype.constructor = SymlinkDereferencer
+function SymlinkDereferencer (inputTree) {
+  if (!(this instanceof SymlinkDereferencer)) return new SymlinkDereferencer(inputTree)
+  this.inputTree = inputTree
+}
+
+SymlinkDereferencer.prototype.write = function (readTree, destDir) {
+  return readTree(this.inputTree)
+    .then(function (srcDir) {
+      return RSVP.denodeify(ncp)(srcDir, destDir, {
+        dereference: true
+      })
+    })
+}

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

22091

increased by28.63%

Dependency count

3

decreased by-25%

Number of package files

12

increased by9.09%

Lines of code

222

increased by79.03%

Worsened metrics

Dev dependency count

6

increased by20%

Dependency changes

• + Addedsymlink-or-copy@^1.0.0

• + Addedsymlink-or-copy@1.3.1(transitive)

• - Removedbroccoli-kitchen-sink-helpers@^0.2.0

• - Removedwalk-sync@^0.1.2

• - Removedbroccoli-kitchen-sink-helpers@0.2.9(transitive)

• - Removedglob@5.0.15(transitive)

• - Removedminimist@1.2.8(transitive)

• - Removedmkdirp@0.5.6(transitive)

• - Removedwalk-sync@0.1.3(transitive)