powerwalk - npm Package Compare versions

Comparing version 1.0.0 to 2.0.0

index.js

@@ -0,93 +1,282 @@
+const debug = require('debug')('powerwalk')
+const path = require('path')
+const Transform = require('readable-stream/transform')
+const eos = require('end-of-stream')
+const inherits = require('inherits')
+const prr = require('prr')
+const objectType = require('./object-type')
+const extend = require('xtend')
+const errno = require('errno')
+const format = require('util').format
+const defaults = {
+  symlinks: false,
+  highWaterMark: 16,
+  emit: 'file',
+  fs: require('graceful-fs')
+}
 
-var through2 = require('through2')
-  , glob = require('glob')
-  , fs = require('graceful-fs')
-  , stat = fs.stat
-  , path = require('path')
+module.exports = walk
+module.exports.Powerwalk = Powerwalk
 
-module.exports = function(dirname){
-  var stream = through2(write, flush)
-    , options = { cwd: dirname
-      , strict: true
-      , nosort: true
-      }
-    , queue = []
-
-  fs.exists(dirname, function(exists){
-    if (! exists) {
-      return stream.emit('error', new Error(dirname + ' does not exist'))
+function walk(dirname, options, callback) {
+  var length = arguments.length
+  for (var i = 0; i < length; i++) {
+    switch (typeof arguments[i]) {
+      case 'string':
+        dirname = arguments[i]
+        break;
+      case 'object':
+        options = arguments[i]
+        break;
+      case 'function':
+        callback = arguments[i]
+        break;
     }
+  }
 
-    // NOTE: this used to intentionally wait until the glob end event, there is some
-    // clean up that happens there which prevents things like double entries etc.
-    // see glob option nounique if this becomes an issue again.
-    glob('**', options)
-    .on('error', function(err){ stream.emit('error', err) })
-    .on('match', onmatch)
-  })
+  debug('starting walk at %s', dirname)
 
-  // For non pipe/stream consumption, end will not emit without a consumer
-  stream.on('data', function(chunk) {
-    stream.emit('file', chunk.toString())
-  })
+  var stream = new Powerwalk(options)
 
+  if (callback) {
+    var results = []
+
+    stream.on('data', push(results))
+
+    eos(stream, function endofstream(err) {
+      if (err) return callback(err)
+      else return callback(err, results)
+    })
+  }
+
+  // maybe do an fs.exisits to provide a non-mysterious error here.
+  if (dirname) {
+    // TODO: move this resolution into the _transform method
+    dirname = path.resolve(dirname || '')
+    stream.write(dirname)
+  }
+
   return stream
+}
 
-  function onmatch(match) {
-    // don't stat empty strings
-    if (match.length === 0) return
+function Powerwalk(options) {
+  if (! (this instanceof Powerwalk)) {
+    return new Powerwalk(options)
+  }
 
-    var pathname = path.resolve(options.cwd, match)
+  options = extend(defaults, options)
 
-    stream.write(pathname)
+  if (!(options.ignore instanceof Array)) {
+    options.ignore = [ options.ignore ]
   }
 
-  function write(chunk, enc, callback) {
-    var pathname = chunk.toString()
+  // TODO: gaurd against invalid object types so options.emit: 'garabage'
+  // doesn't cause problems
+  // TODO: assert options.depth is a number
 
-    queue.push(pathname)
+  debug('initializing: %o', options)
 
-    stat(pathname, function(err, stats) {
-      dequeue(pathname)
+  var powerwalk = this
 
-      if (err) return callback(err)
-      if (! stats.isFile()) return callback()
+  Transform.call(powerwalk, options)
 
-      var file = {
-        filename: pathname,
-        stats: stats
-      }
+  prr(powerwalk, 'options', options)
+  prr(powerwalk, 'depth', 0, { writable: true })
+  prr(powerwalk, '_q', [])
+  prr(powerwalk, '_walked', [])
 
-      if (wants('stat')) stream.emit('stat', file)
+  powerwalk.on('path', push(powerwalk._walked))
+}
 
-      if (wants('read')) read(file, callback)
-      else callback(null, pathname)
+inherits(Powerwalk, Transform)
 
-      if (queue.length === 0) stream.end()
-    })
-  }
+Powerwalk.prototype._transform = function (buffer, enc, callback) {
+  debug('transform %s', buffer)
 
-  function flush(callback) {
+  var powerwalk = this
+  var options = powerwalk.options
+  var pathname = buffer.toString()
+  var fs = powerwalk.options.fs
+
+  if (contains(powerwalk.options.ignore, pathname)) {
+    debug('ignoring: %s', pathname)
     callback()
+    return
   }
 
-  function wants(event){
-    return stream.listeners(event).length > 0
+  // // before anything setup things on first write
+  // // * path resolver
+  // if (! powerwalk._started) {
+  //   powerwalk._started = true
+  // }
+  // // end first write setup
+
+  powerwalk.queue(pathname)
+
+  fs.lstat(pathname, function(err, stats) {
+    if (err) {
+      return callback(error(err, 'lstat', pathname))
+    }
+
+    powerwalk.emit('stat', pathname, stats)
+
+    var type = objectType(stats)
+
+    switch (type) {
+      case 'file':
+        powerwalk.dequeue(pathname, type, callback)
+        break
+      case 'directory':
+        powerwalk.depth++
+        debug('depth: %s', powerwalk.depth)
+
+        // stop recursing if depth has been reached...
+        if (powerwalk.options.depth && powerwalk.options.depth === powerwalk.depth) {
+          powerwalk.dequeue(pathname, type, callback)
+          break
+        }
+
+        fs.readdir(pathname, function ondir(err, results) {
+          if (err) return callback(err)
+
+          var length = results.length
+          for (var i = 0; i < length; i++) {
+            var resolved = path.resolve(pathname, results[i])
+            powerwalk.write(resolved)
+          }
+
+          powerwalk.dequeue(pathname, type, callback)
+        })
+
+        break
+      case 'symlink':
+        // On a symlink there are two properties:
+        // * The linkname
+        // * the actual path of the link
+        //
+        // For instance one/two-symlink is actually ../two
+        //
+        // This should be kept track of for each symlink to possilby prevent recurrion
+        // loops.
+        //
+        // Or keep track of emitted paths and DRY
+        var walked = powerwalk.walked(pathname)
+        var shouldSkip = ! options.symlinks || walked
+
+        debug('shouldSkip: %s', shouldSkip)
+
+        if (shouldSkip) {
+          debug('skipping link: %s', pathname)
+          powerwalk.dequeue(pathname, type, callback)
+          break
+        }
+
+        fs.readlink(pathname, function onlinl(err, link) {
+          if (err) {
+            return callback(error(err, 'readlink', pathname))
+          }
+
+          var dirname = path.dirname(pathname)
+          var resolved = path.resolve(dirname, link)
+
+          debug('symlink %s', pathname)
+          debug('symlink resolved: %s', resolved)
+          powerwalk.write(resolved)
+          powerwalk.dequeue(pathname, type, callback)
+        })
+
+        break
+      default:
+        powerwalk.dequeue(pathname, type, callback)
+        break
+    }
+  })
+}
+
+// NOTE: this is to keep track of walked symlinks, instead of tracking every
+// path it would be better to only treat symlinks in this way.
+Powerwalk.prototype.walked = function(pathname) {
+  return contains(this._walked, pathname)
+}
+
+Powerwalk.prototype.queue = function(pathname) {
+  this._q.push(pathname)
+}
+
+Powerwalk.prototype.dequeue = function(pathname, type, callback) {
+  var powerwalk = this
+  var start = powerwalk._q.indexOf(pathname)
+  var deleteCount = 1
+  var removed = powerwalk._q.splice(start, deleteCount)[0]
+
+  if (! removed) {
+    var err = new Error('Can not dequeue items that have not been queued.')
+    powerwalk.emit('error', err)
+    return
   }
 
-  function dequeue(pathname) {
-    queue.splice(queue.indexOf(pathname), 1)
+  if (! powerwalk.walked(pathname)) {
+    powerwalk.emit('path', pathname)
+    powerwalk.emit(type, pathname)
   }
 
-  function read(file, callback){
-    fs.readFile(file.filename, 'utf8', function(err, data){
-      if (err) return callback(err)
+  if (type === powerwalk.options.emit) {
+    debug('%s: %s', type, pathname)
+    callback(null, pathname)
+  } else {
+    callback()
+  }
 
-      file.data = data
+  if (powerwalk._q.length === 0) {
+    powerwalk.end()
+  }
 
-      stream.emit('read', file)
+  return removed
+}
 
-      callback(null, file.filename)
-    })
+Powerwalk.prototype._flush = function(callback) {
+  debug('_flush')
+
+  var powerwalk = this
+
+  // Experimental: This might be a bad idea since data events are queued and the
+  // read stream might not be hooked up til later.
+  // if (powerwalk.listeners('data').length === 0) {
+  //   powerwalk.on('data', noop)
+  // }
+
+  callback()
+}
+
+
+function contains(array, item) {
+  return array.indexOf(item) !== -1
+}
+
+function push(array) {
+  return callback
+
+  function callback(buffer) {
+    array.push(buffer.toString())
   }
 }
+
+function noop(){}
+
+function error(err, method, pathname) {
+  var code = err.code || -1
+  var description = ''
+
+  if (errno.code[code]) {
+    description = errno.code[code].description
+  } else {
+    description = 'unknown error'
+  }
+
+  var message = format('%s "%s" failed: %s', method, pathname, description)
+
+  err.message = message
+  err.pathname = pathname
+
+  return err
+}

package.json

 {
   "name": "powerwalk",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "description": "Recursively walks a directory and emits filenames. Supports additional stat and read events (if you want them).",
@@ -21,11 +21,19 @@ "main": "index.js",
   ],
-  "author": "Jason Campbell <[e-mail]> (http://artifact.sh)",
+  "author": "Jason Campbell <[e-mail]> (http://artifact.sh)",
   "license": "MIT",
   "dependencies": {
-    "glob": "^4.0.5",
+    "debug": "^2.2.0",
+    "end-of-stream": "^1.1.0",
+    "errno": "^0.1.2",
     "graceful-fs": "^3.0.2",
-    "through2": "^0.6.1"
+    "inherits": "^2.0.1",
+    "prr": "^1.0.1",
+    "readable-stream": "^2.0.0",
+    "xtend": "^4.0.0"
   },
   "devDependencies": {
-    "prova": "^1.14.0"
+    "coveralls": "^2.11.2",
+    "istanbul": "^0.3.17",
+    "tape": "^4.0.0",
+    "through2": "^0.6.5"
   },
@@ -32,0 +40,0 @@ "bugs": {

test/test-powerwalk.js

@@ -1,27 +0,22 @@
-
 var powerwalk = require('../')
-var test = require('prova')
-var path = require('path')
-var fixtures = path.resolve(__dirname, './fixtures')
-var through2 = require('through2')
-var fs = require('graceful-fs')
+var test = require('tape')
+var EventEmitter = require('events').EventEmitter
+var Stream = require('stream').Stream
+var helpers = require('./helpers')
 
-test('powerwalk(source).pipe(stream)', function(assert) {
-  var stream = through2(write)
+test('var stream = powerwalk(options)', function(t) {
+  t.ok(powerwalk() instanceof EventEmitter, 'should be an event emitter')
+  t.ok(powerwalk() instanceof Stream, 'should be a stream')
+  t.end()
+})
 
-  assert.plan(8)
+test('powerwalk(dirname, callback)', function(t) {
+  var dirname = helpers.resolve('dreams')
+  var expected = helpers.expected('dreams')
 
-  powerwalk(fixtures)
-  .pipe(stream)
-
-  function write(chunk, enc, callback){
-    var filename = chunk.toString()
-
-    fs.exists(filename, function(exists) {
-      assert.ok(exists, 'filename should exist')
-      assert.ok(filename.match(/fixtures\/(.*)\.md$/), 'should be a fixture')
-    })
-
-    callback()
-  }
+  powerwalk(dirname, function(err, results) {
+    t.error(err)
+    t.same(results.sort(), expected('files'), 'should callback with files')
+    t.end()
+  })
 })

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Major refactor

Supply chain risk

Package has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.

Found 1 instance in 1 package

Dynamic require

Supply chain risk

Dynamic require can indicate the package is performing dangerous or unsafe dynamic code execution.

Found 1 instance in 1 package

Filesystem access

Supply chain risk

Accesses the file system, and could potentially read sensitive data.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

22217

increased by260.37%

Number of package files

27

increased by145.45%

Lines of code

610

increased by585.39%

Worsened metrics

Dependency count

8

increased by166.67%

Dev dependency count

4

increased by300%

Number of low supply chain risk alerts

2

increased byInfinity%

Number of medium supply chain risk alerts

1

increased byInfinity%

Dependency changes

• + Addeddebug@^2.2.0

• + Addedend-of-stream@^1.1.0

• + Addederrno@^0.1.2

• + Addedinherits@^2.0.1

• + Addedprr@^1.0.1

• + Addedreadable-stream@^2.0.0

• + Addedxtend@^4.0.0

• + Addeddebug@2.6.9(transitive)

• + Addedend-of-stream@1.4.4(transitive)

• + Addederrno@0.1.8(transitive)

• + Addedisarray@1.0.0(transitive)

• + Addedms@2.0.0(transitive)

• + Addedprocess-nextick-args@2.0.1(transitive)

• + Addedprr@1.0.1(transitive)

• + Addedreadable-stream@2.3.8(transitive)

• + Addedsafe-buffer@5.1.2(transitive)

• + Addedstring_decoder@1.1.1(transitive)

• + Addedutil-deprecate@1.0.2(transitive)

• - Removedglob@^4.0.5

• - Removedthrough2@^0.6.1

• - Removedbalanced-match@1.0.2(transitive)

• - Removedbrace-expansion@1.1.11(transitive)

• - Removedconcat-map@0.0.1(transitive)

• - Removedglob@4.5.3(transitive)

• - Removedinflight@1.0.6(transitive)

• - Removedisarray@0.0.1(transitive)

• - Removedminimatch@2.0.10(transitive)

• - Removedreadable-stream@1.0.34(transitive)

• - Removedstring_decoder@0.10.31(transitive)

• - Removedthrough2@0.6.5(transitive)