pacote - npm Package Compare versions

Comparing version 4.0.0 to 5.0.0

CHANGELOG.md

@@ -5,2 +5,27 @@ # Change Log
 
+<a name="5.0.0"></a>
+# [5.0.0](https://github.com/zkat/pacote/compare/v4.0.0...v5.0.0) (2017-08-16)
+
+
+### Bug Fixes
+
+* **registry:** Pass maxSockets options down (#110) ([3f05b79](https://github.com/zkat/pacote/commit/3f05b79))
+
+
+### Features
+
+* **deps:** replace tar-fs/tar-stream with tar[@3](https://github.com/3) ([28c80a9](https://github.com/zkat/pacote/commit/28c80a9))
+* **tar:** switch to tarv3 ([53899c7](https://github.com/zkat/pacote/commit/53899c7))
+
+
+### BREAKING CHANGES
+
+* **tar:** this changes the underlying tar library, and thus may introduce some subtle low-level incompatibility. Also:
+
+* The tarball packer built into pacote works much closer to how the one npm injects does.
+* Special characters on Windows will now be escaped the way tar(1) usually does: by replacing them with the `0xf000` masked character on the way out.
+* Directories won't be chowned.
+
+
+
 <a name="4.0.0"></a>
@@ -7,0 +32,0 @@ # [4.0.0](https://github.com/zkat/pacote/compare/v3.0.0...v4.0.0) (2017-06-29)

extract.js

@@ -7,4 +7,4 @@ 'use strict'
 const extractStream = require('./lib/extract-stream')
+const mkdirp = BB.promisify(require('mkdirp'))
 const npa = require('npm-package-arg')
-const pipe = BB.promisify(require('mississippi').pipe)
 const optCheck = require('./lib/util/opt-check')
@@ -63,5 +63,12 @@ const retry = require('promise-retry')
 function extractByDigest (start, spec, dest, opts) {
-  const xtractor = extractStream(dest, opts)
-  const cached = cacache.get.stream.byDigest(opts.cache, opts.integrity, opts)
-  return pipe(cached, xtractor).then(() => {
+  return mkdirp(dest).then(() => {
+    const xtractor = extractStream(dest, opts)
+    const cached = cacache.get.stream.byDigest(opts.cache, opts.integrity, opts)
+    cached.pipe(xtractor)
+    return new BB((resolve, reject) => {
+      cached.on('error', reject)
+      xtractor.on('error', reject)
+      xtractor.on('close', resolve)
+    })
+  }).then(() => {
     opts.log.silly('pacote', `${spec} extracted to ${dest} by content address ${Date.now() - start}ms`)
@@ -73,8 +80,14 @@ })
 function extractByManifest (start, spec, dest, opts) {
-  const xtractor = extractStream(dest, opts)
-  return BB.resolve(null).then(() => {
+  return mkdirp(dest).then(() => {
+    const xtractor = extractStream(dest, opts)
     if (!fetch) {
       fetch = require('./lib/fetch')
     }
-    return pipe(fetch.tarball(spec, opts), xtractor)
+    const tardata = fetch.tarball(spec, opts)
+    tardata.pipe(xtractor)
+    return new BB((resolve, reject) => {
+      tardata.on('error', reject)
+      xtractor.on('error', reject)
+      xtractor.on('close', resolve)
+    })
   }).then(() => {
@@ -81,0 +94,0 @@ opts.log.silly('pacote', `${spec} extracted in ${Date.now() - start}ms`)

lib/extract-stream.js

 'use strict'
 
-const gunzip = require('./util/gunzip-maybe')
-const path = require('path')
-const pipeline = require('mississippi').pipeline
-const tar = require('tar-fs')
+const tar = require('tar')
 
 module.exports = extractStream
-function extractStream (dest, opts) {
+function extractStream (dest, opts, cb) {
   opts = opts || {}
-  const sawIgnores = {}
-  return pipeline(gunzip(), tar.extract(dest, {
-    map: (header) => {
-      if (process.platform !== 'win32') {
-        header.uid = opts.uid == null ? header.uid : opts.uid
-        header.gid = opts.gid == null ? header.gid : opts.gid
-      }
-      // Note: This mirrors logic in the fs read operations that are
-      // employed during tarball creation, in the fstream-npm module.
-      // It is duplicated here to handle tarballs that are created
-      // using other means, such as system tar or git archive.
-      if (header.type === 'file') {
-        const base = path.basename(header.name)
-        if (base === '.npmignore') {
-          sawIgnores[header.name] = true
-        } else if (base === '.gitignore') {
-          const npmignore = header.name.replace(/\.gitignore$/, '.npmignore')
-          if (!sawIgnores[npmignore]) {
-            // Rename, may be clobbered later.
-            header.name = npmignore
-          }
-        }
-      }
-      return header
-    },
-    ignore: makeIgnore(opts.log),
-    dmode: opts.dmode,
-    fmode: opts.fmode,
-    umask: opts.umask,
-    strip: 1
-  }))
+  return tar.x({
+    cwd: dest,
+    filter: (name, entry) => !entry.header.type.match(/^.*link$/i),
+    strip: 1,
+    onwarn: msg => opts.log && opts.log.warn('tar', msg),
+    onentry: makeOnEntry(opts),
+    preserveOwner: opts.uid != null || opts.gid != null
+  })
 }
 
-function makeIgnore (log) {
-  const sawIgnores = {}
-  return (name, header) => _ignore(name, header, sawIgnores, log)
+function makeOnEntry (opts) {
+  const sawEntry = {}
+  return entry => _onentry(entry, sawEntry, opts)
 }
 
-function _ignore (name, header, sawIgnores, logger) {
-  if (header.type.match(/^.*link$/)) {
-    if (logger) {
-      logger.warn(
-        'extract-stream',
-        'excluding symbolic link',
-        header.name, '->', header.linkname)
-    }
-    return true
+function _onentry (entry, sawIgnores, opts) {
+  if (process.getuid) {
+    entry.uid = opts.uid == null ? entry.uid : opts.uid
+    entry.gid = opts.gid == null ? entry.gid : opts.gid
   }
-
-  return false
+  if (entry.type.toLowerCase() === 'file') {
+    entry.mode = opts.fmode & ~(opts.umask || 0)
+  } else if (entry.type.toLowerCase() === 'directory') {
+    entry.mode = opts.dmode & ~(opts.umask || 0)
+  }
 }

lib/fetchers/registry/fetch.js

@@ -25,2 +25,3 @@ 'use strict'
     localAddress: opts.localAddress,
+    maxSockets: opts.maxSockets,
     memoize: opts.memoize,
@@ -27,0 +28,0 @@ noProxy: opts.noProxy,

lib/finalize-manifest.js

@@ -9,3 +9,2 @@ 'use strict'
 const finished = BB.promisify(require('mississippi').finished)
-const gunzip = require('./util/gunzip-maybe')
 const minimatch = require('minimatch')
@@ -17,7 +16,7 @@ const normalize = require('normalize-package-data')
 const ssri = require('ssri')
-const tar = require('tar-stream')
+const tar = require('tar')
 
 // `finalizeManifest` takes as input the various kinds of manifests that
-// manifest handlers ('lib/handlers/*/manifest.js') return, and makes sure they
-// are:
+// manifest handlers ('lib/fetchers/*.js#manifest()') return, and makes sure
+// they are:
 //
@@ -154,7 +153,3 @@ // * filled out with any required data that the handler couldn't fill in
     const tarStream = fetchFromManifest(pkg, spec, opts)
-    const extracted = needsExtract && tar.extract()
-    extracted && extracted.on('entry', (h, str, next) => {
-      // Drain it
-      str.on('data', () => {}).on('end', next).on('error', next)
-    })
+    const extracted = needsExtract && new tar.Parse()
     return BB.join(
@@ -165,3 +160,3 @@ needsShrinkwrap && jsonFromStream('npm-shrinkwrap.json', extracted),
       needsHash && ssri.fromStream(tarStream, { algorithms: ['sha1'] }),
-      needsExtract && pipe(tarStream, gunzip(), extracted),
+      needsExtract && pipe(tarStream, extracted),
       (sr, mani, paths, hash) => {
@@ -171,4 +166,3 @@ const extraProps = mani || {}
         // drain out the rest of the tarball
-        tarStream.unpipe()
-        tarStream.on('data', () => {})
+        tarStream.resume()
         // if we have directories.bin, we need to collect any matching files
@@ -207,16 +201,14 @@ // to add to bin
     dataStream.on('error', cb)
-    dataStream.on('finish', cb)
-    dataStream.on('entry', function handler (header, stream, next) {
-      const filePath = header.name.replace(/[^/]+\//, '')
+    dataStream.on('close', cb)
+    dataStream.on('entry', entry => {
+      const filePath = entry.header.path.replace(/[^/]+\//, '')
       if (filePath !== filename) {
-        next()
+        entry.resume()
       } else {
         let data = ''
-        stream.on('data', d => { data += d })
-        stream.on('error', cb)
-        finished(stream).then(() => {
-          dataStream.removeListener('entry', handler)
+        entry.on('data', d => { data += d })
+        entry.on('error', cb)
+        finished(entry).then(() => {
           try {
             cb(null, JSON.parse(data))
-            next()
           } catch (err) {
@@ -226,3 +218,2 @@ cb(err)
         }, err => {
-          dataStream.removeListener('entry', handler)
           cb(err)
@@ -239,8 +230,7 @@ })
     dataStream.on('error', cb)
-    dataStream.on('finish', () => cb(null, paths))
-    dataStream.on('entry', function handler (header, stream, next) {
-      const filePath = header.name.replace(/[^/]+\//, '')
-      stream.on('data', () => {})
+    dataStream.on('close', () => cb(null, paths))
+    dataStream.on('entry', function handler (entry) {
+      const filePath = entry.header.path.replace(/[^/]+\//, '')
+      entry.resume()
       paths.push(filePath)
-      next()
     })
@@ -247,0 +237,0 @@ })

lib/util/pack-dir.js

@@ -8,4 +8,5 @@ 'use strict'
 const optCheck = require('./opt-check')
+const packlist = require('npm-packlist')
 const pipe = BB.promisify(require('mississippi').pipe)
-const tar = require('tar-fs')
+const tar = require('tar')
 
@@ -17,18 +18,7 @@ module.exports = packDir
   const packer = opts.dirPacker
-  ? opts.dirPacker(manifest, dir)
-  : tar.pack(dir, {
-    map: header => {
-      header.name = 'package/' + header.name
-      header.mtime = 0 // make tarballs idempotent
-      return header
-    },
-    ignore: (name) => {
-      return name.match(/\.git/)
-    }
-  })
+  ? BB.resolve(opts.dirPacker(manifest, dir))
+  : mkPacker(dir)
 
   if (!opts.cache) {
-    return pipe(packer, target).catch(err => {
-      throw err
-    })
+    return packer.then(packer => pipe(packer, target))
   } else {
@@ -40,7 +30,18 @@ const cacher = cacache.put.stream(
     })
-    return BB.all([
+    return packer.then(packer => BB.all([
       pipe(packer, cacher),
       pipe(packer, target)
-    ])
+    ]))
   }
 }
+
+function mkPacker (dir) {
+  return packlist({path: dir}).then(files => {
+    return tar.c({
+      cwd: dir,
+      gzip: true,
+      portable: true,
+      prefix: 'package/'
+    }, files)
+  })
+}

package.json

 {
   "name": "pacote",
-  "version": "4.0.0",
+  "version": "5.0.0",
   "description": "JavaScript package downloader",
@@ -53,2 +53,3 @@ "main": "index.js",
     "npm-package-arg": "^5.1.2",
+    "npm-packlist": "^1.1.6",
     "npm-pick-manifest": "^1.0.4",
@@ -60,20 +61,20 @@ "osenv": "^0.1.4",
     "safe-buffer": "^5.1.1",
-    "semver": "^5.3.0",
+    "semver": "^5.4.1",
     "ssri": "^4.1.6",
-    "tar-fs": "^1.15.3",
-    "tar-stream": "^1.5.4",
+    "tar": "^3.1.13",
     "unique-filename": "^1.1.0",
-    "which": "^1.2.12"
+    "which": "^1.3.0"
   },
   "devDependencies": {
     "mkdirp": "^0.5.1",
-    "nock": "^9.0.13",
+    "nock": "^9.0.14",
     "npmlog": "^4.1.2",
-    "nyc": "^11.0.3",
+    "nyc": "^11.1.0",
     "require-inject": "^1.4.2",
     "rimraf": "^2.5.4",
-    "standard": "^10.0.1",
+    "standard": "^10.0.3",
     "standard-version": "^4.2.0",
     "tacks": "^1.2.6",
-    "tap": "^10.7.0",
+    "tap": "^10.7.2",
+    "tar-stream": "^1.5.4",
     "weallbehave": "^1.2.0",
@@ -80,0 +81,0 @@ "weallcontribute": "^1.0.7"

README.md

@@ -25,3 +25,3 @@ # pacote [![npm version](https://img.shields.io/npm/v/pacote.svg)](https://npm.im/pacote) [![license](https://img.shields.io/npm/l/pacote.svg)](https://npm.im/pacote) [![Travis](https://img.shields.io/travis/zkat/pacote.svg)](https://travis-ci.org/zkat/pacote) [![AppVeyor](https://ci.appveyor.com/api/projects/status/github/zkat/pacote?svg=true)](https://ci.appveyor.com/project/zkat/pacote) [![Coverage Status](https://coveralls.io/repos/github/zkat/pacote/badge.svg?branch=latest)](https://coveralls.io/github/zkat/pacote?branch=latest)
   * [`options`](#options)
-  * [`clearMemoized`](#clear-memoized)
+  * [`clearMemoized`](#clearMemoized)
 
@@ -155,2 +155,3 @@ ### Example
 ##### `opts.log`
+##### `opts.maxSockets`
 
@@ -157,0 +158,0 @@ Default: `silentNpmLog`

No alert changes

Improved metrics

Total package byte prevSize

84561

increased by0.05%

Number of lines in readme file

160

increased by0.63%

Worsened metrics

Dev dependency count

13

increased by8.33%

Lines of code

1534

decreased by-1.48%

Dependency changes

• + Addednpm-packlist@^1.1.6

• + Addedtar@^3.1.13

• + Addedignore-walk@3.0.4(transitive)

• + Addedminipass@2.9.0(transitive)

• + Addedminizlib@1.3.3(transitive)

• + Addednpm-bundled@1.1.2(transitive)

• + Addednpm-normalize-package-bin@1.0.1(transitive)

• + Addednpm-packlist@1.4.8(transitive)

• + Addedtar@3.2.3(transitive)

• + Addedyallist@3.1.1(transitive)

• - Removedtar-fs@^1.15.3

• - Removedtar-stream@^1.5.4

• - Removedbl@1.2.3(transitive)

• - Removedbuffer-alloc@1.2.0(transitive)

• - Removedbuffer-alloc-unsafe@1.1.0(transitive)

• - Removedbuffer-fill@1.0.0(transitive)

• - Removedfs-constants@1.0.0(transitive)

• - Removedtar-fs@1.16.3(transitive)

• - Removedtar-stream@1.6.2(transitive)

• - Removedto-buffer@1.1.1(transitive)

• Updatedsemver@^5.4.1

• Updatedwhich@^1.3.0