pacote - npm Package Compare versions

Comparing version 2.2.2 to 2.3.0

CHANGELOG.md

@@ -5,2 +5,18 @@ # Change Log
 
+<a name="2.3.0"></a>
+# [2.3.0](https://github.com/zkat/pacote/compare/v2.2.2...v2.3.0) (2017-04-26)
+
+
+### Bug Fixes
+
+* **git:** had ENOTSUP error on windows ([ee17c35](https://github.com/zkat/pacote/commit/ee17c35))
+* **memoization:** actually memoize package metadata ([e2078c0](https://github.com/zkat/pacote/commit/e2078c0))
+
+
+### Features
+
+* **memoization:** better packument memoization + pacote.clearMemoized() ([eb1bd4f](https://github.com/zkat/pacote/commit/eb1bd4f))
+
+
+
 <a name="2.2.2"></a>
@@ -7,0 +23,0 @@ ## [2.2.2](https://github.com/zkat/pacote/compare/v2.2.1...v2.2.2) (2017-04-24)

index.js

@@ -6,3 +6,4 @@ 'use strict'
   manifest: require('./manifest'),
-  prefetch: require('./prefetch')
+  prefetch: require('./prefetch'),
+  clearMemoized: require('./lib/fetch').clearMemoized
 }

lib/fetch.js

@@ -8,3 +8,4 @@ 'use strict'
   tarball: ['spec', 'opts'],
-  fromManifest: ['manifest', 'spec', 'opts']
+  fromManifest: ['manifest', 'spec', 'opts'],
+  clearMemoized () {}
 }, {name: 'Fetcher'})
@@ -42,2 +43,9 @@ module.exports = Fetcher
 
+module.exports.clearMemoized = clearMemoized
+function clearMemoized () {
+  Object.keys(fetchers).forEach(k => {
+    fetchers[k].clearMemoized()
+  })
+}
+
 function getFetcher (type) {
@@ -44,0 +52,0 @@ if (!TYPES.has(type)) {

lib/fetchers/registry/index.js

 'use strict'
 
+const cacache = require('cacache')
 const Fetcher = require('../../fetch')
@@ -20,3 +21,8 @@ const regManifest = require('./manifest')
     return regTarball.fromManifest(manifest, spec, opts)
+  },
+
+  clearMemoized () {
+    cacache.clearMemoized()
+    regManifest.clearMemoized()
   }
 })

lib/fetchers/registry/manifest.js

@@ -5,11 +5,14 @@ 'use strict'
 
-const checkWarnings = require('./check-warning-header')
-const fetch = require('make-fetch-happen')
+const fetch = require('./fetch')
+const LRU = require('lru-cache')
 const optCheck = require('../../util/opt-check')
 const pickManifest = require('npm-pick-manifest')
 const pickRegistry = require('./pick-registry')
-const registryKey = require('./registry-key')
 const ssri = require('ssri')
 const url = require('url')
 
+// Corgis are cute. 🐕🐶
+const CORGI_DOC = 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*'
+const JSON_DOC = 'application/json'
+
 module.exports = manifest
@@ -22,2 +25,15 @@ function manifest (spec, opts) {
 
+  return getManifest(uri, registry, spec, opts).then(manifest => {
+    return annotateManifest(uri, registry, manifest)
+  })
+}
+
+function metadataUrl (registry, name) {
+  const normalized = registry.slice(-1) !== '/'
+  ? registry + '/'
+  : registry
+  return url.resolve(normalized, name)
+}
+
+function getManifest (uri, registry, spec, opts) {
   return fetchPackument(uri, spec, registry, opts).then(packument => {
@@ -29,7 +45,3 @@ try {
     } catch (err) {
-      if (
-        err.code === 'ETARGET' &&
-        packument._cached &&
-        !opts.offline
-      ) {
+      if (err.code === 'ETARGET' && packument._cached && !opts.offline) {
         opts.log.silly(
@@ -50,94 +62,58 @@ 'registry:manifest',
     }
-  }).then(manifest => {
-    const shasum = manifest.dist && manifest.dist.shasum
-    manifest._integrity = manifest.dist && manifest.dist.integrity
-    if (!manifest._integrity && shasum) {
-      // Use legacy dist.shasum field if available.
-      manifest._integrity = ssri.fromHex(shasum, 'sha1').toString()
-    }
-    manifest._resolved = (
-      manifest.dist && manifest.dist.tarball
-    )
-    if (!manifest._resolved) {
-      const err = new Error(
-        `Manifest for ${manifest.name}@${manifest.version} from ${uri} is missing a tarball url (pkg.dist.tarball). Guessing a default.`
-      )
-      err.code = 'ENOTARBALL'
-      err.manifest = manifest
-      if (!manifest._warnings) { manifest._warnings = [] }
-      manifest._warnings.push(err.message)
-      manifest._resolved =
-      `${registry}/${manifest.name}/-/${manifest.name}-${manifest.version}.tgz`
-    }
-    return manifest
   })
 }
 
-function metadataUrl (registry, name) {
-  const normalized = registry.slice(-1) !== '/'
-  ? registry + '/'
-  : registry
-  return url.resolve(normalized, name)
+// TODO - make this an opt
+const MEMO = new LRU({
+  length: m => m._contentLength,
+  max: 200 * 1024 * 1024, // 200MB
+  maxAge: 30 * 1000 // 30s
+})
+
+module.exports.clearMemoized = clearMemoized
+function clearMemoized () {
+  MEMO.reset()
 }
 
 function fetchPackument (uri, spec, registry, opts) {
-  const startTime = Date.now()
-  const auth = opts.auth && opts.auth[registryKey(registry)]
-  const memo = opts.memoize && opts.memoize.get // `memoize` is a Map-like
-  const cacheMode = opts.offline
-  ? 'only-if-cached'
-  : opts.preferOffline
-  ? 'force-cache'
-  : opts.preferOnline
-  ? 'no-cache'
-  : 'default'
-  return memo && memo.has(uri) ? BB.resolve(memo.get(uri)).then(p => {
-    opts.log.http('registry:manifest', `MEMOIZED ${uri}`)
-    return p
-  }) : fetch(uri, {
-    agent: opts.agent,
-    cache: cacheMode,
-    cacheManager: opts.cache,
+  if (MEMO.has(uri)) {
+    return BB.resolve(MEMO.get(uri))
+  }
+
+  return fetch(uri, registry, Object.assign({
     headers: {
-      accept: opts.fullMetadata
-      ? 'application/json'
-      // Corgis are cute. 🐕🐶
-      : 'application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*',
-      authorization: (auth && auth.token && `Bearer ${auth.token}`) || '',
-      'npm-in-ci': opts.isFromCI,
-      'npm-scope': opts.projectScope,
-      'user-agent': opts.userAgent,
       'pacote-req-type': 'packument',
       'pacote-pkg-id': `registry:${manifest.name}`,
-      'referer': opts.refer
-    },
-    proxy: opts.proxy,
-    retry: opts.retry,
-    timeout: opts.timeout
-  }).then(res => res.json().then(packument => {
-    if (res.headers.has('npm-notice') && !res.headers.has('x-local-cache')) {
-      opts.log.warn('notice', res.headers.get('npm-notice'))
+      accept: opts.fullMetadata ? JSON_DOC : CORGI_DOC
     }
-    checkWarnings(res, registry, opts)
-    const elapsedTime = Date.now() - startTime
-    const attempt = res.headers.get('x-fetch-attempts')
-    const attemptStr = attempt && attempt > 1 ? ` attempt #${attempt}` : ''
-    const cacheStr = res.headers.get('x-local-cache') ? ' (from cache)' : ''
-    opts.log.http(
-      'registry:manifest',
-      `GET ${res.status} ${uri} ${elapsedTime}ms${attemptStr}${cacheStr}`
-    )
-    if (res.status >= 400) {
-      const err = new Error(`${res.statusText}: ${spec} (${uri}). ${JSON.stringify(packument)}`)
-      err.code = `E${res.status}`
-      err.spec = spec
-      err.packument = packument
-      err.response = res
-      throw err
-    }
+  }, opts)).then(res => res.json().then(packument => {
     packument._cached = res.headers.has('x-local-cache')
     packument._contentLength = +res.headers.get('content-length')
-    memo && memo.set(uri, packument)
+    MEMO.set(uri, packument)
     return packument
   }))
 }
+
+function annotateManifest (uri, registry, manifest) {
+  const shasum = manifest.dist && manifest.dist.shasum
+  manifest._integrity = manifest.dist && manifest.dist.integrity
+  if (!manifest._integrity && shasum) {
+    // Use legacy dist.shasum field if available.
+    manifest._integrity = ssri.fromHex(shasum, 'sha1').toString()
+  }
+  manifest._resolved = (
+    manifest.dist && manifest.dist.tarball
+  )
+  if (!manifest._resolved) {
+    const err = new Error(
+      `Manifest for ${manifest.name}@${manifest.version} from ${uri} is missing a tarball url (pkg.dist.tarball). Guessing a default.`
+    )
+    err.code = 'ENOTARBALL'
+    err.manifest = manifest
+    if (!manifest._warnings) { manifest._warnings = [] }
+    manifest._warnings.push(err.message)
+    manifest._resolved =
+    `${registry}/${manifest.name}/-/${manifest.name}-${manifest.version}.tgz`
+  }
+  return manifest
+}

lib/fetchers/registry/tarball.js

@@ -5,4 +5,3 @@ 'use strict'
 
-const checkWarnings = require('./check-warning-header')
-const fetch = require('make-fetch-happen')
+const fetch = require('./fetch')
 const manifest = require('./manifest')
@@ -13,3 +12,2 @@ const optCheck = require('../../util/opt-check')
 const pipe = BB.promisify(require('mississippi').pipe)
-const registryKey = require('./registry-key')
 const ssri = require('ssri')
@@ -41,28 +39,7 @@ const url = require('url')
   const uri = getTarballUrl(registry, manifest)
-  const auth = (
-    opts.auth &&
-    // If a tarball is on a different registry, don't leak creds
-    url.parse(uri).host === url.parse(registry).host &&
-    opts.auth[registryKey(registry)]
-  )
-  const startTime = Date.now()
-  fetch(uri, {
-    agent: opts.agent,
-    cache: opts.offline
-    ? 'only-if-cached'
-    : opts.preferOffline
-    ? 'force-cache'
-    : opts.preferOnline
-    ? 'no-cache'
-    : 'default',
-    cacheManager: opts.cache,
+  fetch(uri, registry, Object.assign({
     headers: {
-      authorization: (auth && auth.token && `Bearer ${auth.token}`) || '',
-      'npm-in-ci': opts.isFromCI,
-      'npm-scope': opts.projectScope,
-      'user-agent': opts.userAgent,
       'pacote-req-type': 'tarball',
       'pacote-pkg-id': `registry:${manifest.name}@${manifest.version}`,
-      'pacote-pkg-manifest': JSON.stringify(manifest),
-      'referer': opts.refer
+      'pacote-pkg-manifest': JSON.stringify(manifest)
     },
@@ -74,32 +51,7 @@ integrity: manifest._integrity,
       : 'sha1'
-    ],
-    proxy: opts.proxy,
-    retry: opts.retry,
-    timeout: opts.timeout
-  }).then(res => {
+    ]
+  }, opts)).then(res => {
     stream.emit('integrity', res.headers.get('x-local-cache-hash'))
     res.body.on('error', err => stream.emit('error', err))
-    if (res.headers.has('npm-notice') && !res.headers.has('x-local-cache')) {
-      opts.log.warn('notice', res.headers.get('npm-notice'))
-    }
-    checkWarnings(res, registry, opts)
-    if (res.status >= 400) {
-      const err = new Error(`${res.statusText}: tarball for ${spec} (${uri}).`)
-      err.code = `E${res.status}`
-      err.spec = spec
-      err.response = res
-      throw err
-    } else {
-      res.body.pipe(stream)
-      stream.on('end', () => {
-        const elapsedTime = Date.now() - startTime
-        const attempt = res.headers.get('x-fetch-attempts')
-        const attemptStr = attempt && attempt > 1 ? ` attempt #${attempt}` : ''
-        const cacheStr = res.headers.get('x-local-cache') ? ' (from cache)' : ''
-        opts.log.http(
-          'registry:tarball',
-          `GET ${res.status} ${uri} ${elapsedTime}ms${attemptStr}${cacheStr}`
-        )
-      })
-    }
+    res.body.pipe(stream)
   }).catch(err => stream.emit('error', err))
@@ -106,0 +58,0 @@ return stream

lib/util/git.js

@@ -179,6 +179,10 @@ 'use strict'
   const gitOpts = {
-    env: gitEnv(),
-    uid: opts.uid,
-    gid: opts.gid
+    env: gitEnv()
   }
+  if (!isNaN(opts.uid)) {
+    gitOpts.uid = +opts.uid
+  }
+  if (!isNaN(opts.gid)) {
+    gitOpts.gid = +opts.gid
+  }
   Object.keys(_gitOpts || {}).forEach(k => {
@@ -196,6 +200,10 @@ gitOpts[k] = _gitOpts[k]
   const gitOpts = {
-    env: gitEnv(),
-    uid: opts.uid,
-    gid: opts.gid
+    env: gitEnv()
   }
+  if (!isNaN(opts.uid)) {
+    gitOpts.uid = +opts.uid
+  }
+  if (!isNaN(opts.gid)) {
+    gitOpts.gid = +opts.gid
+  }
   Object.keys(_gitOpts).forEach(k => {
@@ -202,0 +210,0 @@ gitOpts[k] = _gitOpts[k]

package.json

 {
   "name": "pacote",
-  "version": "2.2.2",
+  "version": "2.3.0",
   "description": "JavaScript package downloader",
@@ -5,0 +5,0 @@ "main": "index.js",

README.md

@@ -25,2 +25,3 @@ # pacote [![npm version](https://img.shields.io/npm/v/pacote.svg)](https://npm.im/pacote) [![license](https://img.shields.io/npm/l/pacote.svg)](https://npm.im/pacote) [![Travis](https://img.shields.io/travis/zkat/pacote.svg)](https://travis-ci.org/zkat/pacote) [![AppVeyor](https://ci.appveyor.com/api/projects/status/github/zkat/pacote?svg=true)](https://ci.appveyor.com/project/zkat/pacote) [![Coverage Status](https://coveralls.io/repos/github/zkat/pacote/badge.svg?branch=latest)](https://coveralls.io/github/zkat/pacote?branch=latest)
   * [`options`](#options)
+  * [`clearMemoized`](#clear-memoized)
 
@@ -124,2 +125,13 @@ ### Example
 
+#### <a name="clearMemoized"></a> `> pacote.clearMemoized()`
+
+This utility function can be used to force pacote to release its references
+to any memoized data in its various internal caches. It might help free
+some memory.
+
+```javascript
+pacote.manifest(...).then(() => pacote.clearMemoized)
+
+```
+
 #### <a name="options"></a> `> options`
@@ -126,0 +138,0 @@

No alert changes

Improved metrics

Total package byte prevSize

68573

increased by0.92%

Number of package files

32

increased by3.23%

Lines of code

1482

increased by1.02%

Number of lines in readme file

159

increased by8.16%

Worsened metrics

Number of medium supply chain risk alerts

8

increased by14.29%

No dependency changes