make-fetch-happen - npm Package Compare versions

Comparing version 1.7.0 to 2.0.0

CHANGELOG.md

@@ -5,2 +5,28 @@ # Change Log
 
+<a name="2.0.0"></a>
+# [2.0.0](https://github.com/zkat/make-fetch-happen/compare/v1.7.0...v2.0.0) (2017-04-09)
+
+
+### Bug Fixes
+
+* **deps:** manually pull in newer node-fetch to avoid babel prod dep ([66e5e87](https://github.com/zkat/make-fetch-happen/commit/66e5e87))
+* **retry:** be more specific about when we retry ([a47b782](https://github.com/zkat/make-fetch-happen/commit/a47b782))
+
+
+### Features
+
+* **agent:** add ca/cert/key support to auto-agent (#15) ([57585a7](https://github.com/zkat/make-fetch-happen/commit/57585a7))
+
+
+### BREAKING CHANGES
+
+* **agent:** pac proxies are no longer supported.
+* **retry:** Retry logic has changes.
+
+* 404s, 420s, and 429s all retry now.
+* ENOTFOUND no longer retries.
+* Only ECONNRESET, ECONNREFUSED, EADDRINUSE, ETIMEDOUT, and `request-timeout` errors are retried.
+
+
+
 <a name="1.7.0"></a>
@@ -7,0 +33,0 @@ # [1.7.0](https://github.com/zkat/make-fetch-happen/compare/v1.6.0...v1.7.0) (2017-04-08)

index.js

@@ -5,3 +5,3 @@ 'use strict'
 const fetch = require('node-fetch')
-let ProxyAgent
+const LRU = require('lru-cache')
 const pkg = require('./package.json')
@@ -11,3 +11,17 @@ const retry = require('promise-retry')
 const Stream = require('stream')
+const url = require('url')
 
+const RETRY_ERRORS = [
+  'ECONNRESET', // remote socket closed on us
+  'ECONNREFUSED', // remote host refused to open connection
+  'EADDRINUSE', // failed to bind to a local port (proxy?)
+  'ETIMEDOUT' // someone in the transaction is WAY TOO SLOW
+  // Known codes we do NOT retry on:
+  // ENOTFOUND (getaddrinfo failure. Either bad hostname, or offline)
+]
+
+const RETRY_TYPES = [
+  'request-timeout'
+]
+
 // https://fetch.spec.whatwg.org/#http-network-or-cache-fetch
@@ -201,3 +215,3 @@ module.exports = cachingFetch
     } else {
-      setWarning(cachedRes, 111, `Unexpected error: ${err.message}`)
+      setWarning(cachedRes, 111, `${err.code}: ${err.message}`)
       return cachedRes
@@ -286,4 +300,14 @@ }
         })
-      } else if (res.status === 408 || res.status >= 500) {
-        // 408 === timeout
+      } else if (
+        // Retriable + rate-limiting status codes
+        // When hitting an API with rate-limiting features,
+        // be sure to set the `retry` settings according to
+        // documentation for that.
+        res.status === 404 || // Not Found ("subsequent requests permissible")
+        res.status === 408 || // Request Timeout
+        res.status === 420 || // Enhance Your Calm (usually Twitter rate-limit)
+        res.status === 429 || // Too Many Requests ("standard" rate-limiting)
+        // Assume server errors are momentary hiccups
+        res.status >= 500
+      ) {
         if (req.method === 'POST') {
@@ -300,3 +324,11 @@ return res
     }).catch(err => {
-      if (req.method !== 'POST') {
+      const code = err.code === 'EPROMISERETRY'
+      ? err.retried.code
+      : err.code
+      if (
+        req.method !== 'POST' && (
+          RETRY_ERRORS.indexOf(code) >= 0 ||
+          RETRY_TYPES.indexOf(err.type) >= 0
+        )
+      ) {
         return retryHandler(err)
@@ -308,3 +340,3 @@ } else {
   }, opts.retry === false ? { retries: 0 } : opts.retry).catch(err => {
-    if (err.status >= 500 || err.status === 408) {
+    if (err.status >= 400) {
       return err
@@ -317,28 +349,97 @@ } else {
 
-let httpsAgent
-let httpAgent
+let AGENT_CACHE = new LRU({
+  max: 50
+})
+let HttpsAgent
+let HttpAgent
 function getAgent (uri, opts) {
+  const parsedUri = url.parse(uri)
+  const isHttps = parsedUri.protocol === 'https:'
+  const pxuri = getProxyUri(uri, opts)
+  const key = [
+    `https:${isHttps}`,
+    pxuri
+    ? `proxy:${pxuri.protocol}//${pxuri.host}:${pxuri.port}`
+    : '>no-proxy<',
+    `ca:${(isHttps && opts.ca) || '>no-ca<'}`,
+    `cert:${(isHttps && opts.cert) || '>no-cert<'}`,
+    `key:${(isHttps && opts.key) || '>no-key<'}`
+  ].join(':')
   if (opts.agent != null) {
     // `agent: false` has special behavior!
     return opts.agent
-  } else if (opts.proxy) {
-    if (!ProxyAgent) {
-      ProxyAgent = require('proxy-agent')
-    }
-    return new ProxyAgent(opts.proxy)
-  } else if (uri.trim().startsWith('https:')) {
-    if (!httpsAgent) {
-      const Agent = require('agentkeepalive').HttpsAgent
-      httpsAgent = new Agent({maxSockets: 15})
-    }
-    return httpsAgent
+  } else if (AGENT_CACHE.peek(key)) {
+    return AGENT_CACHE.get(key)
+  } else if (pxuri) {
+    const proxy = getProxy(pxuri, opts)
+    AGENT_CACHE.set(key, proxy)
+    return proxy
   } else {
-    if (!httpAgent) {
-      const Agent = require('agentkeepalive')
-      httpAgent = new Agent({maxSockets: 15})
+    if (isHttps && !HttpsAgent) {
+      HttpsAgent = require('agentkeepalive').HttpsAgent
+    } else if (!isHttps && !HttpAgent) {
+      HttpAgent = require('agentkeepalive')
     }
-    return httpAgent
+    const agent = isHttps
+    ? new HttpsAgent({
+      maxSockets: opts.maxSockets || 15,
+      ca: opts.ca,
+      cert: opts.cert,
+      key: opts.key
+    })
+    : new HttpAgent({
+      maxSockets: opts.maxSockets || 15
+    })
+    AGENT_CACHE.set(key, agent)
+    return agent
   }
 }
 
+function getProxyUri (uri, opts) {
+  const puri = url.parse(uri)
+  const proxy = opts.proxy || (
+    puri.protocol === 'https:' && process.env.https_proxy
+  ) || (
+    puri.protocol === 'http:' && (
+      process.env.https_proxy || process.env.http_proxy || process.env.proxy
+    )
+  )
+  return !checkNoProxy(uri) && (
+    typeof proxy === 'string'
+    ? url.parse(proxy)
+    : proxy
+  )
+}
+
+let HttpProxyAgent
+let HttpsProxyAgent
+let SocksProxyAgent
+function getProxy (proxyUrl, opts) {
+  let popts = {
+    host: proxyUrl.hostname,
+    port: proxyUrl.port,
+    protocol: proxyUrl.protocol,
+    path: proxyUrl.path,
+    ca: opts.ca,
+    cert: opts.cert,
+    key: opts.key,
+    maxSockets: opts.maxSockets || 15
+  }
+  if (proxyUrl.protocol === 'http:') {
+    if (!HttpProxyAgent) { HttpProxyAgent = require('http-proxy-agent') }
+    return new HttpProxyAgent(popts)
+  } else if (proxyUrl.protocol === 'https:') {
+    if (!HttpsProxyAgent) { HttpsProxyAgent = require('https-proxy-agent') }
+    return new HttpsProxyAgent(popts)
+  } else if (proxyUrl.startsWith('socks')) {
+    if (!SocksProxyAgent) { SocksProxyAgent = require('socks-proxy-agent') }
+    return new SocksProxyAgent(popts)
+  }
+}
+
+function checkNoProxy (uri) {
+  // TODO
+  return false
+}
+
 function isConditional (headers) {
@@ -345,0 +446,0 @@ return Object.keys(headers).some(h => {

package.json

 {
   "name": "make-fetch-happen",
-  "version": "1.7.0",
+  "version": "2.0.0",
   "description": "Opinionated, caching, retrying fetch client",
@@ -8,3 +8,4 @@ "main": "index.js",
     "*.js",
-    "lib"
+    "lib",
+    "node-fetch-pkg.tgz"
   ],
@@ -41,8 +42,10 @@ "scripts": {
     "checksum-stream": "^1.0.2",
+    "http-proxy-agent": "^1.0.0",
+    "https-proxy-agent": "^1.0.0",
     "lru-cache": "^4.0.2",
     "mississippi": "^1.2.0",
-    "node-fetch": "^2.0.0-alpha.3",
+    "node-fetch": "file:node-fetch-pkg.tgz",
     "promise-retry": "^1.1.1",
-    "proxy-agent": "^2.0.0",
     "safe-buffer": "^5.0.1",
+    "socks-proxy-agent": "^2.0.0",
     "ssri": "^4.0.0"
@@ -49,0 +52,0 @@ },

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Network access

Supply chain risk

This module accesses the network.

Found 2 instances in 1 package

Environment variable access

Supply chain risk

Package accesses environment variables, which may be a sign of credential stuffing or data theft.

Found 3 instances in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

70124

increased by72.85%

Number of package files

7

increased by16.67%

Lines of code

675

increased by16.58%

Worsened metrics

Dependency count

13

increased by18.18%

Number of low supply chain risk alerts

6

increased by200%

Number of medium supply chain risk alerts

12

increased by20%

Dependency changes

• + Addedhttp-proxy-agent@^1.0.0

• + Addedhttps-proxy-agent@^1.0.0

• + Addedsocks-proxy-agent@^2.0.0

• + Addedagent-base@2.1.1(transitive)

• + Addedhttp-proxy-agent@1.0.0(transitive)

• + Addedhttps-proxy-agent@1.0.0(transitive)

• + Addedsemver@5.0.3(transitive)

• + Addedsocks-proxy-agent@2.1.1(transitive)

• - Removedproxy-agent@^2.0.0

• - Removedagent-base@4.3.0(transitive)

• - Removedast-types@0.16.1(transitive)

• - Removedbytes@3.1.2(transitive)

• - Removedco@4.6.0(transitive)

• - Removeddata-uri-to-buffer@1.2.0(transitive)

• - Removeddebug@3.1.03.2.7(transitive)

• - Removeddeep-is@0.1.4(transitive)

• - Removeddegenerator@1.0.4(transitive)

• - Removeddepd@2.0.0(transitive)

• - Removedes6-promise@4.2.8(transitive)

• - Removedes6-promisify@5.0.0(transitive)

• - Removedescodegen@1.14.3(transitive)

• - Removedesprima@3.1.34.0.1(transitive)

• - Removedestraverse@4.3.0(transitive)

• - Removedesutils@2.0.3(transitive)

• - Removedfast-levenshtein@2.0.6(transitive)

• - Removedfile-uri-to-path@1.0.0(transitive)

• - Removedftp@0.3.10(transitive)

• - Removedget-uri@2.0.4(transitive)

• - Removedhttp-errors@2.0.0(transitive)

• - Removedhttp-proxy-agent@2.1.0(transitive)

• - Removedhttps-proxy-agent@2.2.4(transitive)

• - Removediconv-lite@0.4.24(transitive)

• - Removedisarray@0.0.1(transitive)

• - Removedlevn@0.3.0(transitive)

• - Removedms@2.1.3(transitive)

• - Removednetmask@1.0.6(transitive)

• - Removednode-fetch@2.7.0(transitive)

• - Removedoptionator@0.8.3(transitive)

• - Removedpac-proxy-agent@2.0.2(transitive)

• - Removedpac-resolver@3.0.0(transitive)

• - Removedprelude-ls@1.1.2(transitive)

• - Removedproxy-agent@2.3.1(transitive)

• - Removedproxy-from-env@1.1.0(transitive)

• - Removedraw-body@2.5.2(transitive)

• - Removedreadable-stream@1.1.14(transitive)

• - Removedsafer-buffer@2.1.2(transitive)

• - Removedsetprototypeof@1.2.0(transitive)

• - Removedsocks-proxy-agent@3.0.1(transitive)

• - Removedsource-map@0.6.1(transitive)

• - Removedstatuses@2.0.1(transitive)

• - Removedstring_decoder@0.10.31(transitive)

• - Removedthunkify@2.1.2(transitive)

• - Removedtoidentifier@1.0.1(transitive)

• - Removedtr46@0.0.3(transitive)

• - Removedtslib@2.7.0(transitive)

• - Removedtype-check@0.3.2(transitive)

• - Removedunpipe@1.0.0(transitive)

• - Removedwebidl-conversions@3.0.1(transitive)

• - Removedwhatwg-url@5.0.0(transitive)

• - Removedword-wrap@1.2.5(transitive)

• - Removedxregexp@2.0.0(transitive)

• Updatednode-fetch@file:node-fetch-pkg.tgz