Socket
Socket
Sign inDemoInstall

make-fetch-happen

Package Overview
Dependencies
Maintainers
1
Versions
105
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

make-fetch-happen - npm Package Compare versions

Comparing version 2.2.5 to 2.2.6

agent.js

2

cache.js
'use strict'
const cacache = require('cacache')
const fetch = require('node-fetch')
const fetch = require('node-fetch-npm')
const pipe = require('mississippi').pipe

@@ -6,0 +6,0 @@ const ssri = require('ssri')

@@ -5,2 +5,13 @@ # Change Log

<a name="2.2.6"></a>
## [2.2.6](https://github.com/zkat/make-fetch-happen/compare/v2.2.5...v2.2.6) (2017-04-26)
### Bug Fixes
* **agent:** check uppercase & lowercase proxy env (#24) ([acf2326](https://github.com/zkat/make-fetch-happen/commit/acf2326)), closes [#22](https://github.com/zkat/make-fetch-happen/issues/22)
* **deps:** switch to node-fetch-npm and stop bundling ([3db603b](https://github.com/zkat/make-fetch-happen/commit/3db603b))
<a name="2.2.5"></a>

@@ -7,0 +18,0 @@ ## [2.2.5](https://github.com/zkat/make-fetch-happen/compare/v2.2.4...v2.2.5) (2017-04-23)

@@ -5,4 +5,3 @@ 'use strict'

const CachePolicy = require('http-cache-semantics')
const fetch = require('node-fetch')
const LRU = require('lru-cache')
const fetch = require('node-fetch-npm')
const pkg = require('./package.json')

@@ -12,5 +11,7 @@ const retry = require('promise-retry')

const Stream = require('stream')
const url = require('url')
const getAgent = require('./agent')
const setWarning = require('./warning')
const USER_AGENT = `${pkg.name}/${pkg.version} (+https://npm.im/${pkg.name})`
const RETRY_ERRORS = [

@@ -37,17 +38,8 @@ 'ECONNRESET', // remote socket closed on us

}
function defaultedFetch (uri, opts) {
let finalOpts
if (opts && _opts) {
finalOpts = {}
Object.keys(_opts).forEach(k => { finalOpts[k] = _opts[k] })
Object.keys(opts).forEach(k => { finalOpts[k] = opts[k] })
} else if (opts) {
finalOpts = opts
} else if (_opts) {
finalOpts = _opts
} else {
finalOpts = {}
}
const finalOpts = Object.assign({}, _opts || {}, opts || {})
return fetch(uri || _uri, finalOpts)
}
defaultedFetch.defaults = fetch.defaults

@@ -57,29 +49,15 @@ return defaultedFetch

function cachingFetch (uri, _opts) {
const opts = {}
Object.keys(_opts || {}).forEach(k => { opts[k] = _opts[k] })
opts.method = (opts.method || 'GET').toUpperCase()
if (typeof opts.cacheManager === 'string' && !Cache) {
// Default cacache-based cache
Cache = require('./cache')
function initializeCache (opts) {
if (typeof opts.cacheManager === 'string') {
if (!Cache) {
// Default cacache-based cache
Cache = require('./cache')
}
opts.cacheManager = new Cache(opts.cacheManager)
}
if (opts.integrity && !ssri) {
ssri = require('ssri')
}
if (opts.retry && typeof opts.retry === 'number') {
opts.retry = {retries: opts.retry}
} else if (opts.retry === false) {
opts.retry = {retries: 0}
}
opts.cacheManager = opts.cacheManager && (
typeof opts.cacheManager === 'string'
? new Cache(opts.cacheManager)
: opts.cacheManager
)
opts.cache = opts.cacheManager && (opts.cache || 'default')
if (
opts.cacheManager &&
opts.cache === 'default' &&
isConditional(opts.headers || {})
) {
opts.cache = opts.cache || 'default'
if (opts.cache === 'default' && isHeaderConditional(opts.headers)) {
// If header list contains `If-Modified-Since`, `If-None-Match`,

@@ -90,8 +68,42 @@ // `If-Unmodified-Since`, `If-Match`, or `If-Range`, fetch will set cache

}
if (
(opts.method === 'GET' || opts.method === 'HEAD') &&
}
function configureOptions (_opts) {
const opts = Object.assign({}, _opts || {})
opts.method = (opts.method || 'GET').toUpperCase()
if (opts.retry && typeof opts.retry === 'number') {
opts.retry = { retries: opts.retry }
}
if (opts.retry === false) {
opts.retry = { retries: 0 }
}
if (opts.cacheManager) {
initializeCache(opts)
}
return opts
}
function initializeSsri () {
if (!ssri) {
ssri = require('ssri')
}
}
function cachingFetch (uri, _opts) {
const opts = configureOptions(_opts)
if (opts.integrity) {
initializeSsri()
}
const isCachable = (opts.method === 'GET' || opts.method === 'HEAD') &&
opts.cacheManager &&
opts.cache !== 'no-store' &&
opts.cache !== 'reload'
) {
if (isCachable) {
const req = new fetch.Request(uri, {

@@ -101,2 +113,3 @@ method: opts.method,

})
return opts.cacheManager.match(req, opts).then(res => {

@@ -118,40 +131,45 @@ if (res) {

}
if (opts.cache === 'default' && !isStale(req, res)) {
return res
}
if (opts.cache === 'default' || opts.cache === 'no-cache') {
return conditionalFetch(req, res, opts)
}
if (opts.cache === 'force-cache' || opts.cache === 'only-if-cached') {
// 112 Disconnected operation
// SHOULD be included if the cache is intentionally disconnected from
// the rest of the network for a period of time.
// (https://tools.ietf.org/html/rfc2616#section-14.46)
setWarning(res, 112, 'Disconnected operation')
return res
}
}
if (res && opts.cache === 'default' && !isStale(req, res)) {
return res
} else if (res && (opts.cache === 'default' || opts.cache === 'no-cache')) {
return condFetch(req, res, opts)
} else if (res && (
opts.cache === 'force-cache' || opts.cache === 'only-if-cached'
)) {
// 112 Disconnected operation
// SHOULD be included if the cache is intentionally disconnected from
// the rest of the network for a period of time.
// (https://tools.ietf.org/html/rfc2616#section-14.46)
setWarning(res, 112, 'Disconnected operation')
return res
} else if (!res && opts.cache === 'only-if-cached') {
const err = new Error(
`request to ${
uri
} failed: cache mode is 'only-if-cached' but no cached response available.`
)
if (!res && opts.cache === 'only-if-cached') {
const errorMsg = `request to ${
uri
} failed: cache mode is 'only-if-cached' but no cached response available.`
const err = new Error(errorMsg)
err.code = 'ENOTCACHED'
throw err
} else {
// Missing cache entry, or mode is default (if stale), reload, no-store
return remoteFetch(req.url, opts)
}
// Missing cache entry, or mode is default (if stale), reload, no-store
return remoteFetch(req.url, opts)
})
} else {
return remoteFetch(uri, opts)
}
return remoteFetch(uri, opts)
}
function adaptHeaders (headers) {
const newHs = {}
for (let k of headers.keys()) {
newHs[k] = headers.get(k)
function iterableToObject (iter) {
const obj = {}
for (let k of iter.keys()) {
obj[k] = iter.get(k)
}
return newHs
return obj
}

@@ -163,11 +181,10 @@

method: req.method,
headers: adaptHeaders(req.headers)
headers: iterableToObject(req.headers)
}
const _res = {
status: res.status,
headers: adaptHeaders(res.headers)
headers: iterableToObject(res.headers)
}
return new CachePolicy(_req, _res, {
shared: false
})
return new CachePolicy(_req, _res, { shared: false })
}

@@ -177,14 +194,20 @@

function isStale (req, res) {
if (!res) { return null }
if (!res) {
return null
}
const _req = {
url: req.url,
method: req.method,
headers: adaptHeaders(req.headers)
headers: iterableToObject(req.headers)
}
const policy = makePolicy(req, res)
policy._responseTime = new Date(
res.headers.get('x-local-cache-time') ||
const responseTime = res.headers.get('x-local-cache-time') ||
res.headers.get('date') ||
0 // better to pretend everything is stale
)
0
policy._responseTime = new Date(responseTime)
const bool = !policy.satisfiesWithoutRevalidation(_req)

@@ -198,78 +221,80 @@ return bool

function condFetch (req, cachedRes, opts) {
let newHeaders = {}
Object.keys(opts.headers || {}).forEach(k => {
newHeaders[k] = opts.headers[k]
})
const policy = makePolicy(req, cachedRes)
function conditionalFetch (req, cachedRes, opts) {
const _req = {
url: req.url,
method: req.method,
headers: newHeaders
headers: Object.assign({}, opts.headers || {})
}
const policy = makePolicy(req, cachedRes)
opts.headers = policy.revalidationHeaders(_req)
return remoteFetch(req.url, opts).then(condRes => {
const revaled = policy.revalidatedPolicy(_req, {
status: condRes.status,
headers: adaptHeaders(condRes.headers)
})
if (condRes.status >= 500 && !mustRevalidate(cachedRes)) {
// 111 Revalidation failed
// MUST be included if a cache returns a stale response because an
// attempt to revalidate the response failed, due to an inability to
// reach the server.
// (https://tools.ietf.org/html/rfc2616#section-14.46)
setWarning(cachedRes, 111, `Revalidation failed`)
return cachedRes
} else if (condRes.status === 304) {
condRes.body = cachedRes.body
return opts.cacheManager.put(req, condRes, opts).then(newRes => {
newRes.headers = new fetch.Headers(revaled.policy.responseHeaders())
return newRes
return remoteFetch(req.url, opts)
.then(condRes => {
const revalidatedPolicy = policy.revalidatedPolicy(_req, {
status: condRes.status,
headers: iterableToObject(condRes.headers)
})
} else {
if (condRes.status >= 500 && !mustRevalidate(cachedRes)) {
// 111 Revalidation failed
// MUST be included if a cache returns a stale response because an
// attempt to revalidate the response failed, due to an inability to
// reach the server.
// (https://tools.ietf.org/html/rfc2616#section-14.46)
setWarning(cachedRes, 111, 'Revalidation failed')
return cachedRes
}
if (condRes.status === 304) { // 304 Not Modified
condRes.body = cachedRes.body
return opts.cacheManager.put(req, condRes, opts)
.then(newRes => {
newRes.headers = new fetch.Headers(revalidatedPolicy.policy.responseHeaders())
return newRes
})
}
return condRes
}
}).then(res => {
return res
}).catch(err => {
if (mustRevalidate(cachedRes)) {
throw err
} else {
// 111 Revalidation failed
// MUST be included if a cache returns a stale response because an
// attempt to revalidate the response failed, due to an inability to
// reach the server.
// (https://tools.ietf.org/html/rfc2616#section-14.46)
setWarning(cachedRes, 111, `Revalidation failed`)
// 199 Miscellaneous warning
// The warning text MAY include arbitrary information to be presented to
// a human user, or logged. A system receiving this warning MUST NOT take
// any automated action, besides presenting the warning to the user.
// (https://tools.ietf.org/html/rfc2616#section-14.46)
setWarning(
cachedRes, 199, `Miscellaneous Warning ${err.code}: ${err.message}`)
return cachedRes
}
})
})
.then(res => res)
.catch(err => {
if (mustRevalidate(cachedRes)) {
throw err
} else {
// 111 Revalidation failed
// MUST be included if a cache returns a stale response because an
// attempt to revalidate the response failed, due to an inability to
// reach the server.
// (https://tools.ietf.org/html/rfc2616#section-14.46)
setWarning(cachedRes, 111, 'Revalidation failed')
// 199 Miscellaneous warning
// The warning text MAY include arbitrary information to be presented to
// a human user, or logged. A system receiving this warning MUST NOT take
// any automated action, besides presenting the warning to the user.
// (https://tools.ietf.org/html/rfc2616#section-14.46)
setWarning(
cachedRes,
199,
`Miscellaneous Warning ${err.code}: ${err.message}`
)
return cachedRes
}
})
}
function setWarning (reqOrRes, code, message, replace) {
// Warning = "Warning" ":" 1#warning-value
// warning-value = warn-code SP warn-agent SP warn-text [SP warn-date]
// warn-code = 3DIGIT
// warn-agent = ( host [ ":" port ] ) | pseudonym
// ; the name or pseudonym of the server adding
// ; the Warning header, for use in debugging
// warn-text = quoted-string
// warn-date = <"> HTTP-date <">
// (https://tools.ietf.org/html/rfc2616#section-14.46)
reqOrRes.headers[replace ? 'set' : 'append'](
'Warning',
`${code} ${url.parse(reqOrRes.url).host} ${
JSON.stringify(message)
} ${
JSON.stringify(new Date().toUTCString())
}`
)
function remoteFetchHandleIntegrity (res, integrity) {
const oldBod = res.body
const newBod = ssri.integrityStream({
integrity
})
oldBod.pipe(newBod)
res.body = newBod
oldBod.once('error', err => {
newBod.emit('error', err)
})
newBod.once('error', err => {
oldBod.emit('error', err)
})
}

@@ -279,12 +304,7 @@

const agent = getAgent(uri, opts)
let headers = {
const headers = Object.assign({
'connection': agent ? 'keep-alive' : 'close',
'user-agent': USER_AGENT
}
if (opts.headers) {
Object.keys(opts.headers).forEach(k => {
headers[k] = opts.headers[k]
})
}
headers = new fetch.Headers(headers)
}, opts.headers || {})
const reqOpts = {

@@ -295,3 +315,3 @@ agent,

follow: opts.follow,
headers,
headers: new fetch.Headers(headers),
method: opts.method,

@@ -302,196 +322,92 @@ redirect: opts.redirect,

}
return retry((retryHandler, attemptNum) => {
const req = new fetch.Request(uri, reqOpts)
return fetch(req).then(res => {
res.headers.set('x-fetch-attempts', attemptNum)
if (opts.integrity) {
const oldBod = res.body
const newBod = ssri.integrityStream({
integrity: opts.integrity
})
oldBod.pipe(newBod)
res.body = newBod
oldBod.once('error', err => {
newBod.emit('error', err)
})
newBod.once('error', err => {
oldBod.emit('error', err)
})
}
if (
opts.cacheManager &&
opts.cache !== 'no-store' &&
(req.method === 'GET' || req.method === 'HEAD') &&
makePolicy(req, res).storable() &&
// No other statuses should be stored!
res.status === 200
) {
return opts.cacheManager.put(req, res, opts)
} else if (opts.cacheManager && (
(req.method !== 'GET' && req.method !== 'HEAD')
)) {
return opts.cacheManager.delete(req).then(() => {
if (res.status >= 500) {
if (req.method === 'POST') {
return res
} else if (req.body instanceof Stream) {
return res
} else {
return retryHandler(res)
return retry(
(retryHandler, attemptNum) => {
const req = new fetch.Request(uri, reqOpts)
return fetch(req)
.then(res => {
res.headers.set('x-fetch-attempts', attemptNum)
if (opts.integrity) {
remoteFetchHandleIntegrity(res, opts.integrity)
}
const isStream = req.body instanceof Stream
if (opts.cacheManager) {
const isMethodGetHead = req.method === 'GET' ||
req.method === 'HEAD'
const isCachable = opts.cache !== 'no-store' &&
isMethodGetHead &&
makePolicy(req, res).storable() &&
res.status === 200 // No other statuses should be stored!
if (isCachable) {
return opts.cacheManager.put(req, res, opts)
}
} else {
return res
if (!isMethodGetHead) {
return opts.cacheManager.delete(req).then(() => {
if (res.status >= 500 && req.method !== 'POST' && !isStream) {
return retryHandler(res)
}
return res
})
}
}
const isRetriable = req.method !== 'POST' &&
!isStream && (
res.status === 408 || // Request Timeout
res.status === 420 || // Enhance Your Calm (usually Twitter rate-limit)
res.status === 429 || // Too Many Requests ("standard" rate-limiting)
res.status >= 500 // Assume server errors are momentary hiccups
)
if (isRetriable) {
return retryHandler(res)
}
return res
})
} else if (
// Retriable + rate-limiting status codes
// When hitting an API with rate-limiting features,
// be sure to set the `retry` settings according to
// documentation for that.
res.status === 408 || // Request Timeout
res.status === 420 || // Enhance Your Calm (usually Twitter rate-limit)
res.status === 429 || // Too Many Requests ("standard" rate-limiting)
// Assume server errors are momentary hiccups
res.status >= 500
) {
if (req.method === 'POST') {
return res
} else if (req.body instanceof Stream) {
return res
} else {
return retryHandler(res)
}
} else {
return res
}
}).catch(err => {
const code = err.code === 'EPROMISERETRY'
? err.retried.code
: err.code
if (
req.method !== 'POST' && (
RETRY_ERRORS.indexOf(code) >= 0 ||
RETRY_TYPES.indexOf(err.type) >= 0
)
) {
return retryHandler(err)
} else {
throw err
}
})
}, opts.retry).catch(err => {
.catch(err => {
const code = err.code === 'EPROMISERETRY' ? err.retried.code : err.code
const isRetryError = RETRY_ERRORS.indexOf(code) === -1 &&
RETRY_TYPES.indexOf(err.type) === -1
if (req.method === 'POST' || isRetryError) {
throw err
}
return retryHandler(err)
})
},
opts.retry
).catch(err => {
if (err.status >= 400) {
return err
} else {
throw err
}
throw err
})
}
let AGENT_CACHE = new LRU({
max: 50
})
let HttpsAgent
let HttpAgent
function getAgent (uri, opts) {
const parsedUri = url.parse(typeof uri === 'string' ? uri : uri.url)
const isHttps = parsedUri.protocol === 'https:'
const pxuri = getProxyUri(uri, opts)
const key = [
`https:${isHttps}`,
pxuri
? `proxy:${pxuri.protocol}//${pxuri.host}:${pxuri.port}`
: '>no-proxy<',
`ca:${(isHttps && opts.ca) || '>no-ca<'}`,
`cert:${(isHttps && opts.cert) || '>no-cert<'}`,
`key:${(isHttps && opts.key) || '>no-key<'}`
].join(':')
if (opts.agent != null) {
// `agent: false` has special behavior!
return opts.agent
} else if (AGENT_CACHE.peek(key)) {
return AGENT_CACHE.get(key)
} else if (pxuri) {
const proxy = getProxy(pxuri, opts)
AGENT_CACHE.set(key, proxy)
return proxy
} else {
if (isHttps && !HttpsAgent) {
HttpsAgent = require('agentkeepalive').HttpsAgent
} else if (!isHttps && !HttpAgent) {
HttpAgent = require('agentkeepalive')
}
const agent = isHttps
? new HttpsAgent({
maxSockets: opts.maxSockets || 15,
ca: opts.ca,
cert: opts.cert,
key: opts.key
})
: new HttpAgent({
maxSockets: opts.maxSockets || 15
})
AGENT_CACHE.set(key, agent)
return agent
function isHeaderConditional (headers) {
if (!headers || typeof headers !== 'object') {
return false
}
}
function getProxyUri (uri, opts) {
const puri = url.parse(uri)
const proxy = opts.proxy || (
puri.protocol === 'https:' && process.env.https_proxy
) || (
puri.protocol === 'http:' && (
process.env.https_proxy || process.env.http_proxy || process.env.proxy
)
)
return !checkNoProxy(uri) && (
typeof proxy === 'string'
? url.parse(proxy)
: proxy
)
}
const modifiers = [
'if-modified-since',
'if-none-match',
'if-unmodified-since',
'if-match',
'if-range'
]
let HttpProxyAgent
let HttpsProxyAgent
let SocksProxyAgent
function getProxy (proxyUrl, opts) {
let popts = {
host: proxyUrl.hostname,
port: proxyUrl.port,
protocol: proxyUrl.protocol,
path: proxyUrl.path,
ca: opts.ca,
cert: opts.cert,
key: opts.key,
maxSockets: opts.maxSockets || 15
}
if (proxyUrl.protocol === 'http:') {
if (!HttpProxyAgent) { HttpProxyAgent = require('http-proxy-agent') }
return new HttpProxyAgent(popts)
} else if (proxyUrl.protocol === 'https:') {
if (!HttpsProxyAgent) { HttpsProxyAgent = require('https-proxy-agent') }
return new HttpsProxyAgent(popts)
} else if (proxyUrl.startsWith('socks')) {
if (!SocksProxyAgent) { SocksProxyAgent = require('socks-proxy-agent') }
return new SocksProxyAgent(popts)
}
return Object.keys(headers)
.some(h => modifiers.indexOf(h.toLowerCase()) !== -1)
}
function checkNoProxy (uri) {
// TODO
return false
}
function isConditional (headers) {
return Object.keys(headers).some(h => {
h = h.toLowerCase()
return (
h === 'if-modified-since' ||
h === 'if-none-match' ||
h === 'if-unmodified-since' ||
h === 'if-match' ||
h === 'if-range'
)
})
}
{
"name": "make-fetch-happen",
"version": "2.2.5",
"version": "2.2.6",
"description": "Opinionated, caching, retrying fetch client",

@@ -35,5 +35,2 @@ "main": "index.js",

"license": "CC0-1.0",
"bundleDependencies": [
"node-fetch"
],
"dependencies": {

@@ -47,3 +44,3 @@ "agentkeepalive": "^3.1.0",

"mississippi": "^1.2.0",
"node-fetch": "2.0.0-alpha.3",
"node-fetch-npm": "^1.0.0",
"promise-retry": "^1.1.1",

@@ -50,0 +47,0 @@ "socks-proxy-agent": "^2.0.0",

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc