Socket
Socket
Sign inDemoInstall

make-fetch-happen

Package Overview
Dependencies
Maintainers
9
Versions
106
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

make-fetch-happen - npm Package Compare versions

Comparing version 6.1.0 to 7.0.0

utils/configure-options.js
utils/initialize-cache.js
utils/is-header-conditional.js
utils/iterable-to-object.js
utils/make-policy.js

92

agent.js

@@ -5,3 +5,3 @@ 'use strict'

let AGENT_CACHE = new LRU({ max: 50 })
const AGENT_CACHE = new LRU({ max: 50 })
let HttpsAgent

@@ -12,7 +12,20 @@ let HttpAgent

const getAgentTimeout = timeout =>
typeof timeout !== 'number' || !timeout ? 0 : timeout + 1
const getMaxSockets = maxSockets => maxSockets || 15
function getAgent (uri, opts) {
const parsedUri = url.parse(typeof uri === 'string' ? uri : uri.url)
const parsedUri = new url.URL(typeof uri === 'string' ? uri : uri.url)
const isHttps = parsedUri.protocol === 'https:'
const pxuri = getProxyUri(uri, opts)
const pxuri = getProxyUri(parsedUri.href, opts)
// If opts.timeout is zero, set the agentTimeout to zero as well. A timeout
// of zero disables the timeout behavior (OS limits still apply). Else, if
// opts.timeout is a non-zero value, set it to timeout + 1, to ensure that
// the node-fetch-npm timeout will always fire first, giving us more
// consistent errors.
const agentTimeout = getAgentTimeout(opts.timeout)
const agentMaxSockets = getMaxSockets(opts.maxSockets)
const key = [

@@ -27,3 +40,5 @@ `https:${isHttps}`,

`cert:${(isHttps && opts.cert) || '>no-cert<'}`,
`key:${(isHttps && opts.key) || '>no-key<'}`
`key:${(isHttps && opts.key) || '>no-key<'}`,
`timeout:${agentTimeout}`,
`maxSockets:${agentMaxSockets}`
].join(':')

@@ -45,17 +60,9 @@

if (isHttps && !HttpsAgent) {
HttpsAgent = require('agentkeepalive').HttpsAgent
} else if (!isHttps && !HttpAgent) {
if (!HttpsAgent) {
HttpAgent = require('agentkeepalive')
HttpsAgent = HttpAgent.HttpsAgent
}
// If opts.timeout is zero, set the agentTimeout to zero as well. A timeout
// of zero disables the timeout behavior (OS limits still apply). Else, if
// opts.timeout is a non-zero value, set it to timeout + 1, to ensure that
// the node-fetch-npm timeout will always fire first, giving us more
// consistent errors.
const agentTimeout = opts.timeout === 0 ? 0 : opts.timeout + 1
const agent = isHttps ? new HttpsAgent({
maxSockets: opts.maxSockets || 15,
maxSockets: agentMaxSockets,
ca: opts.ca,

@@ -68,3 +75,3 @@ cert: opts.cert,

}) : new HttpAgent({
maxSockets: opts.maxSockets || 15,
maxSockets: agentMaxSockets,
localAddress: opts.localAddress,

@@ -78,3 +85,3 @@ timeout: agentTimeout

function checkNoProxy (uri, opts) {
const host = url.parse(uri).hostname.split('.').reverse()
const host = new url.URL(uri).hostname.split('.').reverse()
let noproxy = (opts.noProxy || getProcessEnv('no_proxy'))

@@ -99,3 +106,5 @@ if (typeof noproxy === 'string') {

function getProcessEnv (env) {
if (!env) { return }
if (!env) {
return
}

@@ -105,3 +114,3 @@ let value

if (Array.isArray(env)) {
for (let e of env) {
for (const e of env) {
value = process.env[e] ||

@@ -123,13 +132,18 @@ process.env[e.toUpperCase()] ||

module.exports.getProxyUri = getProxyUri
function getProxyUri (uri, opts) {
const protocol = url.parse(uri).protocol
const protocol = new url.URL(uri).protocol
const proxy = opts.proxy || (
protocol === 'https:' && getProcessEnv('https_proxy')
) || (
protocol === 'http:' && getProcessEnv(['https_proxy', 'http_proxy', 'proxy'])
const proxy = opts.proxy ||
(
protocol === 'https:' &&
getProcessEnv('https_proxy')
) ||
(
protocol === 'http:' &&
getProcessEnv(['https_proxy', 'http_proxy', 'proxy'])
)
if (!proxy) { return null }
const parsedProxy = (typeof proxy === 'string') ? url.parse(proxy) : proxy
const parsedProxy = (typeof proxy === 'string') ? new url.URL(proxy) : proxy

@@ -139,18 +153,26 @@ return !checkNoProxy(uri, opts) && parsedProxy

const getAuth = u =>
u.username && u.password ? `${u.username}:${u.password}`
: u.username ? u.username
: null
const getPath = u => u.pathname + u.search + u.hash
let HttpProxyAgent
let HttpsProxyAgent
let SocksProxyAgent
module.exports.getProxy = getProxy
function getProxy (proxyUrl, opts, isHttps) {
let popts = {
const popts = {
host: proxyUrl.hostname,
port: proxyUrl.port,
protocol: proxyUrl.protocol,
path: proxyUrl.path,
auth: proxyUrl.auth,
path: getPath(proxyUrl),
auth: getAuth(proxyUrl),
ca: opts.ca,
cert: opts.cert,
key: opts.key,
timeout: opts.timeout === 0 ? 0 : opts.timeout + 1,
timeout: getAgentTimeout(opts.timeout),
localAddress: opts.localAddress,
maxSockets: opts.maxSockets || 15,
maxSockets: getMaxSockets(opts.maxSockets),
rejectUnauthorized: opts.strictSSL

@@ -173,4 +195,3 @@ }

}
}
if (proxyUrl.protocol.startsWith('socks')) {
} else if (proxyUrl.protocol.startsWith('socks')) {
if (!SocksProxyAgent) {

@@ -181,3 +202,10 @@ SocksProxyAgent = require('socks-proxy-agent')

return new SocksProxyAgent(popts)
} else {
throw Object.assign(
new Error(`unsupported proxy protocol: '${proxyUrl.protocol}'`),
{
url: proxyUrl.href
}
)
}
}

49

cache.js

@@ -16,8 +16,8 @@ 'use strict'

function cacheKey (req) {
const parsed = url.parse(req.url)
const parsed = new url.URL(req.url)
return `make-fetch-happen:request-cache:${
url.format({
protocol: parsed.protocol,
slashes: parsed.slashes,
host: parsed.host,
slashes: true,
port: parsed.port,
hostname: parsed.hostname,

@@ -42,3 +42,2 @@ pathname: parsed.pathname

match (req, opts) {
opts = opts || {}
const key = cacheKey(req)

@@ -78,3 +77,5 @@ return cacache.get.info(this._path, key).then(info => {

})
c.on('error', err => body.emit('error', err))
c.on('error', /* istanbul ignore next */ err => {
body.emit('error', err)
})
c.pipe(body)

@@ -88,3 +89,3 @@ }

.then(data => body.end(data))
.catch(err => {
.catch(/* istanbul ignore next */ err => {
body.emit('error', err)

@@ -132,13 +133,19 @@ })

cacache.get.stream.byDigest(this._path, info.integrity, cacheOpts),
cacache.put.stream(this._path, cacheKey(req), cacheOpts)
).promise().then(() => response)
cacache.put.stream(this._path, ckey, cacheOpts)
).promise().then(() => {
return response
})
})
}
const oldBody = response.body
const newBody = new MinipassFlush({
// the flush is the last thing in the pipeline. Build the pipeline
// back-to-front so we don't consume the data before we use it!
// We unshift in either a tee-stream to the cache put stream,
// or a collecter that dumps it to cache in one go, then the
// old body to bring in the data.
const newBody = new MinipassPipeline(new MinipassFlush({
flush () {
return cacheWritePromise
}
})
}))

@@ -157,3 +164,3 @@ let cacheWriteResolve, cacheWriteReject

cachePath,
cacheKey(req),
ckey,
data,

@@ -163,7 +170,3 @@ cacheOpts

})
oldBody
.on('error', er => collecter.emit('error', er))
.pipe(collecter)
.on('error', er => newBody.emit('error', er))
.pipe(newBody)
newBody.unshift(collecter)
} else {

@@ -173,13 +176,11 @@ const tee = new Minipass()

cachePath,
cacheKey(req),
ckey,
cacheOpts
)
tee.pipe(cacheStream)
tee.pipe(newBody)
cacheStream.promise().then(cacheWriteResolve, cacheWriteReject)
oldBody.on('error', er => tee.emit('error', er))
.pipe(tee)
.on('error', er => newBody.emit('error', er))
newBody.unshift(tee)
}
newBody.unshift(oldBody)
return Promise.resolve(new fetch.Response(newBody, response))

@@ -213,4 +214,4 @@ }

function matchDetails (req, cached) {
const reqUrl = url.parse(req.url)
const cacheUrl = url.parse(cached.url)
const reqUrl = new url.URL(req.url)
const cacheUrl = new url.URL(cached.url)
const vary = cached.resHeaders.get('Vary')

@@ -217,0 +218,0 @@ // https://tools.ietf.org/html/rfc7234#section-4.1

29

CHANGELOG.md

@@ -5,2 +5,31 @@ # Changelog

## [7.0.0](https://github.com/npm/make-fetch-happen/compare/v6.1.0...v7.0.0) (2019-12-17)
### ⚠ BREAKING CHANGES
* drops support for node v8, since it's EOL as of 2020-01-01
### Features
* **github:** added github actions with coveralls integration ([1913c1b](https://github.com/npm/make-fetch-happen/commit/1913c1b51aaac6044b4dab65b3d19ec943a35f39))
* updated fetch module; linting mostly; based on testing ([063f28e](https://github.com/npm/make-fetch-happen/commit/063f28ea1ac23f7e9d9d79e15949ca82b634ce97))
* **utils:** fixed configure-options based on testing ([9dd4f6f](https://github.com/npm/make-fetch-happen/commit/9dd4f6f108442dc247de44e1ddc0341edcb84c9b))
* fixed test dep requires; added mockRequire function to mock tests properly ([95de7a1](https://github.com/npm/make-fetch-happen/commit/95de7a171110907e30f41f489e4be983cd8184a5))
* refactored functions into utilities ([74620dd](https://github.com/npm/make-fetch-happen/commit/74620dd7c2262ac46d9b4f6ac2dc9ff45a4f19ee))
* updated dev deps; update tap; updated standard ([dce6eec](https://github.com/npm/make-fetch-happen/commit/dce6eece130fb20164a62eeabc6090811d8f14a4))
* updated fetch tests; linting, logic, added tests ([d50aeaf](https://github.com/npm/make-fetch-happen/commit/d50aeafebeb5d8f7118d7f6660208f40ac487804))
### Bug Fixes
* format cache key with new URL object shape ([21cb6cc](https://github.com/npm/make-fetch-happen/commit/21cb6cc968aabff8b5c5c02e3666fb093fd6578c))
* polish out an unnecessary URL object creation ([67a01d4](https://github.com/npm/make-fetch-happen/commit/67a01d46b2cacbadc22f49604ee524526cee3912)), closes [#14](https://github.com/npm/make-fetch-happen/issues/14)
* support user without password in proxy auth ([e24bbf9](https://github.com/npm/make-fetch-happen/commit/e24bbf935bc8a2c49070cdb2518e5ee290143191))
* updated 'files' property in package file ([945e40c](https://github.com/npm/make-fetch-happen/commit/945e40c7fbb59333e0c632c490683e4babc68dc1))
* Use WhatWG URL objects over deprecated legacy url API ([28aca97](https://github.com/npm/make-fetch-happen/commit/28aca97dfb63ca003ebf62d1b961771cfbb2481d))
* drop node 8 ([9fa7944](https://github.com/npm/make-fetch-happen/commit/9fa7944cbc603f3a194dfb440f519a7d5265653e))
## [6.1.0](https://github.com/npm/make-fetch-happen/compare/v6.0.1...v6.1.0) (2019-11-14)

@@ -7,0 +36,0 @@

172

index.js
'use strict'
let Cache
const url = require('url')
const CachePolicy = require('http-cache-semantics')
const fetch = require('minipass-fetch')

@@ -12,5 +10,10 @@ const pkg = require('./package.json')

const Minipass = require('minipass')
const MinipassPipeline = require('minipass-pipeline')
const getAgent = require('./agent')
const setWarning = require('./warning')
const configureOptions = require('./utils/configure-options')
const iterableToObject = require('./utils/iterable-to-object')
const makePolicy = require('./utils/make-policy')
const isURL = /^https?:/

@@ -63,41 +66,2 @@ const USER_AGENT = `${pkg.name}/${pkg.version} (+https://npm.im/${pkg.name})`

function initializeCache (opts) {
if (typeof opts.cacheManager === 'string') {
if (!Cache) {
// Default cacache-based cache
Cache = require('./cache')
}
opts.cacheManager = new Cache(opts.cacheManager, opts)
}
opts.cache = opts.cache || 'default'
if (opts.cache === 'default' && isHeaderConditional(opts.headers)) {
// If header list contains `If-Modified-Since`, `If-None-Match`,
// `If-Unmodified-Since`, `If-Match`, or `If-Range`, fetch will set cache
// mode to "no-store" if it is "default".
opts.cache = 'no-store'
}
}
function configureOptions (_opts) {
const opts = Object.assign({}, _opts || {})
opts.method = (opts.method || 'GET').toUpperCase()
if (opts.retry && typeof opts.retry === 'number') {
opts.retry = { retries: opts.retry }
}
if (opts.retry === false) {
opts.retry = { retries: 0 }
}
if (opts.cacheManager) {
initializeCache(opts)
}
return opts
}
function initializeSsri () {

@@ -118,6 +82,11 @@ if (!ssri) {

const isCachable = (opts.method === 'GET' || opts.method === 'HEAD') &&
opts.cacheManager &&
opts.cache !== 'no-store' &&
opts.cache !== 'reload'
const isCachable = (
(
opts.method === 'GET' ||
opts.method === 'HEAD'
) &&
Boolean(opts.cacheManager) &&
opts.cache !== 'no-store' &&
opts.cache !== 'reload'
)

@@ -182,30 +151,4 @@ if (isCachable) {

function iterableToObject (iter) {
const obj = {}
for (let k of iter.keys()) {
obj[k] = iter.get(k)
}
return obj
}
function makePolicy (req, res) {
const _req = {
url: req.url,
method: req.method,
headers: iterableToObject(req.headers)
}
const _res = {
status: res.status,
headers: iterableToObject(res.headers)
}
return new CachePolicy(_req, _res, { shared: false })
}
// https://tools.ietf.org/html/rfc7234#section-4.2
function isStale (req, res) {
if (!res) {
return null
}
const _req = {

@@ -220,4 +163,5 @@ url: req.url,

const responseTime = res.headers.get('x-local-cache-time') ||
res.headers.get('date') ||
0
/* istanbul ignore next - would be weird to get a 'stale'
* response that didn't come from cache with a cache time header */
(res.headers.get('date') || 0)

@@ -324,6 +268,3 @@ policy._responseTime = new Date(responseTime)

})
oldBod.pipe(newBod)
oldBod.on('error', er => newBod.emit('error'))
return new fetch.Response(newBod, res)
return new fetch.Response(new MinipassPipeline(oldBod, newBod), res)
}

@@ -334,3 +275,3 @@

const headers = Object.assign({
'connection': agent ? 'keep-alive' : 'close',
connection: agent ? 'keep-alive' : 'close',
'user-agent': USER_AGENT

@@ -356,3 +297,3 @@ }, opts.headers || {})

return fetch(req)
.then(res => {
.then((res) => {
if (opts.integrity) {

@@ -367,9 +308,13 @@ res = remoteFetchHandleIntegrity(res, opts.integrity)

if (opts.cacheManager) {
const isMethodGetHead = req.method === 'GET' ||
const isMethodGetHead = (
req.method === 'GET' ||
req.method === 'HEAD'
)
const isCachable = opts.cache !== 'no-store' &&
const isCachable = (
opts.cache !== 'no-store' &&
isMethodGetHead &&
makePolicy(req, res).storable() &&
res.status === 200 // No other statuses should be stored!
)

@@ -395,4 +340,6 @@ if (isCachable) {

const isRetriable = req.method !== 'POST' &&
!isStream && (
const isRetriable = (
req.method !== 'POST' &&
!isStream &&
(
res.status === 408 || // Request Timeout

@@ -403,2 +350,3 @@ res.status === 420 || // Enhance Your Calm (usually Twitter rate-limit)

)
)

@@ -413,5 +361,11 @@ if (isRetriable) {

if (!fetch.isRedirect(res.status) || opts.redirect === 'manual') {
if (!fetch.isRedirect(res.status)) {
return res
}
if (opts.redirect === 'manual') {
return res
}
// if (!fetch.isRedirect(res.status) || opts.redirect === 'manual') {
// return res
// }

@@ -434,13 +388,12 @@ // handle redirects - matches behavior of fetch: https://github.com/bitinn/node-fetch

const resolvedUrl = url.resolve(req.url, res.headers.get('location'))
let redirectURL = url.parse(resolvedUrl)
const resolvedUrlParsed = new url.URL(res.headers.get('location'), req.url)
const resolvedUrl = url.format(resolvedUrlParsed)
const redirectURL = (isURL.test(res.headers.get('location')))
? new url.URL(res.headers.get('location'))
: resolvedUrlParsed
if (isURL.test(res.headers.get('location'))) {
redirectURL = url.parse(res.headers.get('location'))
}
// Remove authorization if changing hostnames (but not if just
// changing ports or protocols). This matches the behavior of request:
// https://github.com/request/request/blob/b12a6245/lib/redirect.js#L134-L138
if (url.parse(req.url).hostname !== redirectURL.hostname) {
if (new url.URL(req.url).hostname !== redirectURL.hostname) {
req.headers.delete('authorization')

@@ -451,4 +404,12 @@ }

// use GET when following redirect
if (res.status === 303 ||
((res.status === 301 || res.status === 302) && req.method === 'POST')) {
if (
res.status === 303 ||
(
req.method === 'POST' &&
(
res.status === 301 ||
res.status === 302
)
)
) {
opts.method = 'GET'

@@ -468,6 +429,10 @@ opts.body = null

.catch(err => {
const code = err.code === 'EPROMISERETRY' ? err.retried.code : err.code
const code = (err.code === 'EPROMISERETRY')
? err.retried.code
: err.code
const isRetryError = RETRY_ERRORS.indexOf(code) === -1 &&
const isRetryError = (
RETRY_ERRORS.indexOf(code) === -1 &&
RETRY_TYPES.indexOf(err.type) === -1
)

@@ -495,18 +460,1 @@ if (req.method === 'POST' || isRetryError) {

}
function isHeaderConditional (headers) {
if (!headers || typeof headers !== 'object') {
return false
}
const modifiers = [
'if-modified-since',
'if-none-match',
'if-unmodified-since',
'if-match',
'if-range'
]
return Object.keys(headers)
.some(h => modifiers.indexOf(h.toLowerCase()) !== -1)
}

38

package.json
{
"name": "make-fetch-happen",
"version": "6.1.0",
"version": "7.0.0",
"description": "Opinionated, caching, retrying fetch client",

@@ -8,3 +8,4 @@ "main": "index.js",

"*.js",
"lib"
"lib",
"utils"
],

@@ -14,7 +15,6 @@ "scripts": {

"release": "standard-version -s",
"postrelease": "npm publish && git push --follow-tags",
"pretest": "standard",
"test": "tap --coverage --nyc-arg=--all --timeout=35 -J test/*.js",
"update-coc": "weallbehave -o . && git add CODE_OF_CONDUCT.md && git commit -m 'docs(coc): updated CODE_OF_CONDUCT.md'",
"update-contrib": "weallcontribute -o . && git add CONTRIBUTING.md && git commit -m 'docs(contributing): updated CONTRIBUTING.md'"
"prepublishOnly": "git push --follow-tags",
"postrelease": "npm publish",
"posttest": "standard",
"test": "tap test/*.js"
},

@@ -38,7 +38,7 @@ "repository": "https://github.com/npm/make-fetch-happen",

"dependencies": {
"agentkeepalive": "^3.4.1",
"agentkeepalive": "^4.1.0",
"cacache": "^13.0.1",
"http-cache-semantics": "^3.8.1",
"http-proxy-agent": "^2.1.0",
"https-proxy-agent": "^3.0.1",
"http-cache-semantics": "^4.0.3",
"http-proxy-agent": "^3.0.0",
"https-proxy-agent": "^4.0.0",
"lru-cache": "^5.1.1",

@@ -56,17 +56,15 @@ "minipass": "^3.0.0",

"mkdirp": "^0.5.1",
"nock": "^9.2.3",
"nock": "^11.7.0",
"npmlog": "^4.1.2",
"require-inject": "^1.4.2",
"rimraf": "^2.6.2",
"safe-buffer": "^5.1.1",
"standard": "^11.0.1",
"standard-version": "^7.0.0",
"rimraf": "^2.7.1",
"safe-buffer": "^5.2.0",
"standard": "^14.3.1",
"standard-version": "^7.0.1",
"tacks": "^1.2.6",
"tap": "^14.6.9",
"weallbehave": "^1.0.0",
"weallcontribute": "^1.0.7"
"tap": "^14.10.2"
},
"engines": {
"node": ">= 8"
"node": ">= 10"
}
}

8

README.md

@@ -1,5 +0,5 @@

# make-fetch-happen [![npm version](https://img.shields.io/npm/v/make-fetch-happen.svg)](https://npm.im/make-fetch-happen) [![license](https://img.shields.io/npm/l/make-fetch-happen.svg)](https://npm.im/make-fetch-happen) [![Travis](https://img.shields.io/travis/zkat/make-fetch-happen.svg)](https://travis-ci.org/zkat/make-fetch-happen) [![AppVeyor](https://ci.appveyor.com/api/projects/status/github/zkat/make-fetch-happen?svg=true)](https://ci.appveyor.com/project/zkat/make-fetch-happen) [![Coverage Status](https://coveralls.io/repos/github/zkat/make-fetch-happen/badge.svg?branch=latest)](https://coveralls.io/github/zkat/make-fetch-happen?branch=latest)
# make-fetch-happen
[![npm version](https://img.shields.io/npm/v/make-fetch-happen.svg)](https://npm.im/make-fetch-happen) [![license](https://img.shields.io/npm/l/make-fetch-happen.svg)](https://npm.im/make-fetch-happen) [![Travis](https://img.shields.io/travis/npm/make-fetch-happen.svg)](https://travis-ci.org/npm/make-fetch-happen) [![Coverage Status](https://coveralls.io/repos/github/npm/make-fetch-happen/badge.svg?branch=latest)](https://coveralls.io/github/npm/make-fetch-happen?branch=latest)
[`make-fetch-happen`](https://github.com/zkat/make-fetch-happen) is a Node.js
[`make-fetch-happen`](https://github.com/npm/make-fetch-happen) is a Node.js
library that wraps [`node-fetch-npm`](https://github.com/npm/node-fetch-npm) with additional

@@ -269,3 +269,3 @@ features [`node-fetch`](https://github.com/bitinn/node-fetch) doesn't intend to include, including HTTP Cache support, request

A string or `url.parse`-d URI to proxy through. Different Proxy handlers will be
A string or `new url.URL()`-d URI to proxy through. Different Proxy handlers will be
used depending on the proxy's protocol.

@@ -272,0 +272,0 @@

2

warning.js

@@ -15,3 +15,3 @@ const url = require('url')

// (https://tools.ietf.org/html/rfc2616#section-14.46)
const host = url.parse(reqOrRes.url).host
const host = new url.URL(reqOrRes.url).host
const jsonMessage = JSON.stringify(message)

@@ -18,0 +18,0 @@ const jsonDate = JSON.stringify(new Date().toUTCString())

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

About

Packages

npm

Go

Maven

PyPI

Rubygems

Stay in touch

Get open source security insights delivered straight into your inbox.

  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc