exp-fetch
Advanced tools
Comparing version
148
index.js
"use strict"; | ||
const request = require("request"); | ||
const got = require("got"); | ||
const VError = require("verror"); | ||
@@ -27,23 +27,24 @@ const AsyncCache = require("exp-asynccache"); | ||
// Options | ||
var freeze = true; | ||
var deepFreeze = false; | ||
var cache = new AsyncCache(initCache({age: 60})); | ||
var cacheKeyFn = behavior.cacheKeyFn || calculateCacheKey; | ||
var cacheValueFn = behavior.cacheValueFn || passThrough; | ||
var maxAgeFn = behavior.maxAgeFn || passThrough; | ||
var onNotFound = behavior.onNotFound; | ||
var onError = behavior.onError; | ||
var onSuccess = behavior.onSuccess; | ||
var cacheNotFound = false; | ||
var logger = behavior.logger || dummyLogger(); | ||
var errorOnRemoteError = true; | ||
var contentType = (behavior.contentType || "json").toLowerCase(); | ||
var keepAliveAgent = behavior.agent; | ||
var followRedirect = true; | ||
var performClone = true; | ||
var maximumNumberOfRedirects = 10; | ||
var httpMethod = (behavior.httpMethod || "GET").toUpperCase(); | ||
var timeout = behavior.timeout || 20000; | ||
var stats = {calls: 0, misses: 0}; | ||
var globalHeaders = behavior.headers || {}; | ||
let freeze = true; | ||
let deepFreeze = false; | ||
let cache = new AsyncCache(initCache({age: 60})); | ||
const cacheKeyFn = behavior.cacheKeyFn || calculateCacheKey; | ||
const cacheValueFn = behavior.cacheValueFn || passThrough; | ||
const maxAgeFn = behavior.maxAgeFn || passThrough; | ||
const onNotFound = behavior.onNotFound; | ||
const onError = behavior.onError; | ||
const onSuccess = behavior.onSuccess; | ||
let cacheNotFound = false; | ||
const logger = behavior.logger || dummyLogger(); | ||
let errorOnRemoteError = true; | ||
const contentType = (behavior.contentType || "json").toLowerCase(); | ||
const keepAliveAgent = behavior.agent; | ||
let followRedirect = true; | ||
let performClone = true; | ||
const maximumNumberOfRedirects = 10; | ||
const httpMethod = (behavior.httpMethod || "GET").toUpperCase(); | ||
const timeout = behavior.timeout || 20000; | ||
const stats = {calls: 0, misses: 0}; | ||
const globalHeaders = behavior.headers || {}; | ||
const retry = "retry" in behavior ? behavior.retry : 0; | ||
@@ -53,3 +54,3 @@ function defaultRequestTimeFn(requestOptions, took) { | ||
} | ||
var requestTimeFn = behavior.requestTimeFn || defaultRequestTimeFn; | ||
const requestTimeFn = behavior.requestTimeFn || defaultRequestTimeFn; | ||
@@ -86,3 +87,3 @@ if (behavior.hasOwnProperty("clone")) { | ||
logger.info("404 Not Found for: %j", url); | ||
var notFoundAge = -1; | ||
let notFoundAge = -1; | ||
@@ -101,3 +102,3 @@ if (onNotFound) { | ||
logger.warning("HTTP Fetching error %d for: %j", res.statusCode, url); | ||
var errorAge = -1; | ||
let errorAge = -1; | ||
@@ -111,3 +112,3 @@ if (onError) { | ||
if (errorOnRemoteError) { | ||
var error = new VError("%s yielded %s (%s)", url, res.statusCode, util.inspect(content)); | ||
const error = new VError("%s yielded %s (%s)", url, res.statusCode, util.inspect(content)); | ||
error.statusCode = res.statusCode; | ||
@@ -122,6 +123,6 @@ | ||
function deepFreezeObj(obj) { | ||
var propNames = Object.getOwnPropertyNames(obj); | ||
const propNames = Object.getOwnPropertyNames(obj); | ||
propNames.forEach(function (name) { | ||
var prop = obj[name]; | ||
propNames.forEach((name) => { | ||
const prop = obj[name]; | ||
@@ -137,5 +138,5 @@ if (typeof prop === "object" && prop !== null) { | ||
function handleSuccess(url, cacheKey, res, content, resolvedCallback) { | ||
var maxAge = maxAgeFn(getMaxAge(res.headers["cache-control"]), cacheKey, res, content); | ||
const maxAge = maxAgeFn(getMaxAge(res.headers["cache-control"]), cacheKey, res, content); | ||
var typeOfContent = typeof content; | ||
const typeOfContent = typeof content; | ||
@@ -156,5 +157,5 @@ if (deepFreeze && typeOfContent === "object") { | ||
function handleRedirect(url, cacheKey, res, body, resolvedCallback) { | ||
var maxAge = maxAgeFn(getMaxAge(res.headers["cache-control"]), cacheKey, res, body); | ||
const maxAge = maxAgeFn(getMaxAge(res.headers["cache-control"]), cacheKey, res, body); | ||
var content = { | ||
const content = { | ||
statusCode: res.statusCode, | ||
@@ -167,32 +168,36 @@ headers: res.headers | ||
function performRequest(url, headers, explicitTimeout, body, redirectCount, callback, onRequestInit) { | ||
var cacheKey = cacheKeyFn(url, body); | ||
var startTime = new Date().getTime(); | ||
const cacheKey = cacheKeyFn(url, body); | ||
const startTime = new Date().getTime(); | ||
stats.calls++; | ||
cache.lookup(cacheKey, function (resolveFunction) { | ||
cache.lookup(cacheKey, (resolveFunction) => { | ||
stats.misses++; | ||
logger.debug("fetching %s cacheKey '%s'", url, cacheKey); | ||
var options = { | ||
const options = { | ||
url: url, | ||
json: contentType === "json", | ||
responseType: contentType === "json" ? contentType : undefined, | ||
agent: keepAliveAgent, | ||
followRedirect: false, | ||
retry, | ||
method: httpMethod, | ||
timeout: explicitTimeout || timeout, | ||
headers: headers | ||
headers: headers, | ||
cache: false | ||
}; | ||
if (body) { | ||
if (body && typeof body === "object") { | ||
options.json = body; | ||
} else if (body) { | ||
options.body = body; | ||
} | ||
var passOptions = { | ||
const passOptions = { | ||
url: options.url, | ||
json: options.json, | ||
method: options.method, | ||
responseType: contentType === "json" ? contentType : undefined, | ||
followRedirect: followRedirect, | ||
headers: options.headers | ||
}; | ||
if (onRequestInit && !onRequestInit.called) { | ||
onRequestInit(passOptions, cacheKey); | ||
@@ -206,20 +211,24 @@ } | ||
request(options, function (err, res, content) { | ||
if (err) return resolvedCallback(new VError(err, "Fetching error for: %j", url)); | ||
if (isRedirect(res)) return handleRedirect(url, cacheKey, res, content, resolvedCallback); | ||
if (res.statusCode === 404) { | ||
return handleNotFound(url, cacheKey, res, content, resolvedCallback); | ||
} else if (res.statusCode > 299) { | ||
return handleError(url, cacheKey, res, content, resolvedCallback); | ||
return request(options).then((res) => { | ||
if (isRedirect(res)) return handleRedirect(url, cacheKey, res, res.body, resolvedCallback); | ||
return parseResponse(res.body, contentType, (_, transformed) => { | ||
return handleSuccess(url, cacheKey, res, transformed, resolvedCallback); | ||
}); | ||
}).catch((err) => { | ||
if (err instanceof got.HTTPError) { | ||
if (err.response.statusCode === 404) { | ||
return handleNotFound(url, cacheKey, err.response, err.response.body, resolvedCallback); | ||
} else if (err.response.statusCode > 299) { | ||
return handleError(url, cacheKey, err.response, err.response.body, resolvedCallback); | ||
} | ||
} else if (err instanceof got.TimeoutError) { | ||
return resolvedCallback(new VError("ESOCKETTIMEDOUT")); | ||
} | ||
return parseResponse(content, contentType, function (_, transformed) { | ||
return handleSuccess(url, cacheKey, res, transformed, resolvedCallback); | ||
}); | ||
return resolvedCallback(err); | ||
}); | ||
}, function (err, response) { | ||
}, (err, response) => { | ||
if (followRedirect && isRedirect(response)) { | ||
if (redirectCount++ < maximumNumberOfRedirects) { | ||
var location = ensureAbsoluteUrl(response.headers, url); | ||
const location = ensureAbsoluteUrl(response.headers, url); | ||
return performRequest(location, headers, explicitTimeout, body, redirectCount, callback); | ||
@@ -236,5 +245,5 @@ } else { | ||
fetch: function (options, optionalBody, resultCallback) { | ||
var url = options; | ||
var headers = Object.assign({}, globalHeaders, options.headers); | ||
var explicitTimeout = null; | ||
let url = options; | ||
const headers = Object.assign({}, globalHeaders, options.headers); | ||
let explicitTimeout = null; | ||
if (typeof options === "object") { | ||
@@ -257,8 +266,2 @@ if (options.url) { | ||
} | ||
var onRequestInit = function () { | ||
if (behavior.onRequestInit) { | ||
behavior.onRequestInit.apply(null, arguments); | ||
} | ||
onRequestInit.called = true; | ||
}; | ||
@@ -268,4 +271,4 @@ if (resultCallback) { | ||
} else { | ||
return new Promise(function (resolve, reject) { | ||
performRequest(url, headers, explicitTimeout, optionalBody, 0, function (err, content) { | ||
return new Promise((resolve, reject) => { | ||
performRequest(url, headers, explicitTimeout, optionalBody, 0, (err, content) => { | ||
if (err) return reject(err); | ||
@@ -276,2 +279,9 @@ return resolve(content); | ||
} | ||
function onRequestInit() { | ||
if (behavior.onRequestInit) { | ||
behavior.onRequestInit.apply(null, arguments); | ||
} | ||
onRequestInit.called = true; | ||
} | ||
}, | ||
@@ -292,1 +302,5 @@ | ||
} | ||
function request({url, ...options}) { | ||
return got(url, {cache: false, ...options}); | ||
} |
"use strict"; | ||
var crypto = require("crypto"); | ||
const crypto = require("crypto"); | ||
@@ -4,0 +4,0 @@ function calculateCacheKey(url, body) { |
"use strict"; | ||
var path = require("path"); | ||
var currentPackage = {}; | ||
const path = require("path"); | ||
let currentPackage = {}; | ||
@@ -6,0 +6,0 @@ try { |
"use strict"; | ||
var url = require("url"); | ||
var util = require("util"); | ||
const url = require("url"); | ||
const util = require("util"); | ||
function ensureAbsoluteUrl(headers, uri) { | ||
var newLocation = url.parse(headers.location); | ||
var oldLocation = url.parse(uri); | ||
var protocol = newLocation.protocol || oldLocation.protocol; | ||
var host = newLocation.host || oldLocation.host; | ||
const newLocation = url.parse(headers.location); | ||
const oldLocation = url.parse(uri); | ||
const protocol = newLocation.protocol || oldLocation.protocol; | ||
const host = newLocation.host || oldLocation.host; | ||
@@ -11,0 +11,0 @@ return util.format("%s//%s%s", protocol, host, newLocation.path); |
@@ -53,4 +53,4 @@ "use strict"; | ||
} else { | ||
var maxAge = 1000 * Number(cacheConfig.age || cacheConfig.maxAge || 60); | ||
var lruCache = new LRU({ | ||
const maxAge = 1000 * Number(cacheConfig.age || cacheConfig.maxAge || 60); | ||
const lruCache = new LRU({ | ||
maxAge: maxAge, | ||
@@ -57,0 +57,0 @@ length: cacheConfig.length || defaultLengthFn, |
"use strict"; | ||
var maxAgeRexExp = /max-age=\s*\d+/; | ||
var maxAgeValueRexExp = /\d+/; | ||
const maxAgeRexExp = /max-age=\s*\d+/; | ||
const maxAgeValueRexExp = /\d+/; | ||
var noCacheKeywords = ["private", "max-age=0", "no-cache", "must-revalidate"]; | ||
const noCacheKeywords = ["private", "max-age=0", "no-cache", "must-revalidate"]; | ||
function containsAny(haystack, listOfHits) { | ||
for (var i = 0; i < listOfHits.length; i++) { | ||
for (let i = 0; i < listOfHits.length; i++) { | ||
if (haystack.indexOf(listOfHits[i]) > -1) { | ||
@@ -26,4 +26,4 @@ return true; | ||
var maxAge = null; | ||
var maxAgeStringMatch = maxAgeRexExp.exec(cacheHeader); | ||
let maxAge = null; | ||
let maxAgeStringMatch = maxAgeRexExp.exec(cacheHeader); | ||
if (maxAgeStringMatch) { | ||
@@ -30,0 +30,0 @@ maxAgeStringMatch = maxAgeValueRexExp.exec(maxAgeStringMatch[0]); |
"use strict"; | ||
var xml2js = require("xml2js"); | ||
const xml2js = require("xml2js"); | ||
@@ -4,0 +4,0 @@ function parseResponse(content, contentType, callback) { |
{ | ||
"name": "exp-fetch", | ||
"version": "4.2.0", | ||
"version": "5.0.0", | ||
"description": "A small pluggable fetch lib", | ||
"main": "index.js", | ||
"engines": { | ||
"node": ">=10" | ||
}, | ||
"scripts": { | ||
"test": "mocha -R dot && eslint . --cache" | ||
"test": "mocha -R dot", | ||
"posttest": "eslint . --cache" | ||
}, | ||
@@ -27,16 +25,19 @@ "repository": { | ||
"devDependencies": { | ||
"chai": "^4.1.2", | ||
"eslint": "^5.3.0", | ||
"mocha": "^9.1.0", | ||
"nock": "^11.9.0" | ||
"chai": "^4.2.0", | ||
"eslint": "^7.32.0", | ||
"mocha": "^9.1.1", | ||
"nock": "^13.1.3" | ||
}, | ||
"dependencies": { | ||
"clone": "^2.1.1", | ||
"exp-asynccache": "^1.2.1", | ||
"log": "^1.4.0", | ||
"lru-cache": "^4.1.1", | ||
"request": "^2.88.0", | ||
"exp-asynccache": "^2.0.0", | ||
"got": "^11.8.2", | ||
"lru-cache": "^6.0.0", | ||
"verror": "^1.10.0", | ||
"xml2js": "^0.4.19" | ||
} | ||
"xml2js": "^0.4.22" | ||
}, | ||
"files": [ | ||
"lib/", | ||
"index.js" | ||
] | ||
} |
fetch | ||
===== | ||
[](https://travis-ci.org/BonnierNews/exp-fetch) | ||
[](https://github.com/BonnierNews/exp-fetch/actions/workflows/build-latest.yaml) | ||
@@ -8,3 +8,3 @@ A small and pluggable lib to fetch a resource and cache the result. | ||
### Usage | ||
By default fetch will treat all response codes except 200 and 404 as errors. 404 will yield `null` and 200 the body. | ||
By default fetch will treat all response codes except 200, 301 and 404 as errors. 404 will yield `null` and 200 the body. | ||
@@ -62,3 +62,3 @@ #### Caching | ||
var body = { | ||
"query": "some string" | ||
"query": "some string" | ||
}; | ||
@@ -74,21 +74,23 @@ | ||
* `freeze`: (default:`true`). When this option is set to false it will not freeze the response so it can be modified. ("use strict" is needed) | ||
* `deepFreeze`: (default:`false`). When this option is set to true it will freeze the response _recursively_ so that it or any objects it contains can't be modified. ("use strict" is needed) | ||
* `agent`: (default: null), keepAlive Agent instance. | ||
* `cache`: (default: `an instance of AsyncCache`) (https://github.com/ExpressenAB/exp-asynccache). To disable caching set `{cache: null}` | ||
* `cacheKeyFn`: (default: caches on the url + sha1 of the body) An optional formatting function for finding the cache-key. One might, for example, want to cache on an url with the get params stripped. | ||
* `cacheNotFound`: (default: false). If set it will cache 404s, if given a number it will cache the 404 for that time. If the `maxAgeFn` is given, it will get this time as the first parameter. | ||
* `cacheValueFn`: (default: caches the response body) An optional function for change what will be returned and cached from fetch. | ||
* `clone`: (default: true), should fetch clone objects before handing them from the cache. | ||
* `contentType`: (default: `json`), expected content type. Fetch will try to parse the given content type. (supported: `xml`|`json`) | ||
* `deepFreeze`: (default:`false`). When this option is set to true it will freeze the response _recursively_ so that it or any objects it contains can't be modified. ("use strict" is needed) | ||
* `errorOnRemoteError`: (default: true). If set it will treat a remote > 200 statusCode as an error. | ||
* `followRedirect`: (default: true), should fetch follow redirects (and cache the redirect chain) | ||
* `freeze`: (default:`true`). When this option is set to false it will not freeze the response so it can be modified. ("use strict" is needed) | ||
* `httpMethod`: (default: `"GET"`), the HTTP-method that should be used to make requests. | ||
* `logger`: A logger object implementing `error`, `warning`, `info`, `debug` for example https://github.com/tj/log.js | ||
* `maxAgeFn`: (default: respects the `cache-control` header) | ||
* `onError`: If given a function, it will be called each time fetch encounters a non 200 nor 404 response | ||
* `onNotFound`: If given a function, it will be called each time fetch encounters a 404 | ||
* `onRequestInit`: If given a function, it will be called before the actual request is made, see [Hooks](#hooks) for signature | ||
* `onNotFound`: If given a function, it will be called each time fetch encounters a 404 | ||
* `onError`: If given a function, it will be called each time fetch encounters a non 200 nor 404 response | ||
* `onSuccess`: If given a function, it will be called each time fetch encounters a 200 | ||
* `requestTimeFn`: (default log with level `debug`) If given a function, it will be called when the request returned and processed from remote end. | ||
* `logger`: A logger object implementing `error`, `warning`, `info`, `debug` for example https://github.com/tj/log.js | ||
* `cacheNotFound`: (default: false). If set it will cache 404s, if given a number it will cache the 404 for that time. If the `maxAgeFn` is given, it will get this time as the first parameter. | ||
* `errorOnRemoteError`: (default: true). If set it will treat a remote > 200 statusCode as an error. | ||
* `contentType`: (default: `json`), expected content type. Fetch will try to parse the given content type. (supported: `xml`|`json`) | ||
* `agentOptions`: (default: `{}`), options passed to the keepAliveAgent. | ||
* `followRedirect`: (default: true), should fetch follow redirects (and cache the redirect chain) | ||
* `clone`: (default: true), should fetch clone objects before handing them from the cache. | ||
* `httpMethod`: (default: `"GET"`), the HTTP-method that should be used to make requests. | ||
* `requestTimeFn`: (default log with level `debug`) If given a function, it will be called when the request returned and processed from remote end. | ||
* `retry`: see [got](https://github.com/sindresorhus/got) for details, defaults to 0 | ||
* `timeout`: see [got](https://github.com/sindresorhus/got) for details, defaults to 20000ms | ||
@@ -117,3 +119,2 @@ The difference between `freeze` and `deepFreeze` is that `deepFreeze` walks the object graph and freezes any | ||
```javascript | ||
@@ -186,3 +187,3 @@ var fetchBuilder = require("exp-fetch"); | ||
function requestTimeFn(requestOptions, took) { | ||
console.log("REQUEST", requestOption.method, ":", requestOption.url, "took", took, "ms"); | ||
console.log("REQUEST", requestOption.method, ":", requestOption.url, "took", took, "ms"); | ||
} | ||
@@ -189,0 +190,0 @@ ``` |
Major refactor
Supply chain riskPackage has recently undergone a major refactor. It may be unstable or indicate significant internal changes. Use caution when updating to versions that include significant changes.
Found 1 instance in 1 package
New author
Supply chain riskA new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.
Found 1 instance in 1 package
Network access
Supply chain riskThis module accesses the network.
Found 1 instance in 1 package
New author
Supply chain riskA new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.
Found 1 instance in 1 package
Environment variable access
Supply chain riskPackage accesses environment variables, which may be a sign of credential stuffing or data theft.
Found 2 instances in 1 package
6
-14.29%213
0.47%1
-66.67%2
-97.4%23280
-60.65%14
-48.15%404
-71.96%+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
+ Added
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
- Removed
Updated
Updated
Updated