New Case Study:See how Anthropic automated 95% of dependency reviews with Socket.Learn More
Socket
Sign inDemoInstall
Socket

exp-fetch

Package Overview
Dependencies
Maintainers
14
Versions
44
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

exp-fetch - npm Package Compare versions

Comparing version

to
5.0.0

148

index.js
"use strict";
const request = require("request");
const got = require("got");
const VError = require("verror");

@@ -27,23 +27,24 @@ const AsyncCache = require("exp-asynccache");

// Options
var freeze = true;
var deepFreeze = false;
var cache = new AsyncCache(initCache({age: 60}));
var cacheKeyFn = behavior.cacheKeyFn || calculateCacheKey;
var cacheValueFn = behavior.cacheValueFn || passThrough;
var maxAgeFn = behavior.maxAgeFn || passThrough;
var onNotFound = behavior.onNotFound;
var onError = behavior.onError;
var onSuccess = behavior.onSuccess;
var cacheNotFound = false;
var logger = behavior.logger || dummyLogger();
var errorOnRemoteError = true;
var contentType = (behavior.contentType || "json").toLowerCase();
var keepAliveAgent = behavior.agent;
var followRedirect = true;
var performClone = true;
var maximumNumberOfRedirects = 10;
var httpMethod = (behavior.httpMethod || "GET").toUpperCase();
var timeout = behavior.timeout || 20000;
var stats = {calls: 0, misses: 0};
var globalHeaders = behavior.headers || {};
let freeze = true;
let deepFreeze = false;
let cache = new AsyncCache(initCache({age: 60}));
const cacheKeyFn = behavior.cacheKeyFn || calculateCacheKey;
const cacheValueFn = behavior.cacheValueFn || passThrough;
const maxAgeFn = behavior.maxAgeFn || passThrough;
const onNotFound = behavior.onNotFound;
const onError = behavior.onError;
const onSuccess = behavior.onSuccess;
let cacheNotFound = false;
const logger = behavior.logger || dummyLogger();
let errorOnRemoteError = true;
const contentType = (behavior.contentType || "json").toLowerCase();
const keepAliveAgent = behavior.agent;
let followRedirect = true;
let performClone = true;
const maximumNumberOfRedirects = 10;
const httpMethod = (behavior.httpMethod || "GET").toUpperCase();
const timeout = behavior.timeout || 20000;
const stats = {calls: 0, misses: 0};
const globalHeaders = behavior.headers || {};
const retry = "retry" in behavior ? behavior.retry : 0;

@@ -53,3 +54,3 @@ function defaultRequestTimeFn(requestOptions, took) {

}
var requestTimeFn = behavior.requestTimeFn || defaultRequestTimeFn;
const requestTimeFn = behavior.requestTimeFn || defaultRequestTimeFn;

@@ -86,3 +87,3 @@ if (behavior.hasOwnProperty("clone")) {

logger.info("404 Not Found for: %j", url);
var notFoundAge = -1;
let notFoundAge = -1;

@@ -101,3 +102,3 @@ if (onNotFound) {

logger.warning("HTTP Fetching error %d for: %j", res.statusCode, url);
var errorAge = -1;
let errorAge = -1;

@@ -111,3 +112,3 @@ if (onError) {

if (errorOnRemoteError) {
var error = new VError("%s yielded %s (%s)", url, res.statusCode, util.inspect(content));
const error = new VError("%s yielded %s (%s)", url, res.statusCode, util.inspect(content));
error.statusCode = res.statusCode;

@@ -122,6 +123,6 @@

function deepFreezeObj(obj) {
var propNames = Object.getOwnPropertyNames(obj);
const propNames = Object.getOwnPropertyNames(obj);
propNames.forEach(function (name) {
var prop = obj[name];
propNames.forEach((name) => {
const prop = obj[name];

@@ -137,5 +138,5 @@ if (typeof prop === "object" && prop !== null) {

function handleSuccess(url, cacheKey, res, content, resolvedCallback) {
var maxAge = maxAgeFn(getMaxAge(res.headers["cache-control"]), cacheKey, res, content);
const maxAge = maxAgeFn(getMaxAge(res.headers["cache-control"]), cacheKey, res, content);
var typeOfContent = typeof content;
const typeOfContent = typeof content;

@@ -156,5 +157,5 @@ if (deepFreeze && typeOfContent === "object") {

function handleRedirect(url, cacheKey, res, body, resolvedCallback) {
var maxAge = maxAgeFn(getMaxAge(res.headers["cache-control"]), cacheKey, res, body);
const maxAge = maxAgeFn(getMaxAge(res.headers["cache-control"]), cacheKey, res, body);
var content = {
const content = {
statusCode: res.statusCode,

@@ -167,32 +168,36 @@ headers: res.headers

function performRequest(url, headers, explicitTimeout, body, redirectCount, callback, onRequestInit) {
var cacheKey = cacheKeyFn(url, body);
var startTime = new Date().getTime();
const cacheKey = cacheKeyFn(url, body);
const startTime = new Date().getTime();
stats.calls++;
cache.lookup(cacheKey, function (resolveFunction) {
cache.lookup(cacheKey, (resolveFunction) => {
stats.misses++;
logger.debug("fetching %s cacheKey '%s'", url, cacheKey);
var options = {
const options = {
url: url,
json: contentType === "json",
responseType: contentType === "json" ? contentType : undefined,
agent: keepAliveAgent,
followRedirect: false,
retry,
method: httpMethod,
timeout: explicitTimeout || timeout,
headers: headers
headers: headers,
cache: false
};
if (body) {
if (body && typeof body === "object") {
options.json = body;
} else if (body) {
options.body = body;
}
var passOptions = {
const passOptions = {
url: options.url,
json: options.json,
method: options.method,
responseType: contentType === "json" ? contentType : undefined,
followRedirect: followRedirect,
headers: options.headers
};
if (onRequestInit && !onRequestInit.called) {
onRequestInit(passOptions, cacheKey);

@@ -206,20 +211,24 @@ }

request(options, function (err, res, content) {
if (err) return resolvedCallback(new VError(err, "Fetching error for: %j", url));
if (isRedirect(res)) return handleRedirect(url, cacheKey, res, content, resolvedCallback);
if (res.statusCode === 404) {
return handleNotFound(url, cacheKey, res, content, resolvedCallback);
} else if (res.statusCode > 299) {
return handleError(url, cacheKey, res, content, resolvedCallback);
return request(options).then((res) => {
if (isRedirect(res)) return handleRedirect(url, cacheKey, res, res.body, resolvedCallback);
return parseResponse(res.body, contentType, (_, transformed) => {
return handleSuccess(url, cacheKey, res, transformed, resolvedCallback);
});
}).catch((err) => {
if (err instanceof got.HTTPError) {
if (err.response.statusCode === 404) {
return handleNotFound(url, cacheKey, err.response, err.response.body, resolvedCallback);
} else if (err.response.statusCode > 299) {
return handleError(url, cacheKey, err.response, err.response.body, resolvedCallback);
}
} else if (err instanceof got.TimeoutError) {
return resolvedCallback(new VError("ESOCKETTIMEDOUT"));
}
return parseResponse(content, contentType, function (_, transformed) {
return handleSuccess(url, cacheKey, res, transformed, resolvedCallback);
});
return resolvedCallback(err);
});
}, function (err, response) {
}, (err, response) => {
if (followRedirect && isRedirect(response)) {
if (redirectCount++ < maximumNumberOfRedirects) {
var location = ensureAbsoluteUrl(response.headers, url);
const location = ensureAbsoluteUrl(response.headers, url);
return performRequest(location, headers, explicitTimeout, body, redirectCount, callback);

@@ -236,5 +245,5 @@ } else {

fetch: function (options, optionalBody, resultCallback) {
var url = options;
var headers = Object.assign({}, globalHeaders, options.headers);
var explicitTimeout = null;
let url = options;
const headers = Object.assign({}, globalHeaders, options.headers);
let explicitTimeout = null;
if (typeof options === "object") {

@@ -257,8 +266,2 @@ if (options.url) {

}
var onRequestInit = function () {
if (behavior.onRequestInit) {
behavior.onRequestInit.apply(null, arguments);
}
onRequestInit.called = true;
};

@@ -268,4 +271,4 @@ if (resultCallback) {

} else {
return new Promise(function (resolve, reject) {
performRequest(url, headers, explicitTimeout, optionalBody, 0, function (err, content) {
return new Promise((resolve, reject) => {
performRequest(url, headers, explicitTimeout, optionalBody, 0, (err, content) => {
if (err) return reject(err);

@@ -276,2 +279,9 @@ return resolve(content);

}
function onRequestInit() {
if (behavior.onRequestInit) {
behavior.onRequestInit.apply(null, arguments);
}
onRequestInit.called = true;
}
},

@@ -292,1 +302,5 @@

}
function request({url, ...options}) {
return got(url, {cache: false, ...options});
}
"use strict";
var crypto = require("crypto");
const crypto = require("crypto");

@@ -4,0 +4,0 @@ function calculateCacheKey(url, body) {

"use strict";
var path = require("path");
var currentPackage = {};
const path = require("path");
let currentPackage = {};

@@ -6,0 +6,0 @@ try {

"use strict";
var url = require("url");
var util = require("util");
const url = require("url");
const util = require("util");
function ensureAbsoluteUrl(headers, uri) {
var newLocation = url.parse(headers.location);
var oldLocation = url.parse(uri);
var protocol = newLocation.protocol || oldLocation.protocol;
var host = newLocation.host || oldLocation.host;
const newLocation = url.parse(headers.location);
const oldLocation = url.parse(uri);
const protocol = newLocation.protocol || oldLocation.protocol;
const host = newLocation.host || oldLocation.host;

@@ -11,0 +11,0 @@ return util.format("%s//%s%s", protocol, host, newLocation.path);

@@ -53,4 +53,4 @@ "use strict";

} else {
var maxAge = 1000 * Number(cacheConfig.age || cacheConfig.maxAge || 60);
var lruCache = new LRU({
const maxAge = 1000 * Number(cacheConfig.age || cacheConfig.maxAge || 60);
const lruCache = new LRU({
maxAge: maxAge,

@@ -57,0 +57,0 @@ length: cacheConfig.length || defaultLengthFn,

"use strict";
var maxAgeRexExp = /max-age=\s*\d+/;
var maxAgeValueRexExp = /\d+/;
const maxAgeRexExp = /max-age=\s*\d+/;
const maxAgeValueRexExp = /\d+/;
var noCacheKeywords = ["private", "max-age=0", "no-cache", "must-revalidate"];
const noCacheKeywords = ["private", "max-age=0", "no-cache", "must-revalidate"];
function containsAny(haystack, listOfHits) {
for (var i = 0; i < listOfHits.length; i++) {
for (let i = 0; i < listOfHits.length; i++) {
if (haystack.indexOf(listOfHits[i]) > -1) {

@@ -26,4 +26,4 @@ return true;

var maxAge = null;
var maxAgeStringMatch = maxAgeRexExp.exec(cacheHeader);
let maxAge = null;
let maxAgeStringMatch = maxAgeRexExp.exec(cacheHeader);
if (maxAgeStringMatch) {

@@ -30,0 +30,0 @@ maxAgeStringMatch = maxAgeValueRexExp.exec(maxAgeStringMatch[0]);

"use strict";
var xml2js = require("xml2js");
const xml2js = require("xml2js");

@@ -4,0 +4,0 @@ function parseResponse(content, contentType, callback) {

{
"name": "exp-fetch",
"version": "4.2.0",
"version": "5.0.0",
"description": "A small pluggable fetch lib",
"main": "index.js",
"engines": {
"node": ">=10"
},
"scripts": {
"test": "mocha -R dot && eslint . --cache"
"test": "mocha -R dot",
"posttest": "eslint . --cache"
},

@@ -27,16 +25,19 @@ "repository": {

"devDependencies": {
"chai": "^4.1.2",
"eslint": "^5.3.0",
"mocha": "^9.1.0",
"nock": "^11.9.0"
"chai": "^4.2.0",
"eslint": "^7.32.0",
"mocha": "^9.1.1",
"nock": "^13.1.3"
},
"dependencies": {
"clone": "^2.1.1",
"exp-asynccache": "^1.2.1",
"log": "^1.4.0",
"lru-cache": "^4.1.1",
"request": "^2.88.0",
"exp-asynccache": "^2.0.0",
"got": "^11.8.2",
"lru-cache": "^6.0.0",
"verror": "^1.10.0",
"xml2js": "^0.4.19"
}
"xml2js": "^0.4.22"
},
"files": [
"lib/",
"index.js"
]
}
fetch
=====
[![Build Status](https://travis-ci.org/BonnierNews/exp-fetch.svg?branch=master)](https://travis-ci.org/BonnierNews/exp-fetch)
[![Built latest](https://github.com/BonnierNews/exp-fetch/actions/workflows/build-latest.yaml/badge.svg)](https://github.com/BonnierNews/exp-fetch/actions/workflows/build-latest.yaml)

@@ -8,3 +8,3 @@ A small and pluggable lib to fetch a resource and cache the result.

### Usage
By default fetch will treat all response codes except 200 and 404 as errors. 404 will yield `null` and 200 the body.
By default fetch will treat all response codes except 200, 301 and 404 as errors. 404 will yield `null` and 200 the body.

@@ -62,3 +62,3 @@ #### Caching

var body = {
"query": "some string"
"query": "some string"
};

@@ -74,21 +74,23 @@

* `freeze`: (default:`true`). When this option is set to false it will not freeze the response so it can be modified. ("use strict" is needed)
* `deepFreeze`: (default:`false`). When this option is set to true it will freeze the response _recursively_ so that it or any objects it contains can't be modified. ("use strict" is needed)
* `agent`: (default: null), keepAlive Agent instance.
* `cache`: (default: `an instance of AsyncCache`) (https://github.com/ExpressenAB/exp-asynccache). To disable caching set `{cache: null}`
* `cacheKeyFn`: (default: caches on the url + sha1 of the body) An optional formatting function for finding the cache-key. One might, for example, want to cache on an url with the get params stripped.
* `cacheNotFound`: (default: false). If set it will cache 404s, if given a number it will cache the 404 for that time. If the `maxAgeFn` is given, it will get this time as the first parameter.
* `cacheValueFn`: (default: caches the response body) An optional function for change what will be returned and cached from fetch.
* `clone`: (default: true), should fetch clone objects before handing them from the cache.
* `contentType`: (default: `json`), expected content type. Fetch will try to parse the given content type. (supported: `xml`|`json`)
* `deepFreeze`: (default:`false`). When this option is set to true it will freeze the response _recursively_ so that it or any objects it contains can't be modified. ("use strict" is needed)
* `errorOnRemoteError`: (default: true). If set it will treat a remote > 200 statusCode as an error.
* `followRedirect`: (default: true), should fetch follow redirects (and cache the redirect chain)
* `freeze`: (default:`true`). When this option is set to false it will not freeze the response so it can be modified. ("use strict" is needed)
* `httpMethod`: (default: `"GET"`), the HTTP-method that should be used to make requests.
* `logger`: A logger object implementing `error`, `warning`, `info`, `debug` for example https://github.com/tj/log.js
* `maxAgeFn`: (default: respects the `cache-control` header)
* `onError`: If given a function, it will be called each time fetch encounters a non 200 nor 404 response
* `onNotFound`: If given a function, it will be called each time fetch encounters a 404
* `onRequestInit`: If given a function, it will be called before the actual request is made, see [Hooks](#hooks) for signature
* `onNotFound`: If given a function, it will be called each time fetch encounters a 404
* `onError`: If given a function, it will be called each time fetch encounters a non 200 nor 404 response
* `onSuccess`: If given a function, it will be called each time fetch encounters a 200
* `requestTimeFn`: (default log with level `debug`) If given a function, it will be called when the request returned and processed from remote end.
* `logger`: A logger object implementing `error`, `warning`, `info`, `debug` for example https://github.com/tj/log.js
* `cacheNotFound`: (default: false). If set it will cache 404s, if given a number it will cache the 404 for that time. If the `maxAgeFn` is given, it will get this time as the first parameter.
* `errorOnRemoteError`: (default: true). If set it will treat a remote > 200 statusCode as an error.
* `contentType`: (default: `json`), expected content type. Fetch will try to parse the given content type. (supported: `xml`|`json`)
* `agentOptions`: (default: `{}`), options passed to the keepAliveAgent.
* `followRedirect`: (default: true), should fetch follow redirects (and cache the redirect chain)
* `clone`: (default: true), should fetch clone objects before handing them from the cache.
* `httpMethod`: (default: `"GET"`), the HTTP-method that should be used to make requests.
* `requestTimeFn`: (default log with level `debug`) If given a function, it will be called when the request returned and processed from remote end.
* `retry`: see [got](https://github.com/sindresorhus/got) for details, defaults to 0
* `timeout`: see [got](https://github.com/sindresorhus/got) for details, defaults to 20000ms

@@ -117,3 +119,2 @@ The difference between `freeze` and `deepFreeze` is that `deepFreeze` walks the object graph and freezes any

```javascript

@@ -186,3 +187,3 @@ var fetchBuilder = require("exp-fetch");

function requestTimeFn(requestOptions, took) {
console.log("REQUEST", requestOption.method, ":", requestOption.url, "took", took, "ms");
console.log("REQUEST", requestOption.method, ":", requestOption.url, "took", took, "ms");
}

@@ -189,0 +190,0 @@ ```