Launch Week Day 5: Introducing Reachability for PHP.Learn More →
Socket
Book a DemoSign in
Socket
Book a DemoSign in

got

Package Overview
Dependencies
Maintainers
4
Versions
195
Alerts
File Explorer

Advanced tools

License
Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

got - npm Package Compare versions

Comparing version
7.1.0
 to
8.0.0
+244
-41
index.js

@@ -6,5 +6,9 @@ 'use strict';

const PassThrough = require('stream').PassThrough;
const Transform = require('stream').Transform;
const urlLib = require('url');
const fs = require('fs');
const querystring = require('querystring');
const CacheableRequest = require('cacheable-request');
const duplexer3 = require('duplexer3');
const intoStream = require('into-stream');
const isStream = require('is-stream');

@@ -17,4 +21,4 @@ const getStream = require('get-stream');

const decompressResponse = require('decompress-response');
const mimicResponse = require('mimic-response');
const isRetryAllowed = require('is-retry-allowed');
const Buffer = require('safe-buffer').Buffer;
const isURL = require('isurl');

@@ -24,3 +28,5 @@ const isPlainObj = require('is-plain-obj');

const pTimeout = require('p-timeout');
const pkg = require('./package');
const pify = require('pify');
const Buffer = require('safe-buffer').Buffer;
const pkg = require('./package.json');

@@ -30,2 +36,34 @@ const getMethodRedirectCodes = new Set([300, 301, 302, 303, 304, 305, 307, 308]);

const isFormData = body => isStream(body) && typeof body.getBoundary === 'function';
const getBodySize = opts => {
const body = opts.body;
if (opts.headers['content-length']) {
return Number(opts.headers['content-length']);
}
if (!body && !opts.stream) {
return 0;
}
if (typeof body === 'string') {
return Buffer.byteLength(body);
}
if (isFormData(body)) {
return pify(body.getLength.bind(body))();
}
if (body instanceof fs.ReadStream) {
return pify(fs.stat)(body.path).then(stat => stat.size);
}
if (isStream(body) && Buffer.isBuffer(body._buffer)) {
return body._buffer.length;
}
return null;
};
function requestAsEventEmitter(opts) {

@@ -37,4 +75,7 @@ opts = opts || {};

const redirects = [];
const agents = typeof opts.agent === 'object' ? opts.agent : null;
let retryCount = 0;
let redirectUrl;
let uploadBodySize;
let uploaded = 0;

@@ -49,2 +90,7 @@ const get = opts => {

if (agents) {
const protocolName = opts.protocol === 'https:' ? 'https' : 'http';
opts.agent = agents[protocolName] || opts.agent;
}
if (opts.useElectronNet && process.versions.electron) {

@@ -55,3 +101,14 @@ const electron = require('electron');

const req = fn.request(opts, res => {
let progressInterval;
const cacheableRequest = new CacheableRequest(fn.request, opts.cache);
const cacheReq = cacheableRequest(opts, res => {
clearInterval(progressInterval);
ee.emit('uploadProgress', {
percent: 1,
transferred: uploaded,
total: uploadBodySize
});
const statusCode = res.statusCode;

@@ -95,6 +152,41 @@

const downloadBodySize = Number(res.headers['content-length']) || null;
let downloaded = 0;
setImmediate(() => {
const progressStream = new Transform({
transform(chunk, encoding, callback) {
downloaded += chunk.length;
const percent = downloadBodySize ? downloaded / downloadBodySize : 0;
// Let flush() be responsible for emitting the last event
if (percent < 1) {
ee.emit('downloadProgress', {
percent,
transferred: downloaded,
total: downloadBodySize
});
}
callback(null, chunk);
},
flush(callback) {
ee.emit('downloadProgress', {
percent: 1,
transferred: downloaded,
total: downloadBodySize
});
callback();
}
});
mimicResponse(res, progressStream);
progressStream.redirectUrls = redirects;
const response = opts.decompress === true &&
typeof decompressResponse === 'function' &&
req.method !== 'HEAD' ? decompressResponse(res) : res;
opts.method !== 'HEAD' ? decompressResponse(progressStream) : progressStream;

@@ -105,25 +197,80 @@ if (!opts.decompress && ['gzip', 'deflate'].indexOf(res.headers['content-encoding']) !== -1) {

response.redirectUrls = redirects;
ee.emit('response', response);
ee.emit('response', response);
ee.emit('downloadProgress', {
percent: 0,
transferred: 0,
total: downloadBodySize
});
res.pipe(progressStream);
});
});
req.once('error', err => {
const backoff = opts.retries(++retryCount, err);
if (backoff) {
setTimeout(get, backoff, opts);
return;
cacheReq.on('error', err => {
if (err instanceof CacheableRequest.RequestError) {
ee.emit('error', new got.RequestError(err, opts));
} else {
ee.emit('error', new got.CacheError(err, opts));
}
ee.emit('error', new got.RequestError(err, opts));
});
if (opts.gotTimeout) {
timedOut(req, opts.gotTimeout);
}
cacheReq.once('request', req => {
req.once('error', err => {
clearInterval(progressInterval);
setImmediate(() => {
ee.emit('request', req);
const backoff = opts.retries(++retryCount, err);
if (backoff) {
setTimeout(get, backoff, opts);
return;
}
ee.emit('error', new got.RequestError(err, opts));
});
ee.once('request', req => {
ee.emit('uploadProgress', {
percent: 0,
transferred: 0,
total: uploadBodySize
});
req.connection.once('connect', () => {
const uploadEventFrequency = 150;
progressInterval = setInterval(() => {
const lastUploaded = uploaded;
const headersSize = Buffer.byteLength(req._header);
uploaded = req.connection.bytesWritten - headersSize;
// Prevent the known issue of `bytesWritten` being larger than body size
if (uploadBodySize && uploaded > uploadBodySize) {
uploaded = uploadBodySize;
}
// Don't emit events with unchanged progress and
// prevent last event from being emitted, because
// it's emitted when `response` is emitted
if (uploaded === lastUploaded || uploaded === uploadBodySize) {
return;
}
ee.emit('uploadProgress', {
percent: uploadBodySize ? uploaded / uploadBodySize : 0,
transferred: uploaded,
total: uploadBodySize
});
}, uploadEventFrequency);
});
});
if (opts.gotTimeout) {
clearInterval(progressInterval);
timedOut(req, opts.gotTimeout);
}
setImmediate(() => {
ee.emit('request', req);
});
});

@@ -133,4 +280,12 @@ };

setImmediate(() => {
get(opts);
Promise.resolve(getBodySize(opts))
.then(size => {
uploadBodySize = size;
get(opts);
})
.catch(err => {
ee.emit('error', err);
});
});
return ee;

@@ -144,3 +299,5 @@ }

return timeoutFn(new PCancelable((onCancel, resolve, reject) => {
const proxy = new EventEmitter();
const cancelable = new PCancelable((onCancel, resolve, reject) => {
const ee = requestAsEventEmitter(opts);

@@ -185,5 +342,5 @@ let cancelOnRequest = false;

res.body = JSON.parse(res.body);
} catch (e) {
} catch (err) {
if (statusCode >= 200 && statusCode < 300) {
throw new got.ParseError(e, statusCode, opts, data);
throw new got.ParseError(err, statusCode, opts, data);
}

@@ -194,3 +351,3 @@ }

if (statusCode !== 304 && (statusCode < 200 || statusCode > limitStatusCode)) {
throw new got.HTTPError(statusCode, res.headers, opts);
throw new got.HTTPError(statusCode, res.statusMessage, res.headers, opts);
}

@@ -206,7 +363,23 @@

ee.on('error', reject);
}));
ee.once('error', reject);
ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));
});
const promise = timeoutFn(cancelable);
promise.cancel = cancelable.cancel.bind(cancelable);
promise.on = (name, fn) => {
proxy.on(name, fn);
return promise;
};
return promise;
}
function asStream(opts) {
opts.stream = true;
const input = new PassThrough();

@@ -224,3 +397,3 @@ const output = new PassThrough();

if (opts.json) {
throw new Error('got can not be used as stream when options.json is used');
throw new Error('Got can not be used as a stream when the `json` option is used');
}

@@ -230,3 +403,3 @@

proxy.write = () => {
throw new Error('got\'s stream is not writable when options.body is used');
throw new Error('Got\'s stream is not writable when the `body` option is used');
};

@@ -266,3 +439,3 @@ }

if (statusCode !== 304 && (statusCode < 200 || statusCode > 299)) {
proxy.emit('error', new got.HTTPError(statusCode, res.headers, opts), null, res);
proxy.emit('error', new got.HTTPError(statusCode, res.statusMessage, res.headers, opts), null, res);
return;

@@ -274,4 +447,6 @@ }

ee.on('error', proxy.emit.bind(proxy, 'error'));
ee.on('redirect', proxy.emit.bind(proxy, 'redirect'));
ee.on('error', proxy.emit.bind(proxy, 'error'));
ee.on('uploadProgress', proxy.emit.bind(proxy, 'uploadProgress'));
ee.on('downloadProgress', proxy.emit.bind(proxy, 'downloadProgress'));

@@ -287,2 +462,5 @@ return proxy;

url = urlParseLax(url);
if (url.auth) {
throw new Error('Basic authentication must be done with the `auth` option');
}
} else if (isURL.lenient(url)) {

@@ -292,6 +470,2 @@ url = urlToOptions(url);

if (url.auth) {
throw new Error('Basic authentication must be done with auth option');
}
opts = Object.assign(

@@ -301,4 +475,5 @@ {

retries: 2,
cache: false,
decompress: true,
useElectronNet: true
useElectronNet: false
},

@@ -312,6 +487,13 @@ url,

const headers = lowercaseKeys(opts.headers);
for (const key of Object.keys(headers)) {
if (headers[key] === null || headers[key] === undefined) {
delete headers[key];
}
}
opts.headers = Object.assign({
'user-agent': `${pkg.name}/${pkg.version} (https://github.com/sindresorhus/got)`,
'accept-encoding': 'gzip,deflate'
}, lowercaseKeys(opts.headers));
}, headers);

@@ -337,3 +519,3 @@ const query = opts.query;

if (!isStream(body) && typeof body !== 'string' && !Buffer.isBuffer(body) && !(opts.form || opts.json)) {
throw new TypeError('options.body must be a ReadableStream, string, Buffer or plain Object');
throw new TypeError('The `body` option must be a stream.Readable, string, Buffer or plain Object');
}

@@ -343,6 +525,6 @@

if ((opts.form || opts.json) && !canBodyBeStringified) {
throw new TypeError('options.body must be a plain Object or Array when options.form or options.json is used');
throw new TypeError('The `body` option must be a plain Object or Array when the `form` or `json` option is used');
}
if (isStream(body) && typeof body.getBoundary === 'function') {
if (isFormData(body)) {
// Special case for https://github.com/form-data/form-data

@@ -363,2 +545,9 @@ headers['content-type'] = headers['content-type'] || `multipart/form-data; boundary=${body.getBoundary()}`;

// Convert buffer to stream to receive upload progress events
// see https://github.com/sindresorhus/got/pull/322
if (Buffer.isBuffer(body)) {
opts.body = intoStream(body);
opts.body._buffer = body;
}
opts.method = (opts.method || 'POST').toUpperCase();

@@ -436,2 +625,3 @@ } else {

super(message);
Error.captureStackTrace(this, this.constructor);
this.name = 'StdError';

@@ -454,2 +644,9 @@

got.CacheError = class extends StdError {
constructor(error, opts) {
super(error.message, error, opts);
this.name = 'CacheError';
}
};
got.RequestError = class extends StdError {

@@ -479,4 +676,8 @@ constructor(error, opts) {

got.HTTPError = class extends StdError {
constructor(statusCode, headers, opts) {
const statusMessage = http.STATUS_CODES[statusCode];
constructor(statusCode, statusMessage, headers, opts) {
if (statusMessage) {
statusMessage = statusMessage.replace(/\r?\n/g, ' ').trim();
} else {
statusMessage = http.STATUS_CODES[statusCode];
}
super(`Response code ${statusCode} (${statusMessage})`, {}, opts);

@@ -507,2 +708,4 @@ this.name = 'HTTPError';

got.CancelError = PCancelable.CancelError;
module.exports = got;
+92
-86
package.json
{
"name": "got",
"version": "7.1.0",
"description": "Simplified HTTP requests",
"license": "MIT",
"repository": "sindresorhus/got",
"maintainers": [
{
"name": "Sindre Sorhus",
"email": "sindresorhus@gmail.com",
"url": "sindresorhus.com"
},
{
"name": "Vsevolod Strukchinsky",
"email": "floatdrop@gmail.com",
"url": "github.com/floatdrop"
},
{
"name": "Alexander Tesfamichael",
"email": "alex.tesfamichael@gmail.com",
"url": "alextes.me"
}
],
"engines": {
"node": ">=4"
},
"scripts": {
"test": "xo && nyc ava",
"coveralls": "nyc report --reporter=text-lcov | coveralls"
},
"files": [
"index.js"
],
"keywords": [
"http",
"https",
"get",
"got",
"url",
"uri",
"request",
"util",
"utility",
"simple",
"curl",
"wget",
"fetch",
"net",
"network",
"electron"
],
"dependencies": {
"decompress-response": "^3.2.0",
"duplexer3": "^0.1.4",
"get-stream": "^3.0.0",
"is-plain-obj": "^1.1.0",
"is-retry-allowed": "^1.0.0",
"is-stream": "^1.0.0",
"isurl": "^1.0.0-alpha5",
"lowercase-keys": "^1.0.0",
"p-cancelable": "^0.3.0",
"p-timeout": "^1.1.1",
"safe-buffer": "^5.0.1",
"timed-out": "^4.0.0",
"url-parse-lax": "^1.0.0",
"url-to-options": "^1.0.1"
},
"devDependencies": {
"ava": "^0.20.0",
"coveralls": "^2.11.4",
"form-data": "^2.1.1",
"get-port": "^3.0.0",
"into-stream": "^3.0.0",
"nyc": "^11.0.2",
"pem": "^1.4.4",
"pify": "^3.0.0",
"tempfile": "^2.0.0",
"tempy": "^0.1.0",
"universal-url": "^1.0.0-alpha",
"xo": "^0.18.0"
},
"ava": {
"concurrency": 4
},
"browser": {
"decompress-response": false
}
"name": "got",
"version": "8.0.0",
"description": "Simplified HTTP requests",
"license": "MIT",
"repository": "sindresorhus/got",
"maintainers": [
{
"name": "Sindre Sorhus",
"email": "sindresorhus@gmail.com",
"url": "sindresorhus.com"
},
{
"name": "Vsevolod Strukchinsky",
"email": "floatdrop@gmail.com",
"url": "github.com/floatdrop"
},
{
"name": "Alexander Tesfamichael",
"email": "alex.tesfamichael@gmail.com",
"url": "alextes.me"
}
],
"engines": {
"node": ">=4"
},
"scripts": {
"test": "xo && nyc ava",
"coveralls": "nyc report --reporter=text-lcov | coveralls"
},
"files": [
"index.js"
],
"keywords": [
"http",
"https",
"get",
"got",
"url",
"uri",
"request",
"util",
"utility",
"simple",
"curl",
"wget",
"fetch",
"net",
"network",
"electron"
],
"dependencies": {
"cacheable-request": "^2.1.1",
"decompress-response": "^3.3.0",
"duplexer3": "^0.1.4",
"get-stream": "^3.0.0",
"into-stream": "^3.1.0",
"is-plain-obj": "^1.1.0",
"is-retry-allowed": "^1.1.0",
"is-stream": "^1.1.0",
"isurl": "^1.0.0-alpha5",
"lowercase-keys": "^1.0.0",
"mimic-response": "^1.0.0",
"p-cancelable": "^0.3.0",
"p-timeout": "^1.2.0",
"pify": "^3.0.0",
"safe-buffer": "^5.1.1",
"timed-out": "^4.0.1",
"url-parse-lax": "^3.0.0",
"url-to-options": "^1.0.1"
},
"devDependencies": {
"ava": "^0.23.0",
"coveralls": "^3.0.0",
"form-data": "^2.1.1",
"get-port": "^3.0.0",
"nyc": "^11.0.2",
"p-event": "^1.3.0",
"pem": "^1.4.4",
"sinon": "^4.0.0",
"slow-stream": "0.0.4",
"tempfile": "^2.0.0",
"tempy": "^0.2.1",
"universal-url": "1.0.0-alpha",
"xo": "^0.18.0"
},
"ava": {
"concurrency": 4
},
"browser": {
"decompress-response": false,
"electron": false
}
}
+173
-20
readme.md

@@ -22,4 +22,6 @@ <h1 align="center">

- [Request cancelation](#aborting-the-request)
- [RFC compliant caching](#cache-adapters)
- [Follows redirects](#followredirect)
- [Retries on network failure](#retries)
- [Progress events](#onuploadprogress-progress)
- [Handles gzip/deflate](#decompress)

@@ -36,3 +38,3 @@ - [Timeout handling](#timeout)

```
$ npm install --save got
$ npm install got
```

@@ -44,20 +46,26 @@

```js
const fs = require('fs');
const got = require('got');
got('todomvc.com')
.then(response => {
(async () => {
try {
const response = await got('sindresorhus.com');
console.log(response.body);
//=> '<!doctype html> ...'
})
.catch(error => {
} catch (error) {
console.log(error.response.body);
//=> 'Internal server error ...'
});
}
})();
```
// Streams
got.stream('todomvc.com').pipe(fs.createWriteStream('index.html'));
###### Streams
// For POST, PUT and PATCH methods got.stream returns a WritableStream
fs.createReadStream('index.html').pipe(got.stream.post('todomvc.com'));
```js
const fs = require('fs');
const got = require('got');
got.stream('sindresorhus.com').pipe(fs.createWriteStream('index.html'));
// For POST, PUT, and PATCH methods `got.stream` returns a `stream.Writable`
fs.createReadStream('index.html').pipe(got.stream.post('sindresorhus.com'));
```

@@ -68,3 +76,3 @@

It's a `GET` request by default, but can be changed in `options`.
It's a `GET` request by default, but can be changed by using different methods or in the `options`.

@@ -75,2 +83,6 @@ #### got(url, [options])

The response object will normally be a [Node.js HTTP response stream](https://nodejs.org/api/http.html#http_class_http_incomingmessage), however if returned from the cache it will be a [responselike object](https://github.com/lukechilds/responselike) which behaves in the same way.
The response will also have a `fromCache` property set with a boolean value.
##### url

@@ -84,2 +96,4 @@

If no protocol is specified, it will default to `https`.
##### options

@@ -178,8 +192,15 @@

###### cache
Type: `Object`<br>
Default: `false`
[Cache adapter instance](#cache-adapters) for storing cached data.
###### useElectronNet
Type: `boolean`<br>
Default: `true`
Default: `false`
When used in Electron, Got will automatically use [`electron.net`](https://electron.atom.io/docs/api/net/) instead of the Node.js `http` module. It should be fully compatible, but you can turn it off here if you encounter a problem. Please open an issue if you do!
When used in Electron, Got will use [`electron.net`](https://electron.atom.io/docs/api/net/) instead of the Node.js `http` module. According to the Electron docs, it should be fully compatible, but it's not entirely. See [#315](https://github.com/sindresorhus/got/issues/315).

@@ -212,2 +233,33 @@

##### .on('uploadProgress', progress)
##### .on('downloadProgress', progress)
Progress events for uploading (sending request) and downloading (receiving response). The `progress` argument is an object like:
```js
{
percent: 0.1,
transferred: 1024,
total: 10240
}
```
If it's not possible to retrieve the body size (can happen when streaming), `total` will be `null`.
**Note**: Progress events can also be used with promises.
```js
(async () => {
const response = await got('sindresorhus.com')
.on('downloadProgress', progress => {
// Report download progress
})
.on('uploadProgress', progress => {
// Report upload progress
});
console.log(response);
})();
```
##### .on('error', error, body, response)

@@ -233,2 +285,6 @@

#### got.CacheError
When a cache method fails, for example if the database goes down, or there's a filesystem error.
#### got.RequestError

@@ -258,8 +314,89 @@

#### got.CancelError
When the request is aborted with `.cancel()`.
## Aborting the request
The promise returned by Got has a `.cancel()` function which, when called, aborts the request.
The promise returned by Got has a [`.cancel()`](https://github.com/sindresorhus/p-cancelable) method which, when called, aborts the request.
```js
(async () => {
const request = got(url, options);
…
// In another part of the code
if (something) {
request.cancel();
}
…
try {
await request;
} catch (error) {
if (request.canceled) { // Or `error instanceof got.CancelError`
// Handle cancelation
}
// Handle other errors
}
})();
```
<a name="cache-adapters"></a>
## Cache
You can use the JavaScript `Map` type as an in memory cache:
```js
const got = require('got');
const map = new Map();
(async () => {
let response = await got('sindresorhus.com', {cache: map});
console.log(response.fromCache);
//=> false
response = await got('sindresorhus.com', {cache: map});
console.log(response.fromCache);
//=> true
})();
```
Got uses [Keyv](https://github.com/lukechilds/keyv) internally to support a wide range of storage adapters. For something more scalable you could use an [official Keyv storage adapter](https://github.com/lukechilds/keyv#official-storage-adapters):
```
$ npm install @keyv/redis
```
```js
const got = require('got');
const KeyvRedis = require('@keyv/redis');
const redis = new KeyvRedis('redis://user:pass@localhost:6379');
got('sindresorhus.com', {cache: redis});
```
Got supports anything that follows the Map API, so it's easy to write your own storage adapter or use a third-party solution.
For example, the following are all valid storage adapters:
```js
const storageAdapter = new Map();
// or
const storageAdapter = require('./my-storage-adapter');
// or
const QuickLRU = require('quick-lru');
const storageAdapter = new QuickLRU({maxSize: 1000});
got('sindresorhus.com', {cache: storageAdapter});
```
View the [Keyv docs](https://github.com/lukechilds/keyv) for more information on how to use storage adapters.
## Proxies

@@ -273,3 +410,3 @@

got('todomvc.com', {
got('sindresorhus.com', {
agent: tunnel.httpOverHttp({

@@ -283,3 +420,18 @@ proxy: {

If you require different agents for different protocols, you can pass a map of agents to the `agent` option. This is necessary because a request to one protocol might redirect to another. In such a scenario, `got` will switch over to the right protocol agent for you.
```js
const got = require('got');
const HttpAgent = require('agentkeepalive');
const HttpsAgent = HttpAgent.HttpsAgent;
got('sindresorhus.com', {
agent: {
http: new HttpAgent(),
https: new HttpsAgent()
}
});
```
## Cookies

@@ -416,3 +568,3 @@

got('todomvc.com', {
got('sindresorhus.com', {
headers: {

@@ -433,2 +585,3 @@ 'user-agent': `my-module/${pkg.version} (https://github.com/username/my-module)`

- [travis-got](https://github.com/samverschueren/travis-got) - Convenience wrapper for interacting with the Travis API
- [graphql-got](https://github.com/kevva/graphql-got) - Convenience wrapper for got to interact with GraphQL

@@ -438,5 +591,5 @@

[![Sindre Sorhus](https://avatars.githubusercontent.com/u/170270?v=3&s=100)](https://sindresorhus.com) | [![Vsevolod Strukchinsky](https://avatars.githubusercontent.com/u/365089?v=3&s=100)](https://github.com/floatdrop) | [![Alexander Tesfamichael](https://avatars.githubusercontent.com/u/2011351?v=3&s=100)](https://alextes.me)
---|---|---
[Sindre Sorhus](https://sindresorhus.com) | [Vsevolod Strukchinsky](https://github.com/floatdrop) | [Alexander Tesfamichael](https://alextes.me)
[![Sindre Sorhus](https://github.com/sindresorhus.png?size=100)](https://sindresorhus.com) | [![Vsevolod Strukchinsky](https://github.com/floatdrop.png?size=100)](https://github.com/floatdrop) | [![Alexander Tesfamichael](https://github.com/AlexTes.png?size=100)](https://github.com/AlexTes) | [![Luke Childs](https://github.com/lukechilds.png?size=100)](https://github.com/lukechilds)
---|---|---|---
[Sindre Sorhus](https://sindresorhus.com) | [Vsevolod Strukchinsky](https://github.com/floatdrop) | [Alexander Tesfamichael](https://alextes.me) | [Luke Childs](https://github.com/lukechilds)

@@ -443,0 +596,0 @@ 

license

Sorry, the diff of this file is not supported yet