Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More
Socket
Sign inDemoInstall
Socket

got

Package Overview
Dependencies
Maintainers
2
Versions
178
Alerts
File Explorer

Advanced tools

Socket logo

Install Socket

Detect and block malicious and high-risk dependencies

Install

got - npm Package Compare versions

Comparing version 10.4.0 to 10.5.0

7

dist/source/create.d.ts

@@ -5,3 +5,3 @@ /// <reference types="node" />

import * as errors from './errors';
import { CancelableRequest, Defaults, ExtendOptions, HandlerFunction, NormalizedOptions, Options, Response, URLOrOptions } from './types';
import { CancelableRequest, Defaults, ExtendOptions, HandlerFunction, NormalizedOptions, Options, Response, URLOrOptions, PaginationOptions } from './types';
export declare type HTTPAlias = 'get' | 'post' | 'put' | 'patch' | 'head' | 'delete';

@@ -55,4 +55,9 @@ export declare type ReturnStream = <T>(url: string | Merge<Options, {

}
export interface GotPaginate {
<T>(url: URLOrOptions & PaginationOptions<T>, options?: Options & PaginationOptions<T>): AsyncIterableIterator<T>;
all<T>(url: URLOrOptions & PaginationOptions<T>, options?: Options & PaginationOptions<T>): Promise<T[]>;
}
export interface Got extends Record<HTTPAlias, GotRequestMethod>, GotRequestMethod {
stream: GotStream;
paginate: GotPaginate;
defaults: Defaults;

@@ -59,0 +64,0 @@ GotError: typeof errors.GotError;

"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
const is_1 = require("@sindresorhus/is");
const as_promise_1 = require("./as-promise");

@@ -92,2 +93,44 @@ const as_stream_1 = require("./as-stream");

}
// @ts-ignore The missing property is added below
got.paginate = async function* (url, options) {
let normalizedOptions = normalize_arguments_1.normalizeArguments(url, options, defaults);
const pagination = normalizedOptions._pagination;
if (!is_1.default.object(pagination)) {
throw new Error('`options._pagination` must be implemented');
}
const all = [];
while (true) {
// @ts-ignore See https://github.com/sindresorhus/got/issues/954
// eslint-disable-next-line no-await-in-loop
const result = await got(normalizedOptions);
// eslint-disable-next-line no-await-in-loop
const parsed = await pagination.transform(result);
for (const item of parsed) {
if (pagination.filter(item, all)) {
if (!pagination.shouldContinue(item, all)) {
return;
}
yield item;
all.push(item);
if (all.length === pagination.countLimit) {
return;
}
}
}
const optionsToMerge = pagination.paginate(result);
if (optionsToMerge === false) {
return;
}
if (optionsToMerge !== undefined) {
normalizedOptions = normalize_arguments_1.normalizeArguments(normalizedOptions, optionsToMerge);
}
}
};
got.paginate.all = async (url, options) => {
const results = [];
for await (const item of got.paginate(url, options)) {
results.push(item);
}
return results;
};
Object.assign(got, { ...errors, mergeOptions: normalize_arguments_1.mergeOptions });

@@ -94,0 +137,0 @@ Object.defineProperty(got, 'defaults', {

@@ -6,2 +6,3 @@ "use strict";

Object.defineProperty(exports, "__esModule", { value: true });
const url_1 = require("url");
const create_1 = require("./create");

@@ -71,3 +72,33 @@ const defaults = {

ignoreInvalidCookies: false,
context: {}
context: {},
_pagination: {
transform: (response) => {
return JSON.parse(response.body);
},
paginate: response => {
if (!Reflect.has(response.headers, 'link')) {
return false;
}
const items = response.headers.link.split(',');
let next;
for (const item of items) {
const parsed = item.split(';');
if (parsed[1].includes('next')) {
next = parsed[0].trimStart().trim();
next = next.slice(1, -1);
break;
}
}
if (next) {
const options = {
url: new url_1.URL(next)
};
return options;
}
return false;
},
filter: () => true,
shouldContinue: () => true,
countLimit: Infinity
}
},

@@ -74,0 +105,0 @@ handlers: [create_1.defaultHandler],

@@ -143,3 +143,3 @@ "use strict";

if (!Reflect.has(setCookie, util_1.promisify.custom)) {
// @ts-ignore TS is dumb.
// @ts-ignore TS is dumb - it says `setCookie` is `never`.
setCookie = util_1.promisify(setCookie.bind(options.cookieJar));

@@ -169,2 +169,24 @@ getCookieString = util_1.promisify(getCookieString.bind(options.cookieJar));

}
// `options._pagination`
if (is_1.default.object(options._pagination)) {
if (defaults && !(Reflect.has(options, 'pagination') && is_1.default.undefined(options._pagination))) {
options._pagination = {
...defaults.pagination,
...options._pagination
};
}
const pagination = options._pagination;
if (!is_1.default.function_(pagination.transform)) {
throw new Error('`options._pagination.transform` must be implemented');
}
if (!is_1.default.function_(pagination.shouldContinue)) {
throw new Error('`options._pagination.shouldContinue` must be implemented');
}
if (!is_1.default.function_(pagination.filter)) {
throw new Error('`options._pagination.filter` must be implemented');
}
if (!is_1.default.function_(pagination.paginate)) {
throw new Error('`options._pagination.paginate` must be implemented');
}
}
// Other values

@@ -171,0 +193,0 @@ options.decompress = Boolean(options.decompress);

@@ -95,3 +95,3 @@ /// <reference types="node" />

export declare const requestSymbol: unique symbol;
export declare type DefaultOptions = Merge<Required<Except<GotOptions, 'hooks' | 'retry' | 'timeout' | 'context' | 'agent' | 'body' | 'cookieJar' | 'encoding' | 'form' | 'json' | 'lookup' | 'request' | 'url' | typeof requestSymbol>>, {
export declare type DefaultOptions = Merge<Required<Except<GotOptions, 'hooks' | 'retry' | 'timeout' | 'context' | '_pagination' | 'agent' | 'body' | 'cookieJar' | 'encoding' | 'form' | 'json' | 'lookup' | 'request' | 'url' | typeof requestSymbol>>, {
hooks: Required<Hooks>;

@@ -103,4 +103,14 @@ retry: DefaultRetryOptions;

};
_pagination?: PaginationOptions<unknown>['_pagination'];
}>;
export interface GotOptions {
export interface PaginationOptions<T> {
_pagination?: {
transform?: (response: Response) => Promise<T[]> | T[];
filter?: (item: T, allItems: T[]) => boolean;
paginate?: (response: Response) => Options | false;
shouldContinue?: (item: T, allItems: T[]) => boolean;
countLimit?: number;
};
}
export interface GotOptions extends PaginationOptions<unknown> {
[requestSymbol]?: RequestFunction;

@@ -156,2 +166,3 @@ url?: URL | string;

maxRedirects: number;
pagination?: Required<PaginationOptions<unknown>['_pagination']>;
[requestSymbol]: RequestFunction;

@@ -158,0 +169,0 @@ decompress: boolean;

12

package.json
{
"name": "got",
"version": "10.4.0",
"version": "10.5.0",
"description": "Human-friendly and powerful HTTP request library for Node.js",

@@ -47,3 +47,3 @@ "license": "MIT",

"@types/cacheable-request": "^6.0.1",
"cacheable-lookup": "^1.0.0",
"cacheable-lookup": "^2.0.0",
"cacheable-request": "^7.0.1",

@@ -61,3 +61,3 @@ "decompress-response": "^5.0.0",

"devDependencies": {
"@ava/typescript": "^1.0.0",
"@ava/typescript": "^1.1.0",
"@sindresorhus/tsconfig": "^0.7.0",

@@ -73,3 +73,3 @@ "@types/duplexer3": "^0.1.0",

"@typescript-eslint/parser": "^2.17.0",
"ava": "^3.1.0",
"ava": "^3.2.0",
"coveralls": "^3.0.4",

@@ -107,3 +107,2 @@ "create-test-server": "^3.0.1",

],
"concurrency": 4,
"timeout": "1m",

@@ -119,2 +118,5 @@ "typescript": {

".ts"
],
"exclude": [
"**/test/**"
]

@@ -121,0 +123,0 @@ },

@@ -29,3 +29,5 @@ <div align="center">

- [Promise & stream API](#api)
- [Promise API](#api)
- [Stream API](#streams)
- [Pagination API (experimental)](#pagination)
- [Request cancelation](#aborting-the-request)

@@ -250,3 +252,3 @@ - [RFC compliant caching](#cache-adapters)

The promise has also `.text()`, `.json()` and `.buffer()` methods which set this option automatically.
The promise also has `.text()`, `.json()` and `.buffer()` methods which sets this and the `resolveBodyOnly` option automatically.

@@ -260,3 +262,3 @@ Example:

// is the same as this
const body = await got(url, {responseType: 'json'});
const body = await got(url, {responseType: 'json', resolveBodyOnly: true});
```

@@ -641,2 +643,45 @@

##### \_pagination
Type: `object`
**Note:** This feature is marked as experimental as we're [looking for feedback](https://github.com/sindresorhus/got/issues/1052) on the API and how it works. The feature itself is stable, but the API may change based on feedback. So if you decide to try it out, we suggest locking down the `got` dependency semver range or use a lockfile.
###### \_pagination.transform
Type: `Function`\
Default: `response => JSON.parse(response.body)`
A function that transform [`Response`](#response) into an array of items. This is where you should do the parsing.
###### \_pagination.paginate
Type: `Function`\
Default: [`Link` header logic](source/index.ts)
A function that returns an object representing Got options pointing to the next page. If there are no more pages, `false` should be returned.
###### \_pagination.filter
Type: `Function`\
Default: `(item, allItems) => true`
Checks whether the item should be emitted or not.
###### \_pagination.shouldContinue
Type: `Function`\
Default: `(item, allItems) => true`
Checks whether the pagination should continue.
For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems) => allItems.some(entry => entry.flag)` instead.
###### \_pagination.countLimit
Type: `number`\
Default: `Infinity`
The maximum amount of items that should be emitted.
#### Response

@@ -792,2 +837,26 @@

#### Pagination
#### got.paginate(url, options?)
Returns an async iterator:
```js
(async () => {
const countLimit = 10;
const pagination = got.paginate('https://api.github.com/repos/sindresorhus/got/commits', {
_pagination: {countLimit}
});
console.log(`Printing latest ${countLimit} Got commits (newest to oldest):`);
for await (const commitData of pagination) {
console.log(commitData.commit.message);
}
})();
```
See [`options._pagination`](#_pagination) for more pagination options.
#### got.get(url, options?)

@@ -1470,2 +1539,3 @@ #### got.post(url, options?)

| Stream API | :heavy_check_mark: | :heavy_check_mark: | Node.js only | :x: | :x: | :heavy_check_mark: |
| Pagination API | :sparkle: | :x: | :x: | :x: | :x: | :x: |
| Request cancelation | :heavy_check_mark: | :x: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: | :heavy_check_mark: |

@@ -1497,3 +1567,4 @@ | RFC compliant caching | :heavy_check_mark: | :x: | :x: | :x: | :x: | :x: |

\*\*\* Currently, only `DownloadProgress` event is supported, `UploadProgress` event is not supported.\
:grey_question: Experimental support.
:sparkle: Almost-stable feature, but the API may change. Don't hestitate to try it out!\
:grey_question: Feature in early stage of development. Very experimental.

@@ -1500,0 +1571,0 @@ <!-- GITHUB -->

SocketSocket SOC 2 Logo

Product

  • Package Alerts
  • Integrations
  • Docs
  • Pricing
  • FAQ
  • Roadmap
  • Changelog

Packages

npm

Stay in touch

Get open source security insights delivered straight into your inbox.


  • Terms
  • Privacy
  • Security

Made with ⚡️ by Socket Inc