got - npm Package Compare versions

Comparing version 10.4.0 to 10.5.0

dist/source/create.d.ts

@@ -5,3 +5,3 @@ /// <reference types="node" />
 import * as errors from './errors';
-import { CancelableRequest, Defaults, ExtendOptions, HandlerFunction, NormalizedOptions, Options, Response, URLOrOptions } from './types';
+import { CancelableRequest, Defaults, ExtendOptions, HandlerFunction, NormalizedOptions, Options, Response, URLOrOptions, PaginationOptions } from './types';
 export declare type HTTPAlias = 'get' | 'post' | 'put' | 'patch' | 'head' | 'delete';
@@ -55,4 +55,9 @@ export declare type ReturnStream = <T>(url: string | Merge<Options, {
 }
+export interface GotPaginate {
+    <T>(url: URLOrOptions & PaginationOptions<T>, options?: Options & PaginationOptions<T>): AsyncIterableIterator<T>;
+    all<T>(url: URLOrOptions & PaginationOptions<T>, options?: Options & PaginationOptions<T>): Promise<T[]>;
+}
 export interface Got extends Record<HTTPAlias, GotRequestMethod>, GotRequestMethod {
     stream: GotStream;
+    paginate: GotPaginate;
     defaults: Defaults;
@@ -59,0 +64,0 @@ GotError: typeof errors.GotError;

dist/source/create.js

 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const is_1 = require("@sindresorhus/is");
 const as_promise_1 = require("./as-promise");
@@ -92,2 +93,44 @@ const as_stream_1 = require("./as-stream");
     }
+    // @ts-ignore The missing property is added below
+    got.paginate = async function* (url, options) {
+        let normalizedOptions = normalize_arguments_1.normalizeArguments(url, options, defaults);
+        const pagination = normalizedOptions._pagination;
+        if (!is_1.default.object(pagination)) {
+            throw new Error('`options._pagination` must be implemented');
+        }
+        const all = [];
+        while (true) {
+            // @ts-ignore See https://github.com/sindresorhus/got/issues/954
+            // eslint-disable-next-line no-await-in-loop
+            const result = await got(normalizedOptions);
+            // eslint-disable-next-line no-await-in-loop
+            const parsed = await pagination.transform(result);
+            for (const item of parsed) {
+                if (pagination.filter(item, all)) {
+                    if (!pagination.shouldContinue(item, all)) {
+                        return;
+                    }
+                    yield item;
+                    all.push(item);
+                    if (all.length === pagination.countLimit) {
+                        return;
+                    }
+                }
+            }
+            const optionsToMerge = pagination.paginate(result);
+            if (optionsToMerge === false) {
+                return;
+            }
+            if (optionsToMerge !== undefined) {
+                normalizedOptions = normalize_arguments_1.normalizeArguments(normalizedOptions, optionsToMerge);
+            }
+        }
+    };
+    got.paginate.all = async (url, options) => {
+        const results = [];
+        for await (const item of got.paginate(url, options)) {
+            results.push(item);
+        }
+        return results;
+    };
     Object.assign(got, { ...errors, mergeOptions: normalize_arguments_1.mergeOptions });
@@ -94,0 +137,0 @@ Object.defineProperty(got, 'defaults', {

dist/source/index.js

@@ -6,2 +6,3 @@ "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+const url_1 = require("url");
 const create_1 = require("./create");
@@ -71,3 +72,33 @@ const defaults = {
         ignoreInvalidCookies: false,
-        context: {}
+        context: {},
+        _pagination: {
+            transform: (response) => {
+                return JSON.parse(response.body);
+            },
+            paginate: response => {
+                if (!Reflect.has(response.headers, 'link')) {
+                    return false;
+                }
+                const items = response.headers.link.split(',');
+                let next;
+                for (const item of items) {
+                    const parsed = item.split(';');
+                    if (parsed[1].includes('next')) {
+                        next = parsed[0].trimStart().trim();
+                        next = next.slice(1, -1);
+                        break;
+                    }
+                }
+                if (next) {
+                    const options = {
+                        url: new url_1.URL(next)
+                    };
+                    return options;
+                }
+                return false;
+            },
+            filter: () => true,
+            shouldContinue: () => true,
+            countLimit: Infinity
+        }
     },
@@ -74,0 +105,0 @@ handlers: [create_1.defaultHandler],

dist/source/normalize-arguments.js

@@ -143,3 +143,3 @@ "use strict";
             if (!Reflect.has(setCookie, util_1.promisify.custom)) {
-                // @ts-ignore TS is dumb.
+                // @ts-ignore TS is dumb - it says `setCookie` is `never`.
                 setCookie = util_1.promisify(setCookie.bind(options.cookieJar));
@@ -169,2 +169,24 @@ getCookieString = util_1.promisify(getCookieString.bind(options.cookieJar));
     }
+    // `options._pagination`
+    if (is_1.default.object(options._pagination)) {
+        if (defaults && !(Reflect.has(options, 'pagination') && is_1.default.undefined(options._pagination))) {
+            options._pagination = {
+                ...defaults.pagination,
+                ...options._pagination
+            };
+        }
+        const pagination = options._pagination;
+        if (!is_1.default.function_(pagination.transform)) {
+            throw new Error('`options._pagination.transform` must be implemented');
+        }
+        if (!is_1.default.function_(pagination.shouldContinue)) {
+            throw new Error('`options._pagination.shouldContinue` must be implemented');
+        }
+        if (!is_1.default.function_(pagination.filter)) {
+            throw new Error('`options._pagination.filter` must be implemented');
+        }
+        if (!is_1.default.function_(pagination.paginate)) {
+            throw new Error('`options._pagination.paginate` must be implemented');
+        }
+    }
     // Other values
@@ -171,0 +193,0 @@ options.decompress = Boolean(options.decompress);

dist/source/types.d.ts

@@ -95,3 +95,3 @@ /// <reference types="node" />
 export declare const requestSymbol: unique symbol;
-export declare type DefaultOptions = Merge<Required<Except<GotOptions, 'hooks' | 'retry' | 'timeout' | 'context' | 'agent' | 'body' | 'cookieJar' | 'encoding' | 'form' | 'json' | 'lookup' | 'request' | 'url' | typeof requestSymbol>>, {
+export declare type DefaultOptions = Merge<Required<Except<GotOptions, 'hooks' | 'retry' | 'timeout' | 'context' | '_pagination' | 'agent' | 'body' | 'cookieJar' | 'encoding' | 'form' | 'json' | 'lookup' | 'request' | 'url' | typeof requestSymbol>>, {
     hooks: Required<Hooks>;
@@ -103,4 +103,14 @@ retry: DefaultRetryOptions;
     };
+    _pagination?: PaginationOptions<unknown>['_pagination'];
 }>;
-export interface GotOptions {
+export interface PaginationOptions<T> {
+    _pagination?: {
+        transform?: (response: Response) => Promise<T[]> | T[];
+        filter?: (item: T, allItems: T[]) => boolean;
+        paginate?: (response: Response) => Options | false;
+        shouldContinue?: (item: T, allItems: T[]) => boolean;
+        countLimit?: number;
+    };
+}
+export interface GotOptions extends PaginationOptions<unknown> {
     [requestSymbol]?: RequestFunction;
@@ -156,2 +166,3 @@ url?: URL | string;
     maxRedirects: number;
+    pagination?: Required<PaginationOptions<unknown>['_pagination']>;
     [requestSymbol]: RequestFunction;
@@ -158,0 +169,0 @@ decompress: boolean;

package.json

 {
 	"name": "got",
-	"version": "10.4.0",
+	"version": "10.5.0",
 	"description": "Human-friendly and powerful HTTP request library for Node.js",
@@ -47,3 +47,3 @@ "license": "MIT",
 		"@types/cacheable-request": "^6.0.1",
-		"cacheable-lookup": "^1.0.0",
+		"cacheable-lookup": "^2.0.0",
 		"cacheable-request": "^7.0.1",
@@ -61,3 +61,3 @@ "decompress-response": "^5.0.0",
 	"devDependencies": {
-		"@ava/typescript": "^1.0.0",
+		"@ava/typescript": "^1.1.0",
 		"@sindresorhus/tsconfig": "^0.7.0",
@@ -73,3 +73,3 @@ "@types/duplexer3": "^0.1.0",
 		"@typescript-eslint/parser": "^2.17.0",
-		"ava": "^3.1.0",
+		"ava": "^3.2.0",
 		"coveralls": "^3.0.4",
@@ -107,3 +107,2 @@ "create-test-server": "^3.0.1",
 		],
-		"concurrency": 4,
 		"timeout": "1m",
@@ -119,2 +118,5 @@ "typescript": {
 			".ts"
+		],
+		"exclude": [
+			"**/test/**"
 		]
@@ -121,0 +123,0 @@ },

readme.md

@@ -29,3 +29,5 @@ <div align="center">
 
-- [Promise & stream API](#api)
+- [Promise API](#api)
+- [Stream API](#streams)
+- [Pagination API (experimental)](#pagination)
 - [Request cancelation](#aborting-the-request)
@@ -250,3 +252,3 @@ - [RFC compliant caching](#cache-adapters)
 
-The promise has also `.text()`, `.json()` and `.buffer()` methods which set this option automatically.
+The promise also has `.text()`, `.json()` and `.buffer()` methods which sets this and the `resolveBodyOnly` option automatically.
 
@@ -260,3 +262,3 @@ Example:
 // is the same as this
-const body = await got(url, {responseType: 'json'});
+const body = await got(url, {responseType: 'json', resolveBodyOnly: true});
 ```
@@ -641,2 +643,45 @@
 
+##### \_pagination
+
+Type: `object`
+
+**Note:** This feature is marked as experimental as we're [looking for feedback](https://github.com/sindresorhus/got/issues/1052) on the API and how it works. The feature itself is stable, but the API may change based on feedback. So if you decide to try it out, we suggest locking down the `got` dependency semver range or use a lockfile.
+
+###### \_pagination.transform
+
+Type: `Function`\
+Default: `response => JSON.parse(response.body)`
+
+A function that transform [`Response`](#response) into an array of items. This is where you should do the parsing.
+
+###### \_pagination.paginate
+
+Type: `Function`\
+Default: [`Link` header logic](source/index.ts)
+
+A function that returns an object representing Got options pointing to the next page. If there are no more pages, `false` should be returned.
+
+###### \_pagination.filter
+
+Type: `Function`\
+Default: `(item, allItems) => true`
+
+Checks whether the item should be emitted or not.
+
+###### \_pagination.shouldContinue
+
+Type: `Function`\
+Default: `(item, allItems) => true`
+
+Checks whether the pagination should continue.
+
+For example, if you need to stop **before** emitting an entry with some flag, you should use `(item, allItems) => !item.flag`. If you want to stop **after** emitting the entry, you should use `(item, allItems) => allItems.some(entry => entry.flag)` instead.
+
+###### \_pagination.countLimit
+
+Type: `number`\
+Default: `Infinity`
+
+The maximum amount of items that should be emitted.
+
 #### Response
@@ -792,2 +837,26 @@
 
+#### Pagination
+
+#### got.paginate(url, options?)
+
+Returns an async iterator:
+
+```js
+(async () => {
+	const countLimit = 10;
+
+	const pagination = got.paginate('https://api.github.com/repos/sindresorhus/got/commits', {
+		_pagination: {countLimit}
+	});
+
+	console.log(`Printing latest ${countLimit} Got commits (newest to oldest):`);
+
+	for await (const commitData of pagination) {
+		console.log(commitData.commit.message);
+	}
+})();
+```
+
+See [`options._pagination`](#_pagination) for more pagination options.
+
 #### got.get(url, options?)
@@ -1470,2 +1539,3 @@ #### got.post(url, options?)
 | Stream API            | :heavy_check_mark: | :heavy_check_mark: | Node.js only         | :x:                      | :x:                | :heavy_check_mark:     |
+| Pagination API        | :sparkle:          | :x:                | :x:                  | :x:                      | :x:                | :x:                    |
 | Request cancelation   | :heavy_check_mark: | :x:                | :heavy_check_mark:   | :heavy_check_mark:       | :heavy_check_mark: | :heavy_check_mark:     |
@@ -1497,3 +1567,4 @@ | RFC compliant caching | :heavy_check_mark: | :x:                | :x:                  | :x:                      | :x:                | :x:                    |
 \*\*\* Currently, only `DownloadProgress` event is supported, `UploadProgress` event is not supported.\
-:grey_question: Experimental support.
+:sparkle: Almost-stable feature, but the API may change. Don't hestitate to try it out!\
+:grey_question: Feature in early stage of development. Very experimental.
 
@@ -1500,0 +1571,0 @@ <!-- GITHUB -->

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

162125

increased by4.6%

Lines of code

2370

increased by4.96%

Number of lines in readme file

1728

increased by4.28%

Dependency changes

• + Addedcacheable-lookup@2.0.1(transitive)

• - Removedcacheable-lookup@1.0.0(transitive)

• Updatedcacheable-lookup@^2.0.0