paged-request
Advanced tools
Comparing version 1.0.2 to 2.0.0
11
index.js
'use strict'; | ||
const needle = require('needle'); | ||
const axios = require('axios'); | ||
@@ -16,3 +16,3 @@ module.exports = async function(url, options, next) { | ||
const opts = Object.assign({}, options); | ||
const acc = { url, options: opts, pages: [], hrefs: [] }; | ||
const acc = { url, options, pages: [], urls: [] }; | ||
let prev; | ||
@@ -23,5 +23,6 @@ let res; | ||
prev = url; | ||
acc.hrefs.push(url); | ||
res = await needle('get', url, null, opts); | ||
url = await next(acc.url, res, acc); | ||
acc.urls.push(url); | ||
res = await axios.get(url, opts); | ||
res.body = res.data; //<= backwards compatibility with 1.0 | ||
url = await next(url, res, acc); | ||
acc.pages.push(res); | ||
@@ -28,0 +29,0 @@ } |
{ | ||
"name": "paged-request", | ||
"description": "Simplified requests for paged (paginated) content.", | ||
"version": "1.0.2", | ||
"version": "2.0.0", | ||
"homepage": "https://github.com/jonschlinkert/paged-request", | ||
@@ -22,9 +22,5 @@ "author": "Jon Schlinkert (https://github.com/jonschlinkert)", | ||
}, | ||
"dependencies": { | ||
"needle": "^2.1.1" | ||
}, | ||
"devDependencies": { | ||
"gulp-format-md": "^1.0.0", | ||
"is-number": "^5.0.0", | ||
"mocha": "^3.5.3" | ||
"mocha": "^5.2.0" | ||
}, | ||
@@ -55,4 +51,11 @@ "keywords": [ | ||
"reflinks": true | ||
} | ||
}, | ||
"reflinks": [ | ||
"axios", | ||
"needle" | ||
] | ||
}, | ||
"dependencies": { | ||
"axios": "^0.18.0" | ||
} | ||
} |
@@ -15,5 +15,9 @@ # paged-request [![NPM version](https://img.shields.io/npm/v/paged-request.svg?style=flat)](https://www.npmjs.com/package/paged-request) [![NPM monthly downloads](https://img.shields.io/npm/dm/paged-request.svg?style=flat)](https://npmjs.org/package/paged-request) [![NPM total downloads](https://img.shields.io/npm/dt/paged-request.svg?style=flat)](https://npmjs.org/package/paged-request) [![Linux Build Status](https://img.shields.io/travis/jonschlinkert/paged-request.svg?style=flat&label=Travis)](https://travis-ci.org/jonschlinkert/paged-request) | ||
## Heads up! | ||
See the [release notes](#release-notes) for information about changes made in v2.0. | ||
## Usage | ||
This library recursively calls [needle's](https://github.com/tomas/needle#needlemethod-url-data-options-callback--20x) `.get` method as long as the user-provided `next()` function returns a string (the next url to get). See [an example](#example). | ||
This library recursively calls [needle's](https://github.com/tomas/needle#needlemethod-url-data-options-callback--20x) `.get` method as long as the user-provide `next()` function returns a string (the next url to get). See [an example](#example). | ||
@@ -53,12 +57,10 @@ **Example** | ||
const request = require('paged-request'); | ||
const isNumber = require('is-number'); | ||
async function next(url, resp, acc) { | ||
// do stuff to check response first if necessary | ||
const regex = /pagination__next[\s\S]+?href=".*?\/(\d+)\/"/; | ||
const num = (regex.exec(resp.body) || [])[1]; | ||
const regex = /href="\/categories\/css\/page\/(\d+)\/"/; | ||
const num = (regex.exec(resp.data) || [])[1]; | ||
// is-number smooths out edge cases, since "num" will be a string | ||
if (isNumber(num)) { | ||
return url + `/page/${num}/`; | ||
if (/^[0-9]+$/.test(num) && +num <= n) { | ||
return `${url}/page/${num}/`; | ||
} | ||
@@ -72,2 +74,9 @@ } | ||
## Release notes | ||
### v2.0 | ||
* renamed `.hrefs` to `.urls` in response object | ||
* now using [axios](https://github.com/axios/axios) instead of [needle](https://github.com/tomas/needle). Please see the axios documentation for API information. | ||
## About | ||
@@ -116,2 +125,9 @@ | ||
### Contributors | ||
| **Commits** | **Contributor** | | ||
| --- | --- | | ||
| 8 | [jonschlinkert](https://github.com/jonschlinkert) | | ||
| 8 | [doowb](https://github.com/doowb) | | ||
### Author | ||
@@ -121,5 +137,5 @@ | ||
* [linkedin/in/jonschlinkert](https://linkedin.com/in/jonschlinkert) | ||
* [github/jonschlinkert](https://github.com/jonschlinkert) | ||
* [twitter/jonschlinkert](https://twitter.com/jonschlinkert) | ||
* [LinkedIn Profile](https://linkedin.com/in/jonschlinkert) | ||
* [GitHub Profile](https://github.com/jonschlinkert) | ||
* [Twitter Profile](https://twitter.com/jonschlinkert) | ||
@@ -133,2 +149,2 @@ ### License | ||
_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.6.0, on January 31, 2018._ | ||
_This file was generated by [verb-generate-readme](https://github.com/verbose/verb-generate-readme), v0.6.0, on May 29, 2018._ |
New author
Supply chain riskA new npm collaborator published a version of the package for the first time. New collaborators are usually benign additions to a project, but do indicate a change to the security surface area of a package.
Found 1 instance in 1 package
8401
2
24
144
0
+ Addedaxios@^0.18.0
+ Addedaxios@0.18.1(transitive)
+ Addedfollow-redirects@1.5.10(transitive)
+ Addedis-buffer@2.0.5(transitive)
- Removedneedle@^2.1.1
- Removeddebug@3.2.7(transitive)
- Removediconv-lite@0.4.24(transitive)
- Removedms@2.1.3(transitive)
- Removedneedle@2.9.1(transitive)
- Removedsafer-buffer@2.1.2(transitive)
- Removedsax@1.3.0(transitive)