Huge News!Announcing our $40M Series B led by Abstract Ventures.Learn More →
rock-req - npm Package Compare versions
Comparing version 5.0.3 to 5.1.0
56
index.js
@@ -1,3 +0,1 @@ | ||
| module.exports = extend() | ||
| const http = require('http') | ||
@@ -8,3 +6,3 @@ const https = require('https') | ||
| const zlib = require('zlib') | ||
| const { pipeline, Writable, Readable } = require('stream') | ||
| const { Writable } = require('stream') | ||
@@ -20,5 +18,6 @@ const isStream = o => o !== null && typeof o === 'object' && typeof o.pipe === 'function' | ||
| maxRedirects: 10, | ||
| maxRetry: 0, | ||
| retryDelay: 100, // ms | ||
| retryOnCode: [408, 429, 500, 502, 503, 504, 521, 522, 524], | ||
| maxRetry: 1, | ||
| retryDelay: 10, // ms | ||
| keepAliveDuration: 3000, // ms | ||
| retryOnCode: [408, 429, 502, 503, 504, 521, 522, 524], | ||
| retryOnError: ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED', 'EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN'], | ||
@@ -29,4 +28,4 @@ beforeRequest: o => o | ||
| _default = applyDefault(defaultOptions, _default) // inherits of parent options | ||
| const agents = [http, https].map(h => (_default.keepAliveDuration > 0) ? new h.Agent({ keepAlive: true, keepAliveMsecs: _default.keepAliveDuration, timeout: _default.keepAliveDuration /* remove free socket node < 19 */ }) : undefined) | ||
| // all options https://nodejs.org/dist/latest-v18.x/docs/api/http.html#http_http_request_url_options_callback | ||
| function rock (opts, directBody, cb) { | ||
@@ -64,5 +63,7 @@ if (typeof opts === 'string') opts = { url: opts } | ||
| if (opts.method) opts.method = opts.method.toUpperCase() | ||
| if (!opts.agent || opts.agent.protocol !== opts.protocol) opts.agent = (opts.protocol === 'https:' ? agents[1] : agents[0]) | ||
| const protocol = opts.protocol === 'https:' ? https : http // Support http/https urls | ||
| const chunks = [] | ||
| const streamToCleanOnError = [] | ||
| let requestAbortedOrEnded = false | ||
@@ -73,7 +74,10 @@ let response = null | ||
| requestAbortedOrEnded = true | ||
| if (opts.retryOnError.indexOf(err?.code) !== -1 && --opts.remainingRetry > 0) { | ||
| opts.prevError = err | ||
| return setTimeout(rock, opts.retryDelay, opts, cb) // retry in 100ms | ||
| if (err) { | ||
| streamToCleanOnError.forEach(s => s.destroy(err)) | ||
| if (opts.retryOnError.indexOf(err?.code) !== -1 && --opts.remainingRetry > 0) { | ||
| opts.prevError = err | ||
| return setTimeout(rock, opts.retryDelay, opts, cb) // retry in 100ms | ||
| } | ||
| return cb(err) | ||
| } | ||
| if (err) return cb(err) | ||
| let data = Buffer.concat(chunks) | ||
@@ -85,2 +89,11 @@ if (opts.json) { | ||
| } | ||
| function listen (stream, isLast = false) { | ||
| streamToCleanOnError.push(stream) | ||
| stream.once('error', onRequestEnd) | ||
| stream.once('close', () => { | ||
| if (stream.readableEnded === false || stream.writableEnded === false) return onRequestEnd(new Error('ERR_STREAM_PREMATURE_CLOSE')) | ||
| if (isLast === true) return onRequestEnd() | ||
| }) | ||
| return stream | ||
| } | ||
| const req = protocol.request(opts, res => { | ||
@@ -125,23 +138,18 @@ opts.prevStatusCode = res.statusCode | ||
| case 'br': | ||
| pipeline(res, zlib.createBrotliDecompress(), output, onRequestEnd); break | ||
| listen(res).pipe(listen(zlib.createBrotliDecompress())).pipe(listen(output, true)); break | ||
| case 'gzip': | ||
| case 'deflate': | ||
| pipeline(res, zlib.createUnzip(), output, onRequestEnd); break | ||
| listen(res).pipe(listen(zlib.createUnzip())).pipe(listen(output, true)); break | ||
| default: | ||
| pipeline(res, output, onRequestEnd); break | ||
| listen(res).pipe(listen(output, true)); break | ||
| } | ||
| }) | ||
| req.once('timeout', () => { | ||
| // This timeout event can come after the input pipeline is finished (ex. timeout with no body) | ||
| const _error = new Error('TimeoutError'); _error.code = 'ETIMEDOUT' | ||
| req.destroy(_error) // we must destroy manually and send the error to the error listener to call onRequestEnd | ||
| }) | ||
| req.once('error', (e) => { | ||
| onRequestEnd(e) // error can happen before pipeline is executed when some interceptor are used such as nock | ||
| req.destroy() | ||
| }) | ||
| const _inputStream = isFnStream(body) ? body(opts) : Readable.from([body], { objectMode: false }) | ||
| pipeline(_inputStream, req, (e) => { | ||
| if (e) onRequestEnd(e) | ||
| }) | ||
| if (isFnStream(body) === true) listen(body(opts)).pipe(listen(req)) | ||
| else listen(req).end(body) | ||
| return req | ||
@@ -165,1 +173,3 @@ } | ||
| } | ||
| module.exports = extend() |
| { | ||
| "name": "rock-req", | ||
| "description": "Zero dependencies (160 LOC) & rock-solid request library: http/https, reliable retry on failure, redirects, gzip/deflate/brotli, extensible, proxy, streams, JSON mode, forms, timeout", | ||
| "version": "5.0.3", | ||
| "description": "Zero deps (160 LOC) & ultra-fast request library: http/https, reliable retry on failure, redirects, gzip/deflate/brotli, extensible, proxy, streams, JSON mode, forms, timeout", | ||
| "version": "5.1.0", | ||
| "author": { | ||
@@ -6,0 +6,0 @@ "name": "David Grelaud & Feross Aboukhadijeh" |
132
README.md
@@ -8,3 +8,3 @@ | ||
| <p align="center">Ensure your HTTP requests always reach their destination!</p> | ||
| <p align="center">⭐️⭐️ Ensure your HTTP requests always reach their destination as <b>efficiently</b> as possible! ⭐️⭐️</p> | ||
@@ -20,3 +20,3 @@ [![npm][npm-image]][npm-url] [![ci][ci-image]][ci-url] [![javascript style guide][standard-image]][standard-url] | ||
| ## 🔥 Why should you need this? | ||
| ## 🔥 Why? | ||
@@ -29,5 +29,5 @@ In most existing libraries (2023): | ||
| - Many request libraries are heavy: node-fetch, superagent, needle, got, axios, request | ||
| - Lightweight alternatives are not as light as they claim due to dependencies (simple-get, tiny-req, puny-req, ...) | ||
| - Lightweight alternatives are not as light as they claim due to dependencies (simple-get, tiny-req, puny-req, phin, ...) | ||
| ⚡️ **Rock-req** solves these problems with only **160 lines of code** and **zero dependencies** | ||
| ⚡️ **Rock-req** solves these problems with only **150 lines of code** and **zero dependencies** | ||
@@ -37,6 +37,10 @@ It also supports many features: | ||
| - Follows redirects | ||
| - Handles gzip/deflate/brotli responses | ||
| - Handles **gzip/deflate/brotli** responses | ||
| - Modify defaults | ||
| - Extend and create new instances | ||
| - Automatically destroy input/output stream on error (pipeline) | ||
| - Automatically destroy input/output **stream** on error and premature close event | ||
| - **Advanced retries** | ||
| - URL Rewrite | ||
| - **Ultra-fast (> 20k req/s)** | ||
| - Keep Alive by default (3000ms) | ||
| - Composable | ||
@@ -48,7 +52,32 @@ - Timeouts | ||
| Like NodeJS pipeline, when the callback is called, the request is 100% finished, even with streams. | ||
| When the callback is called, the request is 100% finished, even with streams. | ||
| ## 🚀 Benchmark Rock-req vs got, axios, node-fetch, phin, simple-get, superagent, ... | ||
| Stop using "slow by-default" and "false-light" HTTP request libraries! | ||
| | Library | Speed | Size deps inc. | | ||
| | ------------ |-----------------:| --------------:| | ||
| | rock-req 🙋♂️ | 21797 req/s | 144 LOC | | ||
| | simple-get | 3260 req/s | 317 LOC | | ||
| | axios | 4910 req/s | 13983 LOC | | ||
| | got | 1762 req/s | 9227 LOC | | ||
| | fetch | 2102 req/s | 13334 LOC | | ||
| | request | 1869 req/s | 46572 LOC | | ||
| | superagent | 2100 req/s | 16109 LOC | | ||
| | phin | 1164 req/s | 331 LOC | | ||
| | undici* | 24378 req/s | 16225 LOC | | ||
| > `undici` is a low-level API, faster alternative to the native NodeJS http module. It is the glass ceiling limit for NodeJS. | ||
| > `rock-req` uses only the native NodeJS http module and provides many high-level features, a lot more than `phin` and `simple-get` with fewer lines | ||
| > Tested with NodeJS 18.x LTS on Macbook Pro M1 Max | ||
| ## Install | ||
| ``` | ||
@@ -144,3 +173,3 @@ npm install rock-req | ||
| - `maxRedirects <number>`overwrite global maximum number of redirects. Defaults to 10 | ||
| - `maxRetry <number>` overwrite global maximum number of retries. Defaults to 0 | ||
| - `maxRetry <number>` overwrite global maximum number of retries. Defaults to 1 | ||
| - `followRedirects <boolean>` do not follow redirects | ||
@@ -171,4 +200,5 @@ - `body <buffer> | <string> | <object> | <function>` body to post | ||
| This function is invoked by rock-req for every request retry. | ||
| If something goes wrong, the old stream is destroyed. | ||
| If something goes wrong, the Readable stream is destroyed automatically and the error can be captured with `'error'` event or `stream.finished` (optional). | ||
| ```js | ||
@@ -201,8 +231,10 @@ const rock = require('rock-req') | ||
| Rock-req requires that output stream is initialized in a function. | ||
| This function is invoked by rock-req for every request retry. | ||
| If something goes wrong, the Writable stream is destroyed automatically and the error can be captured with `'error'` event or `stream.finished` (optional). | ||
| ```js | ||
| const rock = require('rock-req') | ||
| const fs = require('fs') | ||
| const { finished } = require('stream') | ||
@@ -214,17 +246,4 @@ // opts contains options passed in rock(opts). DO NOT MODIFY IT | ||
| const writer = fs.createWriteStream('test_gfg.txt') | ||
| // Internally, rock-req uses pipeline. If something goes wrong, the stream is destroyed automatically. | ||
| // If you need to do some action (removing temporary files, ...), uses this native NodeJS method: | ||
| const cleanup = finished(writer, (err) => { | ||
| if (err) { | ||
| // clean up things | ||
| } | ||
| // When using the finished() method in NodeJS, it's important to be aware that it can leave some event listeners | ||
| // (specifically, the 'error', 'end', 'finish', and 'close' events) hanging around even after this callback function has been called. | ||
| // This is intentional, as it helps prevent unexpected crashes if an error occurs due to incorrect stream implementations. | ||
| // However, if you don't want these event listeners to stick around after the callback function has been called, | ||
| // you can use the cleanup function that's returned by stream.finished() to remove them. | ||
| // You'll need to explicitly call this cleanup function within your callback function to ensure that the event listeners get removed properly. | ||
| cleanup(); | ||
| }); | ||
| // It must return a Writable stream. Otherwise, the request is cancel with an error | ||
| writer.on('error', (e) => { /* clean up your stuff */ }) | ||
| return writer | ||
@@ -242,3 +261,3 @@ } | ||
| By default, rock-req retries with the following errors if `maxRetry > 1`. | ||
| By default, rock-req retries with the following errors if `maxRetry > 0`. | ||
@@ -254,3 +273,2 @@ The callback is called when the request succeed or all retries are done | ||
| 429, /* Too Many Requests */ | ||
| 500, /* Internal Server Error */ | ||
| 502, /* Bad Gateway */ | ||
@@ -277,3 +295,3 @@ 503, /* Service Unavailable */ | ||
| body : 'this is the POST body', | ||
| maxRetry : 2 // 0 is the default value (= no retries) | ||
| maxRetry : 1 | ||
| } | ||
@@ -284,3 +302,3 @@ rock(opts, function (err, res, data) {} ); | ||
| ### Global options | ||
| ### Global options & Extend | ||
@@ -291,8 +309,8 @@ Change default parameters globally (not recommended), or create a new instance with specific paramaters (see below) | ||
| rock.defaults = { | ||
| headers : { 'accept-encoding': 'gzip, deflate, br' }, | ||
| maxRedirects : 10, | ||
| maxRetry : 0, | ||
| retryDelay : 100, //ms | ||
| retryOnCode : [408, 429, 500, 502, 503, 504, 521, 522, 524 ], | ||
| retryOnError : ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED','EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN' ], | ||
| headers : { 'accept-encoding': 'gzip, deflate, br' }, | ||
| maxRedirects : 10, | ||
| maxRetry : 1, | ||
| retryDelay : 10, //ms | ||
| retryOnCode : [408, 429, 502, 503, 504, 521, 522, 524 ], | ||
| retryOnError : ['ETIMEDOUT', 'ECONNRESET', 'EADDRINUSE', 'ECONNREFUSED','EPIPE', 'ENOTFOUND', 'ENETUNREACH', 'EAI_AGAIN' ], | ||
| // beforeRequest is called for each request, retry and redirect | ||
@@ -311,2 +329,3 @@ beforeRequest : (opts) => { | ||
| opts.remainingRedirects; | ||
| opts.agent = otherHttpAgent; | ||
@@ -326,12 +345,24 @@ // READ-ONLY options (not exhaustive) | ||
| ### Extend and intercept retries | ||
| Create a new instance with specific parameters instead of modifying global `rock.defaults`. | ||
| Create a new instance with specific parameter instead of modifying `rock.defaults` | ||
| By default, this new instance inherits values of the instance source if options are not overwritten. | ||
| Headers are merged. Then only the first level of the options object is merged (no deep travelling in sub-objects or arrays). | ||
| Here is a basic example of `beforeRequest` interceptor to use [HAProxy as a forward proxy](https://www.haproxy.com/user-spotlight-series/haproxy-as-egress-controller/). | ||
| The `keepAliveDuration` can be changed only with `extend` method because `rock-req` creates new http Agent on new instances. | ||
| `beforeRequest` is always called on each redirect/retry. | ||
| ```js | ||
| const myInstance = rock.extend({ | ||
| keepAliveDuration : 0, // Change keep alive duration. Default to 3000ms. Set 0 to deactivate keep alive. | ||
| headers: { | ||
| 'Custom-header': 'x-for-proxy' | ||
| }, | ||
| timeout : 1000 | ||
| }); | ||
| myInstance.get('http://example.com', function (err, res, data) {}) | ||
| ``` | ||
| ### Intercept retries for Higher Availability / Higher bandwidth | ||
| `beforeRequest` is always called on each request, each redirect and each retry. | ||
| - on redirect, `opts.url` (and `hostname`, `port`, `protocol`, `path`) is updated to the new location. `opts.url` is null if it is a relative redirect. | ||
@@ -341,3 +372,9 @@ - on retry, `opts.url` (and `hostname`, `port`, `protocol`, `path`) have the same value as they did | ||
| For example, you can dynamically change the http Agent to use a another proxy on each request. | ||
| Be careful, in this case, you must provide the right http/https Agent if there is a redirection from http to https. | ||
| Otherwise, rock-req automatically replaces your Agent with the correct one if the protocol changes after redirection. | ||
| Or, you can rewrite the URL if you want to use [HAProxy as a forward proxy](https://www.haproxy.com/user-spotlight-series/haproxy-as-egress-controller/). | ||
| ```js | ||
@@ -352,7 +389,3 @@ const myInstance = rock.extend({ | ||
| return opts; | ||
| }, | ||
| headers: { | ||
| 'Custom-header': 'x-for-proxy' | ||
| }, | ||
| timeout : 1000 | ||
| } | ||
| }); | ||
@@ -365,3 +398,2 @@ | ||
| ### Timeout | ||
@@ -579,8 +611,12 @@ | ||
| ## Notes: | ||
| ## TODO: | ||
| - [ ] replace deprecated `url.parse` by `new URL` but new URL is slower than url.parse. Let's see if Node 20 LTS is faster | ||
| - [ ] agent keep Alive | ||
| - [ ] add advanced timeout (response timeout) | ||
| - [ ] test prevError | ||
| - [ ] test HTTP abort signal option | ||
| - [ ] test input stream error with 502 error retry. Does stream.resume destroy all streams? | ||
| - [ ] promisify | ||
| - [ ] typescript type | ||
| - [ ] NodesJS 19 doesn't need agent.timeout to exit https://github.com/nodejs/node/issues/47228 https://github.com/nodejs/node/issues/2642 | ||
@@ -587,0 +623,0 @@ |
New alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Fixed alerts
License Policy Violation
LicenseThis package is not allowed per your license policy. Review the package's license to ensure compliance.
Found 1 instance in 1 package
Improved metrics
- Total package byte prevSize
- increased by6.93%
29522
- Lines of code
- increased by5.37%
157
- Number of lines in readme file
- increased by6.17%
619
No dependency changes