urllib - npm Package Compare versions

Comparing version 3.16.1 to 3.17.0

package.json

 {
   "name": "urllib",
-  "version": "3.16.1",
+  "version": "3.17.0",
   "publishConfig": {
@@ -56,2 +56,3 @@ "tag": "latest"
     "test": "npm run lint && vitest run",
+    "test-keepalive": "cross-env TEST_KEEPALIVE_COUNT=50 vitest run --test-timeout 120000 keep-alive-header.test.ts",
     "cov": "vitest run --coverage",
@@ -82,4 +83,5 @@ "ci": "npm run lint && npm run cov && npm run build:test",
     "@types/tar-stream": "^2.2.2",
-    "@vitest/coverage-c8": "^0.29.7",
+    "@vitest/coverage-v8": "^0.32.0",
     "busboy": "^1.6.0",
+    "cross-env": "^7.0.3",
     "eslint": "^8.25.0",
@@ -93,3 +95,3 @@ "eslint-config-egg": "^12.1.0",
     "typescript": "^5.0.4",
-    "vitest": "^0.31.1"
+    "vitest": "^0.32.0"
   },
@@ -96,0 +98,0 @@ "engines": {

README.md

@@ -203,6 +203,3 @@ # urllib
 - `timing`: timing object if timing enable.
-- `remoteAddress`: http server ip address
-- `remotePort`: http server ip port
-- `socketHandledRequests`: socket already handled request count
-- `socketHandledResponses`: socket already handled response count
+- `socket`: socket info
 
@@ -209,0 +206,0 @@ ## Run test with debug log

src/cjs/diagnosticsChannel.js

@@ -24,2 +24,21 @@ "use strict";
 //   -> undici:request:trailers => { request, trailers }
+function subscribe(name, listener) {
+    if (typeof node_diagnostics_channel_1.default.subscribe === 'function') {
+        node_diagnostics_channel_1.default.subscribe(name, listener);
+    }
+    else {
+        // TODO: support Node.js 14, will be removed on the next major version
+        node_diagnostics_channel_1.default.channel(name).subscribe(listener);
+    }
+}
+function formatSocket(socket) {
+    if (!socket)
+        return socket;
+    return {
+        localAddress: socket[symbols_1.default.kSocketLocalAddress],
+        localPort: socket[symbols_1.default.kSocketLocalPort],
+        remoteAddress: socket.remoteAddress,
+        remotePort: socket.remotePort,
+    };
+}
 function initDiagnosticsChannel() {
@@ -33,3 +52,3 @@ // makre sure init global DiagnosticsChannel once
     // Note: a request is only loosely completed to a given socket.
-    node_diagnostics_channel_1.default.channel('undici:request:create').subscribe((message, name) => {
+    subscribe('undici:request:create', (message, name) => {
         const { request } = message;
@@ -57,12 +76,16 @@ if (!kHandler) {
     // This message is published after a connection is established.
-    node_diagnostics_channel_1.default.channel('undici:client:connected').subscribe((message, name) => {
+    subscribe('undici:client:connected', (message, name) => {
         const { socket } = message;
         socket[symbols_1.default.kSocketId] = (0, utils_1.globalId)('UndiciSocket');
         socket[symbols_1.default.kSocketStartTime] = node_perf_hooks_1.performance.now();
+        socket[symbols_1.default.kSocketConnectedTime] = new Date();
         socket[symbols_1.default.kHandledRequests] = 0;
         socket[symbols_1.default.kHandledResponses] = 0;
-        debug('[%s] Socket#%d connected', name, socket[symbols_1.default.kSocketId]);
+        // copy local address to symbol, avoid them be reset after request error throw
+        socket[symbols_1.default.kSocketLocalAddress] = socket.localAddress;
+        socket[symbols_1.default.kSocketLocalPort] = socket.localPort;
+        debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols_1.default.kSocketId], formatSocket(socket));
     });
     // This message is published right before the first byte of the request is written to the socket.
-    node_diagnostics_channel_1.default.channel('undici:client:sendHeaders').subscribe((message, name) => {
+    subscribe('undici:client:sendHeaders', (message, name) => {
         const { request, socket } = message;
@@ -77,3 +100,3 @@ if (!kHandler)
         opaque[symbols_1.default.kRequestSocket] = socket;
-        debug('[%s] Request#%d send headers on Socket#%d (handled %d requests)', name, opaque[symbols_1.default.kRequestId], socket[symbols_1.default.kSocketId], socket[symbols_1.default.kHandledRequests]);
+        debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)', name, opaque[symbols_1.default.kRequestId], socket[symbols_1.default.kSocketId], socket[symbols_1.default.kHandledRequests], formatSocket(socket));
         if (!opaque[symbols_1.default.kEnableRequestTiming])
@@ -89,3 +112,3 @@ return;
     });
-    node_diagnostics_channel_1.default.channel('undici:request:bodySent').subscribe((message, name) => {
+    subscribe('undici:request:bodySent', (message, name) => {
         const { request } = message;
@@ -103,3 +126,3 @@ if (!kHandler)
     // This message is published after the response headers have been received, i.e. the response has been completed.
-    node_diagnostics_channel_1.default.channel('undici:request:headers').subscribe((message, name) => {
+    subscribe('undici:request:headers', (message, name) => {
         const { request, response } = message;
@@ -114,3 +137,3 @@ if (!kHandler)
         socket[symbols_1.default.kHandledResponses]++;
-        debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses)', name, opaque[symbols_1.default.kRequestId], response.statusCode, socket[symbols_1.default.kSocketId], socket[symbols_1.default.kHandledResponses]);
+        debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)', name, opaque[symbols_1.default.kRequestId], response.statusCode, socket[symbols_1.default.kSocketId], socket[symbols_1.default.kHandledResponses], formatSocket(socket));
         if (!opaque[symbols_1.default.kEnableRequestTiming])
@@ -121,3 +144,3 @@ return;
     // This message is published after the response body and trailers have been received, i.e. the response has been completed.
-    node_diagnostics_channel_1.default.channel('undici:request:trailers').subscribe((message, name) => {
+    subscribe('undici:request:trailers', (message, name) => {
         const { request } = message;
@@ -134,4 +157,13 @@ if (!kHandler)
     });
-    // diagnosticsChannel.channel('undici:request:error')
+    // This message is published if the request is going to error, but it has not errored yet.
+    // subscribe('undici:request:error', (message, name) => {
+    //   const { request, error } = message as DiagnosticsChannel.RequestErrorMessage;
+    //   const opaque = request[kHandler]?.opts?.opaque;
+    //   if (!opaque || !opaque[symbols.kRequestId]) return;
+    //   const socket = opaque[symbols.kRequestSocket];
+    //   debug('[%s] Request#%d error on Socket#%d (handled %d responses, sock: %o), error: %o',
+    //     name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledResponses],
+    //     formatSocket(socket), error);
+    // });
 }
 exports.initDiagnosticsChannel = initDiagnosticsChannel;

src/cjs/HttpClient.d.ts

@@ -7,4 +7,4 @@ /// <reference types="node" />
 import { CheckAddressFunction } from './HttpAgent';
-import { RequestURL, RequestOptions } from './Request';
-import { HttpClientResponse } from './Response';
+import { RequestURL, RequestOptions, RequestMeta } from './Request';
+import { RawResponseWithMeta, HttpClientResponse } from './Response';
 export type ClientOptions = {
@@ -41,2 +41,10 @@ defaultArgs?: RequestOptions;
 export declare const HEADER_USER_AGENT: string;
+export type RequestDiagnosticsMessage = {
+    request: RequestMeta;
+};
+export type ResponseDiagnosticsMessage = {
+    request: RequestMeta;
+    response: RawResponseWithMeta;
+    error?: Error;
+};
 export declare class HttpClient extends EventEmitter {
@@ -43,0 +51,0 @@ #private;

src/cjs/HttpClient.js

@@ -7,2 +7,3 @@ "use strict";
 exports.HttpClient = exports.HEADER_USER_AGENT = void 0;
+const node_diagnostics_channel_1 = __importDefault(require("node:diagnostics_channel"));
 const node_events_1 = require("node:events");
@@ -73,3 +74,3 @@ const node_http_1 = require("node:http");
 }
-exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.16.1');
+exports.HEADER_USER_AGENT = (0, default_user_agent_1.default)('node-urllib', '3.17.0');
 function getFileName(stream) {
@@ -85,2 +86,6 @@ const filePath = stream.path;
 }
+const channels = {
+    request: node_diagnostics_channel_1.default.channel('urllib:request'),
+    response: node_diagnostics_channel_1.default.channel('urllib:response'),
+};
 class HttpClient extends node_events_1.EventEmitter {
@@ -132,2 +137,3 @@ #defaultArgs;
             retry: 0,
+            timing: true,
             ...this.#defaultArgs,
@@ -143,3 +149,6 @@ ...options,
         };
-        const requestStartTime = node_perf_hooks_1.performance.now();
+        if (!requestContext.requestStartTime) {
+            requestContext.requestStartTime = node_perf_hooks_1.performance.now();
+        }
+        const requestStartTime = requestContext.requestStartTime;
         // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
@@ -392,2 +401,5 @@ const timing = {
             requestOptions.headers = headers;
+            channels.request.publish({
+                request: reqMeta,
+            });
             if (this.listenerCount('request') > 0) {
@@ -511,2 +523,6 @@ this.emit('request', reqMeta);
             }
+            channels.response.publish({
+                request: reqMeta,
+                response: res,
+            });
             if (this.listenerCount('response') > 0) {
@@ -535,2 +551,12 @@ this.emit('response', {
             }
+            else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
+                // auto retry on socket error, https://github.com/node-modules/urllib/issues/454
+                if (args.retry > 0 && requestContext.retries < args.retry) {
+                    if (args.retryDelay) {
+                        await (0, utils_1.sleep)(args.retryDelay);
+                    }
+                    requestContext.retries++;
+                    return await this.#requestInternal(url, options, requestContext);
+                }
+            }
             err.opaque = orginalOpaque;
@@ -540,2 +566,7 @@ err.status = res.status;
             err.res = res;
+            if (err.socket) {
+                // store rawSocket
+                err._rawSocket = err.socket;
+            }
+            err.socket = socketInfo;
             // make sure requestUrls not empty
@@ -547,2 +578,7 @@ if (res.requestUrls.length === 0) {
             this.#updateSocketInfo(socketInfo, internalOpaque);
+            channels.response.publish({
+                request: reqMeta,
+                response: res,
+                error: err,
+            });
             if (this.listenerCount('response') > 0) {
@@ -569,4 +605,4 @@ this.emit('response', {
             socketInfo.handledResponses = socket[symbols_1.default.kHandledResponses];
-            socketInfo.localAddress = socket.localAddress;
-            socketInfo.localPort = socket.localPort;
+            socketInfo.localAddress = socket[symbols_1.default.kSocketLocalAddress];
+            socketInfo.localPort = socket[symbols_1.default.kSocketLocalPort];
             socketInfo.remoteAddress = socket.remoteAddress;
@@ -577,2 +613,5 @@ socketInfo.remotePort = socket.remotePort;
             socketInfo.bytesWritten = socket.bytesWritten;
+            socketInfo.connectedTime = socket[symbols_1.default.kSocketConnectedTime];
+            socketInfo.lastRequestEndTime = socket[symbols_1.default.kSocketRequestEndTime];
+            socket[symbols_1.default.kSocketRequestEndTime] = new Date();
         }
@@ -579,0 +618,0 @@ }

src/cjs/index.d.ts

@@ -5,3 +5,3 @@ import { RequestOptions, RequestURL } from './Request';
 export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
-export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient';
+export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, RequestDiagnosticsMessage, ResponseDiagnosticsMessage, } from './HttpClient';
 export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request';
@@ -8,0 +8,0 @@ export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response';

src/cjs/Request.d.ts

@@ -100,3 +100,3 @@ /// <reference types="node" />
     /**
-     * Enable timing or not, default is false.
+     * Enable timing or not, default is true.
      * */
@@ -136,1 +136,8 @@ timing?: boolean;
 };
+export type RequestMeta = {
+    requestId: number;
+    url: string;
+    args: RequestOptions;
+    ctx?: unknown;
+    retries: number;
+};

src/cjs/Response.d.ts

@@ -16,2 +16,4 @@ /// <reference types="node" />
     handledResponses: number;
+    connectedTime?: Date;
+    lastRequestEndTime?: Date;
 };
@@ -18,0 +20,0 @@ /**

src/cjs/symbols.d.ts

 declare const _default: {
     kSocketId: symbol;
     kSocketStartTime: symbol;
+    kSocketConnectedTime: symbol;
+    kSocketRequestEndTime: symbol;
+    kSocketLocalAddress: symbol;
+    kSocketLocalPort: symbol;
     kHandledRequests: symbol;
@@ -5,0 +9,0 @@ kHandledResponses: symbol;

src/cjs/symbols.js

@@ -6,2 +6,6 @@ "use strict";
     kSocketStartTime: Symbol('socket start time'),
+    kSocketConnectedTime: Symbol('socket connected time'),
+    kSocketRequestEndTime: Symbol('socket request end time'),
+    kSocketLocalAddress: Symbol('socket local address'),
+    kSocketLocalPort: Symbol('socket local port'),
     kHandledRequests: Symbol('handled requests per socket'),
@@ -8,0 +12,0 @@ kHandledResponses: Symbol('handled responses per socket'),

src/diagnosticsChannel.ts

 import diagnosticsChannel from 'node:diagnostics_channel';
 import { performance } from 'node:perf_hooks';
 import { debuglog } from 'node:util';
+import { Socket } from 'node:net';
 import { DiagnosticsChannel } from 'undici';
@@ -20,2 +21,22 @@ import symbols from './symbols';
 //   -> undici:request:trailers => { request, trailers }
+
+function subscribe(name: string, listener: (message: unknown, channelName: string | symbol) => void) {
+  if (typeof diagnosticsChannel.subscribe === 'function') {
+    diagnosticsChannel.subscribe(name, listener);
+  } else {
+    // TODO: support Node.js 14, will be removed on the next major version
+    diagnosticsChannel.channel(name).subscribe(listener);
+  }
+}
+
+function formatSocket(socket: Socket) {
+  if (!socket) return socket;
+  return {
+    localAddress: socket[symbols.kSocketLocalAddress],
+    localPort: socket[symbols.kSocketLocalPort],
+    remoteAddress: socket.remoteAddress,
+    remotePort: socket.remotePort,
+  };
+}
+
 export function initDiagnosticsChannel() {
@@ -29,3 +50,3 @@ // makre sure init global DiagnosticsChannel once
   // Note: a request is only loosely completed to a given socket.
-  diagnosticsChannel.channel('undici:request:create').subscribe((message, name) => {
+  subscribe('undici:request:create', (message, name) => {
     const { request } = message as DiagnosticsChannel.RequestCreateMessage;
@@ -53,13 +74,17 @@ if (!kHandler) {
   // This message is published after a connection is established.
-  diagnosticsChannel.channel('undici:client:connected').subscribe((message, name) => {
+  subscribe('undici:client:connected', (message, name) => {
     const { socket } = message as DiagnosticsChannel.ClientConnectedMessage;
     socket[symbols.kSocketId] = globalId('UndiciSocket');
     socket[symbols.kSocketStartTime] = performance.now();
+    socket[symbols.kSocketConnectedTime] = new Date();
     socket[symbols.kHandledRequests] = 0;
     socket[symbols.kHandledResponses] = 0;
-    debug('[%s] Socket#%d connected', name, socket[symbols.kSocketId]);
+    // copy local address to symbol, avoid them be reset after request error throw
+    socket[symbols.kSocketLocalAddress] = socket.localAddress;
+    socket[symbols.kSocketLocalPort] = socket.localPort;
+    debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols.kSocketId], formatSocket(socket));
   });
 
   // This message is published right before the first byte of the request is written to the socket.
-  diagnosticsChannel.channel('undici:client:sendHeaders').subscribe((message, name) => {
+  subscribe('undici:client:sendHeaders', (message, name) => {
     const { request, socket } = message as DiagnosticsChannel.ClientSendHeadersMessage;
@@ -73,4 +98,5 @@ if (!kHandler) return;
     opaque[symbols.kRequestSocket] = socket;
-    debug('[%s] Request#%d send headers on Socket#%d (handled %d requests)',
-      name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledRequests]);
+    debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)',
+      name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledRequests],
+      formatSocket(socket));
 
@@ -87,3 +113,3 @@ if (!opaque[symbols.kEnableRequestTiming]) return;
 
-  diagnosticsChannel.channel('undici:request:bodySent').subscribe((message, name) => {
+  subscribe('undici:request:bodySent', (message, name) => {
     const { request } = message as DiagnosticsChannel.RequestBodySentMessage;
@@ -100,3 +126,3 @@ if (!kHandler) return;
   // This message is published after the response headers have been received, i.e. the response has been completed.
-  diagnosticsChannel.channel('undici:request:headers').subscribe((message, name) => {
+  subscribe('undici:request:headers', (message, name) => {
     const { request, response } = message as DiagnosticsChannel.RequestHeadersMessage;
@@ -110,4 +136,5 @@ if (!kHandler) return;
     socket[symbols.kHandledResponses]++;
-    debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses)',
-      name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses]);
+    debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)',
+      name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses],
+      formatSocket(socket));
 
@@ -119,3 +146,3 @@ if (!opaque[symbols.kEnableRequestTiming]) return;
   // This message is published after the response body and trailers have been received, i.e. the response has been completed.
-  diagnosticsChannel.channel('undici:request:trailers').subscribe((message, name) => {
+  subscribe('undici:request:trailers', (message, name) => {
     const { request } = message as DiagnosticsChannel.RequestTrailersMessage;
@@ -131,3 +158,13 @@ if (!kHandler) return;
   });
-  // diagnosticsChannel.channel('undici:request:error')
+
+  // This message is published if the request is going to error, but it has not errored yet.
+  // subscribe('undici:request:error', (message, name) => {
+  //   const { request, error } = message as DiagnosticsChannel.RequestErrorMessage;
+  //   const opaque = request[kHandler]?.opts?.opaque;
+  //   if (!opaque || !opaque[symbols.kRequestId]) return;
+  //   const socket = opaque[symbols.kRequestSocket];
+  //   debug('[%s] Request#%d error on Socket#%d (handled %d responses, sock: %o), error: %o',
+  //     name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledResponses],
+  //     formatSocket(socket), error);
+  // });
 }

src/esm/diagnosticsChannel.js

@@ -18,2 +18,21 @@ import diagnosticsChannel from 'node:diagnostics_channel';
 //   -> undici:request:trailers => { request, trailers }
+function subscribe(name, listener) {
+    if (typeof diagnosticsChannel.subscribe === 'function') {
+        diagnosticsChannel.subscribe(name, listener);
+    }
+    else {
+        // TODO: support Node.js 14, will be removed on the next major version
+        diagnosticsChannel.channel(name).subscribe(listener);
+    }
+}
+function formatSocket(socket) {
+    if (!socket)
+        return socket;
+    return {
+        localAddress: socket[symbols.kSocketLocalAddress],
+        localPort: socket[symbols.kSocketLocalPort],
+        remoteAddress: socket.remoteAddress,
+        remotePort: socket.remotePort,
+    };
+}
 export function initDiagnosticsChannel() {
@@ -27,3 +46,3 @@ // makre sure init global DiagnosticsChannel once
     // Note: a request is only loosely completed to a given socket.
-    diagnosticsChannel.channel('undici:request:create').subscribe((message, name) => {
+    subscribe('undici:request:create', (message, name) => {
         const { request } = message;
@@ -51,12 +70,16 @@ if (!kHandler) {
     // This message is published after a connection is established.
-    diagnosticsChannel.channel('undici:client:connected').subscribe((message, name) => {
+    subscribe('undici:client:connected', (message, name) => {
         const { socket } = message;
         socket[symbols.kSocketId] = globalId('UndiciSocket');
         socket[symbols.kSocketStartTime] = performance.now();
+        socket[symbols.kSocketConnectedTime] = new Date();
         socket[symbols.kHandledRequests] = 0;
         socket[symbols.kHandledResponses] = 0;
-        debug('[%s] Socket#%d connected', name, socket[symbols.kSocketId]);
+        // copy local address to symbol, avoid them be reset after request error throw
+        socket[symbols.kSocketLocalAddress] = socket.localAddress;
+        socket[symbols.kSocketLocalPort] = socket.localPort;
+        debug('[%s] Socket#%d connected (sock: %o)', name, socket[symbols.kSocketId], formatSocket(socket));
     });
     // This message is published right before the first byte of the request is written to the socket.
-    diagnosticsChannel.channel('undici:client:sendHeaders').subscribe((message, name) => {
+    subscribe('undici:client:sendHeaders', (message, name) => {
         const { request, socket } = message;
@@ -71,3 +94,3 @@ if (!kHandler)
         opaque[symbols.kRequestSocket] = socket;
-        debug('[%s] Request#%d send headers on Socket#%d (handled %d requests)', name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledRequests]);
+        debug('[%s] Request#%d send headers on Socket#%d (handled %d requests, sock: %o)', name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledRequests], formatSocket(socket));
         if (!opaque[symbols.kEnableRequestTiming])
@@ -83,3 +106,3 @@ return;
     });
-    diagnosticsChannel.channel('undici:request:bodySent').subscribe((message, name) => {
+    subscribe('undici:request:bodySent', (message, name) => {
         const { request } = message;
@@ -97,3 +120,3 @@ if (!kHandler)
     // This message is published after the response headers have been received, i.e. the response has been completed.
-    diagnosticsChannel.channel('undici:request:headers').subscribe((message, name) => {
+    subscribe('undici:request:headers', (message, name) => {
         const { request, response } = message;
@@ -108,3 +131,3 @@ if (!kHandler)
         socket[symbols.kHandledResponses]++;
-        debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses)', name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses]);
+        debug('[%s] Request#%d get %s response headers on Socket#%d (handled %d responses, sock: %o)', name, opaque[symbols.kRequestId], response.statusCode, socket[symbols.kSocketId], socket[symbols.kHandledResponses], formatSocket(socket));
         if (!opaque[symbols.kEnableRequestTiming])
@@ -115,3 +138,3 @@ return;
     // This message is published after the response body and trailers have been received, i.e. the response has been completed.
-    diagnosticsChannel.channel('undici:request:trailers').subscribe((message, name) => {
+    subscribe('undici:request:trailers', (message, name) => {
         const { request } = message;
@@ -128,3 +151,12 @@ if (!kHandler)
     });
-    // diagnosticsChannel.channel('undici:request:error')
+    // This message is published if the request is going to error, but it has not errored yet.
+    // subscribe('undici:request:error', (message, name) => {
+    //   const { request, error } = message as DiagnosticsChannel.RequestErrorMessage;
+    //   const opaque = request[kHandler]?.opts?.opaque;
+    //   if (!opaque || !opaque[symbols.kRequestId]) return;
+    //   const socket = opaque[symbols.kRequestSocket];
+    //   debug('[%s] Request#%d error on Socket#%d (handled %d responses, sock: %o), error: %o',
+    //     name, opaque[symbols.kRequestId], socket[symbols.kSocketId], socket[symbols.kHandledResponses],
+    //     formatSocket(socket), error);
+    // });
 }

src/esm/HttpClient.d.ts

@@ -7,4 +7,4 @@ /// <reference types="node" />
 import { CheckAddressFunction } from './HttpAgent';
-import { RequestURL, RequestOptions } from './Request';
-import { HttpClientResponse } from './Response';
+import { RequestURL, RequestOptions, RequestMeta } from './Request';
+import { RawResponseWithMeta, HttpClientResponse } from './Response';
 export type ClientOptions = {
@@ -41,2 +41,10 @@ defaultArgs?: RequestOptions;
 export declare const HEADER_USER_AGENT: string;
+export type RequestDiagnosticsMessage = {
+    request: RequestMeta;
+};
+export type ResponseDiagnosticsMessage = {
+    request: RequestMeta;
+    response: RawResponseWithMeta;
+    error?: Error;
+};
 export declare class HttpClient extends EventEmitter {
@@ -43,0 +51,0 @@ #private;

src/esm/HttpClient.js

@@ -0,1 +1,2 @@
+import diagnosticsChannel from 'node:diagnostics_channel';
 import { EventEmitter } from 'node:events';
@@ -66,3 +67,3 @@ import { STATUS_CODES } from 'node:http';
 }
-export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.16.1');
+export const HEADER_USER_AGENT = createUserAgent('node-urllib', '3.17.0');
 function getFileName(stream) {
@@ -78,2 +79,6 @@ const filePath = stream.path;
 }
+const channels = {
+    request: diagnosticsChannel.channel('urllib:request'),
+    response: diagnosticsChannel.channel('urllib:response'),
+};
 export class HttpClient extends EventEmitter {
@@ -125,2 +130,3 @@ #defaultArgs;
             retry: 0,
+            timing: true,
             ...this.#defaultArgs,
@@ -136,3 +142,6 @@ ...options,
         };
-        const requestStartTime = performance.now();
+        if (!requestContext.requestStartTime) {
+            requestContext.requestStartTime = performance.now();
+        }
+        const requestStartTime = requestContext.requestStartTime;
         // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
@@ -385,2 +394,5 @@ const timing = {
             requestOptions.headers = headers;
+            channels.request.publish({
+                request: reqMeta,
+            });
             if (this.listenerCount('request') > 0) {
@@ -504,2 +516,6 @@ this.emit('request', reqMeta);
             }
+            channels.response.publish({
+                request: reqMeta,
+                response: res,
+            });
             if (this.listenerCount('response') > 0) {
@@ -528,2 +544,12 @@ this.emit('response', {
             }
+            else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
+                // auto retry on socket error, https://github.com/node-modules/urllib/issues/454
+                if (args.retry > 0 && requestContext.retries < args.retry) {
+                    if (args.retryDelay) {
+                        await sleep(args.retryDelay);
+                    }
+                    requestContext.retries++;
+                    return await this.#requestInternal(url, options, requestContext);
+                }
+            }
             err.opaque = orginalOpaque;
@@ -533,2 +559,7 @@ err.status = res.status;
             err.res = res;
+            if (err.socket) {
+                // store rawSocket
+                err._rawSocket = err.socket;
+            }
+            err.socket = socketInfo;
             // make sure requestUrls not empty
@@ -540,2 +571,7 @@ if (res.requestUrls.length === 0) {
             this.#updateSocketInfo(socketInfo, internalOpaque);
+            channels.response.publish({
+                request: reqMeta,
+                response: res,
+                error: err,
+            });
             if (this.listenerCount('response') > 0) {
@@ -562,4 +598,4 @@ this.emit('response', {
             socketInfo.handledResponses = socket[symbols.kHandledResponses];
-            socketInfo.localAddress = socket.localAddress;
-            socketInfo.localPort = socket.localPort;
+            socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
+            socketInfo.localPort = socket[symbols.kSocketLocalPort];
             socketInfo.remoteAddress = socket.remoteAddress;
@@ -570,4 +606,7 @@ socketInfo.remotePort = socket.remotePort;
             socketInfo.bytesWritten = socket.bytesWritten;
+            socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
+            socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
+            socket[symbols.kSocketRequestEndTime] = new Date();
         }
     }
 }

src/esm/index.d.ts

@@ -5,3 +5,3 @@ import { RequestOptions, RequestURL } from './Request';
 export { MockAgent, ProxyAgent, Agent, Dispatcher, setGlobalDispatcher, getGlobalDispatcher, } from 'undici';
-export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient';
+export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, RequestDiagnosticsMessage, ResponseDiagnosticsMessage, } from './HttpClient';
 export { RequestOptions, RequestOptions as RequestOptions2, RequestURL, HttpMethod, FixJSONCtlCharsHandler, FixJSONCtlChars, } from './Request';
@@ -8,0 +8,0 @@ export { SocketInfo, Timing, RawResponseWithMeta, HttpClientResponse } from './Response';

src/esm/index.js

@@ -31,3 +31,3 @@ import LRU from 'ylru';
 // HttpClient2 is keep compatible with urlib@2 HttpClient2
-export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient.js';
+export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT, } from './HttpClient.js';
 export default {
@@ -34,0 +34,0 @@ request,

src/esm/Request.d.ts

@@ -100,3 +100,3 @@ /// <reference types="node" />
     /**
-     * Enable timing or not, default is false.
+     * Enable timing or not, default is true.
      * */
@@ -136,1 +136,8 @@ timing?: boolean;
 };
+export type RequestMeta = {
+    requestId: number;
+    url: string;
+    args: RequestOptions;
+    ctx?: unknown;
+    retries: number;
+};

src/esm/Response.d.ts

@@ -16,2 +16,4 @@ /// <reference types="node" />
     handledResponses: number;
+    connectedTime?: Date;
+    lastRequestEndTime?: Date;
 };
@@ -18,0 +20,0 @@ /**

src/esm/symbols.d.ts

 declare const _default: {
     kSocketId: symbol;
     kSocketStartTime: symbol;
+    kSocketConnectedTime: symbol;
+    kSocketRequestEndTime: symbol;
+    kSocketLocalAddress: symbol;
+    kSocketLocalPort: symbol;
     kHandledRequests: symbol;
@@ -5,0 +9,0 @@ kHandledResponses: symbol;

src/esm/symbols.js

 export default {
     kSocketId: Symbol('socket id'),
     kSocketStartTime: Symbol('socket start time'),
+    kSocketConnectedTime: Symbol('socket connected time'),
+    kSocketRequestEndTime: Symbol('socket request end time'),
+    kSocketLocalAddress: Symbol('socket local address'),
+    kSocketLocalPort: Symbol('socket local port'),
     kHandledRequests: Symbol('handled requests per socket'),
@@ -5,0 +9,0 @@ kHandledResponses: Symbol('handled responses per socket'),

src/HttpClient.ts

@@ -0,1 +1,2 @@
+import diagnosticsChannel from 'node:diagnostics_channel';
 import { EventEmitter } from 'node:events';
@@ -32,3 +33,3 @@ import { LookupFunction } from 'node:net';
 import { HttpAgent, CheckAddressFunction } from './HttpAgent';
-import { RequestURL, RequestOptions, HttpMethod } from './Request';
+import { RequestURL, RequestOptions, HttpMethod, RequestMeta } from './Request';
 import { RawResponseWithMeta, HttpClientResponse, SocketInfo } from './Response';
@@ -141,4 +142,21 @@ import { parseJSON, sleep, digestAuthHeader, globalId, performanceTime, isReadable } from './utils';
   retries: number;
+  requestStartTime?: number;
 };
 
+const channels = {
+  request: diagnosticsChannel.channel('urllib:request'),
+  response: diagnosticsChannel.channel('urllib:response'),
+};
+
+export type RequestDiagnosticsMessage = {
+  request: RequestMeta;
+};
+
+export type ResponseDiagnosticsMessage = {
+  request: RequestMeta;
+  response: RawResponseWithMeta;
+  error?: Error;
+};
+
+
 export class HttpClient extends EventEmitter {
@@ -193,2 +211,3 @@ #defaultArgs?: RequestOptions;
       retry: 0,
+      timing: true,
       ...this.#defaultArgs,
@@ -204,3 +223,6 @@ ...options,
     };
-    const requestStartTime = performance.now();
+    if (!requestContext.requestStartTime) {
+      requestContext.requestStartTime = performance.now();
+    }
+    const requestStartTime = requestContext.requestStartTime;
 
@@ -239,4 +261,4 @@ // https://developer.chrome.com/docs/devtools/network/reference/?utm_source=devtools#timing-explanation
       retries: requestContext.retries,
-    };
-    const socketInfo = {
+    } as RequestMeta;
+    const socketInfo: SocketInfo = {
       id: 0,
@@ -446,2 +468,5 @@ localAddress: '',
       requestOptions.headers = headers;
+      channels.request.publish({
+        request: reqMeta,
+      } as RequestDiagnosticsMessage);
       if (this.listenerCount('request') > 0) {
@@ -565,2 +590,6 @@ this.emit('request', reqMeta);
 
+      channels.response.publish({
+        request: reqMeta,
+        response: res,
+      } as ResponseDiagnosticsMessage);
       if (this.listenerCount('response') > 0) {
@@ -587,2 +616,11 @@ this.emit('response', {
         err = new HttpClientRequestTimeoutError(bodyTimeout, { cause: e });
+      } else if (err.code === 'UND_ERR_SOCKET' || err.code === 'ECONNRESET') {
+        // auto retry on socket error, https://github.com/node-modules/urllib/issues/454
+        if (args.retry > 0 && requestContext.retries < args.retry) {
+          if (args.retryDelay) {
+            await sleep(args.retryDelay);
+          }
+          requestContext.retries++;
+          return await this.#requestInternal(url, options, requestContext);
+        }
       }
@@ -593,2 +631,7 @@ err.opaque = orginalOpaque;
       err.res = res;
+      if (err.socket) {
+        // store rawSocket
+        err._rawSocket = err.socket;
+      }
+      err.socket = socketInfo;
       // make sure requestUrls not empty
@@ -601,2 +644,7 @@ if (res.requestUrls.length === 0) {
 
+      channels.response.publish({
+        request: reqMeta,
+        response: res,
+        error: err,
+      } as ResponseDiagnosticsMessage);
       if (this.listenerCount('response') > 0) {
@@ -624,4 +672,4 @@ this.emit('response', {
       socketInfo.handledResponses = socket[symbols.kHandledResponses];
-      socketInfo.localAddress = socket.localAddress;
-      socketInfo.localPort = socket.localPort;
+      socketInfo.localAddress = socket[symbols.kSocketLocalAddress];
+      socketInfo.localPort = socket[symbols.kSocketLocalPort];
       socketInfo.remoteAddress = socket.remoteAddress;
@@ -632,4 +680,7 @@ socketInfo.remotePort = socket.remotePort;
       socketInfo.bytesWritten = socket.bytesWritten;
+      socketInfo.connectedTime = socket[symbols.kSocketConnectedTime];
+      socketInfo.lastRequestEndTime = socket[symbols.kSocketRequestEndTime];
+      socket[symbols.kSocketRequestEndTime] = new Date();
     }
   }
 }

src/index.ts

@@ -39,3 +39,6 @@ import LRU from 'ylru';
 // HttpClient2 is keep compatible with urlib@2 HttpClient2
-export { HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT } from './HttpClient';
+export {
+  HttpClient, HttpClient as HttpClient2, HEADER_USER_AGENT as USER_AGENT,
+  RequestDiagnosticsMessage, ResponseDiagnosticsMessage,
+} from './HttpClient';
 // RequestOptions2 is keep compatible with urlib@2 RequestOptions2
@@ -42,0 +45,0 @@ export {

src/Request.ts

@@ -102,3 +102,3 @@ import { Readable, Writable } from 'node:stream';
   /**
-   * Enable timing or not, default is false.
+   * Enable timing or not, default is true.
    * */
@@ -138,1 +138,9 @@ timing?: boolean;
 };
+
+export type RequestMeta = {
+  requestId: number;
+  url: string;
+  args: RequestOptions;
+  ctx?: unknown;
+  retries: number;
+};

src/Response.ts

@@ -15,2 +15,4 @@ import { Readable } from 'node:stream';
   handledResponses: number;
+  connectedTime?: Date;
+  lastRequestEndTime?: Date;
 };
@@ -17,0 +19,0 @@

src/symbols.ts

 export default {
   kSocketId: Symbol('socket id'),
   kSocketStartTime: Symbol('socket start time'),
+  kSocketConnectedTime: Symbol('socket connected time'),
+  kSocketRequestEndTime: Symbol('socket request end time'),
+  kSocketLocalAddress: Symbol('socket local address'),
+  kSocketLocalPort: Symbol('socket local port'),
   kHandledRequests: Symbol('handled requests per socket'),
@@ -5,0 +9,0 @@ kHandledResponses: Symbol('handled responses per socket'),

New alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Fixed alerts

License Policy Violation

License

This package is not allowed per your license policy. Review the package's license to ensure compliance.

Found 1 instance in 1 package

Improved metrics

Total package byte prevSize

184313

increased by6.78%

Lines of code

3962

increased by7.84%

Worsened metrics

Dev dependency count

19

increased by5.56%

Number of lines in readme file

306

decreased by-0.97%

No dependency changes