tokio - cargo Package Compare versions

Comparing version 1.52.4 to 1.53.0

src/io/uring/rename.rs

+use super::utils::cstr;
+
+use crate::runtime::driver::op::{CancelData, Cancellable, Completable, CqeResult, Op};
+
+use io_uring::{opcode, types};
+use std::ffi::CString;
+use std::io;
+use std::path::Path;
+
+#[derive(Debug)]
+pub(crate) struct Rename {
+    /// This field will be read by the kernel during the operation, so we
+    /// need to ensure it is valid for the entire duration of the operation.
+    _from: CString,
+    _to: CString,
+}
+
+impl Completable for Rename {
+    type Output = io::Result<()>;
+
+    fn complete(self, cqe: CqeResult) -> Self::Output {
+        cqe.result.map(drop)
+    }
+
+    fn complete_with_error(self, error: io::Error) -> Self::Output {
+        Err(error)
+    }
+}
+
+impl Cancellable for Rename {
+    fn cancel(self) -> CancelData {
+        CancelData::Rename(self)
+    }
+}
+
+impl Op<Rename> {
+    pub(crate) fn rename(from: &Path, to: &Path) -> io::Result<Op<Rename>> {
+        let from = cstr(from)?;
+        let to = cstr(to)?;
+
+        let rename_op = opcode::RenameAt::new(
+            types::Fd(libc::AT_FDCWD),
+            from.as_ptr(),
+            types::Fd(libc::AT_FDCWD),
+            to.as_ptr(),
+        )
+        .build();
+
+        // SAFETY: Parameters are valid for the entire duration of the operation
+        Ok(unsafe {
+            Op::new(
+                rename_op,
+                Rename {
+                    _from: from,
+                    _to: to,
+                },
+            )
+        })
+    }
+}

src/io/uring/statx.rs

+#![cfg(all(
+    tokio_unstable,
+    feature = "io-uring",
+    feature = "rt",
+    feature = "fs",
+    // libc::statx is only supported on these platforms
+    // FIXME: Add musl target env when our minimum supported
+    // rust version is 1.93. To clarify, statx support is
+    // introduced to musl in 1.25 as mentioned officially here:
+    // https://musl.libc.org/releases.html.
+    // However, rustup target_env building for *-linux-musl
+    // uses 1.25 musl on all *-linux-musl platforms starting
+    // in 1.93 stable rust version.
+    // https://blog.rust-lang.org/2025/12/05/Updating-musl-1.2.5/
+    any(target_env = "gnu", target_os = "android")
+))]
+
+use crate::fs::File;
+use crate::io::uring::utils::cstr;
+use crate::runtime::driver::op::{CancelData, Cancellable, Completable, CqeResult, Op};
+use io_uring::{opcode, types};
+use libc::statx;
+use std::ffi::{CStr, CString};
+use std::fmt::{Debug, Formatter};
+use std::io;
+use std::mem::MaybeUninit;
+use std::os::fd::AsRawFd;
+use std::path::Path;
+
+pub(crate) struct Metadata(statx);
+
+impl Metadata {
+    /// Returns the size of the file, in bytes, this metadata is for.
+    pub(crate) fn len(&self) -> u64 {
+        self.0.stx_size
+    }
+}
+
+impl Debug for Metadata {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("Metadata");
+        debug.field("len", &self.len());
+        debug.finish_non_exhaustive()
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct Statx {
+    /// This field will be read by the kernel during the operation, so we
+    /// need to ensure it is valid for the entire duration of the operation.
+    _path: CString,
+    buffer: Box<MaybeUninit<statx>>,
+}
+
+impl Completable for Statx {
+    type Output = io::Result<Metadata>;
+
+    fn complete(self, cqe: CqeResult) -> Self::Output {
+        // SAFETY: On success, we always receive 0, which should guarantee
+        // that the information about a file is stored inside the
+        // statx buffer. On failure, we'll receive an Error value.
+        // Refer to man page description and return value:
+        // https://man7.org/linux/man-pages/man2/statx.2.html
+        cqe.result
+            .map(|_| Metadata(unsafe { *self.buffer.as_ptr() }))
+    }
+
+    fn complete_with_error(self, error: io::Error) -> Self::Output {
+        Err(error)
+    }
+}
+
+impl Cancellable for Statx {
+    fn cancel(self) -> CancelData {
+        CancelData::Statx(self)
+    }
+}
+
+impl Op<Statx> {
+    /// Submit a request to retrieve a file's status.
+    #[inline]
+    fn statx(path: &Path, flags: i32) -> io::Result<Op<Statx>> {
+        let path = cstr(path)?;
+        let mut buffer = Box::new(MaybeUninit::<statx>::uninit());
+
+        let statx_op = opcode::Statx::new(
+            types::Fd(libc::AT_FDCWD),
+            path.as_ptr(),
+            buffer.as_mut_ptr().cast(),
+        )
+        .flags(flags)
+        .mask(libc::STATX_BASIC_STATS | libc::STATX_BTIME)
+        .build();
+
+        // SAFETY: Parameters are valid for the entire duration of the operation
+        Ok(unsafe {
+            Op::new(
+                statx_op,
+                Statx {
+                    _path: path,
+                    buffer,
+                },
+            )
+        })
+    }
+
+    /// Retrieves the metadata information of the given path, following symlinks
+    /// if the path provided points to a symlink location.
+    #[inline]
+    pub(crate) fn metadata(path: &Path) -> io::Result<Op<Statx>> {
+        Op::statx(path, libc::AT_STATX_SYNC_AS_STAT)
+    }
+
+    /// Retrieves the metadata information of the given file
+    pub(crate) fn file_metadata(file: &File) -> io::Result<Op<Statx>> {
+        let mut buffer = Box::new(MaybeUninit::<statx>::uninit());
+        let empty_path: &'static CStr = c"";
+
+        // io-uring was introduced in linux 5.1
+        // pass in an empty path instead of null to target the file descriptor
+        // status as specified by man:
+        // https://man7.org/linux/man-pages/man2/statx.2.html
+        let statx_op = opcode::Statx::new(
+            types::Fd(file.as_raw_fd()),
+            // it should be fine to pass in `empty_path` whose lifetime
+            // does not exceed the `file_metadata()` function as a ptr here
+            // because we want to stat the dirfd not this pathname
+            empty_path.as_ptr(),
+            buffer.as_mut_ptr().cast(),
+        )
+        .flags(libc::AT_STATX_SYNC_AS_STAT | libc::AT_EMPTY_PATH)
+        .mask(libc::STATX_BASIC_STATS | libc::STATX_BTIME)
+        .build();
+
+        // SAFETY: Parameters are valid for the entire duration of the operation
+        Ok(unsafe {
+            Op::new(
+                statx_op,
+                Statx {
+                    _path: empty_path.into(),
+                    buffer,
+                },
+            )
+        })
+    }
+
+    // TODO: Once `Metadata::from_statx` is stabilized, we can use use this function
+    // to enable io-uring support on `tokio::fs::symlink_metadata`.
+    // See this PR for more detail: https://github.com/tokio-rs/tokio/pull/8080
+    // See `Metadata::from_statx` tracking issue to see progress:
+    // https://github.com/rust-lang/rust/issues/156268
+    /// Retrieves the metadata information of the given path without following symlinks.
+    #[inline]
+    #[allow(dead_code)]
+    pub(crate) fn symlink_metadata(path: &Path) -> io::Result<Op<Statx>> {
+        Op::statx(
+            path,
+            libc::AT_STATX_SYNC_AS_STAT | libc::AT_SYMLINK_NOFOLLOW,
+        )
+    }
+}

src/runtime/metrics/schedule_latency_mock.rs

+//! A mock implementation of the types in `schedule_latency`. These types
+//! are zero-sized so that the size of types using them are not increased
+//! unless schedule latency tracking is explicitly enabled.
+
+use std::time::Instant;
+
+#[derive(Copy, Clone)]
+pub(crate) struct ScheduleLatencyInstant();
+
+impl ScheduleLatencyInstant {
+    pub(crate) fn new(_runtime_start: Option<Instant>) -> Self {
+        Self()
+    }
+
+    pub(crate) fn prepare(self, _runtime_start: Option<Instant>) -> Option<ScheduleLatencyContext> {
+        None
+    }
+}
+
+pub(crate) struct ScheduleLatencyContext {
+    _private: (),
+}
+
+impl ScheduleLatencyContext {
+    // This method is referenced (but never called) when the `schedule-latency`
+    // feature is disabled and `tokio_unstable` is enabled.
+    #[allow(dead_code)]
+    pub(crate) fn elapsed_nanos(&self, _now: Instant) -> u64 {
+        unimplemented!("This should never be called because prepare() always returns None")
+    }
+}

src/runtime/metrics/schedule_latency.rs

+use std::num::NonZeroU64;
+use std::time::Instant;
+
+/// `ScheduleLatencyInstant` tracks the time a task was scheduled.
+///
+/// The time a task was scheduled is stored as the number of nanoseconds
+/// since startup of the task's scheduler.
+///
+/// **Note**: This is an [unstable API][unstable]. The public API of this type
+/// may break in 1.x releases. See [the documentation on unstable
+/// features][unstable] for details.
+///
+/// [unstable]: crate#unstable-features
+#[derive(Copy, Clone)]
+pub(crate) struct ScheduleLatencyInstant(Option<NonZeroU64>);
+
+impl ScheduleLatencyInstant {
+    /// Create a new `ScheduleLatencyInstant` using the provided scheduler startup Instant.
+    pub(crate) fn new(scheduler_start: Option<Instant>) -> Self {
+        Self(scheduler_start.map(|scheduler_start| {
+            NonZeroU64::new(scheduler_start.elapsed().as_nanos() as u64).unwrap_or(NonZeroU64::MIN)
+        }))
+    }
+
+    /// Prepare a context that can calculate the number of nanoseconds elapsed
+    /// since this task was scheduled.
+    pub(crate) fn prepare(
+        self,
+        scheduler_start: Option<Instant>,
+    ) -> Option<ScheduleLatencyContext> {
+        match (scheduler_start, self.0) {
+            (Some(scheduler_start), Some(scheduled_at_delta)) => Some(ScheduleLatencyContext {
+                scheduler_start,
+                scheduled_at_delta,
+            }),
+            _ => None,
+        }
+    }
+}
+
+/// `ScheduleLatencyContext` contains all the data required to calculate the time elapsed
+/// since a task was scheduled.
+///
+/// `ScheduleLatencyInstant` on its own in insufficient because it only contains a delta.
+/// The scheduler startup time is required to convert the delta back into an actual time
+/// but is omitted from `ScheduleLatencyInstant` to keep its memory size minimal.
+pub(crate) struct ScheduleLatencyContext {
+    scheduler_start: Instant,
+    scheduled_at_delta: NonZeroU64,
+}
+
+impl ScheduleLatencyContext {
+    /// Calculate how many nanoseconds have elapsed between `now` and when this task
+    /// was last scheduled.
+    pub(crate) fn elapsed_nanos(&self, now: Instant) -> u64 {
+        let nanos_since_start = now
+            .saturating_duration_since(self.scheduler_start)
+            .as_nanos() as u64;
+        nanos_since_start.saturating_sub(self.scheduled_at_delta.get())
+    }
+}

tests/fs_uring_rename.rs

+//! Uring file operations tests.
+
+#![cfg(all(
+    tokio_unstable,
+    feature = "io-uring",
+    feature = "rt",
+    feature = "fs",
+    target_os = "linux"
+))]
+
+use futures::future::Future;
+use futures::future::FutureExt;
+use libc::PATH_MAX;
+use std::future::poll_fn;
+use std::io::Write;
+use std::path::PathBuf;
+use std::sync::mpsc;
+use std::task::Poll;
+use std::time::Duration;
+use tempfile::{tempdir, NamedTempFile};
+use tokio::fs::{rename, try_exists};
+use tokio::runtime::{Builder, Runtime};
+use tokio_test::assert_pending;
+use tokio_util::task::TaskTracker;
+
+use crate::support::io_uring::io_uring_supported;
+
+mod support {
+    pub(crate) mod io_uring;
+}
+
+fn multi_rt(n: usize) -> Box<dyn Fn() -> Runtime> {
+    Box::new(move || {
+        Builder::new_multi_thread()
+            .worker_threads(n)
+            .enable_all()
+            .build()
+            .unwrap()
+    })
+}
+
+fn current_rt() -> Box<dyn Fn() -> Runtime> {
+    Box::new(|| Builder::new_current_thread().enable_all().build().unwrap())
+}
+
+fn rt_combinations() -> Vec<Box<dyn Fn() -> Runtime>> {
+    vec![
+        current_rt(),
+        multi_rt(1),
+        multi_rt(2),
+        multi_rt(8),
+        multi_rt(64),
+        multi_rt(256),
+    ]
+}
+
+#[test]
+fn shutdown_runtime_while_performing_io_uring_ops() {
+    if !io_uring_supported() {
+        return;
+    }
+
+    fn run(rt: Runtime) {
+        let (done_tx, done_rx) = mpsc::channel();
+        let (_tmp, path) = create_tmp_files(1);
+        // keep 100 permits
+        const N: i32 = 100;
+        rt.spawn(async move {
+            let path = path[0].clone();
+            let dst = path.with_extension("renamed");
+
+            // spawning a bunch of uring operations.
+            let mut futs = vec![];
+
+            // spawning a bunch of uring operations.
+            for _ in 0..N {
+                let mut fut = Box::pin(rename(path.clone(), dst.clone()));
+
+                poll_fn(|cx| {
+                    assert_pending!(fut.as_mut().poll(cx));
+                    Poll::<()>::Pending
+                })
+                .await;
+
+                futs.push(fut);
+            }
+
+            tokio::task::yield_now().await;
+        });
+
+        std::thread::spawn(move || {
+            rt.shutdown_timeout(Duration::from_millis(300));
+            done_tx.send(()).unwrap();
+        });
+
+        done_rx.recv().unwrap();
+    }
+
+    for rt in rt_combinations() {
+        run(rt());
+    }
+}
+
+#[test]
+fn rename_many_files() {
+    fn run(rt: Runtime) {
+        const NUM_FILES: usize = 512;
+
+        let dir = tempdir().unwrap();
+
+        rt.block_on(async move {
+            let tracker = TaskTracker::new();
+
+            for i in 0..NUM_FILES {
+                let src = dir.path().join(format!("src-{i}"));
+                let dst = dir.path().join(format!("dst-{i}"));
+                tokio::fs::write(&src, b"contents").await.unwrap();
+
+                tracker.spawn(async move {
+                    rename(&src, &dst).await.unwrap();
+
+                    assert!(!try_exists(&src).await.unwrap());
+                    assert!(try_exists(&dst).await.unwrap());
+                });
+            }
+            tracker.close();
+            tracker.wait().await;
+        });
+    }
+
+    for rt in rt_combinations() {
+        run(rt());
+    }
+}
+
+#[tokio::test]
+async fn rename_file() {
+    let dir = tempdir().unwrap();
+    let src = dir.path().join("src.txt");
+    let dst = dir.path().join("dst.txt");
+
+    tokio::fs::write(&src, b"Hello File!").await.unwrap();
+
+    rename(&src, &dst).await.unwrap();
+
+    assert!(!try_exists(&src).await.unwrap());
+    assert!(try_exists(&dst).await.unwrap());
+    assert_eq!(tokio::fs::read(&dst).await.unwrap(), b"Hello File!");
+}
+
+#[tokio::test]
+async fn rename_replaces_destination() {
+    let dir = tempdir().unwrap();
+    let src = dir.path().join("src.txt");
+    let dst = dir.path().join("dst.txt");
+
+    tokio::fs::write(&src, b"source").await.unwrap();
+    tokio::fs::write(&dst, b"destination").await.unwrap();
+
+    rename(&src, &dst).await.unwrap();
+
+    assert!(!try_exists(&src).await.unwrap());
+    assert_eq!(tokio::fs::read(&dst).await.unwrap(), b"source");
+}
+
+#[tokio::test]
+async fn rename_directory() {
+    let dir = tempdir().unwrap();
+    let src = dir.path().join("src_dir");
+    let dst = dir.path().join("dst_dir");
+
+    tokio::fs::create_dir(&src).await.unwrap();
+    tokio::fs::write(src.join("file.txt"), b"contents")
+        .await
+        .unwrap();
+
+    rename(&src, &dst).await.unwrap();
+
+    assert!(!try_exists(&src).await.unwrap());
+    assert!(try_exists(&dst).await.unwrap());
+    assert_eq!(
+        tokio::fs::read(dst.join("file.txt")).await.unwrap(),
+        b"contents"
+    );
+}
+
+#[tokio::test]
+async fn rename_nonexistent_source() {
+    let dir = tempdir().unwrap();
+    let src = dir.path().join("nonexistent");
+    let dst = dir.path().join("dst");
+
+    let result = rename(&src, &dst).await;
+    assert_eq!(result.err().unwrap().raw_os_error().unwrap(), libc::ENOENT);
+}
+
+#[tokio::test]
+async fn rename_path_name_too_long() {
+    let dir = tempdir().unwrap();
+    let src = dir.path().join("src.txt");
+    tokio::fs::write(&src, b"Hello File!").await.unwrap();
+
+    // if the destination name is above PATH_MAX (Linux is 4096 bytes), we should
+    // receive a `std::io::ErrorKind::InvalidFilename` error.
+    let long_dst = dir.path().join(vec!["a"; (PATH_MAX + 1) as usize].join(""));
+    let result = rename(&src, &long_dst).await;
+    assert_eq!(
+        result.err().unwrap().kind(),
+        std::io::ErrorKind::InvalidFilename
+    );
+}
+
+#[tokio::test]
+async fn cancel_op_future() {
+    if !io_uring_supported() {
+        return;
+    }
+
+    let (_tmp_file, path): (Vec<NamedTempFile>, Vec<PathBuf>) = create_tmp_files(1);
+    let path = path[0].clone();
+    let dst = path.with_extension("renamed");
+
+    let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
+
+    let handle = tokio::spawn(async move {
+        poll_fn(|cx| {
+            let fut = rename(path.clone(), dst.clone());
+
+            // the first poll should return Pending.
+            assert_pending!(Box::pin(fut).poll_unpin(cx));
+            tx.send(true).unwrap();
+
+            Poll::<()>::Pending
+        })
+        .await;
+    });
+
+    // Wait for the first poll
+    let val = rx.recv().await;
+    assert!(val.unwrap());
+
+    handle.abort();
+
+    let res = handle.await.unwrap_err();
+    assert!(res.is_cancelled());
+}
+
+fn create_tmp_files(num_files: usize) -> (Vec<NamedTempFile>, Vec<PathBuf>) {
+    let mut files = Vec::with_capacity(num_files);
+    for _ in 0..num_files {
+        let mut tmp = NamedTempFile::new().unwrap();
+        let buf = vec![20; 1023];
+        tmp.write_all(&buf).unwrap();
+        let path = tmp.path().to_path_buf();
+        files.push((tmp, path));
+    }
+
+    files.into_iter().unzip()
+}

tests/fs_uring_statx_fd_leak_test.rs

+#![cfg(all(
+    tokio_unstable,
+    feature = "io-uring",
+    feature = "rt",
+    feature = "fs",
+    target_os = "linux"
+))]
+
+mod support {
+    pub(crate) mod io_uring;
+}
+
+use std::fmt::Debug;
+use std::fs;
+use std::future::Future;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::task::{Context, Poll};
+use std::time::Duration;
+
+use tempfile::NamedTempFile;
+use tokio::fs::read;
+use tokio::sync::mpsc::{unbounded_channel, UnboundedSender};
+use tokio::time::timeout;
+use tokio_test::assert_pending;
+
+use crate::support::io_uring::{assert_fds_are_not_leaking, io_uring_supported};
+
+/// Count currently-open fds in this process.
+fn fd_count() -> usize {
+    fs::read_dir("/proc/self/fd").unwrap().count()
+}
+
+/// First poll:
+/// - polls the inner `tokio::fs::read()` future once,
+/// - expects `Pending` so we know we took the io_uring path,
+/// - registers the task waker with Tokio's uring machinery.
+///
+/// Second poll:
+/// - happens after the kernel completes the open and Tokio stores the CQE as
+///   `Lifecycle::Completed(cqe)` and wakes the task,
+/// - polls inner future again to execute `Op::file_metadata`/Statx operation
+///
+/// Third poll:
+/// - happens after the kernel completes the statx and Tokio stores the CQE as
+///   `Lifecycle::Completed(cqe)` and wakes the task,
+/// - stays pending forever so the task can be aborted.
+///
+/// Aborting the task here drops the inner `Op::file_metadata()` future while Tokio
+/// still has a completed CQE sitting in the slab.
+struct PollOpenOnceThenNeverRepoll<F> {
+    inner: Pin<Box<F>>,
+    poll_senders: Vec<UnboundedSender<()>>,
+    poll_pending_counts: usize,
+}
+
+impl<F: Future> Future for PollOpenOnceThenNeverRepoll<F>
+where
+    F::Output: Debug,
+{
+    type Output = F::Output;
+
+    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let poll_count = self.poll_pending_counts;
+        let sender_count = self.poll_senders.len();
+        if poll_count < sender_count {
+            // The first two poll should return `Poll::Pending`` because it verifies
+            //  that io_uring is enabled and then executes the open operation in
+            // `tokio::fs::read`
+            if poll_count != sender_count - 1 {
+                assert_pending!(self.inner.as_mut().poll(cx));
+            }
+            self.poll_pending_counts += 1;
+            self.poll_senders[poll_count].send(()).unwrap();
+        }
+
+        Poll::Pending
+    }
+}
+
+async fn completed_then_dropped_before_repoll(path: PathBuf) {
+    // `tokio::fs::read` has an Open operation occurring (via OpenOptions) before doing a
+    // Statx operation (via `Op::file_metadata`), we must poll and complete the Open operation
+    // first to then be able to poll the Statx operation.
+    const POLL_PENDING_COUNT: usize = 3;
+    let mut poll_senders: Vec<UnboundedSender<()>> = Vec::new();
+    let mut poll_receivers = Vec::new();
+
+    for _ in 0..POLL_PENDING_COUNT {
+        let (sender, receiver) = unbounded_channel();
+        poll_senders.push(sender);
+        poll_receivers.push(receiver);
+    }
+
+    let handle = tokio::spawn(async move {
+        let fut = read(&path);
+        PollOpenOnceThenNeverRepoll {
+            inner: Box::pin(fut),
+            poll_senders,
+            poll_pending_counts: 0,
+        }
+        .await
+    });
+
+    for (i, poll_receiver) in poll_receivers
+        .iter_mut()
+        .enumerate()
+        .take(POLL_PENDING_COUNT)
+    {
+        // Wait until the inner open has been polled once and registered with io_uring.
+        if i == 0 {
+            poll_receiver.recv().await.unwrap();
+        } else {
+            // Wait until Tokio wakes the task because the open/statx completed. At this point
+            // the CQE should already be stored as `Lifecycle::Completed(cqe)`.
+            let _ = timeout(Duration::from_secs(2), poll_receiver.recv()).await;
+        }
+    }
+
+    // Abort now, before the inner statx future gets re-polled and consumes the CQE.
+    handle.abort();
+    let err = handle.await.unwrap_err();
+    assert!(err.is_cancelled(), "task was not cancelled as expected");
+}
+
+#[test]
+fn uring_completed_then_dropped() {
+    if !io_uring_supported() {
+        return;
+    }
+
+    let rt = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(1)
+        .enable_all()
+        .build()
+        .unwrap();
+
+    rt.block_on(async {
+        let before = fd_count();
+        let tmp = NamedTempFile::new().unwrap();
+        let path = tmp.path().to_path_buf();
+        let file_number = 128;
+
+        for _ in 0..file_number {
+            completed_then_dropped_before_repoll(path.clone()).await;
+        }
+
+        assert_fds_are_not_leaking(before, file_number, 1).await
+    });
+}

tests/fs_uring_statx.rs

+//! Uring file operations tests.
+
+#![cfg(all(
+    tokio_unstable,
+    feature = "io-uring",
+    feature = "rt",
+    feature = "fs",
+    target_os = "linux"
+))]
+
+use futures::future::Future;
+use futures::future::FutureExt;
+use libc::PATH_MAX;
+use std::future::poll_fn;
+use std::io::Write;
+use std::os::unix::fs::PermissionsExt;
+use std::path::PathBuf;
+use std::sync::mpsc;
+use std::task::Poll;
+use std::time::Duration;
+use tempfile::{tempdir, NamedTempFile};
+use tokio::fs::{create_dir, metadata, set_permissions, symlink, try_exists, write};
+use tokio::runtime::{Builder, Runtime};
+use tokio_test::assert_pending;
+use tokio_util::task::TaskTracker;
+
+use crate::support::io_uring::io_uring_supported;
+
+mod support {
+    pub(crate) mod io_uring;
+}
+
+fn multi_rt(n: usize) -> Box<dyn Fn() -> Runtime> {
+    Box::new(move || {
+        Builder::new_multi_thread()
+            .worker_threads(n)
+            .enable_all()
+            .build()
+            .unwrap()
+    })
+}
+
+fn current_rt() -> Box<dyn Fn() -> Runtime> {
+    Box::new(|| Builder::new_current_thread().enable_all().build().unwrap())
+}
+
+fn rt_combinations() -> Vec<Box<dyn Fn() -> Runtime>> {
+    vec![
+        current_rt(),
+        multi_rt(1),
+        multi_rt(2),
+        multi_rt(8),
+        multi_rt(64),
+        multi_rt(256),
+    ]
+}
+
+#[test]
+fn shutdown_runtime_while_performing_io_uring_ops() {
+    if !io_uring_supported() {
+        return;
+    }
+
+    fn run(rt: Runtime) {
+        let (done_tx, done_rx) = mpsc::channel();
+        let (_tmp, path) = create_tmp_files(1);
+        // keep 100 permits
+        const N: i32 = 100;
+        rt.spawn(async move {
+            let path = path[0].clone();
+
+            // spawning a bunch of uring operations.
+            let mut futs = vec![];
+
+            // spawning a bunch of uring operations.
+            for _ in 0..N {
+                let path = path.clone();
+                let mut fut = Box::pin(try_exists(path));
+
+                poll_fn(|cx| {
+                    assert_pending!(fut.as_mut().poll(cx));
+                    Poll::<()>::Pending
+                })
+                .await;
+
+                futs.push(fut);
+            }
+
+            tokio::task::yield_now().await;
+        });
+
+        std::thread::spawn(move || {
+            rt.shutdown_timeout(Duration::from_millis(300));
+            done_tx.send(()).unwrap();
+        });
+
+        done_rx.recv().unwrap();
+    }
+
+    for rt in rt_combinations() {
+        run(rt());
+    }
+}
+
+#[test]
+fn stat_many_files() {
+    fn run(rt: Runtime) {
+        const NUM_FILES: usize = 512;
+
+        let (_tmp_files, paths): (Vec<NamedTempFile>, Vec<PathBuf>) = create_tmp_files(NUM_FILES);
+
+        rt.block_on(async move {
+            let tracker = TaskTracker::new();
+
+            for i in 0..10_000 {
+                let path = paths.get(i % NUM_FILES).unwrap().clone();
+                tracker.spawn(async move {
+                    let exists = try_exists(path).await.unwrap();
+                    assert!(exists);
+                });
+            }
+            tracker.close();
+            tracker.wait().await;
+        });
+    }
+
+    for rt in rt_combinations() {
+        run(rt());
+    }
+}
+
+#[tokio::test]
+async fn stat_small_large_files() {
+    let (_tmp, path) = create_large_temp_file();
+
+    let exists = try_exists(path).await.unwrap();
+    assert!(exists);
+
+    let (_tmp, path) = create_small_temp_file();
+
+    let exists = try_exists(path).await.unwrap();
+    assert!(exists);
+}
+
+#[tokio::test]
+async fn stat_nonexistent_file() {
+    let path = tempdir().unwrap().path().join("nonexistent_path");
+    let exists = try_exists(path).await.unwrap();
+    assert!(!exists);
+}
+
+// Error is not produced on Linux 4.19 (Linux 7.1 it works)
+#[tokio::test]
+#[cfg_attr(miri, ignore)] // No `chmod` in miri.
+#[cfg(unix)]
+async fn stat_permission_denied() {
+    if !io_uring_supported() {
+        return;
+    }
+
+    let dir = tempdir().unwrap();
+    let permission_denied_directory_path = dir.path().join("baz");
+    create_dir(&permission_denied_directory_path).await.unwrap();
+    let permission_denied_file_path = permission_denied_directory_path.join("baz.txt");
+    write(&permission_denied_file_path, b"Hello File!")
+        .await
+        .unwrap();
+    let mut perms = metadata(&permission_denied_directory_path)
+        .await
+        .unwrap()
+        .permissions();
+
+    perms.set_mode(0o244);
+    set_permissions(&permission_denied_directory_path, perms)
+        .await
+        .unwrap();
+    let permission_denied_result = try_exists(permission_denied_file_path).await;
+    assert_eq!(
+        permission_denied_result
+            .err()
+            .unwrap()
+            .raw_os_error()
+            .unwrap(),
+        libc::EACCES
+    );
+}
+
+#[tokio::test]
+#[cfg(unix)]
+async fn stat_filesystem_loop() {
+    let dir = tempdir().unwrap();
+    let first_symlink = dir.path().join("bar");
+    let second_symlink = dir.path().join("foo");
+    symlink(&first_symlink, &second_symlink).await.unwrap();
+    symlink(&second_symlink, &first_symlink).await.unwrap();
+
+    // Both symlinks loop on each other, so stating either one should produce
+    // a file system loop error. This produces a `std::io::ErrorKind::FilesystemLoop`
+    // error, but that error is gated behind io_error_more feature, and we can't be
+    // sure if that name will ever be changed, so preferred using libc::ELOOP instead
+    let filesystem_loop_result = try_exists(first_symlink).await;
+    assert_eq!(
+        filesystem_loop_result
+            .err()
+            .unwrap()
+            .raw_os_error()
+            .unwrap(),
+        libc::ELOOP
+    );
+    let filesystem_loop_result = try_exists(second_symlink).await;
+    assert_eq!(
+        filesystem_loop_result
+            .err()
+            .unwrap()
+            .raw_os_error()
+            .unwrap(),
+        libc::ELOOP
+    );
+}
+
+#[tokio::test]
+async fn stat_path_name_too_long() {
+    let dir = tempdir().unwrap();
+    // if we stat a file whose name is above PATH_MAX (Linux is 4096 bytes, Windows 260 chars or 32767 chars if extended
+    // path is permitted), we should receive an std::io::ErrorKind::InvalidFilename error
+    let long_nonexistent_path = dir.path().join(vec!["a"; (PATH_MAX + 1) as usize].join(""));
+    let name_too_long_result = try_exists(long_nonexistent_path).await;
+    assert_eq!(
+        name_too_long_result.err().unwrap().kind(),
+        std::io::ErrorKind::InvalidFilename
+    );
+}
+
+#[tokio::test]
+async fn cancel_op_future() {
+    if !io_uring_supported() {
+        return;
+    }
+
+    let (_tmp_file, path): (Vec<NamedTempFile>, Vec<PathBuf>) = create_tmp_files(1);
+    let path = path[0].clone();
+
+    let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
+
+    let handle = tokio::spawn(async move {
+        poll_fn(|cx| {
+            let fut = try_exists(path.clone());
+
+            // the first poll should return Pending.
+            assert_pending!(Box::pin(fut).poll_unpin(cx));
+            tx.send(true).unwrap();
+
+            Poll::<()>::Pending
+        })
+        .await;
+    });
+
+    // Wait for the first poll
+    let val = rx.recv().await;
+    assert!(val.unwrap());
+
+    handle.abort();
+
+    let res = handle.await.unwrap_err();
+    assert!(res.is_cancelled());
+}
+
+fn create_tmp_files(num_files: usize) -> (Vec<NamedTempFile>, Vec<PathBuf>) {
+    let mut files = Vec::with_capacity(num_files);
+    for _ in 0..num_files {
+        let mut tmp = NamedTempFile::new().unwrap();
+        let buf = vec![20; 1023];
+        tmp.write_all(&buf).unwrap();
+        let path = tmp.path().to_path_buf();
+        files.push((tmp, path));
+    }
+
+    files.into_iter().unzip()
+}
+
+fn create_large_temp_file() -> (NamedTempFile, PathBuf) {
+    let mut tmp = NamedTempFile::new().unwrap();
+    let buf = create_buf(5000);
+
+    tmp.write_all(&buf).unwrap();
+    let path = tmp.path().to_path_buf();
+
+    (tmp, path)
+}
+
+fn create_small_temp_file() -> (NamedTempFile, PathBuf) {
+    let mut tmp = NamedTempFile::new().unwrap();
+    let buf = create_buf(20);
+
+    tmp.write_all(&buf).unwrap();
+    let path = tmp.path().to_path_buf();
+
+    (tmp, path)
+}
+
+fn create_buf(length: usize) -> Vec<u8> {
+    (0..length).map(|i| i as u8).collect()
+}

tests/support/io_uring.rs

+#![cfg(all(
+    tokio_unstable,
+    feature = "io-uring",
+    feature = "rt",
+    feature = "fs",
+    target_os = "linux"
+))]
+
+use std::{
+    fs,
+    time::{Duration, Instant},
+};
+
+use io_uring::IoUring;
+
+// Currently, we are running some of the tests on Kernels where io_uring is not supported
+// to check if the fallback mechanism works, this comes with the limitation that we are not
+// able to run some checks (e.g., asserting a poll returns pending). This utility function
+// is useful when we want to run a test only in Linux targets where io_uring is supported.
+pub fn io_uring_supported() -> bool {
+    match IoUring::new(256) {
+        Ok(_) => true,
+        // The Kernel does not support io-uring
+        Err(e) if e.raw_os_error() == Some(libc::ENOSYS) => false,
+        Err(_) => unreachable!(
+            "The target should either support io_uring or return ENOSYS if not supported"
+        ),
+    }
+}
+
+#[allow(dead_code)]
+pub async fn assert_fds_are_not_leaking(count_before: usize, opened_files: usize, timeout: u64) {
+    let fd_check_start = Instant::now();
+
+    let max_leaked_fd = opened_files / 2;
+
+    while fd_check_start.elapsed() < Duration::from_secs(timeout) {
+        tokio::task::yield_now().await;
+
+        let fd_count_after_cancel = fs::read_dir("/proc/self/fd").unwrap().count();
+        let leaked = fd_count_after_cancel.saturating_sub(count_before);
+
+        // Since we are opening {opened_files} files, we expect that the related fds
+        // related to this operation will be closed. Since some other fds
+        // can be opened in the meantime, we expect this number to be higher
+        // than the counter before opening the files. This number could be
+        // lower, but to avoid test flakiness we check that this is at most
+        // half the number of the file we opened to check if there's a leak.
+        if leaked <= max_leaked_fd {
+            // test success
+            return;
+        }
+    }
+    panic!("Number of FDs is staying above {max_leaked_fd}. There is probably an FD leak.");
+}

.cargo_vcs_info.json

 {
   "git": {
-    "sha1": "efdba5fcf02c4b93d379114df136b994c3b21445"
+    "sha1": "be689a35f5ade5a39e507f79d3ec85cdab27806f"
   },
   "path_in_vcs": "tokio"
 }

Cargo.lock

@@ -1088,5 +1088,5 @@ # This file is automatically @generated by Cargo.
 name = "tokio"
-version = "1.52.3"
+version = "1.52.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fc7f01b389ac15039e4dc9531aa973a135d7a4135281b12d7c1bc79fd57fffe"
+checksum = "317fafbbe3f02fc663dad00ea6186197de963cd4190e86a26d8d0fae095539af"
 dependencies = [
@@ -1098,3 +1098,3 @@ "pin-project-lite",
 name = "tokio"
-version = "1.52.4"
+version = "1.53.0"
 dependencies = [
@@ -1134,5 +1134,5 @@ "async-stream",
 name = "tokio-macros"
-version = "2.7.0"
+version = "2.7.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496"
+checksum = "6328af13490e73a9b4694030fafd93f8c8c6a9dede33e821c3fc63eddf8042ba"
 dependencies = [
@@ -1152,3 +1152,3 @@ "proc-macro2",
  "pin-project-lite",
- "tokio 1.52.3",
+ "tokio 1.52.4",
 ]
@@ -1163,3 +1163,3 @@
  "futures-core",
- "tokio 1.52.3",
+ "tokio 1.52.4",
  "tokio-stream",
@@ -1179,3 +1179,3 @@ ]
  "pin-project-lite",
- "tokio 1.52.3",
+ "tokio 1.52.4",
 ]
@@ -1182,0 +1182,0 @@

Cargo.toml

@@ -16,3 +16,3 @@ # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
 name = "tokio"
-version = "1.52.4"
+version = "1.53.0"
 authors = ["Tokio Contributors <[e-mail]>"]
@@ -122,2 +122,3 @@ build = false
 rt-multi-thread = ["rt"]
+schedule-latency = []
 signal = [
@@ -246,2 +247,6 @@ "libc",
 [[test]]
+name = "fs_uring_rename"
+path = "tests/fs_uring_rename.rs"
+
+[[test]]
 name = "fs_uring_runtime_shutdown"
@@ -251,2 +256,10 @@ path = "tests/fs_uring_runtime_shutdown.rs"
 [[test]]
+name = "fs_uring_statx"
+path = "tests/fs_uring_statx.rs"
+
+[[test]]
+name = "fs_uring_statx_fd_leak_test"
+path = "tests/fs_uring_statx_fd_leak_test.rs"
+
+[[test]]
 name = "fs_write"
@@ -253,0 +266,0 @@ path = "tests/fs_write.rs"

docs/reactor-refactor.md

@@ -55,2 +55,3 @@ # Refactor I/O driver
     async fn clear_readiness(&self, ready_event: ReadyEvent);
+}
 ```
@@ -57,0 +58,0 @@

README.md

@@ -63,3 +63,3 @@ *[TokioConf 2026 program and tickets are now available!](https://tokioconf.com)*
 [dependencies]
-tokio = { version = "1.52.4", features = ["full"] }
+tokio = { version = "1.53.0", features = ["full"] }
 ```
@@ -66,0 +66,0 @@ Then, on your main.rs:

src/fs/create_dir_all.rs

@@ -29,2 +29,5 @@ use crate::fs::asyncify;
 ///
+/// If the final component at `path` already exists and is a directory, this
+/// function will also return successfully without error.
+///
 /// Notable exception is made for situations where any of the directories
@@ -31,0 +34,0 @@ /// specified in the `path` could not be created as it was being created concurrently.

src/fs/file.rs

@@ -602,3 +602,3 @@ //! Types for working with [`File`].
     ) -> Poll<io::Result<()>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
 
@@ -698,3 +698,3 @@ let me = self.get_mut();
     fn poll_complete(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<u64>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         let inner = self.inner.get_mut();
@@ -735,3 +735,3 @@
     ) -> Poll<io::Result<usize>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         let me = self.get_mut();
@@ -807,3 +807,3 @@ let inner = me.inner.get_mut();
     ) -> Poll<Result<usize, io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         let me = self.get_mut();
@@ -879,3 +879,3 @@ let inner = me.inner.get_mut();
     fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         let inner = self.inner.get_mut();
@@ -886,3 +886,3 @@ inner.poll_flush(cx)
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         self.poll_flush(cx)
@@ -907,2 +907,9 @@ }
 #[cfg(unix)]
+impl From<std::os::fd::OwnedFd> for File {
+    fn from(fd: std::os::fd::OwnedFd) -> Self {
+        Self::from_std(StdFile::from(fd))
+    }
+}
+
+#[cfg(unix)]
 impl std::os::unix::io::AsRawFd for File {
@@ -933,4 +940,10 @@ fn as_raw_fd(&self) -> std::os::unix::io::RawFd {
 cfg_windows! {
-    use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle, AsHandle, BorrowedHandle};
+    use crate::os::windows::io::{AsRawHandle, FromRawHandle, RawHandle, AsHandle, BorrowedHandle, OwnedHandle};
 
+    impl From<OwnedHandle> for File {
+        fn from(handle: OwnedHandle) -> Self {
+            Self::from_std(StdFile::from(handle))
+        }
+    }
+
     impl AsRawHandle for File {
@@ -1088,3 +1101,3 @@ fn as_raw_handle(&self) -> RawHandle {
     fn poll_complete_inflight(&mut self, cx: &mut Context<'_>) -> Poll<()> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         match self.poll_flush(cx) {
@@ -1091,0 +1104,0 @@ Poll::Ready(Err(e)) => {

src/fs/mocks.rs

@@ -41,2 +41,6 @@ //! Mock version of std::fs::File;
     #[cfg(windows)]
+    impl From<std::os::windows::io::OwnedHandle> for File {
+        fn from(handle: std::os::windows::io::OwnedHandle) -> Self;
+    }
+    #[cfg(windows)]
     impl std::os::windows::io::AsRawHandle for File {
@@ -110,2 +114,11 @@ fn as_raw_handle(&self) -> std::os::windows::io::RawHandle;
 
+#[cfg(all(test, unix))]
+impl From<OwnedFd> for MockFile {
+    #[inline]
+    fn from(file: OwnedFd) -> MockFile {
+        use std::os::fd::IntoRawFd;
+        unsafe { MockFile::from_raw_fd(IntoRawFd::into_raw_fd(file)) }
+    }
+}
+
 tokio_thread_local! {
@@ -112,0 +125,0 @@ static QUEUE: RefCell<VecDeque<Box<dyn FnOnce() + Send>>> = RefCell::new(VecDeque::new())

src/fs/open_options.rs

@@ -521,2 +521,6 @@ use crate::fs::{asyncify, File};
     pub async fn open(&self, path: impl AsRef<Path>) -> io::Result<File> {
+        self.open_inner(path.as_ref()).await
+    }
+
+    async fn open_inner(&self, path: &Path) -> io::Result<File> {
         match &self.inner {
@@ -539,3 +543,3 @@ Kind::Std(opts) => Self::std_open(opts, path).await,
                 {
-                    Op::open(path.as_ref(), opts)?.await
+                    Op::open(path, opts)?.await
                 } else {
@@ -549,8 +553,7 @@ let opts = opts.clone().into();
 
-    async fn std_open(opts: &StdOpenOptions, path: impl AsRef<Path>) -> io::Result<File> {
-        let path = path.as_ref().to_owned();
+    async fn std_open(opts: &StdOpenOptions, path: &Path) -> io::Result<File> {
+        let path = path.to_owned();
         let opts = opts.clone();
 
-        let std = asyncify(move || opts.open(path)).await?;
-        Ok(File::from_std(std))
+        Ok(asyncify(move || opts.open(path)).await?.into())
     }
@@ -557,0 +560,0 @@

src/fs/read_dir.rs

@@ -77,3 +77,3 @@ use crate::fs::asyncify;
     ///
-    /// This method is cancellation safe.
+    /// This method is cancel safe.
     pub async fn next_entry(&mut self) -> io::Result<Option<DirEntry>> {
@@ -80,0 +80,0 @@ use std::future::poll_fn;

src/fs/read_uring.rs

@@ -22,5 +22,22 @@ use crate::fs::OpenOptions;
 
-    // TODO: use io uring in the future to obtain metadata
+    #[cfg(not(any(target_env = "gnu", target_os = "android")))]
     let size_hint: Option<usize> = file.metadata().await.map(|m| m.len() as usize).ok();
 
+    #[cfg(
+        // libc::statx is only supported on these platforms
+        // FIXME: Add musl target env when our minimum supported
+        // rust version is 1.93. To clarify, statx support is
+        // introduced to musl in 1.25 as mentioned officially here:
+        // https://musl.libc.org/releases.html.
+        // However, rustup target_env building for *-linux-musl
+        // uses 1.25 musl on all *-linux-musl platforms starting
+        // in 1.93 stable rust version.
+        // https://blog.rust-lang.org/2025/12/05/Updating-musl-1.2.5/
+        any(target_env = "gnu", target_os = "android")
+    )]
+    let size_hint = Op::file_metadata(&file)?
+        .await
+        .map(|m| m.len() as usize)
+        .ok();
+
     let fd: OwnedFd = file
@@ -27,0 +44,0 @@ .try_into_std()

src/fs/read.rs

@@ -57,3 +57,3 @@ use crate::fs::asyncify;
 pub async fn read(path: impl AsRef<Path>) -> io::Result<Vec<u8>> {
-    let path = path.as_ref().to_owned();
+    let path = path.as_ref();
 
@@ -65,3 +65,12 @@ #[cfg(all(
         feature = "fs",
-        target_os = "linux"
+        // libc::statx is only supported on these platforms
+        // FIXME: Add musl target env when our minimum supported
+        // rust version is 1.93. To clarify, statx support is
+        // introduced to musl in 1.25 as mentioned officially here:
+        // https://musl.libc.org/releases.html.
+        // However, rustup target_env building for *-linux-musl
+        // uses 1.25 musl on all *-linux-musl platforms starting
+        // in 1.93 stable rust version.
+        // https://blog.rust-lang.org/2025/12/05/Updating-musl-1.2.5/
+        any(target_env = "gnu", target_os = "android")
     ))]
@@ -77,7 +86,12 @@ {
         {
-            return read_uring(&path).await;
+            return read_uring(path).await;
         }
     }
 
+    read_spawn_blocking(path).await
+}
+
+async fn read_spawn_blocking(path: &Path) -> io::Result<Vec<u8>> {
+    let path = path.to_owned();
     asyncify(move || std::fs::read(path)).await
 }

src/fs/rename.rs

@@ -13,6 +13,35 @@ use crate::fs::asyncify;
 pub async fn rename(from: impl AsRef<Path>, to: impl AsRef<Path>) -> io::Result<()> {
-    let from = from.as_ref().to_owned();
-    let to = to.as_ref().to_owned();
+    let from = from.as_ref();
+    let to = to.as_ref();
 
+    #[cfg(all(
+        tokio_unstable,
+        feature = "io-uring",
+        feature = "rt",
+        feature = "fs",
+        target_os = "linux",
+    ))]
+    {
+        use crate::io::uring::rename::Rename;
+        use crate::runtime::driver::op::Op;
+
+        let handle = crate::runtime::Handle::current();
+        let driver_handle = handle.inner.driver().io();
+
+        type RenameOp = Op<Rename>;
+
+        if driver_handle
+            .check_and_init(io_uring::opcode::RenameAt::CODE)
+            .await?
+        {
+            return RenameOp::rename(from, to)?.await;
+        }
+    }
+
+    rename_blocking(from, to).await
+}
+
+async fn rename_blocking(from: &Path, to: &Path) -> io::Result<()> {
+    let [from, to] = [from, to].map(Path::to_owned);
     asyncify(move || std::fs::rename(from, to)).await
 }

src/fs/try_exists.rs

@@ -26,4 +26,65 @@ use crate::fs::asyncify;
 pub async fn try_exists(path: impl AsRef<Path>) -> io::Result<bool> {
-    let path = path.as_ref().to_owned();
+    let path = path.as_ref();
+
+    #[cfg(all(
+        tokio_unstable,
+        feature = "io-uring",
+        feature = "rt",
+        feature = "fs",
+        // libc::statx is only supported on these platforms
+        // FIXME: Add musl target env when our minimum supported
+        // rust version is 1.93. To clarify, statx support is
+        // introduced to musl in 1.25 as mentioned officially here:
+        // https://musl.libc.org/releases.html.
+        // However, rustup target_env building for *-linux-musl
+        // uses 1.25 musl on all *-linux-musl platforms starting
+        // in 1.93 stable rust version.
+        // https://blog.rust-lang.org/2025/12/05/Updating-musl-1.2.5/
+        any(target_env = "gnu", target_os = "android")
+    ))]
+    {
+        let handle = crate::runtime::Handle::current();
+        let driver_handle = handle.inner.driver().io();
+        if driver_handle
+            .check_and_init(io_uring::opcode::Statx::CODE)
+            .await?
+        {
+            return try_exists_uring(path).await;
+        }
+    }
+
+    try_exists_spawn_blocking(path).await
+}
+
+cfg_io_uring! {
+    #[inline]
+    #[cfg(
+        // libc::statx is only supported on these platforms
+        // FIXME: Add musl target env when our minimum supported
+        // rust version is 1.93. To clarify, statx support is
+        // introduced to musl in 1.25 as mentioned officially here:
+        // https://musl.libc.org/releases.html.
+        // However, rustup target_env building for *-linux-musl
+        // uses 1.25 musl on all *-linux-musl platforms starting
+        // in 1.93 stable rust version.
+        // https://blog.rust-lang.org/2025/12/05/Updating-musl-1.2.5/
+        any(target_env = "gnu", target_os = "android")
+    )]
+    async fn try_exists_uring(path: &Path) -> io::Result<bool> {
+        use crate::runtime::driver::op::Op;
+
+        match Op::metadata(path)?.await {
+            Ok(_) => Ok(true),
+            Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(false),
+            Err(error) => Err(error),
+        }
+    }
+}
+
+async fn try_exists_spawn_blocking(path: &Path) -> io::Result<bool> {
+    let path = path.to_owned();
+    // FIXME: When MSRV is 1.81, change this to
+    // std::fs::exists() to be consistent with
+    // all other tokio::fs operations
     asyncify(move || path.try_exists()).await
 }

src/io/async_buf_read.rs

@@ -100,2 +100,3 @@ use crate::io::AsyncRead;
 impl AsyncBufRead for &[u8] {
+    #[inline]
     fn poll_fill_buf(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
@@ -105,2 +106,3 @@ Poll::Ready(Ok(*self))
 
+    #[inline]
     fn consume(mut self: Pin<&mut Self>, amt: usize) {
@@ -112,2 +114,3 @@ *self = &self[amt..];
 impl<T: AsRef<[u8]> + Unpin> AsyncBufRead for io::Cursor<T> {
+    #[inline]
     fn poll_fill_buf(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
@@ -117,2 +120,3 @@ Poll::Ready(io::BufRead::fill_buf(self.get_mut()))
 
+    #[inline]
     fn consume(self: Pin<&mut Self>, amt: usize) {
@@ -119,0 +123,0 @@ io::BufRead::consume(self.get_mut(), amt);

src/io/async_read.rs

@@ -97,2 +97,3 @@ use super::ReadBuf;
 impl AsyncRead for &[u8] {
+    #[inline]
     fn poll_read(
@@ -112,2 +113,3 @@ mut self: Pin<&mut Self>,
 impl<T: AsRef<[u8]> + Unpin> AsyncRead for io::Cursor<T> {
+    #[inline]
     fn poll_read(
@@ -114,0 +116,0 @@ mut self: Pin<&mut Self>,

src/io/async_seek.rs

@@ -90,5 +90,7 @@ use std::io::{self, SeekFrom};
 impl<T: AsRef<[u8]> + Unpin> AsyncSeek for io::Cursor<T> {
+    #[inline]
     fn start_seek(mut self: Pin<&mut Self>, pos: SeekFrom) -> io::Result<()> {
         io::Seek::seek(&mut *self, pos).map(drop)
     }
+    #[inline]
     fn poll_complete(self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<io::Result<u64>> {
@@ -95,0 +97,0 @@ Poll::Ready(Ok(self.get_mut().position()))

src/io/async_write.rs

@@ -254,2 +254,3 @@ use std::io::{self, IoSlice};
 impl AsyncWrite for Vec<u8> {
+    #[inline]
     fn poll_write(
@@ -264,2 +265,3 @@ self: Pin<&mut Self>,
 
+    #[inline]
     fn poll_write_vectored(
@@ -273,2 +275,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn is_write_vectored(&self) -> bool {
@@ -278,2 +281,3 @@ true
 
+    #[inline]
     fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -283,2 +287,3 @@ Poll::Ready(Ok(()))
 
+    #[inline]
     fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -290,2 +295,3 @@ Poll::Ready(Ok(()))
 impl AsyncWrite for io::Cursor<&mut [u8]> {
+    #[inline]
     fn poll_write(
@@ -299,2 +305,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn poll_write_vectored(
@@ -308,2 +315,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn is_write_vectored(&self) -> bool {
@@ -313,2 +321,3 @@ true
 
+    #[inline]
     fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -318,2 +327,3 @@ Poll::Ready(io::Write::flush(&mut *self))
 
+    #[inline]
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -325,2 +335,3 @@ self.poll_flush(cx)
 impl AsyncWrite for io::Cursor<&mut Vec<u8>> {
+    #[inline]
     fn poll_write(
@@ -334,2 +345,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn poll_write_vectored(
@@ -343,2 +355,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn is_write_vectored(&self) -> bool {
@@ -348,2 +361,3 @@ true
 
+    #[inline]
     fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -353,2 +367,3 @@ Poll::Ready(io::Write::flush(&mut *self))
 
+    #[inline]
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -360,2 +375,3 @@ self.poll_flush(cx)
 impl AsyncWrite for io::Cursor<Vec<u8>> {
+    #[inline]
     fn poll_write(
@@ -369,2 +385,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn poll_write_vectored(
@@ -378,2 +395,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn is_write_vectored(&self) -> bool {
@@ -383,2 +401,3 @@ true
 
+    #[inline]
     fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -388,2 +407,3 @@ Poll::Ready(io::Write::flush(&mut *self))
 
+    #[inline]
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -395,2 +415,3 @@ self.poll_flush(cx)
 impl AsyncWrite for io::Cursor<Box<[u8]>> {
+    #[inline]
     fn poll_write(
@@ -404,2 +425,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn poll_write_vectored(
@@ -413,2 +435,3 @@ mut self: Pin<&mut Self>,
 
+    #[inline]
     fn is_write_vectored(&self) -> bool {
@@ -418,2 +441,3 @@ true
 
+    #[inline]
     fn poll_flush(mut self: Pin<&mut Self>, _: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -423,2 +447,3 @@ Poll::Ready(io::Write::flush(&mut *self))
 
+    #[inline]
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
@@ -425,0 +450,0 @@ self.poll_flush(cx)

src/io/blocking.rs

@@ -217,3 +217,3 @@ use crate::io::sys;
         if self.pos == self.buf.len() {
-            self.buf.truncate(0);
+            self.buf.clear();
             self.pos = 0;
@@ -319,3 +319,3 @@ }
             self.pos = 0;
-            self.buf.truncate(0);
+            self.buf.clear();
             ret
@@ -322,0 +322,0 @@ }

src/io/bsd/poll_aio.rs

@@ -18,4 +18,4 @@ //! Use POSIX AIO futures with Tokio.
 ///
-/// Tokio's consumer must pass an implementor of this trait to create a
-/// [`Aio`] object.  Implementors must implement at least one of [`AioSource::register`] and
+/// Tokio's consumer must pass an implementer of this trait to create a
+/// [`Aio`] object.  Implementers must implement at least one of [`AioSource::register`] and
 /// [`AioSource::register_borrowed`].
@@ -31,3 +31,3 @@ pub trait AioSource {
     fn register(&mut self, _kq: RawFd, _token: usize) {
-        // This default implementation exists so new AioSource implementors that implement the
+        // This default implementation exists so new AioSource implementers that implement the
         // register_borrowed method can compile without the need to implement register.
@@ -40,3 +40,3 @@ unimplemented!("Use AioSource::register_borrowed instead")
         // This default implementation serves to provide backwards compatibility with AioSource
-        // implementors written before 1.52.0 that only implemented the unsafe `register` method.
+        // implementers written before 1.52.0 that only implemented the unsafe `register` method.
         #[allow(deprecated)]
@@ -43,0 +43,0 @@ self.register(kq.as_raw_fd(), token)

src/io/stdout.rs

@@ -18,2 +18,41 @@ use crate::io::blocking::Blocking;
     ///
+    /// # Warning
+    ///
+    /// Each call to [`stdout()`] creates a **new** handle with its own
+    /// internal state. Writes through different handles are not
+    /// coordinated, so creating a new handle in a loop can cause output
+    /// to appear out of order:
+    ///
+    /// ```no_run
+    /// # use tokio::io::{self, AsyncWriteExt};
+    /// # #[tokio::main]
+    /// # async fn main() -> std::io::Result<()> {
+    /// // WRONG: creates a new handle each iteration
+    /// for i in 0..10 {
+    ///     let mut out = io::stdout();
+    ///     out.write_all(b"data").await?;
+    ///     out.write_all(b"\n").await?;
+    ///     // out is dropped here; its last write may still be
+    ///     // running when the next iteration starts
+    /// }
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
+    /// To preserve order, create one handle outside the loop and
+    /// reuse it:
+    ///
+    /// ```no_run
+    /// # use tokio::io::{self, AsyncWriteExt};
+    /// # #[tokio::main]
+    /// # async fn main() -> std::io::Result<()> {
+    /// let mut out = io::stdout();
+    /// for i in 0..10 {
+    ///     out.write_all(b"data").await?;
+    ///     out.write_all(b"\n").await?;
+    /// }
+    /// # Ok(())
+    /// # }
+    /// ```
+    ///
     /// Created by the [`stdout`] function.
@@ -20,0 +59,0 @@ ///

src/io/uring/mod.rs

 pub(crate) mod open;
 pub(crate) mod read;
+pub(crate) mod rename;
+pub(crate) mod statx;
 pub(crate) mod utils;
 pub(crate) mod write;

src/io/util/async_buf_read_ext.rs

@@ -42,4 +42,4 @@ use crate::io::util::fill_buf::{fill_buf, FillBuf};
         ///
-        /// If the method is used as the event in a
-        /// [`tokio::select!`](crate::select) statement and some other branch
+        /// If used as a branch in
+        /// [`tokio::select!`](crate::select) and another branch
         /// completes first, then some data may have been partially read. Any
@@ -133,5 +133,5 @@ /// partially read bytes are appended to `buf`, and the method can be
         ///
-        /// This method is not cancellation safe. If the method is used as the
-        /// event in a [`tokio::select!`](crate::select) statement and some
-        /// other branch completes first, then some data may have been partially
+        /// This method is not cancel safe. If the method is used as a
+        /// branch in [`tokio::select!`](crate::select) and another
+        /// branch completes first, then some data may have been partially
         /// read, and this data is lost. There are no guarantees regarding the
@@ -274,4 +274,4 @@ /// contents of `buf` when the call is cancelled. The current
         ///
-        /// This method is cancel safe. If you use it as the event in a
-        /// [`tokio::select!`](crate::select) statement and some other branch
+        /// This method is cancel safe. If used as a branch in
+        /// [`tokio::select!`](crate::select) and another branch
         /// completes first, then it is guaranteed that no data was read.
@@ -278,0 +278,0 @@ ///

src/io/util/async_read_ext.rs

@@ -150,4 +150,4 @@ use crate::io::util::chain::{chain, Chain};
         ///
-        /// This method is cancel safe. If you use it as the event in a
-        /// [`tokio::select!`](crate::select) statement and some other branch
+        /// This method is cancel safe. If you use it as a branch in
+        /// [`tokio::select!`](crate::select) and another branch
         /// completes first, then it is guaranteed that no data was read.
@@ -219,4 +219,4 @@ ///
         ///
-        /// This method is cancel safe. If you use it as the event in a
-        /// [`tokio::select!`](crate::select) statement and some other branch
+        /// This method is cancel safe. If you use it as a branch in
+        /// [`tokio::select!`](crate::select) and another branch
         /// completes first, then it is guaranteed that no data was read.
@@ -296,5 +296,5 @@ ///
         ///
-        /// This method is not cancellation safe. If the method is used as the
-        /// event in a [`tokio::select!`](crate::select) statement and some
-        /// other branch completes first, then some data may already have been
+        /// This method is not cancel safe. If the method is used as a
+        /// branch in [`tokio::select!`](crate::select) and another
+        /// branch completes first, then some data may already have been
         /// read into `buf`.
@@ -353,4 +353,4 @@ ///
             ///
-            /// This method is cancel safe. If this method is used as an event in a
-            /// [`tokio::select!`](crate::select) statement and some other branch
+            /// This method is cancel safe. If this method is used as a branch in
+            /// [`tokio::select!`](crate::select) and another branch
             /// completes first, it is guaranteed that no data were read.
@@ -398,4 +398,4 @@ ///
             ///
-            /// This method is cancel safe. If this method is used as an event in a
-            /// [`tokio::select!`](crate::select) statement and some other branch
+            /// This method is cancel safe. If this method is used as a branch in
+            /// [`tokio::select!`](crate::select) and another branch
             /// completes first, it is guaranteed that no data were read.
@@ -444,5 +444,5 @@ ///
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -489,5 +489,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -534,5 +534,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -579,5 +579,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -623,5 +623,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -669,5 +669,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -713,5 +713,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -760,5 +760,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -807,5 +807,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -851,5 +851,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -897,5 +897,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -942,5 +942,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -987,5 +987,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -1032,5 +1032,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -1076,5 +1076,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -1122,5 +1122,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -1166,5 +1166,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -1213,5 +1213,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -1260,5 +1260,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -1304,5 +1304,5 @@ /// # Examples
             ///
-            /// This method is not cancellation safe. If the method is used as the
-            /// event in a [`tokio::select!`](crate::select) statement and some
-            /// other branch completes first, then some data may be lost.
+            /// This method is not cancel safe. If the method is used as a
+            /// branch in [`tokio::select!`](crate::select) and another
+            /// branch completes first, then some data may be lost.
             ///
@@ -1309,0 +1309,0 @@ /// # Examples

src/io/util/async_write_ext.rs

@@ -106,5 +106,5 @@ use crate::io::util::flush::{flush, Flush};
         ///
-        /// This method is cancellation safe in the sense that if it is used as
-        /// the event in a [`tokio::select!`](crate::select) statement and some
-        /// other branch completes first, then it is guaranteed that no data was
+        /// This method is cancel safe. If it is used as a branch in
+        /// [`tokio::select!`](crate::select) and another branch completes
+        /// first, then it is guaranteed that no data was
         /// written to this `AsyncWrite`.
@@ -150,5 +150,5 @@ ///
         ///
-        /// This method is cancellation safe in the sense that if it is used as
-        /// the event in a [`tokio::select!`](crate::select) statement and some
-        /// other branch completes first, then it is guaranteed that no data was
+        /// This method is cancel safe. If it is used as a branch in
+        /// [`tokio::select!`](crate::select) and another branch completes
+        /// first, then it is guaranteed that no data was
         /// written to this `AsyncWrite`.
@@ -228,5 +228,5 @@ ///
         ///
-        /// This method is cancellation safe in the sense that if it is used as
-        /// the event in a [`tokio::select!`](crate::select) statement and some
-        /// other branch completes first, then it is guaranteed that no data was
+        /// This method is cancel safe. If it is used as a branch in
+        /// [`tokio::select!`](crate::select) and another branch completes
+        /// first, then it is guaranteed that no data was
         /// written to this `AsyncWrite`.
@@ -300,4 +300,4 @@ ///
         ///
-        /// If `write_all_buf` is used as the event in a
-        /// [`tokio::select!`](crate::select) statement and some other branch
+        /// If `write_all_buf` is used as a branch in
+        /// [`tokio::select!`](crate::select) and another branch
         /// completes first, then the data in the provided buffer may have been
@@ -361,5 +361,5 @@ /// partially written. However, it is guaranteed that the provided
         ///
-        /// This method is not cancellation safe. If it is used as the event
-        /// in a [`tokio::select!`](crate::select) statement and some other
-        /// branch completes first, then the provided buffer may have been
+        /// This method is not cancel safe. If it is used as a branch in
+        /// [`tokio::select!`](crate::select) and another branch completes
+        /// first, then the provided buffer may have been
         /// partially written, but future calls to `write_all` will start over
@@ -1261,5 +1261,5 @@ /// from the beginning of the buffer.
         ///
-        /// If `flush` is used as the event in a [`tokio::select!`](crate::select)
-        /// statement and some other branch completes first, then the data in the
-        /// buffered data in this `AsyncWrite` may have been partially flushed.
+        /// If `flush` is used as a branch in [`tokio::select!`](crate::select)
+        /// and another branch completes first, then the buffered data in this
+        /// `AsyncWrite` may have been partially flushed.
         /// However, it is guaranteed that the buffer is advanced by the amount of
@@ -1266,0 +1266,0 @@ /// bytes that have been partially flushed.

src/io/util/chain.rs

@@ -97,3 +97,5 @@ use crate::io::{AsyncBufRead, AsyncRead, ReadBuf};
             ready!(me.first.poll_read(cx, buf))?;
-            if buf.remaining() == rem {
+            // A read that fills nothing only indicates EOF if the buffer
+            // had capacity to begin with.
+            if buf.remaining() == rem && rem != 0 {
                 *me.done_first = true;
@@ -100,0 +102,0 @@ } else {

src/io/util/copy.rs

@@ -85,3 +85,3 @@ use crate::io::{AsyncRead, AsyncWrite, ReadBuf};
     {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         #[cfg(any(
@@ -88,0 +88,0 @@ feature = "fs",

src/io/util/empty.rs

@@ -74,3 +74,3 @@ use crate::io::util::poll_proceed_and_make_progress;
     ) -> Poll<io::Result<()>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -84,3 +84,3 @@ Poll::Ready(Ok(()))
     fn poll_fill_buf(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<&[u8]>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -101,3 +101,3 @@ Poll::Ready(Ok(&[]))
     ) -> Poll<io::Result<usize>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -109,3 +109,3 @@ Poll::Ready(Ok(buf.len()))
     fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -117,3 +117,3 @@ Poll::Ready(Ok(()))
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -134,3 +134,3 @@ Poll::Ready(Ok(()))
     ) -> Poll<Result<usize, io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -150,3 +150,3 @@ let num_bytes = bufs.iter().map(|b| b.len()).sum();
     fn poll_complete(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<u64>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -153,0 +153,0 @@ Poll::Ready(Ok(0))

src/io/util/lines.rs

@@ -52,3 +52,3 @@ use crate::io::util::read_line::read_line_internal;
     ///
-    /// This method is cancellation safe.
+    /// This method is cancel safe.
     ///
@@ -55,0 +55,0 @@ /// # Examples

src/io/util/mem.rs

@@ -188,3 +188,3 @@ //! In-process memory IO types.
 /// The `max_buf_size` argument is the maximum amount of bytes that can be
-/// written to a buffer before the it returns `Poll::Pending`.
+/// written to a buffer before it returns `Poll::Pending`.
 ///
@@ -221,3 +221,3 @@ /// # Unify reader and writer
     /// The `max_buf_size` argument is the maximum amount of bytes that can be
-    /// written to a buffer before the it returns `Poll::Pending`.
+    /// written to a buffer before it returns `Poll::Pending`.
     #[cfg_attr(docsrs, doc(cfg(feature = "io-util")))]
@@ -336,3 +336,3 @@ pub fn new_unsplit(max_buf_size: usize) -> SimplexStream {
         ) -> Poll<std::io::Result<()>> {
-            ready!(crate::trace::trace_leaf(cx));
+            ready!(crate::trace::trace_leaf());
             let coop = ready!(crate::task::coop::poll_proceed(cx));
@@ -354,3 +354,3 @@
         ) -> Poll<std::io::Result<()>> {
-            ready!(crate::trace::trace_leaf(cx));
+            ready!(crate::trace::trace_leaf());
             self.poll_read_internal(cx, buf)
@@ -368,3 +368,3 @@ }
         ) -> Poll<std::io::Result<usize>> {
-            ready!(crate::trace::trace_leaf(cx));
+            ready!(crate::trace::trace_leaf());
             let coop = ready!(crate::task::coop::poll_proceed(cx));
@@ -386,3 +386,3 @@
         ) -> Poll<std::io::Result<usize>> {
-            ready!(crate::trace::trace_leaf(cx));
+            ready!(crate::trace::trace_leaf());
             self.poll_write_internal(cx, buf)
@@ -398,3 +398,3 @@ }
         ) -> Poll<Result<usize, std::io::Error>> {
-            ready!(crate::trace::trace_leaf(cx));
+            ready!(crate::trace::trace_leaf());
             let coop = ready!(crate::task::coop::poll_proceed(cx));
@@ -416,3 +416,3 @@
         ) -> Poll<Result<usize, std::io::Error>> {
-            ready!(crate::trace::trace_leaf(cx));
+            ready!(crate::trace::trace_leaf());
             self.poll_write_vectored_internal(cx, bufs)
@@ -419,0 +419,0 @@ }

src/io/util/mod.rs

@@ -87,3 +87,3 @@ #![allow(unreachable_pub)] // https://github.com/rust-lang/rust/issues/57411
     // used by `BufReader` and `BufWriter`
-    // https://github.com/rust-lang/rust/blob/master/library/std/src/sys_common/io.rs#L1
+    // https://github.com/rust-lang/rust/blob/main/library/std/src/sys/io/mod.rs#L72
     const DEFAULT_BUF_SIZE: usize = 8 * 1024;
@@ -90,0 +90,0 @@

src/io/util/repeat.rs

@@ -59,3 +59,3 @@ use bytes::BufMut;
     ) -> Poll<io::Result<()>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -62,0 +62,0 @@ buf.put_bytes(self.byte, buf.remaining());

src/io/util/sink.rs

@@ -60,3 +60,3 @@ use crate::io::util::poll_proceed_and_make_progress;
     ) -> Poll<Result<usize, io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -68,3 +68,3 @@ Poll::Ready(Ok(buf.len()))
     fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -76,3 +76,3 @@ Poll::Ready(Ok(()))
     fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), io::Error>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         ready!(poll_proceed_and_make_progress(cx));
@@ -79,0 +79,0 @@ Poll::Ready(Ok(()))

src/io/util/vec_with_initialized.rs

@@ -8,3 +8,3 @@ use crate::io::ReadBuf;
 ///
-/// The implementor must guarantee that the vector returned by the
+/// The implementer must guarantee that the vector returned by the
 /// `as_mut` and `as_mut` methods do not change from one call to
@@ -11,0 +11,0 @@ /// another.

src/lib.rs

@@ -357,2 +357,3 @@ #![allow(
 //! - `tracing`: Enables tracing events.
+//! - `schedule-latency`: Allows measurement of task scheduling latencies.
 //! - `io-uring`: Enables `io-uring` (Linux only).
@@ -492,3 +493,8 @@ //! - `taskdump`: Enables `taskdump` (Linux only).
         target_os = "linux",
-        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        any(
+            target_arch = "aarch64",
+            target_arch = "x86",
+            target_arch = "x86_64",
+            target_arch = "s390x"
+        )
     ))
@@ -498,5 +504,14 @@ ))]
     "The `taskdump` feature is only currently supported on \
-linux, on `aarch64`, `x86` and `x86_64`."
+linux, on `aarch64`, `x86`, `x86_64` and `s390x`."
 );
 
+#[cfg(all(not(tokio_unstable), feature = "schedule-latency"))]
+compile_error!("The `schedule-latency` feature requires `--cfg tokio_unstable`.");
+
+#[cfg(all(
+    feature = "schedule-latency",
+    not(all(target_pointer_width = "64", target_has_atomic = "64"))
+))]
+compile_error!("The `schedule-latency` feature is only currently supported on 64-bit targets.");
+
 // Includes re-exports used by macros.
@@ -580,3 +595,3 @@ //
         #[allow(dead_code)]
-        pub(crate) fn trace_leaf(_: &mut std::task::Context<'_>) -> std::task::Poll<()> {
+        pub(crate) fn trace_leaf() -> std::task::Poll<()> {
             std::task::Poll::Ready(())
@@ -588,3 +603,3 @@ }
     pub(crate) async fn async_trace_leaf() {
-        std::future::poll_fn(trace_leaf).await
+        std::future::poll_fn(|_cx| trace_leaf()).await
     }
@@ -591,0 +606,0 @@ }

src/macros/cfg.rs

@@ -520,3 +520,4 @@ #![allow(unused_macros)]
                     target_arch = "x86",
-                    target_arch = "x86_64"
+                    target_arch = "x86_64",
+                    target_arch = "s390x"
                 )
@@ -534,3 +535,4 @@ ))]
                         target_arch = "x86",
-                        target_arch = "x86_64"
+                        target_arch = "x86_64",
+                        target_arch = "s390x"
                     )
@@ -555,3 +557,4 @@ )))
                     target_arch = "x86",
-                    target_arch = "x86_64"
+                    target_arch = "x86_64",
+                    target_arch = "s390x"
                 )
@@ -752,1 +755,20 @@ )))]
 }
+
+macro_rules! cfg_schedule_latency {
+    ($($item:item)*) => {
+        $(
+            #[cfg(feature = "schedule-latency")]
+            #[cfg_attr(docsrs, doc(cfg(feature = "schedule-latency")))]
+            $item
+        )*
+    };
+}
+
+macro_rules! cfg_not_schedule_latency {
+    ($($item:item)*) => {
+        $(
+            #[cfg(not(feature = "schedule-latency"))]
+            $item
+        )*
+    }
+}

src/macros/select.rs

@@ -98,2 +98,7 @@ macro_rules! doc {
         ///
+        /// Cancellation safety describes what happens when a future is dropped
+        /// before it completes. Whether something is cancellation safe depends on
+        /// the behavior of the future passed to `select!`, which may come from an
+        /// async method, an async expression, or another future-producing operation.
+        ///
         /// The following methods are cancellation safe:
@@ -100,0 +105,0 @@ ///

src/net/tcp/listener.rs

@@ -72,3 +72,3 @@ use crate::io::{Interest, PollEvented};
         ///
-        /// The address type can be any implementor of the [`ToSocketAddrs`] trait.
+        /// The address type can be any implementer of the [`ToSocketAddrs`] trait.
         /// If `addr` yields multiple addresses, bind will be attempted with each of
@@ -95,3 +95,2 @@ /// the addresses until one succeeds and returns the listener. If none of
         /// async fn main() -> io::Result<()> {
-        /// #   if cfg!(miri) { return Ok(()); } // No `socket` in miri.
         ///     let listener = TcpListener::bind("127.0.0.1:2345").await?;
@@ -139,4 +138,4 @@ ///
     ///
-    /// This method is cancel safe. If the method is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If the method is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, then it is guaranteed that no new connections were
@@ -143,0 +142,0 @@ /// accepted by this method.

src/net/tcp/socket.rs

@@ -289,2 +289,3 @@ use crate::net::{TcpListener, TcpStream};
         not(target_os = "cygwin"),
+        not(target_os = "nuttx"),
     ))]
@@ -298,2 +299,3 @@ #[cfg_attr(
             not(target_os = "cygwin"),
+            not(target_os = "nuttx"),
         )))
@@ -336,2 +338,3 @@ )]
         not(target_os = "cygwin"),
+        not(target_os = "nuttx"),
     ))]
@@ -345,2 +348,3 @@ #[cfg_attr(
             not(target_os = "cygwin"),
+            not(target_os = "nuttx"),
         )))
@@ -952,3 +956,2 @@ )]
     /// async fn main() -> std::io::Result<()> {
-    /// #   if cfg!(miri) { return Ok(()); } // No `socket` in miri.
     ///     let socket2_socket = Socket::new(Domain::IPV4, Type::STREAM, None)?;
@@ -955,0 +958,0 @@ ///     socket2_socket.set_nonblocking(true)?;

src/net/tcp/stream.rs

@@ -234,3 +234,2 @@ cfg_not_wasi! {
     /// async fn main() -> Result<(), Box<dyn Error>> {
-    /// #   if cfg!(miri) { return Ok(()); } // No `socket` in miri.
     ///     let mut data = [0u8; 12];
@@ -1079,4 +1078,4 @@ /// #   if false {
     ///
-    /// This method is cancel safe. If the method is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If the method is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, then it is guaranteed that no peek was performed, and
@@ -1083,0 +1082,0 @@ /// that `buf` has not been modified.

src/net/unix/datagram/socket.rs

@@ -42,3 +42,3 @@ use crate::io::{Interest, PollEvented, ReadBuf, Ready};
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -381,3 +381,3 @@ /// use tempfile::tempdir;
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -474,3 +474,3 @@ /// use tempfile::tempdir;
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -542,3 +542,3 @@ /// use std::os::unix::net::UnixDatagram as StdUDS;
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -583,3 +583,3 @@ /// use tempfile::tempdir;
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -620,4 +620,4 @@ /// use tempfile::tempdir;
     ///
-    /// This method is cancel safe. If `send` is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If `send` is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, then it is guaranteed that the message was not sent.
@@ -751,4 +751,4 @@ ///
     ///
-    /// This method is cancel safe. If `recv` is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If `recv` is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, it is guaranteed that no messages were received on this
@@ -913,3 +913,3 @@ /// socket.
         /// # async fn main() -> Result<(), Box<dyn Error>> {
-        /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+        /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
         /// use tokio::net::UnixDatagram;
@@ -1073,4 +1073,4 @@ /// use tempfile::tempdir;
     ///
-    /// This method is cancel safe. If `send_to` is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If `send_to` is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, then it is guaranteed that the message was not sent.
@@ -1083,3 +1083,3 @@ ///
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -1125,4 +1125,4 @@ /// use tempfile::tempdir;
     ///
-    /// This method is cancel safe. If `recv_from` is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If `recv_from` is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, it is guaranteed that no messages were received on this
@@ -1136,3 +1136,3 @@ /// socket.
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -1454,3 +1454,3 @@ /// use tempfile::tempdir;
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -1478,3 +1478,3 @@ /// use tempfile::tempdir;
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -1504,3 +1504,3 @@ ///
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -1553,3 +1553,3 @@ /// use tempfile::tempdir;
     /// # async fn main() -> Result<(), Box<dyn Error>> {
-    /// # if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// # if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     /// use tokio::net::UnixDatagram;
@@ -1556,0 +1556,0 @@ ///

src/net/unix/listener.rs

@@ -93,3 +93,19 @@ use crate::io::{Interest, PollEvented};
 
-        let listener = mio::net::UnixListener::bind_addr(&addr)?;
+        let addr = SocketAddr::from(addr);
+        UnixListener::bind_addr(&addr)
+    }
+
+    /// Creates a new `UnixListener` bound to the specified address.
+    ///
+    /// # Panics
+    ///
+    /// This function panics if it is not called from within a runtime with
+    /// IO enabled.
+    ///
+    /// The runtime is usually set implicitly when this function is called
+    /// from a future driven by a tokio runtime, otherwise runtime can be set
+    /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+    #[track_caller]
+    pub fn bind_addr(socket_addr: &SocketAddr) -> io::Result<UnixListener> {
+        let listener = mio::net::UnixListener::bind_addr(&socket_addr.0)?;
         let io = PollEvented::new(listener)?;
@@ -190,4 +206,4 @@ Ok(UnixListener { io })
     ///
-    /// This method is cancel safe. If the method is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If the method is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, then it is guaranteed that no new connections were
@@ -194,0 +210,0 @@ /// accepted by this method.

src/net/unix/stream.rs

@@ -72,2 +72,11 @@ use crate::io::{AsyncRead, AsyncWrite, Interest, PollEvented, ReadBuf, Ready};
     /// handle.
+    ///
+    /// # Panics
+    ///
+    /// This function panics if it is not called from within a runtime with
+    /// IO enabled.
+    ///
+    /// The runtime is usually set implicitly when this function is called
+    /// from a future driven by a tokio runtime, otherwise runtime can be set
+    /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
     pub async fn connect<P>(path: P) -> io::Result<UnixStream>
@@ -90,3 +99,22 @@ where
 
-        let stream = mio::net::UnixStream::connect_addr(&addr)?;
+        let addr = SocketAddr::from(addr);
+        UnixStream::connect_addr(&addr).await
+    }
+
+    /// Connects to the socket named by `socket_addr`.
+    ///
+    /// This function will create a new Unix socket and connect to the address
+    /// specified, associating the returned stream with the default event
+    /// loop's handle.
+    ///
+    /// # Panics
+    ///
+    /// This function panics if it is not called from within a runtime with
+    /// IO enabled.
+    ///
+    /// The runtime is usually set implicitly when this function is called
+    /// from a future driven by a tokio runtime, otherwise runtime can be set
+    /// explicitly with [`Runtime::enter`](crate::runtime::Runtime::enter) function.
+    pub async fn connect_addr(socket_addr: &SocketAddr) -> io::Result<UnixStream> {
+        let stream = mio::net::UnixStream::connect_addr(&socket_addr.0)?;
         let stream = UnixStream::new(stream)?;
@@ -853,3 +881,3 @@
     /// async fn main() -> Result<(), Box<dyn Error>> {
-    /// #   if cfg!(miri) { return Ok(()); } // No `socket` in miri.
+    /// #   if cfg!(miri) { return Ok(()); } // No Unix domain sockets in miri.
     ///     let dir = tempfile::tempdir().unwrap();
@@ -856,0 +884,0 @@ ///     let bind_path = dir.path().join("bind_path");

src/net/unix/ucred.rs

@@ -27,4 +27,6 @@ use crate::net::unix;
     ///
-    /// This is only implemented under Linux, Android, iOS, macOS, Solaris,
-    /// Illumos and Cygwin. On other platforms this will always return `None`.
+    /// This is implemented under Linux, Android, OpenBSD, FreeBSD (since
+    /// FreeBSD 13), NetBSD, NTO, iOS, macOS, tvOS, watchOS, visionOS,
+    /// Solaris, Illumos, Cygwin, Haiku, and Redox. On other platforms this
+    /// will always return `None`.
     pub fn pid(&self) -> Option<unix::pid_t> {
@@ -45,8 +47,11 @@ self.pid
 
-#[cfg(any(target_os = "netbsd", target_os = "nto"))]
+#[cfg(target_os = "netbsd")]
 pub(crate) use self::impl_netbsd::get_peer_cred;
 
-#[cfg(any(target_os = "dragonfly", target_os = "freebsd"))]
-pub(crate) use self::impl_bsd::get_peer_cred;
+#[cfg(target_os = "dragonfly")]
+pub(crate) use self::impl_dragonfly::get_peer_cred;
 
+#[cfg(target_os = "freebsd")]
+pub(crate) use self::impl_freebsd::get_peer_cred;
+
 #[cfg(any(
@@ -67,5 +72,13 @@ target_os = "macos",
 
-#[cfg(any(target_os = "espidf", target_os = "vita", target_os = "hurd"))]
+#[cfg(any(
+    target_os = "espidf",
+    target_os = "nuttx",
+    target_os = "vita",
+    target_os = "hurd"
+))]
 pub(crate) use self::impl_noproc::get_peer_cred;
 
+#[cfg(target_os = "nto")]
+pub(crate) use self::impl_nto::get_peer_cred;
+
 #[cfg(any(
@@ -178,4 +191,4 @@ target_os = "linux",
 
-#[cfg(any(target_os = "dragonfly", target_os = "freebsd"))]
-pub(crate) mod impl_bsd {
+#[cfg(target_os = "dragonfly")]
+pub(crate) mod impl_dragonfly {
     use crate::net::unix::{self, UnixStream};
@@ -210,2 +223,70 @@
 
+#[cfg(target_os = "freebsd")]
+pub(crate) mod impl_freebsd {
+    use crate::net::unix::{self, UnixStream};
+
+    use libc::{c_void, getsockopt, socklen_t, xucred, LOCAL_PEERCRED, XUCRED_VERSION};
+    use std::io;
+    use std::mem::{size_of, MaybeUninit};
+    use std::os::unix::io::AsRawFd;
+
+    pub(crate) fn get_peer_cred(sock: &UnixStream) -> io::Result<super::UCred> {
+        // `SOL_LOCAL` is not re-exported by `libc` for FreeBSD; it is defined
+        // as 0 in `<sys/un.h>`.
+        const SOL_LOCAL: libc::c_int = 0;
+
+        unsafe {
+            let raw_fd = sock.as_raw_fd();
+
+            let mut xucred = MaybeUninit::<xucred>::zeroed();
+            let mut len = size_of::<xucred>() as socklen_t;
+
+            let ret = getsockopt(
+                raw_fd,
+                SOL_LOCAL,
+                LOCAL_PEERCRED,
+                xucred.as_mut_ptr() as *mut c_void,
+                &mut len,
+            );
+
+            if ret != 0 {
+                return Err(io::Error::last_os_error());
+            }
+            if len as usize != size_of::<xucred>() {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    "unexpected xucred size from LOCAL_PEERCRED",
+                ));
+            }
+
+            let xucred = xucred.assume_init();
+
+            // Match `getpeereid(3)` and reject any `xucred` whose version we
+            // don't know how to interpret.
+            if xucred.cr_version != XUCRED_VERSION {
+                return Err(io::Error::new(
+                    io::ErrorKind::InvalidData,
+                    "unexpected xucred version from LOCAL_PEERCRED",
+                ));
+            }
+
+            // `cr_pid` is populated by the kernel since FreeBSD 13. PID 0 is
+            // the kernel scheduler and never a real userland peer, so we
+            // surface it as `None` rather than a misleading `Some(0)`.
+            let pid = match xucred.cr_pid__c_anonymous_union.cr_pid {
+                0 => None,
+                p => Some(p as unix::pid_t),
+            };
+
+            // `xucred` carries the effective uid in `cr_uid` and the effective
+            // gid in `cr_groups[0]`, matching what `getpeereid(2)` returns.
+            Ok(super::UCred {
+                uid: xucred.cr_uid as unix::uid_t,
+                gid: xucred.cr_groups[0] as unix::gid_t,
+                pid,
+            })
+        }
+    }
+}
+
 #[cfg(any(
@@ -325,3 +406,8 @@ target_os = "macos",
 
-#[cfg(any(target_os = "espidf", target_os = "vita", target_os = "hurd"))]
+#[cfg(any(
+    target_os = "espidf",
+    target_os = "nuttx",
+    target_os = "vita",
+    target_os = "hurd"
+))]
 pub(crate) mod impl_noproc {
@@ -339,1 +425,32 @@ use crate::net::unix::UnixStream;
 }
+
+#[cfg(target_os = "nto")]
+pub(crate) mod impl_nto {
+    use crate::net::unix::{self, UnixStream};
+
+    use libc::getpeereid;
+    use std::io;
+    use std::mem::MaybeUninit;
+    use std::os::unix::io::AsRawFd;
+
+    pub(crate) fn get_peer_cred(sock: &UnixStream) -> io::Result<super::UCred> {
+        unsafe {
+            let raw_fd = sock.as_raw_fd();
+
+            let mut uid = MaybeUninit::uninit();
+            let mut gid = MaybeUninit::uninit();
+
+            let ret = getpeereid(raw_fd, uid.as_mut_ptr(), gid.as_mut_ptr());
+
+            if ret == 0 {
+                Ok(super::UCred {
+                    uid: uid.assume_init() as unix::uid_t,
+                    gid: gid.assume_init() as unix::gid_t,
+                    pid: None,
+                })
+            } else {
+                Err(io::Error::last_os_error())
+            }
+        }
+    }
+}

src/process/mod.rs

@@ -1138,3 +1138,3 @@ //! An implementation of asynchronous process management for Tokio.
     fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         // Keep track of task budget
@@ -1141,0 +1141,0 @@ let coop = ready!(crate::task::coop::poll_proceed(cx));

src/runtime/config.rs

@@ -51,2 +51,5 @@ #![cfg_attr(
 
+    /// How to build schedule latency histograms
+    pub(crate) metrics_schedule_latency_histogram: Option<crate::runtime::HistogramBuilder>,
+
     #[cfg(tokio_unstable)]
@@ -53,0 +56,0 @@ /// How to respond to unhandled task panics.

src/runtime/context.rs

@@ -72,3 +72,8 @@ use crate::loom::thread::AccessError;
         target_os = "linux",
-        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        any(
+            target_arch = "aarch64",
+            target_arch = "x86",
+            target_arch = "x86_64",
+            target_arch = "s390x"
+        )
     ))]
@@ -117,3 +122,4 @@ trace: trace::Context,
                     target_arch = "x86",
-                    target_arch = "x86_64"
+                    target_arch = "x86_64",
+                    target_arch = "s390x"
                 )
@@ -120,0 +126,0 @@ ))]

src/runtime/driver/op.rs

 use crate::io::blocking::Buf;
 use crate::io::uring::open::Open;
 use crate::io::uring::read::Read;
+use crate::io::uring::rename::Rename;
 use crate::io::uring::utils::ArcFd;
@@ -9,6 +10,19 @@ use crate::io::uring::write::Write;
 
+#[cfg(
+    // libc::statx is only supported on these platforms
+    // FIXME: Add musl target env when our minimum supported
+    // rust version is 1.93. To clarify, statx support is
+    // introduced to musl in 1.25 as mentioned officially here:
+    // https://musl.libc.org/releases.html.
+    // However, rustup target_env building for *-linux-musl
+    // uses 1.25 musl on all *-linux-musl platforms starting
+    // in 1.93 stable rust version.
+    // https://blog.rust-lang.org/2025/12/05/Updating-musl-1.2.5/
+    any(target_env = "gnu", target_os = "android")
+)]
+use crate::io::uring::statx::Statx;
 use io_uring::cqueue;
 use io_uring::squeue::Entry;
 use std::future::Future;
-use std::io::{self, Error};
+use std::io;
 use std::mem;
@@ -26,4 +40,18 @@ use std::os::fd::OwnedFd;
     Write(Write),
+    Rename(Rename),
     ReadVec(Read<Vec<u8>, OwnedFd>),
     ReadBuf(Read<Buf, ArcFd>),
+    #[cfg(
+        // libc::statx is only supported on these platforms
+        // FIXME: Add musl target env when our minimum supported
+        // rust version is 1.93. To clarify, statx support is
+        // introduced to musl in 1.25 as mentioned officially here:
+        // https://musl.libc.org/releases.html.
+        // However, rustup target_env building for *-linux-musl
+        // uses 1.25 musl on all *-linux-musl platforms starting
+        // in 1.93 stable rust version.
+        // https://blog.rust-lang.org/2025/12/05/Updating-musl-1.2.5/
+        any(target_env = "gnu", target_os = "android")
+    )]
+    Statx(Statx),
 }
@@ -48,3 +76,3 @@
     /// The operation has completed with a single cqe result
-    Completed(io_uring::cqueue::Entry),
+    Completed(cqueue::Entry),
 }
@@ -129,3 +157,3 @@
     // upstream by embedding it in the `Output`.
-    fn complete_with_error(self, error: Error) -> Self::Output;
+    fn complete_with_error(self, error: io::Error) -> Self::Output;
 }
@@ -132,0 +160,0 @@

src/runtime/dump.rs

@@ -230,5 +230,6 @@ //! Snapshots of runtime state.
     /// Due to the way tracing is implemented, Tokio leaf futures will usually, instead of doing their
-    /// actual work, do the equivalent of a `yield_now` (returning a `Poll::Pending` and scheduling the
-    /// current context for execution), which means forward progress will probably not happen unless
-    /// you eventually call your future outside of `capture`.
+    /// actual work, return `Poll::Pending` without registering the task's waker with any driver.
+    /// This means forward progress will probably not happen unless you eventually call your future
+    /// outside of `capture`, or explicitly re-schedule the task (e.g. by calling
+    /// [`cx.waker().wake_by_ref()`][std::task::Waker::wake_by_ref]) after `capture` returns.
     ///
@@ -235,0 +236,0 @@ /// [`Handle::dump`]: crate::runtime::Handle::dump

src/runtime/handle.rs

@@ -358,3 +358,8 @@ use crate::runtime;
             target_os = "linux",
-            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+            any(
+                target_arch = "aarch64",
+                target_arch = "x86",
+                target_arch = "x86_64",
+                target_arch = "s390x"
+            )
         ))]
@@ -386,3 +391,8 @@ let future = super::task::trace::Trace::root(future);
             target_os = "linux",
-            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+            any(
+                target_arch = "aarch64",
+                target_arch = "x86",
+                target_arch = "x86_64",
+                target_arch = "s390x"
+            )
         ))]
@@ -416,3 +426,8 @@ let future = super::task::trace::Trace::root(future);
             target_os = "linux",
-            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+            any(
+                target_arch = "aarch64",
+                target_arch = "x86",
+                target_arch = "x86_64",
+                target_arch = "s390x"
+            )
         ))]
@@ -603,3 +618,3 @@ let future = super::task::trace::Trace::root(future);
         ///
-        /// Task dumps are supported on Linux atop `aarch64`, `x86` and `x86_64`.
+        /// Task dumps are supported on Linux atop `aarch64`, `x86`, `x86_64` and `s390x`.
         ///
@@ -606,0 +621,0 @@ /// ## Current Thread Runtime Requirements

src/runtime/io/driver.rs

@@ -236,2 +236,14 @@ // Signal handling
             ctx.dispatch_completions();
+
+            // There might be some cases where the CQ overflows, so we need to flush
+            // the remaining buffered CQEs.
+            while ctx
+                .uring
+                .as_mut()
+                .is_some_and(|uring| uring.submission().cq_overflow())
+            {
+                ctx.submit()
+                    .expect("failed to flush io_uring completion queue overflow");
+                ctx.dispatch_completions();
+            }
         }
@@ -238,0 +250,0 @@

src/runtime/io/registration_set.rs

@@ -23,3 +23,3 @@ use crate::loom::sync::atomic::AtomicUsize;
     // List of all registrations tracked by the set
-    registrations: LinkedList<Arc<ScheduledIo>, ScheduledIo>,
+    registrations: LinkedList<Arc<ScheduledIo>>,
 
@@ -26,0 +26,0 @@ // Registrations that are pending drop. When a `Registration` is dropped, it

src/runtime/io/registration.rs

@@ -149,3 +149,3 @@ #![cfg_attr(not(feature = "net"), allow(dead_code))]
     ) -> Poll<io::Result<ReadyEvent>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         // Keep track of task budget
@@ -152,0 +152,0 @@ let coop = ready!(crate::task::coop::poll_proceed(cx));

src/runtime/io/scheduled_io.rs

@@ -110,8 +110,6 @@ use crate::io::interest::Interest;
 
-type WaitList = LinkedList<Waiter, <Waiter as linked_list::Link>::Target>;
-
 #[derive(Debug, Default)]
 struct Waiters {
     /// List of all current waiters.
-    list: WaitList,
+    list: LinkedList<Waiter>,
 
@@ -118,0 +116,0 @@ /// Waker used for `AsyncRead`.

src/runtime/local_runtime/runtime.rs

@@ -237,3 +237,8 @@ #![allow(irrefutable_let_patterns)]
             target_os = "linux",
-            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+            any(
+                target_arch = "aarch64",
+                target_arch = "x86",
+                target_arch = "x86_64",
+                target_arch = "s390x"
+            )
         ))]
@@ -240,0 +245,0 @@ let future = crate::runtime::task::trace::Trace::root(future);

src/runtime/metrics/batch.rs

@@ -7,2 +7,3 @@ use crate::runtime::metrics::WorkerMetrics;
 
+use crate::runtime::metrics::ScheduleLatencyContext;
 use std::sync::atomic::Ordering::Relaxed;
@@ -57,2 +58,5 @@ use std::time::{Duration, Instant};
     poll_timer: Option<PollTimer>,
+
+    #[cfg(feature = "schedule-latency")]
+    schedule_latencies: Option<HistogramBatch>,
 }
@@ -100,2 +104,10 @@
                 });
+                // Schedule latencies cannot be tracked if `Instant::now()` is unavailable
+                #[cfg(feature = "schedule-latency")]
+                let schedule_latencies = maybe_now.and_then(|_| {
+                    worker_metrics
+                        .schedule_latency_histogram
+                        .as_ref()
+                        .map(HistogramBatch::from_histogram)
+                });
                 MetricsBatch {
@@ -114,2 +126,4 @@ park_count: 0,
                     poll_timer,
+                    #[cfg(feature = "schedule-latency")]
+                    schedule_latencies,
                 }
@@ -162,2 +176,8 @@ }
                 }
+
+                #[cfg(feature = "schedule-latency")]
+                if let Some(schedule_latencies) = &self.schedule_latencies {
+                    let dst = worker.schedule_latency_histogram.as_ref().unwrap();
+                    schedule_latencies.submit(dst);
+                }
             }
@@ -214,7 +234,13 @@ }
             /// Start polling an individual task
-            pub(crate) fn start_poll(&mut self) {}
+            pub(crate) fn start_poll(&mut self, _task_scheduled_at: Option<ScheduleLatencyContext>) {}
         },
         unstable: {
             /// Start polling an individual task
-            pub(crate) fn start_poll(&mut self) {
+            ///
+            /// # Arguments
+            ///
+            /// `task_scheduled_at` is used to calculate task schedule latency.
+            /// A `ScheduleLatencyContext` can be obtained by calling `prepare` on a task's
+            /// `ScheduleLatencyInstant`.
+            pub(crate) fn start_poll(&mut self, _task_scheduled_at: Option<ScheduleLatencyContext>) {
                 self.poll_count += 1;
@@ -224,2 +250,11 @@ if let Some(poll_timer) = &mut self.poll_timer {
                 }
+                #[cfg(feature = "schedule-latency")]
+                if let Some(task_scheduled_at) = _task_scheduled_at {
+                    if let Some(schedule_latencies) = &mut self.schedule_latencies {
+                        if let Some(now) = self.poll_timer.as_ref().map(|p| p.poll_started_at).or_else(now) {
+                            let elapsed = task_scheduled_at.elapsed_nanos(now);
+                            schedule_latencies.measure(elapsed, 1);
+                        }
+                    }
+                }
             }
@@ -226,0 +261,0 @@ }

src/runtime/metrics/mod.rs

@@ -41,1 +41,11 @@ //! This module contains information need to view information about how the
 }
+
+cfg_schedule_latency! {
+    mod schedule_latency;
+    pub(crate) use schedule_latency::{ScheduleLatencyInstant, ScheduleLatencyContext};
+}
+
+cfg_not_schedule_latency! {
+    mod schedule_latency_mock;
+    pub(crate) use schedule_latency_mock::{ScheduleLatencyInstant, ScheduleLatencyContext};
+}

src/runtime/metrics/runtime.rs

@@ -1033,2 +1033,190 @@ use crate::runtime::Handle;
     feature! {
+        #![feature = "schedule-latency"]
+        /// Returns `true` if the runtime is tracking the distribution of task
+        /// schedule latencies.
+        ///
+        /// Task schedule latencies are not instrumented by default as doing so
+        /// requires calling [`Instant::now()`] when a task is scheduled and when
+        /// it is polled. The feature is enabled by calling
+        /// [`enable_metrics_schedule_latency_histogram()`] when building the runtime.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime::{self, Handle};
+        ///
+        /// fn main() {
+        ///     runtime::Builder::new_current_thread()
+        ///         .enable_metrics_schedule_latency_histogram()
+        ///         .build()
+        ///         .unwrap()
+        ///         .block_on(async {
+        ///             let metrics = Handle::current().metrics();
+        ///             let enabled = metrics.schedule_latency_histogram_enabled();
+        ///
+        ///             println!("Tracking task schedule latency distribution: {:?}", enabled);
+        ///         });
+        /// }
+        /// ```
+        ///
+        /// [`enable_metrics_schedule_latency_histogram()`]: crate::runtime::Builder::enable_metrics_schedule_latency_histogram
+        /// [`Instant::now()`]: std::time::Instant::now
+        pub fn schedule_latency_histogram_enabled(&self) -> bool {
+            self.handle.inner.worker_metrics(0).schedule_latency_histogram.is_some()
+        }
+
+        /// Returns the number of histogram buckets tracking the distribution of
+        /// task schedule latencies.
+        ///
+        /// This value is configured by calling
+        /// [`metrics_schedule_latency_histogram_configuration()`] when building the runtime.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime::{self, Handle};
+        ///
+        /// fn main() {
+        ///     runtime::Builder::new_current_thread()
+        ///         .enable_metrics_schedule_latency_histogram()
+        ///         .build()
+        ///         .unwrap()
+        ///         .block_on(async {
+        ///             let metrics = Handle::current().metrics();
+        ///             let buckets = metrics.schedule_latency_histogram_num_buckets();
+        ///
+        ///             println!("Histogram buckets: {:?}", buckets);
+        ///         });
+        /// }
+        /// ```
+        ///
+        /// [`metrics_schedule_latency_histogram_configuration()`]: crate::runtime::Builder::metrics_schedule_latency_histogram_configuration
+        pub fn schedule_latency_histogram_num_buckets(&self) -> usize {
+            self.handle
+                .inner
+                .worker_metrics(0)
+                .schedule_latency_histogram
+                .as_ref()
+                .map(|histogram| histogram.num_buckets())
+                .unwrap_or_default()
+        }
+
+        /// Returns the range of task schedule latencies tracked by the given bucket.
+        ///
+        /// This value is configured by calling
+        /// [`metrics_schedule_latency_histogram_configuration()`] when building the runtime.
+        ///
+        /// # Panics
+        ///
+        /// The method panics if `bucket` represents an invalid bucket index, i.e.
+        /// is greater than or equal to `schedule_latency_histogram_num_buckets()`.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime::{self, Handle};
+        ///
+        /// fn main() {
+        ///     runtime::Builder::new_current_thread()
+        ///         .enable_metrics_schedule_latency_histogram()
+        ///         .build()
+        ///         .unwrap()
+        ///         .block_on(async {
+        ///             let metrics = Handle::current().metrics();
+        ///             let buckets = metrics.schedule_latency_histogram_num_buckets();
+        ///
+        ///             for i in 0..buckets {
+        ///                 let range = metrics.schedule_latency_histogram_bucket_range(i);
+        ///                 println!("Histogram bucket {} range: {:?}", i, range);
+        ///             }
+        ///         });
+        /// }
+        /// ```
+        ///
+        /// [`metrics_schedule_latency_histogram_configuration()`]: crate::runtime::Builder::metrics_schedule_latency_histogram_configuration
+        #[track_caller]
+        pub fn schedule_latency_histogram_bucket_range(&self, bucket: usize) -> Range<Duration> {
+            self.handle
+                .inner
+                .worker_metrics(0)
+                .schedule_latency_histogram
+                .as_ref()
+                .map(|histogram| {
+                    let range = histogram.bucket_range(bucket);
+                    std::ops::Range {
+                        start: Duration::from_nanos(range.start),
+                        end: Duration::from_nanos(range.end),
+                    }
+                })
+                .unwrap_or_default()
+        }
+
+        /// Returns the number of times the given worker polled tasks with a schedule
+        /// latency within the given bucket's range.
+        ///
+        /// Each worker maintains its own histogram and the counts for each bucket
+        /// starts at zero when the runtime is created. Each time the worker polls a
+        /// task, it tracks the time elapsed between when the task was scheduled and
+        /// when it was polled and increments the associated bucket by 1.
+        ///
+        /// Each bucket is a monotonically increasing counter. It is never
+        /// decremented or reset to zero.
+        ///
+        /// # Arguments
+        ///
+        /// `worker` is the index of the worker being queried. The given value must
+        /// be between 0 and `num_workers()`. The index uniquely identifies a single
+        /// worker and will continue to identify the worker throughout the lifetime
+        /// of the runtime instance.
+        ///
+        /// `bucket` is the index of the bucket being queried. The bucket is scoped
+        /// to the worker. The range represented by the bucket can be queried by
+        /// calling [`schedule_latency_histogram_bucket_range()`]. Each worker maintains
+        /// identical bucket ranges.
+        ///
+        /// # Panics
+        ///
+        /// The method panics when `worker` represents an invalid worker, i.e. is
+        /// greater than or equal to `num_workers()` or if `bucket` represents an
+        /// invalid bucket.
+        ///
+        /// # Examples
+        ///
+        /// ```
+        /// use tokio::runtime::{self, Handle};
+        ///
+        /// fn main() {
+        ///     runtime::Builder::new_current_thread()
+        ///         .enable_metrics_schedule_latency_histogram()
+        ///         .build()
+        ///         .unwrap()
+        ///         .block_on(async {
+        ///             let metrics = Handle::current().metrics();
+        ///             let buckets = metrics.schedule_latency_histogram_num_buckets();
+        ///
+        ///             for worker in 0..metrics.num_workers() {
+        ///                 for i in 0..buckets {
+        ///                     let range = metrics.schedule_latency_histogram_bucket_range(i);
+        ///                     let count = metrics.schedule_latency_histogram_bucket_count(worker, i);
+        ///                     println!("{} tasks encountered a scheduling latency between {}us and {}us", count, range.start.as_micros(), range.end.as_micros());
+        ///                 }
+        ///             }
+        ///         });
+        /// }
+        /// ```
+        ///
+        /// [`schedule_latency_histogram_bucket_range()`]: crate::runtime::RuntimeMetrics::schedule_latency_histogram_bucket_range
+        #[track_caller]
+        pub fn schedule_latency_histogram_bucket_count(&self, worker: usize, bucket: usize) -> u64 {
+            self.handle
+                .inner
+                .worker_metrics(worker)
+                .schedule_latency_histogram
+                .as_ref()
+                .map(|histogram| histogram.get(bucket))
+                .unwrap_or_default()
+        }
+    }
+
+    feature! {
         #![all(
@@ -1035,0 +1223,0 @@ tokio_unstable,

src/runtime/metrics/worker.rs

@@ -68,2 +68,6 @@ use crate::runtime::Config;
     pub(super) poll_count_histogram: Option<Histogram>,
+
+    #[cfg(feature = "schedule-latency")]
+    /// If `Some`, tracks the number of times tasks were scheduled by duration range.
+    pub(super) schedule_latency_histogram: Option<Histogram>,
 }
@@ -97,2 +101,10 @@
                     .map(|histogram_builder| histogram_builder.build());
+                #[cfg(feature = "schedule-latency")]
+                {
+                    worker_metrics.schedule_latency_histogram = config
+                        .metrics_schedule_latency_histogram
+                        .as_ref()
+                        .map(|histogram_builder| histogram_builder.build());
+                }
+
                 worker_metrics
@@ -99,0 +111,0 @@ }

src/runtime/mod.rs

@@ -239,2 +239,16 @@ //! The Tokio runtime.
 //!
+//! ## Unix `fork`
+//!
+//! User code that calls `fork(2)` without immediately calling `exec` must not
+//! reuse Tokio in the child process. Tokio supports this kind of fork only in
+//! two cases:
+//!
+//! - The fork happens before the parent process has used Tokio in any way.
+//! - The child process does not use Tokio after the fork.
+//!
+//! Creating or using a Tokio runtime in a child process after the parent has
+//! used Tokio is not supported, even if the runtime in the child is newly
+//! created. Some Tokio modules, including process and signal handling, use
+//! process-global state that cannot currently be reset after `fork`.
+//!
 //! [tasks]: crate::task
@@ -420,3 +434,3 @@ //! [`Runtime`]: Runtime
 
-#[cfg_attr(not(feature = "time"), allow(dead_code))]
+#[allow(dead_code)]
 #[derive(Debug, Copy, Clone, PartialEq)]
@@ -435,2 +449,4 @@ pub(crate) enum TimerFlavor {
 
+    use crate::time::Instant;
+
     use std::task::{Context, Poll};
@@ -448,13 +464,11 @@ use std::pin::Pin;
     impl Timer {
+        #[cfg_attr(not(all(tokio_unstable, feature = "rt-multi-thread")), allow(unused_variables))]
         #[track_caller]
-        pub(crate) fn new(
-            handle: crate::runtime::scheduler::Handle,
-            deadline: crate::time::Instant,
-        ) -> Self {
+        pub(crate) fn new(handle: scheduler::Handle, deadline: Instant) -> Self {
             match handle.timer_flavor() {
-                crate::runtime::TimerFlavor::Traditional => {
-                    Timer::Traditional(time::TimerEntry::new(handle, deadline))
+                TimerFlavor::Traditional => {
+                    Timer::Traditional(time::TimerEntry::new(handle))
                 }
                 #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-                crate::runtime::TimerFlavor::Alternative => {
+                TimerFlavor::Alternative => {
                     Timer::Alternative(time_alt::Timer::new(handle, deadline))
@@ -465,7 +479,12 @@ }
 
-        pub(crate) fn deadline(&self) -> crate::time::Instant {
-            match self {
-                Timer::Traditional(entry) => entry.deadline(),
+        pub(crate) fn init(self: Pin<&mut Self>, deadline: Instant) {
+            // Safety: we never move the inner entries.
+            let this = unsafe { self.get_unchecked_mut() };
+            match this {
+                // Safety: we never move the inner entries.
+                Timer::Traditional(entry) => unsafe {
+                    Pin::new_unchecked(entry).init(deadline)
+                }
                 #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-                Timer::Alternative(entry) => entry.deadline(),
+                Timer::Alternative(_) => {},
             }
@@ -482,24 +501,16 @@ }
 
-        pub(crate) fn flavor(self: Pin<&Self>) -> TimerFlavor {
-            match self.get_ref() {
-                Timer::Traditional(_) => TimerFlavor::Traditional,
-                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-                Timer::Alternative(_) => TimerFlavor::Alternative,
-            }
-        }
-
-        pub(crate) fn reset(
-            self: Pin<&mut Self>,
-            new_time: crate::time::Instant,
-            reregister: bool
-        ) {
+        #[cfg_attr(not(all(tokio_unstable, feature = "rt-multi-thread")), allow(unused_variables))]
+        pub(crate) fn reset(self: Pin<&mut Self>, handle: scheduler::Handle, deadline: Instant) {
             // Safety: we never move the inner entries.
             let this = unsafe { self.get_unchecked_mut() };
             match this {
-                Timer::Traditional(entry) => {
-                    // Safety: we never move the inner entries.
-                    unsafe { Pin::new_unchecked(entry).reset(new_time, reregister); }
+                // Safety: we never move the inner entries.
+                Timer::Traditional(entry) => unsafe {
+                    Pin::new_unchecked(entry).reset(deadline)
                 }
+                // Safety: we never move the inner entries.
                 #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-                Timer::Alternative(_) => panic!("not implemented yet"),
+                Timer::Alternative(entry) => unsafe {
+                    Pin::new_unchecked(entry).set(time_alt::Timer::new(handle, deadline))
+                },
             }
@@ -515,40 +526,13 @@ }
             match this {
-                Timer::Traditional(entry) => {
-                    // Safety: we never move the inner entries.
-                    unsafe { Pin::new_unchecked(entry).poll_elapsed(cx) }
+                // Safety: we never move the inner entries.
+                Timer::Traditional(entry) => unsafe {
+                    Pin::new_unchecked(entry).poll_elapsed(cx)
                 }
+                // Safety: we never move the inner entries.
                 #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-                Timer::Alternative(entry) => {
-                    // Safety: we never move the inner entries.
-                    unsafe { Pin::new_unchecked(entry).poll_elapsed(cx).map(Ok) }
+                Timer::Alternative(entry) => unsafe {
+                    Pin::new_unchecked(entry).poll_elapsed(cx).map(Ok)
                 }
             }
         }
-
-        #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-        pub(crate) fn scheduler_handle(&self) -> &crate::runtime::scheduler::Handle {
-            match self {
-                Timer::Traditional(_) => unreachable!("we should not call this on Traditional Timer"),
-                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-                Timer::Alternative(entry) => entry.scheduler_handle(),
-            }
-        }
-
-        #[cfg(all(tokio_unstable, feature = "tracing"))]
-        pub(crate) fn driver(self: Pin<&Self>) -> &crate::runtime::time::Handle {
-            match self.get_ref() {
-                Timer::Traditional(entry) => entry.driver(),
-                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-                Timer::Alternative(entry) => entry.driver(),
-            }
-        }
-
-        #[cfg(all(tokio_unstable, feature = "tracing"))]
-        pub(crate) fn clock(self: Pin<&Self>) -> &crate::time::Clock {
-            match self.get_ref() {
-                Timer::Traditional(entry) => entry.clock(),
-                #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-                Timer::Alternative(entry) => entry.clock(),
-            }
-        }
     }
@@ -555,0 +539,0 @@ }

src/runtime/runtime.rs

@@ -356,3 +356,8 @@ use super::BOX_FUTURE_THRESHOLD;
             target_os = "linux",
-            any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+            any(
+                target_arch = "aarch64",
+                target_arch = "x86",
+                target_arch = "x86_64",
+                target_arch = "s390x"
+            )
         ))]
@@ -359,0 +364,0 @@ let future = super::task::trace::Trace::root(future);

src/runtime/scheduler/current_thread/mod.rs

@@ -6,3 +6,4 @@ use crate::loom::sync::atomic::AtomicBool;
 use crate::runtime::task::{
-    self, JoinHandle, OwnedTasks, Schedule, SpawnLocation, Task, TaskHarnessScheduleHooks,
+    self, JoinHandle, LocalNotified, OwnedTasks, Schedule, SpawnLocation, Task,
+    TaskHarnessScheduleHooks,
 };
@@ -24,2 +25,3 @@ use crate::runtime::{
 use std::time::Duration;
+use std::time::Instant;
 use std::{fmt, thread};
@@ -105,2 +107,7 @@
     worker_metrics: WorkerMetrics,
+
+    /// Startup time of this scheduler.
+    ///
+    /// This instant is used as the basis of task `scheduled_at` measurements.
+    started_at: Option<Instant>,
 }
@@ -151,2 +158,7 @@
 
+        let started_at = config
+            .metrics_schedule_latency_histogram
+            .as_ref()
+            .map(|_| Instant::now());
+
         let handle = Arc::new(Handle {
@@ -169,2 +181,3 @@ name,
                 worker_metrics,
+                started_at,
             },
@@ -376,7 +389,24 @@ driver: driver_handle,
     /// thread-local context.
-    fn run_task<R>(&self, mut core: Box<Core>, f: impl FnOnce() -> R) -> (Box<Core>, R) {
-        core.metrics.start_poll();
-        let mut ret = self.enter(core, || crate::task::coop::budget(f));
-        ret.0.metrics.end_poll();
-        ret
+    fn run_task(&self, task: LocalNotified<Arc<Handle>>, mut core: Box<Core>) -> Box<Core> {
+        #[cfg(tokio_unstable)]
+        let task_meta = task.task_meta();
+
+        core.metrics.start_poll(
+            task.get_scheduled_at()
+                .prepare(self.handle.shared.started_at),
+        );
+
+        let (mut c, ()) = self.enter(core, || {
+            crate::task::coop::budget(|| {
+                #[cfg(tokio_unstable)]
+                self.handle.task_hooks.poll_start_callback(&task_meta);
+
+                task.run();
+
+                #[cfg(tokio_unstable)]
+                self.handle.task_hooks.poll_stop_callback(&task_meta);
+            })
+        });
+        c.metrics.end_poll();
+        c
     }
@@ -549,3 +579,8 @@
         target_os = "linux",
-        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+        any(
+            target_arch = "aarch64",
+            target_arch = "x86",
+            target_arch = "x86_64",
+            target_arch = "s390x"
+        )
     ))]
@@ -655,2 +690,3 @@ pub(crate) fn dump(&self) -> crate::runtime::Dump {
 
+use crate::runtime::metrics::ScheduleLatencyInstant;
 use std::num::NonZeroU64;
@@ -684,2 +720,11 @@
 
+        if self
+            .shared
+            .config
+            .metrics_schedule_latency_histogram
+            .is_some()
+        {
+            task.set_scheduled_at(ScheduleLatencyInstant::new(self.shared.started_at));
+        }
+
         context::with_scheduler(|maybe_cx| match maybe_cx {
@@ -834,15 +879,4 @@ Some(CurrentThread(cx)) if Arc::ptr_eq(self, &cx.handle) => {
 
-                    #[cfg(tokio_unstable)]
-                    let task_meta = task.task_meta();
+                    let c = context.run_task(task, core);
 
-                    let (c, ()) = context.run_task(core, || {
-                        #[cfg(tokio_unstable)]
-                        context.handle.task_hooks.poll_start_callback(&task_meta);
-
-                        task.run();
-
-                        #[cfg(tokio_unstable)]
-                        context.handle.task_hooks.poll_stop_callback(&task_meta);
-                    });
-
                     core = c;
@@ -849,0 +883,0 @@ }

src/runtime/scheduler/mod.rs

@@ -123,14 +123,2 @@ cfg_rt! {
         #[cfg(all(tokio_unstable, feature = "rt-multi-thread", feature = "time"))]
-        /// Returns true if both handles belong to the same runtime instance.
-        pub(crate) fn is_same_runtime(&self, other: &Handle) -> bool {
-            match (self, other) {
-                (Handle::CurrentThread(a), Handle::CurrentThread(b)) => Arc::ptr_eq(a, b),
-                #[cfg(feature = "rt-multi-thread")]
-                (Handle::MultiThread(a), Handle::MultiThread(b)) => Arc::ptr_eq(a, b),
-                #[cfg(feature = "rt-multi-thread")]
-                _ => false, // different runtime types
-            }
-        }
-
-        #[cfg(all(tokio_unstable, feature = "rt-multi-thread", feature = "time"))]
         /// Returns true if the runtime is shutting down.
@@ -306,13 +294,2 @@ pub(crate) fn is_shutdown(&self) -> bool {
 
-        #[cfg(all(tokio_unstable, feature = "time", feature = "rt-multi-thread"))]
-        pub(crate) fn with_time_temp_local_context<F, R>(&self, f: F) -> R
-        where
-            F: FnOnce(Option<crate::runtime::time_alt::TempLocalContext<'_>>) -> R,
-        {
-            match self {
-                Context::CurrentThread(_) => panic!("the alternative timer implementation is not supported on CurrentThread runtime"),
-                Context::MultiThread(context) => context.with_time_temp_local_context(f),
-            }
-        }
-
         cfg_rt_multi_thread! {
@@ -319,0 +296,0 @@ #[track_caller]

src/runtime/scheduler/multi_thread/queue.rs

@@ -86,12 +86,8 @@ //! Run-queue structures to support a work-stealing scheduler
 pub(crate) fn local<T: 'static>() -> (Steal<T>, Local<T>) {
-    let mut buffer = Vec::with_capacity(LOCAL_QUEUE_CAPACITY);
+    let buffer = std::iter::repeat_with(|| UnsafeCell::new(MaybeUninit::uninit()));
 
-    for _ in 0..LOCAL_QUEUE_CAPACITY {
-        buffer.push(UnsafeCell::new(MaybeUninit::uninit()));
-    }
-
     let inner = Arc::new(Inner {
         head: AtomicUnsignedLong::new(0),
         tail: AtomicUnsignedShort::new(0),
-        buffer: make_fixed_size(buffer.into_boxed_slice()),
+        buffer: make_fixed_size(buffer.take(LOCAL_QUEUE_CAPACITY).collect()),
     });
@@ -98,0 +94,0 @@

src/runtime/scheduler/multi_thread/stats.rs

@@ -0,1 +1,2 @@
+use crate::runtime::metrics::ScheduleLatencyContext;
 use crate::runtime::{Config, MetricsBatch, WorkerMetrics};
@@ -116,4 +117,4 @@
 
-    pub(crate) fn start_poll(&mut self) {
-        self.batch.start_poll();
+    pub(crate) fn start_poll(&mut self, task_scheduled_at: Option<ScheduleLatencyContext>) {
+        self.batch.start_poll(task_scheduled_at);
 
@@ -120,0 +121,0 @@ self.tasks_polled_in_batch += 1;

src/runtime/scheduler/multi_thread/worker.rs

@@ -77,3 +77,3 @@ //! A scheduler is initialized with a fixed number of workers. Each worker is
 use std::thread;
-use std::time::Duration;
+use std::time::{Duration, Instant};
 
@@ -96,2 +96,3 @@ mod metrics;
 
+use crate::runtime::metrics::ScheduleLatencyInstant;
 #[cfg(all(tokio_unstable, feature = "time"))]
@@ -207,2 +208,7 @@ use crate::runtime::scheduler::util;
 
+    /// Startup time of this scheduler.
+    ///
+    /// This instant is used as the basis of task `scheduled_at` measurements.
+    started_at: Option<Instant>,
+
     /// Only held to trigger some code on drop. This is used to get internal
@@ -317,2 +323,6 @@ /// runtime metrics that can be useful when doing performance
     let (inject, inject_synced) = inject::Shared::new();
+    let started_at = config
+        .metrics_schedule_latency_histogram
+        .as_ref()
+        .map(|_| Instant::now());
 
@@ -339,2 +349,3 @@ let remotes_len = remotes.len();
             worker_metrics: worker_metrics.into_boxed_slice(),
+            started_at,
             _counters: Counters,
@@ -674,3 +685,6 @@ },
         // purposes. These tasks inherent the "parent"'s limits.
-        core.stats.start_poll();
+        core.stats.start_poll(
+            task.get_scheduled_at()
+                .prepare(self.worker.handle.shared.started_at),
+        );
 
@@ -1334,2 +1348,11 @@ // Make the core available to the runtime context
     pub(super) fn schedule_task(&self, task: Notified, is_yield: bool) {
+        if self
+            .shared
+            .config
+            .metrics_schedule_latency_histogram
+            .is_some()
+        {
+            task.set_scheduled_at(ScheduleLatencyInstant::new(self.shared.started_at));
+        }
+
         with_current(|maybe_cx| {
@@ -1336,0 +1359,0 @@ if let Some(cx) = maybe_cx {

src/runtime/task/core.rs

@@ -24,2 +24,3 @@ //! Core task module.
 use crate::runtime::context;
+use crate::runtime::metrics::ScheduleLatencyInstant;
 use crate::runtime::task::raw::{self, Vtable};
@@ -195,2 +196,5 @@ use crate::runtime::task::state::State;
     pub(super) tracing_id: Option<tracing::Id>,
+
+    /// The last time this task was scheduled. Used to measure schedule latency.
+    pub(super) scheduled_at: UnsafeCell<ScheduleLatencyInstant>,
 }
@@ -252,2 +256,3 @@
                 tracing_id,
+                scheduled_at: UnsafeCell::new(ScheduleLatencyInstant::new(None)),
             }
@@ -540,2 +545,19 @@ }
     }
+
+    /// Updates the last time this task was scheduled. Used to calculate
+    /// the time elapsed between task scheduling and polling.
+    ///
+    /// # Safety
+    ///
+    /// The caller must guarantee exclusive access to this field.
+    pub(super) unsafe fn set_scheduled_at(&self, scheduled_at: ScheduleLatencyInstant) {
+        self.scheduled_at.with_mut(|ptr| *ptr = scheduled_at);
+    }
+
+    /// Gets the last time this task was scheduled.
+    pub(super) fn get_scheduled_at(&self) -> ScheduleLatencyInstant {
+        // Safety: If there are concurrent writes, then that write has violated
+        // the safety requirements on `set_scheduled_at`.
+        unsafe { self.scheduled_at.with(|ptr| *ptr) }
+    }
 }
@@ -542,0 +564,0 @@

src/runtime/task/join.rs

@@ -31,5 +31,5 @@ use crate::runtime::task::{AbortHandle, Header, RawTask};
     ///
-    /// The `&mut JoinHandle<T>` type is cancel safe. If it is used as the event
-    /// in a `tokio::select!` statement and some other branch completes first,
-    /// then it is guaranteed that the output of the task is not lost.
+    /// Awaiting a `&mut JoinHandle<T>` is cancel safe. If it is used as a
+    /// branch in `tokio::select!` and another branch completes first, then it
+    /// is guaranteed that the output of the task is not lost.
     ///
@@ -329,3 +329,3 @@ /// If a `JoinHandle` is dropped, then the task continues running in the
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         let mut ret = Poll::Pending;
@@ -332,0 +332,0 @@

src/runtime/task/list.rs

@@ -12,4 +12,4 @@ //! This module has containers for storing the tasks spawned on a scheduler. The
 use crate::runtime::task::{JoinHandle, LocalNotified, Notified, Schedule, SpawnLocation, Task};
-use crate::util::linked_list::{Link, LinkedList};
-use crate::util::sharded_list;
+use crate::util::linked_list::LinkedList;
+use crate::util::sharded_list::ShardedList;
 
@@ -60,3 +60,3 @@ use crate::loom::sync::atomic::{AtomicBool, Ordering};
 pub(crate) struct OwnedTasks<S: 'static> {
-    list: List<S>,
+    list: ShardedList<Task<S>>,
     pub(crate) id: NonZeroU64,
@@ -66,4 +66,2 @@ closed: AtomicBool,
 
-type List<S> = sharded_list::ShardedList<Task<S>, <Task<S> as Link>::Target>;
-
 pub(crate) struct LocalOwnedTasks<S: 'static> {
@@ -76,3 +74,3 @@ inner: UnsafeCell<OwnedTasksInner<S>>,
 struct OwnedTasksInner<S: 'static> {
-    list: LinkedList<Task<S>, <Task<S> as Link>::Target>,
+    list: LinkedList<Task<S>>,
     closed: bool,
@@ -85,3 +83,3 @@ }
         Self {
-            list: List::new(shard_size),
+            list: ShardedList::new(shard_size),
             closed: AtomicBool::new(false),
@@ -88,0 +86,0 @@ id: get_next_id(),

src/runtime/task/mod.rs

@@ -224,2 +224,3 @@ //! The task module.
 
+use crate::runtime::metrics::ScheduleLatencyInstant;
 use crate::runtime::TaskCallback;
@@ -251,2 +252,11 @@ use std::marker::PhantomData;
     }
+
+    pub(crate) fn set_scheduled_at(&self, scheduled_at: ScheduleLatencyInstant) {
+        // SAFETY: There are no concurrent writes because there is only ever one `Notified`
+        // reference per task. There are no concurrent reads because this field is only read
+        // when polling the task, which can only happen after it's scheduled.
+        unsafe {
+            self.0.header().set_scheduled_at(scheduled_at);
+        }
+    }
 }
@@ -273,2 +283,6 @@
     }
+
+    pub(crate) fn get_scheduled_at(&self) -> ScheduleLatencyInstant {
+        self.task.header().get_scheduled_at()
+    }
 }
@@ -406,11 +420,6 @@
 
-    #[cfg(all(
-        tokio_unstable,
-        feature = "taskdump",
-        feature = "rt",
-        target_os = "linux",
-        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
-    ))]
-    pub(super) fn as_raw(&self) -> RawTask {
-        self.raw
+    cfg_taskdump! {
+        pub(super) fn as_raw(&self) -> RawTask {
+            self.raw
+        }
     }
@@ -516,2 +525,11 @@
     }
+
+    cfg_taskdump! {
+        /// Returns a `WakerRef` borrowing from this task.
+        ///
+        /// `WakerRef` derefs to `Waker` without bumping the task's refcount.
+        pub(crate) fn waker_ref(&self) -> waker::WakerRef<'_, S> {
+            waker::waker_ref::<S>(self.task.raw.header_ptr_ref())
+        }
+    }
 }
@@ -518,0 +536,0 @@

src/runtime/task/raw.rs

@@ -245,2 +245,8 @@ // It doesn't make sense to enforce `unsafe_op_in_unsafe_fn` for this module because
 
+    cfg_taskdump! {
+        pub(super) fn header_ptr_ref(&self) -> &NonNull<Header> {
+            &self.ptr
+        }
+    }
+
     pub(super) fn trailer_ptr(&self) -> NonNull<Trailer> {
@@ -247,0 +253,0 @@ unsafe { Header::get_trailer(self.ptr) }

src/runtime/task/state.rs

@@ -280,24 +280,19 @@ use crate::loom::sync::atomic::AtomicUsize;
 
-    /// Transitions the state to `NOTIFIED`, unconditionally increasing the ref
-    /// count.
-    ///
-    /// Returns `true` if the notified bit was transitioned from `0` to `1`;
-    /// otherwise `false.`
-    #[cfg(all(
-        tokio_unstable,
-        feature = "taskdump",
-        feature = "rt",
-        target_os = "linux",
-        any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
-    ))]
-    pub(super) fn transition_to_notified_for_tracing(&self) -> bool {
-        self.fetch_update_action(|mut snapshot| {
-            if snapshot.is_notified() {
-                (false, None)
-            } else {
-                snapshot.set_notified();
-                snapshot.ref_inc();
-                (true, Some(snapshot))
-            }
-        })
+    cfg_taskdump! {
+        /// Transitions the state to `NOTIFIED`, unconditionally increasing the ref
+        /// count.
+        ///
+        /// Returns `true` if the notified bit was transitioned from `0` to `1`;
+        /// otherwise `false.`
+        pub(super) fn transition_to_notified_for_tracing(&self) -> bool {
+            self.fetch_update_action(|mut snapshot| {
+                if snapshot.is_notified() {
+                    (false, None)
+                } else {
+                    snapshot.set_notified();
+                    snapshot.ref_inc();
+                    (true, Some(snapshot))
+                }
+            })
+        }
     }
@@ -304,0 +299,0 @@

src/runtime/task/trace/mod.rs

@@ -260,3 +260,5 @@ use crate::loom::sync::Arc;
     {
-        trace_impl::capture(f)
+        let mut trace = Trace::empty();
+        let result = trace_with(f, |meta| trace_impl::trace_leaf(meta, &mut trace));
+        (result, trace)
     }
@@ -283,8 +285,8 @@
 
-/// If this is a sub-invocation of [`Trace::capture`], capture a backtrace.
+/// If this is a sub-invocation of [`trace_with`], capture a backtrace.
 ///
-/// The captured backtrace will be returned by [`Trace::capture`].
+/// The captured backtrace will be returned by [`trace_with`].
 ///
 /// Invoking this function does nothing when it is not a sub-invocation
-/// [`Trace::capture`].
+/// [`trace_with`].
 // This function is marked `#[inline(never)]` to ensure that it gets a distinct `Frame` in the
@@ -294,3 +296,3 @@ // backtrace, below which frames should not be included in the backtrace (since they reflect the
 #[inline(never)]
-pub(crate) fn trace_leaf(cx: &mut task::Context<'_>) -> Poll<()> {
+pub(crate) fn trace_leaf() -> Poll<()> {
     let root_addr = Context::current_frame_addr();
@@ -304,14 +306,2 @@
         leaf_fn(&meta);
-
-        // Use the same logic that `yield_now` uses to send out wakeups after
-        // the task yields.
-        context::with_scheduler(|scheduler| {
-            if let Some(scheduler) = scheduler {
-                match scheduler {
-                    scheduler::Context::CurrentThread(s) => s.defer.defer(cx.waker()),
-                    #[cfg(feature = "rt-multi-thread")]
-                    scheduler::Context::MultiThread(s) => s.defer.defer(cx.waker()),
-                }
-            }
-        });
     });
@@ -466,2 +456,23 @@
             let id = local_notified.task.id();
+
+            // Re-enqueue the task's waker on the scheduler's defer queue so
+            // the task is polled again after the dump completes. This is the
+            // same mechanism `yield_now` uses; the defer queue is drained
+            // after `trace_current_thread` / `trace_multi_thread` returns.
+            //
+            // We do this before polling so the borrow of the task ends before
+            // the `LocalNotified` is consumed in `run()`. `defer` clones the
+            // waker into its own queue, so the deferred entry outlives the
+            // `WakerRef` here.
+            let waker_ref = local_notified.waker_ref();
+            context::with_scheduler(|scheduler| {
+                if let Some(scheduler) = scheduler {
+                    match scheduler {
+                        scheduler::Context::CurrentThread(s) => s.defer.defer(&waker_ref),
+                        #[cfg(feature = "rt-multi-thread")]
+                        scheduler::Context::MultiThread(s) => s.defer.defer(&waker_ref),
+                    }
+                }
+            });
+
             let ((), trace) = Trace::capture(|| local_notified.run());
@@ -468,0 +479,0 @@ (id, trace)

src/runtime/task/trace/trace_impl.rs

@@ -7,17 +7,4 @@ //! Current `backtrace::trace` + collector based backtrace implementation
 
-use crate::runtime::task::trace::{trace_with, Trace, TraceMeta};
+use crate::runtime::task::trace::{Trace, TraceMeta};
 
-/// Capture using the default `backtrace::trace`-based implementation.
-#[inline(never)]
-pub(super) fn capture<F, R>(f: F) -> (R, Trace)
-where
-    F: FnOnce() -> R,
-{
-    let mut trace = Trace::empty();
-
-    let result = trace_with(f, |meta| trace_leaf(meta, &mut trace));
-
-    (result, trace)
-}
-
 /// Capture a backtrace via `backtrace::trace` and collect it into `trace`.
@@ -24,0 +11,0 @@ pub(crate) fn trace_leaf(meta: &TraceMeta, trace: &mut Trace) {

src/runtime/task/waker.rs

@@ -9,3 +9,3 @@ use crate::runtime::task::{Header, RawTask, Schedule};
 
-pub(super) struct WakerRef<'a, S: 'static> {
+pub(crate) struct WakerRef<'a, S: 'static> {
     waker: ManuallyDrop<Waker>,
@@ -12,0 +12,0 @@ _p: PhantomData<(&'a Header, S)>,

src/runtime/time_alt/cancellation_queue.rs

@@ -1,10 +0,8 @@
-use super::{CancellationQueueEntry, Entry, EntryHandle};
+use super::{CancellationQueueEntry, EntryHandle};
 use crate::loom::sync::{Arc, Mutex};
-use crate::util::linked_list;
+use crate::util::linked_list::LinkedList;
 
-type EntryList = linked_list::LinkedList<CancellationQueueEntry, Entry>;
-
 #[derive(Debug, Default)]
 struct Inner {
-    list: EntryList,
+    list: LinkedList<CancellationQueueEntry>,
 }
@@ -24,3 +22,3 @@
         Self {
-            list: EntryList::new(),
+            list: LinkedList::new(),
         }
@@ -27,0 +25,0 @@ }

src/runtime/time_alt/cancellation_queue/tests.rs

 use super::*;
 
-use futures::task::noop_waker;
-
 #[cfg(loom)]
@@ -12,3 +10,3 @@ const NUM_ITEMS: usize = 16;
 fn new_handle() -> EntryHandle {
-    EntryHandle::new(0, noop_waker())
+    EntryHandle::new(0)
 }
@@ -15,0 +13,0 @@

src/runtime/time_alt/context.rs

@@ -28,3 +28,2 @@ use super::{cancellation_queue, RegistrationQueue, Wheel};
         registration_queue: &'a mut RegistrationQueue,
-        elapsed: u64,
     },
@@ -40,3 +39,2 @@ #[cfg(feature = "rt-multi-thread")]
             registration_queue: &mut cx.registration_queue,
-            elapsed: cx.wheel.elapsed(),
         }
@@ -43,0 +41,0 @@ }

src/runtime/time_alt/entry.rs

@@ -7,7 +7,5 @@ use super::cancellation_queue::Sender;
 use std::ptr::NonNull;
-use std::task::Waker;
+use std::task::{Context, Poll, Waker};
 
-pub(super) type EntryList = linked_list::LinkedList<Entry, Entry>;
-
-#[derive(Debug)]
+#[derive(Debug, Default)]
 struct State {
@@ -38,3 +36,3 @@ cancelled: bool,
     /// the scheduler removes the entry from the [`RegistrationQueue`]
-    /// [`RegistrationQueue`] and insert it into the [`Wheel`].
+    /// and insert it into the [`Wheel`].
     ///
@@ -191,10 +189,3 @@ /// Finally, after parking the resource driver, the scheduler removes
 impl Handle {
-    pub(crate) fn new(deadline: u64, waker: Waker) -> Self {
-        let state = State {
-            cancelled: false,
-            woken_up: false,
-            waker: Some(waker),
-            cancel_tx: None,
-        };
-
+    pub(crate) fn new(deadline: u64) -> Self {
         let entry = Arc::new(Entry {
@@ -204,3 +195,3 @@ cancel_pointers: linked_list::Pointers::new(),
             deadline,
-            state: Mutex::new(state),
+            state: Mutex::new(State::default()),
             _pin: PhantomPinned,
@@ -235,10 +226,20 @@ });
 
-    pub(crate) fn register_waker(&self, waker: Waker) {
+    pub(crate) fn poll(&self, cx: &mut Context<'_>) -> Poll<()> {
         let mut lock = self.entry.state.lock();
-        if !lock.cancelled && !lock.woken_up {
-            let maybe_old_waker = lock.waker.replace(waker);
-            // unlock before calling waker
-            drop(lock);
-            drop(maybe_old_waker);
+        if lock.woken_up {
+            return Poll::Ready(());
+        } else if lock.cancelled {
+            return Poll::Pending;
         }
+        // PANIC: no intermediary state is possible should the user-controllable `Waker`
+        // panic on `Clone` or `Drop`.
+        let maybe_old_waker = match &lock.waker {
+            Some(current_waker) if current_waker.will_wake(cx.waker()) => None,
+            _ => lock.waker.replace(cx.waker().clone()),
+        };
+        // unlock before dropping waker
+        drop(lock);
+        drop(maybe_old_waker);
+
+        Poll::Pending
     }
@@ -245,0 +246,0 @@

src/runtime/time_alt/mod.rs

@@ -8,4 +8,3 @@ pub(crate) mod context;
 pub(crate) use entry::Handle as EntryHandle;
-use entry::{CancellationQueueEntry, RegistrationQueueEntry, WakeQueueEntry};
-use entry::{Entry, EntryList};
+use entry::{CancellationQueueEntry, Entry, RegistrationQueueEntry, WakeQueueEntry};
 
@@ -12,0 +11,0 @@ mod registration_queue;

src/runtime/time_alt/registration_queue.rs

@@ -1,10 +0,8 @@
-use super::{Entry, EntryHandle, RegistrationQueueEntry};
-use crate::util::linked_list;
+use super::{EntryHandle, RegistrationQueueEntry};
+use crate::util::linked_list::LinkedList;
 
-type EntryList = linked_list::LinkedList<RegistrationQueueEntry, Entry>;
-
 /// A queue of entries that need to be registered in the timer wheel.
 #[derive(Debug)]
 pub(crate) struct RegistrationQueue {
-    list: EntryList,
+    list: LinkedList<RegistrationQueueEntry>,
 }
@@ -24,3 +22,3 @@
         Self {
-            list: EntryList::new(),
+            list: LinkedList::new(),
         }
@@ -33,3 +31,3 @@ }
     ///
-    /// - [`Entry::extra_pointers`] of `hdl` must not being used.
+    /// - `Entry::extra_pointers` of `hdl` must not being used.
     pub(crate) unsafe fn push_front(&mut self, hdl: EntryHandle) {
@@ -36,0 +34,0 @@ self.list.push_front(hdl);

src/runtime/time_alt/registration_queue/tests.rs

 use super::*;
 
-use futures::task::noop_waker;
-
 #[cfg(loom)]
@@ -12,3 +10,3 @@ const NUM_ITEMS: usize = 16;
 fn new_handle() -> EntryHandle {
-    EntryHandle::new(0, noop_waker())
+    EntryHandle::new(0)
 }
@@ -15,0 +13,0 @@

src/runtime/time_alt/tests.rs

 use super::*;
 use crate::loom::thread;
 
+use std::task::Context;
+
 use futures_test::task::{new_count_waker, AwokenCount};
@@ -14,3 +16,5 @@
     let (waker, count) = new_count_waker();
-    (EntryHandle::new(0, waker), count)
+    let entry = EntryHandle::new(0);
+    _ = entry.poll(&mut Context::from_waker(&waker));
+    (entry, count)
 }
@@ -17,0 +21,0 @@

src/runtime/time_alt/timer.rs

 use super::{EntryHandle, TempLocalContext};
-use crate::runtime::scheduler::Handle as SchedulerHandle;
+use crate::runtime::scheduler;
 use crate::time::Instant;
@@ -12,12 +12,4 @@
 pub(crate) struct Timer {
-    sched_handle: SchedulerHandle,
-
     /// The entry in the timing wheel.
-    ///
-    /// - `Some` if the timer is registered / pending / woken up / cancelling.
-    /// - `None` if the timer is unregistered.
-    entry: Option<EntryHandle>,
-
-    /// The deadline for the timer.
-    deadline: Instant,
+    entry: EntryHandle,
 }
@@ -27,5 +19,3 @@
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("Timer")
-            .field("deadline", &self.deadline)
-            .finish()
+        f.debug_struct("Timer").finish()
     }
@@ -36,5 +26,3 @@ }
     fn drop(&mut self) {
-        if let Some(entry) = self.entry.take() {
-            entry.cancel();
-        }
+        self.entry.cancel();
     }
@@ -45,83 +33,33 @@ }
     #[track_caller]
-    pub(crate) fn new(sched_hdl: SchedulerHandle, deadline: Instant) -> Self {
-        // Panic if the time driver is not enabled
-        let _ = sched_hdl.driver().time();
-        Timer {
-            sched_handle: sched_hdl,
-            entry: None,
-            deadline,
-        }
-    }
-
-    pub(crate) fn deadline(&self) -> Instant {
-        self.deadline
-    }
-
-    pub(crate) fn is_elapsed(&self) -> bool {
-        self.entry.as_ref().is_some_and(|entry| entry.is_woken_up())
-    }
-
-    fn register(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
-        let this = self.get_mut();
-
-        with_current_temp_local_context(&this.sched_handle, |maybe_time_cx| {
-            let deadline = deadline_to_tick(&this.sched_handle, this.deadline);
-
-            match maybe_time_cx {
-                Some(TempLocalContext::Running {
-                    registration_queue: _,
-                    elapsed,
-                }) if deadline <= elapsed => Poll::Ready(()),
-
-                Some(TempLocalContext::Running {
-                    registration_queue,
-                    elapsed: _,
-                }) => {
-                    let hdl = EntryHandle::new(deadline, cx.waker().clone());
-                    this.entry = Some(hdl.clone());
-                    unsafe {
-                        registration_queue.push_front(hdl);
-                    }
-                    Poll::Pending
-                }
-                #[cfg(feature = "rt-multi-thread")]
-                Some(TempLocalContext::Shutdown) => panic!("{RUNTIME_SHUTTING_DOWN_ERROR}"),
-
-                _ => {
-                    let hdl = EntryHandle::new(deadline, cx.waker().clone());
-                    this.entry = Some(hdl.clone());
-                    push_from_remote(&this.sched_handle, hdl);
-                    Poll::Pending
-                }
+    pub(crate) fn new(handle: scheduler::Handle, deadline: Instant) -> Self {
+        let tick = deadline_to_tick(&handle, deadline);
+        let entry = with_current_temp_local_context(&handle, |ctx| match ctx {
+            Some(TempLocalContext::Running { registration_queue }) => {
+                let entry = EntryHandle::new(tick);
+                unsafe { registration_queue.push_front(entry.clone()) }
+                entry
             }
-        })
-    }
+            #[cfg(feature = "rt-multi-thread")]
+            Some(TempLocalContext::Shutdown) => panic!("{RUNTIME_SHUTTING_DOWN_ERROR}"),
 
-    pub(crate) fn poll_elapsed(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
-        match self.entry.as_ref() {
-            Some(entry) if entry.is_woken_up() => Poll::Ready(()),
-            Some(entry) => {
-                entry.register_waker(cx.waker().clone());
-                Poll::Pending
+            _ => {
+                let entry = EntryHandle::new(tick);
+                push_from_remote(&handle, entry.clone());
+                entry
             }
-            None => self.register(cx),
-        }
-    }
+        });
 
-    pub(crate) fn scheduler_handle(&self) -> &SchedulerHandle {
-        &self.sched_handle
+        Timer { entry }
     }
 
-    #[cfg(all(tokio_unstable, feature = "tracing"))]
-    pub(crate) fn driver(&self) -> &crate::runtime::time::Handle {
-        self.sched_handle.driver().time()
+    pub(crate) fn is_elapsed(&self) -> bool {
+        self.entry.is_woken_up()
     }
 
-    #[cfg(all(tokio_unstable, feature = "tracing"))]
-    pub(crate) fn clock(&self) -> &crate::time::Clock {
-        self.sched_handle.driver().clock()
+    pub(crate) fn poll_elapsed(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> {
+        self.entry.poll(cx)
     }
 }
 
-fn with_current_temp_local_context<F, R>(hdl: &SchedulerHandle, f: F) -> R
+fn with_current_temp_local_context<F, R>(sched_hdl: &scheduler::Handle, f: F) -> R
 where
@@ -132,3 +70,3 @@ F: FnOnce(Option<TempLocalContext<'_>>) -> R,
     {
-        let (_, _) = (hdl, f);
+        let _ = f;
         panic!("Tokio runtime is not enabled, cannot access the current wheel");
@@ -139,21 +77,43 @@ }
     {
+        use crate::loom::sync::Arc;
         use crate::runtime::context;
 
-        let is_same_rt =
-            context::with_current(|cur_hdl| cur_hdl.is_same_runtime(hdl)).unwrap_or_default();
+        // There is no compile-time guarantee that the timer is
+        // always registered in the same runtime as it was created in,
+        // so we need to check it at runtime.
+        let is_same_rt = context::with_current(|cur_sched_hdl| {
+            use crate::runtime::scheduler::Handle;
 
+            match (sched_hdl, cur_sched_hdl) {
+                (Handle::CurrentThread(_), _) => {
+                    // this case is impossible as `tokio::runtime::Builder::enable_alt_timer`
+                    // is not supported in the current-thread runtime, but we'd better handle it
+                    // in case the API is misused in the future.
+                    unreachable!("alternative timer is not supported in the current-thread runtime")
+                }
+                (_, Handle::CurrentThread(_)) => false,
+                (Handle::MultiThread(sched_hdl), Handle::MultiThread(cur_sched_hdl)) => {
+                    Arc::as_ptr(sched_hdl) == Arc::as_ptr(cur_sched_hdl)
+                }
+            }
+        })
+        .unwrap_or_default();
+
         if !is_same_rt {
-            // We don't want to create the timer in one runtime,
-            // but register it in a different runtime's timer wheel.
-            f(None)
-        } else {
-            context::with_scheduler(|maybe_cx| match maybe_cx {
-                Some(cx) => cx.with_time_temp_local_context(f),
-                None => f(None),
-            })
+            // The timer is being registered from a runtime
+            // that is different from the runtime that the timer is created in,
+            // so we cannot access `TempLocalContext` of the original runtime.
+            return f(None);
         }
+
+        // The timer is being registered from the same runtime that the timer is created in,
+        // so we can access `TempLocalContext`.
+        context::with_scheduler(|maybe_cx| match maybe_cx {
+            Some(cx) => cx.expect_multi_thread().with_time_temp_local_context(f),
+            None => f(None),
+        })
     }
 }
 
-fn push_from_remote(sched_hdl: &SchedulerHandle, entry_hdl: EntryHandle) {
+fn push_from_remote(sched_hdl: &scheduler::Handle, entry_hdl: EntryHandle) {
     #[cfg(not(feature = "rt"))]
@@ -172,5 +132,5 @@ {
 
-fn deadline_to_tick(sched_hdl: &SchedulerHandle, deadline: Instant) -> u64 {
+fn deadline_to_tick(sched_hdl: &scheduler::Handle, deadline: Instant) -> u64 {
     let time_hdl = sched_hdl.driver().time();
     time_hdl.time_source().deadline_to_tick(deadline)
 }

src/runtime/time_alt/wake_queue.rs

@@ -1,10 +0,8 @@
-use super::{Entry, EntryHandle, WakeQueueEntry};
-use crate::util::linked_list;
+use super::{EntryHandle, WakeQueueEntry};
+use crate::util::linked_list::LinkedList;
 
-type EntryList = linked_list::LinkedList<WakeQueueEntry, Entry>;
-
 /// A queue of entries that need to be woken up.
 #[derive(Debug)]
 pub(crate) struct WakeQueue {
-    list: EntryList,
+    list: LinkedList<WakeQueueEntry>,
 }
@@ -24,3 +22,3 @@
         Self {
-            list: EntryList::new(),
+            list: LinkedList::new(),
         }
@@ -37,3 +35,3 @@ }
     ///
-    /// - [`Entry::extra_pointers`] of `hdl` must not being used.
+    /// - `Entry::extra_pointers` of `hdl` must not being used.
     pub(crate) unsafe fn push_front(&mut self, hdl: EntryHandle) {
@@ -40,0 +38,0 @@ self.list.push_front(hdl);

src/runtime/time_alt/wake_queue/tests.rs

 use super::*;
 
+use std::task::Context;
+
 use futures_test::task::{new_count_waker, AwokenCount};
@@ -13,3 +15,5 @@
     let (waker, count) = new_count_waker();
-    (EntryHandle::new(0, waker), count)
+    let entry = EntryHandle::new(0);
+    _ = entry.poll(&mut Context::from_waker(&waker));
+    (entry, count)
 }
@@ -16,0 +20,0 @@

src/runtime/time_alt/wheel/level.rs

@@ -1,2 +0,4 @@
-use super::{EntryHandle, EntryList};
+use crate::util::linked_list::LinkedList;
+
+use super::{Entry, EntryHandle};
 use std::ptr::NonNull;
@@ -19,3 +21,3 @@ use std::{array, fmt};
     /// Slots. We access these via the EntryInner `current_list` as well, so this needs to be an `UnsafeCell`.
-    slot: [EntryList; LEVEL_MULT],
+    slot: [LinkedList<Entry>; LEVEL_MULT],
 }
@@ -46,3 +48,3 @@
             occupied: 0,
-            slot: array::from_fn(|_| EntryList::default()),
+            slot: array::from_fn(|_| LinkedList::default()),
         }
@@ -147,3 +149,3 @@ }
 
-    pub(crate) fn take_slot(&mut self, slot: usize) -> EntryList {
+    pub(crate) fn take_slot(&mut self, slot: usize) -> LinkedList<Entry> {
         self.occupied &= !occupied_bit(slot);
@@ -150,0 +152,0 @@

src/runtime/time_alt/wheel/mod.rs

@@ -6,4 +6,6 @@ mod level;
 use super::cancellation_queue::Sender;
-use super::{EntryHandle, EntryList, WakeQueue};
+use super::{Entry, EntryHandle, WakeQueue};
 
+use crate::util::linked_list::LinkedList;
+
 /// Hashed timing wheel implementation.
@@ -43,9 +45,6 @@ ///
     pub(crate) fn new() -> Wheel {
-        let mut levels = Vec::with_capacity(NUM_LEVELS);
-        for i in 0..NUM_LEVELS {
-            levels.push(Level::new(i));
-        }
+        let levels = (0..NUM_LEVELS).map(Level::new).collect::<Box<_>>();
         Wheel {
             elapsed: 0,
-            levels: levels.into_boxed_slice().try_into().unwrap(),
+            levels: levels.try_into().unwrap(),
         }
@@ -210,3 +209,3 @@ }
     /// Obtains the list of entries that need processing for the given expiration.
-    fn take_entries(&mut self, expiration: &Expiration) -> EntryList {
+    fn take_entries(&mut self, expiration: &Expiration) -> LinkedList<Entry> {
         self.levels[expiration.level].take_slot(expiration.slot)
@@ -213,0 +212,0 @@ }

src/runtime/time/entry.rs

@@ -297,8 +297,3 @@ //! Timer state structures.
         #[pin]
-        inner: Option<TimerShared>,
-        // Deadline for the timer. This is used to register on the first
-        // poll, as we can't register prior to being pinned.
-        deadline: Instant,
-        // Whether the deadline has been registered.
-        registered: bool,
+        inner: TimerShared,
     }
@@ -331,4 +326,2 @@
 
-pub(super) type EntryList = crate::util::linked_list::LinkedList<TimerShared, TimerShared>;
-
 /// The shared state structure of a timer. This structure is shared between the
@@ -483,58 +476,21 @@ /// frontend (`Entry`) and driver backend.
 impl TimerEntry {
-    #[track_caller]
-    pub(crate) fn new(handle: scheduler::Handle, deadline: Instant) -> Self {
-        // Panic if the time driver is not enabled
-        let _ = handle.driver().time();
-
+    pub(crate) fn new(handle: scheduler::Handle) -> Self {
         Self {
             driver: handle,
-            inner: None,
-            deadline,
-            registered: false,
+            inner: TimerShared::new(),
         }
     }
 
-    fn inner(&self) -> Option<&TimerShared> {
-        self.inner.as_ref()
-    }
+    pub(crate) fn init(self: Pin<&mut Self>, deadline: Instant) {
+        let tick = self.driver().time_source().deadline_to_tick(deadline);
 
-    fn init_inner(self: Pin<&mut Self>) {
-        match self.inner {
-            Some(_) => {}
-            None => self.project().inner.set(Some(TimerShared::new())),
+        unsafe {
+            self.driver()
+                .reregister(&self.driver.driver().io, tick, (&self.inner).into());
         }
     }
 
-    pub(crate) fn deadline(&self) -> Instant {
-        self.deadline
-    }
-
     pub(crate) fn is_elapsed(&self) -> bool {
-        let Some(inner) = self.inner() else {
-            return false;
-        };
-
         // Is this timer still in the timer wheel?
-        let deregistered = !inner.might_be_registered();
-
-        // Once the timer has expired,
-        // it will be taken out of the wheel and be fired.
-        //
-        // So if we have already registered the timer into the wheel,
-        // but now it is not in the wheel, it means that it has been
-        // fired.
-        //
-        // +--------------+-----------------+----------+
-        // | deregistered | self.registered |  output  |
-        // +--------------+-----------------+----------+
-        // |     true     |      false      |  false   | <- never been registered
-        // +--------------+-----------------+----------+
-        // |     false    |      false      |  false   | <- never been registered
-        // +--------------+-----------------+----------+
-        // |     true     |      true       |  true    | <- registered into the wheel,
-        // |              |                 |          |    and then taken out of the wheel.
-        // +--------------+-----------------+----------+
-        // |     false    |      true       |  false   | <- still registered in the wheel
-        // +--------------+-----------------+----------+
-        deregistered && self.registered
+        !self.inner.might_be_registered()
     }
@@ -544,7 +500,2 @@
     pub(crate) fn cancel(self: Pin<&mut Self>) {
-        // Avoid calling the `clear_entry` method, because it has not been initialized yet.
-        let Some(inner) = self.inner() else {
-            return;
-        };
-
         // We need to perform an acq/rel fence with the driver thread, and the
@@ -572,29 +523,15 @@ // simplest way to do so is to grab the driver lock.
         // deregister the entry if necessary.
-        unsafe { self.driver().clear_entry(NonNull::from(inner)) };
+        unsafe { self.driver().clear_entry(NonNull::from(&self.inner)) };
     }
 
-    pub(crate) fn reset(mut self: Pin<&mut Self>, new_time: Instant, reregister: bool) {
-        let this = self.as_mut().project();
-        *this.deadline = new_time;
-        *this.registered = reregister;
+    pub(crate) fn reset(self: Pin<&mut Self>, deadline: Instant) {
+        let tick = self.driver().time_source().deadline_to_tick(deadline);
 
-        let tick = self.driver().time_source().deadline_to_tick(new_time);
-        let inner = match self.inner() {
-            Some(inner) => inner,
-            None => {
-                self.as_mut().init_inner();
-                self.inner()
-                    .expect("inner should already be initialized by `this.init_inner()`")
-            }
-        };
-
-        if inner.extend_expiration(tick).is_ok() {
+        if self.inner.extend_expiration(tick).is_ok() {
             return;
         }
 
-        if reregister {
-            unsafe {
-                self.driver()
-                    .reregister(&self.driver.driver().io, tick, inner.into());
-            }
+        unsafe {
+            self.driver()
+                .reregister(&self.driver.driver().io, tick, (&self.inner).into());
         }
@@ -604,3 +541,3 @@ }
     pub(crate) fn poll_elapsed(
-        mut self: Pin<&mut Self>,
+        self: Pin<&mut Self>,
         cx: &mut Context<'_>,
@@ -614,21 +551,8 @@ ) -> Poll<Result<(), super::Error>> {
 
-        if !self.registered {
-            let deadline = self.deadline;
-            self.as_mut().reset(deadline, true);
-        }
-
-        let inner = self
-            .inner()
-            .expect("inner should already be initialized by `self.reset()`");
-        inner.state.poll(cx.waker())
+        self.inner.state.poll(cx.waker())
     }
 
-    pub(crate) fn driver(&self) -> &super::Handle {
+    fn driver(&self) -> &super::Handle {
         self.driver.driver().time()
     }
-
-    #[cfg(all(tokio_unstable, feature = "tracing"))]
-    pub(crate) fn clock(&self) -> &super::Clock {
-        self.driver.driver().clock()
-    }
 }
@@ -635,0 +559,0 @@

src/runtime/time/mod.rs

@@ -11,3 +11,3 @@ // Currently, rust warns when an unsafe fn contains an unsafe {} block. However,
 pub(crate) use entry::TimerEntry;
-use entry::{EntryList, TimerHandle, TimerShared, MAX_SAFE_MILLIS_DURATION};
+use entry::{TimerHandle, TimerShared, MAX_SAFE_MILLIS_DURATION};
 
@@ -14,0 +14,0 @@ mod handle;

src/runtime/time/tests/mod.rs

@@ -51,7 +51,7 @@ #![cfg(not(target_os = "wasi"))]
         let jh = thread::spawn(move || {
-            let entry = TimerEntry::new(
-                handle_.inner.clone(),
-                handle_.inner.driver().clock().now() + Duration::from_secs(1),
-            );
+            let entry = TimerEntry::new(handle_.inner.clone());
             pin!(entry);
+            entry
+                .as_mut()
+                .init(handle_.inner.driver().clock().now() + Duration::from_secs(1));
 
@@ -81,7 +81,7 @@ block_on(std::future::poll_fn(|cx| entry.as_mut().poll_elapsed(cx))).unwrap();
         let jh = thread::spawn(move || {
-            let entry = TimerEntry::new(
-                handle_.inner.clone(),
-                handle_.inner.driver().clock().now() + Duration::from_secs(1),
-            );
+            let entry = TimerEntry::new(handle_.inner.clone());
             pin!(entry);
+            entry
+                .as_mut()
+                .init(handle_.inner.driver().clock().now() + Duration::from_secs(1));
 
@@ -116,7 +116,7 @@ let _ = entry
         let jh = thread::spawn(move || {
-            let entry = TimerEntry::new(
-                handle_.inner.clone(),
-                handle_.inner.driver().clock().now() + Duration::from_secs(1),
-            );
+            let entry = TimerEntry::new(handle_.inner.clone());
             pin!(entry);
+            entry
+                .as_mut()
+                .init(handle_.inner.driver().clock().now() + Duration::from_secs(1));
 
@@ -155,4 +155,5 @@ let _ = entry
         let jh = thread::spawn(move || {
-            let entry = TimerEntry::new(handle_.inner.clone(), start + Duration::from_secs(1));
+            let entry = TimerEntry::new(handle_.inner.clone());
             pin!(entry);
+            entry.as_mut().init(start + Duration::from_secs(1));
 
@@ -163,3 +164,3 @@ let _ = entry
 
-            entry.as_mut().reset(start + Duration::from_secs(2), true);
+            entry.as_mut().reset(start + Duration::from_secs(2));
 
@@ -214,6 +215,6 @@ // shouldn't complete before 2s
     for i in 0..normal_or_miri(1024, 64) {
-        let mut entry = Box::pin(TimerEntry::new(
-            handle.inner.clone(),
-            handle.inner.driver().clock().now() + Duration::from_millis(i),
-        ));
+        let mut entry = Box::pin(TimerEntry::new(handle.inner.clone()));
+        entry
+            .as_mut()
+            .init(handle.inner.driver().clock().now() + Duration::from_millis(i));
 
@@ -249,7 +250,6 @@ let _ = entry
 
-    let e1 = TimerEntry::new(
-        handle.inner.clone(),
-        handle.inner.driver().clock().now() + Duration::from_millis(193),
-    );
+    let e1 = TimerEntry::new(handle.inner.clone());
     pin!(e1);
+    e1.as_mut()
+        .init(handle.inner.driver().clock().now() + Duration::from_millis(193));
 
@@ -256,0 +256,0 @@ let handle = handle.inner.driver().time();

src/runtime/time/wheel/level.rs

@@ -1,2 +0,3 @@
-use crate::runtime::time::{EntryList, TimerHandle, TimerShared};
+use crate::runtime::time::{TimerHandle, TimerShared};
+use crate::util::linked_list::LinkedList;
 
@@ -19,3 +20,3 @@ use std::{array, fmt, ptr::NonNull};
     /// Slots. We access these via the EntryInner `current_list` as well, so this needs to be an `UnsafeCell`.
-    slot: [EntryList; LEVEL_MULT],
+    slot: [LinkedList<TimerShared>; LEVEL_MULT],
 }
@@ -46,3 +47,3 @@
             occupied: 0,
-            slot: array::from_fn(|_| EntryList::default()),
+            slot: array::from_fn(|_| LinkedList::default()),
         }
@@ -145,3 +146,3 @@ }
 
-    pub(crate) fn take_slot(&mut self, slot: usize) -> EntryList {
+    pub(crate) fn take_slot(&mut self, slot: usize) -> LinkedList<TimerShared> {
         self.occupied &= !occupied_bit(slot);
@@ -148,0 +149,0 @@

src/runtime/time/wheel/mod.rs

 use crate::runtime::time::{TimerHandle, TimerShared};
 use crate::time::error::InsertError;
+use crate::util::linked_list::LinkedList;
 
@@ -11,3 +12,2 @@ mod level;
 use super::entry::STATE_DEREGISTERED;
-use super::EntryList;
 
@@ -40,3 +40,3 @@ /// Timing wheel implementation.
     /// Entries queued for firing
-    pending: EntryList,
+    pending: LinkedList<TimerShared>,
 }
@@ -55,10 +55,7 @@
     pub(crate) fn new() -> Wheel {
-        let mut levels = Vec::with_capacity(NUM_LEVELS);
-        for i in 0..NUM_LEVELS {
-            levels.push(Level::new(i));
-        }
+        let levels = (0..NUM_LEVELS).map(Level::new).collect::<Box<_>>();
         Wheel {
             elapsed: 0,
-            levels: levels.into_boxed_slice().try_into().unwrap(),
-            pending: EntryList::new(),
+            levels: levels.try_into().unwrap(),
+            pending: LinkedList::new(),
         }
@@ -272,3 +269,3 @@ }
     /// Obtains the list of entries that need processing for the given expiration.
-    fn take_entries(&mut self, expiration: &Expiration) -> EntryList {
+    fn take_entries(&mut self, expiration: &Expiration) -> LinkedList<TimerShared> {
         self.levels[expiration.level].take_slot(expiration.slot)
@@ -275,0 +272,0 @@ }

src/signal/unix.rs

@@ -429,4 +429,4 @@ //! Unix-specific types for signal handling.
     ///
-    /// This method is cancel safe. If you use it as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If you use it as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, then it is guaranteed that no signal is lost.
@@ -433,0 +433,0 @@ ///

src/signal/windows/sys.rs

 use std::io;
+use std::io::Error;
+use std::ops::Index;
 use std::sync::OnceLock;
@@ -12,41 +14,62 @@
 
+#[derive(Clone, Copy)]
+#[repr(u32)]
+enum SignalKind {
+    CtrlC = console::CTRL_C_EVENT,
+    CtrlBreak = console::CTRL_BREAK_EVENT,
+    CtrlClose = console::CTRL_CLOSE_EVENT,
+    CtrlLogoff = console::CTRL_LOGOFF_EVENT,
+    CtrlShutdown = console::CTRL_SHUTDOWN_EVENT,
+}
+
+impl SignalKind {
+    const fn terminates(&self) -> bool {
+        // Returning from the handler function of those events immediately terminates the process.
+        // So for async systems, the easiest solution is to simply never return from
+        // the handler function.
+        //
+        // For more information, see:
+        // https://learn.microsoft.com/en-us/windows/console/handlerroutine#remarks
+        matches!(
+            self,
+            Self::CtrlClose | Self::CtrlLogoff | Self::CtrlShutdown
+        )
+    }
+}
+
 pub(super) fn ctrl_break() -> io::Result<RxFuture> {
-    new(&registry().ctrl_break)
+    new(SignalKind::CtrlBreak)
 }
 
 pub(super) fn ctrl_close() -> io::Result<RxFuture> {
-    new(&registry().ctrl_close)
+    new(SignalKind::CtrlClose)
 }
 
 pub(super) fn ctrl_c() -> io::Result<RxFuture> {
-    new(&registry().ctrl_c)
+    new(SignalKind::CtrlC)
 }
 
 pub(super) fn ctrl_logoff() -> io::Result<RxFuture> {
-    new(&registry().ctrl_logoff)
+    new(SignalKind::CtrlLogoff)
 }
 
 pub(super) fn ctrl_shutdown() -> io::Result<RxFuture> {
-    new(&registry().ctrl_shutdown)
+    new(SignalKind::CtrlShutdown)
 }
 
-fn new(event_info: &EventInfo) -> io::Result<RxFuture> {
-    global_init()?;
-    let rx = event_info.subscribe();
+fn new(signal: SignalKind) -> io::Result<RxFuture> {
+    let registry = REGISTRY
+        .get_or_init(
+            || match unsafe { console::SetConsoleCtrlHandler(Some(handler), 1) } {
+                0 => Err(Error::last_os_error().raw_os_error().expect("unreachable")),
+                _ => Ok(Registry::default()),
+            },
+        )
+        .as_ref()
+        .map_err(|&code| Error::from_raw_os_error(code))?;
+
+    let rx = registry[signal].subscribe();
     Ok(RxFuture::new(rx))
 }
 
-fn event_requires_infinite_sleep_in_handler(signum: u32) -> bool {
-    // Returning from the handler function of those events immediately terminates the process.
-    // So for async systems, the easiest solution is to simply never return from
-    // the handler function.
-    //
-    // For more information, see:
-    // https://learn.microsoft.com/en-us/windows/console/handlerroutine#remarks
-    matches!(
-        signum,
-        console::CTRL_CLOSE_EVENT | console::CTRL_LOGOFF_EVENT | console::CTRL_SHUTDOWN_EVENT
-    )
-}
-
 #[derive(Debug, Default)]
@@ -61,11 +84,12 @@ struct Registry {
 
-impl Registry {
-    fn event_info(&self, signum: u32) -> Option<&EventInfo> {
-        match signum {
-            console::CTRL_BREAK_EVENT => Some(&self.ctrl_break),
-            console::CTRL_CLOSE_EVENT => Some(&self.ctrl_close),
-            console::CTRL_C_EVENT => Some(&self.ctrl_c),
-            console::CTRL_LOGOFF_EVENT => Some(&self.ctrl_logoff),
-            console::CTRL_SHUTDOWN_EVENT => Some(&self.ctrl_shutdown),
-            _ => None,
+impl Index<SignalKind> for Registry {
+    type Output = EventInfo;
+
+    fn index(&self, signal: SignalKind) -> &Self::Output {
+        match signal {
+            SignalKind::CtrlC => &self.ctrl_c,
+            SignalKind::CtrlBreak => &self.ctrl_break,
+            SignalKind::CtrlClose => &self.ctrl_close,
+            SignalKind::CtrlLogoff => &self.ctrl_logoff,
+            SignalKind::CtrlShutdown => &self.ctrl_shutdown,
         }
@@ -75,30 +99,20 @@ }
 
-fn registry() -> &'static Registry {
-    static REGISTRY: OnceLock<Registry> = OnceLock::new();
+static REGISTRY: OnceLock<Result<Registry, i32>> = OnceLock::new();
 
-    REGISTRY.get_or_init(Default::default)
-}
+unsafe extern "system" fn handler(ty: u32) -> BOOL {
+    let signal = match ty {
+        console::CTRL_C_EVENT => SignalKind::CtrlC,
+        console::CTRL_BREAK_EVENT => SignalKind::CtrlBreak,
+        console::CTRL_CLOSE_EVENT => SignalKind::CtrlClose,
+        console::CTRL_LOGOFF_EVENT => SignalKind::CtrlLogoff,
+        console::CTRL_SHUTDOWN_EVENT => SignalKind::CtrlShutdown,
+        // Ignore unknown signals.
+        _ => return 0,
+    };
 
-fn global_init() -> io::Result<()> {
-    static INIT: OnceLock<Result<(), Option<i32>>> = OnceLock::new();
-
-    INIT.get_or_init(|| {
-        let rc = unsafe { console::SetConsoleCtrlHandler(Some(handler), 1) };
-        if rc == 0 {
-            Err(io::Error::last_os_error().raw_os_error())
-        } else {
-            Ok(())
-        }
-    })
-    .map_err(|e| {
-        e.map_or_else(
-            || io::Error::new(io::ErrorKind::Other, "registering signal handler failed"),
-            io::Error::from_raw_os_error,
-        )
-    })
-}
-
-unsafe extern "system" fn handler(ty: u32) -> BOOL {
-    // Ignore unknown control signal types.
-    let Some(event_info) = registry().event_info(ty) else {
+    // Note that `OnceLock::get` does not handle the small window between calling
+    // `SetConsoleCtrlHandler` and `REGISTRY` being initialized.
+    let Ok(registry) = REGISTRY.wait().as_ref() else {
+        // Technically unreachable since `handler` is only called if
+        // `SetConsoleCtrlHandler` succeeded.
         return 0;
@@ -111,4 +125,4 @@ };
     // go ahead and perform the broadcast here.
-    match event_info.send(()) {
-        Ok(_) if event_requires_infinite_sleep_in_handler(ty) => loop {
+    match registry[signal].send(()) {
+        Ok(_) if signal.terminates() => loop {
             std::thread::park();
@@ -130,9 +144,9 @@ },
 
-    unsafe fn raise_event(signum: u32) {
-        if event_requires_infinite_sleep_in_handler(signum) {
+    unsafe fn raise_event(signal: SignalKind) {
+        if signal.terminates() {
             // Those events will enter an infinite loop in `handler`, so
             // we need to run them on a separate thread
-            std::thread::spawn(move || unsafe { super::handler(signum) });
+            std::thread::spawn(move || unsafe { super::handler(signal as u32) });
         } else {
-            unsafe { super::handler(signum) };
+            unsafe { super::handler(signal as u32) };
         }
@@ -154,3 +168,3 @@ }
         unsafe {
-            raise_event(console::CTRL_C_EVENT);
+            raise_event(SignalKind::CtrlC);
         }
@@ -172,3 +186,3 @@
             unsafe {
-                raise_event(console::CTRL_BREAK_EVENT);
+                raise_event(SignalKind::CtrlBreak);
             }
@@ -191,3 +205,3 @@
             unsafe {
-                raise_event(console::CTRL_CLOSE_EVENT);
+                raise_event(SignalKind::CtrlClose);
             }
@@ -210,3 +224,3 @@
             unsafe {
-                raise_event(console::CTRL_SHUTDOWN_EVENT);
+                raise_event(SignalKind::CtrlShutdown);
             }
@@ -229,3 +243,3 @@
             unsafe {
-                raise_event(console::CTRL_LOGOFF_EVENT);
+                raise_event(SignalKind::CtrlLogoff);
             }
@@ -232,0 +246,0 @@

src/sync/batch_semaphore.rs

@@ -44,3 +44,3 @@ #![cfg_attr(not(feature = "sync"), allow(unreachable_pub, dead_code))]
 struct Waitlist {
-    queue: LinkedList<Waiter, <Waiter as linked_list::Link>::Target>,
+    queue: LinkedList<Waiter>,
     closed: bool,
@@ -580,3 +580,3 @@ }
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
 
@@ -583,0 +583,0 @@ #[cfg(all(tokio_unstable, feature = "tracing"))]

src/sync/broadcast.rs

@@ -35,11 +35,22 @@ //! A multi-producer, multi-consumer broadcast queue. Each sent value is seen by
 //! time. This upper bound is passed to the [`channel`] function as an argument.
+//! The provided capacity is rounded **up** to the next power of two; that
+//! rounded size is the number of messages the ring buffer can hold, and is what
+//! lag detection is based on. For example, `channel(3)` allocates a buffer of
+//! length 4, so a receiver only lags once it falls more than 4 messages behind
+//! the sender.
 //!
 //! If a value is sent when the channel is at capacity, the oldest value
-//! currently held by the channel is released. This frees up space for the new
-//! value. Any receiver that has not yet seen the released value will return
-//! [`RecvError::Lagged`] the next time [`recv`] is called.
+//! currently held by the channel is overwritten. This frees up space for the
+//! new value. Any receiver that has not yet seen the overwritten value will
+//! return [`RecvError::Lagged`] the next time [`recv`] (or
+//! [`try_recv`](Receiver::try_recv)) is called. The error carries the number of
+//! messages that were dropped before the receiver's cursor and are therefore
+//! no longer available.
 //!
-//! Once [`RecvError::Lagged`] is returned, the lagging receiver's position is
-//! updated to the oldest value contained by the channel. The next call to
-//! [`recv`] will return this value.
+//! Returning [`RecvError::Lagged`] does **not** close or disconnect the
+//! receiver. The lagging receiver's internal cursor is advanced to the oldest
+//! value still retained by the channel. The **next** successful call to
+//! [`recv`] / [`try_recv`](Receiver::try_recv) returns that oldest retained
+//! value (unless further sends overwrite it again before the receiver reads
+//! it). Subsequent receives then continue in send order from there.
 //!
@@ -101,5 +112,7 @@ //! This behavior enables a receiver to detect when it has lagged so far behind
 //! use tokio::sync::broadcast;
+//! use tokio::sync::broadcast::error::RecvError;
 //!
 //! # #[tokio::main(flavor = "current_thread")]
 //! # async fn main() {
+//! // Capacity 2 → ring buffer of length 2.
 //! let (tx, mut rx) = broadcast::channel(2);
@@ -109,9 +122,10 @@ //!
 //! tx.send(20).unwrap();
+//! // Overwrites 10; receiver has not read it yet.
 //! tx.send(30).unwrap();
 //!
-//! // The receiver lagged behind
-//! assert!(rx.recv().await.is_err());
+//! // One message (10) was dropped; cursor moves to the oldest retained value (20).
+//! assert!(matches!(rx.recv().await, Err(RecvError::Lagged(1))));
 //!
-//! // At this point, we can abort or continue with lost messages
-//!
+//! // At this point, we can abort or continue with lost messages.
+//! // Continuing resumes from the oldest retained message.
 //! assert_eq!(20, rx.recv().await.unwrap());
@@ -284,6 +298,14 @@ //! assert_eq!(30, rx.recv().await.unwrap());
 
-        /// The receiver lagged too far behind. Attempting to receive again will
-        /// return the oldest message still retained by the channel.
+        /// The receiver lagged too far behind: one or more messages were
+        /// overwritten in the ring buffer before this receiver could read them.
         ///
-        /// Includes the number of skipped messages.
+        /// The receiver remains subscribed. Its internal cursor has been advanced
+        /// to the oldest message still retained by the channel; the next
+        /// successful [`recv`] call returns that message (unless further sends
+        /// overwrite it first).
+        ///
+        /// The `u64` is the number of messages that were skipped (dropped before
+        /// the receiver's previous cursor position).
+        ///
+        /// [`recv`]: crate::sync::broadcast::Receiver::recv
         Lagged(u64),
@@ -319,7 +341,14 @@ }
 
-        /// The receiver lagged too far behind and has been forcibly disconnected.
-        /// Attempting to receive again will return the oldest message still
-        /// retained by the channel.
+        /// The receiver lagged too far behind: one or more messages were
+        /// overwritten in the ring buffer before this receiver could read them.
         ///
-        /// Includes the number of skipped messages.
+        /// The receiver remains subscribed. Its internal cursor has been advanced
+        /// to the oldest message still retained by the channel; the next
+        /// successful [`try_recv`] call returns that message (unless further sends
+        /// overwrite it first).
+        ///
+        /// The `u64` is the number of messages that were skipped (dropped before
+        /// the receiver's previous cursor position).
+        ///
+        /// [`try_recv`]: crate::sync::broadcast::Receiver::try_recv
         Lagged(u64),
@@ -378,3 +407,3 @@ }
     /// Receivers waiting for a value.
-    waiters: LinkedList<Waiter, <Waiter as linked_list::Link>::Target>,
+    waiters: LinkedList<Waiter>,
 }
@@ -462,3 +491,6 @@
 ///
-/// **Note:** The actual capacity may be greater than the provided `capacity`.
+/// **Note:** The provided `capacity` is rounded **up** to the next power of
+/// two. That rounded size is the number of messages the internal ring buffer
+/// can retain, and is what [lag detection](self#lagging) uses. For example,
+/// `channel(3)` behaves as if the capacity were 4.
 ///
@@ -560,14 +592,12 @@ /// All data sent on [`Sender`] will become available on every active
 
-        let mut buffer = Vec::with_capacity(capacity);
-
-        for i in 0..capacity {
-            buffer.push(Mutex::new(Slot {
+        let buffer = (0..capacity).map(|i| {
+            Mutex::new(Slot {
                 rem: AtomicUsize::new(0),
                 pos: (i as u64).wrapping_sub(capacity as u64),
                 val: None,
-            }));
-        }
+            })
+        });
 
         let shared = Arc::new(Shared {
-            buffer: buffer.into_boxed_slice(),
+            buffer: buffer.collect(),
             mask: capacity - 1,
@@ -956,3 +986,3 @@ tail: Mutex::new(Tail {
 struct WaitersList<'a, T> {
-    list: GuardedLinkedList<Waiter, <Waiter as linked_list::Link>::Target>,
+    list: GuardedLinkedList<Waiter>,
     is_empty: bool,
@@ -975,3 +1005,3 @@ shared: &'a Shared<T>,
     fn new(
-        unguarded_list: LinkedList<Waiter, <Waiter as linked_list::Link>::Target>,
+        unguarded_list: LinkedList<Waiter>,
         guard: Pin<&'a Waiter>,
@@ -1145,9 +1175,13 @@ shared: &'a Shared<T>,
     ///
-    /// If the returned value from `len` is larger than the next largest power of 2
-    /// of the capacity of the channel any call to [`recv`] will return an
-    /// `Err(RecvError::Lagged)` and any call to [`try_recv`] will return an
-    /// `Err(TryRecvError::Lagged)`, e.g. if the capacity of the channel is 10,
-    /// [`recv`] will start to return `Err(RecvError::Lagged)` once `len` returns
-    /// values larger than 16.
+    /// This count includes messages that have already been overwritten in the
+    /// ring buffer and are no longer readable. If `len` is **greater than** the
+    /// channel's effective capacity (the provided capacity rounded up to the
+    /// next power of two), the next call to [`recv`] returns
+    /// `Err(RecvError::Lagged)` and the next call to [`try_recv`] returns
+    /// `Err(TryRecvError::Lagged)`. For example, with `channel(10)` the buffer
+    /// length is 16, so lagging begins once `len` is larger than 16.
     ///
+    /// After a successful receive (including after handling `Lagged` and then
+    /// reading retained messages), `len` decreases accordingly.
+    ///
     /// [`Receiver`]: crate::sync::broadcast::Receiver
@@ -1415,12 +1449,14 @@ /// [`recv`]: crate::sync::broadcast::Receiver::recv
     /// If the [`Receiver`] handle falls behind, once the channel is full, newly
-    /// sent values will overwrite old values. At this point, a call to [`recv`]
-    /// will return with `Err(RecvError::Lagged)` and the [`Receiver`]'s
-    /// internal cursor is updated to point to the oldest value still held by
-    /// the channel. A subsequent call to [`recv`] will return this value
-    /// **unless** it has been since overwritten.
+    /// sent values overwrite old values in the ring buffer. The next call to
+    /// [`recv`] then returns `Err(RecvError::Lagged(n))`, where `n` is the
+    /// number of overwritten messages the receiver missed. The receiver stays
+    /// subscribed; its internal cursor is advanced to the oldest value still
+    /// held by the channel. A subsequent call to [`recv`] returns that value,
+    /// unless further sends overwrite it before the receiver reads it. See
+    /// [lagging](self#lagging) for details.
     ///
     /// # Cancel safety
     ///
-    /// This method is cancel safe. If `recv` is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If `recv` is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, it is guaranteed that no messages were received on this
@@ -1461,2 +1497,3 @@ /// channel.
     /// use tokio::sync::broadcast;
+    /// use tokio::sync::broadcast::error::RecvError;
     ///
@@ -1471,7 +1508,6 @@ /// # #[tokio::main(flavor = "current_thread")]
     ///
-    /// // The receiver lagged behind
-    /// assert!(rx.recv().await.is_err());
+    /// // One message was overwritten before this receiver could read it.
+    /// assert!(matches!(rx.recv().await, Err(RecvError::Lagged(1))));
     ///
-    /// // At this point, we can abort or continue with lost messages
-    ///
+    /// // Resume from the oldest retained message, or abort the task instead.
     /// assert_eq!(20, rx.recv().await.unwrap());
@@ -1497,8 +1533,10 @@ /// assert_eq!(30, rx.recv().await.unwrap());
     /// If the [`Receiver`] handle falls behind, once the channel is full, newly
-    /// sent values will overwrite old values. At this point, a call to [`recv`]
-    /// will return with `Err(TryRecvError::Lagged)` and the [`Receiver`]'s
-    /// internal cursor is updated to point to the oldest value still held by
-    /// the channel. A subsequent call to [`try_recv`] will return this value
-    /// **unless** it has been since overwritten. If there are no values to
-    /// receive, `Err(TryRecvError::Empty)` is returned.
+    /// sent values overwrite old values in the ring buffer. The next call to
+    /// [`try_recv`] then returns `Err(TryRecvError::Lagged(n))`, where `n` is
+    /// the number of overwritten messages the receiver missed. The receiver
+    /// stays subscribed; its internal cursor is advanced to the oldest value
+    /// still held by the channel. A subsequent call to [`try_recv`] returns
+    /// that value, unless further sends overwrite it before the receiver reads
+    /// it. If there are no values to receive, `Err(TryRecvError::Empty)` is
+    /// returned. See [lagging](self#lagging) for details.
     ///
@@ -1625,3 +1663,3 @@ /// [`recv`]: crate::sync::broadcast::Receiver::recv
     fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<T, RecvError>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
 
@@ -1628,0 +1666,0 @@ let (receiver, waiter) = self.project();

src/sync/mpsc/chan.rs

@@ -292,3 +292,3 @@ use crate::loom::cell::UnsafeCell;
 
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
 
@@ -353,3 +353,3 @@ // Keep track of task budget
 
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
 
@@ -524,2 +524,7 @@ // Keep track of task budget
 
+            // When Rx is dropped, there is nothing for a task to poll anymore.
+            // This means we can drop our waker to potentially free up resources.
+            // Do so before draining the channel where panics may occur.
+            self.inner.rx_waker.take_waker();
+
             guard.drain();
@@ -526,0 +531,0 @@ });

src/sync/mpsc/unbounded.rs

@@ -126,5 +126,5 @@ use crate::loom::sync::{atomic::AtomicUsize, Arc};
     ///
-    /// This method is cancel safe. If `recv` is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
-    /// completes first, it is guaranteed that no messages were received on this
+    /// This method is cancel safe. If `recv` is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch completes first,
+    /// it is guaranteed that no messages were received on this
     /// channel.
@@ -195,5 +195,5 @@ ///
     ///
-    /// This method is cancel safe. If `recv_many` is used as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
-    /// completes first, it is guaranteed that no messages were received on this
+    /// This method is cancel safe. If `recv_many` is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch completes first,
+    /// it is guaranteed that no messages were received on this
     /// channel.
@@ -200,0 +200,0 @@ ///

src/sync/notify.rs

@@ -23,5 +23,2 @@ // Allow `unreachable_pub` warnings when sync is not enabled
 
-type WaitList = LinkedList<Waiter, <Waiter as linked_list::Link>::Target>;
-type GuardedWaitList = GuardedLinkedList<Waiter, <Waiter as linked_list::Link>::Target>;
-
 /// Notifies a single task to wake up.
@@ -215,3 +212,3 @@ ///
     state: AtomicUsize,
-    waiters: Mutex<WaitList>,
+    waiters: Mutex<LinkedList<Waiter>>,
 }
@@ -332,3 +329,3 @@
 struct NotifyWaitersList<'a> {
-    list: GuardedWaitList,
+    list: GuardedLinkedList<Waiter>,
     is_empty: bool,
@@ -340,3 +337,3 @@ notify: &'a Notify,
     fn new(
-        unguarded_list: WaitList,
+        unguarded_list: LinkedList<Waiter>,
         guard: Pin<&'a Waiter>,
@@ -356,3 +353,3 @@ notify: &'a Notify,
     /// requires an exclusive access to the main list in `Notify`.
-    fn pop_back_locked(&mut self, _waiters: &mut WaitList) -> Option<NonNull<Waiter>> {
+    fn pop_back_locked(&mut self, _waiters: &mut LinkedList<Waiter>) -> Option<NonNull<Waiter>> {
         let result = self.list.pop_back();
@@ -755,3 +752,3 @@ if result.is_none() {
         curr: usize,
-        mut waiters: crate::loom::sync::MutexGuard<'a, LinkedList<Waiter, Waiter>>,
+        mut waiters: crate::loom::sync::MutexGuard<'a, LinkedList<Waiter>>,
     ) {
@@ -851,3 +848,3 @@ if matches!(get_state(curr), EMPTY | NOTIFIED) {
 fn notify_locked(
-    waiters: &mut WaitList,
+    waiters: &mut LinkedList<Waiter>,
     state: &AtomicUsize,
@@ -1237,5 +1234,4 @@ curr: usize,
                     #[cfg(feature = "taskdump")]
-                    if let Some(waker) = waker {
-                        let mut ctx = Context::from_waker(waker);
-                        std::task::ready!(crate::trace::trace_leaf(&mut ctx));
+                    if let Some(_waker) = waker {
+                        std::task::ready!(crate::trace::trace_leaf());
                     }
@@ -1332,5 +1328,4 @@
                     #[cfg(feature = "taskdump")]
-                    if let Some(waker) = waker {
-                        let mut ctx = Context::from_waker(waker);
-                        std::task::ready!(crate::trace::trace_leaf(&mut ctx));
+                    if let Some(_waker) = waker {
+                        std::task::ready!(crate::trace::trace_leaf());
                     }
@@ -1417,3 +1412,3 @@ return Poll::Ready(());
     guarded_notify: &'a Notify,
-    guarded_waiters: crate::loom::sync::MutexGuard<'a, WaitList>,
+    guarded_waiters: crate::loom::sync::MutexGuard<'a, LinkedList<Waiter>>,
     current_state: usize,
@@ -1420,0 +1415,0 @@ }

src/sync/oneshot.rs

@@ -246,7 +246,7 @@ #![cfg_attr(not(feature = "sync"), allow(dead_code, unreachable_pub))]
 ///
-/// # Cancellation safety
+/// # Cancel safety
 ///
-/// The `Receiver` is cancel safe. If it is used as the event in a
-/// [`tokio::select!`](crate::select) statement and some other branch
-/// completes first, it is guaranteed that no message was received on this
+/// Awaiting a `&mut Receiver<T>` is cancel safe. If it is used as a branch in
+/// [`tokio::select!`](crate::select) and another branch completes first, it is
+/// guaranteed that no message was received on this
 /// channel.
@@ -822,3 +822,3 @@ ///
     pub fn poll_closed(&mut self, cx: &mut Context<'_>) -> Poll<()> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
 
@@ -1273,17 +1273,19 @@ // Keep track of task budget
 
-    fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
-        // If `inner` is `None`, then `poll()` has already completed.
+    fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
+        let this = self.get_mut();
+
         #[cfg(all(tokio_unstable, feature = "tracing"))]
-        let _res_span = self.resource_span.clone().entered();
+        let _res_span = this.resource_span.enter();
         #[cfg(all(tokio_unstable, feature = "tracing"))]
-        let _ao_span = self.async_op_span.clone().entered();
+        let _ao_span = this.async_op_span.enter();
         #[cfg(all(tokio_unstable, feature = "tracing"))]
-        let _ao_poll_span = self.async_op_poll_span.clone().entered();
+        let _ao_poll_span = this.async_op_poll_span.enter();
 
-        let ret = if let Some(inner) = self.as_ref().get_ref().inner.as_ref() {
+        // If `inner` is `None`, then `poll()` has already completed.
+        let ret = if let Some(inner) = this.inner.as_ref() {
             #[cfg(all(tokio_unstable, feature = "tracing"))]
-            let res = ready!(trace_poll_op!("poll_recv", inner.poll_recv(cx))).map_err(Into::into);
+            let res = ready!(trace_poll_op!("poll_recv", inner.poll_recv(cx)));
 
             #[cfg(any(not(tokio_unstable), not(feature = "tracing")))]
-            let res = ready!(inner.poll_recv(cx)).map_err(Into::into);
+            let res = ready!(inner.poll_recv(cx));
 
@@ -1295,3 +1297,3 @@ res
 
-        self.inner = None;
+        this.inner = None;
         Ready(ret)
@@ -1320,3 +1322,3 @@ }
     fn poll_recv(&self, cx: &mut Context<'_>) -> Poll<Result<T, RecvError>> {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
         // Keep track of task budget
@@ -1323,0 +1325,0 @@ let coop = ready!(crate::task::coop::poll_proceed(cx));

src/sync/semaphore.rs

@@ -29,2 +29,31 @@ use super::batch_semaphore as ll; // low level implementation
 ///
+/// # Memory ordering
+///
+/// If a task writes some data and then releases a permit, any task that later
+/// acquires a permit is guaranteed to see that data. This makes it safe to use
+/// a semaphore to hand data off between tasks through shared state.
+///
+/// Stated more precisely in terms of atomic memory orderings: acquiring a
+/// permit (via [`acquire`], [`acquire_many`], [`try_acquire`],
+/// [`try_acquire_many`], or their `_owned` variants), releasing permits (by
+/// dropping a [`SemaphorePermit`] or [`OwnedSemaphorePermit`], or by calling
+/// [`add_permits`] or [`forget_permits`]), and closing the semaphore (via
+/// [`close`]) are all `AcqRel` operations. They are totally ordered, and each
+/// one synchronizes-with all such operations that precede it, giving the same
+/// guarantees as `AcqRel` operations on a single atomic.
+///
+/// A failed acquisition attempt (including [`TryAcquireError::NoPermits`] and
+/// [`TryAcquireError::Closed`]), along with the [`available_permits`] and
+/// [`is_closed`] methods, behave like an `Acquire` load.
+///
+/// [`acquire`]: Semaphore::acquire
+/// [`acquire_many`]: Semaphore::acquire_many
+/// [`try_acquire`]: Semaphore::try_acquire
+/// [`try_acquire_many`]: Semaphore::try_acquire_many
+/// [`add_permits`]: Semaphore::add_permits
+/// [`forget_permits`]: Semaphore::forget_permits
+/// [`close`]: Semaphore::close
+/// [`available_permits`]: Semaphore::available_permits
+/// [`is_closed`]: Semaphore::is_closed
+///
 /// # Examples
@@ -31,0 +60,0 @@ ///

src/sync/watch.rs

@@ -791,4 +791,4 @@ #![cfg_attr(not(feature = "sync"), allow(dead_code, unreachable_pub))]
     ///
-    /// This method is cancel safe. If you use it as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If you use it as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, then it is guaranteed that no values have been marked
@@ -861,4 +861,4 @@ /// seen by this call to `changed`.
     ///
-    /// This method is cancel safe. If you use it as the event in a
-    /// [`tokio::select!`](crate::select) statement and some other branch
+    /// This method is cancel safe. If you use it as a branch in
+    /// [`tokio::select!`](crate::select) and another branch
     /// completes first, then it is guaranteed that the last seen value `val`
@@ -865,0 +865,0 @@ /// (if any) satisfies `f(val) == false`.

src/task/coop/consume_budget.rs

@@ -29,3 +29,3 @@ /// Consumes a unit of budget and returns the execution back to the Tokio
     std::future::poll_fn(move |cx| {
-        std::task::ready!(crate::trace::trace_leaf(cx));
+        std::task::ready!(crate::trace::trace_leaf());
         if status.is_ready() {
@@ -32,0 +32,0 @@ return status;

src/task/join_set.rs

@@ -291,7 +291,7 @@ //! A collection of tasks spawned on a Tokio runtime.
     ///
-    /// # Cancel Safety
+    /// # Cancel safety
     ///
-    /// This method is cancel safe. If `join_next` is used as the event in a `tokio::select!`
-    /// statement and some other branch completes first, it is guaranteed that no tasks were
-    /// removed from this `JoinSet`.
+    /// This method is cancel safe. If `join_next` is used as a branch in
+    /// `tokio::select!` and another branch completes first, it is guaranteed
+    /// that no tasks were removed from this `JoinSet`.
     pub async fn join_next(&mut self) -> Option<Result<T, JoinError>> {
@@ -309,7 +309,7 @@ std::future::poll_fn(|cx| self.poll_join_next(cx)).await
     ///
-    /// # Cancel Safety
+    /// # Cancel safety
     ///
-    /// This method is cancel safe. If `join_next_with_id` is used as the event in a `tokio::select!`
-    /// statement and some other branch completes first, it is guaranteed that no tasks were
-    /// removed from this `JoinSet`.
+    /// This method is cancel safe. If `join_next_with_id` is used as a branch
+    /// in `tokio::select!` and another branch completes first, it is
+    /// guaranteed that no tasks were removed from this `JoinSet`.
     ///
@@ -316,0 +316,0 @@ /// [task ID]: crate::task::Id

src/task/local.rs

@@ -434,3 +434,4 @@ //! Runs `!Send` futures on the current thread.
                         target_arch = "x86",
-                        target_arch = "x86_64"
+                        target_arch = "x86_64",
+                        target_arch = "s390x"
                     )
@@ -437,0 +438,0 @@ ))]

src/task/spawn.rs

@@ -203,3 +203,4 @@ use crate::runtime::BOX_FUTURE_THRESHOLD;
                 target_arch = "x86",
-                target_arch = "x86_64"
+                target_arch = "x86_64",
+                target_arch = "s390x"
             )
@@ -206,0 +207,0 @@ ))]

src/task/yield_now.rs

@@ -41,3 +41,3 @@ use crate::runtime::context;
     poll_fn(|cx| {
-        ready!(crate::trace::trace_leaf(cx));
+        ready!(crate::trace::trace_leaf());
 
@@ -50,2 +50,8 @@ if yielded {
 
+        // Don't wake the task immediately, as that would push it right back
+        // onto the run queue and it could be polled again before other tasks
+        // or the IO/timer driver get a chance to run. Instead, hand the waker
+        // to the scheduler, which wakes deferred tasks only after it has run
+        // out of ready tasks and polled the driver. When polled from outside
+        // a Tokio runtime, the waker is woken immediately.
         context::defer(cx.waker());
@@ -52,0 +58,0 @@

src/time/interval.rs

@@ -134,10 +134,4 @@ use crate::time::{sleep_until, Duration, Instant, Sleep};
 
-    #[cfg(all(tokio_unstable, feature = "tracing"))]
-    let delay = resource_span.in_scope(|| Box::pin(sleep_until(start)));
-
-    #[cfg(not(all(tokio_unstable, feature = "tracing")))]
-    let delay = Box::pin(sleep_until(start));
-
     Interval {
-        delay,
+        delay: Box::pin(sleep_until(start)),
         period,
@@ -412,4 +406,5 @@ missed_tick_behavior: MissedTickBehavior::default(),
     ///
-    /// This method is cancellation safe. If `tick` is used as the branch in a `tokio::select!` and
-    /// another branch completes first, then no tick has been consumed.
+    /// This method is cancel safe. If `tick` is used as a branch in
+    /// [`tokio::select!`](crate::select) and another branch completes first,
+    /// then no tick has been consumed.
     ///
@@ -492,3 +487,3 @@ /// # Examples
         // the next call to [`poll_tick`].
-        self.delay.as_mut().reset_without_reregister(next);
+        self.delay.as_mut().reset_without_timer(next);
 
@@ -495,0 +490,0 @@ // Return the time when we were scheduled to tick

src/time/sleep.rs

@@ -1,2 +0,2 @@
-use crate::runtime::Timer;
+use crate::runtime::{scheduler, Timer};
 use crate::time::{error::Error, Duration, Instant};
@@ -226,7 +226,7 @@ use crate::util::trace;
     pub struct Sleep {
+        deadline: Instant,
+        driver: scheduler::Handle,
         inner: Inner,
-
-        // The link between the `Sleep` instance and the timer that drives it.
         #[pin]
-        entry: Timer,
+        timer: Option<Timer>,
     }
@@ -255,14 +255,7 @@ }
     ) -> Sleep {
-        use crate::runtime::scheduler;
         let handle = scheduler::Handle::current();
-        let entry = Timer::new(handle, deadline);
+        // Panic if the time driver is not enabled (backwards compat)
+        _ = handle.driver().time();
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         let inner = {
-            let handle = scheduler::Handle::current();
-            let clock = handle.driver().clock();
-            let handle = &handle.driver().time();
-            let time_source = handle.time_source();
-            let deadline_tick = time_source.deadline_to_tick(deadline);
-            let duration = deadline_tick.saturating_sub(time_source.now(clock));
-
             let location = location.expect("should have location if tracing");
@@ -279,15 +272,10 @@ let resource_span = tracing::trace_span!(
 
-            let async_op_span = resource_span.in_scope(|| {
-                tracing::trace!(
-                    target: "runtime::resource::state_update",
-                    duration = duration,
-                    duration.unit = "ms",
-                    duration.op = "override",
-                );
+            let async_op_span = tracing::trace_span!(
+                parent: &resource_span,
+                "runtime.resource.async_op",
+                source = "Sleep::new_timeout",
+            );
 
-                tracing::trace_span!("runtime.resource.async_op", source = "Sleep::new_timeout")
-            });
-
             let async_op_poll_span =
-                async_op_span.in_scope(|| tracing::trace_span!("runtime.resource.async_op.poll"));
+                tracing::trace_span!(parent: &async_op_span, "runtime.resource.async_op.poll");
 
@@ -306,3 +294,8 @@ let ctx = trace::AsyncOpTracingCtx {
 
-        Sleep { inner, entry }
+        Sleep {
+            deadline,
+            driver: handle,
+            inner,
+            timer: None,
+        }
     }
@@ -316,3 +309,3 @@
     pub fn deadline(&self) -> Instant {
-        self.entry.deadline()
+        self.deadline
     }
@@ -324,3 +317,3 @@
     pub fn is_elapsed(&self) -> bool {
-        self.entry.is_elapsed()
+        self.timer.as_ref().is_some_and(Timer::is_elapsed)
     }
@@ -358,61 +351,24 @@
     pub fn reset(self: Pin<&mut Self>, deadline: Instant) {
-        self.reset_inner(deadline);
-    }
+        let mut this = self.project();
+        *this.deadline = deadline;
 
-    /// Resets the `Sleep` instance to a new deadline without reregistering it
-    /// to be woken up.
-    ///
-    /// Calling this function allows changing the instant at which the `Sleep`
-    /// future completes without having to create new associated state and
-    /// without having it registered. This is required in e.g. the
-    /// [`crate::time::Interval`] where we want to reset the internal [Sleep]
-    /// without having it wake up the last task that polled it.
-    pub(crate) fn reset_without_reregister(self: Pin<&mut Self>, deadline: Instant) {
-        let mut me = self.project();
-        match me.entry.as_ref().flavor() {
-            crate::runtime::TimerFlavor::Traditional => {
-                me.entry.as_mut().reset(deadline, false);
-            }
-            #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-            crate::runtime::TimerFlavor::Alternative => {
-                let handle = me.entry.as_ref().scheduler_handle().clone();
-                me.entry.set(Timer::new(handle, deadline));
-            }
-        }
-    }
+        let handle = this.driver;
 
-    fn reset_inner(self: Pin<&mut Self>, deadline: Instant) {
-        let mut me = self.project();
-        match me.entry.as_ref().flavor() {
-            crate::runtime::TimerFlavor::Traditional => {
-                me.entry.as_mut().reset(deadline, true);
-            }
-            #[cfg(all(tokio_unstable, feature = "rt-multi-thread"))]
-            crate::runtime::TimerFlavor::Alternative => {
-                let handle = me.entry.as_ref().scheduler_handle().clone();
-                me.entry.set(Timer::new(handle, deadline));
-            }
-        }
-
         #[cfg(all(tokio_unstable, feature = "tracing"))]
         {
-            let _resource_enter = me.inner.ctx.resource_span.enter();
-            me.inner.ctx.async_op_span =
+            let _resource_enter = this.inner.ctx.resource_span.enter();
+            this.inner.ctx.async_op_span =
                 tracing::trace_span!("runtime.resource.async_op", source = "Sleep::reset");
-            let _async_op_enter = me.inner.ctx.async_op_span.enter();
+            let _async_op_enter = this.inner.ctx.async_op_span.enter();
 
-            me.inner.ctx.async_op_poll_span =
+            this.inner.ctx.async_op_poll_span =
                 tracing::trace_span!("runtime.resource.async_op.poll");
 
-            let duration = {
-                let clock = me.entry.as_ref().clock();
-                let time_source = me.entry.as_ref().driver().time_source();
-                let now = time_source.now(clock);
-                let deadline_tick = time_source.deadline_to_tick(deadline);
-                deadline_tick.saturating_sub(now)
-            };
-
+            let clock = handle.driver().clock();
+            let time_source = handle.driver().time().time_source();
+            let now = time_source.now(clock);
+            let tick = time_source.deadline_to_tick(deadline);
             tracing::trace!(
                 target: "runtime::resource::state_update",
-                duration = duration,
+                duration = tick.saturating_sub(now),
                 duration.unit = "ms",
@@ -422,8 +378,32 @@ duration.op = "override",
         }
+
+        match this.timer.as_mut().as_pin_mut() {
+            Some(timer) => timer.reset(handle.clone(), deadline),
+            None => {
+                let timer = Timer::new(handle.clone(), deadline);
+                this.timer.set(Some(timer));
+                this.timer.as_pin_mut().unwrap().init(deadline);
+            }
+        }
     }
 
+    /// Resets the `Sleep` instance to a new deadline.
+    ///
+    /// Unlike [`reset`][Self::reset], this __removes__ the internal timer.
+    pub(super) fn reset_without_timer(self: Pin<&mut Self>, deadline: Instant) {
+        let mut this = self.project();
+        *this.deadline = deadline;
+        this.timer.set(None);
+    }
+
     fn poll_elapsed(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Result<(), Error>> {
-        let me = self.project();
+        ready!(crate::trace::trace_leaf());
+        let mut this = self.project();
 
-        ready!(crate::trace::trace_leaf(cx));
+        #[cfg(all(tokio_unstable, feature = "tracing"))]
+        let _res_span = this.inner.ctx.resource_span.enter();
+        #[cfg(all(tokio_unstable, feature = "tracing"))]
+        let _ao_span = this.inner.ctx.async_op_span.enter();
+        #[cfg(all(tokio_unstable, feature = "tracing"))]
+        let _ao_poll_span = this.inner.ctx.async_op_poll_span.enter();
 
@@ -440,3 +420,30 @@ // Keep track of task budget
 
-        let result = me.entry.poll_elapsed(cx).map(move |r| {
+        let timer = match this.timer.as_mut().as_pin_mut() {
+            Some(timer) => timer,
+            None => {
+                let handle = this.driver;
+
+                #[cfg(all(tokio_unstable, feature = "tracing"))]
+                {
+                    let clock = handle.driver().clock();
+                    let time_source = handle.driver().time().time_source();
+                    let now = time_source.now(clock);
+                    let tick = time_source.deadline_to_tick(*this.deadline);
+                    tracing::trace!(
+                        target: "runtime::resource::state_update",
+                        duration = tick.saturating_sub(now),
+                        duration.unit = "ms",
+                        duration.op = "override",
+                    );
+                }
+
+                let timer = Timer::new(handle.clone(), *this.deadline);
+                this.timer.set(Some(timer));
+                let mut timer = this.timer.as_pin_mut().unwrap();
+                timer.as_mut().init(*this.deadline);
+                timer
+            }
+        };
+
+        let result = timer.poll_elapsed(cx).map(move |r| {
             coop.made_progress();
@@ -466,10 +473,4 @@ r
     // really do much better if we passed the error onwards.
-    fn poll(mut self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Self::Output> {
-        #[cfg(all(tokio_unstable, feature = "tracing"))]
-        let _res_span = self.inner.ctx.resource_span.clone().entered();
-        #[cfg(all(tokio_unstable, feature = "tracing"))]
-        let _ao_span = self.inner.ctx.async_op_span.clone().entered();
-        #[cfg(all(tokio_unstable, feature = "tracing"))]
-        let _ao_poll_span = self.inner.ctx.async_op_poll_span.clone().entered();
-        match ready!(self.as_mut().poll_elapsed(cx)) {
+    fn poll(self: Pin<&mut Self>, cx: &mut task::Context<'_>) -> Poll<Self::Output> {
+        match ready!(self.poll_elapsed(cx)) {
             Ok(()) => Poll::Ready(()),
@@ -476,0 +477,0 @@ Err(e) => panic!("timer error: {e}"),

src/time/timeout.rs

@@ -24,3 +24,3 @@ //! Allows a future to execute for a maximum amount of time.
 ///
-/// Note that the timeout is checked before polling the future, so if the future
+/// Note that the future is polled before the timeout is checked, so if the future
 /// does not yield during execution then it is possible for the future to complete
@@ -27,0 +27,0 @@ /// and exceed the timeout _without_ returning an error.

src/util/blocking_check.rs

@@ -27,4 +27,4 @@ #[cfg(unix)]
     // we cannot retrieve the nonblocking status on windows
-    // and i dont know how to support wasi yet
+    // and i don't know how to support wasi yet
     Ok(())
 }

src/util/idle_notified_set.rs

@@ -16,8 +16,5 @@ //! This module defines an `IdleNotifiedSet`, which is a collection of elements.
 use crate::loom::sync::{Arc, Mutex};
-use crate::util::linked_list::{self, Link};
+use crate::util::linked_list::{self, Link, LinkedList};
 use crate::util::{waker_ref, Wake};
 
-type LinkedList<T> =
-    linked_list::LinkedList<ListEntry<T>, <ListEntry<T> as linked_list::Link>::Target>;
-
 /// This is the main handle to the collection.
@@ -51,4 +48,4 @@ pub(crate) struct IdleNotifiedSet<T> {
 struct ListsInner<T> {
-    notified: LinkedList<T>,
-    idle: LinkedList<T>,
+    notified: LinkedList<ListEntry<T>>,
+    idle: LinkedList<ListEntry<T>>,
     /// Whenever an element in the `notified` list is woken, this waker will be
@@ -238,3 +235,3 @@ /// notified and consumed, if it exists.
     pub(crate) fn for_each<F: FnMut(&mut T)>(&mut self, mut func: F) {
-        fn get_ptrs<T>(list: &mut LinkedList<T>, ptrs: &mut Vec<*mut T>) {
+        fn get_ptrs<T>(list: &mut LinkedList<ListEntry<T>>, ptrs: &mut Vec<*mut T>) {
             let mut node = list.last();
@@ -297,3 +294,3 @@
         struct AllEntries<T, F: FnMut(T)> {
-            all_entries: LinkedList<T>,
+            all_entries: LinkedList<ListEntry<T>>,
             func: F,
@@ -353,3 +350,6 @@ }
 /// that setting `my_list` to `Neither` is ok.
-unsafe fn move_to_new_list<T>(from: &mut LinkedList<T>, to: &mut LinkedList<T>) {
+unsafe fn move_to_new_list<T>(
+    from: &mut LinkedList<ListEntry<T>>,
+    to: &mut LinkedList<ListEntry<T>>,
+) {
     while let Some(entry) = from.pop_back() {
@@ -356,0 +356,0 @@ entry.my_list.with_mut(|ptr| {

src/util/linked_list.rs

@@ -18,3 +18,3 @@ #![cfg_attr(not(feature = "full"), allow(dead_code))]
 use core::fmt;
-use core::marker::{PhantomData, PhantomPinned};
+use core::marker::PhantomPinned;
 use core::mem::ManuallyDrop;
@@ -27,15 +27,12 @@ use core::ptr::{self, NonNull};
 /// responsibility to ensure the list is empty before dropping it.
-pub(crate) struct LinkedList<L, T> {
+pub(crate) struct LinkedList<L: Link> {
     /// Linked list head
-    head: Option<NonNull<T>>,
+    head: Option<NonNull<L::Target>>,
 
     /// Linked list tail
-    tail: Option<NonNull<T>>,
-
-    /// Node type marker.
-    _marker: PhantomData<*const L>,
+    tail: Option<NonNull<L::Target>>,
 }
 
-unsafe impl<L: Link> Send for LinkedList<L, L::Target> where L::Target: Send {}
-unsafe impl<L: Link> Sync for LinkedList<L, L::Target> where L::Target: Sync {}
+unsafe impl<L: Link> Send for LinkedList<L> where L::Target: Send {}
+unsafe impl<L: Link> Sync for LinkedList<L> where L::Target: Sync {}
 
@@ -62,3 +59,2 @@ /// Defines how a type is tracked within a linked list.
     /// Convert the handle to a raw pointer without consuming the handle.
-    #[allow(clippy::wrong_self_convention)]
     fn as_raw(handle: &Self::Handle) -> NonNull<Self::Target>;
@@ -118,14 +114,11 @@
 
-impl<L, T> LinkedList<L, T> {
+impl<L: Link> LinkedList<L> {
     /// Creates an empty linked list.
-    pub(crate) const fn new() -> LinkedList<L, T> {
+    pub(crate) const fn new() -> LinkedList<L> {
         LinkedList {
             head: None,
             tail: None,
-            _marker: PhantomData,
         }
     }
-}
 
-impl<L: Link> LinkedList<L, L::Target> {
     /// Adds an element first in the list.
@@ -248,3 +241,3 @@ pub(crate) fn push_front(&mut self, val: L::Handle) {
 
-impl<L: Link> fmt::Debug for LinkedList<L, L::Target> {
+impl<L: Link> fmt::Debug for LinkedList<L> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -265,3 +258,3 @@ f.debug_struct("LinkedList")
 ))]
-impl<L: Link> LinkedList<L, L::Target> {
+impl<L: Link> LinkedList<L> {
     pub(crate) fn last(&self) -> Option<&L::Target> {
@@ -273,3 +266,3 @@ let tail = self.tail.as_ref()?;
 
-impl<L: Link> Default for LinkedList<L, L::Target> {
+impl<L: Link> Default for LinkedList<L> {
     fn default() -> Self {
@@ -283,12 +276,12 @@ Self::new()
 cfg_io_driver_impl! {
-    pub(crate) struct DrainFilter<'a, T: Link, F> {
-        list: &'a mut LinkedList<T, T::Target>,
+    pub(crate) struct DrainFilter<'a, L: Link, F> {
+        list: &'a mut LinkedList<L>,
         filter: F,
-        curr: Option<NonNull<T::Target>>,
+        curr: Option<NonNull<L::Target>>,
     }
 
-    impl<T: Link> LinkedList<T, T::Target> {
-        pub(crate) fn drain_filter<F>(&mut self, filter: F) -> DrainFilter<'_, T, F>
+    impl<L: Link> LinkedList<L> {
+        pub(crate) fn drain_filter<F>(&mut self, filter: F) -> DrainFilter<'_, L, F>
         where
-            F: FnMut(&T::Target) -> bool,
+            F: FnMut(&L::Target) -> bool,
         {
@@ -304,8 +297,7 @@ let curr = self.head;
 
-    impl<'a, T, F> Iterator for DrainFilter<'a, T, F>
+    impl<'a, L: Link, F> Iterator for DrainFilter<'a, L, F>
     where
-        T: Link,
-        F: FnMut(&T::Target) -> bool,
+        F: FnMut(&L::Target) -> bool,
     {
-        type Item = T::Handle;
+        type Item = L::Handle;
 
@@ -315,3 +307,3 @@ fn next(&mut self) -> Option<Self::Item> {
                 // safety: the pointer references data contained by the list
-                self.curr = unsafe { T::pointers(curr).as_ref() }.get_next();
+                self.curr = unsafe { L::pointers(curr).as_ref() }.get_next();
 
@@ -330,6 +322,6 @@ // safety: the value is still owned by the linked list.
 cfg_taskdump! {
-    impl<T: Link> LinkedList<T, T::Target> {
+    impl<L: Link> LinkedList<L> {
         pub(crate) fn for_each<F>(&mut self, mut f: F)
         where
-            F: FnMut(&T::Handle),
+            F: FnMut(&L::Handle),
         {
@@ -340,5 +332,5 @@ let mut next = self.head;
                 unsafe {
-                    let handle = ManuallyDrop::new(T::from_raw(curr));
+                    let handle = ManuallyDrop::new(L::from_raw(curr));
                     f(&handle);
-                    next = T::pointers(curr).as_ref().get_next();
+                    next = L::pointers(curr).as_ref().get_next();
                 }
@@ -367,15 +359,12 @@ }
     /// at the guard node itself.
-    pub(crate) struct GuardedLinkedList<L, T> {
+    pub(crate) struct GuardedLinkedList<L: Link> {
         /// Pointer to the guard node.
-        guard: NonNull<T>,
-
-        /// Node type marker.
-        _marker: PhantomData<*const L>,
+        guard: NonNull<L::Target>,
     }
 
-    impl<L: Link> LinkedList<L, L::Target> {
+    impl<L: Link> LinkedList<L> {
         /// Turns a linked list into the guarded version by linking the guard node
         /// with the head and tail nodes. Like with other nodes, you should guarantee
         /// that the guard node is pinned in memory.
-        pub(crate) fn into_guarded(self, guard_handle: L::Handle) -> GuardedLinkedList<L, L::Target> {
+        pub(crate) fn into_guarded(self, guard_handle: L::Handle) -> GuardedLinkedList<L> {
             // `guard_handle` is a NonNull pointer, we don't have to care about dropping it.
@@ -402,7 +391,7 @@ let guard = L::as_raw(&guard_handle);
 
-            GuardedLinkedList { guard, _marker: PhantomData }
+            GuardedLinkedList { guard }
         }
     }
 
-    impl<L: Link> GuardedLinkedList<L, L::Target> {
+    impl<L: Link> GuardedLinkedList<L> {
         fn tail(&self) -> Option<NonNull<L::Target>> {
@@ -532,3 +521,3 @@ let tail_ptr = unsafe {
 
-    fn collect_list(list: &mut LinkedList<&'_ Entry, <&'_ Entry as Link>::Target>) -> Vec<i32> {
+    fn collect_list(list: &mut LinkedList<&'_ Entry>) -> Vec<i32> {
         let mut ret = vec![];
@@ -543,6 +532,3 @@
 
-    fn push_all<'a>(
-        list: &mut LinkedList<&'a Entry, <&'_ Entry as Link>::Target>,
-        entries: &[Pin<&'a Entry>],
-    ) {
+    fn push_all<'a>(list: &mut LinkedList<&'a Entry>, entries: &[Pin<&'a Entry>]) {
         for entry in entries.iter() {
@@ -571,3 +557,3 @@ list.push_front(*entry);
     fn const_new() {
-        const _: LinkedList<&Entry, <&Entry as Link>::Target> = LinkedList::new();
+        const _: LinkedList<&Entry> = LinkedList::new();
     }
@@ -600,3 +586,3 @@
 
-        let mut list = LinkedList::<&Entry, <&Entry as Link>::Target>::new();
+        let mut list = LinkedList::<&Entry>::new();
 
@@ -758,3 +744,3 @@ list.push_front(a.as_ref());
             // Remove missing
-            let mut list = LinkedList::<&Entry, <&Entry as Link>::Target>::new();
+            let mut list = LinkedList::<&Entry>::new();
 
@@ -788,3 +774,3 @@ list.push_front(b.as_ref());
 
-        let mut ll = LinkedList::<&Entry, <&Entry as Link>::Target>::new();
+        let mut ll = LinkedList::<&Entry>::new();
         let mut reference = VecDeque::new();
@@ -791,0 +777,0 @@

src/util/rand.rs

@@ -42,9 +42,4 @@ cfg_rt! {
         let one = (seed >> 32) as u32;
-        let mut two = seed as u32;
+        let two = seed as u32;
 
-        if two == 0 {
-            // This value cannot be zero
-            two = 1;
-        }
-
         Self::from_pair(one, two)
@@ -54,3 +49,7 @@ }
     fn from_pair(s: u32, r: u32) -> Self {
-        Self { s, r }
+        if s == 0 && r == 0 {
+            Self { s: 0, r: 1 }
+        } else {
+            Self { s, r }
+        }
     }
@@ -98,1 +97,20 @@ }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn non_zero_seed_from_u64() {
+        let seed = RngSeed::from_u64(0);
+        assert_eq!(seed.s, 0);
+        assert_eq!(seed.r, 1);
+    }
+
+    #[test]
+    fn non_zero_seed_from_pair() {
+        let seed = RngSeed::from_pair(0, 0);
+        assert_eq!(seed.s, 0);
+        assert_eq!(seed.r, 1);
+    }
+}

src/util/sharded_list.rs

@@ -15,4 +15,4 @@ use std::ptr::NonNull;
 /// Note: Due to its inner sharded design, the order of nodes cannot be guaranteed.
-pub(crate) struct ShardedList<L, T> {
-    lists: Box<[Mutex<LinkedList<L, T>>]>,
+pub(crate) struct ShardedList<L: ShardedListItem> {
+    lists: Box<[Mutex<LinkedList<L>>]>,
     added: MetricAtomicU64,
@@ -36,3 +36,11 @@ count: MetricAtomicUsize,
 
-impl<L, T> ShardedList<L, T> {
+/// Used to get the lock of shard.
+pub(crate) struct ShardGuard<'a, L: Link> {
+    lock: MutexGuard<'a, LinkedList<L>>,
+    added: &'a MetricAtomicU64,
+    count: &'a MetricAtomicUsize,
+    id: usize,
+}
+
+impl<L: ShardedListItem> ShardedList<L> {
     /// Creates a new and empty sharded linked list with the specified size.
@@ -43,8 +51,5 @@ pub(crate) fn new(sharded_size: usize) -> Self {
         let shard_mask = sharded_size - 1;
-        let mut lists = Vec::with_capacity(sharded_size);
-        for _ in 0..sharded_size {
-            lists.push(Mutex::new(LinkedList::<L, T>::new()))
-        }
+        let lists = std::iter::repeat_with(|| Mutex::new(LinkedList::new()));
         Self {
-            lists: lists.into_boxed_slice(),
+            lists: lists.take(sharded_size).collect(),
             added: MetricAtomicU64::new(0),
@@ -55,13 +60,3 @@ count: MetricAtomicUsize::new(0),
     }
-}
 
-/// Used to get the lock of shard.
-pub(crate) struct ShardGuard<'a, L, T> {
-    lock: MutexGuard<'a, LinkedList<L, T>>,
-    added: &'a MetricAtomicU64,
-    count: &'a MetricAtomicUsize,
-    id: usize,
-}
-
-impl<L: ShardedListItem> ShardedList<L, L::Target> {
     /// Removes the last element from a list specified by `shard_id` and returns it, or None if it is
@@ -99,3 +94,3 @@ /// empty.
     /// Gets the lock of `ShardedList`, makes us have the write permission.
-    pub(crate) fn lock_shard(&self, val: &L::Handle) -> ShardGuard<'_, L, L::Target> {
+    pub(crate) fn lock_shard(&self, val: &L::Handle) -> ShardGuard<'_, L> {
         let id = unsafe { L::get_shard_id(L::as_raw(val)) };
@@ -137,3 +132,3 @@ ShardGuard {
     #[inline]
-    fn shard_inner(&self, id: usize) -> MutexGuard<'_, LinkedList<L, <L as Link>::Target>> {
+    fn shard_inner(&self, id: usize) -> MutexGuard<'_, LinkedList<L>> {
         // Safety: This modulo operation ensures that the index is not out of bounds.
@@ -144,3 +139,3 @@ unsafe { self.lists.get_unchecked(id & self.shard_mask).lock() }
 
-impl<'a, L: ShardedListItem> ShardGuard<'a, L, L::Target> {
+impl<'a, L: ShardedListItem> ShardGuard<'a, L> {
     /// Push a value to this shard.
@@ -157,3 +152,3 @@ pub(crate) fn push(mut self, val: L::Handle) {
 cfg_taskdump! {
-    impl<L: ShardedListItem> ShardedList<L, L::Target> {
+    impl<L: ShardedListItem> ShardedList<L> {
         pub(crate) fn for_each<F>(&self, mut f: F)
@@ -160,0 +155,0 @@ where

tests/dump.rs

@@ -5,3 +5,8 @@ #![cfg(all(
     target_os = "linux",
-    any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    any(
+        target_arch = "aarch64",
+        target_arch = "x86",
+        target_arch = "x86_64",
+        target_arch = "s390x"
+    )
 ))]
@@ -8,0 +13,0 @@

tests/fs_uring_cancel_open.rs

@@ -16,12 +16,23 @@ //! Uring file operations tests.
 use tempfile::NamedTempFile;
+use tokio_test::assert_pending;
 
+use crate::support::io_uring::{assert_fds_are_not_leaking, io_uring_supported};
+
+mod support {
+    pub(crate) mod io_uring;
+}
+
 // see: https://github.com/tokio-rs/tokio/issues/7979
 #[tokio::test]
 async fn file_descriptors_are_closed_when_cancelling_open_op() {
+    if !io_uring_supported() {
+        return;
+    }
+
     let tmp = NamedTempFile::new().unwrap();
     let path = tmp.path().to_path_buf();
-
+    let file_number = 128;
     let fd_count_before_opens = fs::read_dir("/proc/self/fd").unwrap().count();
 
-    for _ in 0..128 {
+    for _ in 0..file_number {
         let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel();
@@ -41,6 +52,4 @@
                 // If io_uring is enabled (and not falling back to the thread pool),
-                // the first poll should return Pending. We don't check if the result
-                // is actually pending because we run some checks on old kernel that
-                // do not support uring.
-                let _pending = Box::pin(fut).poll_unpin(cx);
+                // the first poll should return Pending.
+                assert_pending!(Box::pin(fut).poll_unpin(cx));
 
@@ -63,12 +72,3 @@ tx.send(()).unwrap();
 
-    let fd_count_after_cancel = fs::read_dir("/proc/self/fd").unwrap().count();
-    let leaked = fd_count_after_cancel.saturating_sub(fd_count_before_opens);
-
-    // Since we are opening 128 files, we expect that the related fds
-    // related to this operation will be closed. Since some other fds
-    // can be opened in the meantime, we expect this number to be higher
-    // than the counter before opening the files. This number could be
-    // lower, but to avoid test flakiness we check that this is at most
-    // half the number of the file we opened to check if there's a leak.
-    assert!(leaked <= 64);
+    assert_fds_are_not_leaking(fd_count_before_opens, file_number, 1).await
 }

tests/fs_uring_completed_then_dropped_before_repoll.rs

@@ -9,2 +9,7 @@ #![cfg(all(
 
+mod support {
+    pub(crate) mod io_uring;
+}
+
+use std::fmt::Debug;
 use std::fs;
@@ -20,3 +25,6 @@ use std::future::Future;
 use tokio::time::timeout;
+use tokio_test::assert_pending;
 
+use crate::support::io_uring::{assert_fds_are_not_leaking, io_uring_supported};
+
 /// Count currently-open fds in this process.
@@ -47,10 +55,13 @@ fn fd_count() -> usize {
 
-impl<F: Future> Future for PollOpenOnceThenNeverRepoll<F> {
-    type Output = ();
+impl<F: Future> Future for PollOpenOnceThenNeverRepoll<F>
+where
+    F::Output: Debug,
+{
+    type Output = F::Output;
 
     fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
         if !self.polled_once {
-            // We don't check if the result is actually pending because
-            // we run some checks on old kernel that do not support uring.
-            let _pending = self.inner.as_mut().poll(cx);
+            // If io_uring is enabled (and not falling back to the thread pool),
+            // the first poll should return Pending.
+            assert_pending!(self.inner.as_mut().poll(cx));
 
@@ -86,3 +97,3 @@ self.polled_once = true;
         }
-        .await;
+        .await
     });
@@ -105,2 +116,6 @@
 fn uring_completed_then_dropped() {
+    if !io_uring_supported() {
+        return;
+    }
+
     let rt = tokio::runtime::Builder::new_multi_thread()
@@ -116,21 +131,10 @@ .worker_threads(1)
         let path = tmp.path().to_path_buf();
+        let file_number = 128;
 
-        for _ in 0..128 {
+        for _ in 0..file_number {
             completed_then_dropped_before_repoll(path.clone()).await;
         }
 
-        // Give completions a moment to settle before counting fds.
-        tokio::time::sleep(Duration::from_millis(250)).await;
-
-        let after = fd_count();
-        let leaked = after.saturating_sub(before);
-
-        // Since we are opening 128 files, we expect that the related fds
-        // related to this operation will be closed. Since some other fds
-        // can be opened in the meantime, we expect this number to be higher
-        // than the counter before opening the files. This number could be
-        // lower, but to avoid test flakiness we check that this is at most
-        // half the number of the file we opened to check if there's a leak.
-        assert!(leaked <= 64);
+        assert_fds_are_not_leaking(before, file_number, 1).await
     });
 }

tests/fs_uring_runtime_shutdown.rs

@@ -15,8 +15,20 @@ //! Uring file operations tests.
 use std::task::Poll;
+use std::time::{Duration, Instant};
 use tempfile::NamedTempFile;
 use tokio::runtime::Builder;
+use tokio_test::assert_pending;
 
+use crate::support::io_uring::io_uring_supported;
+
+mod support {
+    pub(crate) mod io_uring;
+}
+
 // see: https://github.com/tokio-rs/tokio/issues/7979
 #[test]
 fn shutdown_runtime_while_performing_io_uring_ops() {
+    if !io_uring_supported() {
+        return;
+    }
+
     let tmp = NamedTempFile::new().unwrap();
@@ -45,6 +57,4 @@ let path = tmp.path().to_path_buf();
                     // If io_uring is enabled (and not falling back to the thread pool),
-                    // the first poll should return Pending. We don't check if the result
-                    // is actually a pending because we run some CI checks based on old
-                    // kernels that don't support uring.
-                    let _pending = Box::pin(fut).poll_unpin(cx);
+                    // the first poll should return Pending.
+                    assert_pending!(Box::pin(fut).poll_unpin(cx));
 
@@ -70,12 +80,20 @@ tx.send(()).unwrap();
 
-    let fd_count_after_cancel = fs::read_dir("/proc/self/fd").unwrap().count();
-    let leaked = fd_count_after_cancel.saturating_sub(fd_count_before_opens);
+    let fd_check_start = Instant::now();
 
-    // Since we are opening 128 files, we expect that the related fds
-    // related to this operation will be closed. Since some other fds
-    // can be opened in the meantime, we expect this number to be higher
-    // than the counter before opening the files. This number could be
-    // lower, but to avoid test flakiness we check that this is at most
-    // half the number of the file we opened to check if there's a leak.
-    assert!(leaked <= 64);
+    while fd_check_start.elapsed() < Duration::from_secs(1) {
+        let fd_count_after_cancel = fs::read_dir("/proc/self/fd").unwrap().count();
+        let leaked = fd_count_after_cancel.saturating_sub(fd_count_before_opens);
+
+        // Since we are opening 128 files, we expect that the related fds
+        // related to this operation will be closed. Since some other fds
+        // can be opened in the meantime, we expect this number to be higher
+        // than the counter before opening the files. This number could be
+        // lower, but to avoid test flakiness we check that this is at most
+        // half the number of the file we opened to check if there's a leak.
+        if leaked <= 64 {
+            // test success
+            return;
+        }
+    }
+    panic!("Number of FDs is staying above 64. There is probably an FD leak.");
 }

tests/io_chain.rs

@@ -17,1 +17,13 @@ #![warn(rust_2018_idioms)]
 }
+
+#[tokio::test]
+async fn empty_read_does_not_skip_first() {
+    let mut buf = Vec::new();
+    let rd1: &[u8] = b"hello ";
+    let rd2: &[u8] = b"world";
+
+    let mut rd = rd1.chain(rd2);
+    assert_eq!(assert_ok!(rd.read(&mut []).await), 0);
+    assert_ok!(rd.read_to_end(&mut buf).await);
+    assert_eq!(buf, b"hello world");
+}

tests/io_copy_bidirectional.rs

@@ -62,3 +62,2 @@ #![warn(rust_2018_idioms)]
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 async fn test_basic_transfer() {
@@ -75,3 +74,2 @@ symmetric(|_handle, mut a, mut b| async move {
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 async fn test_transfer_after_close() {
@@ -96,3 +94,2 @@ symmetric(|handle, mut a, mut b| async move {
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 async fn blocking_one_side_does_not_block_other() {
@@ -99,0 +96,0 @@ symmetric(|handle, mut a, mut b| async move {

tests/io_driver_drop.rs

@@ -9,3 +9,2 @@ #![warn(rust_2018_idioms)]
 #[test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn tcp_doesnt_block() {
@@ -33,3 +32,2 @@ let rt = rt();
 #[test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn drop_wakes() {
@@ -36,0 +34,0 @@ let rt = rt();

tests/io_driver.rs

@@ -35,3 +35,2 @@ #![warn(rust_2018_idioms)]
 #[test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn test_drop_on_notify() {
@@ -95,3 +94,2 @@ // When the reactor receives a kernel notification, it notifies the
 )]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn panics_when_io_disabled() {
@@ -98,0 +96,0 @@ let rt = runtime::Builder::new_current_thread().build().unwrap();

tests/net_bind_resource.rs

 #![warn(rust_2018_idioms)]
 // WASIp1 doesn't support bind
-// No `socket` on miri.
 #![cfg(all(
@@ -10,3 +9,2 @@ feature = "net",
     not(all(target_os = "wasi", target_env = "p1")),
-    not(miri)
 ))]
@@ -20,3 +18,2 @@
 #[should_panic]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn no_runtime_panics_binding_net_tcp_listener() {
@@ -23,0 +20,0 @@ let listener = net::TcpListener::bind("127.0.0.1:0").expect("failed to bind listener");

tests/net_lookup_host.rs

@@ -40,3 +40,2 @@ // WASIp1 doesn't support direct socket operations
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `getaddrinfo` in miri.
 async fn resolve_dns() -> io::Result<()> {
@@ -43,0 +42,0 @@ let mut hosts = net::lookup_host("localhost:3000").await?;

tests/net_panic.rs

@@ -15,3 +15,3 @@ #![warn(rust_2018_idioms)]
 #[test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No UDP sockets in miri.
 fn udp_socket_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
@@ -39,3 +39,2 @@ use std::net::SocketAddr;
 #[test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn tcp_listener_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
@@ -59,3 +58,2 @@ let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
 #[test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn tcp_stream_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
@@ -82,3 +80,3 @@ let std_listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
 #[cfg(unix)]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets in miri.
 fn unix_listener_bind_panic_caller() -> Result<(), Box<dyn Error>> {
@@ -105,3 +103,3 @@ use tokio::net::UnixListener;
 #[cfg(unix)]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets in miri.
 fn unix_listener_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
@@ -129,3 +127,3 @@ use tokio::net::UnixListener;
 #[cfg(unix)]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets in miri.
 fn unix_stream_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
@@ -154,3 +152,3 @@ use tokio::net::UnixStream;
 #[cfg(unix)]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets in miri.
 fn unix_datagram_from_std_panic_caller() -> Result<(), Box<dyn Error>> {
@@ -157,0 +155,0 @@ use std::os::unix::net::UnixDatagram as StdUDS;

tests/net_quickack.rs

@@ -11,3 +11,3 @@ #![warn(rust_2018_idioms)]
 ))]
-#![cfg(not(miri))] // No `socket` in miri.
+#![cfg(not(miri))] // Miri doesn't support TCP quickack socket option.
 
@@ -14,0 +14,0 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};

tests/net_unix_pipe.rs

@@ -138,3 +138,3 @@ #![cfg(feature = "full")]
 #[cfg(any(target_os = "linux", target_os = "android"))]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No `mkfifo` in miri.
 async fn fifo_resilient_reader() -> io::Result<()> {
@@ -141,0 +141,0 @@ const DATA: &[u8] = b"this is some data to write to the fifo";

tests/no_rt.rs

@@ -23,3 +23,2 @@ #![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi does not support panic recovery
 )]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn panics_when_no_reactor() {
@@ -41,3 +40,2 @@ let srv = TcpListener::bind("127.0.0.1:0").unwrap();
 )]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn io_panics_when_no_tokio_context() {
@@ -44,0 +42,0 @@ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();

tests/rt_common.rs

 #![allow(clippy::needless_range_loop)]
 #![warn(rust_2018_idioms)]
 #![cfg(feature = "full")]
-#![cfg(not(miri))]
 
@@ -191,2 +190,3 @@ // Tests to run on both current-thread & multi-thread runtime variants.
     #[test]
+    #[cfg_attr(miri, ignore)] // Too slow on miri
     fn spawn_many_from_block_on() {
@@ -243,2 +243,3 @@ use tokio::sync::mpsc;
     #[test]
+    #[cfg_attr(miri, ignore)] // Too slow on miri
     fn spawn_many_from_task() {
@@ -512,3 +513,2 @@ use tokio::sync::mpsc;
     #[cfg(not(target_os="wasi"))] // Wasi does not support bind
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     #[test]
@@ -588,3 +588,2 @@ fn block_on_socket() {
     #[cfg(not(target_os="wasi"))] // Wasi does not support bind
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     #[test]
@@ -660,3 +659,2 @@ fn socket_from_blocking() {
     #[cfg(not(windows))]
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     #[cfg(not(target_os="wasi"))] // Wasi does not support bind or threads
@@ -716,3 +714,2 @@ fn io_driver_called_when_under_load() {
     #[cfg(not(target_os="wasi"))]
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     fn yield_defers_until_park() {
@@ -735,3 +732,2 @@ for _ in 0..10 {
     #[cfg(not(target_os="wasi"))]
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     fn coop_yield_defers_until_park() {
@@ -846,3 +842,2 @@ for _ in 0..10 {
     #[cfg(not(target_os="wasi"))] // Wasi does not support threads
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     #[test]
@@ -1013,3 +1008,3 @@ fn client_server_block_on() {
     #[cfg(not(target_os="wasi"))] // Wasi doesn't support UDP or bind()
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
+    #[cfg_attr(miri, ignore)] // No UDP sockets in miri.
     #[test]
@@ -1052,2 +1047,3 @@ fn io_notify_while_shutting_down() {
     #[test]
+    #[cfg_attr(miri, ignore)] // Miri detects leaked threads (see #7010)
     fn shutdown_timeout() {
@@ -1071,2 +1067,3 @@ let (tx, rx) = oneshot::channel();
     #[test]
+    #[cfg_attr(miri, ignore)] // Miri detects leaked threads (see #7010)
     fn shutdown_timeout_0() {
@@ -1087,2 +1084,3 @@ let runtime = rt();
     #[test]
+    #[cfg_attr(miri, ignore)] // Miri detects leaked threads (see #7010)
     fn shutdown_wakeup_time() {
@@ -1149,3 +1147,2 @@ let runtime = rt();
     #[cfg(not(target_os = "wasi"))] // Wasi does not support bind
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     #[test]
@@ -1173,3 +1170,2 @@ fn local_set_block_on_socket() {
     #[cfg(not(target_os = "wasi"))] // Wasi does not support bind
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     #[test]
@@ -1295,2 +1291,3 @@ fn local_set_client_server_block_on() {
     #[test]
+    #[cfg_attr(miri, ignore)] // Too slow on miri
     fn ping_pong_saturation() {
@@ -1359,2 +1356,3 @@ use std::sync::atomic::{Ordering, AtomicBool};
     #[cfg(not(target_os="wasi"))]
+    #[cfg_attr(miri, ignore)] // Too slow on miri
     fn shutdown_concurrent_spawn() {
@@ -1361,0 +1359,0 @@ const NUM_TASKS: usize = 10_000;

tests/rt_handle_block_on.rs

 #![warn(rust_2018_idioms)]
-#![cfg(all(feature = "full", not(miri)))]
+#![cfg(feature = "full")]
 
@@ -244,3 +244,2 @@ // All io tests that deal with shutdown is currently ignored because there are known bugs in with
     #[test]
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
     fn tcp_listener_bind() {
@@ -296,3 +295,3 @@ let rt = rt();
     #[test]
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
+    #[cfg_attr(miri, ignore)] // No UDP sockets in miri.
     fn udp_socket_bind() {
@@ -458,3 +457,3 @@ let rt = rt();
     #[cfg(unix)]
-    #[cfg_attr(miri, ignore)] // No `socket` in miri.
+    #[cfg_attr(miri, ignore)] // No Unix domain sockets in miri.
     #[test]
@@ -461,0 +460,0 @@ fn unix_listener_bind() {

tests/rt_threaded.rs

 #![warn(rust_2018_idioms)]
-// Too slow on miri.
-#![cfg(all(feature = "full", not(target_os = "wasi"), not(miri)))]
+#![cfg(all(feature = "full", not(target_os = "wasi")))]
 
@@ -38,2 +37,3 @@ use tokio::io::{AsyncReadExt, AsyncWriteExt};
 #[test]
+#[cfg_attr(miri, ignore)] // Too slow on miri.
 fn many_oneshot_futures() {
@@ -102,2 +102,3 @@ // used for notifying the main thread
 #[test]
+#[cfg_attr(miri, ignore)] // Too slow on miri
 fn many_multishot_futures() {
@@ -193,3 +194,2 @@ const CHAIN: usize = 200;
 #[test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
 fn spawn_shutdown() {
@@ -238,2 +238,3 @@ let rt = rt();
 #[test]
+#[cfg_attr(miri, ignore)] // Too slow on miri
 fn drop_threadpool_drops_futures() {
@@ -329,4 +330,3 @@ for _ in 0..1_000 {
 #[test]
-// too slow on miri
-#[cfg_attr(miri, ignore)]
+#[cfg_attr(miri, ignore)] // Too slow on miri
 fn blocking() {
@@ -708,3 +708,3 @@ // used for notifying the main thread
     let deferred_task = tokio_test::task::spawn(tokio::task::yield_now());
-    let deffered_task = Arc::new(Mutex::new(deferred_task));
+    let deferred_task = Arc::new(Mutex::new(deferred_task));
 
@@ -717,3 +717,3 @@ let rt = runtime::Builder::new_multi_thread()
     let jh = {
-        let deferred_task = Arc::clone(&deffered_task);
+        let deferred_task = Arc::clone(&deferred_task);
         rt.spawn(async move {
@@ -738,3 +738,3 @@ {
     let is_woken = {
-        let lock = deffered_task.lock().unwrap();
+        let lock = deferred_task.lock().unwrap();
         lock.is_woken()
@@ -758,2 +758,3 @@ };
 #[cfg(not(tokio_no_tuning_tests))]
+#[cfg_attr(miri, ignore)] // Too slow on miri
 fn test_tuning() {
@@ -760,0 +761,0 @@ use std::sync::atomic::AtomicBool;

tests/rt_unstable_metrics.rs

@@ -803,2 +803,51 @@ #![warn(rust_2018_idioms)]
 
+#[cfg(feature = "schedule-latency")]
+#[test]
+fn schedule_latency_counts() {
+    const N: u64 = 50;
+    let rts = [
+        tokio::runtime::Builder::new_current_thread()
+            .enable_all()
+            .enable_metrics_schedule_latency_histogram()
+            .metrics_schedule_latency_histogram_configuration(HistogramConfiguration::linear(
+                Duration::from_millis(50),
+                3,
+            ))
+            .build()
+            .unwrap(),
+        tokio::runtime::Builder::new_multi_thread()
+            .worker_threads(2)
+            .enable_all()
+            .enable_metrics_schedule_latency_histogram()
+            .metrics_schedule_latency_histogram_configuration(HistogramConfiguration::linear(
+                Duration::from_millis(50),
+                3,
+            ))
+            .build()
+            .unwrap(),
+    ];
+
+    for rt in rts {
+        let metrics = rt.metrics();
+        rt.block_on(async {
+            for _ in 0..N {
+                tokio::spawn(async {}).await.unwrap();
+            }
+        });
+        drop(rt);
+
+        let num_workers = metrics.num_workers();
+        let num_buckets = metrics.schedule_latency_histogram_num_buckets();
+
+        assert!(metrics.schedule_latency_histogram_enabled());
+        assert_eq!(num_buckets, 3);
+
+        let n = (0..num_workers)
+            .flat_map(|i| (0..num_buckets).map(move |j| (i, j)))
+            .map(|(worker, bucket)| metrics.schedule_latency_histogram_bucket_count(worker, bucket))
+            .sum();
+        assert_eq!(N, n);
+    }
+}
+
 async fn try_spawn_stealable_task() -> Result<(), mpsc::RecvTimeoutError> {
@@ -805,0 +854,0 @@ // We use a blocking channel to synchronize the tasks.

tests/sync_broadcast.rs

@@ -723,1 +723,238 @@ #![allow(clippy::cognitive_complexity)]
 }
+
+// --- Lagging semantics -------------------------------------------------------
+//
+// Capacity is rounded up to the next power of two. That buffer length is the
+// maximum number of messages retained; exceeding it overwrites the oldest
+// message and causes RecvError::Lagged / TryRecvError::Lagged on the slow
+// receiver. After Lagged, the cursor points at the oldest retained message.
+
+/// A slow receiver that stays within the ring buffer receives every message
+/// in order; lagging only begins once the buffer wraps past the receiver.
+#[test]
+fn slow_receiver_within_capacity_does_not_lag() {
+    // capacity 2 → buffer length 2
+    let (tx, mut slow) = broadcast::channel(2);
+    let mut fast = tx.subscribe();
+
+    assert_ok!(tx.send(1));
+    assert_ok!(tx.send(2));
+
+    // Fast keeps up; slow has not read yet but both messages are still retained.
+    assert_eq!(assert_recv!(fast), 1);
+    assert_eq!(assert_recv!(fast), 2);
+    assert_empty!(fast);
+
+    assert_eq!(assert_recv!(slow), 1);
+    assert_eq!(assert_recv!(slow), 2);
+    assert_empty!(slow);
+}
+
+/// When sends overwrite the oldest retained values, the slow receiver sees
+/// Lagged(n) with the correct miss count, then resumes from the oldest
+/// retained message in send order.
+#[test]
+fn capacity_overflow_reports_lagged_then_oldest_retained() {
+    // capacity 2 → buffer length 2; third send overwrites the first.
+    let (tx, mut rx) = broadcast::channel(2);
+
+    assert_ok!(tx.send(10));
+    assert_ok!(tx.send(20));
+    assert_ok!(tx.send(30));
+
+    assert_lagged!(rx.try_recv(), 1);
+    // After Lagged, cursor is at oldest retained (20).
+    assert_eq!(assert_recv!(rx), 20);
+    assert_eq!(assert_recv!(rx), 30);
+    assert_empty!(rx);
+}
+
+/// Lagged(n) counts every overwritten message since the receiver's previous
+/// cursor, not merely "at least one".
+#[test]
+fn lagged_count_matches_number_of_overwritten_messages() {
+    // capacity 4 → buffer length 4
+    let (tx, mut rx) = broadcast::channel(4);
+
+    for i in 1..=4 {
+        assert_ok!(tx.send(i));
+    }
+    // Still within capacity: no lag.
+    assert_eq!(assert_recv!(rx), 1);
+
+    // Send four more without reading: overwrites 2, 3, 4, and then the slot
+    // that held 1 (already consumed). Receiver still owed 2,3,4 — all gone —
+    // so it missed 3 messages; oldest retained is 5.
+    for i in 5..=8 {
+        assert_ok!(tx.send(i));
+    }
+
+    assert_lagged!(rx.try_recv(), 3);
+    assert_eq!(assert_recv!(rx), 5);
+    assert_eq!(assert_recv!(rx), 6);
+    assert_eq!(assert_recv!(rx), 7);
+    assert_eq!(assert_recv!(rx), 8);
+    assert_empty!(rx);
+}
+
+/// Non-power-of-two capacity is rounded up; lag detection uses the rounded
+/// buffer length (e.g. capacity 3 → buffer 4).
+#[test]
+fn lag_uses_power_of_two_buffer_length() {
+    // capacity 3 → buffer length 4
+    let (tx, mut rx) = broadcast::channel(3);
+
+    for i in 1..=4 {
+        assert_ok!(tx.send(i));
+    }
+    // Four messages fit in the rounded buffer; no lag yet.
+    assert_eq!(rx.len(), 4);
+    assert_eq!(assert_recv!(rx), 1);
+
+    // After reading 1, next points at 2. Buffer holds 2,3,4,5 — still no lag.
+    assert_ok!(tx.send(5));
+    assert_eq!(rx.len(), 4);
+
+    // Overwrites 2; receiver still wants 2 → Lagged(1), resume at 3.
+    assert_ok!(tx.send(6));
+    assert_eq!(rx.len(), 5);
+    assert_lagged!(rx.try_recv(), 1);
+    assert_eq!(assert_recv!(rx), 3);
+    assert_eq!(assert_recv!(rx), 4);
+    assert_eq!(assert_recv!(rx), 5);
+    assert_eq!(assert_recv!(rx), 6);
+    assert_empty!(rx);
+}
+
+/// Async `recv` reports the same Lagged semantics as `try_recv`.
+#[tokio::test]
+async fn async_recv_lagged_then_resumes_from_oldest_retained() {
+    use broadcast::error::RecvError;
+
+    let (tx, mut rx) = broadcast::channel(2);
+
+    assert_ok!(tx.send(10));
+    assert_ok!(tx.send(20));
+    assert_ok!(tx.send(30));
+
+    assert!(matches!(rx.recv().await, Err(RecvError::Lagged(1))));
+    assert_eq!(rx.recv().await.unwrap(), 20);
+    assert_eq!(rx.recv().await.unwrap(), 30);
+}
+
+/// A receiver that lags, catches up, then lags again reports a fresh miss
+/// count based on the new gap only.
+#[test]
+fn lag_catch_up_then_lag_again() {
+    let (tx, mut rx) = broadcast::channel(2);
+
+    assert_ok!(tx.send(1));
+    assert_ok!(tx.send(2));
+    assert_ok!(tx.send(3));
+
+    assert_lagged!(rx.try_recv(), 1);
+    assert_eq!(assert_recv!(rx), 2);
+    assert_eq!(assert_recv!(rx), 3);
+    assert_empty!(rx);
+
+    // Fully caught up; another overflow lags again from a clean cursor.
+    assert_ok!(tx.send(4));
+    assert_ok!(tx.send(5));
+    assert_ok!(tx.send(6));
+
+    assert_lagged!(rx.try_recv(), 1);
+    assert_eq!(assert_recv!(rx), 5);
+    assert_eq!(assert_recv!(rx), 6);
+    assert_empty!(rx);
+}
+
+/// Only the slow receiver lags; a caught-up receiver is unaffected.
+#[test]
+fn lag_is_per_receiver() {
+    let (tx, mut slow) = broadcast::channel(2);
+    let mut fast = tx.subscribe();
+
+    assert_ok!(tx.send(1));
+    assert_ok!(tx.send(2));
+
+    assert_eq!(assert_recv!(fast), 1);
+    assert_eq!(assert_recv!(fast), 2);
+
+    assert_ok!(tx.send(3));
+    assert_ok!(tx.send(4));
+    // Fast has read through 2; buffer holds 3,4 — no lag.
+    assert_eq!(assert_recv!(fast), 3);
+    assert_eq!(assert_recv!(fast), 4);
+
+    // Slow never read; 1 and 2 were overwritten → Lagged(2), oldest is 3.
+    assert_lagged!(slow.try_recv(), 2);
+    assert_eq!(assert_recv!(slow), 3);
+    assert_eq!(assert_recv!(slow), 4);
+    assert_empty!(slow);
+    assert_empty!(fast);
+}
+
+/// If the receiver lags and more messages are sent before it reads the
+/// Lagged error's "resume" position, a second Lagged reflects the additional
+/// overwrites since the cursor was advanced.
+#[test]
+fn lag_again_before_reading_retained_messages() {
+    let (tx, mut rx) = broadcast::channel(2);
+
+    assert_ok!(tx.send(1));
+    assert_ok!(tx.send(2));
+    assert_ok!(tx.send(3));
+
+    // First lag: miss 1, cursor advances to oldest retained (value 2).
+    assert_lagged!(rx.try_recv(), 1);
+
+    // Before reading 2/3, send enough to overwrite them (and one more).
+    assert_ok!(tx.send(4));
+    assert_ok!(tx.send(5));
+    assert_ok!(tx.send(6));
+
+    // Cursor was at value 2; values 2, 3, and 4 are gone; oldest retained is 5.
+    assert_lagged!(rx.try_recv(), 3);
+    assert_eq!(assert_recv!(rx), 5);
+    assert_eq!(assert_recv!(rx), 6);
+    assert_empty!(rx);
+}
+
+/// With capacity 1 (buffer length 1), every send after the first unread one
+/// causes a lag of exactly one when the receiver finally reads.
+#[test]
+fn single_slot_capacity_lag_semantics() {
+    let (tx, mut rx) = broadcast::channel(1);
+
+    assert_ok!(tx.send(1));
+    assert_eq!(assert_recv!(rx), 1);
+
+    assert_ok!(tx.send(2));
+    assert_ok!(tx.send(3));
+
+    assert_lagged!(rx.try_recv(), 1);
+    assert_eq!(assert_recv!(rx), 3);
+    assert_empty!(rx);
+}
+
+/// `len` after lagging still counts from the old cursor until Lagged is
+/// observed and the cursor advances; then `len` reflects retained messages.
+#[test]
+fn receiver_len_after_lag_error_advances_cursor() {
+    let (tx, mut rx) = broadcast::channel(2);
+
+    assert_ok!(tx.send(1));
+    assert_ok!(tx.send(2));
+    assert_ok!(tx.send(3));
+    assert_ok!(tx.send(4));
+
+    // Missed 1 and 2; buffer holds 3,4. len counts from old cursor.
+    assert_eq!(rx.len(), 4);
+    assert_lagged!(rx.try_recv(), 2);
+    // Cursor now at oldest retained (3); two messages remain.
+    assert_eq!(rx.len(), 2);
+    assert_eq!(assert_recv!(rx), 3);
+    assert_eq!(rx.len(), 1);
+    assert_eq!(assert_recv!(rx), 4);
+    assert_eq!(rx.len(), 0);
+}

tests/sync_mpsc.rs

@@ -522,2 +522,106 @@ #![allow(clippy::redundant_clone)]
 
+#[test]
+fn failed_reserve_many_wakes_closed_receiver() {
+    let (tx, mut rx) = mpsc::channel::<()>(2);
+
+    // Hold one of the two slots so the reservation can only take the other one
+    // and has to queue for the second.
+    let permit = tx.try_reserve().unwrap();
+
+    let mut reserve = tokio_test::task::spawn(tx.reserve_many(2));
+    assert_pending!(reserve.poll());
+
+    rx.close();
+
+    let mut recv = tokio_test::task::spawn(rx.recv());
+    assert_pending!(recv.poll());
+
+    // One slot is still outstanding, so the channel is not idle and the
+    // receiver must not be woken yet.
+    drop(permit);
+    assert!(!recv.is_woken());
+
+    // The reservation fails on the closed channel and returns the slot it had
+    // already acquired, leaving the channel idle. The receiver must be woken.
+    assert_ready_err!(reserve.poll());
+    assert!(recv.is_woken());
+    assert_ready!(recv.poll());
+}
+
+#[test]
+fn cancelled_reserve_many_wakes_closed_receiver() {
+    let (tx, mut rx) = mpsc::channel::<()>(2);
+
+    let permit = tx.try_reserve().unwrap();
+
+    let mut reserve = tokio_test::task::spawn(tx.reserve_many(2));
+    assert_pending!(reserve.poll());
+
+    rx.close();
+
+    let mut recv = tokio_test::task::spawn(rx.recv());
+    assert_pending!(recv.poll());
+
+    drop(permit);
+    assert!(!recv.is_woken());
+
+    // Cancelling the reservation returns the slot it had already acquired,
+    // leaving the channel idle. The receiver must be woken.
+    drop(reserve);
+    assert!(recv.is_woken());
+    assert_ready!(recv.poll());
+}
+
+#[test]
+fn failed_reserve_wakes_closed_receiver() {
+    let (tx, mut rx) = mpsc::channel::<()>(1);
+
+    // Hold the only slot so the reservation below has to queue.
+    let permit = tx.try_reserve().unwrap();
+
+    let mut reserve = tokio_test::task::spawn(tx.reserve());
+    assert_pending!(reserve.poll());
+
+    // Returning the slot assigns it to the queued reservation, which is woken
+    // but not yet polled, so the channel stays non-idle.
+    drop(permit);
+    assert!(reserve.is_woken());
+
+    rx.close();
+
+    let mut recv = tokio_test::task::spawn(rx.recv());
+    assert_pending!(recv.poll());
+
+    // The reservation fails on the closed channel despite holding the
+    // assigned slot; returning it leaves the channel idle. The receiver must
+    // be woken.
+    assert_ready_err!(reserve.poll());
+    assert!(recv.is_woken());
+    assert_ready!(recv.poll());
+}
+
+#[test]
+fn cancelled_reserve_wakes_closed_receiver() {
+    let (tx, mut rx) = mpsc::channel::<()>(1);
+
+    let permit = tx.try_reserve().unwrap();
+
+    let mut reserve = tokio_test::task::spawn(tx.reserve());
+    assert_pending!(reserve.poll());
+
+    drop(permit);
+    assert!(reserve.is_woken());
+
+    rx.close();
+
+    let mut recv = tokio_test::task::spawn(rx.recv());
+    assert_pending!(recv.poll());
+
+    // Cancelling the reservation returns the slot it was assigned, leaving
+    // the channel idle. The receiver must be woken.
+    drop(reserve);
+    assert!(recv.is_woken());
+    assert_ready!(recv.poll());
+}
+
 #[maybe_tokio_test]
@@ -1635,2 +1739,36 @@ async fn tx_close_gets_none() {
 
+/// A task's wakers keep its memory allocation alive. This test ensures
+/// `mpsc::channel` releases the rx waker immediately upon drop to potentially
+/// free up resources.
+#[test]
+fn release_waker_on_rx_drop() {
+    use std::task::{Context, Wake, Waker};
+
+    struct DummyWaker;
+    impl Wake for DummyWaker {
+        fn wake(self: Arc<Self>) {}
+    }
+
+    // Create a dummy Arc<impl Wake> to count waker references
+    let dummy = Arc::new(DummyWaker);
+    let waker = Waker::from(dummy.clone());
+    let mut cx = Context::from_waker(&waker);
+
+    // Baseline: 2 references (`dummy` and `waker`)
+    assert_eq!(Arc::strong_count(&dummy), 2);
+
+    // Register `waker` in rx
+    let (_tx, mut rx) = mpsc::channel::<()>(1);
+    assert_pending!(rx.poll_recv(&mut cx));
+    assert_eq!(Arc::strong_count(&dummy), 3);
+
+    // Drop rx, releasing the stored waker
+    drop(rx);
+    assert_eq!(
+        Arc::strong_count(&dummy),
+        2,
+        "dropping rx did not drop its waker",
+    );
+}
+
 fn is_debug<T: fmt::Debug>(_: &T) {}

tests/task_trace_self.rs

@@ -6,3 +6,8 @@ #![allow(unknown_lints, unexpected_cfgs)]
     target_os = "linux",
-    any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")
+    any(
+        target_arch = "aarch64",
+        target_arch = "x86",
+        target_arch = "x86_64",
+        target_arch = "s390x"
+    )
 ))]
@@ -65,2 +70,6 @@
             *this.t_last = State::Alerted;
+            // `Trace::capture` does not reschedule the task. Wake the task
+            // ourselves so the wrapped future gets polled again and can make
+            // progress now that tracing has captured its state.
+            cx.waker().wake_by_ref();
             return res;
@@ -189,17 +198,12 @@ }
 
-        // Poll the future with a real waker. if it returns Ready, exit immediately
-        match this.f.as_mut().poll(cx) {
-            Poll::Ready(result) => return Poll::Ready(result),
-            Poll::Pending => {}
+        // if the future is ready, exit immediately
+        if let Poll::Ready(result) = this.f.as_mut().poll(cx) {
+            return Poll::Ready(result);
         };
 
+        // if is pending, trace its location:
         let mut logs = Vec::new();
 
-        // Tracing poll with a noop waker. If the future is at a yield
-        // point, trace_leaf fires our callback and returns Pending. We discard
-        // the result — this poll is purely for capturing the backtrace.
-        let noop = futures::task::noop_waker();
-        let mut noop_cx = Context::from_waker(&noop);
         let trace_poll = trace_with(
-            || this.f.as_mut().poll(&mut noop_cx),
+            || this.f.as_mut().poll(cx),
             |meta| trace_leaf_for_test(meta, &mut logs),
@@ -206,0 +210,0 @@ );

tests/tcp_accept.rs

 #![warn(rust_2018_idioms)]
 // WASIp1 doesn't support bind
-// No `socket` on miri.
 #![cfg(all(
@@ -10,3 +9,2 @@ feature = "net",
     not(all(target_os = "wasi", target_env = "p1")),
-    not(miri)
 ))]
@@ -13,0 +11,0 @@

tests/tcp_connect.rs

 #![warn(rust_2018_idioms)]
 // WASIp1 doesn't support bind
-// No `socket` on miri.
 #![cfg(all(
@@ -10,3 +9,2 @@ feature = "net",
     not(all(target_os = "wasi", target_env = "p1")),
-    not(miri)
 ))]
@@ -13,0 +11,0 @@

tests/tcp_echo.rs

 #![warn(rust_2018_idioms)]
 // WASIp1 doesn't support bind
-// No `socket` on miri.
 #![cfg(all(
@@ -10,3 +9,2 @@ feature = "net",
     not(all(target_os = "wasi", target_env = "p1")),
-    not(miri)
 ))]
@@ -21,3 +19,7 @@
 async fn echo_server() {
+    const BYTES: &[u8] = b"foo bar baz";
+    #[cfg(not(miri))]
     const ITER: usize = 1024;
+    #[cfg(miri)] // Use a lower iteration count with Miri because it's too slow otherwise.
+    const ITER: usize = 32;
 
@@ -29,3 +31,2 @@ let (tx, rx) = oneshot::channel();
 
-    let msg = "foo bar baz";
     tokio::spawn(async move {
@@ -36,8 +37,8 @@ let mut stream = assert_ok!(TcpStream::connect(&addr).await);
             // write
-            assert_ok!(stream.write_all(msg.as_bytes()).await);
+            assert_ok!(stream.write_all(BYTES).await);
 
             // read
-            let mut buf = [0; 11];
+            let mut buf = [0; BYTES.len()];
             assert_ok!(stream.read_exact(&mut buf).await);
-            assert_eq!(&buf[..], msg.as_bytes());
+            assert_eq!(&buf[..], BYTES);
         }
@@ -52,5 +53,5 @@
     let n = assert_ok!(io::copy(&mut rd, &mut wr).await);
-    assert_eq!(n, (ITER * msg.len()) as u64);
+    assert_eq!(n, (ITER * BYTES.len()) as u64);
 
     assert_ok!(rx.await);
 }

tests/tcp_into_split.rs

 #![warn(rust_2018_idioms)]
-#![cfg(all(feature = "full", not(target_os = "wasi"), not(miri)))] // Wasi doesn't support multithreading or peeking
-                                                                   // No `socket` on miri.
+#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support multithreading or peeking
 
@@ -5,0 +4,0 @@ use std::io::{Error, ErrorKind, Result};

tests/tcp_into_std.rs

 #![warn(rust_2018_idioms)]
 // WASIp1 doesn't support bind
-// No `socket` on miri.
 #![cfg(all(
@@ -10,3 +9,2 @@ feature = "net",
     not(all(target_os = "wasi", target_env = "p1")),
-    not(miri)
 ))]
@@ -13,0 +11,0 @@

tests/tcp_peek.rs

 #![warn(rust_2018_idioms)]
-#![cfg(all(feature = "full", not(target_os = "wasi"), not(miri)))] // Wasi doesn't support peeking
-                                                                   // No `socket` on miri.
+#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support peeking
 
@@ -5,0 +4,0 @@ use tokio::io::AsyncReadExt;

tests/tcp_shutdown.rs

 #![warn(rust_2018_idioms)]
-#![cfg(all(feature = "full", not(target_os = "wasi"), not(miri)))] // Wasi doesn't support mulithreading
-                                                                   // No `socket` on miri.
+#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support multithreading
 
@@ -35,2 +34,3 @@ use tokio::io::{self, AsyncReadExt, AsyncWriteExt};
 #[tokio::test]
+#[cfg_attr(miri, ignore = "Miri doesn't support `SO_LINGER`")]
 async fn shutdown_after_tcp_reset() {
@@ -48,3 +48,12 @@ let srv = assert_ok!(TcpListener::bind("127.0.0.1:0").await);
         dropped_rx.await.unwrap();
-        assert_ok!(AsyncWriteExt::shutdown(&mut stream).await);
+
+        // After the peer's RST (linger = 0), `shutdown` returns `Ok(())` on most
+        // platforms, but FreeBSD can surface the reset as `ConnectionReset` when
+        // the kernel processed it before `shutdown` ran. Both are valid for an
+        // already-reset connection.
+        match AsyncWriteExt::shutdown(&mut stream).await {
+            Ok(()) => {}
+            Err(e) if e.kind() == io::ErrorKind::ConnectionReset => {}
+            Err(e) => panic!("unexpected error after reset: {e:?}"),
+        }
     });
@@ -51,0 +60,0 @@

tests/tcp_socket.rs

 #![warn(rust_2018_idioms)]
 // WASIp1 doesn't support bind
-// No `socket` on miri.
 #![cfg(all(
@@ -10,3 +9,2 @@ feature = "net",
     not(all(target_os = "wasi", target_env = "p1")),
-    not(miri)
 ))]
@@ -55,2 +53,3 @@
 #[tokio::test]
+#[cfg_attr(miri, ignore = "Miri doesn't support binding before connecting")]
 async fn bind_before_connect() {
@@ -76,2 +75,3 @@ // Create server
 #[tokio::test]
+#[cfg_attr(miri, ignore = "Miri doesn't support `SO_LINGER`")]
 async fn basic_linger() {
@@ -144,5 +144,19 @@ // Create server
 
-test!(keepalive, set_keepalive(true));
+test!(
+    #[cfg_attr(
+        miri,
+        ignore = "Miri doesn't support setting the keepalive socket option"
+    )]
+    keepalive,
+    set_keepalive(true)
+);
 
-test!(reuseaddr, set_reuseaddr(true));
+test!(
+    #[cfg_attr(
+        miri,
+        ignore = "Miri doesn't support reading the reuseaddr socket option"
+    )]
+    reuseaddr,
+    set_reuseaddr(true)
+);
 
@@ -155,5 +169,16 @@ #[cfg(all(
 ))]
-test!(reuseport, set_reuseport(true));
+test!(
+    #[cfg_attr(
+        miri,
+        ignore = "Miri doesn't support setting the reuseport socket option"
+    )]
+    reuseport,
+    set_reuseport(true)
+);
 
 test!(
+    #[cfg_attr(
+        miri,
+        ignore = "Miri doesn't support setting the send buffer size socket option"
+    )]
     send_buffer_size,
@@ -165,2 +190,6 @@ set_send_buffer_size(SET_BUF_SIZE),
 test!(
+    #[cfg_attr(
+        miri,
+        ignore = "Miri doesn't support setting the receive buffer size socket option"
+    )]
     recv_buffer_size,
@@ -173,2 +202,3 @@ set_recv_buffer_size(SET_BUF_SIZE),
     #[cfg_attr(target_os = "wasi", ignore = "WASI does not yet support `SO_LINGER`")]
+    #[cfg_attr(miri, ignore = "Miri doesn't support `SO_LINGER`")]
     #[expect(deprecated, reason = "set_linger is deprecated")]
@@ -179,3 +209,7 @@ linger,
 
-test!(nodelay, set_nodelay(true));
+test!(
+    #[cfg_attr(miri, ignore = "Miri only supports `TCP_NODELAY` on connected sockets")]
+    nodelay,
+    set_nodelay(true)
+);
 
@@ -193,3 +227,7 @@ #[cfg(any(
 ))]
-test!(IPv6 tclass_v6, set_tclass_v6(96));
+#[cfg(not(miri))] // Miri doesn't support TClass.
+test!(
+    IPv6 tclass_v6,
+    set_tclass_v6(96)
+);
 
@@ -202,4 +240,5 @@ #[cfg(not(any(
     target_os = "haiku",
-    target_os = "wasi"
+    target_os = "wasi",
+    miri // Miri doesn't support TOS.
 )))]
 test!(IPv4 tos_v4, set_tos_v4(96));

tests/tcp_split.rs

 #![warn(rust_2018_idioms)]
-#![cfg(all(feature = "full", not(target_os = "wasi"), not(miri)))] // Wasi doesn't support peeking
-                                                                   // No `socket` on miri.
+#![cfg(all(feature = "full", not(target_os = "wasi")))] // Wasi doesn't support peeking
 
@@ -5,0 +4,0 @@ use std::io::Result;

tests/tcp_stream.rs

 #![warn(rust_2018_idioms)]
 // WASIp1 doesn't support bind
-// No `socket` on miri.
 #![cfg(all(
@@ -10,3 +9,2 @@ feature = "net",
     not(all(target_os = "wasi", target_env = "p1")),
-    not(miri)
 ))]
@@ -28,3 +26,3 @@
 #[cfg(not(target_os = "wasi"))] // WASI does not yet support `SO_LINGER`
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore = "Miri doesn't support `SO_LINGER`")]
 #[expect(deprecated)] // set_linger is deprecated
@@ -53,5 +51,4 @@ async fn set_linger() {
 )]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
 async fn try_read_write() {
-    const DATA: &[u8] = b"this is some data to write to the socket";
+    const DATA: &[u8] = &[2u8; 4000];
 
@@ -241,3 +238,2 @@ // Create listener
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
 async fn poll_read_ready() {
@@ -266,3 +262,2 @@ let (mut client, mut server) = create_pair().await;
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
 async fn poll_write_ready() {
@@ -325,5 +320,4 @@ let (mut client, server) = create_pair().await;
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
 async fn try_read_buf() {
-    const DATA: &[u8] = b"this is some data to write to the socket";
+    const DATA: &[u8] = &[2u8; 4000];
 
@@ -418,3 +412,2 @@ // Create listener
 )]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
 async fn read_closed() {
@@ -435,3 +428,2 @@ let (client, mut server) = create_pair().await;
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
 async fn write_closed() {
@@ -438,0 +430,0 @@ let (mut client, mut server) = create_pair().await;

tests/time_alt.rs

@@ -7,2 +7,8 @@ #![warn(rust_2018_idioms)]
 
+use std::future::Future;
+
+use futures::FutureExt;
+use futures_test::task::noop_context;
+use tokio_test::assert_pending;
+
 fn rt_combinations() -> Vec<Runtime> {
@@ -110,1 +116,115 @@ let mut rts = vec![];
 }
+
+#[test]
+/// It is possible that a timer is created in one runtime,
+/// but `.reset()` is called in a different runtime.
+/// In this case, the timer should be registered in the original runtime.
+fn reset_should_stay_on_same_runtime() {
+    let rt1 = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(1)
+        .enable_alt_timer()
+        .build()
+        .unwrap();
+
+    let rt2 = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(1)
+        .enable_alt_timer()
+        .build()
+        .unwrap();
+
+    // Register the timer into the local timer wheel of `rt1`.
+    //
+    // We cannot use bare `rt1.block_on` as the local timer wheel of `rt1`
+    // is only accessible from the worker threads of `rt1`,
+    // but `rt1.block_on` runs the future on the current thread.
+    // So we need to use `rt1.spawn` to run the future on the worker thread.
+    let sleep = rt1
+        .block_on(
+            #[allow(clippy::async_yields_async)]
+            rt1.spawn(async {
+                // 1 hour is long enough to make sure the timer is not fired before we call `reset()`.
+                tokio::time::sleep(Duration::from_secs(3600))
+            }),
+        )
+        .unwrap();
+    let mut sleep = Box::pin(sleep);
+    assert_pending!(sleep.as_mut().poll(&mut noop_context()));
+
+    // reset the timer created from `rt1` in `rt2`,
+    // which should register the timer into the local timer wheel of `rt1`.
+    let sleep = rt2
+        .block_on({
+            #[allow(clippy::async_yields_async)]
+            rt2.spawn(async move {
+                sleep
+                    .as_mut()
+                    .reset(Instant::now() + Duration::from_secs(3600));
+                sleep
+            })
+        })
+        .unwrap();
+
+    // drop `rt1` to fire all timers registered in `rt1`,
+    // including the timer we just reset.
+    drop(rt1);
+
+    // If this assertion fails, it means the timer is not registered in `rt1`.
+    // This can happen if the timer is registered in `rt2` instead of `rt1`,
+    assert!(sleep.now_or_never().is_some());
+}
+
+#[test]
+/// It is possible that a timer is created in one runtime,
+/// but `.reset()` is called in a different runtime.
+/// In this case, the timer should be registered in the original runtime.
+fn reset_should_stay_on_same_runtime2() {
+    let rt1 = tokio::runtime::Builder::new_multi_thread()
+        .worker_threads(1)
+        .enable_alt_timer()
+        .build()
+        .unwrap();
+
+    let rt2 = tokio::runtime::Builder::new_current_thread()
+        .build()
+        .unwrap();
+
+    // Register the timer into the local timer wheel of `rt1`.
+    //
+    // We cannot use bare `rt1.block_on` as the local timer wheel of `rt1`
+    // is only accessible from the worker threads of `rt1`,
+    // but `rt1.block_on` runs the future on the current thread.
+    // So we need to use `rt1.spawn` to run the future on the worker thread.
+    let sleep = rt1
+        .block_on(
+            #[allow(clippy::async_yields_async)]
+            rt1.spawn(async {
+                // 1 hour is long enough to make sure the timer is not fired before we call `reset()`.
+                tokio::time::sleep(Duration::from_secs(3600))
+            }),
+        )
+        .unwrap();
+    let mut sleep = Box::pin(sleep);
+    assert_pending!(sleep.as_mut().poll(&mut noop_context()));
+
+    // reset the timer created from `rt1` in `rt2`,
+    // which should register the timer into the local timer wheel of `rt1`.
+    let sleep = rt2
+        .block_on({
+            #[allow(clippy::async_yields_async)]
+            rt2.spawn(async move {
+                sleep
+                    .as_mut()
+                    .reset(Instant::now() + Duration::from_secs(3600));
+                sleep
+            })
+        })
+        .unwrap();
+
+    // drop `rt1` to fire all timers registered in `rt1`,
+    // including the timer we just reset.
+    drop(rt1);
+
+    // If this assertion fails, it means the timer is not registered in `rt1`.
+    // This can happen if the timer is registered in `rt2` instead of `rt1`,
+    assert!(sleep.now_or_never().is_some());
+}

tests/time_rt.rs

 #![warn(rust_2018_idioms)]
 #![cfg(feature = "full")]
 
+use futures_test::task::noop_context;
 use tokio::runtime::Runtime;
 use tokio::time::*;
+use tokio_test::assert_pending;
 
@@ -167,1 +169,29 @@ use std::sync::mpsc;
 }
+
+#[test]
+fn tickspace() {
+    use std::future::Future as _;
+    use std::thread;
+    let rt = || {
+        tokio::runtime::Builder::new_current_thread()
+            .enable_time()
+            .start_paused(true)
+            .build()
+            .unwrap()
+    };
+
+    let rt_past = rt();
+    thread::sleep(Duration::from_millis(1));
+    let rt = rt();
+
+    let _guard = rt_past.enter();
+    let mut sleep = std::pin::pin!(sleep(Duration::from_millis(1)));
+    assert_pending!(sleep.as_mut().poll(&mut noop_context()));
+
+    let deadline = sleep.deadline();
+    rt.block_on(async { sleep.as_mut().reset(deadline + Duration::from_millis(1)) });
+
+    let now = Instant::now();
+    rt_past.block_on(sleep);
+    assert_eq!(now.elapsed(), Duration::from_millis(2));
+}

tests/time_sleep.rs

@@ -13,11 +13,15 @@ #![warn(rust_2018_idioms)]
 
-#[tokio::test]
+#[tokio::test(start_paused = true)]
 async fn immediate_sleep() {
-    time::pause();
-
     let now = Instant::now();
 
-    // Ready!
-    time::sleep_until(now).await;
-    assert_elapsed!(now, ms(1));
+    let sleep = time::sleep_until(now);
+
+    tokio::pin!(sleep);
+
+    assert!(!sleep.is_elapsed());
+
+    sleep.as_mut().await;
+    assert_elapsed!(now, ms(0));
+    assert!(sleep.is_elapsed());
 }
@@ -24,0 +28,0 @@

tests/tracing_time.rs

@@ -10,2 +10,3 @@ //! Tests for time resource instrumentation.
 
+use tokio_test::task;
 use tracing_mock::{expect, subscriber};
@@ -21,2 +22,8 @@
 
+    let poll_op = || {
+        expect::event()
+            .with_target("runtime::resource::poll_op")
+            .with_fields(expect::field("op_name").with_value(&"poll_elapsed"))
+    };
+
     let state_update = expect::event()
@@ -45,18 +52,22 @@ .with_target("runtime::resource::state_update")
         .new_span(sleep_span.clone().with_ancestry(expect::is_explicit_root()))
-        .enter(sleep_span.clone())
-        .event(state_update)
         .new_span(
             async_op_span
                 .clone()
-                .with_ancestry(expect::has_contextual_parent(&sleep_span_id))
+                .with_ancestry(expect::has_explicit_parent(&sleep_span_id))
                 .with_fields(expect::field("source").with_value(&"Sleep::new_timeout")),
         )
-        .exit(sleep_span.clone())
-        .enter(async_op_span.clone())
         .new_span(
             async_op_poll_span
                 .clone()
-                .with_ancestry(expect::has_contextual_parent(&async_op_span_id)),
+                .with_ancestry(expect::has_explicit_parent(&async_op_span_id)),
         )
+        .enter(sleep_span.clone())
+        .enter(async_op_span.clone())
+        .enter(async_op_poll_span.clone())
+        .event(poll_op())
+        .event(state_update)
+        .event(poll_op())
+        .exit(async_op_poll_span.clone())
         .exit(async_op_span.clone())
+        .exit(sleep_span.clone())
         .drop_span(async_op_span)
@@ -70,3 +81,3 @@ .drop_span(async_op_poll_span)
 
-        _ = tokio::time::sleep(Duration::from_millis(7));
+        _ = task::spawn(tokio::time::sleep(Duration::from_millis(7))).poll();
     }
@@ -73,0 +84,0 @@

tests/udp.rs

 #![warn(rust_2018_idioms)]
 // WASIp1 doesn't support bind
-// No `socket` on miri.
+// No UDP sockets on miri.
 #![cfg(all(
@@ -5,0 +5,0 @@ feature = "net",

tests/uds_cred.rs

 #![warn(rust_2018_idioms)]
 #![cfg(feature = "full")]
-#![cfg(all(unix, not(target_os = "dragonfly"), not(miri)))] // No `getsockopt` on miri.
+#![cfg(all(unix, not(target_os = "dragonfly"), not(miri)))] // No `getsockopt` for Unix domain sockets on miri.
 
@@ -26,2 +26,33 @@ use tokio::net::UnixStream;
     assert_eq!(cred_a.gid(), gid);
+
+    // On platforms where `UCred::pid` is implemented and the kernel
+    // populates it, both ends of a `socketpair` must report the current
+    // process's PID.
+    //
+    // FreeBSD COMPAT32 (32-bit binary on a 64-bit kernel) leaves `cr_pid`
+    // zeroed pending FreeBSD bug 294833; the assertion is gated on 64-bit
+    // FreeBSD until that fix ships.
+    #[cfg(any(
+        target_os = "linux",
+        target_os = "android",
+        target_os = "openbsd",
+        all(target_os = "freebsd", target_pointer_width = "64"),
+        target_os = "netbsd",
+        target_os = "nto",
+        target_os = "macos",
+        target_os = "ios",
+        target_os = "tvos",
+        target_os = "watchos",
+        target_os = "visionos",
+        target_os = "solaris",
+        target_os = "illumos",
+        target_os = "redox",
+        target_os = "haiku",
+        target_os = "cygwin",
+    ))]
+    {
+        let pid = unsafe { libc::getpid() };
+        assert_eq!(cred_a.pid(), Some(pid));
+        assert_eq!(cred_b.pid(), Some(pid));
+    }
 }

tests/uds_datagram.rs

@@ -24,3 +24,3 @@ #![warn(rust_2018_idioms)]
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets on miri.
 async fn echo() -> io::Result<()> {
@@ -50,3 +50,3 @@ let dir = tempfile::tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets on miri.
 async fn echo_from() -> io::Result<()> {
@@ -124,3 +124,3 @@ let dir = tempfile::tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets on miri.
 async fn split() -> std::io::Result<()> {
@@ -150,3 +150,3 @@ let dir = tempfile::tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets on miri.
 async fn send_to_recv_from_poll() -> std::io::Result<()> {
@@ -173,3 +173,3 @@ let dir = tempfile::tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets on miri.
 async fn send_recv_poll() -> std::io::Result<()> {
@@ -198,3 +198,3 @@ let dir = tempfile::tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets on miri.
 async fn try_send_to_recv_from() -> std::io::Result<()> {
@@ -247,3 +247,3 @@ let dir = tempfile::tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets on miri.
 async fn try_recv_buf_from() -> std::io::Result<()> {
@@ -296,3 +296,3 @@ let dir = tempfile::tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` on miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets on miri.
 async fn recv_buf_from() -> std::io::Result<()> {
@@ -299,0 +299,0 @@ let tmp = tempfile::tempdir()?;

tests/uds_socket.rs

@@ -13,3 +13,3 @@ #![warn(rust_2018_idioms)]
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets in miri.
 async fn datagram_echo_server() -> io::Result<()> {
@@ -56,3 +56,3 @@ let dir = tempfile::tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets in miri.
 async fn listen_and_stream() -> std::io::Result<()> {
@@ -97,3 +97,3 @@ let dir = tempfile::Builder::new().tempdir().unwrap();
 #[tokio::test]
-#[cfg_attr(miri, ignore)] // No `socket` in miri.
+#[cfg_attr(miri, ignore)] // No Unix domain sockets in miri.
 async fn assert_usage() -> std::io::Result<()> {
@@ -100,0 +100,0 @@ let datagram_socket = UnixSocket::new_datagram()?;

tests/uds_split.rs

 #![warn(rust_2018_idioms)]
 #![cfg(feature = "full")]
 #![cfg(unix)]
-#![cfg(not(miri))] // No `socket` in miri.
+#![cfg(not(miri))] // No Unix domain sockets in miri.
 
@@ -6,0 +6,0 @@ use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};

tests/uds_stream.rs

 #![cfg(feature = "full")]
 #![warn(rust_2018_idioms)]
 #![cfg(unix)]
-#![cfg(not(miri))] // No socket in miri.
+#![cfg(not(miri))] // No Unix domain sockets on miri.
 
@@ -11,2 +11,3 @@ use std::io;
 use std::os::linux::net::SocketAddrExt;
+use std::os::unix::net::SocketAddr;
 use std::task::Poll;
@@ -48,2 +49,30 @@
 #[tokio::test]
+async fn accept_read_write_socketaddr() -> std::io::Result<()> {
+    let dir = tempfile::Builder::new()
+        .prefix("tokio-uds-tests")
+        .tempdir()
+        .unwrap();
+    let sock_path = dir.path().join("connect.sock");
+    let addr = SocketAddr::from_pathname(&sock_path)?.into();
+
+    let listener = UnixListener::bind_addr(&addr)?;
+
+    let accept = listener.accept();
+    let connect = UnixStream::connect_addr(&addr);
+    let ((mut server, _), mut client) = try_join(accept, connect).await?;
+
+    // Write to the client.
+    client.write_all(b"hello").await?;
+    drop(client);
+
+    // Read from the server.
+    let mut buf = vec![];
+    server.read_to_end(&mut buf).await?;
+    assert_eq!(&buf, b"hello");
+    let len = server.read(&mut buf).await?;
+    assert_eq!(len, 0);
+    Ok(())
+}
+
+#[tokio::test]
 async fn shutdown() -> std::io::Result<()> {
@@ -50,0 +79,0 @@ let dir = tempfile::Builder::new()

No alert changes

No metric changes