std: use futex for everything but Mutex on Apple platforms

Author: joboet

library/std/src/sys/pal/unix/futex.rs

@@ -1,5 +1,6 @@
 #![cfg(any(
     target_os = "linux",
+    target_vendor = "apple",
     target_os = "android",
     all(target_os = "emscripten", target_feature = "atomics"),
     target_os = "freebsd",
@@ -145,6 +146,189 @@ pub fn futex_wake_all(futex: &AtomicU32) {
     };
 }
 
+/// With macOS version 14.4, Apple introduced a public futex API. Unfortunately,
+/// our minimum supported version is 10.12, so we need a fallback API. Luckily
+/// for us, the underlying syscalls have been available since exactly that
+/// version, so we just use those when needed. This is private API however,
+/// which means we need to take care to avoid breakage if the syscall is removed
+/// and to avoid apps being rejected from the App Store. To do this, we use weak
+/// linkage emulation for both the public and the private API.
+///
+/// See os/os_sync_wait_on_address.h (Apple has failed to upload the documentation
+/// to their website) for documentation of the public API and
+/// https://github.com/apple-oss-distributions/xnu/blob/1031c584a5e37aff177559b9f69dbd3c8c3fd30a/bsd/sys/ulock.h#L69
+/// for the header file of the private API, along with its usage in libpthread
+/// https://github.com/apple-oss-distributions/libpthread/blob/d8c4e3c212553d3e0f5d76bb7d45a8acd61302dc/src/pthread_cond.c#L463
+#[cfg(target_vendor = "apple")]
+mod apple {
+    use crate::ffi::{c_int, c_void};
+    use crate::sys::pal::weak::weak;
+
+    pub const OS_CLOCK_MACH_ABSOLUTE_TIME: u32 = 32;
+    pub const OS_SYNC_WAIT_ON_ADDRESS_NONE: u32 = 0;
+    pub const OS_SYNC_WAKE_BY_ADDRESS_NONE: u32 = 0;
+
+    pub const UL_COMPARE_AND_WAIT: u32 = 1;
+    pub const ULF_WAKE_ALL: u32 = 0x100;
+    // The syscalls support directly returning errors instead of going through errno.
+    pub const ULF_NO_ERRNO: u32 = 0x1000000;
+
+    // These functions appeared with macOS 14.4, iOS 17.4, tvOS 17.4, watchOS 10.4, visionOS 1.1.
+    weak! {
+        pub fn os_sync_wait_on_address(*mut c_void, u64, usize, u32) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wait_on_address_with_timeout(*mut c_void, u64, usize, u32, u32, u64) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wake_by_address_any(*mut c_void, usize, u32) -> c_int
+    }
+
+    weak! {
+        pub fn os_sync_wake_by_address_all(*mut c_void, usize, u32) -> c_int
+    }
+
+    // This syscall appeared with macOS 11.0.
+    // It is used to support nanosecond precision for timeouts, among other features.
+    weak! {
+        pub fn __ulock_wait2(u32, *mut c_void, u64, u64, u64) -> c_int
+    }
+
+    // These syscalls appeared with macOS 10.12.
+    weak! {
+        pub fn __ulock_wait(u32, *mut c_void, u64, u32) -> c_int
+    }
+
+    weak! {
+        pub fn __ulock_wake(u32, *mut c_void, u64) -> c_int
+    }
+}
+
+#[cfg(target_vendor = "apple")]
+pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
+    use apple::*;
+
+    use crate::mem::size_of;
+
+    let addr = futex.as_ptr().cast();
+    let value = expected as u64;
+    let size = size_of::<u32>();
+    if let Some(timeout) = timeout {
+        let timeout_ns = timeout.as_nanos().clamp(1, u64::MAX as u128) as u64;
+        let timeout_ms = timeout.as_micros().clamp(1, u32::MAX as u128) as u32;
+
+        if let Some(wait) = os_sync_wait_on_address_with_timeout.get() {
+            let r = unsafe {
+                wait(
+                    addr,
+                    value,
+                    size,
+                    OS_SYNC_WAIT_ON_ADDRESS_NONE,
+                    OS_CLOCK_MACH_ABSOLUTE_TIME,
+                    timeout_ns,
+                )
+            };
+
+            // We promote spurious wakeups (reported as EINTR) to normal ones for
+            // simplicity and because Apple's documentation is unclear as to what the
+            // unit for the deadline of `os_sync_wait_on_address_with_deadline` is,
+            // making it hard to support timeouts in a fashion similar to the Linux
+            // futex implementation.
+            r != -1 || super::os::errno() != libc::ETIMEDOUT
+        } else if let Some(wait) = __ulock_wait2.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ns, 0)
+                    != -libc::ETIMEDOUT
+            }
+        } else if let Some(wait) = __ulock_wait.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, timeout_ms)
+                    != -libc::ETIMEDOUT
+            }
+        } else {
+            panic!("your system is below the minimum supported version of Rust");
+        }
+    } else {
+        if let Some(wait) = os_sync_wait_on_address.get() {
+            unsafe {
+                wait(addr, value, size, OS_SYNC_WAIT_ON_ADDRESS_NONE);
+            }
+        } else if let Some(wait) = __ulock_wait.get() {
+            unsafe {
+                wait(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, value, 0);
+            }
+        } else {
+            panic!("your system is below the minimum supported version of Rust");
+        }
+        true
+    }
+}
+
+#[cfg(target_vendor = "apple")]
+pub fn futex_wake(futex: &AtomicU32) -> bool {
+    use apple::*;
+
+    use crate::io::Error;
+    use crate::mem::size_of;
+
+    let addr = futex.as_ptr().cast();
+    if let Some(wake) = os_sync_wake_by_address_any.get() {
+        unsafe { wake(addr, size_of::<u32>(), OS_SYNC_WAKE_BY_ADDRESS_NONE) == 0 }
+    } else if let Some(wake) = __ulock_wake.get() {
+        // __ulock_wake can get interrupted, so retry until either waking up a
+        // waiter or failing because there are no waiters (ENOENT).
+        loop {
+            let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_NO_ERRNO, addr, 0) };
+
+            if r >= 0 {
+                return true;
+            } else {
+                match -r {
+                    libc::ENOENT => return false,
+                    libc::EINTR => continue,
+                    err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)),
+                }
+            }
+        }
+    } else {
+        panic!("your system is below the minimum supported version of Rust");
+    }
+}
+
+#[cfg(target_vendor = "apple")]
+pub fn futex_wake_all(futex: &AtomicU32) {
+    use apple::*;
+
+    use crate::io::Error;
+    use crate::mem::size_of;
+
+    let addr = futex.as_ptr().cast();
+
+    if let Some(wake) = os_sync_wake_by_address_all.get() {
+        unsafe {
+            wake(addr, size_of::<u32>(), OS_SYNC_WAKE_BY_ADDRESS_NONE);
+        }
+    } else if let Some(wake) = __ulock_wake.get() {
+        loop {
+            let r = unsafe { wake(UL_COMPARE_AND_WAIT | ULF_WAKE_ALL | ULF_NO_ERRNO, addr, 0) };
+
+            if r >= 0 {
+                return;
+            } else {
+                match -r {
+                    libc::ENOENT => return,
+                    libc::EINTR => continue,
+                    err => panic!("__ulock_wake failed: {}", Error::from_raw_os_error(err)),
+                }
+            }
+        }
+    } else {
+        panic!("your system is below the minimum supported version of Rust");
+    }
+}
+
 #[cfg(target_os = "openbsd")]
 pub fn futex_wait(futex: &AtomicU32, expected: u32, timeout: Option<Duration>) -> bool {
     use super::time::Timespec;

library/std/src/sys/pal/unix/sync/mod.rs

@@ -9,8 +9,10 @@
 )))]
 #![forbid(unsafe_op_in_unsafe_fn)]
 
+#[cfg(not(target_vendor = "apple"))]
 mod condvar;
 mod mutex;
 
+#[cfg(not(target_vendor = "apple"))]
 pub use condvar::Condvar;
 pub use mutex::Mutex;

library/std/src/sys/sync/condvar/mod.rs

@@ -2,6 +2,7 @@ cfg_if::cfg_if! {
     if #[cfg(any(
         all(target_os = "windows", not(target_vendor="win7")),
         target_os = "linux",
+        target_vendor = "apple",
         target_os = "android",
         target_os = "freebsd",
         target_os = "openbsd",

library/std/src/sys/sync/once/mod.rs

@@ -18,6 +18,10 @@ cfg_if::cfg_if! {
         target_os = "dragonfly",
         target_os = "fuchsia",
         target_os = "hermit",
+        target_os = "macos",
+        target_os = "ios",
+        target_os = "tvos",
+        target_os = "watchos",
     ))] {
         mod futex;
         pub use futex::{Once, OnceState};

library/std/src/sys/sync/rwlock/mod.rs

@@ -2,6 +2,7 @@ cfg_if::cfg_if! {
     if #[cfg(any(
         all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "linux",
+        target_vendor = "apple",
         target_os = "android",
         target_os = "freebsd",
         target_os = "openbsd",

library/std/src/sys/sync/thread_parking/darwin.rs

@@ -2,6 +2,7 @@ cfg_if::cfg_if! {
     if #[cfg(any(
         all(target_os = "windows", not(target_vendor = "win7")),
         target_os = "linux",
+        target_vendor = "apple",
         target_os = "android",
         all(target_arch = "wasm32", target_feature = "atomics"),
         target_os = "freebsd",