ksync/

raw_kernel_mutex.rs

// Copyright 2026 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

use crate::raw_lock::RawLock;
use core::ffi::c_void;
use pin_init::{PinInit, pin_data, pin_init_from_closure};

#[cfg(any(feature = "lock_name_tracing", feature = "spin_lock_tracing"))]
const RAW_MUTEX_SIZE: usize = 32;
#[cfg(not(any(feature = "lock_name_tracing", feature = "spin_lock_tracing")))]
const RAW_MUTEX_SIZE: usize = 24;

unsafe extern "C" {
    fn cpp_mutex_destroy(mutex: *mut c_void);
    fn cpp_mutex_acquire(lock: *mut c_void, entry_storage: *mut c_void);
    fn cpp_mutex_release(lock: *mut c_void, entry_storage: *mut c_void);
    fn cpp_mutex_is_contested(lock: *mut c_void) -> bool;

    fn cpp_critical_mutex_destroy(mutex: *mut c_void);
    fn cpp_critical_mutex_acquire(lock: *mut c_void, entry_storage: *mut c_void) -> bool;
    fn cpp_critical_mutex_release(
        lock: *mut c_void,
        entry_storage: *mut c_void,
        should_clear: bool,
    );
    fn cpp_critical_mutex_is_contested(lock: *mut c_void) -> bool;
}

const MUTEX_MAGIC: u32 = 0x6D757478; // 'mutx'
const INVALID_CPU: u32 = u32::MAX;

const fn make_mutex_storage() -> [u8; RAW_MUTEX_SIZE] {
    let mut bytes = [0u8; RAW_MUTEX_SIZE];
    #[cfg(any(feature = "lock_name_tracing", feature = "spin_lock_tracing"))]
    let offset = 8;
    #[cfg(not(any(feature = "lock_name_tracing", feature = "spin_lock_tracing")))]
    let offset = 0;

    let magic_bytes = MUTEX_MAGIC.to_ne_bytes();
    let cpu_bytes = INVALID_CPU.to_ne_bytes();

    bytes[offset] = magic_bytes[0];
    bytes[offset + 1] = magic_bytes[1];
    bytes[offset + 2] = magic_bytes[2];
    bytes[offset + 3] = magic_bytes[3];

    bytes[offset + 4] = cpu_bytes[0];
    bytes[offset + 5] = cpu_bytes[1];
    bytes[offset + 6] = cpu_bytes[2];
    bytes[offset + 7] = cpu_bytes[3];

    bytes
}

#[repr(C, align(8))]
struct RawMutexStorage(zr::OpaqueBytes<RAW_MUTEX_SIZE>);

#[derive(Default)]
#[repr(C, align(8))]
pub struct LockEntryStorage(zr::OpaqueBytes<40>);

/// Opaque layout block matching the Zircon C++ Mutex exactly.
#[pin_data(PinnedDrop)]
#[repr(C)]
pub struct RawMutex {
    #[cfg(any(feature = "lock_dep", feature = "lock_metadata_only"))]
    class_id: *const c_void,
    storage: RawMutexStorage,
}

impl RawMutex {
    pub const INIT: Self = Self::const_init(core::ptr::null());

    /// Statically initializes a RawMutex in constant context.
    pub const fn const_init(_class_id: *const c_void) -> Self {
        Self {
            #[cfg(any(feature = "lock_dep", feature = "lock_metadata_only"))]
            class_id: _class_id,
            storage: RawMutexStorage(zr::OpaqueBytes::new(make_mutex_storage())),
        }
    }

    /// Is the mutex contested i.e. is at least one thread blocked waiting on it?
    ///
    /// The contested flag does not track threads which are spin-waiting on the
    /// Mutex and have yet to enter a blocking phase.
    #[inline]
    pub fn is_contested(&self) -> bool {
        // SAFETY: The FFI call is safe because the lock is initialized
        unsafe { cpp_mutex_is_contested(self.as_mut_ptr()) }
    }

    /// Returns a slice over the raw mutex storage bytes.
    #[cfg(any(test, ktest))]
    #[inline]
    pub fn raw_storage_slice(&self) -> &[u8] {
        // SAFETY: The storage is valid and allocated with RAW_MUTEX_SIZE bytes.
        unsafe { core::slice::from_raw_parts(self.storage.0.get() as *const u8, RAW_MUTEX_SIZE) }
    }
}

impl Default for RawMutex {
    fn default() -> Self {
        Self::const_init(core::ptr::null())
    }
}

// SAFETY: RawMutex is safe to share and access across threads.
unsafe impl Sync for RawMutex {}
// SAFETY: RawMutex is safe to send across threads.
unsafe impl Send for RawMutex {}

zr::unsafe_pinned_drop_ffi!(RawMutex, cpp_mutex_destroy);

pub struct RawMutexPolicy;

// SAFETY: `cpp_mutex_acquire` and `cpp_mutex_release` enforce mutual exclusion on `RawMutex`.
unsafe impl crate::LockPolicy<RawMutex> for RawMutexPolicy {
    type AcquireArgs = ();
    type GuardState = ();

    #[inline]
    unsafe fn acquire(
        lock: &RawMutex,
        entry: *mut LockEntryStorage,
        _args: (),
    ) -> Self::GuardState {
        // SAFETY: The FFI call is safe because the lock is initialized, and the caller guarantees
        // that `entry` points to valid storage for a lockdep entry.
        unsafe {
            cpp_mutex_acquire(lock.as_mut_ptr(), entry as *mut c_void);
        }
    }

    #[inline]
    unsafe fn reacquire(
        lock: &RawMutex,
        entry: *mut LockEntryStorage,
        state: &mut Self::GuardState,
    ) {
        // SAFETY: Caller upholds the `LockPolicy::reacquire` contract.
        *state = unsafe { Self::acquire(lock, entry, ()) };
    }

    #[inline]
    unsafe fn release(lock: &RawMutex, entry: *mut LockEntryStorage, _state: Self::GuardState) {
        // SAFETY: The FFI call is safe because the lock is initialized, and the caller guarantees
        // that `entry` points to valid storage for a lockdep entry.
        unsafe {
            cpp_mutex_release(lock.as_mut_ptr(), entry as *mut c_void);
        }
    }
}

// SAFETY: `RawMutex` is a physical kernel mutex with a unique storage address.
unsafe impl crate::RawLock for RawMutex {
    type LockEntry = LockEntryStorage;
    type DefaultPolicy = RawMutexPolicy;

    #[inline]
    unsafe fn init(class_id: *const c_void) -> impl PinInit<Self, core::convert::Infallible> {
        // SAFETY: The closure initializes the provided slot in place before returning Ok(()).
        unsafe {
            pin_init_from_closure(move |slot: *mut Self| {
                slot.write(Self::const_init(class_id));
                Ok(())
            })
        }
    }

    #[inline]
    fn as_mut_ptr(&self) -> *mut c_void {
        self as *const Self as *mut Self as *mut c_void
    }
}

/// Opaque layout block matching the Zircon C++ CriticalMutex exactly.
#[pin_data(PinnedDrop)]
#[repr(C)]
pub struct RawCriticalMutex {
    #[cfg(any(feature = "lock_dep", feature = "lock_metadata_only"))]
    class_id: *const c_void,
    storage: RawMutexStorage,
}

impl RawCriticalMutex {
    pub const INIT: Self = Self::const_init(core::ptr::null());

    /// Statically initializes a RawCriticalMutex in constant context.
    pub const fn const_init(_class_id: *const c_void) -> Self {
        Self {
            #[cfg(any(feature = "lock_dep", feature = "lock_metadata_only"))]
            class_id: _class_id,
            storage: RawMutexStorage(zr::OpaqueBytes::new(make_mutex_storage())),
        }
    }

    /// Is the mutex contested i.e. is at least one thread blocked waiting on it?
    ///
    /// The contested flag does not track threads which are spin-waiting on the
    /// Mutex and have yet to enter a blocking phase.
    #[inline]
    pub fn is_contested(&self) -> bool {
        // SAFETY: The FFI call is safe because the lock is initialized
        unsafe { cpp_critical_mutex_is_contested(self.as_mut_ptr()) }
    }

    /// Returns a slice over the raw critical mutex storage bytes.
    #[cfg(any(test, ktest))]
    #[inline]
    pub fn raw_storage_slice(&self) -> &[u8] {
        // SAFETY: The storage is valid and allocated with RAW_MUTEX_SIZE bytes.
        unsafe { core::slice::from_raw_parts(self.storage.0.get() as *const u8, RAW_MUTEX_SIZE) }
    }
}

impl Default for RawCriticalMutex {
    fn default() -> Self {
        Self::const_init(core::ptr::null())
    }
}

// SAFETY: RawCriticalMutex is safe to share and access across threads.
unsafe impl Sync for RawCriticalMutex {}
// SAFETY: RawCriticalMutex is safe to send across threads.
unsafe impl Send for RawCriticalMutex {}

zr::unsafe_pinned_drop_ffi!(RawCriticalMutex, cpp_critical_mutex_destroy);

pub struct RawCriticalMutexPolicy;

// SAFETY: `cpp_critical_mutex_acquire` and `cpp_critical_mutex_release` enforce mutual exclusion
// on `RawCriticalMutex`.
unsafe impl crate::LockPolicy<RawCriticalMutex> for RawCriticalMutexPolicy {
    type AcquireArgs = ();
    type GuardState = bool;

    #[inline]
    unsafe fn acquire(
        lock: &RawCriticalMutex,
        entry: *mut LockEntryStorage,
        _args: (),
    ) -> Self::GuardState {
        // SAFETY: The FFI call is safe because the lock is initialized, and the caller guarantees
        // that `entry` points to valid storage for a lockdep entry.
        unsafe { cpp_critical_mutex_acquire(lock.as_mut_ptr(), entry as *mut c_void) }
    }

    #[inline]
    unsafe fn reacquire(
        lock: &RawCriticalMutex,
        entry: *mut LockEntryStorage,
        state: &mut Self::GuardState,
    ) {
        // SAFETY: Caller upholds the `LockPolicy::reacquire` contract.
        *state = unsafe { Self::acquire(lock, entry, ()) };
    }

    #[inline]
    unsafe fn release(
        lock: &RawCriticalMutex,
        entry: *mut LockEntryStorage,
        should_clear: Self::GuardState,
    ) {
        // SAFETY: The FFI call is safe because the lock is initialized, and the caller guarantees
        // that `entry` points to valid storage for a lockdep entry.
        unsafe {
            cpp_critical_mutex_release(lock.as_mut_ptr(), entry as *mut c_void, should_clear);
        }
    }
}

// SAFETY: `RawCriticalMutex` is a physical kernel critical mutex with a unique storage address.
unsafe impl crate::RawLock for RawCriticalMutex {
    type LockEntry = LockEntryStorage;
    type DefaultPolicy = RawCriticalMutexPolicy;

    #[inline]
    unsafe fn init(class_id: *const c_void) -> impl PinInit<Self, core::convert::Infallible> {
        // SAFETY: The closure initializes the provided slot in place before returning Ok(()).
        unsafe {
            pin_init_from_closure(move |slot: *mut Self| {
                slot.write(Self::const_init(class_id));
                Ok(())
            })
        }
    }

    #[inline]
    fn as_mut_ptr(&self) -> *mut c_void {
        self as *const Self as *mut Self as *mut c_void
    }
}

const _: () = {
    const HAS_NAME_STORAGE: bool =
        cfg!(any(feature = "lock_name_tracing", feature = "spin_lock_tracing"));
    #[cfg(any(feature = "lock_dep", feature = "lock_metadata_only"))]
    const EXPECTED_SIZE: usize = if HAS_NAME_STORAGE { 40 } else { 32 };
    #[cfg(not(any(feature = "lock_dep", feature = "lock_metadata_only")))]
    const EXPECTED_SIZE: usize = if HAS_NAME_STORAGE { 32 } else { 24 };

    assert!(core::mem::size_of::<RawMutex>() == EXPECTED_SIZE);
    assert!(core::mem::align_of::<RawMutex>() == 8);

    assert!(core::mem::size_of::<RawCriticalMutex>() == EXPECTED_SIZE);
    assert!(core::mem::align_of::<RawCriticalMutex>() == 8);
};