mmio/
arch.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4//! Architecture specific instructions for working with device memory.
5use core::ptr::NonNull;
6
7// TODO(b/423071349): deprecate this module if core::arch::{load, store} get stabilized.
8// https://github.com/rust-lang/unsafe-code-guidelines/issues/321.
9
10// The default implementation uses core::ptr::read_volatile and core::ptr::write_volatile for
11// all loads and stores. On certain platforms this may produce instructions that are
12// incompatible with certain hypervisors.
13#[cfg(all(not(target_arch = "x86_64"), not(target_arch = "aarch64")))]
14mod ops {
15    macro_rules! load_op {
16        ($_:tt, $ptr:expr) => {
17            // Safety: provided by caller.
18            unsafe { core::ptr::read_volatile($ptr.as_ptr()) }
19        };
20    }
21
22    macro_rules! store_op {
23        ($_type:tt, $ptr:expr, $value:expr) => {
24            // Safety: provided by caller.
25            unsafe { core::ptr::write_volatile($ptr.as_ptr(), $value) }
26        };
27    }
28    pub(super) use {load_op, store_op};
29}
30
31// KVM on x86 doesn't support Mmio using vector registers. Specify the instructions for this
32// platform explicitly in order to reduce the demands on the hypervisor.
33//
34// For more details see:
35// https://cs.opensource.google/fuchsia/fuchsia/+/main:zircon/system/ulib/mmio-ptr/include/lib/mmio-ptr/mmio-ptr.h;l=116;drc=36e68808ae97fc54675be5c3f57484726577425c
36#[cfg(target_arch = "x86_64")]
37mod ops {
38    macro_rules! load_op {
39        (u8, $ptr:expr) => {
40            load_op!(INTERNAL u8, $ptr, "mov {d}, [{s}]", reg_byte)
41        };
42        (u16, $ptr:expr) => {
43            load_op!(INTERNAL u16, $ptr, "mov {d:x}, [{s}]", reg)
44        };
45        (u32, $ptr:expr) => {
46            load_op!(INTERNAL u32, $ptr, "mov {d:e}, [{s}]", reg)
47        };
48        (u64, $ptr:expr) => {
49            load_op!(INTERNAL u64, $ptr, "mov {d:r}, [{s}]", reg)
50        };
51        (INTERNAL $type:ty, $ptr:expr, $inst:literal, $reg_class:ident) => {
52            {
53                let ptr = $ptr.as_ptr();
54                let mut res: $type;
55                // Safety: provided by caller.
56                unsafe {
57                    core::arch::asm!($inst, d = out($reg_class) res, s = in(reg) ptr, options(nostack));
58                }
59                res
60            }
61        }
62    }
63
64    macro_rules! store_op {
65        (u8, $ptr:expr, $value:expr) => {
66            store_op!(INTERNAL u8, $ptr, $value, "mov [{d}], {s}", reg_byte)
67        };
68        (u16, $ptr:expr, $value:expr) => {
69            store_op!(INTERNAL u16, $ptr, $value, "mov [{d}], {s:x}", reg)
70        };
71        (u32, $ptr:expr, $value:expr) => {
72            store_op!(INTERNAL u32, $ptr, $value, "mov [{d}], {s:e}", reg)
73        };
74        (u64, $ptr:expr, $value:expr) => {
75            store_op!(INTERNAL u64, $ptr, $value, "mov [{d}], {s:r}", reg)
76        };
77        (INTERNAL $type:ty, $ptr:expr, $value:expr, $inst:literal, $reg_class:ident) => {
78            {
79                let ptr = $ptr.as_ptr();
80                // Safety: provided by caller.
81                unsafe {
82                    core::arch::asm!($inst, d = in(reg) ptr, s = in($reg_class) $value, options(nostack));
83                }
84            }
85        }
86    }
87
88    pub(super) use {load_op, store_op};
89}
90
91// rustc/llvm may generate instructions that are incompatible with being run in certain
92// hypervisors.
93//
94// For more details see:
95//
96// https://github.com/rust-lang/rust/issues/131894 and
97// https://cs.opensource.google/fuchsia/fuchsia/+/main:zircon/system/ulib/mmio-ptr/include/lib/mmio-ptr/mmio-ptr.h;l=57;drc=36e68808ae97fc54675be5c3f57484726577425c
98#[cfg(target_arch = "aarch64")]
99mod ops {
100    macro_rules! load_op {
101        (u8, $ptr:expr) => {
102            load_op!(INTERNAL u8, $ptr, "ldrb {d:w}, [{s}]")
103        };
104        (u16, $ptr:expr) => {
105            load_op!(INTERNAL u16, $ptr, "ldrh {d:w}, [{s}]")
106        };
107        (u32, $ptr:expr) => {
108            load_op!(INTERNAL u32, $ptr, "ldr {d:w}, [{s}]")
109        };
110        (u64, $ptr:expr) => {
111            load_op!(INTERNAL u64, $ptr, "ldr {d:x}, [{s}]")
112        };
113        (INTERNAL $type:ty, $ptr:expr, $inst:literal) => {
114            {
115                let ptr = $ptr.as_ptr();
116                let mut res: $type;
117                unsafe {
118                    core::arch::asm!($inst, d = out(reg) res, s = in(reg) ptr, options(nostack));
119                }
120                res
121            }
122        }
123    }
124
125    macro_rules! store_op {
126        (u8, $ptr:expr, $value:expr) => {
127            store_op!(INTERNAL u8, $ptr, $value, "strb {s:w}, [{d}]")
128        };
129        (u16, $ptr:expr, $value:expr) => {
130            store_op!(INTERNAL u16, $ptr, $value, "strh {s:w}, [{d}]")
131        };
132        (u32, $ptr:expr, $value:expr) => {
133            store_op!(INTERNAL u32, $ptr, $value, "str {s:w}, [{d}]")
134        };
135        (u64, $ptr:expr, $value:expr) => {
136            store_op!(INTERNAL u64, $ptr, $value, "str {s:x}, [{d}]")
137        };
138        (INTERNAL $type:ty, $ptr:expr, $value:expr, $inst:literal) => {
139            {
140                let ptr = $ptr.as_ptr();
141                unsafe {
142                    core::arch::asm!($inst, d = in(reg) ptr, s = in(reg) $value, options(nostack));
143                }
144            }
145        }
146    }
147
148    pub(super) use {load_op, store_op};
149}
150
151use ops::{load_op, store_op};
152
153macro_rules! load_fn {
154    ($name:ident, $type:tt) => {
155        #[inline]
156        /// Loads a value from the pointee suitably for device memory.
157        ///
158        /// # Safety
159        /// - the pointer must be valid
160        /// - the pointer must be aligned
161        /// - there must not be concurrent stores overlapping this load
162        /// - this pointer must be safe to access as with [core::ptr::read_volatile]
163        pub(crate) unsafe fn $name(ptr: NonNull<$type>) -> $type {
164            load_op!($type, ptr)
165        }
166    };
167}
168
169load_fn! {load8, u8}
170load_fn! {load16, u16}
171load_fn! {load32, u32}
172load_fn! {load64, u64}
173
174macro_rules! store_fn {
175    ($type:tt, $name:ident) => {
176        #[inline]
177        /// Stores the given balue to the pointee suitably for device memory.
178        ///
179        /// # Safety
180        /// - the pointer must be valid
181        /// - the pointer must be aligned
182        /// - there must be any concurrent operations overlapping this store
183        /// - this pointer must be safe to access as with [core::ptr::write_volatile]
184        pub(crate) unsafe fn $name(ptr: NonNull<$type>, value: $type) {
185            store_op!($type, ptr, value)
186        }
187    };
188}
189
190store_fn! {u8, store8}
191store_fn! {u16, store16}
192store_fn! {u32, store32}
193store_fn! {u64, store64}
mmio/arch.rs

mmio/
arch.rs
