starnix_core/mm/
memory.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::mm::{
6    MappingOptions, MemoryManager, PAGE_SIZE, VMEX_RESOURCE, ZX_VM_SPECIFIC_OVERWRITE,
7};
8use fuchsia_rcu::RcuDroppable;
9use fuchsia_runtime::UtcClock;
10use mapped_clock::{CLOCK_SIZE, MappedClock};
11use starnix_logging::{CATEGORY_STARNIX_MM, impossible_error, set_zx_name};
12use starnix_uapi::errno;
13use starnix_uapi::errors::Errno;
14use std::mem::MaybeUninit;
15use std::sync::{Arc, OnceLock};
16use zerocopy::FromBytes;
17use zx::Koid;
18
19// This tracks a VMO handle along with basic information about the handle.
20#[derive(Debug, RcuDroppable)]
21pub struct VmoAndBasicInfo {
22    vmo: zx::Vmo,
23    info: OnceLock<(Koid, zx::Rights)>,
24}
25
26impl PartialEq for VmoAndBasicInfo {
27    fn eq(&self, other: &Self) -> bool {
28        self.vmo == other.vmo
29    }
30}
31
32impl Eq for VmoAndBasicInfo {}
33
34impl From<zx::Vmo> for VmoAndBasicInfo {
35    fn from(vmo: zx::Vmo) -> Self {
36        Self { vmo, info: OnceLock::new() }
37    }
38}
39
40impl VmoAndBasicInfo {
41    fn get_info(&self) -> &(Koid, zx::Rights) {
42        self.info.get_or_init(|| {
43            let info = self.vmo.basic_info().map_err(impossible_error).unwrap();
44            (info.koid, info.rights)
45        })
46    }
47
48    pub fn get_koid(&self) -> Koid {
49        self.get_info().0
50    }
51
52    pub fn get_rights(&self) -> zx::Rights {
53        self.get_info().1
54    }
55}
56
57impl Drop for VmoAndBasicInfo {
58    fn drop(&mut self) {
59        #[cfg(debug_assertions)]
60        {
61            if let Some((koid, rights)) = self.info.get() {
62                if let Ok(info) = self.vmo.basic_info() {
63                    debug_assert_eq!(*koid, info.koid, "Cached KOID mismatch");
64                    debug_assert_eq!(*rights, info.rights, "Cached rights mismatch");
65                }
66            }
67        }
68    }
69}
70
71#[derive(Debug, RcuDroppable)]
72pub enum MemoryObject {
73    Vmo(VmoAndBasicInfo),
74    /// The memory object is a bpf ring buffer. The layout it represents is:
75    /// |Page1 - Page2 - Page3 .. PageN - Page3 .. PageN| where the vmo is
76    /// |Page1 - Page2 - Page3 .. PageN|
77    RingBuf(VmoAndBasicInfo),
78    /// A memory mapped clock is backed by kernel memory, not by a VMO. So
79    /// it is handled specially.  The lifecycle of this clock is:
80    /// - starts off as an empty unmapped thing.
81    /// - a MappedClock is created on `map_in_vmar`.
82    /// - a name is added on `set_zx_name`.
83    /// - most clone/resize operations return errors.
84    /// - unmapped at the end of the process lifetime.
85    MemoryMappedClock {
86        // Koid of the `utc_clock`, cached for performance.
87        koid: Koid,
88        // The UTC clock handle to map to memory. Do not use it for clock reads, use
89        // the public functions in `//src/starnix/kernel/core/time/utc.rs` instead
90        utc_clock: UtcClock,
91    },
92}
93
94impl std::cmp::Eq for MemoryObject {}
95
96// Implemented manually as `MemoryMappedClock`'s mutex is not transparent to
97// `PartialEq`.
98impl std::cmp::PartialEq for MemoryObject {
99    fn eq(&self, other: &MemoryObject) -> bool {
100        match (self, other) {
101            (MemoryObject::Vmo(info1), MemoryObject::Vmo(info2)) => info1.vmo == info2.vmo,
102            (MemoryObject::RingBuf(info1), MemoryObject::RingBuf(info2)) => info1.vmo == info2.vmo,
103            (MemoryObject::MemoryMappedClock { .. }, MemoryObject::MemoryMappedClock { .. }) => {
104                self.get_koid() == other.get_koid()
105            }
106            (_, _) => false,
107        }
108    }
109}
110
111impl From<zx::Vmo> for MemoryObject {
112    fn from(vmo: zx::Vmo) -> Self {
113        Self::Vmo(VmoAndBasicInfo::from(vmo))
114    }
115}
116
117impl From<UtcClock> for MemoryObject {
118    fn from(utc_clock: UtcClock) -> MemoryObject {
119        let koid = utc_clock.koid().expect("koid should always be readable");
120        MemoryObject::MemoryMappedClock { koid, utc_clock }
121    }
122}
123
124impl MemoryObject {
125    pub fn as_vmo(&self) -> Option<&zx::Vmo> {
126        match self {
127            Self::Vmo(info) => Some(&info.vmo),
128            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => None,
129        }
130    }
131
132    /// Returns true if this [MemoryObject] is a memory mapped clock.
133    pub fn is_clock(&self) -> bool {
134        match self {
135            Self::Vmo(_) | Self::RingBuf(_) => false,
136            Self::MemoryMappedClock { .. } => true,
137        }
138    }
139
140    pub fn into_vmo(self) -> Option<zx::Vmo> {
141        match self {
142            Self::Vmo(info) => Some(
143                info.vmo
144                    .duplicate_handle(zx::Rights::SAME_RIGHTS)
145                    .expect("duplicate_handle failed in into_vmo"),
146            ),
147            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => None,
148        }
149    }
150
151    pub fn get_content_size(&self) -> u64 {
152        match self {
153            Self::Vmo(info) => info.vmo.get_stream_size().map_err(impossible_error).unwrap(),
154            Self::RingBuf(_) => self.get_size(),
155            Self::MemoryMappedClock { .. } => CLOCK_SIZE as u64,
156        }
157    }
158
159    pub fn set_content_size(&self, size: u64) -> Result<(), zx::Status> {
160        match self {
161            Self::Vmo(info) => info.vmo.set_stream_size(size),
162            Self::RingBuf(_) => Ok(()),
163            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
164        }
165    }
166
167    pub fn get_size(&self) -> u64 {
168        match self {
169            Self::Vmo(info) => info.vmo.get_size().map_err(impossible_error).unwrap(),
170            Self::RingBuf(info) => {
171                let base_size = info.vmo.get_size().map_err(impossible_error).unwrap();
172                (base_size - *PAGE_SIZE) * 2
173            }
174            Self::MemoryMappedClock { .. } => CLOCK_SIZE as u64,
175        }
176    }
177
178    pub fn set_size(&self, size: u64) -> Result<(), zx::Status> {
179        match self {
180            Self::Vmo(info) => info.vmo.set_size(size),
181            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
182        }
183    }
184
185    pub fn create_child(
186        &self,
187        option: zx::VmoChildOptions,
188        offset: u64,
189        size: u64,
190    ) -> Result<Self, zx::Status> {
191        match self {
192            Self::Vmo(info) => info.vmo.create_child(option, offset, size).map(Self::from),
193            Self::RingBuf(info) => info
194                .vmo
195                .create_child(option, offset, size)
196                .map(|vmo| Self::RingBuf(VmoAndBasicInfo::from(vmo))),
197            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
198        }
199    }
200
201    pub fn duplicate_handle(&self, rights: zx::Rights) -> Result<Self, zx::Status> {
202        match self {
203            Self::Vmo(info) => info.vmo.duplicate_handle(rights).map(Self::from),
204            Self::RingBuf(info) => info
205                .vmo
206                .duplicate_handle(rights)
207                .map(|vmo| Self::RingBuf(VmoAndBasicInfo::from(vmo))),
208            Self::MemoryMappedClock { utc_clock, .. } => {
209                utc_clock.duplicate_handle(rights).map(|c| Self::from(c))
210            }
211        }
212    }
213
214    pub fn read(&self, data: &mut [u8], offset: u64) -> Result<(), zx::Status> {
215        match self {
216            Self::Vmo(info) => info.vmo.read(data, offset),
217            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
218        }
219    }
220
221    pub fn read_to_array<T: Copy + FromBytes, const N: usize>(
222        &self,
223        offset: u64,
224    ) -> Result<[T; N], zx::Status> {
225        match self {
226            Self::Vmo(info) => info.vmo.read_to_array(offset),
227            Self::RingBuf(_) => Err(zx::Status::NOT_SUPPORTED),
228            // There does not seem to be an API that allows this read.
229            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
230        }
231    }
232
233    pub fn read_to_vec(&self, offset: u64, length: u64) -> Result<Vec<u8>, zx::Status> {
234        match self {
235            Self::Vmo(info) => info.vmo.read_to_vec(offset, length),
236            Self::RingBuf(_) => Err(zx::Status::NOT_SUPPORTED),
237            // See the note in `read_to_array` above.
238            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
239        }
240    }
241
242    pub fn read_uninit<'a>(
243        &self,
244        data: &'a mut [MaybeUninit<u8>],
245        offset: u64,
246    ) -> Result<&'a mut [u8], zx::Status> {
247        match self {
248            Self::Vmo(info) => info.vmo.read_uninit(data, offset),
249            Self::RingBuf(_) => Err(zx::Status::NOT_SUPPORTED),
250            // See the note in `read_to_array` above.
251            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
252        }
253    }
254
255    /// Reads from the memory.
256    ///
257    /// # Safety
258    ///
259    /// Callers must guarantee that the buffer is valid to write to.
260    ///
261    /// # Errors
262    ///
263    /// Returns `zx::Status::NOT_SUPPORTED` where unsupported.
264    pub unsafe fn read_raw(
265        &self,
266        buffer: *mut u8,
267        buffer_length: usize,
268        offset: u64,
269    ) -> Result<(), zx::Status> {
270        match self {
271            #[allow(clippy::undocumented_unsafe_blocks, reason = "2024 edition migration")]
272            Self::Vmo(info) => unsafe { info.vmo.read_raw(buffer, buffer_length, offset) },
273            Self::RingBuf(_) => Err(zx::Status::NOT_SUPPORTED),
274            // See the note in `read_to_array` above.
275            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
276        }
277    }
278
279    /// Write to memory.
280    ///
281    /// # Errors
282    ///
283    /// Returns `zx::Status::NOT_SUPPORTED` for read-only memory.
284    pub fn write(&self, data: &[u8], offset: u64) -> Result<(), zx::Status> {
285        match self {
286            Self::Vmo(info) => info.vmo.write(data, offset),
287            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
288        }
289    }
290
291    /// Returns the koid of the underlying memory-like object.
292    ///
293    /// Should be cheap to call frequently.
294    pub fn get_koid(&self) -> Koid {
295        match self {
296            Self::Vmo(info) => info.get_koid(),
297            Self::RingBuf(info) => info.get_koid(),
298            Self::MemoryMappedClock { koid, .. } => *koid,
299        }
300    }
301
302    /// Returns the rights of the underlying memory-like object.
303    pub fn get_rights(&self) -> zx::Rights {
304        match self {
305            Self::Vmo(info) => info.get_rights(),
306            Self::RingBuf(info) => info.get_rights(),
307            Self::MemoryMappedClock { utc_clock, .. } => {
308                utc_clock.basic_info().map_err(impossible_error).unwrap().rights
309            }
310        }
311    }
312
313    /// Returns `zx::VmoInfo` for a memory object that supports it.
314    ///
315    /// # Panics
316    ///
317    /// Calling `info` on a `MemoryObject` that is not represented by a VMO
318    /// will panic. To avoid this in code, call `is_clock` before attempting.
319    pub fn info(&self) -> Result<zx::VmoInfo, Errno> {
320        match self {
321            Self::Vmo(info) | Self::RingBuf(info) => info.vmo.info().map_err(|_| errno!(EIO)),
322            // Use `is_clock` to avoid calling info on a clock.
323            Self::MemoryMappedClock { .. } => {
324                panic!("info() is not supported on a memory mapped clock")
325            }
326        }
327    }
328
329    pub fn set_zx_name(&self, name: &[u8]) {
330        match self {
331            Self::Vmo(info) | Self::RingBuf(info) => set_zx_name(&info.vmo, name),
332            Self::MemoryMappedClock { .. } => {
333                // The memory mapped clock is a singleton, so it does not
334                // seem appropriate to give it a zx name.
335            }
336        }
337    }
338
339    pub fn with_zx_name(self, name: &[u8]) -> Self {
340        self.set_zx_name(name);
341        self
342    }
343
344    pub fn op_range(
345        &self,
346        op: zx::VmoOp,
347        mut offset: u64,
348        mut size: u64,
349    ) -> Result<(), zx::Status> {
350        match self {
351            Self::Vmo(info) => info.vmo.op_range(op, offset, size),
352            Self::RingBuf(info) => {
353                let vmo_size = info.vmo.get_size().map_err(impossible_error).unwrap();
354                let data_size = vmo_size - (2 * *PAGE_SIZE);
355                let memory_size = vmo_size + data_size;
356                if offset + size > memory_size {
357                    return Err(zx::Status::OUT_OF_RANGE);
358                }
359                // If `offset` is greater than `vmo_size`, the operation is equivalent to the one
360                // done on the first part of the memory range.
361                if offset >= vmo_size {
362                    offset -= data_size;
363                }
364                // If the operation spill over the end if the vmo, it must be done on the start of
365                // the data part of the vmo.
366                if offset + size > vmo_size {
367                    info.vmo.op_range(op, 2 * *PAGE_SIZE, offset + size - vmo_size)?;
368                    size = vmo_size - offset;
369                }
370                info.vmo.op_range(op, offset, size)
371            }
372            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
373        }
374    }
375
376    pub fn replace_as_executable(self, vmex: &zx::Resource) -> Result<Self, zx::Status> {
377        match self {
378            Self::Vmo(info) => {
379                let vmo = info.vmo.duplicate_handle(zx::Rights::SAME_RIGHTS)?;
380                let exec_vmo = vmo.replace_as_executable(vmex)?;
381                Ok(Self::from(exec_vmo))
382            }
383            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
384        }
385    }
386
387    pub fn map_in_vmar(
388        &self,
389        vmar: &zx::Vmar,
390        vmar_offset: usize,
391        mut memory_offset: u64,
392        len: usize,
393        flags: zx::VmarFlags,
394    ) -> Result<usize, zx::Status> {
395        match self {
396            Self::Vmo(info) => vmar.map(vmar_offset, &info.vmo, memory_offset, len, flags),
397            Self::RingBuf(info) => {
398                let vmo_size = info.vmo.get_size().map_err(impossible_error).unwrap();
399                let data_size = vmo_size - (2 * *PAGE_SIZE);
400                let memory_size = vmo_size + data_size;
401                if memory_offset.checked_add(len as u64).ok_or(zx::Status::OUT_OF_RANGE)?
402                    > memory_size
403                {
404                    return Err(zx::Status::OUT_OF_RANGE);
405                }
406                // If `memory_offset` is greater than `vmo_size`, the operation is equivalent to
407                // the one done on the first part of the memory range.
408                if memory_offset >= vmo_size {
409                    memory_offset -= data_size;
410                }
411                // Map the vmo for the full length. This ensures the kernel will choose a range
412                // that can accommodate the full length so that the second mapping will not erase
413                // another mapping.
414                let result = vmar.map(
415                    vmar_offset,
416                    &info.vmo,
417                    memory_offset,
418                    len,
419                    flags | zx::VmarFlags::ALLOW_FAULTS,
420                )?;
421                // The maximal amount of data that can have been mapped from the vmo with the
422                // previous operation.
423                let max_mapped_len = (vmo_size - memory_offset) as usize;
424                // If more data is needed, the data part of the vmo must be mapped again, replacing
425                // the part of the previous mapping that contained no data.
426                if len > max_mapped_len {
427                    let vmar_info = vmar.info().map_err(|_| errno!(EIO))?;
428                    let base_address = vmar_info.base;
429                    // The request should map the data part of the vmo a second time
430                    let second_mapping_address = vmar
431                        .map(
432                            result + max_mapped_len - base_address,
433                            &info.vmo,
434                            2 * *PAGE_SIZE,
435                            len - max_mapped_len,
436                            flags | ZX_VM_SPECIFIC_OVERWRITE,
437                        )
438                        .expect("Mapping should not fail as the space has been reserved");
439                    debug_assert_eq!(second_mapping_address, result + max_mapped_len);
440                }
441                Ok(result)
442            }
443            Self::MemoryMappedClock { utc_clock, .. } => {
444                // The memory mapped clock API only allows memory offset of 0, and a page-sized
445                // length of the mapping. No offset or partial mappings are allowed.
446                assert_eq!(0, memory_offset, "memory mapped clock must be at memory offset 0");
447
448                // We don't need to remember this, since vmar will know how to unmap it.
449                let memory_mapped_clock = MappedClock::try_new_without_unmap(
450                    &utc_clock,
451                    vmar,
452                    flags,
453                    vmar_offset as u64,
454                )?;
455                Ok(memory_mapped_clock.raw_addr())
456            }
457        }
458    }
459
460    pub fn memmove(
461        &self,
462        options: zx::TransferDataOptions,
463        dst_offset: u64,
464        src_offset: u64,
465        size: u64,
466    ) -> Result<(), zx::Status> {
467        match self {
468            Self::Vmo(info) => {
469                info.vmo.transfer_data(options, dst_offset, size, &info.vmo, src_offset)
470            }
471            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
472        }
473    }
474
475    pub fn clone_memory(
476        self: &Arc<Self>,
477        rights: zx::Rights,
478        options: MappingOptions,
479    ) -> Result<Arc<Self>, Errno> {
480        if self.is_clock() {
481            return Err(errno!(ENOTSUP, "clone_memory not supported on memory mapped clock"));
482        }
483
484        // Non-anonymous memory is pager-backed, and we can clone it if we don't need write
485        // rights.
486        Ok(if !options.contains(MappingOptions::ANONYMOUS) && !rights.contains(zx::Rights::WRITE) {
487            self.clone()
488        } else {
489            fuchsia_trace::duration!(CATEGORY_STARNIX_MM, "pager_backed_memory_snapshot");
490            let mut cloned_memory = self
491                .create_child(
492                    zx::VmoChildOptions::SNAPSHOT_MODIFIED | zx::VmoChildOptions::RESIZABLE,
493                    0,
494                    self.get_size(),
495                )
496                .map_err(MemoryManager::get_errno_for_map_err)?;
497            if rights.contains(zx::Rights::EXECUTE) {
498                cloned_memory = cloned_memory
499                    .replace_as_executable(&VMEX_RESOURCE)
500                    .map_err(impossible_error)?;
501            }
502
503            Arc::new(cloned_memory)
504        })
505    }
506}
starnix_core/mm/memory.rs

starnix_core/mm/
memory.rs
