starnix_core/mm/
memory.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::mm::{
6    MappingOptions, MemoryManager, PAGE_SIZE, VMEX_RESOURCE, ZX_VM_SPECIFIC_OVERWRITE,
7};
8use fuchsia_rcu::RcuDroppable;
9use fuchsia_runtime::UtcClock;
10use mapped_clock::{CLOCK_SIZE, MappedClock};
11use starnix_logging::{CATEGORY_STARNIX_MM, impossible_error, set_zx_name};
12use starnix_uapi::errno;
13use starnix_uapi::errors::Errno;
14use std::mem::MaybeUninit;
15use std::sync::{Arc, OnceLock};
16use zerocopy::FromBytes;
17use zx::Koid;
18
19// This tracks a VMO handle along with basic information about the handle.
20//
21// The VMO is held behind an `Arc` so that every `VmoAndBasicInfo` referring to the same VMO with
22// the same rights can share a single Zircon handle instead of each owning its own duplicate.
23// Sharing a handle this way is equivalent to duplicating it with `ZX_RIGHT_SAME_RIGHTS`: the
24// kernel object and the rights are identical, only the handle value differs.
25#[derive(Debug, RcuDroppable)]
26pub struct VmoAndBasicInfo {
27    vmo: Arc<zx::Vmo>,
28    info: OnceLock<(Koid, zx::Rights)>,
29}
30
31impl PartialEq for VmoAndBasicInfo {
32    fn eq(&self, other: &Self) -> bool {
33        Arc::ptr_eq(&self.vmo, &other.vmo)
34    }
35}
36
37impl Eq for VmoAndBasicInfo {}
38
39impl From<zx::Vmo> for VmoAndBasicInfo {
40    fn from(vmo: zx::Vmo) -> Self {
41        Self::from(Arc::new(vmo))
42    }
43}
44
45impl From<Arc<zx::Vmo>> for VmoAndBasicInfo {
46    fn from(vmo: Arc<zx::Vmo>) -> Self {
47        Self { vmo, info: OnceLock::new() }
48    }
49}
50
51impl VmoAndBasicInfo {
52    fn get_info(&self) -> &(Koid, zx::Rights) {
53        self.info.get_or_init(|| {
54            let info = self.vmo.basic_info().map_err(impossible_error).unwrap();
55            (info.koid, info.rights)
56        })
57    }
58
59    pub fn get_koid(&self) -> Koid {
60        self.get_info().0
61    }
62
63    pub fn get_rights(&self) -> zx::Rights {
64        self.get_info().1
65    }
66
67    /// Returns another reference to this VMO holding `rights`.
68    ///
69    /// If `rights` are the rights this handle already holds, the existing handle is shared rather
70    /// than asking the kernel for a new one. Otherwise a rights-reduced duplicate is created.
71    fn share_or_duplicate(&self, rights: zx::Rights) -> Result<Self, zx::Status> {
72        if rights == zx::Rights::SAME_RIGHTS || rights == self.get_rights() {
73            // The cached info describes the kernel object and its rights, both of which are
74            // unchanged by sharing, so carry the cache over to avoid re-querying it.
75            return Ok(Self { vmo: Arc::clone(&self.vmo), info: self.info.clone() });
76        }
77        // Changing the rights does not change the koid and we know the target rights.
78        let koid = self.get_koid();
79        let vmo = Arc::new(self.vmo.duplicate_handle(rights)?);
80        let info = OnceLock::from((koid, rights));
81        Ok(Self { vmo, info })
82    }
83
84    /// Consumes `self` and returns an owned VMO handle with the same rights.
85    ///
86    /// If this was the last reference to the shared handle it is returned directly, otherwise a
87    /// duplicate is created.
88    fn into_owned_vmo(mut self) -> Result<zx::Vmo, zx::Status> {
89        self.debug_check_cached_info();
90        match Arc::get_mut(&mut self.vmo) {
91            Some(vmo) => {
92                // Move the valid zx::Vmo out of `self`, replacing it with an invalid
93                // pointer which will be dropped when `self` goes out of scope.
94                let owned_vmo = std::mem::replace(vmo, zx::NullableHandle::invalid().into());
95                Ok(owned_vmo)
96            }
97            None => self.vmo.duplicate_handle(zx::Rights::SAME_RIGHTS),
98        }
99    }
100
101    fn debug_check_cached_info(&self) {
102        #[cfg(debug_assertions)]
103        if let Some((koid, rights)) = self.info.get() {
104            if let Ok(info) = self.vmo.basic_info() {
105                debug_assert_eq!(*koid, info.koid, "Cached KOID mismatch");
106                debug_assert_eq!(*rights, info.rights, "Cached rights mismatch");
107            }
108        }
109    }
110}
111
112impl Drop for VmoAndBasicInfo {
113    fn drop(&mut self) {
114        #[cfg(debug_assertions)]
115        self.debug_check_cached_info();
116    }
117}
118
119#[derive(Debug, RcuDroppable)]
120pub enum MemoryObject {
121    Vmo(VmoAndBasicInfo),
122    /// The memory object is a bpf ring buffer. The layout it represents is:
123    /// |Page1 - Page2 - Page3 .. PageN - Page3 .. PageN| where the vmo is
124    /// |Page1 - Page2 - Page3 .. PageN|
125    RingBuf(VmoAndBasicInfo),
126    /// A memory mapped clock is backed by kernel memory, not by a VMO. So
127    /// it is handled specially.  The lifecycle of this clock is:
128    /// - starts off as an empty unmapped thing.
129    /// - a MappedClock is created on `map_in_vmar`.
130    /// - a name is added on `set_zx_name`.
131    /// - most clone/resize operations return errors.
132    /// - unmapped at the end of the process lifetime.
133    MemoryMappedClock {
134        // Koid of the `utc_clock`, cached for performance.
135        koid: Koid,
136        // The UTC clock handle to map to memory. Do not use it for clock reads, use
137        // the public functions in `//src/starnix/kernel/core/time/utc.rs` instead
138        utc_clock: UtcClock,
139    },
140}
141
142impl std::cmp::Eq for MemoryObject {}
143
144// Implemented manually as `MemoryMappedClock`'s mutex is not transparent to
145// `PartialEq`.
146impl std::cmp::PartialEq for MemoryObject {
147    fn eq(&self, other: &MemoryObject) -> bool {
148        match (self, other) {
149            (MemoryObject::Vmo(info1), MemoryObject::Vmo(info2)) => info1.vmo == info2.vmo,
150            (MemoryObject::RingBuf(info1), MemoryObject::RingBuf(info2)) => info1.vmo == info2.vmo,
151            (MemoryObject::MemoryMappedClock { .. }, MemoryObject::MemoryMappedClock { .. }) => {
152                self.get_koid() == other.get_koid()
153            }
154            (_, _) => false,
155        }
156    }
157}
158
159impl From<zx::Vmo> for MemoryObject {
160    fn from(vmo: zx::Vmo) -> Self {
161        Self::Vmo(VmoAndBasicInfo::from(vmo))
162    }
163}
164
165impl From<Arc<zx::Vmo>> for MemoryObject {
166    fn from(vmo: Arc<zx::Vmo>) -> Self {
167        Self::Vmo(VmoAndBasicInfo::from(vmo))
168    }
169}
170
171impl From<UtcClock> for MemoryObject {
172    fn from(utc_clock: UtcClock) -> MemoryObject {
173        let koid = utc_clock.koid().expect("koid should always be readable");
174        MemoryObject::MemoryMappedClock { koid, utc_clock }
175    }
176}
177
178impl MemoryObject {
179    pub fn as_vmo(&self) -> Option<&zx::Vmo> {
180        match self {
181            Self::Vmo(info) => Some(&info.vmo),
182            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => None,
183        }
184    }
185
186    /// Returns true if this [MemoryObject] is a memory mapped clock.
187    pub fn is_clock(&self) -> bool {
188        match self {
189            Self::Vmo(_) | Self::RingBuf(_) => false,
190            Self::MemoryMappedClock { .. } => true,
191        }
192    }
193
194    pub fn into_vmo(self) -> Option<zx::Vmo> {
195        match self {
196            Self::Vmo(info) => {
197                Some(info.into_owned_vmo().expect("duplicate_handle failed in into_vmo"))
198            }
199            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => None,
200        }
201    }
202
203    pub fn get_content_size(&self) -> u64 {
204        match self {
205            Self::Vmo(info) => info.vmo.get_stream_size().map_err(impossible_error).unwrap(),
206            Self::RingBuf(_) => self.get_size(),
207            Self::MemoryMappedClock { .. } => CLOCK_SIZE as u64,
208        }
209    }
210
211    pub fn set_content_size(&self, size: u64) -> Result<(), zx::Status> {
212        match self {
213            Self::Vmo(info) => info.vmo.set_stream_size(size),
214            Self::RingBuf(_) => Ok(()),
215            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
216        }
217    }
218
219    pub fn get_size(&self) -> u64 {
220        match self {
221            Self::Vmo(info) => info.vmo.get_size().map_err(impossible_error).unwrap(),
222            Self::RingBuf(info) => {
223                let base_size = info.vmo.get_size().map_err(impossible_error).unwrap();
224                (base_size - *PAGE_SIZE) * 2
225            }
226            Self::MemoryMappedClock { .. } => CLOCK_SIZE as u64,
227        }
228    }
229
230    pub fn set_size(&self, size: u64) -> Result<(), zx::Status> {
231        match self {
232            Self::Vmo(info) => info.vmo.set_size(size),
233            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
234        }
235    }
236
237    pub fn create_child(
238        &self,
239        option: zx::VmoChildOptions,
240        offset: u64,
241        size: u64,
242    ) -> Result<Self, zx::Status> {
243        match self {
244            Self::Vmo(info) => info.vmo.create_child(option, offset, size).map(Self::from),
245            Self::RingBuf(info) => info
246                .vmo
247                .create_child(option, offset, size)
248                .map(|vmo| Self::RingBuf(VmoAndBasicInfo::from(vmo))),
249            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
250        }
251    }
252
253    /// Creates a writable `SNAPSHOT_AT_LEAST_ON_WRITE` child spanning all of this memory, for use
254    /// as the backing of a private mapping.
255    ///
256    /// Zircon removes `Rights::EXECUTE` from snapshot children created without `NO_WRITE` (see
257    /// `zx_vmo_create_child`). If this memory has `Rights::EXECUTE`, the right is restored on the
258    /// child so that the private mapping can still be made executable.
259    pub fn create_private_child(&self) -> Result<Self, zx::Status> {
260        let child =
261            self.create_child(zx::VmoChildOptions::SNAPSHOT_AT_LEAST_ON_WRITE, 0, self.get_size())?;
262        if self.get_rights().contains(zx::Rights::EXECUTE) {
263            child.replace_as_executable(&VMEX_RESOURCE)
264        } else {
265            Ok(child)
266        }
267    }
268
269    pub fn duplicate_handle(&self, rights: zx::Rights) -> Result<Self, zx::Status> {
270        match self {
271            Self::Vmo(info) => info.share_or_duplicate(rights).map(Self::Vmo),
272            Self::RingBuf(info) => info.share_or_duplicate(rights).map(Self::RingBuf),
273            Self::MemoryMappedClock { utc_clock, .. } => {
274                utc_clock.duplicate_handle(rights).map(|c| Self::from(c))
275            }
276        }
277    }
278
279    pub fn read(&self, data: &mut [u8], offset: u64) -> Result<(), zx::Status> {
280        match self {
281            Self::Vmo(info) => info.vmo.read(data, offset),
282            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
283        }
284    }
285
286    pub fn read_to_array<T: Copy + FromBytes, const N: usize>(
287        &self,
288        offset: u64,
289    ) -> Result<[T; N], zx::Status> {
290        match self {
291            Self::Vmo(info) => info.vmo.read_to_array(offset),
292            Self::RingBuf(_) => Err(zx::Status::NOT_SUPPORTED),
293            // There does not seem to be an API that allows this read.
294            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
295        }
296    }
297
298    pub fn read_to_vec(&self, offset: u64, length: u64) -> Result<Vec<u8>, zx::Status> {
299        match self {
300            Self::Vmo(info) => info.vmo.read_to_vec(offset, length),
301            Self::RingBuf(_) => Err(zx::Status::NOT_SUPPORTED),
302            // See the note in `read_to_array` above.
303            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
304        }
305    }
306
307    pub fn read_uninit<'a>(
308        &self,
309        data: &'a mut [MaybeUninit<u8>],
310        offset: u64,
311    ) -> Result<&'a mut [u8], zx::Status> {
312        match self {
313            Self::Vmo(info) => info.vmo.read_uninit(data, offset),
314            Self::RingBuf(_) => Err(zx::Status::NOT_SUPPORTED),
315            // See the note in `read_to_array` above.
316            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
317        }
318    }
319
320    /// Reads from the memory.
321    ///
322    /// # Safety
323    ///
324    /// Callers must guarantee that the buffer is valid to write to.
325    ///
326    /// # Errors
327    ///
328    /// Returns `zx::Status::NOT_SUPPORTED` where unsupported.
329    pub unsafe fn read_raw(
330        &self,
331        buffer: *mut u8,
332        buffer_length: usize,
333        offset: u64,
334    ) -> Result<(), zx::Status> {
335        match self {
336            #[allow(clippy::undocumented_unsafe_blocks, reason = "2024 edition migration")]
337            Self::Vmo(info) => unsafe { info.vmo.read_raw(buffer, buffer_length, offset) },
338            Self::RingBuf(_) => Err(zx::Status::NOT_SUPPORTED),
339            // See the note in `read_to_array` above.
340            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
341        }
342    }
343
344    /// Write to memory.
345    ///
346    /// # Errors
347    ///
348    /// Returns `zx::Status::NOT_SUPPORTED` for read-only memory.
349    pub fn write(&self, data: &[u8], offset: u64) -> Result<(), zx::Status> {
350        match self {
351            Self::Vmo(info) => info.vmo.write(data, offset),
352            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
353        }
354    }
355
356    /// Returns the koid of the underlying memory-like object.
357    ///
358    /// Should be cheap to call frequently.
359    pub fn get_koid(&self) -> Koid {
360        match self {
361            Self::Vmo(info) => info.get_koid(),
362            Self::RingBuf(info) => info.get_koid(),
363            Self::MemoryMappedClock { koid, .. } => *koid,
364        }
365    }
366
367    /// Returns the rights of the underlying memory-like object.
368    pub fn get_rights(&self) -> zx::Rights {
369        match self {
370            Self::Vmo(info) => info.get_rights(),
371            Self::RingBuf(info) => info.get_rights(),
372            Self::MemoryMappedClock { utc_clock, .. } => {
373                utc_clock.basic_info().map_err(impossible_error).unwrap().rights
374            }
375        }
376    }
377
378    /// Returns `zx::VmoInfo` for a memory object that supports it.
379    ///
380    /// # Panics
381    ///
382    /// Calling `info` on a `MemoryObject` that is not represented by a VMO
383    /// will panic. To avoid this in code, call `is_clock` before attempting.
384    pub fn info(&self) -> Result<zx::VmoInfo, Errno> {
385        match self {
386            Self::Vmo(info) | Self::RingBuf(info) => info.vmo.info().map_err(|_| errno!(EIO)),
387            // Use `is_clock` to avoid calling info on a clock.
388            Self::MemoryMappedClock { .. } => {
389                panic!("info() is not supported on a memory mapped clock")
390            }
391        }
392    }
393
394    pub fn set_zx_name(&self, name: &[u8]) {
395        match self {
396            Self::Vmo(info) | Self::RingBuf(info) => set_zx_name(&*info.vmo, name),
397            Self::MemoryMappedClock { .. } => {
398                // The memory mapped clock is a singleton, so it does not
399                // seem appropriate to give it a zx name.
400            }
401        }
402    }
403
404    pub fn with_zx_name(self, name: &[u8]) -> Self {
405        self.set_zx_name(name);
406        self
407    }
408
409    pub fn op_range(
410        &self,
411        op: zx::VmoOp,
412        mut offset: u64,
413        mut size: u64,
414    ) -> Result<(), zx::Status> {
415        match self {
416            Self::Vmo(info) => info.vmo.op_range(op, offset, size),
417            Self::RingBuf(info) => {
418                let vmo_size = info.vmo.get_size().map_err(impossible_error).unwrap();
419                let data_size = vmo_size - (2 * *PAGE_SIZE);
420                let memory_size = vmo_size + data_size;
421                if offset + size > memory_size {
422                    return Err(zx::Status::OUT_OF_RANGE);
423                }
424                // If `offset` is greater than `vmo_size`, the operation is equivalent to the one
425                // done on the first part of the memory range.
426                if offset >= vmo_size {
427                    offset -= data_size;
428                }
429                // If the operation spill over the end if the vmo, it must be done on the start of
430                // the data part of the vmo.
431                if offset + size > vmo_size {
432                    info.vmo.op_range(op, 2 * *PAGE_SIZE, offset + size - vmo_size)?;
433                    size = vmo_size - offset;
434                }
435                info.vmo.op_range(op, offset, size)
436            }
437            Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
438        }
439    }
440
441    pub fn replace_as_executable(self, vmex: &zx::Resource) -> Result<Self, zx::Status> {
442        match self {
443            Self::Vmo(info) => {
444                // `zx_handle_replace` consumes the handle, so only duplicate it if it is still
445                // shared with another `MemoryObject`.
446                let vmo = info.into_owned_vmo()?;
447                let exec_vmo = vmo.replace_as_executable(vmex)?;
448                Ok(Self::from(exec_vmo))
449            }
450            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
451        }
452    }
453
454    pub fn map_in_vmar(
455        &self,
456        vmar: &zx::Vmar,
457        vmar_offset: usize,
458        mut memory_offset: u64,
459        len: usize,
460        flags: zx::VmarFlags,
461    ) -> Result<usize, zx::Status> {
462        match self {
463            Self::Vmo(info) => vmar.map(vmar_offset, &info.vmo, memory_offset, len, flags),
464            Self::RingBuf(info) => {
465                let vmo_size = info.vmo.get_size().map_err(impossible_error).unwrap();
466                let data_size = vmo_size - (2 * *PAGE_SIZE);
467                let memory_size = vmo_size + data_size;
468                if memory_offset.checked_add(len as u64).ok_or(zx::Status::OUT_OF_RANGE)?
469                    > memory_size
470                {
471                    return Err(zx::Status::OUT_OF_RANGE);
472                }
473                // If `memory_offset` is greater than `vmo_size`, the operation is equivalent to
474                // the one done on the first part of the memory range.
475                if memory_offset >= vmo_size {
476                    memory_offset -= data_size;
477                }
478                // Map the vmo for the full length. This ensures the kernel will choose a range
479                // that can accommodate the full length so that the second mapping will not erase
480                // another mapping.
481                let result = vmar.map(
482                    vmar_offset,
483                    &info.vmo,
484                    memory_offset,
485                    len,
486                    flags | zx::VmarFlags::ALLOW_FAULTS,
487                )?;
488                // The maximal amount of data that can have been mapped from the vmo with the
489                // previous operation.
490                let max_mapped_len = (vmo_size - memory_offset) as usize;
491                // If more data is needed, the data part of the vmo must be mapped again, replacing
492                // the part of the previous mapping that contained no data.
493                if len > max_mapped_len {
494                    let vmar_info = vmar.info().map_err(|_| errno!(EIO))?;
495                    let base_address = vmar_info.base;
496                    // The request should map the data part of the vmo a second time
497                    let second_mapping_address = vmar
498                        .map(
499                            result + max_mapped_len - base_address,
500                            &info.vmo,
501                            2 * *PAGE_SIZE,
502                            len - max_mapped_len,
503                            flags | ZX_VM_SPECIFIC_OVERWRITE,
504                        )
505                        .expect("Mapping should not fail as the space has been reserved");
506                    debug_assert_eq!(second_mapping_address, result + max_mapped_len);
507                }
508                Ok(result)
509            }
510            Self::MemoryMappedClock { utc_clock, .. } => {
511                // The memory mapped clock API only allows memory offset of 0, and a page-sized
512                // length of the mapping. No offset or partial mappings are allowed.
513                assert_eq!(0, memory_offset, "memory mapped clock must be at memory offset 0");
514
515                // We don't need to remember this, since vmar will know how to unmap it.
516                let memory_mapped_clock = MappedClock::try_new_without_unmap(
517                    &utc_clock,
518                    vmar,
519                    flags,
520                    vmar_offset as u64,
521                )?;
522                Ok(memory_mapped_clock.raw_addr())
523            }
524        }
525    }
526
527    pub fn memmove(
528        &self,
529        options: zx::TransferDataOptions,
530        dst_offset: u64,
531        src_offset: u64,
532        size: u64,
533    ) -> Result<(), zx::Status> {
534        match self {
535            Self::Vmo(info) => {
536                info.vmo.transfer_data(options, dst_offset, size, &info.vmo, src_offset)
537            }
538            Self::RingBuf(_) | Self::MemoryMappedClock { .. } => Err(zx::Status::NOT_SUPPORTED),
539        }
540    }
541
542    pub fn clone_memory(
543        self: &Arc<Self>,
544        rights: zx::Rights,
545        options: MappingOptions,
546    ) -> Result<Arc<Self>, Errno> {
547        if self.is_clock() {
548            return Err(errno!(ENOTSUP, "clone_memory not supported on memory mapped clock"));
549        }
550
551        // Non-anonymous memory is pager-backed, and we can clone it if we don't need write
552        // rights.
553        Ok(if !options.contains(MappingOptions::ANONYMOUS) && !rights.contains(zx::Rights::WRITE) {
554            self.clone()
555        } else {
556            fuchsia_trace::duration!(CATEGORY_STARNIX_MM, "pager_backed_memory_snapshot");
557            let mut cloned_memory = self
558                .create_child(
559                    zx::VmoChildOptions::SNAPSHOT_MODIFIED | zx::VmoChildOptions::RESIZABLE,
560                    0,
561                    self.get_size(),
562                )
563                .map_err(MemoryManager::get_errno_for_map_err)?;
564            if rights.contains(zx::Rights::EXECUTE) {
565                cloned_memory = cloned_memory
566                    .replace_as_executable(&VMEX_RESOURCE)
567                    .map_err(impossible_error)?;
568            }
569
570            Arc::new(cloned_memory)
571        })
572    }
573}
starnix_core/mm/memory.rs

starnix_core/mm/
memory.rs
