starnix_core/mm/
mapping.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::mm::memory::MemoryObject;
6use crate::mm::memory_manager::MemoryManagerState;
7use crate::mm::{
8    FaultRegisterMode, GUARD_PAGE_COUNT_FOR_GROWSDOWN_MAPPINGS, MappingOptions, PAGE_SIZE,
9    ProtectionFlags,
10};
11use crate::vfs::FileMapping;
12use crate::vfs::aio::AioContext;
13use bitflags::bitflags;
14use flyweights::FlyByteStr;
15use fuchsia_inspect::HistogramProperty;
16use starnix_uapi::errors::Errno;
17use starnix_uapi::file_mode::Access;
18use starnix_uapi::user_address::UserAddress;
19use starnix_uapi::{PROT_EXEC, PROT_READ, PROT_WRITE, errno};
20use static_assertions::const_assert_eq;
21use std::mem::MaybeUninit;
22use std::ops::Range;
23use std::sync::Arc;
24
25#[derive(Debug, Clone, Copy, PartialEq, Eq)]
26pub enum MappingMode {
27    Eager,
28    Lazy,
29}
30
31/// Describes a single memory mapping entry.
32///
33/// The size of this type *heavily* influences Starnix's heap usage in common scenarios, please
34/// think twice about increasing its size. Compiling in "release" mode includes a compile
35/// time check for size increase.
36#[split_enum_storage::container] // this also derives Clone, Debug, PartialEq, Eq
37#[must_use]
38pub struct Mapping {
39    /// Object backing this mapping.
40    backing: MappingBacking,
41
42    /// The flags used by the mapping, including protection.
43    flags: MappingFlags,
44
45    /// The name for this mapping.
46    ///
47    /// This may be a reference to the filesystem node backing this mapping or a userspace-assigned
48    /// name. The value of the name is orthogonal to whether this mapping is anonymous -
49    /// mappings of the file '/dev/zero' are treated as anonymous mappings and anonymous mappings
50    /// may have a name assigned.
51    ///
52    /// Because of this exception, avoid using this to check if a mapping is anonymous.
53    /// use [private_anonymous] method instead.
54    ///
55    /// NOTE: this *must* be accessed through `name()`/`set_name()` which are generated by below
56    /// macro.
57    #[split_enum_storage::decomposed]
58    name: MappingName,
59}
60
61// The size of this type *heavily* influences Starnix's heap usage in common scenarios, please
62// think twice about increasing the size here.
63#[cfg(not(any(test, debug_assertions)))]
64static_assertions::assert_eq_size!(Mapping, [u8; 24]);
65
66impl Mapping {
67    pub fn new(backing: MappingBacking, flags: MappingFlags, mode: MappingMode) -> Mapping {
68        Self::with_name(backing, flags, MappingName::None, mode)
69    }
70
71    pub fn with_name(
72        backing: MappingBacking,
73        mut flags: MappingFlags,
74        name: MappingName,
75        mode: MappingMode,
76    ) -> Mapping {
77        flags.set(MappingFlags::MAPPED_IN_VMAR, mode == MappingMode::Eager);
78        MappingUnsplit { backing, flags, name }.decompose()
79    }
80
81    pub fn flags(&self) -> MappingFlags {
82        self.flags
83    }
84
85    pub fn mapping_mode(&self) -> MappingMode {
86        if self.flags.contains(MappingFlags::MAPPED_IN_VMAR) {
87            MappingMode::Eager
88        } else {
89            MappingMode::Lazy
90        }
91    }
92
93    pub fn set_mapping_mode(&mut self, mode: MappingMode) {
94        self.flags.set(MappingFlags::MAPPED_IN_VMAR, mode == MappingMode::Eager);
95    }
96
97    pub fn set_flags(&mut self, new_flags: MappingFlags) {
98        self.flags = new_flags;
99    }
100
101    /// Maximum [`Access`] permissions allowed for this mapping, used by [`Mapping::vm_flags`] to
102    /// report `mr`, `mw`, and `me` in `/proc/[pid]/smaps` (see `proc_pid_smaps(5)`).
103    fn max_access(&self) -> Access {
104        match self.name() {
105            MappingNameRef::None
106            | MappingNameRef::Stack
107            | MappingNameRef::Heap
108            | MappingNameRef::Vma(_)
109            | MappingNameRef::Ashmem(_)
110            | MappingNameRef::AioContext(_) => Access::rwx(),
111            // The vDSO and vvar mappings are backed by read-only kernel VMOs (with execute rights
112            // for the vDSO) and cannot be made writable by userspace via `mprotect`.
113            MappingNameRef::Vdso => Access::READ | Access::EXEC,
114            MappingNameRef::Vvar => Access::READ,
115            MappingNameRef::File(file) => {
116                let mut access = file.file().max_access_for_memory_mapping();
117                // Private file mappings use copy-on-write, so they may always be made writable
118                // (e.g. via `mprotect(PROT_WRITE)`) even if the underlying file was opened
119                // read-only.
120                if !self.flags.contains(MappingFlags::SHARED) {
121                    access |= Access::WRITE;
122                }
123                access
124            }
125        }
126    }
127
128    pub fn get_backing_internal(&self) -> &MappingBacking {
129        &self.backing
130    }
131
132    pub fn set_backing_internal(&mut self, backing: MappingBacking) {
133        self.backing = backing;
134    }
135
136    pub fn set_uffd(&mut self, mode: FaultRegisterMode) {
137        self.flags |= MappingFlags::UFFD;
138        if mode == FaultRegisterMode::MISSING {
139            self.flags |= MappingFlags::UFFD_MISSING;
140        }
141    }
142
143    pub fn clear_uffd(&mut self) {
144        self.flags = self.flags.difference(MappingFlags::UFFD | MappingFlags::UFFD_MISSING);
145    }
146
147    pub fn set_mlock(&mut self) {
148        self.flags |= MappingFlags::LOCKED;
149    }
150
151    pub fn clear_mlock(&mut self) {
152        self.flags = self.flags.difference(MappingFlags::LOCKED);
153    }
154
155    pub fn new_private_anonymous(
156        flags: MappingFlags,
157        name: MappingName,
158        mode: MappingMode,
159    ) -> Mapping {
160        Self::with_name(MappingBacking::PrivateAnonymous, flags, name, mode)
161    }
162
163    pub fn inflate_to_include_guard_pages(&self, range: &Range<UserAddress>) -> Range<UserAddress> {
164        let start = if self.flags.contains(MappingFlags::GROWSDOWN) {
165            range
166                .start
167                .saturating_sub(*PAGE_SIZE as usize * GUARD_PAGE_COUNT_FOR_GROWSDOWN_MAPPINGS)
168        } else {
169            range.start
170        };
171        start..range.end
172    }
173
174    /// Converts a `UserAddress` to an offset in this mapping's memory object.
175    pub fn address_to_offset(&self, addr: UserAddress) -> u64 {
176        match &self.backing {
177            MappingBacking::Memory(backing) => backing.address_to_offset(addr),
178            MappingBacking::PrivateAnonymous => {
179                // For private, anonymous allocations the virtual address is the offset in the backing memory object.
180                addr.ptr() as u64
181            }
182        }
183    }
184
185    pub fn can_read(&self) -> bool {
186        self.flags.contains(MappingFlags::READ)
187    }
188
189    pub fn can_write(&self) -> bool {
190        self.flags.contains(MappingFlags::WRITE)
191    }
192
193    pub fn can_exec(&self) -> bool {
194        self.flags.contains(MappingFlags::EXEC)
195    }
196
197    pub fn private_anonymous(&self) -> bool {
198        if let MappingBacking::PrivateAnonymous = &self.backing {
199            return true;
200        }
201        !self.flags.contains(MappingFlags::SHARED) && self.flags.contains(MappingFlags::ANONYMOUS)
202    }
203
204    pub fn vm_flags(&self) -> String {
205        let mut string = String::default();
206        let max_access = self.max_access();
207        // From <https://man7.org/linux/man-pages/man5/proc_pid_smaps.5.html>:
208        //
209        // rd   -   readable
210        if self.flags.contains(MappingFlags::READ) {
211            string.push_str("rd ");
212        }
213        // wr   -   writable
214        if self.flags.contains(MappingFlags::WRITE) {
215            string.push_str("wr ");
216        }
217        // ex   -   executable
218        if self.flags.contains(MappingFlags::EXEC) {
219            string.push_str("ex ");
220        }
221        // sh   -   shared
222        if self.flags.contains(MappingFlags::SHARED) && max_access.contains(Access::WRITE) {
223            string.push_str("sh ");
224        }
225        // mr   -   may read
226        if max_access.contains(Access::READ) {
227            string.push_str("mr ");
228        }
229        // mw   -   may write
230        if max_access.contains(Access::WRITE) {
231            string.push_str("mw ");
232        }
233        // me   -   may execute
234        if max_access.contains(Access::EXEC) {
235            string.push_str("me ");
236        }
237        // ms   -   may share
238        if self.flags.contains(MappingFlags::SHARED) {
239            string.push_str("ms ");
240        }
241        // gd   -   stack segment grows down
242        if self.flags.contains(MappingFlags::GROWSDOWN) {
243            string.push_str("gd ");
244        }
245        // pf   -   pure PFN range
246        // dw   -   disabled write to the mapped file
247        // lo   -   pages are locked in memory
248        if self.flags.contains(MappingFlags::LOCKED) {
249            string.push_str("lo ");
250        }
251        // io   -   memory mapped I/O area
252        // sr   -   sequential read advise provided
253        // rr   -   random read advise provided
254        // dc   -   do not copy area on fork
255        if self.flags.contains(MappingFlags::DONTFORK) {
256            string.push_str("dc ");
257        }
258        // de   -   do not expand area on remapping
259        if self.flags.contains(MappingFlags::DONT_EXPAND) {
260            string.push_str("de ");
261        }
262        // ac   -   area is accountable
263        string.push_str("ac ");
264        // nr   -   swap space is not reserved for the area
265        // ht   -   area uses huge tlb pages
266        // sf   -   perform synchronous page faults (since Linux 4.15)
267        // nl   -   non-linear mapping (removed in Linux 4.0)
268        // ar   -   architecture specific flag
269        // wf   -   wipe on fork (since Linux 4.14)
270        if self.flags.contains(MappingFlags::WIPEONFORK) {
271            string.push_str("wf ");
272        }
273        // dd   -   do not include area into core dump
274        // sd   -   soft-dirty flag (since Linux 3.13)
275        // mm   -   mixed map area
276        // hg   -   huge page advise flag
277        // nh   -   no-huge page advise flag
278        // mg   -   mergeable advise flag
279        // um   -   userfaultfd missing pages tracking (since Linux 4.3)
280        if self.flags.contains(MappingFlags::UFFD_MISSING) {
281            string.push_str("um");
282        }
283        // uw   -   userfaultfd wprotect pages tracking (since Linux 4.3)
284        // ui   -   userfaultfd minor fault pages tracking (since Linux 5.13)
285        string
286    }
287}
288
289#[derive(Debug, Eq, PartialEq, Clone)]
290pub enum MappingBacking {
291    Memory(Box<MappingBackingMemory>),
292
293    PrivateAnonymous,
294}
295
296#[derive(Debug, Eq, PartialEq, Clone, split_enum_storage::SplitStorage)]
297pub enum MappingName {
298    /// No name.
299    None,
300
301    /// This mapping is the initial stack.
302    Stack,
303
304    /// This mapping is the heap.
305    Heap,
306
307    /// This mapping is the vdso.
308    Vdso,
309
310    /// This mapping is the vvar.
311    Vvar,
312
313    /// The file backing this mapping.
314    File(Arc<FileMapping>),
315
316    /// The name associated with the mapping. Set by prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ...).
317    /// An empty name is distinct from an unnamed mapping. Mappings are initially created with no
318    /// name and can be reset to the unnamed state by passing NULL to
319    /// prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, ...).
320    Vma(FlyByteStr),
321
322    /// The name associated with the mapping of an ashmem region.  Set by ioctl(fd, ASHMEM_SET_NAME, ...).
323    /// By default "dev/ashmem".
324    Ashmem(FlyByteStr),
325
326    /// This mapping is a context for asynchronous I/O.
327    AioContext(Arc<AioContext>),
328}
329
330#[derive(Debug, Eq, PartialEq, Clone)]
331pub struct MappingBackingMemory {
332    /// The memory object that contains the memory used in this mapping.
333    memory: Arc<MemoryObject>,
334
335    /// The delta to convert from a user address to an offset in the memory object.
336    address_to_offset_delta: u64,
337}
338
339impl MappingBackingMemory {
340    pub fn new(base: UserAddress, memory: Arc<MemoryObject>, memory_offset: u64) -> Self {
341        let address_to_offset_delta = memory_offset.wrapping_sub(base.ptr() as u64);
342        Self { memory, address_to_offset_delta }
343    }
344
345    pub fn memory(&self) -> &Arc<MemoryObject> {
346        &self.memory
347    }
348
349    /// Reads exactly `bytes.len()` bytes of memory from `addr`.
350    ///
351    /// # Parameters
352    /// - `addr`: The address to read data from.
353    /// - `bytes`: The byte array to read into.
354    pub fn read_memory<'a>(
355        &self,
356        addr: UserAddress,
357        bytes: &'a mut [MaybeUninit<u8>],
358    ) -> Result<&'a mut [u8], Errno> {
359        self.memory.read_uninit(bytes, self.address_to_offset(addr)).map_err(|_| errno!(EFAULT))
360    }
361
362    /// Writes the provided bytes to `addr`.
363    ///
364    /// # Parameters
365    /// - `addr`: The address to write to.
366    /// - `bytes`: The bytes to write to the memory object.
367    pub fn write_memory(&self, addr: UserAddress, bytes: &[u8]) -> Result<(), Errno> {
368        self.memory.write(bytes, self.address_to_offset(addr)).map_err(|_| errno!(EFAULT))
369    }
370
371    pub fn zero(&self, addr: UserAddress, length: usize) -> Result<usize, Errno> {
372        self.memory
373            .op_range(zx::VmoOp::ZERO, self.address_to_offset(addr), length as u64)
374            .map_err(|_| errno!(EFAULT))?;
375        Ok(length)
376    }
377
378    /// Converts a `UserAddress` to an offset in this mapping's memory object.
379    pub fn address_to_offset(&self, addr: UserAddress) -> u64 {
380        (addr.ptr() as u64).wrapping_add(self.address_to_offset_delta)
381    }
382}
383
384bitflags! {
385    #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
386    #[rustfmt::skip]  // Preserve column alignment.
387    pub struct MappingFlags: u16 {
388        const READ           = 1 <<  0;  // PROT_READ
389        const WRITE          = 1 <<  1;  // PROT_WRITE
390        const EXEC           = 1 <<  2;  // PROT_EXEC
391        const SHARED         = 1 <<  3;
392        const ANONYMOUS      = 1 <<  4;
393        const LOWER_32BIT    = 1 <<  5;
394        const GROWSDOWN      = 1 <<  6;
395        const ELF_BINARY     = 1 <<  7;
396        const DONTFORK       = 1 <<  8;
397        const WIPEONFORK     = 1 <<  9;
398        const DONT_SPLIT     = 1 << 10;
399        const DONT_EXPAND    = 1 << 11;
400        const LOCKED         = 1 << 12;
401        const UFFD           = 1 << 13;
402        const UFFD_MISSING   = 1 << 14;
403        const MAPPED_IN_VMAR = 1 << 15;
404    }
405}
406
407// The low three bits of MappingFlags match ProtectionFlags.
408const_assert_eq!(MappingFlags::READ.bits(), PROT_READ as u16);
409const_assert_eq!(MappingFlags::WRITE.bits(), PROT_WRITE as u16);
410const_assert_eq!(MappingFlags::EXEC.bits(), PROT_EXEC as u16);
411
412// The next bits of MappingFlags match MappingOptions, shifted up.
413const_assert_eq!(MappingFlags::SHARED.bits(), MappingOptions::SHARED.bits() << 3);
414const_assert_eq!(MappingFlags::ANONYMOUS.bits(), MappingOptions::ANONYMOUS.bits() << 3);
415const_assert_eq!(MappingFlags::LOWER_32BIT.bits(), MappingOptions::LOWER_32BIT.bits() << 3);
416const_assert_eq!(MappingFlags::GROWSDOWN.bits(), MappingOptions::GROWSDOWN.bits() << 3);
417const_assert_eq!(MappingFlags::ELF_BINARY.bits(), MappingOptions::ELF_BINARY.bits() << 3);
418const_assert_eq!(MappingFlags::DONTFORK.bits(), MappingOptions::DONTFORK.bits() << 3);
419const_assert_eq!(MappingFlags::WIPEONFORK.bits(), MappingOptions::WIPEONFORK.bits() << 3);
420const_assert_eq!(MappingFlags::DONT_SPLIT.bits(), MappingOptions::DONT_SPLIT.bits() << 3);
421const_assert_eq!(MappingFlags::DONT_EXPAND.bits(), MappingOptions::DONT_EXPAND.bits() << 3);
422
423impl MappingFlags {
424    pub fn access_flags(&self) -> ProtectionFlags {
425        ProtectionFlags::from_bits_truncate(
426            self.bits() as u32 & ProtectionFlags::ACCESS_FLAGS.bits(),
427        )
428    }
429
430    pub fn with_access_flags(&self, prot_flags: ProtectionFlags) -> Self {
431        let mapping_flags =
432            *self & (MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXEC).complement();
433        mapping_flags | Self::from_bits_truncate(prot_flags.access_flags().bits() as u16)
434    }
435
436    pub fn options(&self) -> MappingOptions {
437        MappingOptions::from_bits_truncate(self.bits() >> 3)
438    }
439
440    pub fn from_access_flags_and_options(
441        prot_flags: ProtectionFlags,
442        options: MappingOptions,
443    ) -> Self {
444        Self::from_bits_truncate(prot_flags.access_flags().bits() as u16)
445            | Self::from_bits_truncate(options.bits() << 3)
446    }
447}
448
449#[derive(Debug, Default)]
450pub struct MappingSummary {
451    no_kind: MappingKindSummary,
452    stack: MappingKindSummary,
453    heap: MappingKindSummary,
454    vdso: MappingKindSummary,
455    vvar: MappingKindSummary,
456    file: MappingKindSummary,
457    vma: MappingKindSummary,
458    ashmem: MappingKindSummary,
459    aiocontext: MappingKindSummary,
460
461    name_lengths: Vec<usize>,
462}
463
464impl MappingSummary {
465    pub fn add(&mut self, mm_state: &MemoryManagerState, mapping: &Mapping) {
466        let kind_summary = match mapping.name() {
467            MappingNameRef::None => &mut self.no_kind,
468            MappingNameRef::Stack => &mut self.stack,
469            MappingNameRef::Heap => &mut self.heap,
470            MappingNameRef::Vdso => &mut self.vdso,
471            MappingNameRef::Vvar => &mut self.vvar,
472            MappingNameRef::File(_) => &mut self.file,
473            MappingNameRef::Vma(name) => {
474                self.name_lengths.push(name.len());
475                &mut self.vma
476            }
477            MappingNameRef::Ashmem(name) => {
478                self.name_lengths.push(name.len());
479                &mut self.ashmem
480            }
481            MappingNameRef::AioContext(_) => &mut self.aiocontext,
482        };
483
484        kind_summary.count += 1;
485        if mapping.flags.contains(MappingFlags::SHARED) {
486            kind_summary.num_shared += 1;
487        } else {
488            kind_summary.num_private += 1;
489        }
490        match mm_state.get_mapping_backing(mapping) {
491            MappingBacking::Memory(_) => {
492                kind_summary.num_memory_objects += 1;
493            }
494            MappingBacking::PrivateAnonymous => kind_summary.num_private_anon += 1,
495        }
496    }
497
498    pub fn record(self, node: &fuchsia_inspect::Node) {
499        node.record_child("no_kind", |node| self.no_kind.record(node));
500        node.record_child("stack", |node| self.stack.record(node));
501        node.record_child("heap", |node| self.heap.record(node));
502        node.record_child("vdso", |node| self.vdso.record(node));
503        node.record_child("vvar", |node| self.vvar.record(node));
504        node.record_child("file", |node| self.file.record(node));
505        node.record_child("vma", |node| self.vma.record(node));
506        node.record_child("ashmem", |node| self.ashmem.record(node));
507        node.record_child("aiocontext", |node| self.aiocontext.record(node));
508
509        let name_lengths = node.create_uint_linear_histogram(
510            "name_lengths",
511            fuchsia_inspect::LinearHistogramParams { floor: 0, step_size: 8, buckets: 4 },
512        );
513        for l in self.name_lengths {
514            name_lengths.insert(l as u64);
515        }
516        node.record(name_lengths);
517    }
518}
519
520#[derive(Debug, Default)]
521struct MappingKindSummary {
522    count: u64,
523    num_private: u64,
524    num_shared: u64,
525    num_memory_objects: u64,
526    num_private_anon: u64,
527}
528
529impl MappingKindSummary {
530    fn record(&self, node: &fuchsia_inspect::Node) {
531        node.record_uint("count", self.count);
532        node.record_uint("num_private", self.num_private);
533        node.record_uint("num_shared", self.num_shared);
534        node.record_uint("num_memory_objects", self.num_memory_objects);
535        node.record_uint("num_private_anon", self.num_private_anon);
536    }
537}
starnix_core/mm/mapping.rs

starnix_core/mm/
mapping.rs
