starnix_core/mm/
vmsplice.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::mm::MemoryManager;
6use crate::mm::memory::MemoryObject;
7use crate::vfs::buffers::{InputBuffer, MessageData, OutputBuffer};
8use crate::vfs::with_iovec_segments;
9
10use smallvec::SmallVec;
11use starnix_sync::{InflightVmsplicedPayloadsLock, LockDepMutex, VmspliceSegmentsLock};
12use starnix_uapi::errors::Errno;
13use starnix_uapi::range_ext::RangeExt as _;
14use starnix_uapi::user_address::UserAddress;
15use starnix_uapi::{errno, error};
16use std::mem::MaybeUninit;
17use std::ops::Range;
18use std::sync::{Arc, Weak};
19
20/// A single segment of a `VmsplicePayload`.
21#[derive(Clone, Debug)]
22pub struct VmsplicePayloadSegment {
23    pub addr_offset: UserAddress,
24    pub length: usize,
25    /// The `MemoryObject` that contains the memory used in this mapping.
26    pub memory: Arc<MemoryObject>,
27    /// The offset in the `MemoryObject` that corresponds to the base address.
28    pub memory_offset: u64,
29    /// Whether this segment should be snapshotted on unmap.
30    pub should_snapshot_on_unmap: bool,
31}
32
33impl VmsplicePayloadSegment {
34    fn split_off(&mut self, index: usize) -> Option<Self> {
35        if index >= self.length {
36            return None;
37        }
38
39        let mut mapping = self.clone();
40        mapping.length = self.length - index;
41        mapping.addr_offset = match mapping.addr_offset + index {
42            Ok(new_addr) => new_addr,
43            Err(_) => return None,
44        };
45        mapping.memory_offset += index as u64;
46
47        self.length = index;
48        Some(mapping)
49    }
50
51    fn truncate(&mut self, limit: usize) {
52        // TODO(https://fxbug.dev/335701084): Truncating like this may leave
53        // unreachable memory in the VMO that is free to reclaim. We should
54        // reclaim the truncated memory if we can guarantee that it is no
55        // longer reachable by other means (e.g. other mappings, files, shared
56        // memory, etc.).
57        self.length = std::cmp::min(self.length, limit);
58    }
59
60    fn read_uninit(&self, data: &mut [MaybeUninit<u8>]) -> Result<(), zx::Status> {
61        self.memory.read_uninit(data, self.memory_offset)?;
62        Ok(())
63    }
64
65    /// Reads from the backing memory.
66    ///
67    /// # Safety
68    ///
69    /// Callers must guarantee that the buffer is valid to write to.
70    unsafe fn raw_read(&self, buffer: *mut u8, buffer_length: usize) -> Result<(), zx::Status> {
71        #[allow(clippy::undocumented_unsafe_blocks, reason = "2024 edition migration")]
72        unsafe {
73            self.memory.read_raw(buffer, buffer_length, self.memory_offset)
74        }
75    }
76}
77
78/// A single payload that may sit in a pipe as a consequence of a `vmsplice(2)`
79/// to a pipe.
80///
81/// A `VmsplicePayload` originally starts with a single segment. The payload
82/// may be split up into multiple segments as the payload sits in the pipe.
83/// This can happen when a mapping that is also backing a vmsplice-ed payload
84/// is modified such that the original segment is partially unmapped.
85///
86/// When the `VmsplicePayload` is created, it will be appended to its associated
87/// memory manager's [`InflightVmsplicedPayloads`]. The list cleans itself when
88/// `handle_unmapping` is run.
89#[derive(Debug, Default)]
90pub struct VmsplicePayload {
91    mapping: Weak<MemoryManager>,
92    segments: LockDepMutex<SmallVec<[VmsplicePayloadSegment; 1]>, VmspliceSegmentsLock>,
93}
94
95impl VmsplicePayload {
96    pub fn new(mapping: Weak<MemoryManager>, segment: VmsplicePayloadSegment) -> Arc<Self> {
97        Self::new_with_segments(mapping, [segment].into())
98    }
99
100    fn new_with_segments(
101        mapping: Weak<MemoryManager>,
102        segments: SmallVec<[VmsplicePayloadSegment; 1]>,
103    ) -> Arc<Self> {
104        let mapping_strong = mapping.upgrade();
105        let payload = Arc::new(Self { mapping, segments: segments.into() });
106        if let Some(mapping) = mapping_strong {
107            mapping.inflight_vmspliced_payloads.handle_new_payload(&payload);
108        }
109        payload
110    }
111}
112
113impl MessageData for Arc<VmsplicePayload> {
114    fn copy_from_user(_data: &mut dyn InputBuffer, _limit: usize) -> Result<Self, Errno> {
115        error!(ENOTSUP)
116    }
117
118    fn ptr(&self) -> Result<*const u8, Errno> {
119        error!(ENOTSUP)
120    }
121
122    fn with_bytes<O, F: FnMut(&[u8]) -> Result<O, Errno>>(&self, mut f: F) -> Result<O, Errno> {
123        let v = {
124            let segments = self.segments.lock();
125            let mut v = Vec::with_capacity(segments.iter().map(|s| s.length).sum());
126            for segment in segments.iter() {
127                segment
128                    .read_uninit(&mut v.spare_capacity_mut()[..segment.length])
129                    .map_err(|_| errno!(EFAULT))?;
130                // SAFETY: The read above succeeded.
131                unsafe { v.set_len(v.len() + segment.length) }
132            }
133            v
134        };
135        // Don't hold the lock because the callback may perform work which
136        // requires taking memory manager or mapping locks. Note that
137        // VmsplicePayload is modified while such locks are held (e.g. unmap).
138        f(&v)
139    }
140
141    fn len(&self) -> usize {
142        self.segments.lock().iter().map(|s| s.length).sum()
143    }
144
145    fn split_off(&mut self, mut limit: usize) -> Option<Self> {
146        let new_segments = {
147            let mut segments = self.segments.lock();
148
149            let mut split_at = 0;
150            for segment in segments.iter() {
151                if limit >= segment.length {
152                    limit -= segment.length;
153                    split_at += 1;
154                } else {
155                    break;
156                }
157            }
158
159            let mut new_segments = SmallVec::new();
160            if limit != 0 && split_at < segments.len() {
161                new_segments.push(segments[split_at].split_off(limit).unwrap());
162                split_at += 1;
163            };
164            if split_at <= segments.len() {
165                new_segments.extend(segments.drain(split_at..));
166            }
167            new_segments
168        };
169
170        if new_segments.is_empty() {
171            None
172        } else {
173            Some(VmsplicePayload::new_with_segments(self.mapping.clone(), new_segments))
174        }
175    }
176
177    fn truncate(&mut self, mut limit: usize) {
178        let mut segments = self.segments.lock();
179
180        segments.retain_mut(|segment| {
181            if limit >= segment.length {
182                limit -= segment.length;
183                true
184            } else if limit != 0 {
185                segment.truncate(limit);
186                limit = 0;
187                true
188            } else {
189                false
190            }
191        })
192    }
193
194    fn clone_at_most(&self, limit: usize) -> Self {
195        let mut payload =
196            VmsplicePayload::new_with_segments(self.mapping.clone(), self.segments.lock().clone());
197        payload.truncate(limit);
198        payload
199    }
200
201    fn copy_to_user(&self, data: &mut dyn OutputBuffer) -> Result<usize, Errno> {
202        let result = with_iovec_segments(data, |iovecs: &mut [syncio::zxio::zx_iovec]| {
203            let segments = self.segments.lock();
204            let length: usize = segments.iter().map(|s| s.length).sum();
205
206            let mut segments = segments.iter();
207            let mut current_segment = segments.next().cloned();
208            let mut copied = 0;
209
210            for iovec in iovecs {
211                if length == copied {
212                    break;
213                }
214
215                iovec.capacity = std::cmp::min(iovec.capacity, length - copied);
216
217                let mut iovec_pos = 0;
218                while let Some(segment) = &mut current_segment {
219                    if iovec_pos == iovec.capacity {
220                        break;
221                    }
222
223                    let to_read = std::cmp::min(segment.length, iovec.capacity - iovec_pos);
224                    let after_read = segment.split_off(to_read);
225                    // SAFETY: `iovec.buffer` has at least `iovec.capacity` bytes, and
226                    // `iovec_pos + to_read <= iovec.capacity`.
227                    unsafe { segment.raw_read((iovec.buffer as *mut u8).add(iovec_pos), to_read) }
228                        .map_err(|_| errno!(EFAULT))?;
229                    copied += to_read;
230                    iovec_pos += to_read;
231
232                    if let Some(after_read) = after_read {
233                        *segment = after_read;
234                    } else {
235                        current_segment = segments.next().cloned();
236                    }
237                }
238            }
239            Ok(copied)
240        });
241        match result {
242            Some(result) => {
243                let copied = result?;
244                // SAFETY: We just successfully read `copied` bytes from the `MemoryObject`
245                // to `data`.
246                unsafe { data.advance(copied)? };
247                Ok(copied)
248            }
249            None => self.with_bytes(|bytes| data.write(bytes)),
250        }
251    }
252}
253
254/// Keeps track of inflight vmsplice-ed payloads.
255///
256/// This is needed so that when a mapping is unmapped, inflight vmspliced payloads
257/// are updated to hold the (unmapped) bytes without being affected by any writes
258/// to the payload's backing `MemoryObject`.
259#[derive(Debug, Default)]
260pub struct InflightVmsplicedPayloads {
261    /// The inflight vmspliced payloads.
262    ///
263    /// Except when a [`VmsplicePayload`] is dropped, this is modified when the
264    /// memory manager's read lock is held. To allow a `vmsplice` operation to a
265    /// pipe to be performed without taking the memory manager's lock exclusively,
266    /// this is protected by its own `Mutex` instead of relying on the memory
267    /// manager's `RwLock`.
268    payloads: LockDepMutex<Vec<Weak<VmsplicePayload>>, InflightVmsplicedPayloadsLock>,
269}
270
271impl InflightVmsplicedPayloads {
272    fn handle_new_payload(&self, payload: &Arc<VmsplicePayload>) {
273        self.payloads.lock().push(Arc::downgrade(payload));
274    }
275
276    pub fn handle_unmapping(
277        &self,
278        unmapped_memory: &Arc<MemoryObject>,
279        unmapped_range: &Range<UserAddress>,
280    ) -> Result<(), Errno> {
281        // Iterate over payloads while removing any deleted payload.
282        let mut payloads = self.payloads.lock();
283        let mut index = 0;
284        while index < payloads.len() {
285            let Some(payload) = payloads[index].upgrade() else {
286                payloads.swap_remove(index);
287                continue;
288            };
289            index += 1;
290
291            let mut segments = payload.segments.lock();
292            let mut new_segments = SmallVec::new();
293
294            for segment in segments.iter() {
295                let mut segment = segment.clone();
296                let segment_end = (segment.addr_offset + segment.length)?;
297                let segment_range = segment.addr_offset..segment_end;
298                let segment_unmapped_range = unmapped_range.intersect(&segment_range);
299
300                if &segment.memory != unmapped_memory || segment_unmapped_range.is_empty() {
301                    // This can happen when say a partial unmapping was performed
302                    // on a `VmsplicePayloadSegment` which split it into a mapped
303                    // and unmapped set of payloads.
304                    new_segments.push(segment);
305                    continue;
306                }
307
308                // Keep the mapped head.
309                if segment_unmapped_range.start != segment_range.start {
310                    if let Some(tail) =
311                        segment.split_off(segment_unmapped_range.start - segment_range.start)
312                    {
313                        new_segments.push(segment);
314                        segment = tail;
315                    }
316                }
317
318                // Keep the mapped tail.
319                let tail = segment
320                    .split_off(segment.length - (segment_range.end - segment_unmapped_range.end));
321
322                // Snapshot the middle, actually unmapped, region if it should be snapshotted.
323                // For file-backed mappings, we want to keep referencing the shared VMO so that
324                // subsequent writes are visible.
325                //
326                // NB: we can't use `zx_vmo_transfer_data` because
327                // there may be multiple vmsplice payloads mapped
328                // to the same VMO region.
329                if segment.should_snapshot_on_unmap {
330                    let memory = segment
331                        .memory
332                        .create_child(
333                            zx::VmoChildOptions::SNAPSHOT_MODIFIED | zx::VmoChildOptions::NO_WRITE,
334                            segment.memory_offset,
335                            segment.length as u64,
336                        )
337                        .map_err(|_| errno!(EFAULT))?;
338
339                    segment.memory = Arc::new(memory);
340                    segment.memory_offset = 0;
341                }
342                new_segments.push(segment);
343
344                if let Some(tail) = tail {
345                    new_segments.push(tail);
346                }
347            }
348
349            *segments = new_segments;
350        }
351
352        Ok(())
353    }
354}
355
356#[cfg(test)]
357mod tests {
358    use super::*;
359    use crate::mm::PAGE_SIZE;
360    use crate::testing::spawn_kernel_and_run;
361    use crate::vfs::VecOutputBuffer;
362
363    #[::fuchsia::test]
364    async fn lifecycle() {
365        spawn_kernel_and_run(async |current_task| {
366            const NUM_PAGES: u64 = 3;
367            let page_size = *PAGE_SIZE;
368
369            let mm = current_task.mm().unwrap();
370
371            assert!(mm.inflight_vmspliced_payloads.payloads.lock().is_empty());
372
373            let memory_size = page_size * NUM_PAGES;
374            let memory = Arc::new(MemoryObject::from(zx::Vmo::create(memory_size).unwrap()));
375            let mut bytes = vec![0; memory_size as usize];
376            for i in 0..NUM_PAGES {
377                bytes[(page_size * i) as usize..][..(page_size as usize)].fill('A' as u8 + i as u8)
378            }
379            memory.write(&bytes, 0).unwrap();
380
381            let payload = VmsplicePayload::new(
382                Arc::downgrade(&mm),
383                VmsplicePayloadSegment {
384                    addr_offset: UserAddress::NULL,
385                    length: (page_size * NUM_PAGES) as usize,
386                    memory: Arc::clone(&memory),
387                    memory_offset: 0,
388                    should_snapshot_on_unmap: true,
389                },
390            );
391            assert_eq!(mm.inflight_vmspliced_payloads.payloads.lock().len(), 1);
392            assert_eq!(payload.segments.lock().len(), 1);
393
394            // A unmapping a different `MemoryObject` should do nothing.
395            {
396                let memory = Arc::new(MemoryObject::from(zx::Vmo::create(page_size).unwrap()));
397                mm.inflight_vmspliced_payloads
398                    .handle_unmapping(&memory, &(UserAddress::NULL..(u64::MAX.into())))
399                    .unwrap();
400                assert_eq!(payload.segments.lock().len(), 1);
401            }
402
403            mm.inflight_vmspliced_payloads
404                .handle_unmapping(&memory, &(UserAddress::NULL..page_size.into()))
405                .unwrap();
406            {
407                let segments = payload.segments.lock();
408                assert_eq!(segments.len(), 2);
409                assert!(!Arc::ptr_eq(&segments[0].memory, &memory));
410                assert!(Arc::ptr_eq(&segments[1].memory, &memory));
411            }
412            let mut got = VecOutputBuffer::new(memory_size as usize);
413            payload.copy_to_user(&mut got).unwrap();
414            assert_eq!(got.data(), &bytes);
415
416            std::mem::drop(payload);
417
418            // Run the unmapping again to ensure payload is dropped.
419            mm.inflight_vmspliced_payloads
420                .handle_unmapping(&memory, &(UserAddress::NULL..page_size.into()))
421                .unwrap();
422
423            assert!(mm.inflight_vmspliced_payloads.payloads.lock().is_empty());
424        })
425        .await;
426    }
427}
starnix_core/mm/vmsplice.rs

starnix_core/mm/
vmsplice.rs
