starnix_core/vfs/
userfault_file.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::mm::{
6    FaultCopyMode, FaultRegisterMode, FaultZeroMode, MemoryAccessorExt, UserFault,
7    UserFaultFeatures,
8};
9use crate::task::{CurrentTask, EventHandler, WaitCanceler, Waiter};
10use crate::vfs::{
11    Anon, FileHandle, FileObject, FileObjectState, FileOps, InputBuffer, OutputBuffer,
12    fileops_impl_nonseekable, fileops_impl_noop_sync,
13};
14use linux_uapi::{
15    UFFDIO_CONTINUE, UFFDIO_COPY, UFFDIO_WAKE, UFFDIO_WRITEPROTECT, UFFDIO_ZEROPAGE, uffdio_copy,
16    uffdio_zeropage,
17};
18use starnix_logging::track_stub;
19use starnix_sync::{FileOpsCore, LockBefore, LockEqualOrBefore, Locked, Unlocked, UserFaultInner};
20use starnix_uapi::errors::Errno;
21use starnix_uapi::open_flags::OpenFlags;
22use starnix_uapi::user_address::UserRef;
23use starnix_uapi::vfs::FdEvents;
24use starnix_uapi::{
25    _UFFDIO_API, _UFFDIO_REGISTER, _UFFDIO_UNREGISTER, UFFDIO, UFFDIO_API, UFFDIO_MOVE,
26    UFFDIO_POISON, UFFDIO_REGISTER, UFFDIO_UNREGISTER, errno, error, uapi, uffdio_api,
27    uffdio_range, uffdio_register,
28};
29use static_assertions::const_assert_eq;
30use std::sync::Arc;
31
32uapi::check_arch_independent_layout! {
33    uffdio_api {
34        api,
35        features,
36        ioctls,
37    }
38
39    uffdio_range {
40        start,
41        len,
42    }
43
44    uffdio_register {
45        range,
46        mode,
47        ioctls,
48    }
49
50    uffdio_copy {
51        dst,
52        src,
53        len,
54        mode,
55        copy,
56    }
57
58    uffdio_zeropage {
59        range,
60        mode,
61        zeropage,
62    }
63
64    uffdio_writeprotect {
65        range,
66        mode,
67    }
68
69    uffdio_continue {
70        range,
71        mode,
72        mapped,
73    }
74
75    uffdio_poison {
76        range,
77        mode,
78        updated,
79    }
80
81    uffdio_move {
82        dst,
83        src,
84        len,
85        mode,
86        move_,
87    }
88}
89
90pub struct UserFaultFile {
91    inner: Arc<UserFault>,
92}
93
94// API version hasn't changed
95const_assert_eq!(UFFDIO, 0xAA);
96
97impl UserFaultFile {
98    pub fn new<L>(
99        locked: &mut Locked<L>,
100        current_task: &CurrentTask,
101        open_flags: OpenFlags,
102        _user_mode_only: bool,
103    ) -> Result<FileHandle, Errno>
104    where
105        L: LockEqualOrBefore<FileOpsCore>,
106    {
107        let mm = current_task.mm()?;
108        let inner = Arc::new(UserFault::new(Arc::downgrade(&mm)));
109        mm.register_uffd(&inner);
110        Anon::new_file(locked, current_task, Box::new(Self { inner }), open_flags, "[userfaultfd]")
111    }
112
113    fn api_handshake<L>(
114        &self,
115        locked: &mut Locked<L>,
116        _current_task: &CurrentTask,
117        request: uffdio_api,
118    ) -> Result<uffdio_api, Errno>
119    where
120        L: LockBefore<UserFaultInner>,
121    {
122        if self.inner.is_initialized(locked) {
123            return error!(EPERM, "userfault object already initialized");
124        }
125
126        if request.api != UFFDIO as u64 {
127            return error!(EINVAL, format!("unsupported API version {}", request.api));
128        }
129
130        let requested_features =
131            UserFaultFeatures::from_bits(request.features.try_into().map_err(|_| errno!(EINVAL))?)
132                .ok_or_else(|| errno!(EINVAL))?;
133        let requested_unsupported = requested_features.difference(UserFaultFeatures::ALL_SUPPORTED);
134        if !requested_unsupported.is_empty() {
135            return error!(EINVAL);
136        }
137
138        // We can support the client, initialize the object.
139        self.inner.initialize(locked, requested_features);
140
141        Ok(uffdio_api {
142            api: request.api,
143            features: UserFaultFeatures::ALL_SUPPORTED.bits() as u64,
144            ioctls: (1 << _UFFDIO_API) | (1 << _UFFDIO_REGISTER) | (1 << _UFFDIO_UNREGISTER),
145        })
146    }
147}
148
149impl FileOps for UserFaultFile {
150    fileops_impl_nonseekable!();
151    fileops_impl_noop_sync!();
152    fn read(
153        &self,
154        _locked: &mut Locked<FileOpsCore>,
155        _file: &FileObject,
156        _current_task: &CurrentTask,
157        _offset: usize,
158        _data: &mut dyn OutputBuffer,
159    ) -> Result<usize, Errno> {
160        track_stub!(TODO("https://fxbug.dev/391599171"), "event-based uffd operations");
161        error!(ENOTSUP)
162    }
163
164    fn write(
165        &self,
166        _locked: &mut Locked<FileOpsCore>,
167        _file: &FileObject,
168        _current_task: &CurrentTask,
169        _offset: usize,
170        _data: &mut dyn InputBuffer,
171    ) -> Result<usize, Errno> {
172        error!(EINVAL)
173    }
174
175    fn query_events(
176        &self,
177        _locked: &mut Locked<FileOpsCore>,
178        _file: &FileObject,
179        _current_task: &CurrentTask,
180    ) -> Result<FdEvents, Errno> {
181        track_stub!(TODO("https://fxbug.dev/391599171"), "event-based uffd operations");
182        error!(ENOTSUP)
183    }
184
185    fn wait_async(
186        &self,
187        _locked: &mut Locked<FileOpsCore>,
188        _file: &FileObject,
189        _current_task: &CurrentTask,
190        _waiter: &Waiter,
191        _events: FdEvents,
192        _handler: EventHandler,
193    ) -> Option<WaitCanceler> {
194        track_stub!(TODO("https://fxbug.dev/391599171"), "event-based uffd operations");
195        None
196    }
197
198    fn ioctl(
199        &self,
200        locked: &mut Locked<Unlocked>,
201        _file: &FileObject,
202        current_task: &CurrentTask,
203        request: u32,
204        arg: starnix_syscalls::SyscallArg,
205    ) -> Result<starnix_syscalls::SyscallResult, Errno> {
206        match request {
207            UFFDIO_API => {
208                let arg: UserRef<uffdio_api> = arg.into();
209                let request = current_task.read_object(arg)?;
210                match self.api_handshake(locked, current_task, request) {
211                    Ok(reply) => {
212                        current_task.write_object(arg, &reply)?;
213                        Ok(0.into())
214                    }
215                    Err(e) => {
216                        current_task.write_object(arg, &uffdio_api::default())?;
217                        Err(e)
218                    }
219                }
220            }
221
222            UFFDIO_REGISTER => {
223                let arg: UserRef<uffdio_register> = arg.into();
224                let mut request = current_task.read_object(arg)?;
225
226                request.ioctls = self
227                    .inner
228                    .op_register(
229                        locked,
230                        request.range.start.into(),
231                        request.range.len,
232                        FaultRegisterMode::from_bits_truncate(
233                            request.mode.try_into().map_err(|_| errno!(EINVAL))?,
234                        ),
235                    )?
236                    .bits();
237                current_task.write_object(arg, &request)?;
238                Ok(0.into())
239            }
240
241            UFFDIO_UNREGISTER => {
242                let arg: UserRef<uffdio_range> = arg.into();
243                let request = current_task.read_object(arg)?;
244                self.inner.op_unregister(locked, request.start.into(), request.len)?;
245                Ok(0.into())
246            }
247
248            UFFDIO_ZEROPAGE => {
249                let arg: UserRef<uffdio_zeropage> = arg.into();
250                let mut request = current_task.read_object(arg)?;
251                let ioctl_res = self.inner.op_zero(
252                    locked,
253                    request.range.start.into(),
254                    request.range.len,
255                    FaultZeroMode::from_bits_truncate(
256                        request.mode.try_into().map_err(|_| errno!(EINVAL))?,
257                    ),
258                );
259                request.zeropage = match ioctl_res {
260                    Ok(bytes) => bytes as i64,
261                    Err(ref e) => -1 * (e.code.error_code() as i64),
262                };
263                current_task.write_object(arg, &request)?;
264                // EAGAIN is returned if the number of bytes zeroed is not equal to the requested
265                // length
266                match ioctl_res {
267                    Ok(bytes) if bytes == request.range.len as usize => Ok(0.into()),
268                    Err(e) => Err(e),
269                    _ => error!(EAGAIN),
270                }
271            }
272
273            UFFDIO_COPY => {
274                let arg: UserRef<uffdio_copy> = arg.into();
275                let mut request = current_task.read_object(arg)?;
276                let mm = current_task.mm()?;
277                let ioctl_res = self.inner.op_copy(
278                    locked,
279                    &mm,
280                    request.src.into(),
281                    request.dst.into(),
282                    request.len,
283                    FaultCopyMode::from_bits_truncate(
284                        request.mode.try_into().map_err(|_| errno!(EINVAL))?,
285                    ),
286                );
287                request.copy = match ioctl_res {
288                    Ok(bytes) => bytes as i64,
289                    Err(ref e) => -1 * (e.code.error_code() as i64),
290                };
291                current_task.write_object(arg, &request)?;
292                // EAGAIN is returned if the number of bytes copied is not equal to the requested
293                // length
294                match ioctl_res {
295                    Ok(bytes) if bytes == request.len as usize => Ok(0.into()),
296                    Err(e) => Err(e),
297                    _ => error!(EAGAIN),
298                }
299            }
300
301            UFFDIO_MOVE => {
302                track_stub!(TODO("https://fxbug.dev/297375964"), "basic uffd ioctls", request);
303                error!(ENOSYS)
304            }
305
306            UFFDIO_WAKE | UFFDIO_WRITEPROTECT | UFFDIO_CONTINUE | UFFDIO_POISON => {
307                track_stub!(
308                    TODO("https://fxbug.dev/322893681"),
309                    "full set of uffd ioctls",
310                    request
311                );
312                error!(ENOSYS)
313            }
314
315            unknown => error!(EINVAL, format!("unknown ioctl request {unknown}")),
316        }
317    }
318
319    // On closing, clear all the registrations pointing to this userfault object.
320    fn close(
321        self: Box<Self>,
322        locked: &mut Locked<FileOpsCore>,
323        _file: &FileObjectState,
324        _current_task: &CurrentTask,
325    ) {
326        self.inner.cleanup(locked);
327    }
328}
starnix_core/vfs/userfault_file.rs

starnix_core/vfs/
userfault_file.rs
