starnix_modules_userfaultfd/
userfault_file.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use linux_uapi::{
6    UFFDIO_CONTINUE, UFFDIO_COPY, UFFDIO_WAKE, UFFDIO_WRITEPROTECT, UFFDIO_ZEROPAGE, uffdio_copy,
7    uffdio_zeropage,
8};
9use starnix_core::mm::{
10    FaultCopyMode, FaultRegisterMode, FaultZeroMode, MemoryAccessorExt, UserFault,
11    UserFaultFeatures,
12};
13use starnix_core::task::{CurrentTask, EventHandler, WaitCanceler, Waiter};
14use starnix_core::vfs::{
15    Anon, FileHandle, FileObject, FileObjectState, FileOps, InputBuffer, OutputBuffer,
16    fileops_impl_nonseekable, fileops_impl_noop_sync,
17};
18use starnix_logging::track_stub;
19
20use starnix_uapi::errors::Errno;
21use starnix_uapi::open_flags::OpenFlags;
22use starnix_uapi::user_address::UserRef;
23use starnix_uapi::vfs::FdEvents;
24use starnix_uapi::{
25    _UFFDIO_API, _UFFDIO_REGISTER, _UFFDIO_UNREGISTER, UFFDIO, UFFDIO_API, UFFDIO_MOVE,
26    UFFDIO_POISON, UFFDIO_REGISTER, UFFDIO_UNREGISTER, errno, error, uapi, uffdio_api,
27    uffdio_range, uffdio_register,
28};
29use static_assertions::const_assert_eq;
30use std::sync::Arc;
31
32uapi::check_arch_independent_layout! {
33    uffdio_api {
34        api,
35        features,
36        ioctls,
37    }
38
39    uffdio_range {
40        start,
41        len,
42    }
43
44    uffdio_register {
45        range,
46        mode,
47        ioctls,
48    }
49
50    uffdio_copy {
51        dst,
52        src,
53        len,
54        mode,
55        copy,
56    }
57
58    uffdio_zeropage {
59        range,
60        mode,
61        zeropage,
62    }
63
64    uffdio_writeprotect {
65        range,
66        mode,
67    }
68
69    uffdio_continue {
70        range,
71        mode,
72        mapped,
73    }
74
75    uffdio_poison {
76        range,
77        mode,
78        updated,
79    }
80
81    uffdio_move {
82        dst,
83        src,
84        len,
85        mode,
86        move_,
87    }
88}
89
90pub struct UserFaultFile {
91    inner: Arc<UserFault>,
92}
93
94// API version hasn't changed
95const_assert_eq!(UFFDIO, 0xAA);
96
97impl UserFaultFile {
98    pub fn new(
99        current_task: &CurrentTask,
100        open_flags: OpenFlags,
101        _user_mode_only: bool,
102    ) -> Result<FileHandle, Errno> {
103        let mm = current_task.mm()?;
104        let inner = Arc::new(UserFault::new(Arc::downgrade(&mm)));
105        mm.register_uffd(&inner);
106        Anon::new_file(current_task, Box::new(Self { inner }), open_flags, "[userfaultfd]")
107    }
108
109    fn api_handshake(
110        &self,
111        _current_task: &CurrentTask,
112        request: uffdio_api,
113    ) -> Result<uffdio_api, Errno> {
114        if self.inner.is_initialized() {
115            return error!(EPERM, "userfault object already initialized");
116        }
117
118        if request.api != UFFDIO as u64 {
119            return error!(EINVAL, format!("unsupported API version {}", request.api));
120        }
121
122        let requested_features =
123            UserFaultFeatures::from_bits(request.features.try_into().map_err(|_| errno!(EINVAL))?)
124                .ok_or_else(|| errno!(EINVAL))?;
125        let requested_unsupported = requested_features.difference(UserFaultFeatures::ALL_SUPPORTED);
126        if !requested_unsupported.is_empty() {
127            return error!(EINVAL);
128        }
129
130        // We can support the client, initialize the object.
131        self.inner.initialize(requested_features);
132
133        Ok(uffdio_api {
134            api: request.api,
135            features: UserFaultFeatures::ALL_SUPPORTED.bits() as u64,
136            ioctls: (1 << _UFFDIO_API) | (1 << _UFFDIO_REGISTER) | (1 << _UFFDIO_UNREGISTER),
137        })
138    }
139}
140
141impl FileOps for UserFaultFile {
142    fileops_impl_nonseekable!();
143    fileops_impl_noop_sync!();
144    fn read(
145        &self,
146        _file: &FileObject,
147        _current_task: &CurrentTask,
148        _offset: usize,
149        _data: &mut dyn OutputBuffer,
150    ) -> Result<usize, Errno> {
151        track_stub!(TODO("https://fxbug.dev/391599171"), "event-based uffd operations");
152        error!(ENOTSUP)
153    }
154
155    fn write(
156        &self,
157        _file: &FileObject,
158        _current_task: &CurrentTask,
159        _offset: usize,
160        _data: &mut dyn InputBuffer,
161    ) -> Result<usize, Errno> {
162        error!(EINVAL)
163    }
164
165    fn query_events(
166        &self,
167        _file: &FileObject,
168        _current_task: &CurrentTask,
169    ) -> Result<FdEvents, Errno> {
170        track_stub!(TODO("https://fxbug.dev/391599171"), "event-based uffd operations");
171        error!(ENOTSUP)
172    }
173
174    fn wait_async(
175        &self,
176        _file: &FileObject,
177        _current_task: &CurrentTask,
178        _waiter: &Waiter,
179        _events: FdEvents,
180        _handler: EventHandler,
181    ) -> Option<WaitCanceler> {
182        track_stub!(TODO("https://fxbug.dev/391599171"), "event-based uffd operations");
183        None
184    }
185
186    fn ioctl(
187        &self,
188        _file: &FileObject,
189        current_task: &CurrentTask,
190        request: u32,
191        arg: starnix_syscalls::SyscallArg,
192    ) -> Result<starnix_syscalls::SyscallResult, Errno> {
193        match request {
194            UFFDIO_API => {
195                let arg: UserRef<uffdio_api> = arg.into();
196                let request = current_task.read_object(arg)?;
197                match self.api_handshake(current_task, request) {
198                    Ok(reply) => {
199                        current_task.write_object(arg, &reply)?;
200                        Ok(0.into())
201                    }
202                    Err(e) => {
203                        current_task.write_object(arg, &uffdio_api::default())?;
204                        Err(e)
205                    }
206                }
207            }
208
209            UFFDIO_REGISTER => {
210                let arg: UserRef<uffdio_register> = arg.into();
211                let mut request = current_task.read_object(arg)?;
212
213                request.ioctls = self
214                    .inner
215                    .op_register(
216                        request.range.start.into(),
217                        request.range.len,
218                        FaultRegisterMode::from_bits_truncate(
219                            request.mode.try_into().map_err(|_| errno!(EINVAL))?,
220                        ),
221                    )?
222                    .bits();
223                current_task.write_object(arg, &request)?;
224                Ok(0.into())
225            }
226
227            UFFDIO_UNREGISTER => {
228                let arg: UserRef<uffdio_range> = arg.into();
229                let request = current_task.read_object(arg)?;
230                self.inner.op_unregister(request.start.into(), request.len)?;
231                Ok(0.into())
232            }
233
234            UFFDIO_ZEROPAGE => {
235                let arg: UserRef<uffdio_zeropage> = arg.into();
236                let mut request = current_task.read_object(arg)?;
237                let ioctl_res = self.inner.op_zero(
238                    request.range.start.into(),
239                    request.range.len,
240                    FaultZeroMode::from_bits_truncate(
241                        request.mode.try_into().map_err(|_| errno!(EINVAL))?,
242                    ),
243                );
244                request.zeropage = match ioctl_res {
245                    Ok(bytes) => bytes as i64,
246                    Err(ref e) => -1 * (e.code.error_code() as i64),
247                };
248                current_task.write_object(arg, &request)?;
249                // EAGAIN is returned if the number of bytes zeroed is not equal to the requested
250                // length
251                match ioctl_res {
252                    Ok(bytes) if bytes == request.range.len as usize => Ok(0.into()),
253                    Err(e) => Err(e),
254                    _ => error!(EAGAIN),
255                }
256            }
257
258            UFFDIO_COPY => {
259                let arg: UserRef<uffdio_copy> = arg.into();
260                let mut request = current_task.read_object(arg)?;
261                let mm = current_task.mm()?;
262                let ioctl_res = self.inner.op_copy(
263                    &mm,
264                    request.src.into(),
265                    request.dst.into(),
266                    request.len,
267                    FaultCopyMode::from_bits_truncate(
268                        request.mode.try_into().map_err(|_| errno!(EINVAL))?,
269                    ),
270                );
271                request.copy = match ioctl_res {
272                    Ok(bytes) => bytes as i64,
273                    Err(ref e) => -1 * (e.code.error_code() as i64),
274                };
275                current_task.write_object(arg, &request)?;
276                // EAGAIN is returned if the number of bytes copied is not equal to the requested
277                // length
278                match ioctl_res {
279                    Ok(bytes) if bytes == request.len as usize => Ok(0.into()),
280                    Err(e) => Err(e),
281                    _ => error!(EAGAIN),
282                }
283            }
284
285            UFFDIO_MOVE => {
286                track_stub!(TODO("https://fxbug.dev/297375964"), "basic uffd ioctls", request);
287                error!(ENOSYS)
288            }
289
290            UFFDIO_WAKE | UFFDIO_WRITEPROTECT | UFFDIO_CONTINUE | UFFDIO_POISON => {
291                track_stub!(
292                    TODO("https://fxbug.dev/322893681"),
293                    "full set of uffd ioctls",
294                    request
295                );
296                error!(ENOSYS)
297            }
298
299            unknown => error!(EINVAL, format!("unknown ioctl request {unknown}")),
300        }
301    }
302
303    // On closing, clear all the registrations pointing to this userfault object.
304    fn close(self: Box<Self>, _file: &FileObjectState, _current_task: &CurrentTask) {
305        self.inner.cleanup();
306    }
307}
starnix_modules_userfaultfd/userfault_file.rs

starnix_modules_userfaultfd/
userfault_file.rs
