starnix_core/bpf/
map.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5// TODO(https://github.com/rust-lang/rust/issues/39371): remove
6#![allow(non_upper_case_globals)]
7
8use crate::mm::memory::MemoryObject;
9use crate::mm::{PAGE_SIZE, ProtectionFlags};
10use crate::security;
11use crate::task::{CurrentTask, Kernel, register_delayed_release};
12use ebpf::{MapFlags, MapSchema};
13use ebpf_api::{Map, MapError, PinnedMap, compute_map_storage_size};
14use starnix_lifecycle::{ObjectReleaser, ReleaserAction};
15use starnix_sync::{EbpfMapStateLevel, LockDepGuard, LockDepMutex};
16use starnix_types::ownership::{Releasable, ReleaseGuard};
17use starnix_uapi::auth::{CAP_BPF, CAP_NET_ADMIN, CAP_PERFMON, CAP_SYS_ADMIN};
18use starnix_uapi::errors::Errno;
19use starnix_uapi::math::round_up_to_increment;
20use starnix_uapi::{
21    bpf_map_type_BPF_MAP_TYPE_ARRAY, bpf_map_type_BPF_MAP_TYPE_ARRAY_OF_MAPS,
22    bpf_map_type_BPF_MAP_TYPE_BLOOM_FILTER, bpf_map_type_BPF_MAP_TYPE_CGROUP_STORAGE,
23    bpf_map_type_BPF_MAP_TYPE_CGRP_STORAGE, bpf_map_type_BPF_MAP_TYPE_CPUMAP,
24    bpf_map_type_BPF_MAP_TYPE_DEVMAP, bpf_map_type_BPF_MAP_TYPE_DEVMAP_HASH,
25    bpf_map_type_BPF_MAP_TYPE_HASH_OF_MAPS, bpf_map_type_BPF_MAP_TYPE_INODE_STORAGE,
26    bpf_map_type_BPF_MAP_TYPE_LPM_TRIE, bpf_map_type_BPF_MAP_TYPE_LRU_HASH,
27    bpf_map_type_BPF_MAP_TYPE_LRU_PERCPU_HASH, bpf_map_type_BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,
28    bpf_map_type_BPF_MAP_TYPE_QUEUE, bpf_map_type_BPF_MAP_TYPE_RINGBUF,
29    bpf_map_type_BPF_MAP_TYPE_SK_STORAGE, bpf_map_type_BPF_MAP_TYPE_SOCKHASH,
30    bpf_map_type_BPF_MAP_TYPE_SOCKMAP, bpf_map_type_BPF_MAP_TYPE_STACK,
31    bpf_map_type_BPF_MAP_TYPE_STACK_TRACE, bpf_map_type_BPF_MAP_TYPE_STRUCT_OPS,
32    bpf_map_type_BPF_MAP_TYPE_TASK_STORAGE, bpf_map_type_BPF_MAP_TYPE_XSKMAP, errno, error,
33};
34use std::ops::Deref;
35use std::sync::atomic::{AtomicU32, Ordering};
36use std::sync::{Arc, Weak};
37
38pub type BpfMapId = u32;
39
40/// Counter for map identifiers.
41static MAP_IDS: AtomicU32 = AtomicU32::new(1);
42fn new_map_id() -> BpfMapId {
43    MAP_IDS.fetch_add(1, Ordering::Relaxed)
44}
45
46pub(crate) fn map_error_to_errno(e: MapError) -> Errno {
47    match e {
48        MapError::InvalidParam => errno!(EINVAL),
49        MapError::InvalidKey => errno!(ENOENT),
50        MapError::EntryExists => errno!(EEXIST),
51        MapError::NoMemory => errno!(ENOMEM),
52        MapError::SizeLimit => errno!(E2BIG),
53        MapError::MapTypeNotSupported | MapError::NotSupported => errno!(ENOSYS),
54        MapError::InvalidVmo | MapError::Internal => errno!(EIO),
55    }
56}
57
58fn check_map_create_access(current_task: &CurrentTask, schema: &MapSchema) -> Result<(), Errno> {
59    if security::is_task_capable_noaudit(current_task, CAP_SYS_ADMIN) {
60        return Ok(());
61    }
62    let cap_bpf_always_required = matches!(
63        schema.map_type,
64        bpf_map_type_BPF_MAP_TYPE_LPM_TRIE
65            | bpf_map_type_BPF_MAP_TYPE_LRU_HASH
66            | bpf_map_type_BPF_MAP_TYPE_LRU_PERCPU_HASH
67            | bpf_map_type_BPF_MAP_TYPE_QUEUE
68            | bpf_map_type_BPF_MAP_TYPE_STACK
69            | bpf_map_type_BPF_MAP_TYPE_ARRAY_OF_MAPS
70            | bpf_map_type_BPF_MAP_TYPE_HASH_OF_MAPS
71            | bpf_map_type_BPF_MAP_TYPE_BLOOM_FILTER
72            | bpf_map_type_BPF_MAP_TYPE_SK_STORAGE
73            | bpf_map_type_BPF_MAP_TYPE_INODE_STORAGE
74            | bpf_map_type_BPF_MAP_TYPE_TASK_STORAGE
75            | bpf_map_type_BPF_MAP_TYPE_CGROUP_STORAGE
76            | bpf_map_type_BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
77            | bpf_map_type_BPF_MAP_TYPE_CGRP_STORAGE
78    );
79
80    if cap_bpf_always_required || !current_task.kernel().allow_unprivileged_bpf() {
81        security::check_task_capable(current_task, CAP_BPF)?;
82    }
83
84    match schema.map_type {
85        bpf_map_type_BPF_MAP_TYPE_DEVMAP
86        | bpf_map_type_BPF_MAP_TYPE_DEVMAP_HASH
87        | bpf_map_type_BPF_MAP_TYPE_CPUMAP
88        | bpf_map_type_BPF_MAP_TYPE_SOCKMAP
89        | bpf_map_type_BPF_MAP_TYPE_SOCKHASH
90        | bpf_map_type_BPF_MAP_TYPE_XSKMAP => {
91            security::check_task_capable(current_task, CAP_NET_ADMIN)?;
92        }
93        bpf_map_type_BPF_MAP_TYPE_STACK_TRACE => {
94            security::check_task_capable(current_task, CAP_PERFMON)?;
95        }
96        bpf_map_type_BPF_MAP_TYPE_STRUCT_OPS => {
97            return error!(EPERM);
98        }
99        _ => {}
100    }
101    Ok(())
102}
103
104#[derive(Debug, Default)]
105struct BpfMapState {
106    memory_object: Option<Arc<MemoryObject>>,
107    readonly_memory_object: Option<Arc<MemoryObject>>,
108    is_frozen: bool,
109}
110
111/// A BPF map and Starnix-specific metadata.
112#[derive(Debug)]
113pub struct BpfMap {
114    id: BpfMapId,
115    map: PinnedMap,
116
117    /// The internal state of the map object.
118    state: LockDepMutex<BpfMapState, EbpfMapStateLevel>,
119
120    /// The security state associated with this bpf Map.
121    pub security_state: security::BpfMapState,
122
123    /// Reference to the `Kernel`. Used to unregister `self` on drop.
124    kernel: Weak<Kernel>,
125}
126
127impl Deref for BpfMap {
128    type Target = PinnedMap;
129    fn deref(&self) -> &PinnedMap {
130        &self.map
131    }
132}
133
134impl BpfMap {
135    pub fn new(
136        current_task: &CurrentTask,
137        schema: MapSchema,
138        name: &str,
139        security_state: security::BpfMapState,
140    ) -> Result<BpfMapHandle, Errno> {
141        check_map_create_access(current_task, &schema)?;
142
143        let map = Map::new(schema, name).map_err(map_error_to_errno)?;
144        let map = BpfMapHandle::new(
145            Self {
146                id: new_map_id(),
147                map,
148                state: Default::default(),
149                security_state,
150                kernel: Arc::downgrade(current_task.kernel()),
151            }
152            .into(),
153        );
154        current_task.kernel().ebpf_state.register_map(&map);
155        Ok(map)
156    }
157
158    pub fn id(&self) -> BpfMapId {
159        self.id
160    }
161
162    pub(crate) fn frozen<'a>(&'a self) -> impl Deref<Target = bool> + 'a {
163        let guard = self.state.lock();
164        LockDepGuard::map(guard, |s| &mut s.is_frozen)
165    }
166
167    pub(crate) fn freeze(&self) -> Result<(), Errno> {
168        let mut state = self.state.lock();
169        if state.is_frozen {
170            return Ok(());
171        }
172        if let Some(memory) = state.memory_object.take() {
173            // The memory has been computed, check whether it is still in use.
174            if let Err(memory) = Arc::try_unwrap(memory) {
175                // There is other user of the memory. freeze must fail.
176                state.memory_object = Some(memory);
177                return error!(EBUSY);
178            }
179        }
180        state.is_frozen = true;
181        return Ok(());
182    }
183
184    pub(crate) fn get_inner(&self) -> PinnedMap {
185        self.map.clone()
186    }
187
188    pub(crate) fn get_memory(
189        &self,
190        length: usize,
191        prot: ProtectionFlags,
192    ) -> Result<Arc<MemoryObject>, Errno> {
193        let mut state = self.state.lock();
194        if state.is_frozen {
195            return error!(EPERM);
196        }
197
198        let page_size = *PAGE_SIZE as usize;
199        match self.schema.map_type {
200            bpf_map_type_BPF_MAP_TYPE_RINGBUF => {
201                // Only the first page of a ring buffer can be mapped as writable.
202                if length > page_size && prot.contains(ProtectionFlags::WRITE) {
203                    return error!(EPERM);
204                }
205                if length > 2 * page_size + 2 * self.schema.max_entries as usize {
206                    return error!(EINVAL);
207                }
208                if length <= page_size && prot.contains(ProtectionFlags::WRITE) {
209                    if let Some(memory) = state.memory_object.as_ref() {
210                        return Ok(memory.clone());
211                    }
212                    let consumer_vmo = self
213                        .vmo()
214                        .create_child(
215                            zx::VmoChildOptions::SLICE,
216                            page_size as u64,
217                            page_size as u64,
218                        )
219                        .map_err(|_| errno!(EIO))?;
220                    let memory = Arc::new(MemoryObject::from(consumer_vmo));
221                    state.memory_object = Some(memory.clone());
222                    Ok(memory)
223                } else {
224                    if let Some(memory) = state.readonly_memory_object.as_ref() {
225                        return Ok(memory.clone());
226                    }
227                    let clone_size = 2 * page_size + self.schema.max_entries as usize;
228                    let vmo_dup = self
229                        .vmo()
230                        .create_child(
231                            zx::VmoChildOptions::SLICE | zx::VmoChildOptions::NO_WRITE,
232                            page_size as u64,
233                            clone_size as u64,
234                        )
235                        .map_err(|_| errno!(EIO))?
236                        .into();
237                    let memory = Arc::new(MemoryObject::RingBuf(vmo_dup));
238                    state.readonly_memory_object = Some(memory.clone());
239                    Ok(memory)
240                }
241            }
242
243            bpf_map_type_BPF_MAP_TYPE_ARRAY => {
244                if !self.schema.flags.contains(MapFlags::Mmapable) {
245                    return error!(EPERM);
246                }
247
248                let array_size = round_up_to_increment(
249                    compute_map_storage_size(&self.schema).map_err(|_| errno!(EINVAL))?,
250                    page_size,
251                )?;
252                if length > array_size {
253                    return error!(EINVAL);
254                }
255
256                if let Some(memory) = state.memory_object.as_ref() {
257                    return Ok(memory.clone());
258                }
259                let vmo_dup: zx::Vmo = self
260                    .vmo()
261                    .as_handle_ref()
262                    .duplicate_handle(zx::Rights::SAME_RIGHTS)
263                    .map_err(|_| errno!(EIO))?
264                    .into();
265                let memory = Arc::new(MemoryObject::from(vmo_dup));
266                state.memory_object = Some(memory.clone());
267                Ok(memory)
268            }
269
270            _ => error!(ENODEV),
271        }
272    }
273}
274
275impl Releasable for BpfMap {
276    type Context<'a> = &'a CurrentTask;
277
278    fn release<'a>(self, _current_task: &'a CurrentTask) {
279        if let Some(kernel) = self.kernel.upgrade() {
280            kernel.ebpf_state.unregister_map(self.id);
281        }
282    }
283}
284
285pub enum BpfMapReleaserAction {}
286impl ReleaserAction<BpfMap> for BpfMapReleaserAction {
287    fn release(map: ReleaseGuard<BpfMap>) {
288        register_delayed_release(map);
289    }
290}
291pub type BpfMapReleaser = ObjectReleaser<BpfMap, BpfMapReleaserAction>;
292pub type BpfMapHandle = Arc<BpfMapReleaser>;
293pub type WeakBpfMapHandle = Weak<BpfMapReleaser>;
starnix_core/bpf/map.rs

starnix_core/bpf/
map.rs
