starnix_core/bpf/
map.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5// TODO(https://github.com/rust-lang/rust/issues/39371): remove
6#![allow(non_upper_case_globals)]
7
8use crate::bpf::check_cap_or_sys_admin;
9use crate::mm::memory::MemoryObject;
10use crate::mm::{PAGE_SIZE, ProtectionFlags};
11use crate::security;
12use crate::task::{CurrentTask, Kernel, register_delayed_release};
13use ebpf::{MapFlags, MapSchema};
14use ebpf_api::{Map, MapError, PinnedMap, compute_map_storage_size};
15use starnix_lifecycle::{ObjectReleaser, ReleaserAction};
16use starnix_sync::{EbpfMapStateLevel, LockDepGuard, LockDepMutex};
17use starnix_types::ownership::{Releasable, ReleaseGuard};
18use starnix_uapi::auth::{CAP_BPF, CAP_NET_ADMIN, CAP_PERFMON};
19use starnix_uapi::errors::Errno;
20use starnix_uapi::math::round_up_to_increment;
21use starnix_uapi::{
22    bpf_map_type_BPF_MAP_TYPE_ARRAY, bpf_map_type_BPF_MAP_TYPE_ARRAY_OF_MAPS,
23    bpf_map_type_BPF_MAP_TYPE_BLOOM_FILTER, bpf_map_type_BPF_MAP_TYPE_CGROUP_STORAGE,
24    bpf_map_type_BPF_MAP_TYPE_CGRP_STORAGE, bpf_map_type_BPF_MAP_TYPE_CPUMAP,
25    bpf_map_type_BPF_MAP_TYPE_DEVMAP, bpf_map_type_BPF_MAP_TYPE_DEVMAP_HASH,
26    bpf_map_type_BPF_MAP_TYPE_HASH_OF_MAPS, bpf_map_type_BPF_MAP_TYPE_INODE_STORAGE,
27    bpf_map_type_BPF_MAP_TYPE_LPM_TRIE, bpf_map_type_BPF_MAP_TYPE_LRU_HASH,
28    bpf_map_type_BPF_MAP_TYPE_LRU_PERCPU_HASH, bpf_map_type_BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE,
29    bpf_map_type_BPF_MAP_TYPE_QUEUE, bpf_map_type_BPF_MAP_TYPE_RINGBUF,
30    bpf_map_type_BPF_MAP_TYPE_SK_STORAGE, bpf_map_type_BPF_MAP_TYPE_SOCKHASH,
31    bpf_map_type_BPF_MAP_TYPE_SOCKMAP, bpf_map_type_BPF_MAP_TYPE_STACK,
32    bpf_map_type_BPF_MAP_TYPE_STACK_TRACE, bpf_map_type_BPF_MAP_TYPE_STRUCT_OPS,
33    bpf_map_type_BPF_MAP_TYPE_TASK_STORAGE, bpf_map_type_BPF_MAP_TYPE_XSKMAP, errno, error,
34};
35use std::ops::Deref;
36use std::sync::atomic::{AtomicU32, Ordering};
37use std::sync::{Arc, Weak};
38
39pub type BpfMapId = u32;
40
41/// Counter for map identifiers.
42static MAP_IDS: AtomicU32 = AtomicU32::new(1);
43fn new_map_id() -> BpfMapId {
44    MAP_IDS.fetch_add(1, Ordering::Relaxed)
45}
46
47pub(crate) fn map_error_to_errno(e: MapError) -> Errno {
48    match e {
49        MapError::InvalidParam => errno!(EINVAL),
50        MapError::InvalidKey => errno!(ENOENT),
51        MapError::EntryExists => errno!(EEXIST),
52        MapError::NoMemory => errno!(ENOMEM),
53        MapError::SizeLimit => errno!(E2BIG),
54        MapError::MapTypeNotSupported | MapError::NotSupported => errno!(ENOSYS),
55        MapError::InvalidVmo | MapError::Internal => errno!(EIO),
56    }
57}
58
59fn check_map_create_access(current_task: &CurrentTask, schema: &MapSchema) -> Result<(), Errno> {
60    let cap_bpf_always_required = matches!(
61        schema.map_type,
62        bpf_map_type_BPF_MAP_TYPE_LPM_TRIE
63            | bpf_map_type_BPF_MAP_TYPE_LRU_HASH
64            | bpf_map_type_BPF_MAP_TYPE_LRU_PERCPU_HASH
65            | bpf_map_type_BPF_MAP_TYPE_QUEUE
66            | bpf_map_type_BPF_MAP_TYPE_STACK
67            | bpf_map_type_BPF_MAP_TYPE_ARRAY_OF_MAPS
68            | bpf_map_type_BPF_MAP_TYPE_HASH_OF_MAPS
69            | bpf_map_type_BPF_MAP_TYPE_BLOOM_FILTER
70            | bpf_map_type_BPF_MAP_TYPE_SK_STORAGE
71            | bpf_map_type_BPF_MAP_TYPE_INODE_STORAGE
72            | bpf_map_type_BPF_MAP_TYPE_TASK_STORAGE
73            | bpf_map_type_BPF_MAP_TYPE_CGROUP_STORAGE
74            | bpf_map_type_BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE
75            | bpf_map_type_BPF_MAP_TYPE_CGRP_STORAGE
76    );
77
78    if cap_bpf_always_required || !current_task.kernel().allow_unprivileged_bpf() {
79        check_cap_or_sys_admin(current_task, CAP_BPF)?;
80    }
81
82    match schema.map_type {
83        bpf_map_type_BPF_MAP_TYPE_DEVMAP
84        | bpf_map_type_BPF_MAP_TYPE_DEVMAP_HASH
85        | bpf_map_type_BPF_MAP_TYPE_CPUMAP
86        | bpf_map_type_BPF_MAP_TYPE_SOCKMAP
87        | bpf_map_type_BPF_MAP_TYPE_SOCKHASH
88        | bpf_map_type_BPF_MAP_TYPE_XSKMAP => {
89            check_cap_or_sys_admin(current_task, CAP_NET_ADMIN)?;
90        }
91        bpf_map_type_BPF_MAP_TYPE_STACK_TRACE => {
92            check_cap_or_sys_admin(current_task, CAP_PERFMON)?;
93        }
94        bpf_map_type_BPF_MAP_TYPE_STRUCT_OPS => {
95            return error!(EPERM);
96        }
97        _ => {}
98    }
99    Ok(())
100}
101
102#[derive(Debug, Default)]
103struct BpfMapState {
104    memory_object: Option<Arc<MemoryObject>>,
105    readonly_memory_object: Option<Arc<MemoryObject>>,
106    is_frozen: bool,
107}
108
109/// A BPF map and Starnix-specific metadata.
110#[derive(Debug)]
111pub struct BpfMap {
112    id: BpfMapId,
113    map: PinnedMap,
114
115    /// The internal state of the map object.
116    state: LockDepMutex<BpfMapState, EbpfMapStateLevel>,
117
118    /// The security state associated with this bpf Map.
119    pub security_state: security::BpfMapState,
120
121    /// Reference to the `Kernel`. Used to unregister `self` on drop.
122    kernel: Weak<Kernel>,
123}
124
125impl Deref for BpfMap {
126    type Target = PinnedMap;
127    fn deref(&self) -> &PinnedMap {
128        &self.map
129    }
130}
131
132impl BpfMap {
133    pub fn new(
134        current_task: &CurrentTask,
135        schema: MapSchema,
136        name: &str,
137        security_state: security::BpfMapState,
138    ) -> Result<BpfMapHandle, Errno> {
139        check_map_create_access(current_task, &schema)?;
140
141        let map = Map::new(schema, name).map_err(map_error_to_errno)?;
142        let map = BpfMapHandle::new(
143            Self {
144                id: new_map_id(),
145                map,
146                state: Default::default(),
147                security_state,
148                kernel: Arc::downgrade(current_task.kernel()),
149            }
150            .into(),
151        );
152        current_task.kernel().ebpf_state.register_map(&map);
153        Ok(map)
154    }
155
156    pub fn id(&self) -> BpfMapId {
157        self.id
158    }
159
160    pub(crate) fn frozen<'a>(&'a self) -> impl Deref<Target = bool> + 'a {
161        let guard = self.state.lock();
162        LockDepGuard::map(guard, |s| &mut s.is_frozen)
163    }
164
165    pub(crate) fn freeze(&self) -> Result<(), Errno> {
166        let mut state = self.state.lock();
167        if state.is_frozen {
168            return Ok(());
169        }
170        if let Some(memory) = state.memory_object.take() {
171            // The memory has been computed, check whether it is still in use.
172            if let Err(memory) = Arc::try_unwrap(memory) {
173                // There is other user of the memory. freeze must fail.
174                state.memory_object = Some(memory);
175                return error!(EBUSY);
176            }
177        }
178        state.is_frozen = true;
179        return Ok(());
180    }
181
182    pub(crate) fn get_inner(&self) -> PinnedMap {
183        self.map.clone()
184    }
185
186    pub(crate) fn get_memory(
187        &self,
188        length: usize,
189        prot: ProtectionFlags,
190    ) -> Result<Arc<MemoryObject>, Errno> {
191        let mut state = self.state.lock();
192        let writable = prot.contains(ProtectionFlags::WRITE);
193        if state.is_frozen && writable {
194            return error!(EPERM);
195        }
196
197        let page_size = *PAGE_SIZE as usize;
198        match self.schema.map_type {
199            bpf_map_type_BPF_MAP_TYPE_RINGBUF => {
200                // Only the first page of a ring buffer can be mapped as writable.
201                if length > page_size && writable {
202                    return error!(EPERM);
203                }
204                if length > 2 * page_size + 2 * self.schema.max_entries as usize {
205                    return error!(EINVAL);
206                }
207            }
208
209            bpf_map_type_BPF_MAP_TYPE_ARRAY => {
210                if !self.schema.flags.contains(MapFlags::Mmapable) {
211                    return error!(EPERM);
212                }
213
214                let array_size = round_up_to_increment(
215                    compute_map_storage_size(&self.schema).map_err(|_| errno!(EINVAL))?,
216                    page_size,
217                )?;
218                if length > array_size {
219                    return error!(EINVAL);
220                }
221
222                if self.schema.flags.contains(MapFlags::ProgReadOnly) && writable {
223                    return error!(EACCES);
224                }
225            }
226
227            _ => return error!(ENODEV),
228        };
229
230        let memory_object =
231            if writable { &mut state.memory_object } else { &mut state.readonly_memory_object };
232        if let Some(memory) = memory_object.as_ref() {
233            return Ok(memory.clone());
234        }
235
236        let new_memory_object = match self.schema.map_type {
237            bpf_map_type_BPF_MAP_TYPE_RINGBUF => {
238                if writable {
239                    MemoryObject::from(
240                        self.vmo()
241                            .create_child(
242                                zx::VmoChildOptions::SLICE,
243                                page_size as u64,
244                                page_size as u64,
245                            )
246                            .map_err(|_| errno!(EIO))?,
247                    )
248                } else {
249                    let clone_size = 2 * page_size + self.schema.max_entries as usize;
250                    MemoryObject::RingBuf(
251                        self.vmo()
252                            .create_child(
253                                zx::VmoChildOptions::SLICE | zx::VmoChildOptions::NO_WRITE,
254                                page_size as u64,
255                                clone_size as u64,
256                            )
257                            .map_err(|_| errno!(EIO))?
258                            .into(),
259                    )
260                }
261            }
262
263            bpf_map_type_BPF_MAP_TYPE_ARRAY => {
264                let rights = if writable {
265                    zx::Rights::SAME_RIGHTS
266                } else {
267                    self.vmo().basic_info().map_err(|_| errno!(EIO))?.rights - zx::Rights::WRITE
268                };
269                MemoryObject::from(self.vmo().duplicate_handle(rights).map_err(|_| errno!(EIO))?)
270            }
271            _ => return error!(ENODEV),
272        };
273
274        let memory = Arc::new(new_memory_object);
275        *memory_object = Some(memory.clone());
276        Ok(memory)
277    }
278}
279
280impl Releasable for BpfMap {
281    type Context<'a> = &'a CurrentTask;
282
283    fn release<'a>(self, _current_task: &'a CurrentTask) {
284        if let Some(kernel) = self.kernel.upgrade() {
285            kernel.ebpf_state.unregister_map(self.id);
286        }
287    }
288}
289
290pub enum BpfMapReleaserAction {}
291impl ReleaserAction<BpfMap> for BpfMapReleaserAction {
292    fn release(map: ReleaseGuard<BpfMap>) {
293        register_delayed_release(map);
294    }
295}
296pub type BpfMapReleaser = ObjectReleaser<BpfMap, BpfMapReleaserAction>;
297pub type BpfMapHandle = Arc<BpfMapReleaser>;
298pub type WeakBpfMapHandle = Weak<BpfMapReleaser>;
starnix_core/bpf/map.rs

starnix_core/bpf/
map.rs
