starnix_core/bpf/
program.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::bpf::BpfMapHandle;
6use crate::bpf::fs::get_bpf_object;
7use crate::security;
8use crate::task::{CurrentTask, CurrentTaskAndLocked, Kernel, register_delayed_release};
9use crate::vfs::{FdNumber, OutputBuffer};
10use ebpf::{
11    BPF_LDDW, BPF_PSEUDO_BTF_ID, BPF_PSEUDO_FUNC, BPF_PSEUDO_MAP_FD, BPF_PSEUDO_MAP_IDX,
12    BPF_PSEUDO_MAP_IDX_VALUE, BPF_PSEUDO_MAP_VALUE, EbpfError, EbpfInstruction, EbpfProgram,
13    EbpfProgramContext, StaticHelperSet, VerifiedEbpfProgram, VerifierLogger, link_program,
14    verify_program,
15};
16use ebpf_api::{AttachType, EbpfApiError, MapsContext, PinnedMap, ProgramType};
17use fidl_fuchsia_ebpf as febpf;
18use starnix_lifecycle::{AtomicU32Counter, ObjectReleaser, ReleaserAction};
19use starnix_logging::{log_error, log_warn, track_stub};
20use starnix_sync::{EbpfStateLock, LockBefore, Locked};
21use starnix_types::ownership::{Releasable, ReleaseGuard};
22use starnix_uapi::auth::{CAP_BPF, CAP_NET_ADMIN, CAP_PERFMON, CAP_SYS_ADMIN};
23use starnix_uapi::errors::Errno;
24use starnix_uapi::{bpf_attr__bindgen_ty_4, errno, error};
25use std::sync::atomic::Ordering;
26use std::sync::{Arc, Weak};
27use zx::{AsHandleRef as _, HandleBased};
28
29#[derive(Clone, Debug)]
30pub struct ProgramInfo {
31    pub program_type: ProgramType,
32    pub expected_attach_type: AttachType,
33}
34
35impl TryFrom<&bpf_attr__bindgen_ty_4> for ProgramInfo {
36    type Error = Errno;
37
38    fn try_from(info: &bpf_attr__bindgen_ty_4) -> Result<Self, Self::Error> {
39        Ok(Self {
40            program_type: info.prog_type.try_into().map_err(map_ebpf_api_error)?,
41            expected_attach_type: info.expected_attach_type.into(),
42        })
43    }
44}
45pub type ProgramId = u32;
46
47static NEXT_PROGRAM_ID: AtomicU32Counter = AtomicU32Counter::new(1);
48fn new_program_id() -> ProgramId {
49    NEXT_PROGRAM_ID.next()
50}
51
52#[derive(Debug)]
53pub struct Program {
54    /// Program info specified during program initialization.
55    pub info: ProgramInfo,
56
57    /// Verified program.
58    program: VerifiedEbpfProgram,
59
60    /// eBPF maps used by the program. These should match the `maps` field in
61    /// the `program`.
62    maps: Vec<BpfMapHandle>,
63
64    /// Integer program ID. This is dinstinct from `fidl_id` because `fidl_id`
65    /// is 64-bit, while Linux uses 32-bit IDs.
66    id: ProgramId,
67
68    /// Handle used when the program is transferred over FIDL to other services.
69    fidl_handle: febpf::ProgramHandle,
70
71    /// ID of the program used in FIDL
72    fidl_id: febpf::ProgramId,
73
74    /// The service end of the `fidl_handle`. Should be moved to the BPF Service
75    /// once it's implemented.
76    #[allow(dead_code)]
77    service_handle: zx::EventPair,
78
79    /// Weak reference to the Kernel where this program is registered.
80    kernel: Weak<Kernel>,
81
82    /// The security state associated with this bpf Program.
83    pub security_state: security::BpfProgState,
84}
85
86fn map_ebpf_error(e: EbpfError) -> Errno {
87    log_error!("Failed to load eBPF program: {e:?}");
88    errno!(EINVAL)
89}
90
91fn map_ebpf_api_error(e: EbpfApiError) -> Errno {
92    log_error!("Failed to load eBPF program: {e:?}");
93    match e {
94        EbpfApiError::InvalidProgramType(_) | EbpfApiError::InvalidExpectedAttachType(_) => {
95            errno!(EINVAL)
96        }
97        EbpfApiError::UnsupportedProgramType(_) => errno!(ENOTSUP),
98    }
99}
100
101impl Program {
102    pub fn new<L>(
103        locked: &mut Locked<L>,
104        current_task: &CurrentTask,
105        info: ProgramInfo,
106        logger: &mut dyn OutputBuffer,
107        mut code: Vec<EbpfInstruction>,
108    ) -> Result<ProgramHandle, Errno>
109    where
110        L: LockBefore<EbpfStateLock>,
111    {
112        Self::check_load_access(current_task, &info)?;
113        let maps = link_maps_fds(current_task, &mut code)?;
114        let maps_schema = maps.iter().map(|m| m.schema).collect();
115        let mut logger = BufferVeriferLogger::new(logger);
116        let calling_context = info
117            .program_type
118            .create_calling_context(info.expected_attach_type, maps_schema)
119            .map_err(map_ebpf_api_error)?;
120        let program = verify_program(code, calling_context, &mut logger).map_err(map_ebpf_error)?;
121
122        let (fidl_handle, service_handle) = zx::EventPair::create();
123        let fidl_id =
124            febpf::ProgramId { id: fidl_handle.get_koid().expect("Failed to get koid").raw_koid() };
125        let fidl_handle = febpf::ProgramHandle { handle: fidl_handle };
126
127        let program = ProgramHandle::new(
128            Self {
129                info,
130                program,
131                maps,
132                id: new_program_id(),
133                fidl_handle,
134                fidl_id,
135                service_handle,
136                kernel: Arc::downgrade(current_task.kernel()),
137                security_state: security::bpf_prog_alloc(current_task),
138            }
139            .into(),
140        );
141        current_task.kernel().ebpf_state.register_program(locked, &program);
142
143        Ok(program)
144    }
145
146    pub fn id(&self) -> ProgramId {
147        self.id
148    }
149
150    pub fn link<C: EbpfProgramContext<Map = PinnedMap> + StaticHelperSet>(
151        &self,
152        program_type: ProgramType,
153    ) -> Result<EbpfProgram<C>, Errno>
154    where
155        for<'a> C::RunContext<'a>: MapsContext<'a>,
156    {
157        if program_type != self.info.program_type {
158            return error!(EINVAL);
159        }
160
161        let maps = self.maps.iter().map(|map| map.map.clone()).collect();
162        let program = link_program(&self.program, maps).map_err(map_ebpf_error)?;
163
164        Ok(program)
165    }
166
167    fn check_load_access(current_task: &CurrentTask, info: &ProgramInfo) -> Result<(), Errno> {
168        if matches!(info.program_type, ProgramType::CgroupSkb | ProgramType::SocketFilter)
169            && current_task.kernel().disable_unprivileged_bpf.load(Ordering::Relaxed) == 0
170        {
171            return Ok(());
172        }
173        if security::is_task_capable_noaudit(current_task, CAP_SYS_ADMIN) {
174            return Ok(());
175        }
176        security::check_task_capable(current_task, CAP_BPF)?;
177        match info.program_type {
178            // Loading tracing program types additionally require the CAP_PERFMON capability.
179            ProgramType::Kprobe
180            | ProgramType::Tracepoint
181            | ProgramType::PerfEvent
182            | ProgramType::RawTracepoint
183            | ProgramType::RawTracepointWritable
184            | ProgramType::Tracing => security::check_task_capable(current_task, CAP_PERFMON),
185
186            // Loading networking program types additionally require the CAP_NET_ADMIN capability.
187            ProgramType::SocketFilter
188            | ProgramType::SchedCls
189            | ProgramType::SchedAct
190            | ProgramType::Xdp
191            | ProgramType::SockOps
192            | ProgramType::SkSkb
193            | ProgramType::SkMsg
194            | ProgramType::SkLookup
195            | ProgramType::SkReuseport
196            | ProgramType::FlowDissector
197            | ProgramType::Netfilter => security::check_task_capable(current_task, CAP_NET_ADMIN),
198
199            // No additional checks are necessary for other program types.
200            ProgramType::CgroupDevice
201            | ProgramType::CgroupSkb
202            | ProgramType::CgroupSock
203            | ProgramType::CgroupSockAddr
204            | ProgramType::CgroupSockopt
205            | ProgramType::CgroupSysctl
206            | ProgramType::Ext
207            | ProgramType::LircMode2
208            | ProgramType::Lsm
209            | ProgramType::LwtIn
210            | ProgramType::LwtOut
211            | ProgramType::LwtSeg6Local
212            | ProgramType::LwtXmit
213            | ProgramType::StructOps
214            | ProgramType::Syscall
215            | ProgramType::Unspec
216            | ProgramType::Fuse => Ok(()),
217        }
218    }
219
220    pub fn fidl_id(&self) -> febpf::ProgramId {
221        self.fidl_id
222    }
223
224    pub fn fidl_handle(&self) -> febpf::ProgramHandle {
225        let handle = self
226            .fidl_handle
227            .handle
228            .duplicate_handle(zx::Rights::TRANSFER | zx::Rights::SIGNAL | zx::Rights::WAIT)
229            .expect("Failed to duplicate handle");
230        febpf::ProgramHandle { handle }
231    }
232}
233
234impl Releasable for Program {
235    type Context<'a> = CurrentTaskAndLocked<'a>;
236
237    fn release<'a>(self, (locked, _current_task): CurrentTaskAndLocked<'a>) {
238        if let Some(kernel) = self.kernel.upgrade() {
239            kernel.ebpf_state.unregister_program(locked, self.id);
240        }
241
242        // Signal the FIDL handle to indicate that the program handle is defunct
243        // and should be closed.
244        self.fidl_handle
245            .handle
246            .signal_handle(
247                zx::Signals::NONE,
248                zx::Signals::from_bits_truncate(febpf::PROGRAM_DEFUNCT_SIGNAL),
249            )
250            .unwrap();
251    }
252}
253
254pub enum ProgramReleaserAction {}
255impl ReleaserAction<Program> for ProgramReleaserAction {
256    fn release(program: ReleaseGuard<Program>) {
257        register_delayed_release(program);
258    }
259}
260pub type ProgramReleaser = ObjectReleaser<Program, ProgramReleaserAction>;
261pub type ProgramHandle = Arc<ProgramReleaser>;
262pub type WeakProgramHandle = Weak<ProgramReleaser>;
263
264impl TryFrom<&Program> for febpf::VerifiedProgram {
265    type Error = Errno;
266
267    fn try_from(program: &Program) -> Result<febpf::VerifiedProgram, Errno> {
268        let mut maps = Vec::with_capacity(program.maps.len());
269        for map in program.maps.iter() {
270            maps.push(map.share().map_err(|_| errno!(EIO))?);
271        }
272
273        // SAFETY: EbpfInstruction is 64-bit, so it's safe to transmute it to u64.
274        let code = program.program.code();
275        #[allow(
276            clippy::undocumented_unsafe_blocks,
277            reason = "Force documented unsafe blocks in Starnix"
278        )]
279        let code_u64 =
280            unsafe { std::slice::from_raw_parts(code.as_ptr() as *const u64, code.len()) };
281
282        let struct_access_instructions = program
283            .program
284            .struct_access_instructions()
285            .iter()
286            .map(|v| febpf::StructAccess {
287                pc: v.pc.try_into().unwrap(),
288                struct_memory_id: v.memory_id.id(),
289                field_offset: v.field_offset.try_into().unwrap(),
290                is_32_bit_ptr_load: v.is_32_bit_ptr_load,
291            })
292            .collect();
293
294        Ok(febpf::VerifiedProgram {
295            code: Some(code_u64.to_vec()),
296            struct_access_instructions: Some(struct_access_instructions),
297            maps: Some(maps),
298            ..Default::default()
299        })
300    }
301}
302
303/// Links maps referenced by FD, replacing them with by-index references.
304fn link_maps_fds(
305    current_task: &CurrentTask,
306    code: &mut Vec<EbpfInstruction>,
307) -> Result<Vec<BpfMapHandle>, Errno> {
308    let code_len = code.len();
309    let mut maps = Vec::<BpfMapHandle>::new();
310    for (pc, instruction) in code.iter_mut().enumerate() {
311        if instruction.code() == BPF_LDDW {
312            // BPF_LDDW requires 2 instructions.
313            if pc >= code_len - 1 {
314                return error!(EINVAL);
315            }
316
317            match instruction.src_reg() {
318                0 => {}
319                BPF_PSEUDO_MAP_FD | BPF_PSEUDO_MAP_VALUE => {
320                    let lddw_type = if instruction.src_reg() == BPF_PSEUDO_MAP_FD {
321                        BPF_PSEUDO_MAP_IDX
322                    } else {
323                        BPF_PSEUDO_MAP_IDX_VALUE
324                    };
325                    // If the instruction references a map fd, then we need to look up the map fd
326                    // and create a reference from this program to that object.
327                    instruction.set_src_reg(lddw_type);
328
329                    let fd = FdNumber::from_raw(instruction.imm());
330                    let object = get_bpf_object(current_task, fd)?;
331                    let map: &BpfMapHandle = object.as_map()?;
332
333                    // Find the map in `maps` or insert it otherwise.
334                    let maybe_index = maps.iter().position(|v| Arc::ptr_eq(v, map));
335                    let index = match maybe_index {
336                        Some(index) => index,
337                        None => {
338                            let index = maps.len();
339                            maps.push(map.clone());
340                            index
341                        }
342                    };
343
344                    instruction.set_imm(index.try_into().unwrap());
345                }
346                BPF_PSEUDO_MAP_IDX
347                | BPF_PSEUDO_MAP_IDX_VALUE
348                | BPF_PSEUDO_BTF_ID
349                | BPF_PSEUDO_FUNC => {
350                    track_stub!(
351                        TODO("https://fxbug.dev/378564467"),
352                        "unsupported pseudo src for ldimm64",
353                        instruction.src_reg()
354                    );
355                    return error!(ENOTSUP);
356                }
357                _ => {
358                    return error!(EINVAL);
359                }
360            }
361        }
362    }
363    Ok(maps)
364}
365
366struct BufferVeriferLogger<'a> {
367    buffer: &'a mut dyn OutputBuffer,
368    full: bool,
369}
370
371impl BufferVeriferLogger<'_> {
372    fn new<'a>(buffer: &'a mut dyn OutputBuffer) -> BufferVeriferLogger<'a> {
373        BufferVeriferLogger { buffer, full: false }
374    }
375}
376
377impl VerifierLogger for BufferVeriferLogger<'_> {
378    fn log(&mut self, line: &[u8]) {
379        debug_assert!(line.is_ascii());
380
381        if self.full {
382            return;
383        }
384        if line.len() + 1 > self.buffer.available() {
385            self.full = true;
386            return;
387        }
388        match self.buffer.write(line) {
389            Err(e) => {
390                log_warn!("Unable to write verifier log: {e:?}");
391                self.full = true;
392            }
393            _ => {}
394        }
395        match self.buffer.write(b"\n") {
396            Err(e) => {
397                log_warn!("Unable to write verifier log: {e:?}");
398                self.full = true;
399            }
400            _ => {}
401        }
402    }
403}
starnix_core/bpf/program.rs

starnix_core/bpf/
program.rs
