starnix_core/bpf/
program.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::bpf::fs::get_bpf_object;
6use crate::bpf::{BpfMapHandle, check_cap_or_sys_admin};
7use crate::security;
8use crate::task::{CurrentTask, Kernel, register_delayed_release};
9use crate::vfs::{FdNumber, OutputBuffer};
10use ebpf::{
11    BPF_LDDW, BPF_PSEUDO_BTF_ID, BPF_PSEUDO_FUNC, BPF_PSEUDO_MAP_FD, BPF_PSEUDO_MAP_IDX,
12    BPF_PSEUDO_MAP_IDX_VALUE, BPF_PSEUDO_MAP_VALUE, EbpfError, EbpfInstruction, EbpfProgram,
13    EbpfProgramContext, StaticHelperSet, VerifiedEbpfProgram, VerifierError, VerifierLogger,
14    link_program, verify_program,
15};
16use ebpf_api::{AttachType, EbpfApiError, MapsContext, PinnedMap, ProgramType, StructId};
17use fidl_fuchsia_ebpf as febpf;
18use starnix_lifecycle::{AtomicCounter, ObjectReleaser, ReleaserAction};
19use starnix_logging::{log_warn, track_stub};
20use starnix_types::ownership::{Releasable, ReleaseGuard};
21use starnix_uapi::auth::{CAP_BPF, CAP_NET_ADMIN, CAP_PERFMON};
22use starnix_uapi::errors::{Errno, ErrnoCode};
23use starnix_uapi::{bpf_attr__bindgen_ty_4, errno, error};
24use std::sync::{Arc, Weak};
25
26#[derive(Clone, Debug)]
27pub struct ProgramInfo {
28    pub program_type: ProgramType,
29    pub expected_attach_type: AttachType,
30}
31
32impl TryFrom<&bpf_attr__bindgen_ty_4> for ProgramInfo {
33    type Error = Errno;
34
35    fn try_from(info: &bpf_attr__bindgen_ty_4) -> Result<Self, Self::Error> {
36        Ok(Self {
37            program_type: info.prog_type.try_into().map_err(map_ebpf_api_error)?,
38            expected_attach_type: info.expected_attach_type.into(),
39        })
40    }
41}
42pub type ProgramId = u32;
43
44static NEXT_PROGRAM_ID: AtomicCounter<u32> = AtomicCounter::<u32>::new_const(1);
45fn new_program_id() -> ProgramId {
46    NEXT_PROGRAM_ID.next()
47}
48
49#[derive(Debug)]
50pub struct Program {
51    /// Program info specified during program initialization.
52    pub info: ProgramInfo,
53
54    /// Verified program.
55    program: VerifiedEbpfProgram,
56
57    /// eBPF maps used by the program. These should match the `maps` field in
58    /// the `program`.
59    maps: Vec<BpfMapHandle>,
60
61    /// Integer program ID. This is dinstinct from `fidl_id` because `fidl_id`
62    /// is 64-bit, while Linux uses 32-bit IDs.
63    id: ProgramId,
64
65    /// Handle used when the program is transferred over FIDL to other services.
66    fidl_handle: febpf::ProgramHandle,
67
68    /// ID of the program used in FIDL
69    fidl_id: febpf::ProgramId,
70
71    /// The service end of the `fidl_handle`. Should be moved to the BPF Service
72    /// once it's implemented.
73    #[allow(dead_code)]
74    service_handle: zx::EventPair,
75
76    /// Weak reference to the Kernel where this program is registered.
77    kernel: Weak<Kernel>,
78
79    /// The security state associated with this bpf Program.
80    pub security_state: security::BpfProgState,
81}
82
83fn map_ebpf_api_error(e: EbpfApiError) -> Errno {
84    match e {
85        EbpfApiError::InvalidProgramType(_) | EbpfApiError::InvalidExpectedAttachType(_) => {
86            errno!(EINVAL, e)
87        }
88        EbpfApiError::UnsupportedProgramType(_) => errno!(ENOTSUP, e),
89    }
90}
91
92#[track_caller]
93fn verifier_error_to_errno(e: VerifierError) -> Errno {
94    Errno::with_context(ErrnoCode::from_error_code(e.errno() as i16), e)
95}
96
97fn map_ebpf_error(e: EbpfError) -> Errno {
98    match e {
99        EbpfError::ProgramVerifyError(v) => verifier_error_to_errno(v),
100        _ => errno!(EINVAL, e),
101    }
102}
103
104impl Program {
105    pub fn new(
106        current_task: &CurrentTask,
107        info: ProgramInfo,
108        logger: &mut dyn OutputBuffer,
109        mut code: Vec<EbpfInstruction>,
110    ) -> Result<ProgramHandle, Errno> {
111        let maps = link_maps_fds(current_task, &mut code)?;
112        let maps_schema = maps.iter().map(|m| m.schema).collect();
113        let mut logger = BufferVeriferLogger::new(logger);
114        let calling_context = info
115            .program_type
116            .create_calling_context(info.expected_attach_type, maps_schema)
117            .map_err(map_ebpf_api_error)?;
118        let program =
119            verify_program(code, calling_context, &mut logger).map_err(verifier_error_to_errno)?;
120
121        let (fidl_handle, service_handle) = zx::EventPair::create();
122        let fidl_id =
123            febpf::ProgramId { id: fidl_handle.koid().expect("Failed to get koid").raw_koid() };
124        let fidl_handle = febpf::ProgramHandle { handle: fidl_handle };
125
126        let program = ProgramHandle::new(
127            Self {
128                info,
129                program,
130                maps,
131                id: new_program_id(),
132                fidl_handle,
133                fidl_id,
134                service_handle,
135                kernel: Arc::downgrade(current_task.kernel()),
136                security_state: security::bpf_prog_alloc(current_task),
137            }
138            .into(),
139        );
140        current_task.kernel().ebpf_state.register_program(&program);
141
142        Ok(program)
143    }
144
145    pub fn id(&self) -> ProgramId {
146        self.id
147    }
148
149    pub fn link<C: EbpfProgramContext<Map = PinnedMap> + StaticHelperSet>(
150        &self,
151        program_type: ProgramType,
152    ) -> Result<EbpfProgram<C>, Errno>
153    where
154        for<'a> C::RunContext<'a>: MapsContext<'a>,
155    {
156        if program_type != self.info.program_type {
157            return error!(EINVAL);
158        }
159
160        let maps = self.maps.iter().map(|map| map.get_inner()).collect();
161        let program = link_program(&self.program, maps).map_err(map_ebpf_error)?;
162
163        Ok(program)
164    }
165
166    pub(super) fn check_load_access(
167        current_task: &CurrentTask,
168        info: &ProgramInfo,
169    ) -> Result<(), Errno> {
170        if matches!(info.program_type, ProgramType::CgroupSkb | ProgramType::SocketFilter)
171            && current_task.kernel().allow_unprivileged_bpf()
172        {
173            return Ok(());
174        }
175
176        check_cap_or_sys_admin(current_task, CAP_BPF)?;
177
178        match info.program_type {
179            // Loading tracing program types additionally require the CAP_PERFMON capability.
180            ProgramType::Kprobe
181            | ProgramType::Tracepoint
182            | ProgramType::PerfEvent
183            | ProgramType::RawTracepoint
184            | ProgramType::RawTracepointWritable
185            | ProgramType::Tracing => check_cap_or_sys_admin(current_task, CAP_PERFMON),
186
187            // Loading networking program types additionally require the CAP_NET_ADMIN capability.
188            ProgramType::SchedCls
189            | ProgramType::SchedAct
190            | ProgramType::Xdp
191            | ProgramType::SockOps
192            | ProgramType::SkSkb
193            | ProgramType::SkMsg
194            | ProgramType::SkLookup
195            | ProgramType::SkReuseport
196            | ProgramType::FlowDissector
197            | ProgramType::Netfilter => check_cap_or_sys_admin(current_task, CAP_NET_ADMIN),
198
199            // No additional checks are necessary for other program types.
200            ProgramType::CgroupDevice
201            | ProgramType::CgroupSkb
202            | ProgramType::CgroupSock
203            | ProgramType::CgroupSockAddr
204            | ProgramType::CgroupSockopt
205            | ProgramType::CgroupSysctl
206            | ProgramType::Ext
207            | ProgramType::LircMode2
208            | ProgramType::Lsm
209            | ProgramType::LwtIn
210            | ProgramType::LwtOut
211            | ProgramType::LwtSeg6Local
212            | ProgramType::LwtXmit
213            | ProgramType::SocketFilter
214            | ProgramType::StructOps
215            | ProgramType::Syscall
216            | ProgramType::Unspec => Ok(()),
217        }
218    }
219
220    pub fn fidl_id(&self) -> febpf::ProgramId {
221        self.fidl_id
222    }
223
224    pub fn fidl_handle(&self) -> febpf::ProgramHandle {
225        let handle = self
226            .fidl_handle
227            .handle
228            .duplicate_handle(zx::Rights::TRANSFER | zx::Rights::SIGNAL | zx::Rights::WAIT)
229            .expect("Failed to duplicate handle");
230        febpf::ProgramHandle { handle }
231    }
232}
233
234impl Releasable for Program {
235    type Context<'a> = &'a CurrentTask;
236
237    fn release<'a>(self, _current_task: &'a CurrentTask) {
238        if let Some(kernel) = self.kernel.upgrade() {
239            kernel.ebpf_state.unregister_program(self.id);
240        }
241
242        // Signal the FIDL handle to indicate that the program handle is defunct
243        // and should be closed.
244        self.fidl_handle
245            .handle
246            .signal(
247                zx::Signals::NONE,
248                zx::Signals::from_bits_truncate(febpf::PROGRAM_DEFUNCT_SIGNAL),
249            )
250            .unwrap();
251    }
252}
253
254pub enum ProgramReleaserAction {}
255impl ReleaserAction<Program> for ProgramReleaserAction {
256    fn release(program: ReleaseGuard<Program>) {
257        register_delayed_release(program);
258    }
259}
260pub type ProgramReleaser = ObjectReleaser<Program, ProgramReleaserAction>;
261pub type ProgramHandle = Arc<ProgramReleaser>;
262pub type WeakProgramHandle = Weak<ProgramReleaser>;
263
264impl TryFrom<&Program> for febpf::VerifiedProgram {
265    type Error = Errno;
266
267    fn try_from(program: &Program) -> Result<febpf::VerifiedProgram, Errno> {
268        let mut maps = Vec::with_capacity(program.maps.len());
269        for map in program.maps.iter() {
270            maps.push(map.share().map_err(|_| errno!(EIO))?);
271        }
272
273        let code_u64: &[u64] = zerocopy::transmute_ref!(program.program.code());
274
275        let mut struct_access_instructions =
276            Vec::with_capacity(program.program.struct_access_instructions().len());
277        for v in program.program.struct_access_instructions() {
278            let struct_id = StructId::try_from(&v.memory_id).map_err(|()| errno!(EINVAL))?.into();
279            struct_access_instructions.push(febpf::StructAccess {
280                pc: v.pc.try_into().unwrap(),
281                struct_id,
282                field_offset: v.field_offset.try_into().unwrap(),
283                is_32_bit_ptr_load: v.is_32_bit_ptr_load,
284            })
285        }
286        Ok(febpf::VerifiedProgram {
287            code: Some(code_u64.to_vec()),
288            struct_access_instructions: Some(struct_access_instructions),
289            maps: Some(maps),
290            ..Default::default()
291        })
292    }
293}
294
295/// Links maps referenced by FD, replacing them with by-index references.
296fn link_maps_fds(
297    current_task: &CurrentTask,
298    code: &mut Vec<EbpfInstruction>,
299) -> Result<Vec<BpfMapHandle>, Errno> {
300    let code_len = code.len();
301    let mut maps = Vec::<BpfMapHandle>::new();
302    for (pc, instruction) in code.iter_mut().enumerate() {
303        if instruction.code() == BPF_LDDW {
304            // BPF_LDDW requires 2 instructions.
305            if pc >= code_len - 1 {
306                return error!(EINVAL);
307            }
308
309            match instruction.src_reg() {
310                0 => {}
311                BPF_PSEUDO_MAP_FD | BPF_PSEUDO_MAP_VALUE => {
312                    let lddw_type = if instruction.src_reg() == BPF_PSEUDO_MAP_FD {
313                        BPF_PSEUDO_MAP_IDX
314                    } else {
315                        BPF_PSEUDO_MAP_IDX_VALUE
316                    };
317                    // If the instruction references a map fd, then we need to look up the map fd
318                    // and create a reference from this program to that object.
319                    instruction.set_src_reg(lddw_type);
320
321                    let fd = FdNumber::from_raw(instruction.imm());
322                    let object = get_bpf_object(current_task, fd)?;
323                    let map: &BpfMapHandle = object.as_map()?;
324
325                    // Find the map in `maps` or insert it otherwise.
326                    let maybe_index = maps.iter().position(|v| Arc::ptr_eq(v, map));
327                    let index = match maybe_index {
328                        Some(index) => index,
329                        None => {
330                            let index = maps.len();
331                            maps.push(map.clone());
332                            index
333                        }
334                    };
335
336                    instruction.set_imm(index.try_into().unwrap());
337                }
338                BPF_PSEUDO_MAP_IDX
339                | BPF_PSEUDO_MAP_IDX_VALUE
340                | BPF_PSEUDO_BTF_ID
341                | BPF_PSEUDO_FUNC => {
342                    track_stub!(
343                        TODO("https://fxbug.dev/378564467"),
344                        "unsupported pseudo src for ldimm64",
345                        instruction.src_reg()
346                    );
347                    return error!(ENOTSUP);
348                }
349                _ => {
350                    return error!(EINVAL);
351                }
352            }
353        }
354    }
355    Ok(maps)
356}
357
358struct BufferVeriferLogger<'a> {
359    buffer: &'a mut dyn OutputBuffer,
360    full: bool,
361}
362
363impl BufferVeriferLogger<'_> {
364    fn new<'a>(buffer: &'a mut dyn OutputBuffer) -> BufferVeriferLogger<'a> {
365        let full = buffer.available() == 0;
366        BufferVeriferLogger { buffer, full }
367    }
368}
369
370impl VerifierLogger for BufferVeriferLogger<'_> {
371    fn is_enabled(&self) -> bool {
372        !self.full
373    }
374
375    fn log(&mut self, line: &[u8]) {
376        debug_assert!(line.is_ascii());
377
378        if self.full {
379            return;
380        }
381        if line.len() + 1 > self.buffer.available() {
382            self.full = true;
383            return;
384        }
385        match self.buffer.write(line) {
386            Err(e) => {
387                log_warn!("Unable to write verifier log: {e:?}");
388                self.full = true;
389            }
390            _ => {}
391        }
392        match self.buffer.write(b"\n") {
393            Err(e) => {
394                log_warn!("Unable to write verifier log: {e:?}");
395                self.full = true;
396            }
397            _ => {}
398        }
399        if self.buffer.available() == 0 {
400            self.full = true;
401        }
402    }
403}
starnix_core/bpf/program.rs

starnix_core/bpf/
program.rs
