starnix_core/bpf/
program.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::bpf::fs::get_bpf_object;
6use crate::bpf::{BpfMapHandle, check_cap_or_sys_admin};
7use crate::security;
8use crate::task::{CurrentTask, Kernel, register_delayed_release};
9use crate::vfs::{FdNumber, OutputBuffer};
10use ebpf::{
11    BPF_LDDW, BPF_PSEUDO_BTF_ID, BPF_PSEUDO_FUNC, BPF_PSEUDO_MAP_FD, BPF_PSEUDO_MAP_IDX,
12    BPF_PSEUDO_MAP_IDX_VALUE, BPF_PSEUDO_MAP_VALUE, CallingContext, EbpfError, EbpfInstruction,
13    EbpfProgram, EbpfProgramContext, NullVerifierLogger, StaticHelperSet, VerifiedEbpfProgram,
14    VerifierError, VerifierLogger, link_program, verify_program,
15};
16use ebpf_api::{AttachType, EbpfApiError, MapsContext, PinnedMap, ProgramType, StructId};
17use fidl_fuchsia_ebpf as febpf;
18use starnix_lifecycle::{AtomicCounter, ObjectReleaser, ReleaserAction};
19use starnix_logging::track_stub;
20use starnix_types::ownership::{Releasable, ReleaseGuard};
21use starnix_uapi::auth::{CAP_BPF, CAP_NET_ADMIN, CAP_PERFMON};
22use starnix_uapi::errors::{Errno, ErrnoCode};
23use starnix_uapi::{bpf_attr__bindgen_ty_4, errno, error};
24use std::sync::{Arc, Weak};
25
26bitflags::bitflags! {
27    /// Verifier log level flags (`BPF_LOG_*` in Linux `include/linux/bpf_verifier.h`).
28    #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
29    pub struct VerifierLogLevel: u32 {
30        /// `BPF_LOG_LEVEL1`: log instruction trace, but reset the log buffer on verification
31        /// success so only verification failures retain the trace.
32        const LEVEL1 = 1 << 0;
33        /// `BPF_LOG_LEVEL2`: verbose trace retained even when verification succeeds.
34        const LEVEL2 = 1 << 1;
35        /// `BPF_LOG_STATS`: log verifier statistics.
36        const STATS = 1 << 2;
37    }
38}
39
40#[derive(Clone, Debug)]
41pub struct ProgramInfo {
42    pub program_type: ProgramType,
43    pub expected_attach_type: AttachType,
44    pub log_level: VerifierLogLevel,
45}
46
47impl TryFrom<&bpf_attr__bindgen_ty_4> for ProgramInfo {
48    type Error = Errno;
49
50    fn try_from(info: &bpf_attr__bindgen_ty_4) -> Result<Self, Self::Error> {
51        Ok(Self {
52            program_type: info.prog_type.try_into().map_err(map_ebpf_api_error)?,
53            expected_attach_type: info.expected_attach_type.into(),
54            log_level: VerifierLogLevel::from_bits_retain(info.log_level),
55        })
56    }
57}
58pub type ProgramId = u32;
59
60static NEXT_PROGRAM_ID: AtomicCounter<u32> = AtomicCounter::<u32>::new_const(1);
61fn new_program_id() -> ProgramId {
62    NEXT_PROGRAM_ID.next()
63}
64
65#[derive(Debug)]
66pub struct Program {
67    /// Program info specified during program initialization.
68    pub info: ProgramInfo,
69
70    /// Verified program.
71    program: VerifiedEbpfProgram,
72
73    /// eBPF maps used by the program. These should match the `maps` field in
74    /// the `program`.
75    maps: Vec<BpfMapHandle>,
76
77    /// Integer program ID. This is dinstinct from `fidl_id` because `fidl_id`
78    /// is 64-bit, while Linux uses 32-bit IDs.
79    id: ProgramId,
80
81    /// Handle used when the program is transferred over FIDL to other services.
82    fidl_handle: febpf::ProgramHandle,
83
84    /// ID of the program used in FIDL
85    fidl_id: febpf::ProgramId,
86
87    /// The service end of the `fidl_handle`. Should be moved to the BPF Service
88    /// once it's implemented.
89    #[allow(dead_code)]
90    service_handle: zx::EventPair,
91
92    /// Weak reference to the Kernel where this program is registered.
93    kernel: Weak<Kernel>,
94
95    /// The security state associated with this bpf Program.
96    pub security_state: security::BpfProgState,
97}
98
99fn map_ebpf_api_error(e: EbpfApiError) -> Errno {
100    match e {
101        EbpfApiError::InvalidProgramType(_) | EbpfApiError::InvalidExpectedAttachType(_) => {
102            errno!(EINVAL, e)
103        }
104        EbpfApiError::UnsupportedProgramType(_) => errno!(ENOTSUP, e),
105    }
106}
107
108#[track_caller]
109fn verifier_error_to_errno(e: VerifierError) -> Errno {
110    Errno::with_context(ErrnoCode::from_error_code(e.errno() as i16), e)
111}
112
113fn map_ebpf_error(e: EbpfError) -> Errno {
114    match e {
115        EbpfError::ProgramVerifyError(v) => verifier_error_to_errno(v),
116        _ => errno!(EINVAL, e),
117    }
118}
119
120fn verify_with_logger(
121    code: Vec<EbpfInstruction>,
122    log_level: VerifierLogLevel,
123    logger: &mut dyn OutputBuffer,
124    make_calling_context: impl Fn() -> Result<CallingContext, Errno>,
125) -> Result<VerifiedEbpfProgram, Errno> {
126    if log_level.is_empty() || logger.available() == 0 {
127        return verify_program(code, make_calling_context()?, &mut NullVerifierLogger)
128            .map_err(verifier_error_to_errno);
129    }
130    if !log_level.contains(VerifierLogLevel::LEVEL2) {
131        if let Ok(program) =
132            verify_program(code.clone(), make_calling_context()?, &mut NullVerifierLogger)
133        {
134            logger.write_all(b"\0")?;
135            return Ok(program);
136        }
137    }
138    let mut buffered_logger = BufferVerifierLogger::new(logger);
139    let res = verify_program(code, make_calling_context()?, &mut buffered_logger);
140    buffered_logger.flush()?;
141    res.map_err(verifier_error_to_errno)
142}
143
144impl Program {
145    pub fn new(
146        current_task: &CurrentTask,
147        info: ProgramInfo,
148        logger: &mut dyn OutputBuffer,
149        mut code: Vec<EbpfInstruction>,
150    ) -> Result<ProgramHandle, Errno> {
151        let maps = link_maps_fds(current_task, &mut code)?;
152        let program = verify_with_logger(code, info.log_level, logger, || {
153            let maps_schema = maps.iter().map(|m| m.schema).collect();
154            info.program_type
155                .create_calling_context(info.expected_attach_type, maps_schema)
156                .map_err(map_ebpf_api_error)
157        })?;
158
159        let (fidl_handle, service_handle) = zx::EventPair::create();
160        let fidl_id =
161            febpf::ProgramId { id: fidl_handle.koid().expect("Failed to get koid").raw_koid() };
162        let fidl_handle = febpf::ProgramHandle { handle: fidl_handle };
163
164        let program = ProgramHandle::new(
165            Self {
166                info,
167                program,
168                maps,
169                id: new_program_id(),
170                fidl_handle,
171                fidl_id,
172                service_handle,
173                kernel: Arc::downgrade(current_task.kernel()),
174                security_state: security::bpf_prog_alloc(current_task),
175            }
176            .into(),
177        );
178        current_task.kernel().ebpf_state.register_program(&program);
179
180        Ok(program)
181    }
182
183    pub fn id(&self) -> ProgramId {
184        self.id
185    }
186
187    pub fn link<C: EbpfProgramContext<Map = PinnedMap> + StaticHelperSet>(
188        &self,
189        program_type: ProgramType,
190    ) -> Result<EbpfProgram<C>, Errno>
191    where
192        for<'a> C::RunContext<'a>: MapsContext<'a>,
193    {
194        if program_type != self.info.program_type {
195            return error!(EINVAL);
196        }
197
198        let maps = self.maps.iter().map(|map| map.get_inner()).collect();
199        let program = link_program(&self.program, maps).map_err(map_ebpf_error)?;
200
201        Ok(program)
202    }
203
204    pub(super) fn check_load_access(
205        current_task: &CurrentTask,
206        info: &ProgramInfo,
207    ) -> Result<(), Errno> {
208        if matches!(info.program_type, ProgramType::CgroupSkb | ProgramType::SocketFilter)
209            && current_task.kernel().allow_unprivileged_bpf()
210        {
211            return Ok(());
212        }
213
214        check_cap_or_sys_admin(current_task, CAP_BPF)?;
215
216        match info.program_type {
217            // Loading tracing program types additionally require the CAP_PERFMON capability.
218            ProgramType::Kprobe
219            | ProgramType::Tracepoint
220            | ProgramType::PerfEvent
221            | ProgramType::RawTracepoint
222            | ProgramType::RawTracepointWritable
223            | ProgramType::Tracing => check_cap_or_sys_admin(current_task, CAP_PERFMON),
224
225            // Loading networking program types additionally require the CAP_NET_ADMIN capability.
226            ProgramType::SchedCls
227            | ProgramType::SchedAct
228            | ProgramType::Xdp
229            | ProgramType::SockOps
230            | ProgramType::SkSkb
231            | ProgramType::SkMsg
232            | ProgramType::SkLookup
233            | ProgramType::SkReuseport
234            | ProgramType::FlowDissector
235            | ProgramType::Netfilter => check_cap_or_sys_admin(current_task, CAP_NET_ADMIN),
236
237            // No additional checks are necessary for other program types.
238            ProgramType::CgroupDevice
239            | ProgramType::CgroupSkb
240            | ProgramType::CgroupSock
241            | ProgramType::CgroupSockAddr
242            | ProgramType::CgroupSockopt
243            | ProgramType::CgroupSysctl
244            | ProgramType::Ext
245            | ProgramType::LircMode2
246            | ProgramType::Lsm
247            | ProgramType::LwtIn
248            | ProgramType::LwtOut
249            | ProgramType::LwtSeg6Local
250            | ProgramType::LwtXmit
251            | ProgramType::SocketFilter
252            | ProgramType::StructOps
253            | ProgramType::Syscall
254            | ProgramType::Unspec => Ok(()),
255        }
256    }
257
258    pub fn fidl_id(&self) -> febpf::ProgramId {
259        self.fidl_id
260    }
261
262    pub fn fidl_handle(&self) -> febpf::ProgramHandle {
263        let handle = self
264            .fidl_handle
265            .handle
266            .duplicate_handle(zx::Rights::TRANSFER | zx::Rights::SIGNAL | zx::Rights::WAIT)
267            .expect("Failed to duplicate handle");
268        febpf::ProgramHandle { handle }
269    }
270}
271
272impl Releasable for Program {
273    type Context<'a> = &'a CurrentTask;
274
275    fn release<'a>(self, _current_task: &'a CurrentTask) {
276        if let Some(kernel) = self.kernel.upgrade() {
277            kernel.ebpf_state.unregister_program(self.id);
278        }
279
280        // Signal the FIDL handle to indicate that the program handle is defunct
281        // and should be closed.
282        self.fidl_handle
283            .handle
284            .signal(
285                zx::Signals::NONE,
286                zx::Signals::from_bits_truncate(febpf::PROGRAM_DEFUNCT_SIGNAL),
287            )
288            .unwrap();
289    }
290}
291
292pub enum ProgramReleaserAction {}
293impl ReleaserAction<Program> for ProgramReleaserAction {
294    fn release(program: ReleaseGuard<Program>) {
295        register_delayed_release(program);
296    }
297}
298pub type ProgramReleaser = ObjectReleaser<Program, ProgramReleaserAction>;
299pub type ProgramHandle = Arc<ProgramReleaser>;
300pub type WeakProgramHandle = Weak<ProgramReleaser>;
301
302impl TryFrom<&Program> for febpf::VerifiedProgram {
303    type Error = Errno;
304
305    fn try_from(program: &Program) -> Result<febpf::VerifiedProgram, Errno> {
306        let mut maps = Vec::with_capacity(program.maps.len());
307        for map in program.maps.iter() {
308            maps.push(map.share().map_err(|_| errno!(EIO))?);
309        }
310
311        let code_u64: &[u64] = zerocopy::transmute_ref!(program.program.code());
312
313        let mut struct_access_instructions =
314            Vec::with_capacity(program.program.struct_access_instructions().len());
315        for v in program.program.struct_access_instructions() {
316            let struct_id = StructId::try_from(&v.memory_id).map_err(|()| errno!(EINVAL))?.into();
317            struct_access_instructions.push(febpf::StructAccess {
318                pc: v.pc.try_into().unwrap(),
319                struct_id,
320                field_offset: v.field_offset.try_into().unwrap(),
321                is_32_bit_ptr_load: v.is_32_bit_ptr_load,
322            })
323        }
324        Ok(febpf::VerifiedProgram {
325            code: Some(code_u64.to_vec()),
326            struct_access_instructions: Some(struct_access_instructions),
327            maps: Some(maps),
328            ..Default::default()
329        })
330    }
331}
332
333/// Links maps referenced by FD, replacing them with by-index references.
334fn link_maps_fds(
335    current_task: &CurrentTask,
336    code: &mut Vec<EbpfInstruction>,
337) -> Result<Vec<BpfMapHandle>, Errno> {
338    let code_len = code.len();
339    let mut maps = Vec::<BpfMapHandle>::new();
340    for (pc, instruction) in code.iter_mut().enumerate() {
341        if instruction.code() == BPF_LDDW {
342            // BPF_LDDW requires 2 instructions.
343            if pc >= code_len - 1 {
344                return error!(EINVAL);
345            }
346
347            match instruction.src_reg() {
348                0 => {}
349                BPF_PSEUDO_MAP_FD | BPF_PSEUDO_MAP_VALUE => {
350                    let lddw_type = if instruction.src_reg() == BPF_PSEUDO_MAP_FD {
351                        BPF_PSEUDO_MAP_IDX
352                    } else {
353                        BPF_PSEUDO_MAP_IDX_VALUE
354                    };
355                    // If the instruction references a map fd, then we need to look up the map fd
356                    // and create a reference from this program to that object.
357                    instruction.set_src_reg(lddw_type);
358
359                    let fd = FdNumber::from_raw(instruction.imm());
360                    let object = get_bpf_object(current_task, fd)?;
361                    let map: &BpfMapHandle = object.as_map()?;
362
363                    // Find the map in `maps` or insert it otherwise.
364                    let maybe_index = maps.iter().position(|v| Arc::ptr_eq(v, map));
365                    let index = match maybe_index {
366                        Some(index) => index,
367                        None => {
368                            let index = maps.len();
369                            maps.push(map.clone());
370                            index
371                        }
372                    };
373
374                    instruction.set_imm(index.try_into().unwrap());
375                }
376                BPF_PSEUDO_MAP_IDX
377                | BPF_PSEUDO_MAP_IDX_VALUE
378                | BPF_PSEUDO_BTF_ID
379                | BPF_PSEUDO_FUNC => {
380                    track_stub!(
381                        TODO("https://fxbug.dev/378564467"),
382                        "unsupported pseudo src for ldimm64",
383                        instruction.src_reg()
384                    );
385                    return error!(ENOTSUP);
386                }
387                _ => {
388                    return error!(EINVAL);
389                }
390            }
391        }
392    }
393    Ok(maps)
394}
395
396struct BufferVerifierLogger<'a> {
397    buffer: &'a mut dyn OutputBuffer,
398    data: Vec<u8>,
399    max_len: usize,
400    full: bool,
401}
402
403impl<'a> BufferVerifierLogger<'a> {
404    fn new(buffer: &'a mut dyn OutputBuffer) -> Self {
405        // Reserve 1 byte in the output buffer for the trailing NUL terminator.
406        let max_len = buffer.available().saturating_sub(1);
407        let full = max_len == 0;
408        Self { buffer, data: Vec::new(), max_len, full }
409    }
410
411    fn flush(mut self) -> Result<(), Errno> {
412        if self.buffer.available() > 0 {
413            self.data.push(0);
414            self.buffer.write_all(&self.data)?;
415        }
416        Ok(())
417    }
418}
419
420impl VerifierLogger for BufferVerifierLogger<'_> {
421    fn is_enabled(&self) -> bool {
422        !self.full
423    }
424
425    fn log(&mut self, line: &[u8]) {
426        debug_assert!(line.is_ascii());
427
428        if self.full {
429            return;
430        }
431        if self.data.len() + line.len() + 1 > self.max_len {
432            self.full = true;
433            return;
434        }
435        self.data.extend_from_slice(line);
436        self.data.push(b'\n');
437        if self.data.len() == self.max_len {
438            self.full = true;
439        }
440    }
441}
442
443#[cfg(test)]
444mod tests {
445    use super::*;
446    use crate::vfs::buffers::VecOutputBuffer;
447    use ebpf::{BPF_ADD, BPF_ALU64, BPF_EXIT, BPF_JMP, BPF_K, BPF_MOV};
448
449    #[::fuchsia::test]
450    fn test_buffer_verifier_logger_batches_and_nul_terminates() {
451        // Buffer has room for "abc\n" (4 bytes) + "\0" (1 byte) = 5 bytes, so second line is
452        // dropped while still NUL-terminating.
453        let mut out = VecOutputBuffer::new(6);
454        let mut logger = BufferVerifierLogger::new(&mut out);
455        assert!(logger.is_enabled());
456        logger.log(b"abc");
457        assert!(logger.is_enabled());
458        logger.log(b"def");
459        assert!(!logger.is_enabled());
460        logger.flush().expect("flush");
461        assert_eq!(out.data(), b"abc\n\0");
462    }
463
464    #[::fuchsia::test]
465    fn test_verify_with_logger_log_levels() {
466        let valid_code = vec![
467            EbpfInstruction::new(BPF_ALU64 | BPF_MOV | BPF_K, 0, 0, 0, 0),
468            EbpfInstruction::new(BPF_JMP | BPF_EXIT, 0, 0, 0, 0),
469        ];
470        let invalid_code = vec![
471            EbpfInstruction::new(BPF_ALU64 | BPF_ADD | BPF_K, 0, 0, 0, 1),
472            EbpfInstruction::new(BPF_JMP | BPF_EXIT, 0, 0, 0, 0),
473        ];
474        let make_context = || {
475            ProgramType::SocketFilter
476                .create_calling_context(AttachType::Unspecified, Vec::new())
477                .map_err(map_ebpf_api_error)
478        };
479
480        // log_level = empty: nothing written to log buffer even when provided.
481        let mut out = VecOutputBuffer::new(256);
482        let _prog = verify_with_logger(
483            valid_code.clone(),
484            VerifierLogLevel::empty(),
485            &mut out,
486            make_context,
487        )
488        .expect("verify valid program with log_level=0");
489        assert!(out.data().is_empty());
490
491        // log_level = LEVEL1 (BPF_LOG_LEVEL1) on success: resets log buffer to empty NUL-terminated
492        // string without formatting instruction trace.
493        let mut out = VecOutputBuffer::new(256);
494        let _prog = verify_with_logger(
495            valid_code.clone(),
496            VerifierLogLevel::LEVEL1,
497            &mut out,
498            make_context,
499        )
500        .expect("verify valid program with log_level=1");
501        assert_eq!(out.data(), b"\0");
502
503        // log_level = LEVEL1 (BPF_LOG_LEVEL1) on failure: populates full instruction trace + error
504        // and NUL-terminates.
505        let mut out = VecOutputBuffer::new(256);
506        let Err(err) =
507            verify_with_logger(invalid_code, VerifierLogLevel::LEVEL1, &mut out, make_context)
508        else {
509            panic!("invalid program must fail");
510        };
511        assert_eq!(err, errno!(EACCES));
512        assert!(out.data().ends_with(b"\0"));
513        let log_str = std::str::from_utf8(&out.data()[..out.data().len() - 1]).unwrap();
514        assert!(log_str.contains("0: (07) add %r0, 0x1"), "unexpected log: {log_str}");
515
516        // log_level = LEVEL2 (BPF_LOG_LEVEL2) on success: retains full instruction trace and
517        // NUL-terminates.
518        let mut out = VecOutputBuffer::new(256);
519        let _prog =
520            verify_with_logger(valid_code, VerifierLogLevel::LEVEL2, &mut out, make_context)
521                .expect("verify valid program with log_level=2");
522        assert!(out.data().ends_with(b"\0"));
523        let log_str = std::str::from_utf8(&out.data()[..out.data().len() - 1]).unwrap();
524        assert_eq!(log_str, "0: (b7) mov %r0, 0x0\n1: (95) exit\n");
525    }
526}
starnix_core/bpf/program.rs

starnix_core/bpf/
program.rs
