starnix_core/bpf/
program.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::bpf::BpfMapHandle;
6use crate::bpf::fs::get_bpf_object;
7use crate::security;
8use crate::task::{CurrentTask, CurrentTaskAndLocked, Kernel, register_delayed_release};
9use crate::vfs::{FdNumber, OutputBuffer};
10use ebpf::{
11    BPF_LDDW, BPF_PSEUDO_BTF_ID, BPF_PSEUDO_FUNC, BPF_PSEUDO_MAP_FD, BPF_PSEUDO_MAP_IDX,
12    BPF_PSEUDO_MAP_IDX_VALUE, BPF_PSEUDO_MAP_VALUE, EbpfError, EbpfInstruction, EbpfProgram,
13    EbpfProgramContext, StaticHelperSet, VerifiedEbpfProgram, VerifierLogger, link_program,
14    verify_program,
15};
16use ebpf_api::{AttachType, EbpfApiError, MapsContext, PinnedMap, ProgramType, StructId};
17use fidl_fuchsia_ebpf as febpf;
18use starnix_lifecycle::{AtomicU32Counter, ObjectReleaser, ReleaserAction};
19use starnix_logging::{log_error, log_warn, track_stub};
20use starnix_sync::{EbpfStateLock, LockBefore, Locked};
21use starnix_types::ownership::{Releasable, ReleaseGuard};
22use starnix_uapi::auth::{CAP_BPF, CAP_NET_ADMIN, CAP_PERFMON, CAP_SYS_ADMIN};
23use starnix_uapi::errors::Errno;
24use starnix_uapi::{bpf_attr__bindgen_ty_4, errno, error};
25use std::sync::atomic::Ordering;
26use std::sync::{Arc, Weak};
27use zx::HandleBased;
28
29#[derive(Clone, Debug)]
30pub struct ProgramInfo {
31    pub program_type: ProgramType,
32    pub expected_attach_type: AttachType,
33}
34
35impl TryFrom<&bpf_attr__bindgen_ty_4> for ProgramInfo {
36    type Error = Errno;
37
38    fn try_from(info: &bpf_attr__bindgen_ty_4) -> Result<Self, Self::Error> {
39        Ok(Self {
40            program_type: info.prog_type.try_into().map_err(map_ebpf_api_error)?,
41            expected_attach_type: info.expected_attach_type.into(),
42        })
43    }
44}
45pub type ProgramId = u32;
46
47static NEXT_PROGRAM_ID: AtomicU32Counter = AtomicU32Counter::new(1);
48fn new_program_id() -> ProgramId {
49    NEXT_PROGRAM_ID.next()
50}
51
52#[derive(Debug)]
53pub struct Program {
54    /// Program info specified during program initialization.
55    pub info: ProgramInfo,
56
57    /// Verified program.
58    program: VerifiedEbpfProgram,
59
60    /// eBPF maps used by the program. These should match the `maps` field in
61    /// the `program`.
62    maps: Vec<BpfMapHandle>,
63
64    /// Integer program ID. This is dinstinct from `fidl_id` because `fidl_id`
65    /// is 64-bit, while Linux uses 32-bit IDs.
66    id: ProgramId,
67
68    /// Handle used when the program is transferred over FIDL to other services.
69    fidl_handle: febpf::ProgramHandle,
70
71    /// ID of the program used in FIDL
72    fidl_id: febpf::ProgramId,
73
74    /// The service end of the `fidl_handle`. Should be moved to the BPF Service
75    /// once it's implemented.
76    #[allow(dead_code)]
77    service_handle: zx::EventPair,
78
79    /// Weak reference to the Kernel where this program is registered.
80    kernel: Weak<Kernel>,
81
82    /// The security state associated with this bpf Program.
83    pub security_state: security::BpfProgState,
84}
85
86fn map_ebpf_error(e: EbpfError) -> Errno {
87    log_error!("Failed to load eBPF program: {e:?}");
88    errno!(EINVAL)
89}
90
91fn map_ebpf_api_error(e: EbpfApiError) -> Errno {
92    log_error!("Failed to load eBPF program: {e:?}");
93    match e {
94        EbpfApiError::InvalidProgramType(_) | EbpfApiError::InvalidExpectedAttachType(_) => {
95            errno!(EINVAL)
96        }
97        EbpfApiError::UnsupportedProgramType(_) => errno!(ENOTSUP),
98    }
99}
100
101impl Program {
102    pub fn new<L>(
103        locked: &mut Locked<L>,
104        current_task: &CurrentTask,
105        info: ProgramInfo,
106        logger: &mut dyn OutputBuffer,
107        mut code: Vec<EbpfInstruction>,
108    ) -> Result<ProgramHandle, Errno>
109    where
110        L: LockBefore<EbpfStateLock>,
111    {
112        Self::check_load_access(current_task, &info)?;
113        let maps = link_maps_fds(current_task, &mut code)?;
114        let maps_schema = maps.iter().map(|m| m.schema).collect();
115        let mut logger = BufferVeriferLogger::new(logger);
116        let calling_context = info
117            .program_type
118            .create_calling_context(info.expected_attach_type, maps_schema)
119            .map_err(map_ebpf_api_error)?;
120        let program = verify_program(code, calling_context, &mut logger).map_err(map_ebpf_error)?;
121
122        let (fidl_handle, service_handle) = zx::EventPair::create();
123        let fidl_id =
124            febpf::ProgramId { id: fidl_handle.koid().expect("Failed to get koid").raw_koid() };
125        let fidl_handle = febpf::ProgramHandle { handle: fidl_handle };
126
127        let program = ProgramHandle::new(
128            Self {
129                info,
130                program,
131                maps,
132                id: new_program_id(),
133                fidl_handle,
134                fidl_id,
135                service_handle,
136                kernel: Arc::downgrade(current_task.kernel()),
137                security_state: security::bpf_prog_alloc(current_task),
138            }
139            .into(),
140        );
141        current_task.kernel().ebpf_state.register_program(locked, &program);
142
143        Ok(program)
144    }
145
146    pub fn id(&self) -> ProgramId {
147        self.id
148    }
149
150    pub fn link<C: EbpfProgramContext<Map = PinnedMap> + StaticHelperSet>(
151        &self,
152        program_type: ProgramType,
153    ) -> Result<EbpfProgram<C>, Errno>
154    where
155        for<'a> C::RunContext<'a>: MapsContext<'a>,
156    {
157        if program_type != self.info.program_type {
158            return error!(EINVAL);
159        }
160
161        let maps = self.maps.iter().map(|map| map.map.clone()).collect();
162        let program = link_program(&self.program, maps).map_err(map_ebpf_error)?;
163
164        Ok(program)
165    }
166
167    fn check_load_access(current_task: &CurrentTask, info: &ProgramInfo) -> Result<(), Errno> {
168        if matches!(info.program_type, ProgramType::CgroupSkb | ProgramType::SocketFilter)
169            && current_task.kernel().disable_unprivileged_bpf.load(Ordering::Relaxed) == 0
170        {
171            return Ok(());
172        }
173        if security::is_task_capable_noaudit(current_task, CAP_SYS_ADMIN) {
174            return Ok(());
175        }
176        security::check_task_capable(current_task, CAP_BPF)?;
177        match info.program_type {
178            // Loading tracing program types additionally require the CAP_PERFMON capability.
179            ProgramType::Kprobe
180            | ProgramType::Tracepoint
181            | ProgramType::PerfEvent
182            | ProgramType::RawTracepoint
183            | ProgramType::RawTracepointWritable
184            | ProgramType::Tracing => security::check_task_capable(current_task, CAP_PERFMON),
185
186            // Loading networking program types additionally require the CAP_NET_ADMIN capability.
187            ProgramType::SocketFilter
188            | ProgramType::SchedCls
189            | ProgramType::SchedAct
190            | ProgramType::Xdp
191            | ProgramType::SockOps
192            | ProgramType::SkSkb
193            | ProgramType::SkMsg
194            | ProgramType::SkLookup
195            | ProgramType::SkReuseport
196            | ProgramType::FlowDissector
197            | ProgramType::Netfilter => security::check_task_capable(current_task, CAP_NET_ADMIN),
198
199            // No additional checks are necessary for other program types.
200            ProgramType::CgroupDevice
201            | ProgramType::CgroupSkb
202            | ProgramType::CgroupSock
203            | ProgramType::CgroupSockAddr
204            | ProgramType::CgroupSockopt
205            | ProgramType::CgroupSysctl
206            | ProgramType::Ext
207            | ProgramType::LircMode2
208            | ProgramType::Lsm
209            | ProgramType::LwtIn
210            | ProgramType::LwtOut
211            | ProgramType::LwtSeg6Local
212            | ProgramType::LwtXmit
213            | ProgramType::StructOps
214            | ProgramType::Syscall
215            | ProgramType::Unspec
216            | ProgramType::Fuse => Ok(()),
217        }
218    }
219
220    pub fn fidl_id(&self) -> febpf::ProgramId {
221        self.fidl_id
222    }
223
224    pub fn fidl_handle(&self) -> febpf::ProgramHandle {
225        let handle = self
226            .fidl_handle
227            .handle
228            .duplicate_handle(zx::Rights::TRANSFER | zx::Rights::SIGNAL | zx::Rights::WAIT)
229            .expect("Failed to duplicate handle");
230        febpf::ProgramHandle { handle }
231    }
232}
233
234impl Releasable for Program {
235    type Context<'a> = CurrentTaskAndLocked<'a>;
236
237    fn release<'a>(self, (locked, _current_task): CurrentTaskAndLocked<'a>) {
238        if let Some(kernel) = self.kernel.upgrade() {
239            kernel.ebpf_state.unregister_program(locked, self.id);
240        }
241
242        // Signal the FIDL handle to indicate that the program handle is defunct
243        // and should be closed.
244        self.fidl_handle
245            .handle
246            .signal(
247                zx::Signals::NONE,
248                zx::Signals::from_bits_truncate(febpf::PROGRAM_DEFUNCT_SIGNAL),
249            )
250            .unwrap();
251    }
252}
253
254pub enum ProgramReleaserAction {}
255impl ReleaserAction<Program> for ProgramReleaserAction {
256    fn release(program: ReleaseGuard<Program>) {
257        register_delayed_release(program);
258    }
259}
260pub type ProgramReleaser = ObjectReleaser<Program, ProgramReleaserAction>;
261pub type ProgramHandle = Arc<ProgramReleaser>;
262pub type WeakProgramHandle = Weak<ProgramReleaser>;
263
264impl TryFrom<&Program> for febpf::VerifiedProgram {
265    type Error = Errno;
266
267    fn try_from(program: &Program) -> Result<febpf::VerifiedProgram, Errno> {
268        let mut maps = Vec::with_capacity(program.maps.len());
269        for map in program.maps.iter() {
270            maps.push(map.share().map_err(|_| errno!(EIO))?);
271        }
272
273        let code_u64: &[u64] = zerocopy::transmute_ref!(program.program.code());
274
275        let mut struct_access_instructions =
276            Vec::with_capacity(program.program.struct_access_instructions().len());
277        for v in program.program.struct_access_instructions() {
278            let struct_id = StructId::try_from(&v.memory_id).map_err(|()| errno!(EINVAL))?.into();
279            struct_access_instructions.push(febpf::StructAccess {
280                pc: v.pc.try_into().unwrap(),
281                struct_id,
282                field_offset: v.field_offset.try_into().unwrap(),
283                is_32_bit_ptr_load: v.is_32_bit_ptr_load,
284            })
285        }
286        Ok(febpf::VerifiedProgram {
287            code: Some(code_u64.to_vec()),
288            struct_access_instructions: Some(struct_access_instructions),
289            maps: Some(maps),
290            ..Default::default()
291        })
292    }
293}
294
295/// Links maps referenced by FD, replacing them with by-index references.
296fn link_maps_fds(
297    current_task: &CurrentTask,
298    code: &mut Vec<EbpfInstruction>,
299) -> Result<Vec<BpfMapHandle>, Errno> {
300    let code_len = code.len();
301    let mut maps = Vec::<BpfMapHandle>::new();
302    for (pc, instruction) in code.iter_mut().enumerate() {
303        if instruction.code() == BPF_LDDW {
304            // BPF_LDDW requires 2 instructions.
305            if pc >= code_len - 1 {
306                return error!(EINVAL);
307            }
308
309            match instruction.src_reg() {
310                0 => {}
311                BPF_PSEUDO_MAP_FD | BPF_PSEUDO_MAP_VALUE => {
312                    let lddw_type = if instruction.src_reg() == BPF_PSEUDO_MAP_FD {
313                        BPF_PSEUDO_MAP_IDX
314                    } else {
315                        BPF_PSEUDO_MAP_IDX_VALUE
316                    };
317                    // If the instruction references a map fd, then we need to look up the map fd
318                    // and create a reference from this program to that object.
319                    instruction.set_src_reg(lddw_type);
320
321                    let fd = FdNumber::from_raw(instruction.imm());
322                    let object = get_bpf_object(current_task, fd)?;
323                    let map: &BpfMapHandle = object.as_map()?;
324
325                    // Find the map in `maps` or insert it otherwise.
326                    let maybe_index = maps.iter().position(|v| Arc::ptr_eq(v, map));
327                    let index = match maybe_index {
328                        Some(index) => index,
329                        None => {
330                            let index = maps.len();
331                            maps.push(map.clone());
332                            index
333                        }
334                    };
335
336                    instruction.set_imm(index.try_into().unwrap());
337                }
338                BPF_PSEUDO_MAP_IDX
339                | BPF_PSEUDO_MAP_IDX_VALUE
340                | BPF_PSEUDO_BTF_ID
341                | BPF_PSEUDO_FUNC => {
342                    track_stub!(
343                        TODO("https://fxbug.dev/378564467"),
344                        "unsupported pseudo src for ldimm64",
345                        instruction.src_reg()
346                    );
347                    return error!(ENOTSUP);
348                }
349                _ => {
350                    return error!(EINVAL);
351                }
352            }
353        }
354    }
355    Ok(maps)
356}
357
358struct BufferVeriferLogger<'a> {
359    buffer: &'a mut dyn OutputBuffer,
360    full: bool,
361}
362
363impl BufferVeriferLogger<'_> {
364    fn new<'a>(buffer: &'a mut dyn OutputBuffer) -> BufferVeriferLogger<'a> {
365        BufferVeriferLogger { buffer, full: false }
366    }
367}
368
369impl VerifierLogger for BufferVeriferLogger<'_> {
370    fn log(&mut self, line: &[u8]) {
371        debug_assert!(line.is_ascii());
372
373        if self.full {
374            return;
375        }
376        if line.len() + 1 > self.buffer.available() {
377            self.full = true;
378            return;
379        }
380        match self.buffer.write(line) {
381            Err(e) => {
382                log_warn!("Unable to write verifier log: {e:?}");
383                self.full = true;
384            }
385            _ => {}
386        }
387        match self.buffer.write(b"\n") {
388            Err(e) => {
389                log_warn!("Unable to write verifier log: {e:?}");
390                self.full = true;
391            }
392            _ => {}
393        }
394    }
395}
starnix_core/bpf/program.rs

starnix_core/bpf/
program.rs
