starnix_core/bpf/
attachments.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5// TODO(https://github.com/rust-lang/rust/issues/39371): remove
6#![allow(non_upper_case_globals)]
7
8use crate::bpf::context::EbpfRunContextImpl;
9use crate::bpf::fs::{BpfHandle, get_bpf_object};
10use crate::bpf::program::ProgramHandle;
11use crate::mm::PAGE_SIZE;
12use crate::task::CurrentTask;
13use crate::vfs::FdNumber;
14use crate::vfs::socket::{
15    SockOptValue, SocketDomain, SocketProtocol, SocketType, ZxioBackedSocket,
16};
17use ebpf::{EbpfProgram, EbpfProgramContext, EbpfPtr, ProgramArgument, Type};
18use ebpf_api::{
19    AttachType, BPF_SOCK_ADDR_TYPE, BPF_SOCK_TYPE, CgroupSockAddrProgramContext,
20    CgroupSockOptProgramContext, CgroupSockProgramContext, CurrentTaskContext, Map, MapValueRef,
21    MapsContext, PinnedMap, ProgramType, ReturnValueContext, SocketCookieContext,
22};
23use fidl_fuchsia_net_filter as fnet_filter;
24use fuchsia_component::client::connect_to_protocol_sync;
25use linux_uapi::bpf_sockopt;
26use starnix_logging::{log_error, log_warn, track_stub};
27use starnix_sync::{EbpfStateLock, FileOpsCore, Locked, OrderedRwLock, Unlocked};
28use starnix_syscalls::{SUCCESS, SyscallResult};
29use starnix_uapi::errors::{Errno, ErrnoCode, is_error_return_value};
30use starnix_uapi::{
31    CGROUP2_SUPER_MAGIC, bpf_attr__bindgen_ty_6, bpf_sock, bpf_sock_addr, errno, error, gid_t,
32    pid_t, uid_t,
33};
34use std::ops::{Deref, DerefMut};
35use std::sync::{Arc, OnceLock};
36use zerocopy::FromBytes;
37
38pub type BpfAttachAttr = bpf_attr__bindgen_ty_6;
39
40fn check_root_cgroup_fd(
41    locked: &mut Locked<Unlocked>,
42    current_task: &CurrentTask,
43    cgroup_fd: FdNumber,
44) -> Result<(), Errno> {
45    let file = current_task.get_file(cgroup_fd)?;
46
47    // Check that `cgroup_fd` is from the CGROUP2 file system.
48    let is_cgroup =
49        file.node().fs().statfs(locked, current_task)?.f_type == CGROUP2_SUPER_MAGIC as i64;
50    if !is_cgroup {
51        log_warn!("bpf_prog_attach(BPF_PROG_ATTACH) is called with an invalid cgroup2 FD.");
52        return error!(EINVAL);
53    }
54
55    // Currently cgroup attachments are supported only for the root cgroup.
56    // TODO(https://fxbug.dev//388077431) Allow attachments to any cgroup once cgroup
57    // hierarchy is moved to starnix_core.
58    let is_root = file
59        .node()
60        .fs()
61        .maybe_root()
62        .map(|root| Arc::ptr_eq(&root.node, file.node()))
63        .unwrap_or(false);
64    if !is_root {
65        log_warn!("bpf_prog_attach(BPF_PROG_ATTACH) is supported only for root cgroup.");
66        return error!(EINVAL);
67    }
68
69    Ok(())
70}
71
72pub fn bpf_prog_attach(
73    locked: &mut Locked<Unlocked>,
74    current_task: &CurrentTask,
75    attr: BpfAttachAttr,
76) -> Result<SyscallResult, Errno> {
77    // SAFETY: reading i32 field from a union is always safe.
78    let bpf_fd = FdNumber::from_raw(attr.attach_bpf_fd as i32);
79    let object = get_bpf_object(current_task, bpf_fd)?;
80    if matches!(object, BpfHandle::ProgramStub(_)) {
81        log_warn!("Stub program. Faking successful attach");
82        return Ok(SUCCESS);
83    }
84    let program = object.as_program()?.clone();
85    let attach_type = AttachType::from(attr.attach_type);
86
87    let program_type = program.info.program_type;
88    if attach_type.get_program_type() != program_type {
89        log_warn!(
90            "bpf_prog_attach(BPF_PROG_ATTACH): program not compatible with attach_type \
91                   attach_type: {attach_type:?}, program_type: {program_type:?}"
92        );
93        return error!(EINVAL);
94    }
95
96    if !attach_type.is_compatible_with_expected_attach_type(program.info.expected_attach_type) {
97        log_warn!(
98            "bpf_prog_attach(BPF_PROG_ATTACH): expected_attach_type didn't match attach_type \
99                   expected_attach_type: {:?}, attach_type: {:?}",
100            program.info.expected_attach_type,
101            attach_type
102        );
103        return error!(EINVAL);
104    }
105
106    // SAFETY: reading i32 field from a union is always safe.
107    let target_fd = unsafe { attr.__bindgen_anon_1.target_fd };
108    let target_fd = FdNumber::from_raw(target_fd as i32);
109
110    current_task.kernel().ebpf_state.attachments.attach_prog(
111        locked,
112        current_task,
113        attach_type,
114        target_fd,
115        program,
116    )
117}
118
119pub fn bpf_prog_detach(
120    locked: &mut Locked<Unlocked>,
121    current_task: &CurrentTask,
122    attr: BpfAttachAttr,
123) -> Result<SyscallResult, Errno> {
124    let attach_type = AttachType::from(attr.attach_type);
125
126    // SAFETY: reading i32 field from a union is always safe.
127    let target_fd = unsafe { attr.__bindgen_anon_1.target_fd };
128    let target_fd = FdNumber::from_raw(target_fd as i32);
129
130    current_task.kernel().ebpf_state.attachments.detach_prog(
131        locked,
132        current_task,
133        attach_type,
134        target_fd,
135    )
136}
137
138// Wrapper for `bpf_sock_addr` used to implement `ProgramArgument` trait.
139#[repr(C)]
140pub struct BpfSockAddr<'a> {
141    sock_addr: bpf_sock_addr,
142
143    socket: &'a ZxioBackedSocket,
144}
145
146impl<'a> Deref for BpfSockAddr<'a> {
147    type Target = bpf_sock_addr;
148    fn deref(&self) -> &Self::Target {
149        &self.sock_addr
150    }
151}
152
153impl<'a> DerefMut for BpfSockAddr<'a> {
154    fn deref_mut(&mut self) -> &mut Self::Target {
155        &mut self.sock_addr
156    }
157}
158
159impl<'a> ProgramArgument for &'_ mut BpfSockAddr<'a> {
160    fn get_type() -> &'static Type {
161        &*BPF_SOCK_ADDR_TYPE
162    }
163}
164
165impl<'a, 'b> SocketCookieContext<&'a mut BpfSockAddr<'a>> for EbpfRunContextImpl<'b> {
166    fn get_socket_cookie(&self, bpf_sock_addr: &'a mut BpfSockAddr<'a>) -> u64 {
167        let v = bpf_sock_addr.socket.get_socket_cookie();
168        v.unwrap_or_else(|errno| {
169            log_error!("Failed to get socket cookie: {:?}", errno);
170            0
171        })
172    }
173}
174
175// Context for eBPF programs of type BPF_PROG_TYPE_CGROUP_SOCKADDR.
176struct SockAddrProgram(EbpfProgram<SockAddrProgram>);
177
178impl EbpfProgramContext for SockAddrProgram {
179    type RunContext<'a> = EbpfRunContextImpl<'a>;
180    type Packet<'a> = ();
181    type Arg1<'a> = &'a mut BpfSockAddr<'a>;
182    type Arg2<'a> = ();
183    type Arg3<'a> = ();
184    type Arg4<'a> = ();
185    type Arg5<'a> = ();
186
187    type Map = PinnedMap;
188}
189
190ebpf_api::ebpf_program_context_type!(SockAddrProgram, CgroupSockAddrProgramContext);
191
192#[derive(Debug, PartialEq, Eq)]
193pub enum SockAddrProgramResult {
194    Allow,
195    Block,
196}
197
198impl SockAddrProgram {
199    fn run<'a>(
200        &self,
201        locked: &'a mut Locked<EbpfStateLock>,
202        current_task: &'a CurrentTask,
203        addr: &'a mut BpfSockAddr<'a>,
204        can_block: bool,
205    ) -> SockAddrProgramResult {
206        let mut run_context = EbpfRunContextImpl::new(locked, current_task);
207        match self.0.run_with_1_argument(&mut run_context, addr) {
208            // UDP_RECVMSG programs are not allowed to block the packet.
209            0 if can_block => SockAddrProgramResult::Block,
210            1 => SockAddrProgramResult::Allow,
211            result => {
212                // TODO(https://fxbug.dev/413490751): Change this to panic once
213                // result validation is implemented in the eBPF verifier.
214                log_error!("eBPF program returned invalid result: {}", result);
215                SockAddrProgramResult::Allow
216            }
217        }
218    }
219}
220
221type AttachedSockAddrProgramCell = OrderedRwLock<Option<SockAddrProgram>, EbpfStateLock>;
222
223// Wrapper for `bpf_sock` used to implement `ProgramArgument` trait.
224#[repr(C)]
225pub struct BpfSock<'a> {
226    // Must be first field.
227    value: bpf_sock,
228
229    socket: &'a ZxioBackedSocket,
230}
231
232impl<'a> Deref for BpfSock<'a> {
233    type Target = bpf_sock;
234    fn deref(&self) -> &Self::Target {
235        &self.value
236    }
237}
238
239impl<'a> DerefMut for BpfSock<'a> {
240    fn deref_mut(&mut self) -> &mut Self::Target {
241        &mut self.value
242    }
243}
244
245impl<'a> ProgramArgument for &'_ BpfSock<'a> {
246    fn get_type() -> &'static Type {
247        &*BPF_SOCK_TYPE
248    }
249}
250
251impl<'a, 'b> SocketCookieContext<&'a BpfSock<'a>> for EbpfRunContextImpl<'b> {
252    fn get_socket_cookie(&self, bpf_sock: &'a BpfSock<'a>) -> u64 {
253        let v = bpf_sock.socket.get_socket_cookie();
254        v.unwrap_or_else(|errno| {
255            log_error!("Failed to get socket cookie: {:?}", errno);
256            0
257        })
258    }
259}
260
261// Context for eBPF programs of type BPF_PROG_TYPE_CGROUP_SOCK.
262struct SockProgram(EbpfProgram<SockProgram>);
263
264impl EbpfProgramContext for SockProgram {
265    type RunContext<'a> = EbpfRunContextImpl<'a>;
266    type Packet<'a> = ();
267    type Arg1<'a> = &'a BpfSock<'a>;
268    type Arg2<'a> = ();
269    type Arg3<'a> = ();
270    type Arg4<'a> = ();
271    type Arg5<'a> = ();
272
273    type Map = PinnedMap;
274}
275
276ebpf_api::ebpf_program_context_type!(SockProgram, CgroupSockProgramContext);
277
278#[derive(Debug, PartialEq, Eq)]
279pub enum SockProgramResult {
280    Allow,
281    Block,
282}
283
284impl SockProgram {
285    fn run<'a>(
286        &self,
287        locked: &mut Locked<EbpfStateLock>,
288        current_task: &'a CurrentTask,
289        sock: &'a BpfSock<'a>,
290    ) -> SockProgramResult {
291        let mut run_context = EbpfRunContextImpl::new(locked, current_task);
292        if self.0.run_with_1_argument(&mut run_context, sock) == 0 {
293            SockProgramResult::Block
294        } else {
295            SockProgramResult::Allow
296        }
297    }
298}
299
300type AttachedSockProgramCell = OrderedRwLock<Option<SockProgram>, EbpfStateLock>;
301
302mod internal {
303    use ebpf::{EbpfPtr, ProgramArgument, Type};
304    use ebpf_api::BPF_SOCKOPT_TYPE;
305    use starnix_uapi::{bpf_sockopt, uaddr};
306    use std::ops::Deref;
307    use zerocopy::{FromBytes, IntoBytes};
308
309    // Wrapper for `bpf_sockopt` that implements `ProgramArgument` trait.
310    #[repr(C)]
311    #[derive(IntoBytes, FromBytes)]
312    pub struct BpfSockOpt(bpf_sockopt);
313
314    impl ProgramArgument for &'_ mut BpfSockOpt {
315        fn get_type() -> &'static Type {
316            &*BPF_SOCKOPT_TYPE
317        }
318    }
319
320    /// Wrapper for `bpf_sockopt` that keeps a buffer for the `optval`.
321    pub struct BpfSockOptWithValue {
322        sockopt: BpfSockOpt,
323
324        // Buffer used to store the option value. A pointer to the buffer
325        // contents is stored in `sockopt`. `Vec::as_mut_ptr()` guarantees that
326        // the pointer remains valid only as long as the `Vec` is not modified,
327        // so this field should not be updated directly. `take_value()` can be
328        // used to extract the value when `BpfSockOpt` is no longer needed.
329        value_buf: Vec<u8>,
330    }
331
332    impl BpfSockOptWithValue {
333        pub fn new(level: u32, optname: u32, value_buf: Vec<u8>, optlen: u32, retval: i32) -> Self {
334            let mut sockopt = Self {
335                sockopt: BpfSockOpt(bpf_sockopt {
336                    level: level as i32,
337                    optname: optname as i32,
338                    optlen: optlen as i32,
339                    retval: retval as i32,
340                    ..Default::default()
341                }),
342                value_buf,
343            };
344
345            // SAFETY: Setting buffer bounds in unions is safe.
346            unsafe {
347                sockopt.sockopt.0.__bindgen_anon_2.optval =
348                    uaddr { addr: sockopt.value_buf.as_mut_ptr() as u64 };
349                sockopt.sockopt.0.__bindgen_anon_3.optval_end = uaddr {
350                    addr: sockopt.value_buf.as_mut_ptr().add(sockopt.value_buf.len()) as u64,
351                };
352            }
353
354            sockopt
355        }
356
357        pub fn as_ptr<'a>(&'a mut self) -> EbpfPtr<'a, BpfSockOpt> {
358            EbpfPtr::from(&mut self.sockopt)
359        }
360
361        // Returns the value. Consumes `self` since it's not safe to use again
362        // after the value buffer is moved.
363        pub fn take_value(self) -> Vec<u8> {
364            self.value_buf
365        }
366    }
367
368    impl Deref for BpfSockOptWithValue {
369        type Target = bpf_sockopt;
370        fn deref(&self) -> &Self::Target {
371            &self.sockopt.0
372        }
373    }
374}
375
376use internal::{BpfSockOpt, BpfSockOptWithValue};
377
378// Context for eBPF programs of type BPF_PROG_TYPE_CGROUP_SOCKOPT.
379struct SockOptProgram(EbpfProgram<SockOptProgram>);
380
381// RunContext for eBPF programs of type BPF_PROG_TYPE_CGROUP_SOCKOPT.
382pub struct SockOptEbpfRunContextImpl<'a> {
383    ebpf_run_context: EbpfRunContextImpl<'a>,
384
385    // Pointer to the BpfSockOpt passed to the program. Used for
386    // `bpf_set_retval` and `bpf_get_retval`.
387    sockopt: EbpfPtr<'a, BpfSockOpt>,
388}
389
390const BPF_SOCKOPT_RETVAL_OFFSET: usize = std::mem::offset_of!(bpf_sockopt, retval);
391
392impl<'a> SockOptEbpfRunContextImpl<'a> {
393    pub fn new(
394        locked: &'a mut Locked<EbpfStateLock>,
395        current_task: &'a CurrentTask,
396        sockopt: EbpfPtr<'a, BpfSockOpt>,
397    ) -> Self {
398        Self { ebpf_run_context: EbpfRunContextImpl::new(locked, current_task), sockopt }
399    }
400}
401
402impl<'a> MapsContext<'a> for SockOptEbpfRunContextImpl<'a> {
403    fn on_map_access(&mut self, map: &Map) {
404        self.ebpf_run_context.on_map_access(map);
405    }
406    fn add_value_ref(&mut self, map_ref: MapValueRef<'a>) {
407        self.ebpf_run_context.add_value_ref(map_ref);
408    }
409}
410
411impl<'a> CurrentTaskContext for SockOptEbpfRunContextImpl<'a> {
412    fn get_uid_gid(&self) -> (uid_t, gid_t) {
413        self.ebpf_run_context.get_uid_gid()
414    }
415    fn get_tid_tgid(&self) -> (pid_t, pid_t) {
416        self.ebpf_run_context.get_tid_tgid()
417    }
418}
419
420impl<'a> ReturnValueContext for SockOptEbpfRunContextImpl<'a> {
421    fn set_retval(&mut self, value: i32) -> i32 {
422        let sockopt = self.sockopt.get_field::<i32, BPF_SOCKOPT_RETVAL_OFFSET>();
423        sockopt.store_relaxed(value);
424        0
425    }
426    fn get_retval(&self) -> i32 {
427        let sockopt = self.sockopt.get_field::<i32, BPF_SOCKOPT_RETVAL_OFFSET>();
428        sockopt.load_relaxed()
429    }
430}
431
432impl EbpfProgramContext for SockOptProgram {
433    type RunContext<'a> = SockOptEbpfRunContextImpl<'a>;
434    type Packet<'a> = ();
435    type Arg1<'a> = EbpfPtr<'a, BpfSockOpt>;
436    type Arg2<'a> = ();
437    type Arg3<'a> = ();
438    type Arg4<'a> = ();
439    type Arg5<'a> = ();
440
441    type Map = PinnedMap;
442}
443
444ebpf_api::ebpf_program_context_type!(SockOptProgram, CgroupSockOptProgramContext);
445
446#[derive(Debug)]
447pub enum SetSockOptProgramResult {
448    /// Fail the syscall.
449    Fail(Errno),
450
451    /// Proceed with the specified option value.
452    Allow(SockOptValue),
453
454    /// Return to userspace without invoking the underlying implementation of
455    /// setsockopt.
456    Bypass,
457}
458
459impl SockOptProgram {
460    fn run<'a>(
461        &self,
462        locked: &mut Locked<EbpfStateLock>,
463        current_task: &'a CurrentTask,
464        sockopt: &'a mut BpfSockOptWithValue,
465    ) -> u64 {
466        let sockopt_ptr = sockopt.as_ptr();
467        let mut run_context = SockOptEbpfRunContextImpl::new(locked, current_task, sockopt_ptr);
468        self.0.run_with_1_argument(&mut run_context, sockopt_ptr)
469    }
470}
471
472type AttachedSockOptProgramCell = OrderedRwLock<Option<SockOptProgram>, EbpfStateLock>;
473
474#[derive(Default)]
475pub struct CgroupEbpfProgramSet {
476    inet4_bind: AttachedSockAddrProgramCell,
477    inet6_bind: AttachedSockAddrProgramCell,
478    inet4_connect: AttachedSockAddrProgramCell,
479    inet6_connect: AttachedSockAddrProgramCell,
480    udp4_sendmsg: AttachedSockAddrProgramCell,
481    udp6_sendmsg: AttachedSockAddrProgramCell,
482    udp4_recvmsg: AttachedSockAddrProgramCell,
483    udp6_recvmsg: AttachedSockAddrProgramCell,
484    sock_create: AttachedSockProgramCell,
485    sock_release: AttachedSockProgramCell,
486    set_sockopt: AttachedSockOptProgramCell,
487    get_sockopt: AttachedSockOptProgramCell,
488}
489
490#[derive(Eq, PartialEq, Debug, Copy, Clone)]
491pub enum SockAddrOp {
492    Bind,
493    Connect,
494    UdpSendMsg,
495    UdpRecvMsg,
496}
497
498#[derive(Eq, PartialEq, Debug, Copy, Clone)]
499pub enum SockOp {
500    Create,
501    Release,
502}
503
504impl CgroupEbpfProgramSet {
505    fn get_sock_addr_program(
506        &self,
507        attach_type: AttachType,
508    ) -> Result<&AttachedSockAddrProgramCell, Errno> {
509        assert!(attach_type.is_cgroup());
510
511        match attach_type {
512            AttachType::CgroupInet4Bind => Ok(&self.inet4_bind),
513            AttachType::CgroupInet6Bind => Ok(&self.inet6_bind),
514            AttachType::CgroupInet4Connect => Ok(&self.inet4_connect),
515            AttachType::CgroupInet6Connect => Ok(&self.inet6_connect),
516            AttachType::CgroupUdp4Sendmsg => Ok(&self.udp4_sendmsg),
517            AttachType::CgroupUdp6Sendmsg => Ok(&self.udp6_sendmsg),
518            AttachType::CgroupUdp4Recvmsg => Ok(&self.udp4_recvmsg),
519            AttachType::CgroupUdp6Recvmsg => Ok(&self.udp6_recvmsg),
520            _ => error!(ENOTSUP),
521        }
522    }
523
524    fn get_sock_program(&self, attach_type: AttachType) -> Result<&AttachedSockProgramCell, Errno> {
525        assert!(attach_type.is_cgroup());
526
527        match attach_type {
528            AttachType::CgroupInetSockCreate => Ok(&self.sock_create),
529            AttachType::CgroupInetSockRelease => Ok(&self.sock_release),
530            _ => error!(ENOTSUP),
531        }
532    }
533
534    fn get_sock_opt_program(
535        &self,
536        attach_type: AttachType,
537    ) -> Result<&AttachedSockOptProgramCell, Errno> {
538        assert!(attach_type.is_cgroup());
539
540        match attach_type {
541            AttachType::CgroupSetsockopt => Ok(&self.set_sockopt),
542            AttachType::CgroupGetsockopt => Ok(&self.get_sockopt),
543            _ => error!(ENOTSUP),
544        }
545    }
546
547    // Executes eBPF program for the operation `op`. `socket_address` contains
548    // socket address as a `sockaddr` struct.
549    pub fn run_sock_addr_prog(
550        &self,
551        locked: &mut Locked<FileOpsCore>,
552        current_task: &CurrentTask,
553        op: SockAddrOp,
554        domain: SocketDomain,
555        socket_type: SocketType,
556        protocol: SocketProtocol,
557        socket_address: &[u8],
558        socket: &ZxioBackedSocket,
559    ) -> Result<SockAddrProgramResult, Errno> {
560        let prog_cell = match (domain, op) {
561            (SocketDomain::Inet, SockAddrOp::Bind) => &self.inet4_bind,
562            (SocketDomain::Inet6, SockAddrOp::Bind) => &self.inet6_bind,
563            (SocketDomain::Inet, SockAddrOp::Connect) => &self.inet4_connect,
564            (SocketDomain::Inet6, SockAddrOp::Connect) => &self.inet6_connect,
565            (SocketDomain::Inet, SockAddrOp::UdpSendMsg) => &self.udp4_sendmsg,
566            (SocketDomain::Inet6, SockAddrOp::UdpSendMsg) => &self.udp6_sendmsg,
567            (SocketDomain::Inet, SockAddrOp::UdpRecvMsg) => &self.udp4_recvmsg,
568            (SocketDomain::Inet6, SockAddrOp::UdpRecvMsg) => &self.udp6_recvmsg,
569            _ => return Ok(SockAddrProgramResult::Allow),
570        };
571
572        let (prog_guard, locked) = prog_cell.read_and(locked);
573        let Some(prog) = prog_guard.as_ref() else {
574            return Ok(SockAddrProgramResult::Allow);
575        };
576
577        let mut bpf_sockaddr = BpfSockAddr { sock_addr: Default::default(), socket };
578        bpf_sockaddr.family = domain.as_raw().into();
579        bpf_sockaddr.type_ = socket_type.as_raw();
580        bpf_sockaddr.protocol = protocol.as_raw();
581
582        let (sa_family, _) = u16::read_from_prefix(socket_address).map_err(|_| errno!(EINVAL))?;
583
584        if domain.as_raw() != sa_family {
585            return error!(EAFNOSUPPORT);
586        }
587        bpf_sockaddr.user_family = sa_family.into();
588
589        match sa_family.into() {
590            linux_uapi::AF_INET => {
591                let (sockaddr, _) = linux_uapi::sockaddr_in::ref_from_prefix(socket_address)
592                    .map_err(|_| errno!(EINVAL))?;
593                bpf_sockaddr.user_port = sockaddr.sin_port.into();
594                bpf_sockaddr.user_ip4 = sockaddr.sin_addr.s_addr;
595            }
596            linux_uapi::AF_INET6 => {
597                let sockaddr = linux_uapi::sockaddr_in6::ref_from_prefix(socket_address)
598                    .map_err(|_| errno!(EINVAL))?
599                    .0;
600                bpf_sockaddr.user_port = sockaddr.sin6_port.into();
601                // SAFETY: reading an array of u32 from a union is safe.
602                bpf_sockaddr.user_ip6 = unsafe { sockaddr.sin6_addr.in6_u.u6_addr32 };
603            }
604            _ => return error!(EAFNOSUPPORT),
605        }
606
607        // UDP recvmsg programs are not allowed to filter packets.
608        let can_block = op != SockAddrOp::UdpRecvMsg;
609        Ok(prog.run(locked, current_task, &mut bpf_sockaddr, can_block))
610    }
611
612    pub fn run_sock_prog(
613        &self,
614        locked: &mut Locked<FileOpsCore>,
615        current_task: &CurrentTask,
616        op: SockOp,
617        domain: SocketDomain,
618        socket_type: SocketType,
619        protocol: SocketProtocol,
620        socket: &ZxioBackedSocket,
621    ) -> SockProgramResult {
622        let prog_cell = match op {
623            SockOp::Create => &self.sock_create,
624            SockOp::Release => &self.sock_release,
625        };
626        let (prog_guard, locked) = prog_cell.read_and(locked);
627        let Some(prog) = prog_guard.as_ref() else {
628            return SockProgramResult::Allow;
629        };
630
631        let bpf_sock = BpfSock {
632            value: bpf_sock {
633                family: domain.as_raw().into(),
634                type_: socket_type.as_raw(),
635                protocol: protocol.as_raw(),
636                ..Default::default()
637            },
638            socket,
639        };
640
641        prog.run(locked, current_task, &bpf_sock)
642    }
643
644    pub fn run_getsockopt_prog(
645        &self,
646        locked: &mut Locked<FileOpsCore>,
647        current_task: &CurrentTask,
648        level: u32,
649        optname: u32,
650        optval: Vec<u8>,
651        optlen: usize,
652        error: Option<Errno>,
653    ) -> Result<(Vec<u8>, usize), Errno> {
654        let (prog_guard, locked) = self.get_sockopt.read_and(locked);
655        let Some(prog) = prog_guard.as_ref() else {
656            return error.map(|e| Err(e)).unwrap_or_else(|| Ok((optval, optlen)));
657        };
658
659        let retval = error.as_ref().map(|e| -(e.code.error_code() as i32)).unwrap_or(0);
660        let mut bpf_sockopt =
661            BpfSockOptWithValue::new(level, optname, optval.clone(), optlen as u32, retval);
662
663        // Run the program.
664        let result = prog.run(locked, current_task, &mut bpf_sockopt);
665
666        let retval = bpf_sockopt.retval;
667
668        let retval = match result {
669            0 if is_error_return_value(retval) => retval,
670            0 => -(linux_uapi::EPERM as i32),
671            1 => retval,
672            _ => {
673                // TODO(https://fxbug.dev/413490751): Change this to panic once
674                // result validation is implemented in the verifier.
675                log_error!("eBPF getsockopt program returned invalid result: {}", result);
676                retval
677            }
678        };
679
680        if retval < 0 {
681            return Err(Errno::new(ErrnoCode::from_error_code(-retval as i16)));
682        }
683
684        let optlen = bpf_sockopt.optlen;
685
686        // Fail if the program has set an invalid `optlen` (except for the
687        // case handled above).
688        if optlen < 0 || (optlen as usize) > optval.len() {
689            return error!(EFAULT);
690        }
691
692        // If `optlen` is set to 0 then proceed with the original value.
693        let value = if optlen == 0 { optval } else { bpf_sockopt.take_value() };
694
695        Ok((value, optlen as usize))
696    }
697
698    pub fn run_setsockopt_prog(
699        &self,
700        locked: &mut Locked<FileOpsCore>,
701        current_task: &CurrentTask,
702        level: u32,
703        optname: u32,
704        value: SockOptValue,
705    ) -> SetSockOptProgramResult {
706        let (prog_guard, locked) = self.set_sockopt.read_and(locked);
707        let Some(prog) = prog_guard.as_ref() else {
708            return SetSockOptProgramResult::Allow(value);
709        };
710
711        let page_size = *PAGE_SIZE as usize;
712
713        // Read only the first page from the user-specified buffer in case it's
714        // larger than that.
715        let buffer = match value.read_bytes(current_task, page_size) {
716            Ok(buffer) => buffer,
717            Err(err) => return SetSockOptProgramResult::Fail(err),
718        };
719
720        let buffer_len = buffer.len();
721        let optlen = value.len();
722        let mut bpf_sockopt = BpfSockOptWithValue::new(level, optname, buffer, optlen as u32, 0);
723        let result = prog.run(locked.cast_locked(), current_task, &mut bpf_sockopt);
724
725        let retval = bpf_sockopt.retval;
726
727        let retval = match result {
728            0 if is_error_return_value(retval) => retval,
729            0 => -(linux_uapi::EPERM as i32),
730            1 => retval,
731            _ => {
732                // TODO(https://fxbug.dev/413490751): Change this to panic once
733                // result validation is implemented in the verifier.
734                log_error!("eBPF getsockopt program returned invalid result: {}", result);
735                retval
736            }
737        };
738
739        if retval < 0 {
740            return SetSockOptProgramResult::Fail(Errno::new(ErrnoCode::from_error_code(
741                -retval as i16,
742            )));
743        }
744
745        match bpf_sockopt.optlen {
746            // `setsockopt` programs can bypass the platform implementation by
747            // setting `optlen` to -1.
748            -1 => SetSockOptProgramResult::Bypass,
749
750            // If the original value is larger than a page and the program
751            // didn't change `optlen` then return the original value. This
752            // allows to avoid `EFAULT` below with a no-op program.
753            new_optlen if optlen > page_size && (new_optlen as usize) == optlen => {
754                SetSockOptProgramResult::Allow(value)
755            }
756
757            // Fail if the program has set an invalid `optlen` (except for the
758            // case handled above).
759            optlen if optlen < 0 || (optlen as usize) > buffer_len => {
760                SetSockOptProgramResult::Fail(errno!(EFAULT))
761            }
762
763            // If `optlen` is set to 0 then proceed with the original value.
764            0 => SetSockOptProgramResult::Allow(value),
765
766            // Return value from `bpf_sockbuf` - it may be different from the
767            // original value.
768            optlen => {
769                let mut value = bpf_sockopt.take_value();
770                value.resize(optlen as usize, 0);
771                SetSockOptProgramResult::Allow(value.into())
772            }
773        }
774    }
775}
776
777fn attach_type_to_netstack_hook(attach_type: AttachType) -> Option<fnet_filter::SocketHook> {
778    let hook = match attach_type {
779        AttachType::CgroupInetEgress => fnet_filter::SocketHook::Egress,
780        AttachType::CgroupInetIngress => fnet_filter::SocketHook::Ingress,
781        _ => return None,
782    };
783    Some(hook)
784}
785
786// Defined a location where eBPF programs can be attached.
787#[derive(Copy, Clone, Debug, PartialEq, Eq)]
788enum AttachLocation {
789    // Attached in Starnix kernel.
790    Kernel,
791
792    // Attached in Netstack.
793    Netstack,
794}
795
796impl TryFrom<AttachType> for AttachLocation {
797    type Error = Errno;
798
799    fn try_from(attach_type: AttachType) -> Result<Self, Self::Error> {
800        match attach_type {
801            AttachType::CgroupInet4Bind
802            | AttachType::CgroupInet6Bind
803            | AttachType::CgroupInet4Connect
804            | AttachType::CgroupInet6Connect
805            | AttachType::CgroupUdp4Sendmsg
806            | AttachType::CgroupUdp6Sendmsg
807            | AttachType::CgroupUdp4Recvmsg
808            | AttachType::CgroupUdp6Recvmsg
809            | AttachType::CgroupInetSockCreate
810            | AttachType::CgroupInetSockRelease
811            | AttachType::CgroupGetsockopt
812            | AttachType::CgroupSetsockopt => Ok(AttachLocation::Kernel),
813
814            AttachType::CgroupInetEgress | AttachType::CgroupInetIngress => {
815                Ok(AttachLocation::Netstack)
816            }
817
818            AttachType::CgroupDevice
819            | AttachType::CgroupInet4Getpeername
820            | AttachType::CgroupInet4Getsockname
821            | AttachType::CgroupInet4PostBind
822            | AttachType::CgroupInet6Getpeername
823            | AttachType::CgroupInet6Getsockname
824            | AttachType::CgroupInet6PostBind
825            | AttachType::CgroupSysctl
826            | AttachType::CgroupUnixConnect
827            | AttachType::CgroupUnixGetpeername
828            | AttachType::CgroupUnixGetsockname
829            | AttachType::CgroupUnixRecvmsg
830            | AttachType::CgroupUnixSendmsg
831            | AttachType::CgroupSockOps
832            | AttachType::SkSkbStreamParser
833            | AttachType::SkSkbStreamVerdict
834            | AttachType::SkMsgVerdict
835            | AttachType::LircMode2
836            | AttachType::FlowDissector
837            | AttachType::TraceRawTp
838            | AttachType::TraceFentry
839            | AttachType::TraceFexit
840            | AttachType::ModifyReturn
841            | AttachType::LsmMac
842            | AttachType::TraceIter
843            | AttachType::XdpDevmap
844            | AttachType::XdpCpumap
845            | AttachType::SkLookup
846            | AttachType::Xdp
847            | AttachType::SkSkbVerdict
848            | AttachType::SkReuseportSelect
849            | AttachType::SkReuseportSelectOrMigrate
850            | AttachType::PerfEvent
851            | AttachType::TraceKprobeMulti
852            | AttachType::LsmCgroup
853            | AttachType::StructOps
854            | AttachType::Netfilter
855            | AttachType::TcxIngress
856            | AttachType::TcxEgress
857            | AttachType::TraceUprobeMulti
858            | AttachType::NetkitPrimary
859            | AttachType::NetkitPeer
860            | AttachType::TraceKprobeSession => {
861                track_stub!(TODO("https://fxbug.dev/322873416"), "BPF_PROG_ATTACH", attach_type);
862                error!(ENOTSUP)
863            }
864
865            AttachType::Unspecified | AttachType::Invalid(_) => {
866                error!(EINVAL)
867            }
868        }
869    }
870}
871
872#[derive(Default)]
873pub struct EbpfAttachments {
874    root_cgroup: CgroupEbpfProgramSet,
875    socket_control: OnceLock<fnet_filter::SocketControlSynchronousProxy>,
876}
877
878impl EbpfAttachments {
879    pub fn root_cgroup(&self) -> &CgroupEbpfProgramSet {
880        &self.root_cgroup
881    }
882
883    fn socket_control(&self) -> &fnet_filter::SocketControlSynchronousProxy {
884        self.socket_control.get_or_init(|| {
885            connect_to_protocol_sync::<fnet_filter::SocketControlMarker>()
886                .expect("Failed to connect to fuchsia.net.filter.SocketControl.")
887        })
888    }
889
890    fn attach_prog(
891        &self,
892        locked: &mut Locked<Unlocked>,
893        current_task: &CurrentTask,
894        attach_type: AttachType,
895        target_fd: FdNumber,
896        program: ProgramHandle,
897    ) -> Result<SyscallResult, Errno> {
898        let location: AttachLocation = attach_type.try_into()?;
899        let program_type = attach_type.get_program_type();
900        match (location, program_type) {
901            (AttachLocation::Kernel, ProgramType::CgroupSockAddr) => {
902                check_root_cgroup_fd(locked, current_task, target_fd)?;
903
904                let linked_program = SockAddrProgram(program.link(attach_type.get_program_type())?);
905                *self.root_cgroup.get_sock_addr_program(attach_type)?.write(locked) =
906                    Some(linked_program);
907
908                Ok(SUCCESS)
909            }
910
911            (AttachLocation::Kernel, ProgramType::CgroupSock) => {
912                check_root_cgroup_fd(locked, current_task, target_fd)?;
913
914                let linked_program = SockProgram(program.link(attach_type.get_program_type())?);
915                *self.root_cgroup.get_sock_program(attach_type)?.write(locked) =
916                    Some(linked_program);
917
918                Ok(SUCCESS)
919            }
920
921            (AttachLocation::Kernel, ProgramType::CgroupSockopt) => {
922                check_root_cgroup_fd(locked, current_task, target_fd)?;
923
924                let linked_program = SockOptProgram(program.link(attach_type.get_program_type())?);
925                *self.root_cgroup.get_sock_opt_program(attach_type)?.write(locked) =
926                    Some(linked_program);
927
928                Ok(SUCCESS)
929            }
930
931            (AttachLocation::Kernel, _) => {
932                unreachable!();
933            }
934
935            (AttachLocation::Netstack, _) => {
936                check_root_cgroup_fd(locked, current_task, target_fd)?;
937                self.attach_prog_in_netstack(attach_type, program)
938            }
939        }
940    }
941
942    fn detach_prog(
943        &self,
944        locked: &mut Locked<Unlocked>,
945        current_task: &CurrentTask,
946        attach_type: AttachType,
947        target_fd: FdNumber,
948    ) -> Result<SyscallResult, Errno> {
949        let location = attach_type.try_into()?;
950        let program_type = attach_type.get_program_type();
951        match (location, program_type) {
952            (AttachLocation::Kernel, ProgramType::CgroupSockAddr) => {
953                check_root_cgroup_fd(locked, current_task, target_fd)?;
954
955                let mut prog_guard =
956                    self.root_cgroup.get_sock_addr_program(attach_type)?.write(locked);
957                if prog_guard.is_none() {
958                    return error!(ENOENT);
959                }
960
961                *prog_guard = None;
962
963                Ok(SUCCESS)
964            }
965
966            (AttachLocation::Kernel, ProgramType::CgroupSock) => {
967                check_root_cgroup_fd(locked, current_task, target_fd)?;
968
969                let mut prog_guard = self.root_cgroup.get_sock_program(attach_type)?.write(locked);
970                if prog_guard.is_none() {
971                    return error!(ENOENT);
972                }
973
974                *prog_guard = None;
975
976                Ok(SUCCESS)
977            }
978
979            (AttachLocation::Kernel, ProgramType::CgroupSockopt) => {
980                check_root_cgroup_fd(locked, current_task, target_fd)?;
981
982                let mut prog_guard =
983                    self.root_cgroup.get_sock_opt_program(attach_type)?.write(locked);
984                if prog_guard.is_none() {
985                    return error!(ENOENT);
986                }
987
988                *prog_guard = None;
989
990                Ok(SUCCESS)
991            }
992
993            (AttachLocation::Kernel, _) => {
994                unreachable!();
995            }
996
997            (AttachLocation::Netstack, _) => {
998                check_root_cgroup_fd(locked, current_task, target_fd)?;
999                self.detach_prog_in_netstack(attach_type)
1000            }
1001        }
1002    }
1003
1004    fn attach_prog_in_netstack(
1005        &self,
1006        attach_type: AttachType,
1007        program: ProgramHandle,
1008    ) -> Result<SyscallResult, Errno> {
1009        let hook = attach_type_to_netstack_hook(attach_type).ok_or_else(|| errno!(ENOTSUP))?;
1010        let opts = fnet_filter::AttachEbpfProgramOptions {
1011            hook: Some(hook),
1012            program: Some((&**program).try_into()?),
1013            ..Default::default()
1014        };
1015        self.socket_control()
1016            .attach_ebpf_program(opts, zx::MonotonicInstant::INFINITE)
1017            .map_err(|e| {
1018                log_error!(
1019                    "failed to send fuchsia.net.filter/SocketControl.AttachEbpfProgram: {}",
1020                    e
1021                );
1022                errno!(EIO)
1023            })?
1024            .map_err(|e| {
1025                use fnet_filter::SocketControlAttachEbpfProgramError as Error;
1026                match e {
1027                    Error::NotSupported => errno!(ENOTSUP),
1028                    Error::LinkFailed => errno!(EINVAL),
1029                    Error::MapFailed => errno!(EIO),
1030                    Error::DuplicateAttachment => errno!(EEXIST),
1031                }
1032            })?;
1033
1034        Ok(SUCCESS)
1035    }
1036
1037    fn detach_prog_in_netstack(&self, attach_type: AttachType) -> Result<SyscallResult, Errno> {
1038        let hook = attach_type_to_netstack_hook(attach_type).ok_or_else(|| errno!(ENOTSUP))?;
1039        self.socket_control()
1040            .detach_ebpf_program(hook, zx::MonotonicInstant::INFINITE)
1041            .map_err(|e| {
1042                log_error!(
1043                    "failed to send fuchsia.net.filter/SocketControl.DetachEbpfProgram: {}",
1044                    e
1045                );
1046                errno!(EIO)
1047            })?
1048            .map_err(|e| {
1049                use fnet_filter::SocketControlDetachEbpfProgramError as Error;
1050                match e {
1051                    Error::NotFound => errno!(ENOENT),
1052                }
1053            })?;
1054        Ok(SUCCESS)
1055    }
1056}
starnix_core/bpf/attachments.rs

starnix_core/bpf/
attachments.rs
