ebpf/
executor.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::memio::EbpfPtr;
6use crate::visitor::{BpfVisitor, ProgramCounter, Register, Source};
7use crate::{
8    BPF_STACK_SIZE, BpfValue, DataWidth, EbpfInstruction, EbpfProgramContext, FromBpfValue,
9    GENERAL_REGISTER_COUNT, HelperSet, Packet, VerifierErrorKind,
10};
11use byteorder::{BigEndian, ByteOrder, LittleEndian};
12use std::mem::MaybeUninit;
13use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
14use zerocopy::IntoBytes;
15
16pub fn execute<C: EbpfProgramContext>(
17    code: &[EbpfInstruction],
18    helpers: &HelperSet<C>,
19    run_context: &mut C::RunContext<'_>,
20    arguments: &[BpfValue],
21) -> u64 {
22    assert!(arguments.len() < 5);
23    let mut context = ComputationContext {
24        code,
25        helpers,
26        registers: Default::default(),
27        stack: [MaybeUninit::uninit(); BPF_STACK_SIZE / std::mem::size_of::<BpfValue>()],
28        pc: 0,
29        result: None,
30    };
31    for (i, v) in arguments.iter().enumerate() {
32        // Arguments are in registers r1 to r5.
33        context.set_reg((i as u8) + 1, *v);
34    }
35
36    // R10 points at the stack.
37    context.registers[10] =
38        BpfValue::from((context.stack.as_mut_ptr() as u64) + (BPF_STACK_SIZE as u64));
39
40    loop {
41        if let Some(result) = context.result {
42            return result;
43        }
44        // The program has been verified, so visiting an instruction cannot fail.
45        let Ok(()) = context.visit(run_context, code[context.pc]);
46        context.next();
47    }
48}
49
50impl BpfValue {
51    #[inline(always)]
52    pub fn add(&self, offset: u64) -> Self {
53        Self::from(self.as_u64().overflowing_add(offset).0)
54    }
55}
56
57/// The state of the computation as known by the interpreter at a given point in time.
58struct ComputationContext<'a, C: EbpfProgramContext> {
59    /// The program being executed.
60    code: &'a [EbpfInstruction],
61    /// Helpers.
62    helpers: &'a HelperSet<C>,
63    /// Registers.
64    registers: [BpfValue; GENERAL_REGISTER_COUNT as usize + 1],
65    /// The state of the stack.
66    stack: [MaybeUninit<BpfValue>; BPF_STACK_SIZE / std::mem::size_of::<BpfValue>()],
67    /// The program counter.
68    pc: ProgramCounter,
69    /// The result, set to Some(value) when the program terminates.
70    result: Option<u64>,
71}
72
73// The interpreter is supposed to be called only with bytecode that passed the
74// verifier.
75struct InterpreterError(!);
76
77impl From<VerifierErrorKind> for InterpreterError {
78    #[inline(always)]
79    fn from(error: VerifierErrorKind) -> Self {
80        unreachable!("Invalid code passed to eBPF interpreter: {:?}", error)
81    }
82}
83
84impl<C: EbpfProgramContext> ComputationContext<'_, C> {
85    #[inline(always)]
86    fn reg(&mut self, index: Register) -> BpfValue {
87        self.registers[index as usize]
88    }
89
90    #[inline(always)]
91    fn set_reg(&mut self, index: Register, value: BpfValue) {
92        self.registers[index as usize] = value;
93    }
94
95    #[inline(always)]
96    fn next(&mut self) {
97        self.advance_pc(1)
98    }
99
100    /// Adds `offset` to the program counter in `ComputationContext`.
101    #[inline(always)]
102    fn advance_pc(&mut self, offset: i16) {
103        let mut pc = self.pc as i64;
104        pc += offset as i64;
105        self.pc = pc as usize;
106    }
107
108    #[inline(always)]
109    fn store_memory(
110        &mut self,
111        addr: BpfValue,
112        value: BpfValue,
113        instruction_offset: u64,
114        width: DataWidth,
115    ) {
116        let addr = addr.add(instruction_offset);
117        match width {
118            DataWidth::U8 => {
119                // SAFETY: Verifier ensures that the store operation is safe.
120                unsafe { EbpfPtr::new(addr.as_ptr::<u8>()).store_relaxed(value.as_u8()) }
121            }
122            DataWidth::U16 => {
123                // SAFETY: Verifier ensures that the store operation is safe.
124                unsafe { EbpfPtr::new(addr.as_ptr::<u16>()).store_relaxed(value.as_u16()) }
125            }
126            DataWidth::U32 => {
127                // SAFETY: Verifier ensures that the store operation is safe.
128                unsafe { EbpfPtr::new(addr.as_ptr::<u32>()).store_relaxed(value.as_u32()) }
129            }
130            DataWidth::U64 => {
131                // SAFETY: Verifier ensures that the store operation is safe.
132                unsafe { EbpfPtr::new(addr.as_ptr::<u64>()).store_relaxed(value.as_u64()) }
133            }
134        }
135    }
136
137    #[inline(always)]
138    fn load_memory(&self, addr: BpfValue, instruction_offset: u64, width: DataWidth) -> BpfValue {
139        let addr = addr.add(instruction_offset);
140        match width {
141            DataWidth::U8 => {
142                // SAFETY: Verifier ensures that the load is safe.
143                BpfValue::from(unsafe { EbpfPtr::new(addr.as_ptr::<u8>()).load_relaxed() })
144            }
145            DataWidth::U16 => {
146                // SAFETY: Verifier ensures that the load is safe.
147                BpfValue::from(unsafe { EbpfPtr::new(addr.as_ptr::<u16>()).load_relaxed() })
148            }
149            DataWidth::U32 => {
150                // SAFETY: Verifier ensures that the load is safe.
151                BpfValue::from(unsafe { EbpfPtr::new(addr.as_ptr::<u32>()).load_relaxed() })
152            }
153            DataWidth::U64 => {
154                // SAFETY: Verifier ensures that the load is safe.
155                BpfValue::from(unsafe { EbpfPtr::new(addr.as_ptr::<u64>()).load_relaxed() })
156            }
157        }
158    }
159
160    #[inline(always)]
161    fn compute_source(&mut self, src: Source) -> BpfValue {
162        match src {
163            Source::Reg(reg) => self.reg(reg),
164            Source::Value(v) => v.into(),
165        }
166    }
167
168    #[inline(always)]
169    fn alu(
170        &mut self,
171        dst: Register,
172        src: Source,
173        op: impl Fn(u64, u64) -> u64,
174    ) -> Result<(), InterpreterError> {
175        let op1 = self.reg(dst).as_u64();
176        let op2 = self.compute_source(src).as_u64();
177        let result = op(op1, op2);
178        self.set_reg(dst, result.into());
179        Ok(())
180    }
181
182    #[inline(always)]
183    fn atomic_operation(
184        &mut self,
185        fetch: bool,
186        dst: Register,
187        offset: i16,
188        src: Register,
189        op: impl Fn(&mut Self, &AtomicU32, u32) -> u32,
190    ) -> Result<(), InterpreterError> {
191        let addr = self.reg(dst).add(offset as u64);
192        // SAFETY
193        //
194        // The address has been verified by the verifier that ensured the memory is valid for
195        // reading and writing, and that it is suitably aligned for an atomic access.
196        #[allow(
197            clippy::undocumented_unsafe_blocks,
198            reason = "Force documented unsafe blocks in Starnix"
199        )]
200        let atomic = unsafe { &*addr.as_ptr::<AtomicU32>() };
201        let value = self.reg(src).as_u32();
202        let old_value = op(self, atomic, value);
203        if fetch {
204            self.set_reg(src, old_value.into());
205        }
206        Ok(())
207    }
208
209    #[inline(always)]
210    fn atomic_operation64(
211        &mut self,
212        fetch: bool,
213        dst: Register,
214        offset: i16,
215        src: Register,
216        op: impl Fn(&mut Self, &AtomicU64, u64) -> u64,
217    ) -> Result<(), InterpreterError> {
218        let addr = self.reg(dst).add(offset as u64);
219        // SAFETY
220        //
221        // The address has been verified by the verifier that ensured the memory is valid for
222        // reading and writing, and that it is suitably aligned for an atomic access.
223        #[allow(
224            clippy::undocumented_unsafe_blocks,
225            reason = "Force documented unsafe blocks in Starnix"
226        )]
227        let atomic = unsafe { &*addr.as_ptr::<AtomicU64>() };
228        let value = self.reg(src).as_u64();
229        let old_value = op(self, atomic, value);
230        if fetch {
231            self.set_reg(src, old_value.into());
232        }
233        Ok(())
234    }
235
236    #[inline(always)]
237    fn endianness<BO: ByteOrder>(
238        &mut self,
239        dst: Register,
240        width: DataWidth,
241    ) -> Result<(), InterpreterError> {
242        let value = self.reg(dst);
243        let new_value = match width {
244            DataWidth::U16 => BO::read_u16((value.as_u64() as u16).as_bytes()) as u64,
245            DataWidth::U32 => BO::read_u32((value.as_u64() as u32).as_bytes()) as u64,
246            DataWidth::U64 => BO::read_u64(value.as_u64().as_bytes()),
247            _ => {
248                panic!("Unexpected bit width for endianness operation");
249            }
250        };
251        self.set_reg(dst, new_value.into());
252        Ok(())
253    }
254
255    #[inline(always)]
256    fn conditional_jump(
257        &mut self,
258        dst: Register,
259        src: Source,
260        offset: i16,
261        op: impl Fn(u64, u64) -> bool,
262    ) -> Result<(), InterpreterError> {
263        let op1 = self.reg(dst).as_u64();
264        let op2 = self.compute_source(src.clone()).as_u64();
265        if op(op1, op2) {
266            self.advance_pc(offset);
267        }
268        Ok(())
269    }
270}
271
272impl<C: EbpfProgramContext> BpfVisitor for ComputationContext<'_, C> {
273    type Context<'a> = C::RunContext<'a>;
274    type Error = InterpreterError;
275
276    #[inline(always)]
277    fn add<'a>(
278        &mut self,
279        _context: &mut Self::Context<'a>,
280        dst: Register,
281        src: Source,
282    ) -> Result<(), InterpreterError> {
283        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_add(y).0))
284    }
285    #[inline(always)]
286    fn add64<'a>(
287        &mut self,
288        _context: &mut Self::Context<'a>,
289        dst: Register,
290        src: Source,
291    ) -> Result<(), InterpreterError> {
292        self.alu(dst, src, |x, y| x.overflowing_add(y).0)
293    }
294    #[inline(always)]
295    fn and<'a>(
296        &mut self,
297        _context: &mut Self::Context<'a>,
298        dst: Register,
299        src: Source,
300    ) -> Result<(), InterpreterError> {
301        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x & y))
302    }
303    #[inline(always)]
304    fn and64<'a>(
305        &mut self,
306        _context: &mut Self::Context<'a>,
307        dst: Register,
308        src: Source,
309    ) -> Result<(), InterpreterError> {
310        self.alu(dst, src, |x, y| x & y)
311    }
312    #[inline(always)]
313    fn arsh<'a>(
314        &mut self,
315        _context: &mut Self::Context<'a>,
316        dst: Register,
317        src: Source,
318    ) -> Result<(), InterpreterError> {
319        self.alu(dst, src, |x, y| {
320            alu32(x, y, |x, y| {
321                let x = x as i32;
322                x.overflowing_shr(y).0 as u32
323            })
324        })
325    }
326    #[inline(always)]
327    fn arsh64<'a>(
328        &mut self,
329        _context: &mut Self::Context<'a>,
330        dst: Register,
331        src: Source,
332    ) -> Result<(), InterpreterError> {
333        self.alu(dst, src, |x, y| {
334            let x = x as i64;
335            // ebpf mask shift operation to the number of bytes, as does `overflowing_sh{rl}`
336            // rust operation. So, it is valid to just cast it to `u32`.
337            let y = y as u32;
338            x.overflowing_shr(y).0 as u64
339        })
340    }
341    #[inline(always)]
342    fn div<'a>(
343        &mut self,
344        _context: &mut Self::Context<'a>,
345        dst: Register,
346        src: Source,
347    ) -> Result<(), InterpreterError> {
348        self.alu(dst, src, |x, y| alu32(x, y, |x, y| if y == 0 { 0 } else { x / y }))
349    }
350    #[inline(always)]
351    fn div64<'a>(
352        &mut self,
353        _context: &mut Self::Context<'a>,
354        dst: Register,
355        src: Source,
356    ) -> Result<(), InterpreterError> {
357        self.alu(dst, src, |x, y| if y == 0 { 0 } else { x / y })
358    }
359    #[inline(always)]
360    fn lsh<'a>(
361        &mut self,
362        _context: &mut Self::Context<'a>,
363        dst: Register,
364        src: Source,
365    ) -> Result<(), InterpreterError> {
366        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_shl(y).0))
367    }
368    #[inline(always)]
369    fn lsh64<'a>(
370        &mut self,
371        _context: &mut Self::Context<'a>,
372        dst: Register,
373        src: Source,
374    ) -> Result<(), InterpreterError> {
375        self.alu(dst, src, |x, y| {
376            // ebpf mask shift operation to the number of bytes, as does `overflowing_sh{rl}`
377            // rust operation. So, it is valid to just cast it to `u32`.
378            let y = y as u32;
379            x.overflowing_shl(y).0
380        })
381    }
382    #[inline(always)]
383    fn r#mod<'a>(
384        &mut self,
385        _context: &mut Self::Context<'a>,
386        dst: Register,
387        src: Source,
388    ) -> Result<(), InterpreterError> {
389        self.alu(dst, src, |x, y| alu32(x, y, |x, y| if y == 0 { x } else { x % y }))
390    }
391    #[inline(always)]
392    fn mod64<'a>(
393        &mut self,
394        _context: &mut Self::Context<'a>,
395        dst: Register,
396        src: Source,
397    ) -> Result<(), InterpreterError> {
398        self.alu(dst, src, |x, y| if y == 0 { x } else { x % y })
399    }
400    #[inline(always)]
401    fn mov<'a>(
402        &mut self,
403        _context: &mut Self::Context<'a>,
404        dst: Register,
405        src: Source,
406    ) -> Result<(), InterpreterError> {
407        self.alu(dst, src, |x, y| alu32(x, y, |_x, y| y))
408    }
409    #[inline(always)]
410    fn mov64<'a>(
411        &mut self,
412        _context: &mut Self::Context<'a>,
413        dst: Register,
414        src: Source,
415    ) -> Result<(), InterpreterError> {
416        self.alu(dst, src, |_x, y| y)
417    }
418    #[inline(always)]
419    fn mul<'a>(
420        &mut self,
421        _context: &mut Self::Context<'a>,
422        dst: Register,
423        src: Source,
424    ) -> Result<(), InterpreterError> {
425        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_mul(y).0))
426    }
427    #[inline(always)]
428    fn mul64<'a>(
429        &mut self,
430        _context: &mut Self::Context<'a>,
431        dst: Register,
432        src: Source,
433    ) -> Result<(), InterpreterError> {
434        self.alu(dst, src, |x, y| x.overflowing_mul(y).0)
435    }
436    #[inline(always)]
437    fn or<'a>(
438        &mut self,
439        _context: &mut Self::Context<'a>,
440        dst: Register,
441        src: Source,
442    ) -> Result<(), InterpreterError> {
443        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x | y))
444    }
445    #[inline(always)]
446    fn or64<'a>(
447        &mut self,
448        _context: &mut Self::Context<'a>,
449        dst: Register,
450        src: Source,
451    ) -> Result<(), InterpreterError> {
452        self.alu(dst, src, |x, y| x | y)
453    }
454    #[inline(always)]
455    fn rsh<'a>(
456        &mut self,
457        _context: &mut Self::Context<'a>,
458        dst: Register,
459        src: Source,
460    ) -> Result<(), InterpreterError> {
461        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_shr(y).0))
462    }
463    #[inline(always)]
464    fn rsh64<'a>(
465        &mut self,
466        _context: &mut Self::Context<'a>,
467        dst: Register,
468        src: Source,
469    ) -> Result<(), InterpreterError> {
470        self.alu(dst, src, |x, y| {
471            // ebpf mask shift operation to the number of bytes, as does `overflowing_sh{rl}`
472            // rust operation. So, it is valid to just cast it to `u32`.
473            let y = y as u32;
474            x.overflowing_shr(y).0
475        })
476    }
477    #[inline(always)]
478    fn sub<'a>(
479        &mut self,
480        _context: &mut Self::Context<'a>,
481        dst: Register,
482        src: Source,
483    ) -> Result<(), InterpreterError> {
484        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_sub(y).0))
485    }
486    #[inline(always)]
487    fn sub64<'a>(
488        &mut self,
489        _context: &mut Self::Context<'a>,
490        dst: Register,
491        src: Source,
492    ) -> Result<(), InterpreterError> {
493        self.alu(dst, src, |x, y| x.overflowing_sub(y).0)
494    }
495    #[inline(always)]
496    fn xor<'a>(
497        &mut self,
498        _context: &mut Self::Context<'a>,
499        dst: Register,
500        src: Source,
501    ) -> Result<(), InterpreterError> {
502        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x ^ y))
503    }
504    #[inline(always)]
505    fn xor64<'a>(
506        &mut self,
507        _context: &mut Self::Context<'a>,
508        dst: Register,
509        src: Source,
510    ) -> Result<(), InterpreterError> {
511        self.alu(dst, src, |x, y| x ^ y)
512    }
513
514    #[inline(always)]
515    fn neg<'a>(
516        &mut self,
517        _context: &mut Self::Context<'a>,
518        dst: Register,
519    ) -> Result<(), InterpreterError> {
520        self.alu(dst, Source::Value(0), |x, y| {
521            alu32(x, y, |x, _y| (x as i32).overflowing_neg().0 as u32)
522        })
523    }
524    #[inline(always)]
525    fn neg64<'a>(
526        &mut self,
527        _context: &mut Self::Context<'a>,
528        dst: Register,
529    ) -> Result<(), InterpreterError> {
530        self.alu(dst, Source::Value(0), |x, _y| (x as i64).overflowing_neg().0 as u64)
531    }
532
533    #[inline(always)]
534    fn be<'a>(
535        &mut self,
536        _context: &mut Self::Context<'a>,
537        dst: Register,
538        width: DataWidth,
539    ) -> Result<(), InterpreterError> {
540        self.endianness::<BigEndian>(dst, width)
541    }
542
543    #[inline(always)]
544    fn le<'a>(
545        &mut self,
546        _context: &mut Self::Context<'a>,
547        dst: Register,
548        width: DataWidth,
549    ) -> Result<(), InterpreterError> {
550        self.endianness::<LittleEndian>(dst, width)
551    }
552
553    #[inline(always)]
554    fn call_external<'a>(
555        &mut self,
556        context: &mut Self::Context<'a>,
557        index: u32,
558    ) -> Result<(), InterpreterError> {
559        let helper = &self.helpers.get_by_index(index).unwrap();
560        let result =
561            helper.0(context, self.reg(1), self.reg(2), self.reg(3), self.reg(4), self.reg(5));
562        self.set_reg(0, result);
563        Ok(())
564    }
565
566    #[inline(always)]
567    fn exit<'a>(&mut self, _context: &mut Self::Context<'a>) -> Result<(), InterpreterError> {
568        self.result = Some(self.reg(0).as_u64());
569        Ok(())
570    }
571
572    #[inline(always)]
573    fn jump<'a>(
574        &mut self,
575        _context: &mut Self::Context<'a>,
576        offset: i16,
577    ) -> Result<(), InterpreterError> {
578        self.advance_pc(offset);
579        Ok(())
580    }
581
582    #[inline(always)]
583    fn jeq<'a>(
584        &mut self,
585        _context: &mut Self::Context<'a>,
586        dst: Register,
587        src: Source,
588        offset: i16,
589    ) -> Result<(), InterpreterError> {
590        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x == y))
591    }
592    #[inline(always)]
593    fn jeq64<'a>(
594        &mut self,
595        _context: &mut Self::Context<'a>,
596        dst: Register,
597        src: Source,
598        offset: i16,
599    ) -> Result<(), InterpreterError> {
600        self.conditional_jump(dst, src, offset, |x, y| x == y)
601    }
602    #[inline(always)]
603    fn jne<'a>(
604        &mut self,
605        _context: &mut Self::Context<'a>,
606        dst: Register,
607        src: Source,
608        offset: i16,
609    ) -> Result<(), InterpreterError> {
610        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x != y))
611    }
612    #[inline(always)]
613    fn jne64<'a>(
614        &mut self,
615        _context: &mut Self::Context<'a>,
616        dst: Register,
617        src: Source,
618        offset: i16,
619    ) -> Result<(), InterpreterError> {
620        self.conditional_jump(dst, src, offset, |x, y| x != y)
621    }
622    #[inline(always)]
623    fn jge<'a>(
624        &mut self,
625        _context: &mut Self::Context<'a>,
626        dst: Register,
627        src: Source,
628        offset: i16,
629    ) -> Result<(), InterpreterError> {
630        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x >= y))
631    }
632    #[inline(always)]
633    fn jge64<'a>(
634        &mut self,
635        _context: &mut Self::Context<'a>,
636        dst: Register,
637        src: Source,
638        offset: i16,
639    ) -> Result<(), InterpreterError> {
640        self.conditional_jump(dst, src, offset, |x, y| x >= y)
641    }
642    #[inline(always)]
643    fn jgt<'a>(
644        &mut self,
645        _context: &mut Self::Context<'a>,
646        dst: Register,
647        src: Source,
648        offset: i16,
649    ) -> Result<(), InterpreterError> {
650        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x > y))
651    }
652    #[inline(always)]
653    fn jgt64<'a>(
654        &mut self,
655        _context: &mut Self::Context<'a>,
656        dst: Register,
657        src: Source,
658        offset: i16,
659    ) -> Result<(), InterpreterError> {
660        self.conditional_jump(dst, src, offset, |x, y| x > y)
661    }
662    #[inline(always)]
663    fn jle<'a>(
664        &mut self,
665        _context: &mut Self::Context<'a>,
666        dst: Register,
667        src: Source,
668        offset: i16,
669    ) -> Result<(), InterpreterError> {
670        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x <= y))
671    }
672    #[inline(always)]
673    fn jle64<'a>(
674        &mut self,
675        _context: &mut Self::Context<'a>,
676        dst: Register,
677        src: Source,
678        offset: i16,
679    ) -> Result<(), InterpreterError> {
680        self.conditional_jump(dst, src, offset, |x, y| x <= y)
681    }
682    #[inline(always)]
683    fn jlt<'a>(
684        &mut self,
685        _context: &mut Self::Context<'a>,
686        dst: Register,
687        src: Source,
688        offset: i16,
689    ) -> Result<(), InterpreterError> {
690        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x < y))
691    }
692    #[inline(always)]
693    fn jlt64<'a>(
694        &mut self,
695        _context: &mut Self::Context<'a>,
696        dst: Register,
697        src: Source,
698        offset: i16,
699    ) -> Result<(), InterpreterError> {
700        self.conditional_jump(dst, src, offset, |x, y| x < y)
701    }
702    #[inline(always)]
703    fn jsge<'a>(
704        &mut self,
705        _context: &mut Self::Context<'a>,
706        dst: Register,
707        src: Source,
708        offset: i16,
709    ) -> Result<(), InterpreterError> {
710        self.conditional_jump(dst, src, offset, |x, y| scomp32(x, y, |x, y| x >= y))
711    }
712    #[inline(always)]
713    fn jsge64<'a>(
714        &mut self,
715        _context: &mut Self::Context<'a>,
716        dst: Register,
717        src: Source,
718        offset: i16,
719    ) -> Result<(), InterpreterError> {
720        self.conditional_jump(dst, src, offset, |x, y| scomp64(x, y, |x, y| x >= y))
721    }
722    #[inline(always)]
723    fn jsgt<'a>(
724        &mut self,
725        _context: &mut Self::Context<'a>,
726        dst: Register,
727        src: Source,
728        offset: i16,
729    ) -> Result<(), InterpreterError> {
730        self.conditional_jump(dst, src, offset, |x, y| scomp32(x, y, |x, y| x > y))
731    }
732    #[inline(always)]
733    fn jsgt64<'a>(
734        &mut self,
735        _context: &mut Self::Context<'a>,
736        dst: Register,
737        src: Source,
738        offset: i16,
739    ) -> Result<(), InterpreterError> {
740        self.conditional_jump(dst, src, offset, |x, y| scomp64(x, y, |x, y| x > y))
741    }
742    #[inline(always)]
743    fn jsle<'a>(
744        &mut self,
745        _context: &mut Self::Context<'a>,
746        dst: Register,
747        src: Source,
748        offset: i16,
749    ) -> Result<(), InterpreterError> {
750        self.conditional_jump(dst, src, offset, |x, y| scomp32(x, y, |x, y| x <= y))
751    }
752    #[inline(always)]
753    fn jsle64<'a>(
754        &mut self,
755        _context: &mut Self::Context<'a>,
756        dst: Register,
757        src: Source,
758        offset: i16,
759    ) -> Result<(), InterpreterError> {
760        self.conditional_jump(dst, src, offset, |x, y| scomp64(x, y, |x, y| x <= y))
761    }
762    #[inline(always)]
763    fn jslt<'a>(
764        &mut self,
765        _context: &mut Self::Context<'a>,
766        dst: Register,
767        src: Source,
768        offset: i16,
769    ) -> Result<(), InterpreterError> {
770        self.conditional_jump(dst, src, offset, |x, y| scomp32(x, y, |x, y| x < y))
771    }
772    #[inline(always)]
773    fn jslt64<'a>(
774        &mut self,
775        _context: &mut Self::Context<'a>,
776        dst: Register,
777        src: Source,
778        offset: i16,
779    ) -> Result<(), InterpreterError> {
780        self.conditional_jump(dst, src, offset, |x, y| scomp64(x, y, |x, y| x < y))
781    }
782    #[inline(always)]
783    fn jset<'a>(
784        &mut self,
785        _context: &mut Self::Context<'a>,
786        dst: Register,
787        src: Source,
788        offset: i16,
789    ) -> Result<(), InterpreterError> {
790        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x & y != 0))
791    }
792    #[inline(always)]
793    fn jset64<'a>(
794        &mut self,
795        _context: &mut Self::Context<'a>,
796        dst: Register,
797        src: Source,
798        offset: i16,
799    ) -> Result<(), InterpreterError> {
800        self.conditional_jump(dst, src, offset, |x, y| x & y != 0)
801    }
802
803    #[inline(always)]
804    fn atomic_add<'a>(
805        &mut self,
806        _context: &mut Self::Context<'a>,
807        fetch: bool,
808        dst: Register,
809        offset: i16,
810        src: Register,
811    ) -> Result<(), InterpreterError> {
812        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.fetch_add(v, Ordering::SeqCst))
813    }
814
815    #[inline(always)]
816    fn atomic_add64<'a>(
817        &mut self,
818        _context: &mut Self::Context<'a>,
819        fetch: bool,
820        dst: Register,
821        offset: i16,
822        src: Register,
823    ) -> Result<(), InterpreterError> {
824        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.fetch_add(v, Ordering::SeqCst))
825    }
826
827    #[inline(always)]
828    fn atomic_and<'a>(
829        &mut self,
830        _context: &mut Self::Context<'a>,
831        fetch: bool,
832        dst: Register,
833        offset: i16,
834        src: Register,
835    ) -> Result<(), InterpreterError> {
836        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.fetch_and(v, Ordering::SeqCst))
837    }
838
839    #[inline(always)]
840    fn atomic_and64<'a>(
841        &mut self,
842        _context: &mut Self::Context<'a>,
843        fetch: bool,
844        dst: Register,
845        offset: i16,
846        src: Register,
847    ) -> Result<(), InterpreterError> {
848        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.fetch_and(v, Ordering::SeqCst))
849    }
850
851    #[inline(always)]
852    fn atomic_or<'a>(
853        &mut self,
854        _context: &mut Self::Context<'a>,
855        fetch: bool,
856        dst: Register,
857        offset: i16,
858        src: Register,
859    ) -> Result<(), InterpreterError> {
860        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.fetch_or(v, Ordering::SeqCst))
861    }
862
863    #[inline(always)]
864    fn atomic_or64<'a>(
865        &mut self,
866        _context: &mut Self::Context<'a>,
867        fetch: bool,
868        dst: Register,
869        offset: i16,
870        src: Register,
871    ) -> Result<(), InterpreterError> {
872        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.fetch_or(v, Ordering::SeqCst))
873    }
874
875    #[inline(always)]
876    fn atomic_xor<'a>(
877        &mut self,
878        _context: &mut Self::Context<'a>,
879        fetch: bool,
880        dst: Register,
881        offset: i16,
882        src: Register,
883    ) -> Result<(), InterpreterError> {
884        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.fetch_xor(v, Ordering::SeqCst))
885    }
886
887    #[inline(always)]
888    fn atomic_xor64<'a>(
889        &mut self,
890        _context: &mut Self::Context<'a>,
891        fetch: bool,
892        dst: Register,
893        offset: i16,
894        src: Register,
895    ) -> Result<(), InterpreterError> {
896        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.fetch_xor(v, Ordering::SeqCst))
897    }
898
899    #[inline(always)]
900    fn atomic_xchg<'a>(
901        &mut self,
902        _context: &mut Self::Context<'a>,
903        fetch: bool,
904        dst: Register,
905        offset: i16,
906        src: Register,
907    ) -> Result<(), InterpreterError> {
908        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.swap(v, Ordering::SeqCst))
909    }
910
911    #[inline(always)]
912    fn atomic_xchg64<'a>(
913        &mut self,
914        _context: &mut Self::Context<'a>,
915        fetch: bool,
916        dst: Register,
917        offset: i16,
918        src: Register,
919    ) -> Result<(), InterpreterError> {
920        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.swap(v, Ordering::SeqCst))
921    }
922
923    #[inline(always)]
924    fn atomic_cmpxchg<'a>(
925        &mut self,
926        _context: &mut Self::Context<'a>,
927        dst: Register,
928        offset: i16,
929        src: Register,
930    ) -> Result<(), InterpreterError> {
931        self.atomic_operation(false, dst, offset, src, |this, a, v| {
932            let r0 = this.reg(0).as_u32();
933            let r0 = match a.compare_exchange(r0, v, Ordering::SeqCst, Ordering::SeqCst) {
934                Ok(v) | Err(v) => v,
935            };
936            this.set_reg(0, r0.into());
937            0
938        })
939    }
940
941    #[inline(always)]
942    fn atomic_cmpxchg64<'a>(
943        &mut self,
944        _context: &mut Self::Context<'a>,
945        dst: Register,
946        offset: i16,
947        src: Register,
948    ) -> Result<(), InterpreterError> {
949        self.atomic_operation64(false, dst, offset, src, |this, a, v| {
950            let r0 = this.reg(0).as_u64();
951            let r0 = match a.compare_exchange(r0, v, Ordering::SeqCst, Ordering::SeqCst) {
952                Ok(v) | Err(v) => v,
953            };
954            this.set_reg(0, r0.into());
955            0
956        })
957    }
958
959    #[inline(always)]
960    fn load<'a>(
961        &mut self,
962        _context: &mut Self::Context<'a>,
963        dst: Register,
964        offset: i16,
965        src: Register,
966        width: DataWidth,
967    ) -> Result<(), InterpreterError> {
968        let addr = self.reg(src);
969        let loaded = self.load_memory(addr, offset as u64, width);
970        self.set_reg(dst, loaded);
971        Ok(())
972    }
973
974    #[inline(always)]
975    fn load64<'a>(
976        &mut self,
977        _context: &mut Self::Context<'a>,
978        dst: Register,
979        _src: u8,
980        lower: u32,
981    ) -> Result<(), InterpreterError> {
982        let value = (lower as u64) | (((self.code[self.pc + 1].imm() as u32) as u64) << 32);
983        self.set_reg(dst, value.into());
984        self.advance_pc(1);
985        Ok(())
986    }
987
988    #[inline(always)]
989    fn load_from_packet<'a>(
990        &mut self,
991        context: &mut Self::Context<'a>,
992        dst_reg: Register,
993        src_reg: Register,
994        offset: i32,
995        register_offset: Option<Register>,
996        width: DataWidth,
997    ) -> Result<(), InterpreterError> {
998        let Some(offset) =
999            register_offset.map(|r| self.reg(r).as_i32()).unwrap_or(0).checked_add(offset as i32)
1000        else {
1001            // Offset overflowed. Exit.
1002            self.result = Some(0);
1003            return Ok(());
1004        };
1005        let src_reg = self.reg(src_reg);
1006        // SAFETY: The verifier checks that the `src_reg` points at packet.
1007        let packet = unsafe { C::Packet::from_bpf_value(context, src_reg) };
1008        if let Some(value) = packet.load(offset, width) {
1009            self.set_reg(dst_reg, value.into());
1010        } else {
1011            self.result = Some(0);
1012        }
1013        Ok(())
1014    }
1015
1016    #[inline(always)]
1017    fn store<'a>(
1018        &mut self,
1019        _context: &mut Self::Context<'a>,
1020        dst: Register,
1021        offset: i16,
1022        src: Source,
1023        width: DataWidth,
1024    ) -> Result<(), InterpreterError> {
1025        let src = self.compute_source(src);
1026        let dst = self.reg(dst);
1027        self.store_memory(dst, src, offset as u64, width);
1028        Ok(())
1029    }
1030}
1031
1032#[inline(always)]
1033fn alu32(x: u64, y: u64, op: impl FnOnce(u32, u32) -> u32) -> u64 {
1034    op(x as u32, y as u32) as u64
1035}
1036
1037#[inline(always)]
1038fn comp32(x: u64, y: u64, op: impl FnOnce(u32, u32) -> bool) -> bool {
1039    op(x as u32, y as u32)
1040}
1041
1042#[inline(always)]
1043fn scomp64(x: u64, y: u64, op: impl FnOnce(i64, i64) -> bool) -> bool {
1044    op(x as i64, y as i64)
1045}
1046
1047#[inline(always)]
1048fn scomp32(x: u64, y: u64, op: impl FnOnce(i32, i32) -> bool) -> bool {
1049    op(x as i32, y as i32)
1050}
ebpf/executor.rs

ebpf/
executor.rs
