ebpf/
executor.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::memio::EbpfPtr;
6use crate::visitor::{BpfVisitor, ProgramCounter, Register, Source};
7use crate::{
8    BPF_STACK_SIZE, BpfValue, DataWidth, EbpfInstruction, EbpfProgramContext, FromBpfValue,
9    GENERAL_REGISTER_COUNT, HelperSet, Packet,
10};
11use byteorder::{BigEndian, ByteOrder, LittleEndian};
12use std::mem::MaybeUninit;
13use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
14use zerocopy::IntoBytes;
15
16pub fn execute<C: EbpfProgramContext>(
17    code: &[EbpfInstruction],
18    helpers: &HelperSet<C>,
19    run_context: &mut C::RunContext<'_>,
20    arguments: &[BpfValue],
21) -> u64 {
22    assert!(arguments.len() < 5);
23    let mut context = ComputationContext {
24        code,
25        helpers,
26        registers: Default::default(),
27        stack: [MaybeUninit::uninit(); BPF_STACK_SIZE / std::mem::size_of::<BpfValue>()],
28        pc: 0,
29        result: None,
30    };
31    for (i, v) in arguments.iter().enumerate() {
32        // Arguments are in registers r1 to r5.
33        context.set_reg((i as u8) + 1, *v);
34    }
35
36    // R10 points at the stack.
37    context.registers[10] =
38        BpfValue::from((context.stack.as_mut_ptr() as u64) + (BPF_STACK_SIZE as u64));
39
40    loop {
41        if let Some(result) = context.result {
42            return result;
43        }
44        context.visit(run_context, code[context.pc]).expect("verifier should have found an issue");
45        context.next();
46    }
47}
48
49impl BpfValue {
50    #[inline(always)]
51    pub fn add(&self, offset: u64) -> Self {
52        Self::from(self.as_u64().overflowing_add(offset).0)
53    }
54}
55
56/// The state of the computation as known by the interpreter at a given point in time.
57struct ComputationContext<'a, C: EbpfProgramContext> {
58    /// The program being executed.
59    code: &'a [EbpfInstruction],
60    /// Helpers.
61    helpers: &'a HelperSet<C>,
62    /// Registers.
63    registers: [BpfValue; GENERAL_REGISTER_COUNT as usize + 1],
64    /// The state of the stack.
65    stack: [MaybeUninit<BpfValue>; BPF_STACK_SIZE / std::mem::size_of::<BpfValue>()],
66    /// The program counter.
67    pc: ProgramCounter,
68    /// The result, set to Some(value) when the program terminates.
69    result: Option<u64>,
70}
71
72impl<C: EbpfProgramContext> ComputationContext<'_, C> {
73    #[inline(always)]
74    fn reg(&mut self, index: Register) -> BpfValue {
75        self.registers[index as usize]
76    }
77
78    #[inline(always)]
79    fn set_reg(&mut self, index: Register, value: BpfValue) {
80        self.registers[index as usize] = value;
81    }
82
83    #[inline(always)]
84    fn next(&mut self) {
85        self.advance_pc(1)
86    }
87
88    /// Adds `offset` to the program counter in `ComputationContext`.
89    #[inline(always)]
90    fn advance_pc(&mut self, offset: i16) {
91        let mut pc = self.pc as i64;
92        pc += offset as i64;
93        self.pc = pc as usize;
94    }
95
96    #[inline(always)]
97    fn store_memory(
98        &mut self,
99        addr: BpfValue,
100        value: BpfValue,
101        instruction_offset: u64,
102        width: DataWidth,
103    ) {
104        let addr = addr.add(instruction_offset);
105        match width {
106            DataWidth::U8 => {
107                // SAFETY: Verifier ensures that the store operation is safe.
108                unsafe { EbpfPtr::new(addr.as_ptr::<u8>()).store_relaxed(value.as_u8()) }
109            }
110            DataWidth::U16 => {
111                // SAFETY: Verifier ensures that the store operation is safe.
112                unsafe { EbpfPtr::new(addr.as_ptr::<u16>()).store_relaxed(value.as_u16()) }
113            }
114            DataWidth::U32 => {
115                // SAFETY: Verifier ensures that the store operation is safe.
116                unsafe { EbpfPtr::new(addr.as_ptr::<u32>()).store_relaxed(value.as_u32()) }
117            }
118            DataWidth::U64 => {
119                // SAFETY: Verifier ensures that the store operation is safe.
120                unsafe { EbpfPtr::new(addr.as_ptr::<u64>()).store_relaxed(value.as_u64()) }
121            }
122        }
123    }
124
125    #[inline(always)]
126    fn load_memory(&self, addr: BpfValue, instruction_offset: u64, width: DataWidth) -> BpfValue {
127        let addr = addr.add(instruction_offset);
128        match width {
129            DataWidth::U8 => {
130                // SAFETY: Verifier ensures that the load is safe.
131                BpfValue::from(unsafe { EbpfPtr::new(addr.as_ptr::<u8>()).load_relaxed() })
132            }
133            DataWidth::U16 => {
134                // SAFETY: Verifier ensures that the load is safe.
135                BpfValue::from(unsafe { EbpfPtr::new(addr.as_ptr::<u16>()).load_relaxed() })
136            }
137            DataWidth::U32 => {
138                // SAFETY: Verifier ensures that the load is safe.
139                BpfValue::from(unsafe { EbpfPtr::new(addr.as_ptr::<u32>()).load_relaxed() })
140            }
141            DataWidth::U64 => {
142                // SAFETY: Verifier ensures that the load is safe.
143                BpfValue::from(unsafe { EbpfPtr::new(addr.as_ptr::<u64>()).load_relaxed() })
144            }
145        }
146    }
147
148    #[inline(always)]
149    fn compute_source(&mut self, src: Source) -> BpfValue {
150        match src {
151            Source::Reg(reg) => self.reg(reg),
152            Source::Value(v) => v.into(),
153        }
154    }
155
156    #[inline(always)]
157    fn alu(
158        &mut self,
159        dst: Register,
160        src: Source,
161        op: impl Fn(u64, u64) -> u64,
162    ) -> Result<(), String> {
163        let op1 = self.reg(dst).as_u64();
164        let op2 = self.compute_source(src).as_u64();
165        let result = op(op1, op2);
166        self.set_reg(dst, result.into());
167        Ok(())
168    }
169
170    #[inline(always)]
171    fn atomic_operation(
172        &mut self,
173        fetch: bool,
174        dst: Register,
175        offset: i16,
176        src: Register,
177        op: impl Fn(&mut Self, &AtomicU32, u32) -> u32,
178    ) -> Result<(), String> {
179        let addr = self.reg(dst).add(offset as u64);
180        // TODO How to statically check alignment?
181        if addr.as_usize() % std::mem::size_of::<AtomicU32>() != 0 {
182            return Err(format!("misaligned access"));
183        }
184        // SAFETY
185        //
186        // The address has been verified by the verifier that ensured the memory is valid for
187        // reading and writing.
188        #[allow(
189            clippy::undocumented_unsafe_blocks,
190            reason = "Force documented unsafe blocks in Starnix"
191        )]
192        let atomic = unsafe { &*addr.as_ptr::<AtomicU32>() };
193        let value = self.reg(src).as_u32();
194        let old_value = op(self, atomic, value);
195        if fetch {
196            self.set_reg(src, old_value.into());
197        }
198        Ok(())
199    }
200
201    #[inline(always)]
202    fn atomic_operation64(
203        &mut self,
204        fetch: bool,
205        dst: Register,
206        offset: i16,
207        src: Register,
208        op: impl Fn(&mut Self, &AtomicU64, u64) -> u64,
209    ) -> Result<(), String> {
210        let addr = self.reg(dst).add(offset as u64);
211        // TODO How to statically check alignment?
212        if addr.as_usize() % std::mem::size_of::<AtomicU64>() != 0 {
213            return Err(format!("misaligned access"));
214        }
215        // SAFETY
216        //
217        // The address has been verified by the verifier that ensured the memory is valid for
218        // reading and writing.
219        #[allow(
220            clippy::undocumented_unsafe_blocks,
221            reason = "Force documented unsafe blocks in Starnix"
222        )]
223        let atomic = unsafe { &*addr.as_ptr::<AtomicU64>() };
224        let value = self.reg(src).as_u64();
225        let old_value = op(self, atomic, value);
226        if fetch {
227            self.set_reg(src, old_value.into());
228        }
229        Ok(())
230    }
231
232    #[inline(always)]
233    fn endianness<BO: ByteOrder>(&mut self, dst: Register, width: DataWidth) -> Result<(), String> {
234        let value = self.reg(dst);
235        let new_value = match width {
236            DataWidth::U16 => BO::read_u16((value.as_u64() as u16).as_bytes()) as u64,
237            DataWidth::U32 => BO::read_u32((value.as_u64() as u32).as_bytes()) as u64,
238            DataWidth::U64 => BO::read_u64(value.as_u64().as_bytes()),
239            _ => {
240                panic!("Unexpected bit width for endianness operation");
241            }
242        };
243        self.set_reg(dst, new_value.into());
244        Ok(())
245    }
246
247    #[inline(always)]
248    fn conditional_jump(
249        &mut self,
250        dst: Register,
251        src: Source,
252        offset: i16,
253        op: impl Fn(u64, u64) -> bool,
254    ) -> Result<(), String> {
255        let op1 = self.reg(dst).as_u64();
256        let op2 = self.compute_source(src.clone()).as_u64();
257        if op(op1, op2) {
258            self.advance_pc(offset);
259        }
260        Ok(())
261    }
262}
263
264impl<C: EbpfProgramContext> BpfVisitor for ComputationContext<'_, C> {
265    type Context<'a> = C::RunContext<'a>;
266
267    #[inline(always)]
268    fn add<'a>(
269        &mut self,
270        _context: &mut Self::Context<'a>,
271        dst: Register,
272        src: Source,
273    ) -> Result<(), String> {
274        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_add(y).0))
275    }
276    #[inline(always)]
277    fn add64<'a>(
278        &mut self,
279        _context: &mut Self::Context<'a>,
280        dst: Register,
281        src: Source,
282    ) -> Result<(), String> {
283        self.alu(dst, src, |x, y| x.overflowing_add(y).0)
284    }
285    #[inline(always)]
286    fn and<'a>(
287        &mut self,
288        _context: &mut Self::Context<'a>,
289        dst: Register,
290        src: Source,
291    ) -> Result<(), String> {
292        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x & y))
293    }
294    #[inline(always)]
295    fn and64<'a>(
296        &mut self,
297        _context: &mut Self::Context<'a>,
298        dst: Register,
299        src: Source,
300    ) -> Result<(), String> {
301        self.alu(dst, src, |x, y| x & y)
302    }
303    #[inline(always)]
304    fn arsh<'a>(
305        &mut self,
306        _context: &mut Self::Context<'a>,
307        dst: Register,
308        src: Source,
309    ) -> Result<(), String> {
310        self.alu(dst, src, |x, y| {
311            alu32(x, y, |x, y| {
312                let x = x as i32;
313                x.overflowing_shr(y).0 as u32
314            })
315        })
316    }
317    #[inline(always)]
318    fn arsh64<'a>(
319        &mut self,
320        _context: &mut Self::Context<'a>,
321        dst: Register,
322        src: Source,
323    ) -> Result<(), String> {
324        self.alu(dst, src, |x, y| {
325            let x = x as i64;
326            // ebpf mask shift operation to the number of bytes, as does `overflowing_sh{rl}`
327            // rust operation. So, it is valid to just cast it to `u32`.
328            let y = y as u32;
329            x.overflowing_shr(y).0 as u64
330        })
331    }
332    #[inline(always)]
333    fn div<'a>(
334        &mut self,
335        _context: &mut Self::Context<'a>,
336        dst: Register,
337        src: Source,
338    ) -> Result<(), String> {
339        self.alu(dst, src, |x, y| alu32(x, y, |x, y| if y == 0 { 0 } else { x / y }))
340    }
341    #[inline(always)]
342    fn div64<'a>(
343        &mut self,
344        _context: &mut Self::Context<'a>,
345        dst: Register,
346        src: Source,
347    ) -> Result<(), String> {
348        self.alu(dst, src, |x, y| if y == 0 { 0 } else { x / y })
349    }
350    #[inline(always)]
351    fn lsh<'a>(
352        &mut self,
353        _context: &mut Self::Context<'a>,
354        dst: Register,
355        src: Source,
356    ) -> Result<(), String> {
357        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_shl(y).0))
358    }
359    #[inline(always)]
360    fn lsh64<'a>(
361        &mut self,
362        _context: &mut Self::Context<'a>,
363        dst: Register,
364        src: Source,
365    ) -> Result<(), String> {
366        self.alu(dst, src, |x, y| {
367            // ebpf mask shift operation to the number of bytes, as does `overflowing_sh{rl}`
368            // rust operation. So, it is valid to just cast it to `u32`.
369            let y = y as u32;
370            x.overflowing_shl(y).0
371        })
372    }
373    #[inline(always)]
374    fn r#mod<'a>(
375        &mut self,
376        _context: &mut Self::Context<'a>,
377        dst: Register,
378        src: Source,
379    ) -> Result<(), String> {
380        self.alu(dst, src, |x, y| alu32(x, y, |x, y| if y == 0 { x } else { x % y }))
381    }
382    #[inline(always)]
383    fn mod64<'a>(
384        &mut self,
385        _context: &mut Self::Context<'a>,
386        dst: Register,
387        src: Source,
388    ) -> Result<(), String> {
389        self.alu(dst, src, |x, y| if y == 0 { x } else { x % y })
390    }
391    #[inline(always)]
392    fn mov<'a>(
393        &mut self,
394        _context: &mut Self::Context<'a>,
395        dst: Register,
396        src: Source,
397    ) -> Result<(), String> {
398        self.alu(dst, src, |x, y| alu32(x, y, |_x, y| y))
399    }
400    #[inline(always)]
401    fn mov64<'a>(
402        &mut self,
403        _context: &mut Self::Context<'a>,
404        dst: Register,
405        src: Source,
406    ) -> Result<(), String> {
407        self.alu(dst, src, |_x, y| y)
408    }
409    #[inline(always)]
410    fn mul<'a>(
411        &mut self,
412        _context: &mut Self::Context<'a>,
413        dst: Register,
414        src: Source,
415    ) -> Result<(), String> {
416        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_mul(y).0))
417    }
418    #[inline(always)]
419    fn mul64<'a>(
420        &mut self,
421        _context: &mut Self::Context<'a>,
422        dst: Register,
423        src: Source,
424    ) -> Result<(), String> {
425        self.alu(dst, src, |x, y| x.overflowing_mul(y).0)
426    }
427    #[inline(always)]
428    fn or<'a>(
429        &mut self,
430        _context: &mut Self::Context<'a>,
431        dst: Register,
432        src: Source,
433    ) -> Result<(), String> {
434        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x | y))
435    }
436    #[inline(always)]
437    fn or64<'a>(
438        &mut self,
439        _context: &mut Self::Context<'a>,
440        dst: Register,
441        src: Source,
442    ) -> Result<(), String> {
443        self.alu(dst, src, |x, y| x | y)
444    }
445    #[inline(always)]
446    fn rsh<'a>(
447        &mut self,
448        _context: &mut Self::Context<'a>,
449        dst: Register,
450        src: Source,
451    ) -> Result<(), String> {
452        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_shr(y).0))
453    }
454    #[inline(always)]
455    fn rsh64<'a>(
456        &mut self,
457        _context: &mut Self::Context<'a>,
458        dst: Register,
459        src: Source,
460    ) -> Result<(), String> {
461        self.alu(dst, src, |x, y| {
462            // ebpf mask shift operation to the number of bytes, as does `overflowing_sh{rl}`
463            // rust operation. So, it is valid to just cast it to `u32`.
464            let y = y as u32;
465            x.overflowing_shr(y).0
466        })
467    }
468    #[inline(always)]
469    fn sub<'a>(
470        &mut self,
471        _context: &mut Self::Context<'a>,
472        dst: Register,
473        src: Source,
474    ) -> Result<(), String> {
475        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x.overflowing_sub(y).0))
476    }
477    #[inline(always)]
478    fn sub64<'a>(
479        &mut self,
480        _context: &mut Self::Context<'a>,
481        dst: Register,
482        src: Source,
483    ) -> Result<(), String> {
484        self.alu(dst, src, |x, y| x.overflowing_sub(y).0)
485    }
486    #[inline(always)]
487    fn xor<'a>(
488        &mut self,
489        _context: &mut Self::Context<'a>,
490        dst: Register,
491        src: Source,
492    ) -> Result<(), String> {
493        self.alu(dst, src, |x, y| alu32(x, y, |x, y| x ^ y))
494    }
495    #[inline(always)]
496    fn xor64<'a>(
497        &mut self,
498        _context: &mut Self::Context<'a>,
499        dst: Register,
500        src: Source,
501    ) -> Result<(), String> {
502        self.alu(dst, src, |x, y| x ^ y)
503    }
504
505    #[inline(always)]
506    fn neg<'a>(&mut self, _context: &mut Self::Context<'a>, dst: Register) -> Result<(), String> {
507        self.alu(dst, Source::Value(0), |x, y| {
508            alu32(x, y, |x, _y| (x as i32).overflowing_neg().0 as u32)
509        })
510    }
511    #[inline(always)]
512    fn neg64<'a>(&mut self, _context: &mut Self::Context<'a>, dst: Register) -> Result<(), String> {
513        self.alu(dst, Source::Value(0), |x, _y| (x as i64).overflowing_neg().0 as u64)
514    }
515
516    #[inline(always)]
517    fn be<'a>(
518        &mut self,
519        _context: &mut Self::Context<'a>,
520        dst: Register,
521        width: DataWidth,
522    ) -> Result<(), String> {
523        self.endianness::<BigEndian>(dst, width)
524    }
525
526    #[inline(always)]
527    fn le<'a>(
528        &mut self,
529        _context: &mut Self::Context<'a>,
530        dst: Register,
531        width: DataWidth,
532    ) -> Result<(), String> {
533        self.endianness::<LittleEndian>(dst, width)
534    }
535
536    #[inline(always)]
537    fn call_external<'a>(
538        &mut self,
539        context: &mut Self::Context<'a>,
540        index: u32,
541    ) -> Result<(), String> {
542        let helper = &self.helpers.get_by_index(index).unwrap();
543        let result =
544            helper.0(context, self.reg(1), self.reg(2), self.reg(3), self.reg(4), self.reg(5));
545        self.set_reg(0, result);
546        Ok(())
547    }
548
549    #[inline(always)]
550    fn exit<'a>(&mut self, _context: &mut Self::Context<'a>) -> Result<(), String> {
551        self.result = Some(self.reg(0).as_u64());
552        Ok(())
553    }
554
555    #[inline(always)]
556    fn jump<'a>(&mut self, _context: &mut Self::Context<'a>, offset: i16) -> Result<(), String> {
557        self.advance_pc(offset);
558        Ok(())
559    }
560
561    #[inline(always)]
562    fn jeq<'a>(
563        &mut self,
564        _context: &mut Self::Context<'a>,
565        dst: Register,
566        src: Source,
567        offset: i16,
568    ) -> Result<(), String> {
569        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x == y))
570    }
571    #[inline(always)]
572    fn jeq64<'a>(
573        &mut self,
574        _context: &mut Self::Context<'a>,
575        dst: Register,
576        src: Source,
577        offset: i16,
578    ) -> Result<(), String> {
579        self.conditional_jump(dst, src, offset, |x, y| x == y)
580    }
581    #[inline(always)]
582    fn jne<'a>(
583        &mut self,
584        _context: &mut Self::Context<'a>,
585        dst: Register,
586        src: Source,
587        offset: i16,
588    ) -> Result<(), String> {
589        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x != y))
590    }
591    #[inline(always)]
592    fn jne64<'a>(
593        &mut self,
594        _context: &mut Self::Context<'a>,
595        dst: Register,
596        src: Source,
597        offset: i16,
598    ) -> Result<(), String> {
599        self.conditional_jump(dst, src, offset, |x, y| x != y)
600    }
601    #[inline(always)]
602    fn jge<'a>(
603        &mut self,
604        _context: &mut Self::Context<'a>,
605        dst: Register,
606        src: Source,
607        offset: i16,
608    ) -> Result<(), String> {
609        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x >= y))
610    }
611    #[inline(always)]
612    fn jge64<'a>(
613        &mut self,
614        _context: &mut Self::Context<'a>,
615        dst: Register,
616        src: Source,
617        offset: i16,
618    ) -> Result<(), String> {
619        self.conditional_jump(dst, src, offset, |x, y| x >= y)
620    }
621    #[inline(always)]
622    fn jgt<'a>(
623        &mut self,
624        _context: &mut Self::Context<'a>,
625        dst: Register,
626        src: Source,
627        offset: i16,
628    ) -> Result<(), String> {
629        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x > y))
630    }
631    #[inline(always)]
632    fn jgt64<'a>(
633        &mut self,
634        _context: &mut Self::Context<'a>,
635        dst: Register,
636        src: Source,
637        offset: i16,
638    ) -> Result<(), String> {
639        self.conditional_jump(dst, src, offset, |x, y| x > y)
640    }
641    #[inline(always)]
642    fn jle<'a>(
643        &mut self,
644        _context: &mut Self::Context<'a>,
645        dst: Register,
646        src: Source,
647        offset: i16,
648    ) -> Result<(), String> {
649        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x <= y))
650    }
651    #[inline(always)]
652    fn jle64<'a>(
653        &mut self,
654        _context: &mut Self::Context<'a>,
655        dst: Register,
656        src: Source,
657        offset: i16,
658    ) -> Result<(), String> {
659        self.conditional_jump(dst, src, offset, |x, y| x <= y)
660    }
661    #[inline(always)]
662    fn jlt<'a>(
663        &mut self,
664        _context: &mut Self::Context<'a>,
665        dst: Register,
666        src: Source,
667        offset: i16,
668    ) -> Result<(), String> {
669        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x < y))
670    }
671    #[inline(always)]
672    fn jlt64<'a>(
673        &mut self,
674        _context: &mut Self::Context<'a>,
675        dst: Register,
676        src: Source,
677        offset: i16,
678    ) -> Result<(), String> {
679        self.conditional_jump(dst, src, offset, |x, y| x < y)
680    }
681    #[inline(always)]
682    fn jsge<'a>(
683        &mut self,
684        _context: &mut Self::Context<'a>,
685        dst: Register,
686        src: Source,
687        offset: i16,
688    ) -> Result<(), String> {
689        self.conditional_jump(dst, src, offset, |x, y| scomp32(x, y, |x, y| x >= y))
690    }
691    #[inline(always)]
692    fn jsge64<'a>(
693        &mut self,
694        _context: &mut Self::Context<'a>,
695        dst: Register,
696        src: Source,
697        offset: i16,
698    ) -> Result<(), String> {
699        self.conditional_jump(dst, src, offset, |x, y| scomp64(x, y, |x, y| x >= y))
700    }
701    #[inline(always)]
702    fn jsgt<'a>(
703        &mut self,
704        _context: &mut Self::Context<'a>,
705        dst: Register,
706        src: Source,
707        offset: i16,
708    ) -> Result<(), String> {
709        self.conditional_jump(dst, src, offset, |x, y| scomp32(x, y, |x, y| x > y))
710    }
711    #[inline(always)]
712    fn jsgt64<'a>(
713        &mut self,
714        _context: &mut Self::Context<'a>,
715        dst: Register,
716        src: Source,
717        offset: i16,
718    ) -> Result<(), String> {
719        self.conditional_jump(dst, src, offset, |x, y| scomp64(x, y, |x, y| x > y))
720    }
721    #[inline(always)]
722    fn jsle<'a>(
723        &mut self,
724        _context: &mut Self::Context<'a>,
725        dst: Register,
726        src: Source,
727        offset: i16,
728    ) -> Result<(), String> {
729        self.conditional_jump(dst, src, offset, |x, y| scomp32(x, y, |x, y| x <= y))
730    }
731    #[inline(always)]
732    fn jsle64<'a>(
733        &mut self,
734        _context: &mut Self::Context<'a>,
735        dst: Register,
736        src: Source,
737        offset: i16,
738    ) -> Result<(), String> {
739        self.conditional_jump(dst, src, offset, |x, y| scomp64(x, y, |x, y| x <= y))
740    }
741    #[inline(always)]
742    fn jslt<'a>(
743        &mut self,
744        _context: &mut Self::Context<'a>,
745        dst: Register,
746        src: Source,
747        offset: i16,
748    ) -> Result<(), String> {
749        self.conditional_jump(dst, src, offset, |x, y| scomp32(x, y, |x, y| x < y))
750    }
751    #[inline(always)]
752    fn jslt64<'a>(
753        &mut self,
754        _context: &mut Self::Context<'a>,
755        dst: Register,
756        src: Source,
757        offset: i16,
758    ) -> Result<(), String> {
759        self.conditional_jump(dst, src, offset, |x, y| scomp64(x, y, |x, y| x < y))
760    }
761    #[inline(always)]
762    fn jset<'a>(
763        &mut self,
764        _context: &mut Self::Context<'a>,
765        dst: Register,
766        src: Source,
767        offset: i16,
768    ) -> Result<(), String> {
769        self.conditional_jump(dst, src, offset, |x, y| comp32(x, y, |x, y| x & y != 0))
770    }
771    #[inline(always)]
772    fn jset64<'a>(
773        &mut self,
774        _context: &mut Self::Context<'a>,
775        dst: Register,
776        src: Source,
777        offset: i16,
778    ) -> Result<(), String> {
779        self.conditional_jump(dst, src, offset, |x, y| x & y != 0)
780    }
781
782    #[inline(always)]
783    fn atomic_add<'a>(
784        &mut self,
785        _context: &mut Self::Context<'a>,
786        fetch: bool,
787        dst: Register,
788        offset: i16,
789        src: Register,
790    ) -> Result<(), String> {
791        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.fetch_add(v, Ordering::SeqCst))
792    }
793
794    #[inline(always)]
795    fn atomic_add64<'a>(
796        &mut self,
797        _context: &mut Self::Context<'a>,
798        fetch: bool,
799        dst: Register,
800        offset: i16,
801        src: Register,
802    ) -> Result<(), String> {
803        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.fetch_add(v, Ordering::SeqCst))
804    }
805
806    #[inline(always)]
807    fn atomic_and<'a>(
808        &mut self,
809        _context: &mut Self::Context<'a>,
810        fetch: bool,
811        dst: Register,
812        offset: i16,
813        src: Register,
814    ) -> Result<(), String> {
815        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.fetch_and(v, Ordering::SeqCst))
816    }
817
818    #[inline(always)]
819    fn atomic_and64<'a>(
820        &mut self,
821        _context: &mut Self::Context<'a>,
822        fetch: bool,
823        dst: Register,
824        offset: i16,
825        src: Register,
826    ) -> Result<(), String> {
827        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.fetch_and(v, Ordering::SeqCst))
828    }
829
830    #[inline(always)]
831    fn atomic_or<'a>(
832        &mut self,
833        _context: &mut Self::Context<'a>,
834        fetch: bool,
835        dst: Register,
836        offset: i16,
837        src: Register,
838    ) -> Result<(), String> {
839        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.fetch_or(v, Ordering::SeqCst))
840    }
841
842    #[inline(always)]
843    fn atomic_or64<'a>(
844        &mut self,
845        _context: &mut Self::Context<'a>,
846        fetch: bool,
847        dst: Register,
848        offset: i16,
849        src: Register,
850    ) -> Result<(), String> {
851        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.fetch_or(v, Ordering::SeqCst))
852    }
853
854    #[inline(always)]
855    fn atomic_xor<'a>(
856        &mut self,
857        _context: &mut Self::Context<'a>,
858        fetch: bool,
859        dst: Register,
860        offset: i16,
861        src: Register,
862    ) -> Result<(), String> {
863        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.fetch_xor(v, Ordering::SeqCst))
864    }
865
866    #[inline(always)]
867    fn atomic_xor64<'a>(
868        &mut self,
869        _context: &mut Self::Context<'a>,
870        fetch: bool,
871        dst: Register,
872        offset: i16,
873        src: Register,
874    ) -> Result<(), String> {
875        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.fetch_xor(v, Ordering::SeqCst))
876    }
877
878    #[inline(always)]
879    fn atomic_xchg<'a>(
880        &mut self,
881        _context: &mut Self::Context<'a>,
882        fetch: bool,
883        dst: Register,
884        offset: i16,
885        src: Register,
886    ) -> Result<(), String> {
887        self.atomic_operation(fetch, dst, offset, src, |_, a, v| a.swap(v, Ordering::SeqCst))
888    }
889
890    #[inline(always)]
891    fn atomic_xchg64<'a>(
892        &mut self,
893        _context: &mut Self::Context<'a>,
894        fetch: bool,
895        dst: Register,
896        offset: i16,
897        src: Register,
898    ) -> Result<(), String> {
899        self.atomic_operation64(fetch, dst, offset, src, |_, a, v| a.swap(v, Ordering::SeqCst))
900    }
901
902    #[inline(always)]
903    fn atomic_cmpxchg<'a>(
904        &mut self,
905        _context: &mut Self::Context<'a>,
906        dst: Register,
907        offset: i16,
908        src: Register,
909    ) -> Result<(), String> {
910        self.atomic_operation(false, dst, offset, src, |this, a, v| {
911            let r0 = this.reg(0).as_u32();
912            let r0 = match a.compare_exchange(r0, v, Ordering::SeqCst, Ordering::SeqCst) {
913                Ok(v) | Err(v) => v,
914            };
915            this.set_reg(0, r0.into());
916            0
917        })
918    }
919
920    #[inline(always)]
921    fn atomic_cmpxchg64<'a>(
922        &mut self,
923        _context: &mut Self::Context<'a>,
924        dst: Register,
925        offset: i16,
926        src: Register,
927    ) -> Result<(), String> {
928        self.atomic_operation64(false, dst, offset, src, |this, a, v| {
929            let r0 = this.reg(0).as_u64();
930            let r0 = match a.compare_exchange(r0, v, Ordering::SeqCst, Ordering::SeqCst) {
931                Ok(v) | Err(v) => v,
932            };
933            this.set_reg(0, r0.into());
934            0
935        })
936    }
937
938    #[inline(always)]
939    fn load<'a>(
940        &mut self,
941        _context: &mut Self::Context<'a>,
942        dst: Register,
943        offset: i16,
944        src: Register,
945        width: DataWidth,
946    ) -> Result<(), String> {
947        let addr = self.reg(src);
948        let loaded = self.load_memory(addr, offset as u64, width);
949        self.set_reg(dst, loaded);
950        Ok(())
951    }
952
953    #[inline(always)]
954    fn load64<'a>(
955        &mut self,
956        _context: &mut Self::Context<'a>,
957        dst: Register,
958        _src: u8,
959        lower: u32,
960    ) -> Result<(), String> {
961        let value = (lower as u64) | (((self.code[self.pc + 1].imm() as u32) as u64) << 32);
962        self.set_reg(dst, value.into());
963        self.advance_pc(1);
964        Ok(())
965    }
966
967    #[inline(always)]
968    fn load_from_packet<'a>(
969        &mut self,
970        context: &mut Self::Context<'a>,
971        dst_reg: Register,
972        src_reg: Register,
973        offset: i32,
974        register_offset: Option<Register>,
975        width: DataWidth,
976    ) -> Result<(), String> {
977        let Some(offset) =
978            register_offset.map(|r| self.reg(r).as_i32()).unwrap_or(0).checked_add(offset as i32)
979        else {
980            // Offset overflowed. Exit.
981            self.result = Some(0);
982            return Ok(());
983        };
984        let src_reg = self.reg(src_reg);
985        // SAFETY: The verifier checks that the `src_reg` points at packet.
986        let packet = unsafe { C::Packet::from_bpf_value(context, src_reg) };
987        if let Some(value) = packet.load(offset, width) {
988            self.set_reg(dst_reg, value.into());
989        } else {
990            self.result = Some(0);
991        }
992        Ok(())
993    }
994
995    #[inline(always)]
996    fn store<'a>(
997        &mut self,
998        _context: &mut Self::Context<'a>,
999        dst: Register,
1000        offset: i16,
1001        src: Source,
1002        width: DataWidth,
1003    ) -> Result<(), String> {
1004        let src = self.compute_source(src);
1005        let dst = self.reg(dst);
1006        self.store_memory(dst, src, offset as u64, width);
1007        Ok(())
1008    }
1009}
1010
1011#[inline(always)]
1012fn alu32(x: u64, y: u64, op: impl FnOnce(u32, u32) -> u32) -> u64 {
1013    op(x as u32, y as u32) as u64
1014}
1015
1016#[inline(always)]
1017fn comp32(x: u64, y: u64, op: impl FnOnce(u32, u32) -> bool) -> bool {
1018    op(x as u32, y as u32)
1019}
1020
1021#[inline(always)]
1022fn scomp64(x: u64, y: u64, op: impl FnOnce(i64, i64) -> bool) -> bool {
1023    op(x as i64, y as i64)
1024}
1025
1026#[inline(always)]
1027fn scomp32(x: u64, y: u64, op: impl FnOnce(i32, i32) -> bool) -> bool {
1028    op(x as i32, y as i32)
1029}
ebpf/executor.rs

ebpf/
executor.rs
