Skip to main content

ebpf/
verifier.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#![allow(unused_variables)]
6
7use crate::scalar_value::{ScalarValueData, U32Range, U32ScalarValueData, U64Range};
8use crate::visitor::{BpfVisitor, ProgramCounter, Register, Source};
9use crate::{
10    BPF_LDDW, BPF_MAX_INSTS, BPF_PSEUDO_MAP_IDX, BPF_PSEUDO_MAP_IDX_VALUE, BPF_STACK_SIZE,
11    DataWidth, EbpfError, EbpfInstruction, GENERAL_REGISTER_COUNT, MapSchema, REGISTER_COUNT,
12};
13use byteorder::{BigEndian, ByteOrder, LittleEndian, NativeEndian};
14use fuchsia_sync::Mutex;
15use linux_uapi::{bpf_map_type, bpf_map_type_BPF_MAP_TYPE_ARRAY};
16use std::cmp::Ordering;
17use std::collections::{BTreeMap, HashMap, HashSet};
18use std::sync::Arc;
19use zerocopy::IntoBytes;
20
21const U32_MAX: u64 = u32::MAX as u64;
22
23/// A trait to receive the log from the verifier.
24pub trait VerifierLogger {
25    /// Log a line. The line is always a correct encoded ASCII string.
26    fn log(&mut self, line: &[u8]);
27}
28
29/// A `VerifierLogger` that drops all its content.
30pub struct NullVerifierLogger;
31
32impl VerifierLogger for NullVerifierLogger {
33    fn log(&mut self, line: &[u8]) {
34        debug_assert!(line.is_ascii());
35    }
36}
37
38/// An identifier for a memory buffer accessible by an ebpf program. The identifiers are built as a
39/// chain of unique identifier so that a buffer can contain multiple pointers to the same type and
40/// the verifier can distinguish between the different instances.
41/// The namespace of a `MemoryId`, used to prevent collisions between different
42/// sources of identifiers.
43#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Ord, PartialOrd)]
44pub enum Namespace {
45    /// For static structure identifiers (e.g., `SkBuff`, `BpfSock`) defined in `program_type.rs`.
46    Hardcoded,
47    /// For IDs generated by `MemoryId::new()` using the global counter.
48    Generated,
49    /// For IDs generated by `VerificationContext::next_id()` using the local counter.
50    Verification,
51}
52
53#[derive(Clone, Debug, PartialEq, Eq, Hash, Ord, PartialOrd)]
54pub struct MemoryId {
55    namespace: Namespace,
56    id: u64,
57    parent: Option<Box<MemoryId>>,
58}
59
60impl MemoryId {
61    pub fn id(&self) -> u64 {
62        self.id
63    }
64}
65
66impl From<u64> for MemoryId {
67    fn from(id: u64) -> Self {
68        Self { namespace: Namespace::Hardcoded, id, parent: None }
69    }
70}
71
72/// A counter that allows to generate new ids for parameters. The namespace is the same as for id
73/// generated for types while verifying an ebpf program. Because different sources of identifiers
74/// use different namespaces, this counter can start at 0.
75static BPF_TYPE_IDENTIFIER_COUNTER: std::sync::atomic::AtomicU64 =
76    std::sync::atomic::AtomicU64::new(0);
77
78impl MemoryId {
79    /// Build a `MemoryId` from a raw u64.
80    ///
81    /// This is supposed to be used by the `ebpf_api` crate to build static identifiers (e.g.
82    /// for hardcoded program contexts like `SkBuff`), but not elsewhere.
83    pub const fn from_raw(id: u64) -> MemoryId {
84        Self { namespace: Namespace::Hardcoded, id, parent: None }
85    }
86
87    pub fn new() -> MemoryId {
88        Self {
89            namespace: Namespace::Generated,
90            id: BPF_TYPE_IDENTIFIER_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
91            parent: None,
92        }
93    }
94
95    /// Build a new id such that `other` is prepended to the chain of parent of `self`.
96    fn prepended(&self, other: MemoryId) -> Self {
97        match &self.parent {
98            None => {
99                MemoryId { namespace: self.namespace, id: self.id, parent: Some(Box::new(other)) }
100            }
101            Some(parent) => MemoryId {
102                namespace: self.namespace,
103                id: self.id,
104                parent: Some(Box::new(parent.prepended(other))),
105            },
106        }
107    }
108
109    /// Returns whether the parent of `self` is `parent`
110    fn has_parent(&self, parent: &MemoryId) -> bool {
111        match &self.parent {
112            None => false,
113            Some(p) => p.as_ref() == parent,
114        }
115    }
116
117    /// Returns whether `self` and `other` represent the same type. This is true if both have the
118    /// same id and if they both have parent, parents must also match.
119    fn matches(&self, other: &MemoryId) -> bool {
120        if self.namespace != other.namespace || self.id != other.id {
121            return false;
122        };
123        match (&self.parent, &other.parent) {
124            (Some(p1), Some(p2)) => p1.matches(p2.as_ref()),
125            _ => true,
126        }
127    }
128}
129
130/// The type of a field in a struct pointed by `Type::PtrToStruct`.
131#[derive(Clone, Debug, PartialEq)]
132pub enum FieldType {
133    /// Read-only scalar value.
134    Scalar { size: usize },
135
136    /// Mutable scalar value.
137    MutableScalar { size: usize },
138
139    /// A pointer to the kernel memory. The full buffer is `buffer_size` bytes long.
140    PtrToMemory { is_32_bit: bool, id: MemoryId, buffer_size: usize },
141
142    /// A nullable pointer to the kernel memory. The full buffer is `buffer_size` bytes long.
143    NullablePtrToMemory { is_32_bit: bool, id: MemoryId, buffer_size: usize },
144
145    /// A pointer to the kernel memory. The full buffer size is determined by an instance of
146    /// `PtrToEndArray` with the same `id`.
147    PtrToArray { is_32_bit: bool, id: MemoryId },
148
149    /// A pointer to the kernel memory that represents the first non accessible byte of a
150    /// `PtrToArray` with the same `id`.
151    PtrToEndArray { is_32_bit: bool, id: MemoryId },
152}
153
154/// Definition of a field in a `Type::PtrToStruct`.
155#[derive(Clone, Debug, PartialEq)]
156pub struct FieldDescriptor {
157    /// The offset at which the field is located.
158    pub offset: usize,
159
160    /// The type of the pointed memory. Currently the verifier supports only `PtrToArray`,
161    /// `PtrToStruct`, `PtrToEndArray` and `ScalarValue`.
162    pub field_type: FieldType,
163}
164
165impl FieldDescriptor {
166    fn size(&self) -> usize {
167        match self.field_type {
168            FieldType::Scalar { size } | FieldType::MutableScalar { size } => size,
169            FieldType::PtrToMemory { is_32_bit, .. }
170            | FieldType::NullablePtrToMemory { is_32_bit, .. }
171            | FieldType::PtrToArray { is_32_bit, .. }
172            | FieldType::PtrToEndArray { is_32_bit, .. } => {
173                if is_32_bit {
174                    4
175                } else {
176                    8
177                }
178            }
179        }
180    }
181}
182
183/// The offset and width of a field in a struct.
184#[derive(Clone, Copy, Debug, Eq, PartialEq)]
185struct Field {
186    offset: i16,
187    width: DataWidth,
188}
189
190impl Field {
191    fn new(offset: i16, width: DataWidth) -> Self {
192        Self { offset, width }
193    }
194}
195
196/// Defines field layout in a struct pointed by `Type::PtrToStruct`.
197#[derive(Debug, PartialEq, Default)]
198pub struct StructDescriptor {
199    /// The list of fields.
200    pub fields: Vec<FieldDescriptor>,
201}
202
203impl StructDescriptor {
204    /// Return true if `self` is a subtype of `super_struct`.
205    fn is_subtype(&self, super_struct: &StructDescriptor) -> bool {
206        // Check that `super_struct.fields` is a subset of `fields`.
207        for super_field in super_struct.fields.iter() {
208            if self.fields.iter().find(|field| *field == super_field).is_none() {
209                return false;
210            }
211        }
212        true
213    }
214
215    /// Finds the field type for load/store at the specified location. None is returned if the
216    /// access is invalid and the program must be rejected.
217    fn find_field(&self, base_offset: ScalarValueData, field: Field) -> Option<&FieldDescriptor> {
218        let offset = base_offset + field.offset;
219        let field_desc = self.fields.iter().find(|f| {
220            f.offset <= offset.min() as usize && (offset.max() as usize) < f.offset + f.size()
221        })?;
222        let is_valid_load = match field_desc.field_type {
223            FieldType::Scalar { size } | FieldType::MutableScalar { size } => {
224                // For scalars check that the access is within the bounds of the field.
225                ((offset + field.width.bytes()).max() as usize) <= field_desc.offset + size
226            }
227            FieldType::PtrToMemory { is_32_bit, .. }
228            | FieldType::NullablePtrToMemory { is_32_bit, .. }
229            | FieldType::PtrToArray { is_32_bit, .. }
230            | FieldType::PtrToEndArray { is_32_bit, .. } => {
231                let expected_width = if is_32_bit { DataWidth::U32 } else { DataWidth::U64 };
232                // Pointer loads are expected to load the whole field.
233                offset.is_known()
234                    && offset.value as usize == field_desc.offset
235                    && field.width == expected_width
236            }
237        };
238
239        is_valid_load.then_some(field_desc)
240    }
241}
242
243#[derive(Clone, Debug, PartialEq)]
244pub enum MemoryParameterSize {
245    /// The memory buffer have the given size.
246    Value(u64),
247    /// The memory buffer size is given by the parameter in the given index.
248    Reference { index: u8 },
249}
250
251impl MemoryParameterSize {
252    fn size(&self, context: &ComputationContext) -> Result<u64, String> {
253        match self {
254            Self::Value(size) => Ok(*size),
255            Self::Reference { index } => {
256                let size_type = context.reg(index + 1)?;
257                match size_type {
258                    Type::ScalarValue(data) if data.is_known() => Ok(data.value),
259                    _ => Err("cannot know buffer size".to_string()),
260                }
261            }
262        }
263    }
264}
265
266#[derive(Clone, Debug, PartialEq)]
267pub enum MapTypeFilter {
268    AllowList(&'static [bpf_map_type]),
269    DenyList(&'static [bpf_map_type]),
270}
271
272impl MapTypeFilter {
273    pub fn is_allowed(&self, map_type: bpf_map_type) -> bool {
274        match self {
275            MapTypeFilter::AllowList(types) => types.contains(&map_type),
276            MapTypeFilter::DenyList(types) => !types.contains(&map_type),
277        }
278    }
279}
280
281#[derive(Clone, Debug, PartialEq)]
282pub enum Type {
283    /// A number.
284    ScalarValue(ScalarValueData),
285    /// A pointer to a map object.
286    ConstPtrToMap { id: u64, schema: MapSchema },
287    /// A pointer into the stack.
288    PtrToStack { offset: StackOffset },
289    /// A pointer to the kernel memory. The full buffer is `buffer_size` bytes long. The pointer is
290    /// situated at `offset` from the start of the buffer.
291    PtrToMemory { id: MemoryId, offset: ScalarValueData, buffer_size: u64 },
292    /// A pointer to a struct with the specified `StructDescriptor`.
293    PtrToStruct { id: MemoryId, offset: ScalarValueData, descriptor: Arc<StructDescriptor> },
294    /// A pointer to the kernel memory. The full buffer size is determined by an instance of
295    /// `PtrToEndArray` with the same `id`. The pointer is situadted at `offset` from the start of
296    /// the buffer.
297    PtrToArray { id: MemoryId, offset: ScalarValueData },
298    /// A pointer to the kernel memory that represents the first non accessible byte of a
299    /// `PtrToArray` with the same `id`.
300    PtrToEndArray { id: MemoryId },
301    /// A pointer that might be null and must be validated before being referenced.
302    NullOr { id: MemoryId, inner: Box<Type> },
303    /// An object that must be passed to a method with an associated `ReleaseParameter` before the
304    /// end of the program.
305    Releasable { id: MemoryId, inner: Box<Type> },
306    /// A function parameter that must be a `ScalarValue` when called.
307    ScalarValueParameter,
308    /// A function parameter that must be a `ConstPtrToMap` when called.
309    ConstPtrToMapParameter { filter: MapTypeFilter },
310    /// A function parameter that must be a key of a map.
311    MapKeyParameter {
312        /// The index in the arguments list that contains a `ConstPtrToMap` for the map this key is
313        /// associated with.
314        map_ptr_index: u8,
315    },
316    /// A function parameter that must be a value of a map.
317    MapValueParameter {
318        /// The index in the arguments list that contains a `ConstPtrToMap` for the map this key is
319        /// associated with.
320        map_ptr_index: u8,
321    },
322    /// A function parameter that must be a pointer to memory.
323    MemoryParameter {
324        /// The index in the arguments list that contains a scalar value containing the size of the
325        /// memory.
326        size: MemoryParameterSize,
327        /// Whether this memory is read by the function.
328        input: bool,
329        /// Whether this memory is written by the function.
330        output: bool,
331    },
332    /// A function return value that is the same type as a parameter.
333    AliasParameter {
334        /// The index in the argument list of the parameter that has the type of this return value.
335        parameter_index: u8,
336    },
337    /// A function return value that is either null, or the given type.
338    NullOrParameter(Box<Type>),
339    /// A function parameter that must be a pointer to memory with the given id.
340    StructParameter { id: MemoryId },
341    /// The parameter should be the same as the specified argument passed to the program.
342    ContextParameter { parameter_index: u8 },
343    /// A function return value that must be passed to a method with an associated
344    /// `ReleaseParameter` before the end of the program.
345    ReleasableParameter { id: MemoryId, inner: Box<Type> },
346    /// A function parameter that will release the value.
347    ReleaseParameter { id: MemoryId },
348    /// A function parameter that is not checked.
349    AnyParameter,
350}
351
352/// Defines a partial ordering on `Type` instances, capturing the notion of how "broad"
353/// a type is in terms of the set of potential values it represents.
354///
355/// The ordering is defined such that `t1 > t2` if a proof that an eBPF program terminates
356/// in a state where a register or memory location has type `t1` is also a proof that
357/// the program terminates in a state where that location has type `t2`.
358///
359/// In other words, a "broader" type represents a larger set of possible values, and
360/// proving termination with a broader type implies termination with any narrower type.
361///
362/// Examples:
363/// * `Type::ScalarValue(ScalarValueData({ unknown_mask: 0, .. }))` (a known scalar value) is less than
364///   `Type::ScalarValue(ScalarValueData({ unknown_mask: u64::MAX, .. }))` (an unknown scalar value).
365impl PartialOrd for Type {
366    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
367        // If the values are equals, return the known result.
368        if self == other {
369            return Some(Ordering::Equal);
370        }
371
372        // If one value is not initialized, the types are ordered.
373        // UNINITIALIZED is treated as the Top element (Greater than all other types).
374        // If a path is proven safe when a register is UNINITIALIZED, it means the path
375        // does not read that register. Consequently, it is safe to prune any state
376        // that has a more specific type (like a pointer) in that register, because
377        // the program will still not read it.
378        if self == &Type::UNINITIALIZED {
379            return Some(Ordering::Greater);
380        }
381        if other == &Type::UNINITIALIZED {
382            return Some(Ordering::Less);
383        }
384
385        // Otherwise, only scalars are comparables.
386        match (self, other) {
387            (Self::ScalarValue(data1), Self::ScalarValue(data2)) => data1.partial_cmp(data2),
388            _ => None,
389        }
390    }
391}
392
393impl From<ScalarValueData> for Type {
394    fn from(value: ScalarValueData) -> Self {
395        Self::ScalarValue(value)
396    }
397}
398
399impl From<u64> for Type {
400    fn from(value: u64) -> Self {
401        Self::ScalarValue(value.into())
402    }
403}
404
405impl Default for Type {
406    /// A new instance of `Type` where no bit has been written yet.
407    fn default() -> Self {
408        Self::UNINITIALIZED.clone()
409    }
410}
411
412impl Type {
413    /// An uninitialized value.
414    pub const UNINITIALIZED: Self = Self::ScalarValue(ScalarValueData::UNINITIALIZED);
415
416    /// A `Type` where the data is usable by userspace, but the value is not known by the verifier.
417    pub const UNKNOWN_SCALAR: Self = Self::ScalarValue(ScalarValueData::UNKNOWN_WRITTEN);
418
419    /// The mask associated with a data of size `width`.
420    fn mask(width: DataWidth) -> u64 {
421        if width == DataWidth::U64 { u64::MAX } else { (1 << width.bits()) - 1 }
422    }
423
424    /// Return true if `self` is a scalar with all bytes initialized.
425    fn is_written_scalar(&self) -> bool {
426        match self {
427            Self::ScalarValue(data) if data.is_fully_initialized() => true,
428            _ => false,
429        }
430    }
431
432    /// Return true if `self` has all bytes initialized.
433    fn is_initialized(&self) -> bool {
434        match self {
435            Self::ScalarValue(data) => data.is_fully_initialized(),
436            _ => true,
437        }
438    }
439
440    /// Return true if `self` is a subtype of `super_type`.
441    pub fn is_subtype(&self, super_type: &Type) -> bool {
442        match (self, super_type) {
443            // Anything can be used in place of an uninitialized value.
444            (_, Self::ScalarValue(data)) if data.is_uninitialized() => true,
445
446            (
447                Self::PtrToStruct { id: id1, offset: offset1, descriptor: descriptor1 },
448                Self::PtrToStruct { id: id2, offset: offset2, descriptor: descriptor2 },
449            ) => id1 == id2 && offset1 <= offset2 && descriptor1.is_subtype(descriptor2),
450
451            // Every type is a subtype of itself.
452            (self_type, super_type) if self_type == super_type => true,
453
454            _ => false,
455        }
456    }
457
458    /// Return true is `self` is guaranteed to be non-zero
459    pub fn is_non_zero(&self) -> bool {
460        match self {
461            Self::ScalarValue(d) => d.min() > 0,
462            Self::NullOr { .. } => false,
463            _ => true,
464        }
465    }
466
467    fn inner(&self, context: &ComputationContext) -> Result<&Type, String> {
468        match self {
469            Self::Releasable { id, inner } => {
470                if context.resources.contains(id) {
471                    Ok(&inner)
472                } else {
473                    Err("Access to released resource".to_string())
474                }
475            }
476            _ => Ok(self),
477        }
478    }
479
480    /// Constraints `type1` and `type2` for a conditional jump with the specified `jump_type` and
481    /// `jump_width`.
482    fn constraint(
483        context: &mut ComputationContext,
484        jump_type: JumpType,
485        jump_width: JumpWidth,
486        type1: Self,
487        type2: Self,
488    ) -> Result<(Self, Self), String> {
489        let result = match (jump_width, jump_type, type1.inner(context)?, type2.inner(context)?) {
490            (JumpWidth::W64, JumpType::Eq, Type::ScalarValue(data1), Type::ScalarValue(data2))
491                if data1.is_fully_initialized() && data2.is_fully_initialized() =>
492            {
493                let umin = std::cmp::max(data1.min(), data2.min());
494                let umax = std::cmp::min(data1.max(), data2.max());
495                let v = Type::ScalarValue(ScalarValueData::new(
496                    data1.value | data2.value,
497                    data1.unknown_mask & data2.unknown_mask,
498                    0,
499                    U64Range::new(umin, umax),
500                ));
501                (v.clone(), v)
502            }
503            (JumpWidth::W32, JumpType::Eq, Type::ScalarValue(data1), Type::ScalarValue(data2))
504                if data1.is_fully_initialized() && data2.is_fully_initialized() =>
505            {
506                let maybe_umin = if data1.min() <= U32_MAX && data2.min() <= U32_MAX {
507                    Some(std::cmp::max(data1.min(), data2.min()))
508                } else {
509                    None
510                };
511                let maybe_umax = if data1.max() <= U32_MAX && data2.max() <= U32_MAX {
512                    Some(std::cmp::min(data1.max(), data2.max()))
513                } else {
514                    None
515                };
516
517                let urange1 = U64Range::new(
518                    maybe_umin.unwrap_or_else(|| data1.min()),
519                    maybe_umax.unwrap_or_else(|| data1.max()),
520                );
521                let v1 = Type::ScalarValue(ScalarValueData::new(
522                    data1.value | (data2.value & U32_MAX),
523                    data1.unknown_mask & (data2.unknown_mask | (U32_MAX << 32)),
524                    0,
525                    urange1,
526                ));
527
528                let urange2 = U64Range::new(
529                    maybe_umin.unwrap_or_else(|| data2.min()),
530                    maybe_umax.unwrap_or_else(|| data2.max()),
531                );
532                let v2 = Type::ScalarValue(ScalarValueData::new(
533                    data2.value | (data1.value & U32_MAX),
534                    data2.unknown_mask & (data1.unknown_mask | (U32_MAX << 32)),
535                    0,
536                    urange2,
537                ));
538                (v1, v2)
539            }
540            (JumpWidth::W64, JumpType::Eq, Type::ScalarValue(data), Type::NullOr { id, .. })
541            | (JumpWidth::W64, JumpType::Eq, Type::NullOr { id, .. }, Type::ScalarValue(data))
542                if data.is_zero() =>
543            {
544                context.set_null(id, true);
545                let zero = Type::from(0);
546                (zero.clone(), zero)
547            }
548            (JumpWidth::W64, jump_type, Type::NullOr { id, inner }, Type::ScalarValue(data))
549                if jump_type.is_strict() && data.is_zero() =>
550            {
551                context.set_null(id, false);
552                let inner = *inner.clone();
553                inner.register_resource(context);
554                (inner, type2)
555            }
556            (JumpWidth::W64, jump_type, Type::ScalarValue(data), Type::NullOr { id, inner })
557                if jump_type.is_strict() && data.is_zero() =>
558            {
559                context.set_null(id, false);
560                let inner = *inner.clone();
561                inner.register_resource(context);
562                (type1, inner)
563            }
564
565            (JumpWidth::W64, JumpType::Lt, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
566                debug_assert!(lhs.min() < u64::MAX);
567                debug_assert!(rhs.max() > 0);
568                let new_max_lhs = std::cmp::min(lhs.max(), rhs.max() - 1);
569                debug_assert!(lhs.min() <= new_max_lhs);
570                let new_min_rhs = std::cmp::max(rhs.min(), lhs.min() + 1);
571                debug_assert!(new_min_rhs <= rhs.max());
572                let new_range_lhs = U64Range::new(lhs.min(), new_max_lhs);
573                let new_range_rhs = U64Range::new(new_min_rhs, rhs.max());
574                (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
575            }
576            (JumpWidth::W64, JumpType::Gt, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
577                debug_assert!(rhs.min() < u64::MAX);
578                debug_assert!(lhs.max() > 0);
579                let new_min_lhs = std::cmp::max(lhs.min(), rhs.min() + 1);
580                debug_assert!(new_min_lhs <= lhs.max());
581                let new_max_rhs = std::cmp::min(rhs.max(), lhs.max() - 1);
582                debug_assert!(rhs.min() <= new_max_rhs);
583                let new_range_lhs = U64Range::new(new_min_lhs, lhs.max());
584                let new_range_rhs = U64Range::new(rhs.min(), new_max_rhs);
585                (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
586            }
587
588            (JumpWidth::W64, JumpType::Le, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
589                let new_max_lhs = std::cmp::min(lhs.max(), rhs.max());
590                debug_assert!(lhs.min() <= new_max_lhs);
591                let new_min_rhs = std::cmp::max(rhs.min(), lhs.min());
592                debug_assert!(new_min_rhs <= rhs.max());
593                let new_range_lhs = U64Range::new(lhs.min(), new_max_lhs);
594                let new_range_rhs = U64Range::new(new_min_rhs, rhs.max());
595                (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
596            }
597            (JumpWidth::W64, JumpType::Ge, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
598                let new_min_lhs = std::cmp::max(lhs.min(), rhs.min());
599                debug_assert!(new_min_lhs <= lhs.max());
600                let new_max_rhs = std::cmp::min(rhs.max(), lhs.max());
601                debug_assert!(rhs.min() <= new_max_rhs);
602                let new_range_lhs = U64Range::new(new_min_lhs, lhs.max());
603                let new_range_rhs = U64Range::new(rhs.min(), new_max_rhs);
604                (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
605            }
606
607            (
608                JumpWidth::W64,
609                JumpType::Eq,
610                Type::PtrToArray { id: id1, offset },
611                Type::PtrToEndArray { id: id2 },
612            )
613            | (
614                JumpWidth::W64,
615                JumpType::Le,
616                Type::PtrToArray { id: id1, offset },
617                Type::PtrToEndArray { id: id2 },
618            )
619            | (
620                JumpWidth::W64,
621                JumpType::Ge,
622                Type::PtrToEndArray { id: id1 },
623                Type::PtrToArray { id: id2, offset },
624            ) if id1 == id2 => {
625                context.update_array_bounds(id1.clone(), *offset);
626                (type1, type2)
627            }
628            (
629                JumpWidth::W64,
630                JumpType::Lt,
631                Type::PtrToArray { id: id1, offset },
632                Type::PtrToEndArray { id: id2 },
633            )
634            | (
635                JumpWidth::W64,
636                JumpType::Gt,
637                Type::PtrToEndArray { id: id1 },
638                Type::PtrToArray { id: id2, offset },
639            ) if id1 == id2 => {
640                context.update_array_bounds(id1.clone(), *offset + 1);
641                (type1, type2)
642            }
643            (JumpWidth::W64, JumpType::Eq, _, _) => (type1.clone(), type1),
644            _ => (type1, type2),
645        };
646        Ok(result)
647    }
648
649    fn match_parameter_type(
650        &self,
651        verification_context: &VerificationContext<'_>,
652        context: &ComputationContext,
653        helper_name: &str,
654        parameter_type: &Type,
655        index: usize,
656        next: &mut ComputationContext,
657    ) -> Result<(), String> {
658        match (parameter_type, self) {
659            (Type::NullOrParameter(t), Type::ScalarValue(data))
660                if data.is_known() && data.value == 0 =>
661            {
662                Ok(())
663            }
664            (Type::NullOrParameter(t), _) => self.match_parameter_type(
665                verification_context,
666                context,
667                helper_name,
668                t,
669                index,
670                next,
671            ),
672            (Type::ScalarValueParameter, Type::ScalarValue(data))
673                if data.is_fully_initialized() =>
674            {
675                Ok(())
676            }
677            (Type::ConstPtrToMapParameter { filter }, Type::ConstPtrToMap { schema, .. }) => {
678                let map_type = schema.map_type;
679                if !filter.is_allowed(map_type) {
680                    return Err(format!("Map type {map_type} not allowed in {helper_name}"));
681                }
682                Ok(())
683            }
684            (
685                Type::MapKeyParameter { map_ptr_index },
686                Type::PtrToMemory { offset, buffer_size, .. },
687            ) => {
688                let schema = context.get_map_schema(*map_ptr_index)?;
689                context.check_memory_access(*offset, *buffer_size, 0, schema.key_size as usize)
690            }
691            (Type::MapKeyParameter { map_ptr_index }, Type::PtrToStack { offset }) => {
692                let schema = context.get_map_schema(*map_ptr_index)?;
693                context.stack.read_data_ptr(context.pc, *offset, schema.key_size as u64)
694            }
695            (
696                Type::MapValueParameter { map_ptr_index },
697                Type::PtrToMemory { offset, buffer_size, .. },
698            ) => {
699                let schema = context.get_map_schema(*map_ptr_index)?;
700                context.check_memory_access(*offset, *buffer_size, 0, schema.value_size as usize)
701            }
702            (Type::MapValueParameter { map_ptr_index }, Type::PtrToStack { offset }) => {
703                let schema = context.get_map_schema(*map_ptr_index)?;
704                context.stack.read_data_ptr(context.pc, *offset, schema.value_size as u64)
705            }
706            (Type::MemoryParameter { size, .. }, Type::PtrToMemory { offset, buffer_size, .. }) => {
707                let expected_size = size.size(context)?;
708                let offset_max = offset.max();
709                if offset_max > *buffer_size {
710                    return Err("out of bound read".to_string());
711                }
712                let size_left = *buffer_size - offset_max;
713                if expected_size > size_left {
714                    return Err("out of bound read".to_string());
715                }
716                Ok(())
717            }
718
719            (Type::MemoryParameter { size, input, output }, Type::PtrToStack { offset }) => {
720                let size = size.size(context)?;
721                let buffer_end = offset.add(size);
722                if !buffer_end.is_within_stack() {
723                    Err("out of bound access".to_string())
724                } else {
725                    if *output {
726                        next.stack.write_data_ptr(context.pc, *offset, size)?;
727                    }
728                    if *input {
729                        context.stack.read_data_ptr(context.pc, *offset, size)?;
730                    }
731                    Ok(())
732                }
733            }
734            (
735                Type::StructParameter { id: id1 },
736                Type::PtrToMemory { id: id2, offset, .. }
737                | Type::PtrToStruct { id: id2, offset, .. },
738            ) if offset.is_zero() && id1.matches(id2) => Ok(()),
739            (
740                Type::ReleasableParameter { id: id1, inner: inner1 },
741                Type::Releasable { id: id2, inner: inner2 },
742            ) if id2.has_parent(id1) => {
743                if next.resources.contains(id2) {
744                    inner2.match_parameter_type(
745                        verification_context,
746                        context,
747                        helper_name,
748                        inner1,
749                        index,
750                        next,
751                    )
752                } else {
753                    Err(format!("Resource already released for index {index}"))
754                }
755            }
756            (Type::ContextParameter { parameter_index }, arg_type) => {
757                if verification_context.calling_context.args.get(*parameter_index as usize)
758                    == Some(arg_type)
759                {
760                    Ok(())
761                } else {
762                    Err(format!("Helper expects program argument {parameter_index}"))
763                }
764            }
765            (Type::ReleaseParameter { id: id1 }, Type::Releasable { id: id2, .. })
766                if id2.has_parent(id1) =>
767            {
768                if next.resources.remove(id2) {
769                    Ok(())
770                } else {
771                    Err(format!("{id2:?} Resource already released for index {index}"))
772                }
773            }
774            (_, Type::Releasable { inner, .. }) => inner.match_parameter_type(
775                verification_context,
776                context,
777                helper_name,
778                parameter_type,
779                index,
780                next,
781            ),
782            (Type::AnyParameter, _) => Ok(()),
783
784            _ => Err(format!("incorrect parameter for index {index}")),
785        }
786    }
787
788    /// If this `Type` is an instance of NullOr with the given `null_id`, replace it wither either
789    /// 0 or the subtype depending on `is_null`
790    fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
791        match self {
792            Type::NullOr { id, inner } if id == null_id => {
793                if is_null {
794                    *self = Type::from(0);
795                } else {
796                    *self = *inner.clone();
797                }
798            }
799            _ => {}
800        }
801    }
802
803    fn register_resource(&self, context: &mut ComputationContext) {
804        match self {
805            Type::Releasable { id, .. } => {
806                context.resources.insert(id.clone());
807            }
808            _ => {}
809        }
810    }
811
812    /// Partially Compares two iterators of comparable items.
813    ///
814    /// This function iterates through both input iterators simultaneously and compares the corresponding elements.
815    /// The comparison continues until:
816    /// 1. Both iterators are exhausted and all elements were considered equal, in which case it returns `Some(Ordering::Equal)`.
817    /// 2. All pairs of corresponding elements that are not equal have the same ordering (`Ordering::Less` or `Ordering::Greater`), in which case it returns `Some(Ordering)` reflecting that consistent ordering.
818    /// 3. One iterator is exhausted before the other, or any comparison between elements yields `None`, or not all non-equal pairs have the same ordering, in which case it returns `None`.
819    fn compare_list<'a>(
820        mut l1: impl Iterator<Item = &'a Self>,
821        mut l2: impl Iterator<Item = &'a Self>,
822    ) -> Option<Ordering> {
823        let mut result = Ordering::Equal;
824        loop {
825            match (l1.next(), l2.next()) {
826                (None, None) => return Some(result),
827                (_, None) | (None, _) => return None,
828                (Some(v1), Some(v2)) => {
829                    result = associate_orderings(result, v1.partial_cmp(v2)?)?;
830                }
831            }
832        }
833    }
834}
835
836#[derive(Clone, Debug)]
837pub struct FunctionSignature {
838    pub args: Vec<Type>,
839    pub return_value: Type,
840    pub invalidate_array_bounds: bool,
841}
842
843#[derive(Clone, Debug)]
844pub struct HelperDefinition {
845    pub index: u32,
846    pub name: &'static str,
847    pub signature: FunctionSignature,
848}
849
850#[derive(Debug, Default)]
851pub struct CallingContext {
852    /// List of map schemas of the associated map. The maps can be accessed using LDDW instruction
853    /// with `src_reg=BPF_PSEUDO_MAP_IDX`.
854    pub maps: Vec<MapSchema>,
855    /// The registered external functions.
856    pub helpers: HashMap<u32, &'static HelperDefinition>,
857    /// The args of the program.
858    pub args: Vec<Type>,
859    /// Packet type. Normally it should be either `None` or `args[0]`.
860    pub packet_type: Option<Type>,
861}
862
863impl CallingContext {
864    pub fn register_map(&mut self, schema: MapSchema) -> usize {
865        let index = self.maps.len();
866        self.maps.push(schema);
867        index
868    }
869}
870
871#[derive(Debug, PartialEq, Clone)]
872pub struct StructAccess {
873    pub pc: ProgramCounter,
874
875    // Memory Id of the struct being accessed.
876    pub memory_id: MemoryId,
877
878    // Offset of the field being loaded.
879    pub field_offset: usize,
880
881    // Indicates that this is a 32-bit pointer load. These instructions must be remapped.
882    pub is_32_bit_ptr_load: bool,
883}
884
885#[derive(Debug, Clone)]
886pub struct VerifiedEbpfProgram {
887    pub(crate) code: Vec<EbpfInstruction>,
888    pub(crate) args: Vec<Type>,
889    pub(crate) struct_access_instructions: Vec<StructAccess>,
890    pub(crate) maps: Vec<MapSchema>,
891}
892
893impl VerifiedEbpfProgram {
894    // Convert the program to raw code. Can be used only when the program doesn't access any
895    // structs and maps.
896    pub fn to_code(self) -> Vec<EbpfInstruction> {
897        debug_assert!(self.struct_access_instructions.is_empty());
898        debug_assert!(self.maps.is_empty());
899        self.code
900    }
901
902    pub fn code(&self) -> &[EbpfInstruction] {
903        &self.code
904    }
905
906    pub fn struct_access_instructions(&self) -> &[StructAccess] {
907        &self.struct_access_instructions
908    }
909
910    pub fn from_verified_code(
911        code: Vec<EbpfInstruction>,
912        args: Vec<Type>,
913        struct_access_instructions: Vec<StructAccess>,
914        maps: Vec<MapSchema>,
915    ) -> Self {
916        Self { code, args, struct_access_instructions, maps }
917    }
918
919    pub fn maps(&self) -> &[MapSchema] {
920        &self.maps
921    }
922}
923
924/// Verify the given code depending on the type of the parameters and the registered external
925/// functions. Returned `VerifiedEbpfProgram` should be linked in order to execute it.
926pub fn verify_program(
927    code: Vec<EbpfInstruction>,
928    calling_context: CallingContext,
929    logger: &mut dyn VerifierLogger,
930) -> Result<VerifiedEbpfProgram, EbpfError> {
931    if code.len() > BPF_MAX_INSTS {
932        return error_and_log(logger, "ebpf program too long");
933    }
934    // Pre-scan the program to ensure all LDDW instructions are structurally well-formed,
935    // even if they are in unreachable code. This is necessary because:
936    // 1. The linker may rewrite LDDW instructions (e.g., to resolve map pointers) and
937    //    expects them to be well-formed.
938    // 2. It ensures that the second part of every LDDW has a `code` of 0 (invalid opcode).
939    //    This guarantees that any attempt to jump directly into the second part of an
940    //    LDDW will be rejected during verification as an invalid instruction.
941    let mut scan_pc = 0;
942    while scan_pc < code.len() {
943        let inst = &code[scan_pc];
944        if inst.code() == BPF_LDDW {
945            let Some(next_instruction) = code.get(scan_pc + 1) else {
946                return error_and_log(logger, "incomplete lddw");
947            };
948            if next_instruction.code() != 0
949                || next_instruction.offset() != 0
950                || next_instruction.src_reg() != 0
951                || next_instruction.dst_reg() != 0
952            {
953                return error_and_log(logger, "invalid lddw");
954            }
955            scan_pc += 2;
956        } else {
957            scan_pc += 1;
958        }
959    }
960
961    let mut context = ComputationContext::default();
962    for (i, t) in calling_context.args.iter().enumerate() {
963        // The parameter registers are r1 to r5.
964        context.set_reg((i + 1) as u8, t.clone()).map_err(EbpfError::ProgramVerifyError)?;
965    }
966    let states = vec![context];
967    let mut verification_context = VerificationContext {
968        calling_context,
969        logger,
970        states,
971        code: &code,
972        counter: 0,
973        iteration: 0,
974        terminating_contexts: Default::default(),
975        struct_access_instructions: Default::default(),
976    };
977    while let Some(mut context) = verification_context.states.pop() {
978        if let Some(terminating_contexts) =
979            verification_context.terminating_contexts.get(&context.pc)
980        {
981            // Check whether there exist a context that terminate and prove that this context does
982            // also terminate.
983            if let Some(ending_context) =
984                terminating_contexts.iter().find(|c| c.computation_context >= context)
985            {
986                // One such context has been found, this proves the current context terminates.
987                // If the context has a parent, register the data dependencies and try to terminate
988                // it.
989                if let Some(parent) = context.parent.take() {
990                    parent.dependencies.lock().push(ending_context.dependencies.clone());
991                    if let Some(parent) = Arc::into_inner(parent) {
992                        parent
993                            .terminate(&mut verification_context)
994                            .map_err(EbpfError::ProgramVerifyError)?;
995                    }
996                }
997                continue;
998            }
999        }
1000        if verification_context.iteration > 10 * BPF_MAX_INSTS {
1001            return error_and_log(verification_context.logger, "bpf byte code does not terminate");
1002        }
1003        if context.pc >= code.len() {
1004            return error_and_log(verification_context.logger, "pc out of bounds");
1005        }
1006        if let Err(message) = context.visit(&mut verification_context, code[context.pc]) {
1007            let message = format!("at PC {}: {}", context.pc, message);
1008            return error_and_log(verification_context.logger, message);
1009        }
1010        if context.terminated {
1011            context.terminate(&mut verification_context).map_err(EbpfError::ProgramVerifyError)?;
1012        }
1013        verification_context.iteration += 1;
1014    }
1015
1016    let struct_access_instructions =
1017        verification_context.struct_access_instructions.into_values().collect::<Vec<_>>();
1018    let CallingContext { maps, args, .. } = verification_context.calling_context;
1019    Ok(VerifiedEbpfProgram { code, struct_access_instructions, maps, args })
1020}
1021
1022struct VerificationContext<'a> {
1023    /// The type information for the program arguments and the registered functions.
1024    calling_context: CallingContext,
1025    /// The logger to use.
1026    logger: &'a mut dyn VerifierLogger,
1027    /// The `ComputationContext` yet to be validated.
1028    states: Vec<ComputationContext>,
1029    /// The program being analyzed.
1030    code: &'a [EbpfInstruction],
1031    /// A counter used to generated unique ids for memory buffers and maps.
1032    counter: u64,
1033    /// The current iteration of the verifier. Used to ensure termination by limiting the number of
1034    /// iteration before bailing out.
1035    iteration: usize,
1036    /// Keep track of the context that terminates at a given pc. The list of context will all be
1037    /// incomparables as each time a bigger context is computed, the smaller ones are removed from
1038    /// the list.
1039    terminating_contexts: BTreeMap<ProgramCounter, Vec<TerminatingContext>>,
1040    /// The current set of struct access instructions that will need to be updated when the
1041    /// program is linked. This is also used to ensure that a given instruction always loads the
1042    /// same field. If this is not the case, the verifier will reject the program.
1043    struct_access_instructions: HashMap<ProgramCounter, StructAccess>,
1044}
1045
1046impl<'a> VerificationContext<'a> {
1047    fn next_id(&mut self) -> MemoryId {
1048        let id = self.counter;
1049        self.counter += 1;
1050        MemoryId { namespace: Namespace::Verification, id, parent: None }
1051    }
1052
1053    /// Register an instruction that loads or stores a struct field. These instructions will need
1054    /// to updated later when the program is linked.
1055    fn register_struct_access(&mut self, struct_access: StructAccess) -> Result<(), String> {
1056        match self.struct_access_instructions.entry(struct_access.pc) {
1057            std::collections::hash_map::Entry::Vacant(entry) => {
1058                entry.insert(struct_access);
1059            }
1060            std::collections::hash_map::Entry::Occupied(entry) => {
1061                if *entry.get() != struct_access {
1062                    return Err("Inconsistent struct field access".to_string());
1063                }
1064            }
1065        }
1066        Ok(())
1067    }
1068}
1069
1070const STACK_ELEMENT_SIZE: usize = std::mem::size_of::<u64>();
1071
1072/// An offset inside the stack. The offset is from the end of the stack.
1073/// downward.
1074#[derive(Clone, Copy, Debug, PartialEq)]
1075pub struct StackOffset(ScalarValueData);
1076
1077impl Default for StackOffset {
1078    fn default() -> Self {
1079        Self(BPF_STACK_SIZE.into())
1080    }
1081}
1082
1083impl StackOffset {
1084    /// Whether the current offset is valid for element access.
1085    fn is_valid_offset(&self) -> bool {
1086        self.0.is_known() && self.0.value < (BPF_STACK_SIZE as u64)
1087    }
1088
1089    /// Whether the current offset is within the stack bounds, inclusive of `BPF_STACK_SIZE`.
1090    fn is_within_stack(&self) -> bool {
1091        self.0.is_known() && self.0.value <= (BPF_STACK_SIZE as u64)
1092    }
1093
1094    /// The value of the register.
1095    fn reg(&self) -> ScalarValueData {
1096        self.0
1097    }
1098
1099    /// The index into the stack array this offset points to. Can be called only if
1100    /// `is_within_stack()` is true.
1101    fn array_index(&self) -> usize {
1102        debug_assert!(self.is_within_stack());
1103        usize::try_from(self.0.value).unwrap() / STACK_ELEMENT_SIZE
1104    }
1105
1106    /// The offset inside the aligned u64 in the stack.
1107    fn sub_index(&self) -> usize {
1108        debug_assert!(self.is_within_stack());
1109        usize::try_from(self.0.value).unwrap() % STACK_ELEMENT_SIZE
1110    }
1111
1112    fn add<T: Into<ScalarValueData>>(self, rhs: T) -> Self {
1113        Self(self.0 + rhs)
1114    }
1115}
1116
1117/// The state of the stack
1118#[derive(Clone, Debug, Default, PartialEq)]
1119struct Stack {
1120    data: HashMap<usize, Type>,
1121}
1122
1123impl Stack {
1124    /// Replace all instances of the NullOr type with the given `null_id` to either 0 or the
1125    /// subtype depending on `is_null`
1126    fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
1127        for (_, t) in self.data.iter_mut() {
1128            t.set_null(null_id, is_null);
1129        }
1130    }
1131
1132    fn get(&self, index: usize) -> &Type {
1133        self.data.get(&index).unwrap_or(&Type::UNINITIALIZED)
1134    }
1135
1136    fn set(&mut self, index: usize, t: Type) {
1137        if t == Type::UNINITIALIZED {
1138            self.data.remove(&index);
1139        } else {
1140            self.data.insert(index, t);
1141        }
1142    }
1143
1144    fn extract_sub_value(value: u64, offset: usize, byte_count: usize) -> u64 {
1145        NativeEndian::read_uint(&value.as_bytes()[offset..], byte_count)
1146    }
1147
1148    fn insert_sub_value(mut original: u64, value: u64, width: DataWidth, offset: usize) -> u64 {
1149        let byte_count = width.bytes();
1150        let original_buf = original.as_mut_bytes();
1151        let value_buf = value.as_bytes();
1152        original_buf[offset..(byte_count + offset)].copy_from_slice(&value_buf[..byte_count]);
1153        original
1154    }
1155
1156    fn write_data_ptr(
1157        &mut self,
1158        pc: ProgramCounter,
1159        mut offset: StackOffset,
1160        bytes: u64,
1161    ) -> Result<(), String> {
1162        for i in 0..bytes {
1163            self.store(offset, Type::UNKNOWN_SCALAR, DataWidth::U8)?;
1164            offset = offset.add(1);
1165        }
1166        Ok(())
1167    }
1168
1169    fn read_data_ptr(
1170        &self,
1171        pc: ProgramCounter,
1172        offset: StackOffset,
1173        bytes: u64,
1174    ) -> Result<(), String> {
1175        let read_element =
1176            |index: usize, start_offset: usize, end_offset: usize| -> Result<(), String> {
1177                match self.get(index) {
1178                    Type::ScalarValue(data) => {
1179                        debug_assert!(end_offset > start_offset);
1180                        let unwritten_bits = Self::extract_sub_value(
1181                            data.unwritten_mask,
1182                            start_offset,
1183                            end_offset - start_offset,
1184                        );
1185                        if unwritten_bits == 0 {
1186                            Ok(())
1187                        } else {
1188                            Err("reading unwritten value from the stack".to_string())
1189                        }
1190                    }
1191                    _ => Err("invalid read from the stack".to_string()),
1192                }
1193            };
1194        if bytes == 0 {
1195            return Ok(());
1196        }
1197
1198        if bytes as usize > BPF_STACK_SIZE {
1199            return Err("stack overflow".to_string());
1200        }
1201
1202        if !offset.is_valid_offset() {
1203            return Err("invalid stack offset".to_string());
1204        }
1205
1206        let end_offset = offset.add(bytes);
1207        if !end_offset.is_within_stack() {
1208            return Err("stack overflow".to_string())?;
1209        }
1210
1211        // Handle the case where all the data is contained in a single element excluding the last
1212        // byte (the case when the read ends at an element edge, i.e. `end_offset.sub_index()==0`,
1213        // is covered by the default path below).
1214        if offset.array_index() == end_offset.array_index() {
1215            return read_element(offset.array_index(), offset.sub_index(), end_offset.sub_index());
1216        }
1217
1218        // Handle the first element, that might be partial.
1219        read_element(offset.array_index(), offset.sub_index(), STACK_ELEMENT_SIZE)?;
1220
1221        // Handle the last element, that might be partial.
1222        if end_offset.sub_index() != 0 {
1223            read_element(end_offset.array_index(), 0, end_offset.sub_index())?;
1224        }
1225
1226        // Handle the any full type between beginning and end.
1227        for i in (offset.array_index() + 1)..end_offset.array_index() {
1228            read_element(i, 0, STACK_ELEMENT_SIZE)?;
1229        }
1230
1231        Ok(())
1232    }
1233
1234    fn store(&mut self, offset: StackOffset, value: Type, width: DataWidth) -> Result<(), String> {
1235        if !offset.is_valid_offset() {
1236            return Err("out of bounds store".to_string());
1237        }
1238        if offset.sub_index() % width.bytes() != 0 {
1239            return Err("misaligned access".to_string());
1240        }
1241
1242        let index = offset.array_index();
1243        if width == DataWidth::U64 {
1244            self.set(index, value);
1245        } else {
1246            match value {
1247                Type::ScalarValue(data) => {
1248                    let old_data = match self.get(index) {
1249                        Type::ScalarValue(data) => *data,
1250                        _ => {
1251                            // The value in the stack is not a scalar. Let consider it an scalar
1252                            // value with no written bits.
1253                            ScalarValueData::UNINITIALIZED
1254                        }
1255                    };
1256                    let sub_index = offset.sub_index();
1257                    let value =
1258                        Self::insert_sub_value(old_data.value, data.value, width, sub_index);
1259                    let unknown_mask = Self::insert_sub_value(
1260                        old_data.unknown_mask,
1261                        data.unknown_mask,
1262                        width,
1263                        sub_index,
1264                    );
1265                    let unwritten_mask = Self::insert_sub_value(
1266                        old_data.unwritten_mask,
1267                        data.unwritten_mask,
1268                        width,
1269                        sub_index,
1270                    );
1271                    let urange = U64Range::compute_range_for_bytes_swap(
1272                        old_data.urange,
1273                        data.urange,
1274                        sub_index,
1275                        0,
1276                        width.bytes(),
1277                    );
1278                    self.set(
1279                        index,
1280                        Type::ScalarValue(ScalarValueData::new(
1281                            value,
1282                            unknown_mask,
1283                            unwritten_mask,
1284                            urange,
1285                        )),
1286                    );
1287                }
1288                _ => {
1289                    return Err("cannot store part of a non scalar value on the stack".to_string());
1290                }
1291            }
1292        }
1293        Ok(())
1294    }
1295
1296    fn load(&self, offset: StackOffset, width: DataWidth) -> Result<Type, String> {
1297        if !offset.is_valid_offset() {
1298            return Err("out of bounds load".to_string());
1299        }
1300        if offset.sub_index() % width.bytes() != 0 {
1301            return Err("misaligned access".to_string());
1302        }
1303
1304        let index = offset.array_index();
1305        let loaded_type = self.get(index).clone();
1306        let result = if width == DataWidth::U64 {
1307            loaded_type
1308        } else {
1309            match loaded_type {
1310                Type::ScalarValue(data) => {
1311                    let sub_index = offset.sub_index();
1312                    let value = Self::extract_sub_value(data.value, sub_index, width.bytes());
1313                    let unknown_mask =
1314                        Self::extract_sub_value(data.unknown_mask, sub_index, width.bytes());
1315                    let unwritten_mask =
1316                        Self::extract_sub_value(data.unwritten_mask, sub_index, width.bytes());
1317                    let urange = U64Range::compute_range_for_bytes_swap(
1318                        0.into(),
1319                        data.urange,
1320                        0,
1321                        sub_index,
1322                        width.bytes(),
1323                    );
1324                    Type::ScalarValue(ScalarValueData::new(
1325                        value,
1326                        unknown_mask,
1327                        unwritten_mask,
1328                        urange,
1329                    ))
1330                }
1331                _ => return Err(format!("incorrect load of {} bytes", width.bytes())),
1332            }
1333        };
1334        if !result.is_initialized() {
1335            return Err("reading unwritten value from the stack".to_string());
1336        }
1337        Ok(result)
1338    }
1339}
1340
1341/// Two types are ordered with `t1` > `t2` if a proof that a program in a state `t1` finish is also
1342/// a proof that a program in a state `t2` finish.
1343impl PartialOrd for Stack {
1344    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1345        let mut result = Ordering::Equal;
1346        let mut data_iter1 = self.data.iter().peekable();
1347        let mut data_iter2 = other.data.iter().peekable();
1348        loop {
1349            let k1 = data_iter1.peek().map(|(k, _)| *k);
1350            let k2 = data_iter2.peek().map(|(k, _)| *k);
1351            let k = match (k1, k2) {
1352                (None, None) => return Some(result),
1353                (Some(k), None) => {
1354                    data_iter1.next();
1355                    *k
1356                }
1357                (None, Some(k)) => {
1358                    data_iter2.next();
1359                    *k
1360                }
1361                (Some(k1), Some(k2)) => {
1362                    if k1 <= k2 {
1363                        data_iter1.next();
1364                    }
1365                    if k2 <= k1 {
1366                        data_iter2.next();
1367                    }
1368                    *std::cmp::min(k1, k2)
1369                }
1370            };
1371            result = associate_orderings(result, self.get(k).partial_cmp(other.get(k))?)?;
1372        }
1373    }
1374}
1375
1376macro_rules! bpf_log {
1377    ($context:ident, $verification_context:ident, $($msg:tt)*) => {
1378        let prefix = format!("{}: ({:02x})", $context.pc, $verification_context.code[$context.pc].code());
1379        let suffix = format!($($msg)*);
1380        $verification_context.logger.log(format!("{prefix} {suffix}").as_bytes());
1381    }
1382}
1383
1384/// The state of the computation as known by the verifier at a given point in time.
1385#[derive(Debug, Default)]
1386struct ComputationContext {
1387    /// The program counter.
1388    pc: ProgramCounter,
1389    /// Register 0 to 9.
1390    registers: [Type; GENERAL_REGISTER_COUNT as usize],
1391    /// The state of the stack.
1392    stack: Stack,
1393    /// The dynamically known bounds of buffers indexed by their ids.
1394    array_bounds: BTreeMap<MemoryId, u64>,
1395    /// The currently allocated resources.
1396    resources: HashSet<MemoryId>,
1397    /// The previous context in the computation.
1398    parent: Option<Arc<ComputationContext>>,
1399    /// The data dependencies of this context. This is used to broaden a known ending context to
1400    /// help cutting computation branches.
1401    dependencies: Mutex<Vec<DataDependencies>>,
1402    /// Whether this context has reached an exit instruction. The main loop uses this flag to
1403    /// call terminate() on the owned context, avoiding the extra Arc reference that would be
1404    /// created by cloning self in the exit handler.
1405    terminated: bool,
1406}
1407
1408impl Clone for ComputationContext {
1409    fn clone(&self) -> Self {
1410        Self {
1411            pc: self.pc,
1412            registers: self.registers.clone(),
1413            stack: self.stack.clone(),
1414            array_bounds: self.array_bounds.clone(),
1415            resources: self.resources.clone(),
1416            parent: self.parent.clone(),
1417            // dependencies are erased as they must always be used on the same instance of the
1418            // context.
1419            dependencies: Default::default(),
1420            terminated: false,
1421        }
1422    }
1423}
1424
1425/// parent and dependencies are ignored for the comparison, as they do not matter for termination.
1426impl PartialEq for ComputationContext {
1427    fn eq(&self, other: &Self) -> bool {
1428        self.pc == other.pc
1429            && self.registers == other.registers
1430            && self.stack == other.stack
1431            && self.array_bounds == other.array_bounds
1432    }
1433}
1434
1435impl ComputationContext {
1436    /// Replace all instances of the NullOr type with the given `null_id` to either 0 or the
1437    /// subtype depending on `is_null`
1438    fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
1439        for i in 0..self.registers.len() {
1440            self.registers[i].set_null(null_id, is_null);
1441        }
1442        self.stack.set_null(null_id, is_null);
1443    }
1444
1445    fn reg(&self, index: Register) -> Result<Type, String> {
1446        if index >= REGISTER_COUNT {
1447            return Err(format!("R{index} is invalid"));
1448        }
1449        if index < GENERAL_REGISTER_COUNT {
1450            Ok(self.registers[index as usize].clone())
1451        } else {
1452            Ok(Type::PtrToStack { offset: StackOffset::default() })
1453        }
1454    }
1455
1456    fn set_reg(&mut self, index: Register, reg_type: Type) -> Result<(), String> {
1457        if index >= GENERAL_REGISTER_COUNT {
1458            return Err(format!("R{index} is invalid"));
1459        }
1460        self.registers[index as usize] = reg_type;
1461        Ok(())
1462    }
1463
1464    fn update_array_bounds(&mut self, id: MemoryId, new_bound: ScalarValueData) {
1465        let new_bound_min = new_bound.min();
1466        self.array_bounds
1467            .entry(id)
1468            .and_modify(|v| *v = std::cmp::max(*v, new_bound_min))
1469            .or_insert(new_bound_min);
1470    }
1471
1472    fn get_map_schema(&self, argument: u8) -> Result<MapSchema, String> {
1473        match self.reg(argument + 1)? {
1474            Type::ConstPtrToMap { schema, .. } => Ok(schema),
1475            _ => Err(format!("No map found at argument {argument}")),
1476        }
1477    }
1478
1479    fn next(&self) -> Result<Self, String> {
1480        let parent = Some(Arc::new(self.clone()));
1481        self.jump_with_offset(0, parent)
1482    }
1483
1484    /// Returns a new `ComputationContext` where the pc has jump by `offset + 1`. In particular,
1485    /// the next instruction is reached with `jump_with_offset(0)`.
1486    fn jump_with_offset(&self, offset: i16, parent: Option<Arc<Self>>) -> Result<Self, String> {
1487        let pc = self
1488            .pc
1489            .checked_add_signed(offset.into())
1490            .and_then(|v| v.checked_add_signed(1))
1491            .ok_or_else(|| "jump outside of program".to_string())?;
1492        let result = Self {
1493            pc,
1494            registers: self.registers.clone(),
1495            stack: self.stack.clone(),
1496            array_bounds: self.array_bounds.clone(),
1497            resources: self.resources.clone(),
1498            parent,
1499            dependencies: Default::default(),
1500            terminated: false,
1501        };
1502        Ok(result)
1503    }
1504
1505    fn check_memory_access(
1506        &self,
1507        dst_offset: ScalarValueData,
1508        dst_buffer_size: u64,
1509        instruction_offset: i16,
1510        width: usize,
1511    ) -> Result<(), String> {
1512        let memory_range = dst_offset.urange + instruction_offset + U64Range::new(0, width as u64);
1513        if memory_range.max > dst_buffer_size {
1514            return Err("out of bound access".to_string());
1515        }
1516        Ok(())
1517    }
1518
1519    fn store_memory(
1520        &mut self,
1521        context: &mut VerificationContext<'_>,
1522        addr: &Type,
1523        field: Field,
1524        value: Type,
1525    ) -> Result<(), String> {
1526        let addr = addr.inner(self)?;
1527        match *addr {
1528            Type::PtrToStack { offset } => {
1529                let offset_sum = offset.add(field.offset);
1530                return self.stack.store(offset_sum, value, field.width);
1531            }
1532            Type::PtrToMemory { offset, buffer_size, .. } => {
1533                self.check_memory_access(offset, buffer_size, field.offset, field.width.bytes())?;
1534            }
1535            Type::PtrToStruct { ref id, offset, ref descriptor, .. } => {
1536                let field_desc = descriptor
1537                    .find_field(offset, field)
1538                    .ok_or_else(|| "incorrect store".to_string())?;
1539
1540                if !matches!(field_desc.field_type, FieldType::MutableScalar { .. }) {
1541                    return Err("store to a read-only field".to_string());
1542                }
1543
1544                context.register_struct_access(StructAccess {
1545                    pc: self.pc,
1546                    memory_id: id.clone(),
1547                    field_offset: field_desc.offset,
1548                    is_32_bit_ptr_load: false,
1549                })?;
1550            }
1551            Type::PtrToArray { ref id, offset } => {
1552                self.check_memory_access(
1553                    offset,
1554                    *self.array_bounds.get(&id).unwrap_or(&0),
1555                    field.offset,
1556                    field.width.bytes(),
1557                )?;
1558            }
1559            _ => return Err("incorrect store".to_string()),
1560        }
1561
1562        match value {
1563            Type::ScalarValue(data) if data.is_fully_initialized() => {}
1564            // Private data should not be leaked.
1565            _ => return Err("incorrect store".to_string()),
1566        }
1567        Ok(())
1568    }
1569
1570    fn load_memory(
1571        &self,
1572        context: &mut VerificationContext<'_>,
1573        addr: &Type,
1574        field: Field,
1575    ) -> Result<Type, String> {
1576        let addr = addr.inner(self)?;
1577        match *addr {
1578            Type::PtrToStack { offset } => {
1579                let offset_sum = offset.add(field.offset);
1580                self.stack.load(offset_sum, field.width)
1581            }
1582            Type::PtrToMemory { ref id, offset, buffer_size, .. } => {
1583                self.check_memory_access(offset, buffer_size, field.offset, field.width.bytes())?;
1584                Ok(Type::UNKNOWN_SCALAR)
1585            }
1586            Type::PtrToStruct { ref id, offset, ref descriptor, .. } => {
1587                let field_desc = descriptor
1588                    .find_field(offset, field)
1589                    .ok_or_else(|| "incorrect load".to_string())?;
1590
1591                let (return_type, is_32_bit_ptr_load) = match &field_desc.field_type {
1592                    FieldType::Scalar { .. } | FieldType::MutableScalar { .. } => {
1593                        (Type::UNKNOWN_SCALAR, false)
1594                    }
1595                    FieldType::PtrToArray { id: array_id, is_32_bit } => (
1596                        Type::PtrToArray { id: array_id.prepended(id.clone()), offset: 0.into() },
1597                        *is_32_bit,
1598                    ),
1599                    FieldType::PtrToEndArray { id: array_id, is_32_bit } => {
1600                        (Type::PtrToEndArray { id: array_id.prepended(id.clone()) }, *is_32_bit)
1601                    }
1602                    FieldType::PtrToMemory { id: memory_id, buffer_size, is_32_bit } => (
1603                        Type::PtrToMemory {
1604                            id: memory_id.prepended(id.clone()),
1605                            offset: 0.into(),
1606                            buffer_size: *buffer_size as u64,
1607                        },
1608                        *is_32_bit,
1609                    ),
1610                    FieldType::NullablePtrToMemory { id: memory_id, buffer_size, is_32_bit } => {
1611                        let id = memory_id.prepended(id.clone());
1612                        (
1613                            Type::NullOr {
1614                                id: id.clone(),
1615                                inner: Box::new(Type::PtrToMemory {
1616                                    id,
1617                                    offset: 0.into(),
1618                                    buffer_size: *buffer_size as u64,
1619                                }),
1620                            },
1621                            *is_32_bit,
1622                        )
1623                    }
1624                };
1625
1626                context.register_struct_access(StructAccess {
1627                    pc: self.pc,
1628                    memory_id: id.clone(),
1629                    field_offset: field_desc.offset,
1630                    is_32_bit_ptr_load,
1631                })?;
1632
1633                Ok(return_type)
1634            }
1635            Type::PtrToArray { ref id, offset } => {
1636                self.check_memory_access(
1637                    offset,
1638                    *self.array_bounds.get(&id).unwrap_or(&0),
1639                    field.offset,
1640                    field.width.bytes(),
1641                )?;
1642                Ok(Type::UNKNOWN_SCALAR)
1643            }
1644            _ => Err("incorrect load".to_string()),
1645        }
1646    }
1647
1648    /**
1649     * Given the given `return_value` in a method signature, return the concrete type to use,
1650     * updating the `next` context is needed.
1651     *
1652     * `maybe_null` is true is the computed value will be a descendant of a `NullOr` type.
1653     */
1654    fn resolve_return_value(
1655        &self,
1656        verification_context: &mut VerificationContext<'_>,
1657        return_value: &Type,
1658        next: &mut ComputationContext,
1659        maybe_null: bool,
1660    ) -> Result<Type, String> {
1661        match return_value {
1662            Type::AliasParameter { parameter_index } => self.reg(parameter_index + 1),
1663            Type::ReleasableParameter { id, inner } => {
1664                let id = verification_context.next_id().prepended(id.clone());
1665                if !maybe_null {
1666                    next.resources.insert(id.clone());
1667                }
1668                Ok(Type::Releasable {
1669                    id,
1670                    inner: Box::new(self.resolve_return_value(
1671                        verification_context,
1672                        inner,
1673                        next,
1674                        maybe_null,
1675                    )?),
1676                })
1677            }
1678            Type::NullOrParameter(t) => {
1679                let id = verification_context.next_id();
1680                Ok(Type::NullOr {
1681                    id,
1682                    inner: Box::new(self.resolve_return_value(
1683                        verification_context,
1684                        t,
1685                        next,
1686                        true,
1687                    )?),
1688                })
1689            }
1690            Type::MapValueParameter { map_ptr_index } => {
1691                let schema = self.get_map_schema(*map_ptr_index)?;
1692                let id = verification_context.next_id();
1693                Ok(Type::PtrToMemory {
1694                    id,
1695                    offset: 0.into(),
1696                    buffer_size: schema.value_size as u64,
1697                })
1698            }
1699            Type::MemoryParameter { size, .. } => {
1700                let buffer_size = size.size(self)?;
1701                let id = verification_context.next_id();
1702                Ok(Type::PtrToMemory { id, offset: 0.into(), buffer_size })
1703            }
1704            t => Ok(t.clone()),
1705        }
1706    }
1707
1708    fn compute_source(&self, src: Source) -> Result<Type, String> {
1709        match src {
1710            Source::Reg(reg) => self.reg(reg),
1711            Source::Value(v) => Ok(v.into()),
1712        }
1713    }
1714
1715    fn apply_computation(
1716        context: &ComputationContext,
1717        op1: Type,
1718        op2: Type,
1719        alu_type: AluType,
1720        op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1721    ) -> Result<Type, String> {
1722        let result: Type = match (alu_type, op1.inner(context)?, op2.inner(context)?) {
1723            (_, Type::ScalarValue(data1), Type::ScalarValue(data2)) => op(*data1, *data2).into(),
1724            (
1725                AluType::Add,
1726                Type::ScalarValue(_),
1727                Type::PtrToStack { .. } | Type::PtrToMemory { .. } | Type::PtrToStruct { .. },
1728            ) => {
1729                return Self::apply_computation(context, op2, op1, alu_type, op);
1730            }
1731            (alu_type, Type::PtrToStack { offset: x }, Type::ScalarValue(data))
1732                if alu_type.is_ptr_compatible() =>
1733            {
1734                Type::PtrToStack { offset: run_on_stack_offset(*x, |x| op(x, *data)) }
1735            }
1736            (
1737                alu_type,
1738                Type::PtrToMemory { id, offset: x, buffer_size },
1739                Type::ScalarValue(data),
1740            ) if alu_type.is_ptr_compatible() => {
1741                let offset = op(*x, *data);
1742                Type::PtrToMemory { id: id.clone(), offset, buffer_size: *buffer_size }
1743            }
1744            (
1745                alu_type,
1746                Type::PtrToStruct { id, offset: x, descriptor },
1747                Type::ScalarValue(data),
1748            ) if alu_type.is_ptr_compatible() => {
1749                let offset = op(*x, *data);
1750                Type::PtrToStruct { id: id.clone(), offset, descriptor: descriptor.clone() }
1751            }
1752            (AluType::Add, Type::PtrToArray { id, offset: x }, Type::ScalarValue(data)) => {
1753                let offset = x.checked_add(*data).ok_or_else(|| format!("XXX"))?;
1754                Type::PtrToArray { id: id.clone(), offset }
1755            }
1756            (AluType::Sub, Type::PtrToArray { id, offset: x }, Type::ScalarValue(data)) => {
1757                let offset = x.checked_sub(*data).ok_or_else(|| format!("XXX"))?;
1758                Type::PtrToArray { id: id.clone(), offset }
1759            }
1760            (
1761                AluType::Sub,
1762                Type::PtrToMemory { id: id1, offset: x1, .. },
1763                Type::PtrToMemory { id: id2, offset: x2, .. },
1764            )
1765            | (
1766                AluType::Sub,
1767                Type::PtrToStruct { id: id1, offset: x1, .. },
1768                Type::PtrToStruct { id: id2, offset: x2, .. },
1769            )
1770            | (
1771                AluType::Sub,
1772                Type::PtrToArray { id: id1, offset: x1 },
1773                Type::PtrToArray { id: id2, offset: x2 },
1774            ) if id1 == id2 => Type::from(op(*x1, *x2)),
1775            (AluType::Sub, Type::PtrToStack { offset: x1 }, Type::PtrToStack { offset: x2 }) => {
1776                Type::from(op(x1.reg(), x2.reg()))
1777            }
1778            (
1779                AluType::Sub,
1780                Type::PtrToArray { id: id1, .. },
1781                Type::PtrToEndArray { id: id2, .. },
1782            )
1783            | (
1784                AluType::Sub,
1785                Type::PtrToEndArray { id: id1, .. },
1786                Type::PtrToArray { id: id2, .. },
1787            ) if id1 == id2 => Type::UNKNOWN_SCALAR,
1788            _ => Type::default(),
1789        };
1790        Ok(result)
1791    }
1792
1793    fn alu(
1794        &mut self,
1795        op_name: Option<&str>,
1796        verification_context: &mut VerificationContext<'_>,
1797        dst: Register,
1798        src: Source,
1799        alu_type: AluType,
1800        op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1801    ) -> Result<(), String> {
1802        if let Some(op_name) = op_name {
1803            bpf_log!(
1804                self,
1805                verification_context,
1806                "{op_name} {}, {}",
1807                display_register(dst),
1808                display_source(src)
1809            );
1810        }
1811        let op1 = self.reg(dst)?;
1812        let op2 = self.compute_source(src)?;
1813        let result = Self::apply_computation(self, op1, op2, alu_type, op)?;
1814        let mut next = self.next()?;
1815        next.set_reg(dst, result)?;
1816        verification_context.states.push(next);
1817        Ok(())
1818    }
1819
1820    fn log_atomic_operation(
1821        &mut self,
1822        op_name: &str,
1823        verification_context: &mut VerificationContext<'_>,
1824        fetch: bool,
1825        dst: Register,
1826        offset: i16,
1827        src: Register,
1828    ) {
1829        bpf_log!(
1830            self,
1831            verification_context,
1832            "lock {}{} [{}{}], {}",
1833            if fetch { "fetch " } else { "" },
1834            op_name,
1835            display_register(dst),
1836            print_offset(offset),
1837            display_register(src),
1838        );
1839    }
1840
1841    fn raw_atomic_operation(
1842        &mut self,
1843        op_name: &str,
1844        verification_context: &mut VerificationContext<'_>,
1845        width: DataWidth,
1846        fetch: bool,
1847        dst: Register,
1848        offset: i16,
1849        src: Register,
1850        op: impl FnOnce(&ComputationContext, Type, Type) -> Result<Type, String>,
1851    ) -> Result<(), String> {
1852        self.log_atomic_operation(op_name, verification_context, fetch, dst, offset, src);
1853        let addr = self.reg(dst)?;
1854        let value = self.reg(src)?;
1855        let field = Field::new(offset, width);
1856        let loaded_type = self.load_memory(verification_context, &addr, field)?;
1857        let result = op(self, loaded_type.clone(), value)?;
1858        let mut next = self.next()?;
1859        next.store_memory(verification_context, &addr, field, result)?;
1860        if fetch {
1861            next.set_reg(src, loaded_type)?;
1862        }
1863        verification_context.states.push(next);
1864        Ok(())
1865    }
1866
1867    fn atomic_operation(
1868        &mut self,
1869        op_name: &str,
1870        verification_context: &mut VerificationContext<'_>,
1871        width: DataWidth,
1872        fetch: bool,
1873        dst: Register,
1874        offset: i16,
1875        src: Register,
1876        alu_type: AluType,
1877        op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1878    ) -> Result<(), String> {
1879        self.raw_atomic_operation(
1880            op_name,
1881            verification_context,
1882            width,
1883            fetch,
1884            dst,
1885            offset,
1886            src,
1887            |context: &ComputationContext, v1: Type, v2: Type| {
1888                Self::apply_computation(context, v1, v2, alu_type, op)
1889            },
1890        )
1891    }
1892
1893    fn raw_atomic_cmpxchg(
1894        &mut self,
1895        op_name: &str,
1896        verification_context: &mut VerificationContext<'_>,
1897        dst: Register,
1898        offset: i16,
1899        src: Register,
1900        jump_width: JumpWidth,
1901        op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
1902    ) -> Result<(), String> {
1903        self.log_atomic_operation(op_name, verification_context, true, dst, offset, src);
1904        let width = match jump_width {
1905            JumpWidth::W32 => DataWidth::U32,
1906            JumpWidth::W64 => DataWidth::U64,
1907        };
1908        let addr = self.reg(dst)?;
1909        let field = Field::new(offset, width);
1910        let dst = self.load_memory(verification_context, &addr, field)?;
1911        let value = self.reg(src)?;
1912        let r0 = self.reg(0)?;
1913        let branch = self.compute_branch(jump_width, &dst, &r0, op)?;
1914        // r0 = dst
1915        if branch.unwrap_or(true) {
1916            let mut next = self.next()?;
1917            let (dst, r0) =
1918                Type::constraint(&mut next, JumpType::Eq, jump_width, dst.clone(), r0.clone())?;
1919            next.set_reg(0, dst)?;
1920            next.store_memory(verification_context, &addr, field, value)?;
1921            verification_context.states.push(next);
1922        }
1923        // r0 != dst
1924        if !branch.unwrap_or(false) {
1925            let mut next = self.next()?;
1926            let (dst, r0) = Type::constraint(&mut next, JumpType::Ne, jump_width, dst, r0)?;
1927            next.set_reg(0, dst.clone())?;
1928            next.store_memory(verification_context, &addr, field, dst)?;
1929            verification_context.states.push(next);
1930        }
1931
1932        Ok(())
1933    }
1934    fn endianness<BO: ByteOrder>(
1935        &mut self,
1936        op_name: &str,
1937        verification_context: &mut VerificationContext<'_>,
1938        dst: Register,
1939        width: DataWidth,
1940    ) -> Result<(), String> {
1941        bpf_log!(self, verification_context, "{op_name}{} {}", width.bits(), display_register(dst),);
1942        let bit_op = |value: u64| match width {
1943            DataWidth::U16 => BO::read_u16((value as u16).as_bytes()) as u64,
1944            DataWidth::U32 => BO::read_u32((value as u32).as_bytes()) as u64,
1945            DataWidth::U64 => BO::read_u64(value.as_bytes()),
1946            _ => {
1947                panic!("Unexpected bit width for endianness operation");
1948            }
1949        };
1950        let value = self.reg(dst)?;
1951        let new_value = match value {
1952            Type::ScalarValue(data) => Type::ScalarValue(ScalarValueData::new(
1953                bit_op(data.value),
1954                bit_op(data.unknown_mask),
1955                bit_op(data.unwritten_mask),
1956                U64Range::max(),
1957            )),
1958            _ => Type::default(),
1959        };
1960        let mut next = self.next()?;
1961        next.set_reg(dst, new_value)?;
1962        verification_context.states.push(next);
1963        Ok(())
1964    }
1965
1966    fn compute_branch(
1967        &self,
1968        jump_width: JumpWidth,
1969        op1: &Type,
1970        op2: &Type,
1971        op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
1972    ) -> Result<Option<bool>, String> {
1973        match (jump_width, op1, op2) {
1974            (_, Type::ScalarValue(data1), Type::ScalarValue(data2)) => op(*data1, *data2),
1975            (JumpWidth::W64, Type::ScalarValue(data), Type::NullOr { .. })
1976            | (JumpWidth::W64, Type::NullOr { .. }, Type::ScalarValue(data))
1977                if data.is_zero() =>
1978            {
1979                Ok(None)
1980            }
1981
1982            (JumpWidth::W64, Type::ScalarValue(data), t) if data.is_zero() && t.is_non_zero() => {
1983                op(1.into(), 0.into())
1984            }
1985
1986            (JumpWidth::W64, t, Type::ScalarValue(data)) if data.is_zero() && t.is_non_zero() => {
1987                op(0.into(), 1.into())
1988            }
1989
1990            (JumpWidth::W64, Type::PtrToStack { offset: x }, Type::PtrToStack { offset: y }) => {
1991                op(x.reg(), y.reg())
1992            }
1993
1994            (
1995                JumpWidth::W64,
1996                Type::PtrToMemory { id: id1, offset: x, .. },
1997                Type::PtrToMemory { id: id2, offset: y, .. },
1998            )
1999            | (
2000                JumpWidth::W64,
2001                Type::PtrToStruct { id: id1, offset: x, .. },
2002                Type::PtrToStruct { id: id2, offset: y, .. },
2003            )
2004            | (
2005                JumpWidth::W64,
2006                Type::PtrToArray { id: id1, offset: x, .. },
2007                Type::PtrToArray { id: id2, offset: y, .. },
2008            ) if *id1 == *id2 => op(*x, *y),
2009
2010            (JumpWidth::W64, Type::PtrToArray { id: id1, .. }, Type::PtrToEndArray { id: id2 })
2011            | (JumpWidth::W64, Type::PtrToEndArray { id: id1 }, Type::PtrToArray { id: id2, .. })
2012                if *id1 == *id2 =>
2013            {
2014                Ok(None)
2015            }
2016
2017            _ => Err(()),
2018        }
2019        .map_err(|_| "non permitted comparison".to_string())
2020    }
2021
2022    fn conditional_jump(
2023        &mut self,
2024        op_name: &str,
2025        verification_context: &mut VerificationContext<'_>,
2026        dst: Register,
2027        src: Source,
2028        offset: i16,
2029        jump_type: JumpType,
2030        jump_width: JumpWidth,
2031        op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
2032    ) -> Result<(), String> {
2033        bpf_log!(
2034            self,
2035            verification_context,
2036            "{op_name} {}, {}, {}",
2037            display_register(dst),
2038            display_source(src),
2039            if offset == 0 { format!("0") } else { print_offset(offset) },
2040        );
2041        let op1 = self.reg(dst)?;
2042        let op2 = self.compute_source(src.clone())?;
2043        let apply_constraints_and_register = |mut next: Self,
2044                                              jump_type: JumpType|
2045         -> Result<Self, String> {
2046            if jump_type != JumpType::Unknown {
2047                let (new_op1, new_op2) =
2048                    Type::constraint(&mut next, jump_type, jump_width, op1.clone(), op2.clone())?;
2049                if dst < REGISTER_COUNT {
2050                    next.set_reg(dst, new_op1)?;
2051                }
2052                match src {
2053                    Source::Reg(r) => {
2054                        next.set_reg(r, new_op2)?;
2055                    }
2056                    _ => {
2057                        // Nothing to do
2058                    }
2059                }
2060            }
2061            Ok(next)
2062        };
2063        let branch = self.compute_branch(jump_width, &op1, &op2, op)?;
2064        let parent = Some(Arc::new(self.clone()));
2065        if branch.unwrap_or(true) {
2066            // Do the jump
2067            verification_context.states.push(apply_constraints_and_register(
2068                self.jump_with_offset(offset, parent.clone())?,
2069                jump_type,
2070            )?);
2071        }
2072        if !branch.unwrap_or(false) {
2073            // Skip the jump
2074            verification_context.states.push(apply_constraints_and_register(
2075                self.jump_with_offset(0, parent)?,
2076                jump_type.invert(),
2077            )?);
2078        }
2079        Ok(())
2080    }
2081
2082    /// Handles the termination of a `ComputationContext`, performing branch cutting optimization.
2083    ///
2084    /// This method is called when it has been proven that the current context terminates (e.g.,
2085    /// reaches an `exit` instruction).
2086    ///
2087    /// The following steps are performed:
2088    /// 1. **Dependency Calculation:** The data dependencies of the context are computed based on
2089    ///    the dependencies of its terminated children and the instruction at the current PC.
2090    /// 2. **Context Broadening:** The context's state is broadened by clearing registers and stack
2091    ///    slots that are *not* in the calculated data dependencies. This optimization assumes that
2092    ///    data not used by the terminated branch is irrelevant for future execution paths.
2093    /// 3. **Termination Registration:** The broadened context is added to the set of terminating
2094    ///    contexts if it is not less than any existing terminating context at the same PC.
2095    /// 4. **Parent Termination:** If all the children of the current context have terminated,
2096    ///    its parent context is recursively terminated.
2097    fn terminate(self, verification_context: &mut VerificationContext<'_>) -> Result<(), String> {
2098        let mut next = Some(self);
2099        // Because of the potential length of the parent chain, do not use recursion.
2100        while let Some(mut current) = next.take() {
2101            // Take the parent to process it at the end and not keep it in the terminating
2102            // contexts.
2103            let parent = current.parent.take();
2104
2105            // 1. Compute the dependencies of the context using the dependencies of its children
2106            //    and the actual operation.
2107            let mut dependencies = DataDependencies::default();
2108            for dependency in current.dependencies.get_mut().iter() {
2109                dependencies.merge(dependency);
2110            }
2111
2112            dependencies.visit(
2113                &mut DataDependenciesVisitorContext {
2114                    calling_context: &verification_context.calling_context,
2115                    computation_context: &current,
2116                },
2117                verification_context.code[current.pc],
2118            )?;
2119
2120            // 2. Clear the state depending on the dependencies states
2121            for register in 0..GENERAL_REGISTER_COUNT {
2122                if !dependencies.registers.contains(&register) {
2123                    current.set_reg(register, Default::default())?;
2124                }
2125            }
2126            current.stack.data.retain(|k, _| dependencies.stack.contains(k));
2127
2128            // 3. Add the cleared state to the set of `terminating_contexts`
2129            let terminating_contexts =
2130                verification_context.terminating_contexts.entry(current.pc).or_default();
2131            let mut is_dominated = false;
2132            terminating_contexts.retain(|c| match c.computation_context.partial_cmp(&current) {
2133                Some(Ordering::Less) => false,
2134                Some(Ordering::Equal) | Some(Ordering::Greater) => {
2135                    // If the list contains a context greater or equal to the current one, it
2136                    // should not be added.
2137                    is_dominated = true;
2138                    true
2139                }
2140                _ => true,
2141            });
2142            if !is_dominated {
2143                terminating_contexts.push(TerminatingContext {
2144                    computation_context: current,
2145                    dependencies: dependencies.clone(),
2146                });
2147            }
2148
2149            // 4. Register the computed dependencies in our parent, and terminate it if all
2150            //    dependencies has been computed.
2151            if let Some(parent) = parent {
2152                parent.dependencies.lock().push(dependencies);
2153                // To check whether all dependencies have been computed, rely on the fact that the Arc
2154                // count of the parent keep track of how many dependencies are left.
2155                next = Arc::into_inner(parent);
2156            }
2157        }
2158        Ok(())
2159    }
2160}
2161
2162impl Drop for ComputationContext {
2163    fn drop(&mut self) {
2164        let mut next = self.parent.take().and_then(Arc::into_inner);
2165        // Because of the potential length of the parent chain, do not use recursion.
2166        while let Some(mut current) = next {
2167            next = current.parent.take().and_then(Arc::into_inner);
2168        }
2169    }
2170}
2171
2172/// Two types are ordered with `t1` > `t2` if a proof that a program in a state `t1` finish is also
2173/// a proof that a program in a state `t2` finish.
2174impl PartialOrd for ComputationContext {
2175    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
2176        if self.pc != other.pc || self.resources != other.resources {
2177            return None;
2178        }
2179        let mut result = self.stack.partial_cmp(&other.stack)?;
2180        result = associate_orderings(
2181            result,
2182            Type::compare_list(self.registers.iter(), other.registers.iter())?,
2183        )?;
2184        let mut array_bound_iter1 = self.array_bounds.iter().peekable();
2185        let mut array_bound_iter2 = other.array_bounds.iter().peekable();
2186        let result = loop {
2187            match (array_bound_iter1.peek().cloned(), array_bound_iter2.peek().cloned()) {
2188                (None, None) => break result,
2189                (None, _) => break associate_orderings(result, Ordering::Greater)?,
2190                (_, None) => break associate_orderings(result, Ordering::Less)?,
2191                (Some((k1, v1)), Some((k2, v2))) => match k1.cmp(k2) {
2192                    Ordering::Equal => {
2193                        array_bound_iter1.next();
2194                        array_bound_iter2.next();
2195                        // The comparison is intentionally reversed (v2.cmp(v1) instead of
2196                        // v1.cmp(v2)). `v1` is the bound of the explored state (`self`) and `v2` is
2197                        // the bound of the current state (`other`). A smaller bound value
2198                        // represents a stronger safety guarantee (fewer assumed valid bytes,
2199                        // meaning a more restricted program). Therefore, a state with a smaller
2200                        // bound dominates (is "Greater" than) a state with a larger bound.
2201                        result = associate_orderings(result, v2.cmp(v1))?;
2202                    }
2203                    v @ Ordering::Less => {
2204                        array_bound_iter1.next();
2205                        result = associate_orderings(result, v)?;
2206                    }
2207                    v @ Ordering::Greater => {
2208                        array_bound_iter2.next();
2209                        result = associate_orderings(result, v)?;
2210                    }
2211                },
2212            }
2213        };
2214        Some(result)
2215    }
2216}
2217
2218/// Represents the read data dependencies of an eBPF program branch.
2219///
2220/// This struct tracks which registers and stack positions are *read* by the
2221/// instructions within a branch of the eBPF program.  This information is used
2222/// during branch cutting optimization to broaden terminating contexts.
2223///
2224/// The verifier assumes that data not read by a terminated branch is irrelevant
2225/// for future execution paths and can be safely cleared.
2226#[derive(Clone, Debug, Default)]
2227struct DataDependencies {
2228    /// The set of registers read by the children of a context.
2229    registers: HashSet<Register>,
2230    /// The stack positions read by the children of a context.
2231    stack: HashSet<usize>,
2232}
2233
2234impl DataDependencies {
2235    fn merge(&mut self, other: &DataDependencies) {
2236        self.registers.extend(other.registers.iter());
2237        self.stack.extend(other.stack.iter());
2238    }
2239
2240    fn alu(&mut self, dst: Register, src: Source) -> Result<(), String> {
2241        // Only do something if the dst is read, otherwise the computation doesn't matter.
2242        if self.registers.contains(&dst) {
2243            if let Source::Reg(src) = src {
2244                self.registers.insert(src);
2245            }
2246        }
2247        Ok(())
2248    }
2249
2250    fn jmp(&mut self, dst: Register, src: Source) -> Result<(), String> {
2251        self.registers.insert(dst);
2252        if let Source::Reg(src) = src {
2253            self.registers.insert(src);
2254        }
2255        Ok(())
2256    }
2257
2258    fn atomic(
2259        &mut self,
2260        context: &ComputationContext,
2261        fetch: bool,
2262        dst: Register,
2263        offset: i16,
2264        src: Register,
2265        width: DataWidth,
2266        is_cmpxchg: bool,
2267    ) -> Result<(), String> {
2268        let mut is_read = false;
2269        if is_cmpxchg && self.registers.contains(&0) {
2270            is_read = true;
2271        }
2272        if fetch && self.registers.contains(&src) {
2273            is_read = true;
2274        }
2275        let addr = context.reg(dst)?;
2276        if let Type::PtrToStack { offset: stack_offset } = addr {
2277            let stack_offset = stack_offset.add(offset);
2278            if !stack_offset.is_valid_offset() {
2279                return Err(format!("Invalid stack offset at {}", context.pc));
2280            }
2281            if is_read || self.stack.contains(&stack_offset.array_index()) {
2282                is_read = true;
2283                self.stack.insert(stack_offset.array_index());
2284            }
2285        }
2286        if is_read {
2287            self.registers.insert(0);
2288            self.registers.insert(src);
2289        }
2290        self.registers.insert(dst);
2291        Ok(())
2292    }
2293}
2294
2295struct DataDependenciesVisitorContext<'a> {
2296    calling_context: &'a CallingContext,
2297    computation_context: &'a ComputationContext,
2298}
2299
2300impl BpfVisitor for DataDependencies {
2301    type Context<'a> = DataDependenciesVisitorContext<'a>;
2302
2303    fn add<'a>(
2304        &mut self,
2305        _context: &mut Self::Context<'a>,
2306        dst: Register,
2307        src: Source,
2308    ) -> Result<(), String> {
2309        self.alu(dst, src)
2310    }
2311    fn add64<'a>(
2312        &mut self,
2313        _context: &mut Self::Context<'a>,
2314        dst: Register,
2315        src: Source,
2316    ) -> Result<(), String> {
2317        self.alu(dst, src)
2318    }
2319    fn and<'a>(
2320        &mut self,
2321        _context: &mut Self::Context<'a>,
2322        dst: Register,
2323        src: Source,
2324    ) -> Result<(), String> {
2325        self.alu(dst, src)
2326    }
2327    fn and64<'a>(
2328        &mut self,
2329        _context: &mut Self::Context<'a>,
2330        dst: Register,
2331        src: Source,
2332    ) -> Result<(), String> {
2333        self.alu(dst, src)
2334    }
2335    fn arsh<'a>(
2336        &mut self,
2337        _context: &mut Self::Context<'a>,
2338        dst: Register,
2339        src: Source,
2340    ) -> Result<(), String> {
2341        self.alu(dst, src)
2342    }
2343    fn arsh64<'a>(
2344        &mut self,
2345        _context: &mut Self::Context<'a>,
2346        dst: Register,
2347        src: Source,
2348    ) -> Result<(), String> {
2349        self.alu(dst, src)
2350    }
2351    fn div<'a>(
2352        &mut self,
2353        _context: &mut Self::Context<'a>,
2354        dst: Register,
2355        src: Source,
2356    ) -> Result<(), String> {
2357        self.alu(dst, src)
2358    }
2359    fn div64<'a>(
2360        &mut self,
2361        _context: &mut Self::Context<'a>,
2362        dst: Register,
2363        src: Source,
2364    ) -> Result<(), String> {
2365        self.alu(dst, src)
2366    }
2367    fn lsh<'a>(
2368        &mut self,
2369        _context: &mut Self::Context<'a>,
2370        dst: Register,
2371        src: Source,
2372    ) -> Result<(), String> {
2373        self.alu(dst, src)
2374    }
2375    fn lsh64<'a>(
2376        &mut self,
2377        _context: &mut Self::Context<'a>,
2378        dst: Register,
2379        src: Source,
2380    ) -> Result<(), String> {
2381        self.alu(dst, src)
2382    }
2383    fn r#mod<'a>(
2384        &mut self,
2385        _context: &mut Self::Context<'a>,
2386        dst: Register,
2387        src: Source,
2388    ) -> Result<(), String> {
2389        self.alu(dst, src)
2390    }
2391    fn mod64<'a>(
2392        &mut self,
2393        _context: &mut Self::Context<'a>,
2394        dst: Register,
2395        src: Source,
2396    ) -> Result<(), String> {
2397        self.alu(dst, src)
2398    }
2399    fn mul<'a>(
2400        &mut self,
2401        _context: &mut Self::Context<'a>,
2402        dst: Register,
2403        src: Source,
2404    ) -> Result<(), String> {
2405        self.alu(dst, src)
2406    }
2407    fn mul64<'a>(
2408        &mut self,
2409        _context: &mut Self::Context<'a>,
2410        dst: Register,
2411        src: Source,
2412    ) -> Result<(), String> {
2413        self.alu(dst, src)
2414    }
2415    fn or<'a>(
2416        &mut self,
2417        _context: &mut Self::Context<'a>,
2418        dst: Register,
2419        src: Source,
2420    ) -> Result<(), String> {
2421        self.alu(dst, src)
2422    }
2423    fn or64<'a>(
2424        &mut self,
2425        _context: &mut Self::Context<'a>,
2426        dst: Register,
2427        src: Source,
2428    ) -> Result<(), String> {
2429        self.alu(dst, src)
2430    }
2431    fn rsh<'a>(
2432        &mut self,
2433        _context: &mut Self::Context<'a>,
2434        dst: Register,
2435        src: Source,
2436    ) -> Result<(), String> {
2437        self.alu(dst, src)
2438    }
2439    fn rsh64<'a>(
2440        &mut self,
2441        _context: &mut Self::Context<'a>,
2442        dst: Register,
2443        src: Source,
2444    ) -> Result<(), String> {
2445        self.alu(dst, src)
2446    }
2447    fn sub<'a>(
2448        &mut self,
2449        _context: &mut Self::Context<'a>,
2450        dst: Register,
2451        src: Source,
2452    ) -> Result<(), String> {
2453        self.alu(dst, src)
2454    }
2455    fn sub64<'a>(
2456        &mut self,
2457        _context: &mut Self::Context<'a>,
2458        dst: Register,
2459        src: Source,
2460    ) -> Result<(), String> {
2461        self.alu(dst, src)
2462    }
2463    fn xor<'a>(
2464        &mut self,
2465        _context: &mut Self::Context<'a>,
2466        dst: Register,
2467        src: Source,
2468    ) -> Result<(), String> {
2469        self.alu(dst, src)
2470    }
2471    fn xor64<'a>(
2472        &mut self,
2473        _context: &mut Self::Context<'a>,
2474        dst: Register,
2475        src: Source,
2476    ) -> Result<(), String> {
2477        self.alu(dst, src)
2478    }
2479
2480    fn mov<'a>(
2481        &mut self,
2482        _context: &mut Self::Context<'a>,
2483        dst: Register,
2484        src: Source,
2485    ) -> Result<(), String> {
2486        if src == Source::Reg(dst) || !self.registers.contains(&dst) {
2487            return Ok(());
2488        }
2489        if let Source::Reg(src) = src {
2490            self.registers.insert(src);
2491        }
2492        self.registers.remove(&dst);
2493        Ok(())
2494    }
2495    fn mov64<'a>(
2496        &mut self,
2497        context: &mut Self::Context<'a>,
2498        dst: Register,
2499        src: Source,
2500    ) -> Result<(), String> {
2501        self.mov(context, dst, src)
2502    }
2503
2504    fn neg<'a>(&mut self, _context: &mut Self::Context<'a>, _dst: Register) -> Result<(), String> {
2505        // This is reading and writing the same value. This induces no change in dependencies.
2506        Ok(())
2507    }
2508    fn neg64<'a>(
2509        &mut self,
2510        _context: &mut Self::Context<'a>,
2511        _dst: Register,
2512    ) -> Result<(), String> {
2513        // This is reading and writing the same value. This induces no change in dependencies.
2514        Ok(())
2515    }
2516
2517    fn be<'a>(
2518        &mut self,
2519        _context: &mut Self::Context<'a>,
2520        _dst: Register,
2521        _width: DataWidth,
2522    ) -> Result<(), String> {
2523        // This is reading and writing the same value. This induces no change in dependencies.
2524        Ok(())
2525    }
2526    fn le<'a>(
2527        &mut self,
2528        _context: &mut Self::Context<'a>,
2529        _dst: Register,
2530        _width: DataWidth,
2531    ) -> Result<(), String> {
2532        // This is reading and writing the same value. This induces no change in dependencies.
2533        Ok(())
2534    }
2535
2536    fn call_external<'a>(
2537        &mut self,
2538        context: &mut Self::Context<'a>,
2539        index: u32,
2540    ) -> Result<(), String> {
2541        let Some(helper) = context.calling_context.helpers.get(&index).cloned() else {
2542            return Err(format!("unknown external function {}", index));
2543        };
2544        // A helper reads the stack region each `input` memory-parameter argument
2545        // points at, so those stack slots are data dependencies, like load/store.
2546        let comp = &context.computation_context;
2547        for (arg_index, arg) in helper.signature.args.iter().enumerate() {
2548            if let Type::MemoryParameter { size, input: true, .. } = arg {
2549                if let Type::PtrToStack { offset } = comp.reg((arg_index + 1) as Register)? {
2550                    let end = offset.add(size.size(comp)?);
2551                    if offset.is_valid_offset() && end.is_within_stack() {
2552                        for slot in offset.array_index()..end.array_index() {
2553                            self.stack.insert(slot);
2554                        }
2555                        if end.sub_index() != 0 {
2556                            self.stack.insert(end.array_index());
2557                        }
2558                    }
2559                }
2560            }
2561        }
2562        // 0 is overwritten and 1 to 5 are scratch registers
2563        for register in 0..helper.signature.args.len() + 1 {
2564            self.registers.remove(&(register as Register));
2565        }
2566        // 1 to k are parameters.
2567        for register in 0..helper.signature.args.len() {
2568            self.registers.insert((register + 1) as Register);
2569        }
2570        Ok(())
2571    }
2572
2573    fn exit<'a>(&mut self, _context: &mut Self::Context<'a>) -> Result<(), String> {
2574        // This read r0 unconditionally.
2575        self.registers.insert(0);
2576        Ok(())
2577    }
2578
2579    fn jump<'a>(&mut self, _context: &mut Self::Context<'a>, _offset: i16) -> Result<(), String> {
2580        // This doesn't do anything with values.
2581        Ok(())
2582    }
2583
2584    fn jeq<'a>(
2585        &mut self,
2586        _context: &mut Self::Context<'a>,
2587        dst: Register,
2588        src: Source,
2589        offset: i16,
2590    ) -> Result<(), String> {
2591        self.jmp(dst, src)
2592    }
2593    fn jeq64<'a>(
2594        &mut self,
2595        _context: &mut Self::Context<'a>,
2596        dst: Register,
2597        src: Source,
2598        offset: i16,
2599    ) -> Result<(), String> {
2600        self.jmp(dst, src)
2601    }
2602    fn jne<'a>(
2603        &mut self,
2604        _context: &mut Self::Context<'a>,
2605        dst: Register,
2606        src: Source,
2607        offset: i16,
2608    ) -> Result<(), String> {
2609        self.jmp(dst, src)
2610    }
2611    fn jne64<'a>(
2612        &mut self,
2613        _context: &mut Self::Context<'a>,
2614        dst: Register,
2615        src: Source,
2616        offset: i16,
2617    ) -> Result<(), String> {
2618        self.jmp(dst, src)
2619    }
2620    fn jge<'a>(
2621        &mut self,
2622        _context: &mut Self::Context<'a>,
2623        dst: Register,
2624        src: Source,
2625        offset: i16,
2626    ) -> Result<(), String> {
2627        self.jmp(dst, src)
2628    }
2629    fn jge64<'a>(
2630        &mut self,
2631        _context: &mut Self::Context<'a>,
2632        dst: Register,
2633        src: Source,
2634        offset: i16,
2635    ) -> Result<(), String> {
2636        self.jmp(dst, src)
2637    }
2638    fn jgt<'a>(
2639        &mut self,
2640        _context: &mut Self::Context<'a>,
2641        dst: Register,
2642        src: Source,
2643        offset: i16,
2644    ) -> Result<(), String> {
2645        self.jmp(dst, src)
2646    }
2647    fn jgt64<'a>(
2648        &mut self,
2649        _context: &mut Self::Context<'a>,
2650        dst: Register,
2651        src: Source,
2652        offset: i16,
2653    ) -> Result<(), String> {
2654        self.jmp(dst, src)
2655    }
2656    fn jle<'a>(
2657        &mut self,
2658        _context: &mut Self::Context<'a>,
2659        dst: Register,
2660        src: Source,
2661        offset: i16,
2662    ) -> Result<(), String> {
2663        self.jmp(dst, src)
2664    }
2665    fn jle64<'a>(
2666        &mut self,
2667        _context: &mut Self::Context<'a>,
2668        dst: Register,
2669        src: Source,
2670        offset: i16,
2671    ) -> Result<(), String> {
2672        self.jmp(dst, src)
2673    }
2674    fn jlt<'a>(
2675        &mut self,
2676        _context: &mut Self::Context<'a>,
2677        dst: Register,
2678        src: Source,
2679        offset: i16,
2680    ) -> Result<(), String> {
2681        self.jmp(dst, src)
2682    }
2683    fn jlt64<'a>(
2684        &mut self,
2685        _context: &mut Self::Context<'a>,
2686        dst: Register,
2687        src: Source,
2688        offset: i16,
2689    ) -> Result<(), String> {
2690        self.jmp(dst, src)
2691    }
2692    fn jsge<'a>(
2693        &mut self,
2694        _context: &mut Self::Context<'a>,
2695        dst: Register,
2696        src: Source,
2697        offset: i16,
2698    ) -> Result<(), String> {
2699        self.jmp(dst, src)
2700    }
2701    fn jsge64<'a>(
2702        &mut self,
2703        _context: &mut Self::Context<'a>,
2704        dst: Register,
2705        src: Source,
2706        offset: i16,
2707    ) -> Result<(), String> {
2708        self.jmp(dst, src)
2709    }
2710    fn jsgt<'a>(
2711        &mut self,
2712        _context: &mut Self::Context<'a>,
2713        dst: Register,
2714        src: Source,
2715        offset: i16,
2716    ) -> Result<(), String> {
2717        self.jmp(dst, src)
2718    }
2719    fn jsgt64<'a>(
2720        &mut self,
2721        _context: &mut Self::Context<'a>,
2722        dst: Register,
2723        src: Source,
2724        offset: i16,
2725    ) -> Result<(), String> {
2726        self.jmp(dst, src)
2727    }
2728    fn jsle<'a>(
2729        &mut self,
2730        _context: &mut Self::Context<'a>,
2731        dst: Register,
2732        src: Source,
2733        offset: i16,
2734    ) -> Result<(), String> {
2735        self.jmp(dst, src)
2736    }
2737    fn jsle64<'a>(
2738        &mut self,
2739        _context: &mut Self::Context<'a>,
2740        dst: Register,
2741        src: Source,
2742        offset: i16,
2743    ) -> Result<(), String> {
2744        self.jmp(dst, src)
2745    }
2746    fn jslt<'a>(
2747        &mut self,
2748        _context: &mut Self::Context<'a>,
2749        dst: Register,
2750        src: Source,
2751        offset: i16,
2752    ) -> Result<(), String> {
2753        self.jmp(dst, src)
2754    }
2755    fn jslt64<'a>(
2756        &mut self,
2757        _context: &mut Self::Context<'a>,
2758        dst: Register,
2759        src: Source,
2760        offset: i16,
2761    ) -> Result<(), String> {
2762        self.jmp(dst, src)
2763    }
2764    fn jset<'a>(
2765        &mut self,
2766        _context: &mut Self::Context<'a>,
2767        dst: Register,
2768        src: Source,
2769        offset: i16,
2770    ) -> Result<(), String> {
2771        self.jmp(dst, src)
2772    }
2773    fn jset64<'a>(
2774        &mut self,
2775        _context: &mut Self::Context<'a>,
2776        dst: Register,
2777        src: Source,
2778        offset: i16,
2779    ) -> Result<(), String> {
2780        self.jmp(dst, src)
2781    }
2782
2783    fn atomic_add<'a>(
2784        &mut self,
2785        context: &mut Self::Context<'a>,
2786        fetch: bool,
2787        dst: Register,
2788        offset: i16,
2789        src: Register,
2790    ) -> Result<(), String> {
2791        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2792    }
2793
2794    fn atomic_add64<'a>(
2795        &mut self,
2796        context: &mut Self::Context<'a>,
2797        fetch: bool,
2798        dst: Register,
2799        offset: i16,
2800        src: Register,
2801    ) -> Result<(), String> {
2802        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2803    }
2804
2805    fn atomic_and<'a>(
2806        &mut self,
2807        context: &mut Self::Context<'a>,
2808        fetch: bool,
2809        dst: Register,
2810        offset: i16,
2811        src: Register,
2812    ) -> Result<(), String> {
2813        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2814    }
2815
2816    fn atomic_and64<'a>(
2817        &mut self,
2818        context: &mut Self::Context<'a>,
2819        fetch: bool,
2820        dst: Register,
2821        offset: i16,
2822        src: Register,
2823    ) -> Result<(), String> {
2824        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2825    }
2826
2827    fn atomic_or<'a>(
2828        &mut self,
2829        context: &mut Self::Context<'a>,
2830        fetch: bool,
2831        dst: Register,
2832        offset: i16,
2833        src: Register,
2834    ) -> Result<(), String> {
2835        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2836    }
2837
2838    fn atomic_or64<'a>(
2839        &mut self,
2840        context: &mut Self::Context<'a>,
2841        fetch: bool,
2842        dst: Register,
2843        offset: i16,
2844        src: Register,
2845    ) -> Result<(), String> {
2846        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2847    }
2848
2849    fn atomic_xor<'a>(
2850        &mut self,
2851        context: &mut Self::Context<'a>,
2852        fetch: bool,
2853        dst: Register,
2854        offset: i16,
2855        src: Register,
2856    ) -> Result<(), String> {
2857        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2858    }
2859
2860    fn atomic_xor64<'a>(
2861        &mut self,
2862        context: &mut Self::Context<'a>,
2863        fetch: bool,
2864        dst: Register,
2865        offset: i16,
2866        src: Register,
2867    ) -> Result<(), String> {
2868        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2869    }
2870
2871    fn atomic_xchg<'a>(
2872        &mut self,
2873        context: &mut Self::Context<'a>,
2874        fetch: bool,
2875        dst: Register,
2876        offset: i16,
2877        src: Register,
2878    ) -> Result<(), String> {
2879        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2880    }
2881
2882    fn atomic_xchg64<'a>(
2883        &mut self,
2884        context: &mut Self::Context<'a>,
2885        fetch: bool,
2886        dst: Register,
2887        offset: i16,
2888        src: Register,
2889    ) -> Result<(), String> {
2890        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2891    }
2892
2893    fn atomic_cmpxchg<'a>(
2894        &mut self,
2895        context: &mut Self::Context<'a>,
2896        dst: Register,
2897        offset: i16,
2898        src: Register,
2899    ) -> Result<(), String> {
2900        self.atomic(&context.computation_context, true, dst, offset, src, DataWidth::U32, true)
2901    }
2902
2903    fn atomic_cmpxchg64<'a>(
2904        &mut self,
2905        context: &mut Self::Context<'a>,
2906        dst: Register,
2907        offset: i16,
2908        src: Register,
2909    ) -> Result<(), String> {
2910        self.atomic(&context.computation_context, true, dst, offset, src, DataWidth::U64, true)
2911    }
2912
2913    fn load<'a>(
2914        &mut self,
2915        context: &mut Self::Context<'a>,
2916        dst: Register,
2917        offset: i16,
2918        src: Register,
2919        width: DataWidth,
2920    ) -> Result<(), String> {
2921        let context = &context.computation_context;
2922        if self.registers.contains(&dst) {
2923            let addr = context.reg(src)?;
2924            if let Type::PtrToStack { offset: stack_offset } = addr {
2925                let stack_offset = stack_offset.add(offset);
2926                if !stack_offset.is_valid_offset() {
2927                    return Err(format!("Invalid stack offset at {}", context.pc));
2928                }
2929                self.stack.insert(stack_offset.array_index());
2930            }
2931        }
2932        self.registers.insert(src);
2933        Ok(())
2934    }
2935
2936    fn load64<'a>(
2937        &mut self,
2938        _context: &mut Self::Context<'a>,
2939        dst: Register,
2940        _src: u8,
2941        _lower: u32,
2942    ) -> Result<(), String> {
2943        self.registers.remove(&dst);
2944        Ok(())
2945    }
2946
2947    fn load_from_packet<'a>(
2948        &mut self,
2949        _context: &mut Self::Context<'a>,
2950        dst: Register,
2951        src: Register,
2952        _offset: i32,
2953        register_offset: Option<Register>,
2954        _width: DataWidth,
2955    ) -> Result<(), String> {
2956        // 1 to 5 are scratch registers
2957        for register in 1..6 {
2958            self.registers.remove(&(register as Register));
2959        }
2960        // Only do something if the dst is read, otherwise the computation doesn't matter.
2961        if self.registers.remove(&dst) {
2962            self.registers.insert(src);
2963            if let Some(reg) = register_offset {
2964                self.registers.insert(reg);
2965            }
2966        }
2967        Ok(())
2968    }
2969
2970    fn store<'a>(
2971        &mut self,
2972        context: &mut Self::Context<'a>,
2973        dst: Register,
2974        offset: i16,
2975        src: Source,
2976        width: DataWidth,
2977    ) -> Result<(), String> {
2978        let context = &context.computation_context;
2979        let addr = context.reg(dst)?;
2980        if let Type::PtrToStack { offset: stack_offset } = addr {
2981            let stack_offset = stack_offset.add(offset);
2982            if !stack_offset.is_valid_offset() {
2983                return Err(format!("Invalid stack offset at {}", context.pc));
2984            }
2985            if self.stack.remove(&stack_offset.array_index()) {
2986                if let Source::Reg(src) = src {
2987                    self.registers.insert(src);
2988                }
2989            }
2990        } else {
2991            if let Source::Reg(src) = src {
2992                self.registers.insert(src);
2993            }
2994            self.registers.insert(dst);
2995        }
2996
2997        Ok(())
2998    }
2999}
3000
3001#[derive(Debug)]
3002struct TerminatingContext {
3003    computation_context: ComputationContext,
3004    dependencies: DataDependencies,
3005}
3006
3007#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3008enum AluType {
3009    Plain,
3010    Sub,
3011    Add,
3012}
3013
3014impl AluType {
3015    /// Can this operation be done one a pointer and a scalar.
3016    fn is_ptr_compatible(&self) -> bool {
3017        match self {
3018            Self::Sub | Self::Add => true,
3019            _ => false,
3020        }
3021    }
3022}
3023
3024#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3025enum JumpWidth {
3026    W32,
3027    W64,
3028}
3029
3030#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3031enum JumpType {
3032    Eq,
3033    Ge,
3034    Gt,
3035    Le,
3036    LooseComparaison,
3037    Lt,
3038    Ne,
3039    StrictComparaison,
3040    Unknown,
3041}
3042
3043impl JumpType {
3044    fn invert(&self) -> Self {
3045        match self {
3046            Self::Eq => Self::Ne,
3047            Self::Ge => Self::Lt,
3048            Self::Gt => Self::Le,
3049            Self::Le => Self::Gt,
3050            Self::LooseComparaison => Self::StrictComparaison,
3051            Self::Lt => Self::Ge,
3052            Self::Ne => Self::Eq,
3053            Self::StrictComparaison => Self::LooseComparaison,
3054            Self::Unknown => Self::Unknown,
3055        }
3056    }
3057
3058    fn is_strict(&self) -> bool {
3059        match self {
3060            Self::Gt | Self::Lt | Self::Ne | Self::StrictComparaison => true,
3061            _ => false,
3062        }
3063    }
3064}
3065
3066fn display_register(register: Register) -> String {
3067    format!("%r{register}")
3068}
3069
3070fn display_source(src: Source) -> String {
3071    match src {
3072        Source::Reg(r) => display_register(r),
3073        Source::Value(v) => format!("0x{v:x}"),
3074    }
3075}
3076
3077impl BpfVisitor for ComputationContext {
3078    type Context<'a> = VerificationContext<'a>;
3079
3080    fn add<'a>(
3081        &mut self,
3082        context: &mut Self::Context<'a>,
3083        dst: Register,
3084        src: Source,
3085    ) -> Result<(), String> {
3086        self.alu(Some("add32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x + y))
3087    }
3088    fn add64<'a>(
3089        &mut self,
3090        context: &mut Self::Context<'a>,
3091        dst: Register,
3092        src: Source,
3093    ) -> Result<(), String> {
3094        self.alu(Some("add"), context, dst, src, AluType::Add, |x, y| x + y)
3095    }
3096    fn and<'a>(
3097        &mut self,
3098        context: &mut Self::Context<'a>,
3099        dst: Register,
3100        src: Source,
3101    ) -> Result<(), String> {
3102        self.alu(Some("and32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x & y))
3103    }
3104    fn and64<'a>(
3105        &mut self,
3106        context: &mut Self::Context<'a>,
3107        dst: Register,
3108        src: Source,
3109    ) -> Result<(), String> {
3110        self.alu(Some("and"), context, dst, src, AluType::Plain, |x, y| x & y)
3111    }
3112    fn arsh<'a>(
3113        &mut self,
3114        context: &mut Self::Context<'a>,
3115        dst: Register,
3116        src: Source,
3117    ) -> Result<(), String> {
3118        self.alu(Some("arsh32"), context, dst, src, AluType::Plain, |x, y| {
3119            alu32(x, y, |x, y| x.ashr(y))
3120        })
3121    }
3122    fn arsh64<'a>(
3123        &mut self,
3124        context: &mut Self::Context<'a>,
3125        dst: Register,
3126        src: Source,
3127    ) -> Result<(), String> {
3128        self.alu(Some("arsh"), context, dst, src, AluType::Plain, |x, y| x.ashr(y))
3129    }
3130    fn div<'a>(
3131        &mut self,
3132        context: &mut Self::Context<'a>,
3133        dst: Register,
3134        src: Source,
3135    ) -> Result<(), String> {
3136        self.alu(Some("div32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x / y))
3137    }
3138    fn div64<'a>(
3139        &mut self,
3140        context: &mut Self::Context<'a>,
3141        dst: Register,
3142        src: Source,
3143    ) -> Result<(), String> {
3144        self.alu(Some("div"), context, dst, src, AluType::Plain, |x, y| x / y)
3145    }
3146    fn lsh<'a>(
3147        &mut self,
3148        context: &mut Self::Context<'a>,
3149        dst: Register,
3150        src: Source,
3151    ) -> Result<(), String> {
3152        self.alu(Some("lsh32"), context, dst, src, AluType::Plain, |x, y| {
3153            alu32(x, y, |x, y| x << y)
3154        })
3155    }
3156    fn lsh64<'a>(
3157        &mut self,
3158        context: &mut Self::Context<'a>,
3159        dst: Register,
3160        src: Source,
3161    ) -> Result<(), String> {
3162        self.alu(Some("lsh"), context, dst, src, AluType::Plain, |x, y| x << y)
3163    }
3164    fn r#mod<'a>(
3165        &mut self,
3166        context: &mut Self::Context<'a>,
3167        dst: Register,
3168        src: Source,
3169    ) -> Result<(), String> {
3170        self.alu(Some("mod32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x % y))
3171    }
3172    fn mod64<'a>(
3173        &mut self,
3174        context: &mut Self::Context<'a>,
3175        dst: Register,
3176        src: Source,
3177    ) -> Result<(), String> {
3178        self.alu(Some("mod"), context, dst, src, AluType::Plain, |x, y| x % y)
3179    }
3180    fn mov<'a>(
3181        &mut self,
3182        context: &mut Self::Context<'a>,
3183        dst: Register,
3184        src: Source,
3185    ) -> Result<(), String> {
3186        bpf_log!(self, context, "mov32 {}, {}", display_register(dst), display_source(src));
3187        let src = self.compute_source(src)?;
3188        let value = match src {
3189            Type::ScalarValue(data) => {
3190                let value = (data.value as u32) as u64;
3191                let unknown_mask = (data.unknown_mask as u32) as u64;
3192                let unwritten_mask = (data.unwritten_mask as u32) as u64;
3193                let urange = U64Range::compute_range_for_bytes_swap(0.into(), data.urange, 0, 0, 4);
3194                Type::ScalarValue(ScalarValueData::new(value, unknown_mask, unwritten_mask, urange))
3195            }
3196            _ => Type::default(),
3197        };
3198        let mut next = self.next()?;
3199        next.set_reg(dst, value)?;
3200        context.states.push(next);
3201        Ok(())
3202    }
3203    fn mov64<'a>(
3204        &mut self,
3205        context: &mut Self::Context<'a>,
3206        dst: Register,
3207        src: Source,
3208    ) -> Result<(), String> {
3209        bpf_log!(self, context, "mov {}, {}", display_register(dst), display_source(src));
3210        let src = self.compute_source(src)?;
3211        let mut next = self.next()?;
3212        next.set_reg(dst, src)?;
3213        context.states.push(next);
3214        Ok(())
3215    }
3216    fn mul<'a>(
3217        &mut self,
3218        context: &mut Self::Context<'a>,
3219        dst: Register,
3220        src: Source,
3221    ) -> Result<(), String> {
3222        self.alu(Some("mul32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x * y))
3223    }
3224    fn mul64<'a>(
3225        &mut self,
3226        context: &mut Self::Context<'a>,
3227        dst: Register,
3228        src: Source,
3229    ) -> Result<(), String> {
3230        self.alu(Some("mul"), context, dst, src, AluType::Plain, |x, y| x * y)
3231    }
3232    fn or<'a>(
3233        &mut self,
3234        context: &mut Self::Context<'a>,
3235        dst: Register,
3236        src: Source,
3237    ) -> Result<(), String> {
3238        self.alu(Some("or32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x | y))
3239    }
3240    fn or64<'a>(
3241        &mut self,
3242        context: &mut Self::Context<'a>,
3243        dst: Register,
3244        src: Source,
3245    ) -> Result<(), String> {
3246        self.alu(Some("or"), context, dst, src, AluType::Plain, |x, y| x | y)
3247    }
3248    fn rsh<'a>(
3249        &mut self,
3250        context: &mut Self::Context<'a>,
3251        dst: Register,
3252        src: Source,
3253    ) -> Result<(), String> {
3254        self.alu(Some("rsh32"), context, dst, src, AluType::Plain, |x, y| {
3255            alu32(x, y, |x, y| x >> y)
3256        })
3257    }
3258    fn rsh64<'a>(
3259        &mut self,
3260        context: &mut Self::Context<'a>,
3261        dst: Register,
3262        src: Source,
3263    ) -> Result<(), String> {
3264        self.alu(Some("rsh"), context, dst, src, AluType::Plain, |x, y| x >> y)
3265    }
3266    fn sub<'a>(
3267        &mut self,
3268        context: &mut Self::Context<'a>,
3269        dst: Register,
3270        src: Source,
3271    ) -> Result<(), String> {
3272        self.alu(Some("sub32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x - y))
3273    }
3274    fn sub64<'a>(
3275        &mut self,
3276        context: &mut Self::Context<'a>,
3277        dst: Register,
3278        src: Source,
3279    ) -> Result<(), String> {
3280        self.alu(Some("sub"), context, dst, src, AluType::Sub, |x, y| x - y)
3281    }
3282    fn xor<'a>(
3283        &mut self,
3284        context: &mut Self::Context<'a>,
3285        dst: Register,
3286        src: Source,
3287    ) -> Result<(), String> {
3288        self.alu(Some("xor32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x ^ y))
3289    }
3290    fn xor64<'a>(
3291        &mut self,
3292        context: &mut Self::Context<'a>,
3293        dst: Register,
3294        src: Source,
3295    ) -> Result<(), String> {
3296        self.alu(Some("xor"), context, dst, src, AluType::Plain, |x, y| x ^ y)
3297    }
3298
3299    fn neg<'a>(&mut self, context: &mut Self::Context<'a>, dst: Register) -> Result<(), String> {
3300        bpf_log!(self, context, "neg32 {}", display_register(dst));
3301        self.alu(None, context, dst, Source::Value(0), AluType::Plain, |x, y| {
3302            alu32(x, y, |x, _y| -x)
3303        })
3304    }
3305    fn neg64<'a>(&mut self, context: &mut Self::Context<'a>, dst: Register) -> Result<(), String> {
3306        bpf_log!(self, context, "neg {}", display_register(dst));
3307        self.alu(None, context, dst, Source::Value(0), AluType::Plain, |x, _y| -x)
3308    }
3309
3310    fn be<'a>(
3311        &mut self,
3312        context: &mut Self::Context<'a>,
3313        dst: Register,
3314        width: DataWidth,
3315    ) -> Result<(), String> {
3316        self.endianness::<BigEndian>("be", context, dst, width)
3317    }
3318
3319    fn le<'a>(
3320        &mut self,
3321        context: &mut Self::Context<'a>,
3322        dst: Register,
3323        width: DataWidth,
3324    ) -> Result<(), String> {
3325        self.endianness::<LittleEndian>("le", context, dst, width)
3326    }
3327
3328    fn call_external<'a>(
3329        &mut self,
3330        context: &mut Self::Context<'a>,
3331        index: u32,
3332    ) -> Result<(), String> {
3333        bpf_log!(self, context, "call 0x{:x}", index);
3334        let Some(helper) = context.calling_context.helpers.get(&index).cloned() else {
3335            return Err(format!("unknown external function {}", index));
3336        };
3337        let HelperDefinition { signature, name, .. } = helper;
3338        debug_assert!(signature.args.len() <= 5);
3339        let mut next = self.next()?;
3340        for (arg_index, arg) in signature.args.iter().enumerate() {
3341            let reg = (arg_index + 1) as u8;
3342            self.reg(reg)?.match_parameter_type(context, self, name, arg, arg_index, &mut next)?
3343        }
3344        // Parameters have been validated, specify the return value on return.
3345        if signature.invalidate_array_bounds {
3346            next.array_bounds.clear();
3347        }
3348        let value =
3349            self.resolve_return_value(context, &signature.return_value, &mut next, false)?;
3350        next.set_reg(0, value)?;
3351        for i in 1..=5 {
3352            next.set_reg(i, Type::default())?;
3353        }
3354        context.states.push(next);
3355        Ok(())
3356    }
3357
3358    fn exit<'a>(&mut self, context: &mut Self::Context<'a>) -> Result<(), String> {
3359        bpf_log!(self, context, "exit");
3360        if !self.reg(0)?.is_written_scalar() {
3361            return Err("register 0 is incorrect at exit time".to_string());
3362        }
3363        if !self.resources.is_empty() {
3364            return Err("some resources have not been released at exit time".to_string());
3365        }
3366        self.terminated = true;
3367        Ok(())
3368    }
3369
3370    fn jump<'a>(&mut self, context: &mut Self::Context<'a>, offset: i16) -> Result<(), String> {
3371        bpf_log!(self, context, "ja {}", offset);
3372        let parent = Some(Arc::new(self.clone()));
3373        context.states.push(self.jump_with_offset(offset, parent)?);
3374        Ok(())
3375    }
3376
3377    fn jeq<'a>(
3378        &mut self,
3379        context: &mut Self::Context<'a>,
3380        dst: Register,
3381        src: Source,
3382        offset: i16,
3383    ) -> Result<(), String> {
3384        self.conditional_jump(
3385            "jeq32",
3386            context,
3387            dst,
3388            src,
3389            offset,
3390            JumpType::Eq,
3391            JumpWidth::W32,
3392            |x, y| {
3393                comp32(x, y, |x, y| {
3394                    // x == y
3395                    if x.min == x.max && x.min == y.min && x.min == y.max {
3396                        return Some(true);
3397                    }
3398                    if x.max < y.min || y.max < x.min {
3399                        return Some(false);
3400                    }
3401                    None
3402                })
3403            },
3404        )
3405    }
3406    fn jeq64<'a>(
3407        &mut self,
3408        context: &mut Self::Context<'a>,
3409        dst: Register,
3410        src: Source,
3411        offset: i16,
3412    ) -> Result<(), String> {
3413        self.conditional_jump(
3414            "jeq",
3415            context,
3416            dst,
3417            src,
3418            offset,
3419            JumpType::Eq,
3420            JumpWidth::W64,
3421            |x, y| {
3422                comp64(x, y, |x, y| {
3423                    // x == y
3424                    if x.min == x.max && x.min == y.min && x.min == y.max {
3425                        return Some(true);
3426                    }
3427                    if x.max < y.min || y.max < x.min {
3428                        return Some(false);
3429                    }
3430                    None
3431                })
3432            },
3433        )
3434    }
3435    fn jne<'a>(
3436        &mut self,
3437        context: &mut Self::Context<'a>,
3438        dst: Register,
3439        src: Source,
3440        offset: i16,
3441    ) -> Result<(), String> {
3442        self.conditional_jump(
3443            "jne32",
3444            context,
3445            dst,
3446            src,
3447            offset,
3448            JumpType::Ne,
3449            JumpWidth::W32,
3450            |x, y| {
3451                comp32(x, y, |x, y| {
3452                    // x != y
3453                    if x.min == x.max && x.min == y.min && x.min == y.max {
3454                        return Some(false);
3455                    }
3456                    if x.max < y.min || y.max < x.min {
3457                        return Some(true);
3458                    }
3459                    None
3460                })
3461            },
3462        )
3463    }
3464    fn jne64<'a>(
3465        &mut self,
3466        context: &mut Self::Context<'a>,
3467        dst: Register,
3468        src: Source,
3469        offset: i16,
3470    ) -> Result<(), String> {
3471        self.conditional_jump(
3472            "jne",
3473            context,
3474            dst,
3475            src,
3476            offset,
3477            JumpType::Ne,
3478            JumpWidth::W64,
3479            |x, y| {
3480                comp64(x, y, |x, y| {
3481                    // x != y
3482                    if x.min == x.max && x.min == y.min && x.min == y.max {
3483                        return Some(false);
3484                    }
3485                    if x.max < y.min || y.max < x.min {
3486                        return Some(true);
3487                    }
3488                    None
3489                })
3490            },
3491        )
3492    }
3493    fn jge<'a>(
3494        &mut self,
3495        context: &mut Self::Context<'a>,
3496        dst: Register,
3497        src: Source,
3498        offset: i16,
3499    ) -> Result<(), String> {
3500        self.conditional_jump(
3501            "jge32",
3502            context,
3503            dst,
3504            src,
3505            offset,
3506            JumpType::Ge,
3507            JumpWidth::W32,
3508            |x, y| {
3509                comp32(x, y, |x, y| {
3510                    // x >= y
3511                    if x.min >= y.max {
3512                        return Some(true);
3513                    }
3514                    if y.min > x.max {
3515                        return Some(false);
3516                    }
3517                    None
3518                })
3519            },
3520        )
3521    }
3522    fn jge64<'a>(
3523        &mut self,
3524        context: &mut Self::Context<'a>,
3525        dst: Register,
3526        src: Source,
3527        offset: i16,
3528    ) -> Result<(), String> {
3529        self.conditional_jump(
3530            "jge",
3531            context,
3532            dst,
3533            src,
3534            offset,
3535            JumpType::Ge,
3536            JumpWidth::W64,
3537            |x, y| {
3538                comp64(x, y, |x, y| {
3539                    // x >= y
3540                    if x.min >= y.max {
3541                        return Some(true);
3542                    }
3543                    if y.min > x.max {
3544                        return Some(false);
3545                    }
3546                    None
3547                })
3548            },
3549        )
3550    }
3551    fn jgt<'a>(
3552        &mut self,
3553        context: &mut Self::Context<'a>,
3554        dst: Register,
3555        src: Source,
3556        offset: i16,
3557    ) -> Result<(), String> {
3558        self.conditional_jump(
3559            "jgt32",
3560            context,
3561            dst,
3562            src,
3563            offset,
3564            JumpType::Gt,
3565            JumpWidth::W32,
3566            |x, y| {
3567                comp32(x, y, |x, y| {
3568                    // x > y
3569                    if x.min > y.max {
3570                        return Some(true);
3571                    }
3572                    if y.min >= x.max {
3573                        return Some(false);
3574                    }
3575                    None
3576                })
3577            },
3578        )
3579    }
3580    fn jgt64<'a>(
3581        &mut self,
3582        context: &mut Self::Context<'a>,
3583        dst: Register,
3584        src: Source,
3585        offset: i16,
3586    ) -> Result<(), String> {
3587        self.conditional_jump(
3588            "jgt",
3589            context,
3590            dst,
3591            src,
3592            offset,
3593            JumpType::Gt,
3594            JumpWidth::W64,
3595            |x, y| {
3596                comp64(x, y, |x, y| {
3597                    // x > y
3598                    if x.min > y.max {
3599                        return Some(true);
3600                    }
3601                    if y.min >= x.max {
3602                        return Some(false);
3603                    }
3604                    None
3605                })
3606            },
3607        )
3608    }
3609    fn jle<'a>(
3610        &mut self,
3611        context: &mut Self::Context<'a>,
3612        dst: Register,
3613        src: Source,
3614        offset: i16,
3615    ) -> Result<(), String> {
3616        self.conditional_jump(
3617            "jle32",
3618            context,
3619            dst,
3620            src,
3621            offset,
3622            JumpType::Le,
3623            JumpWidth::W32,
3624            |x, y| {
3625                comp32(x, y, |x, y| {
3626                    // x <= y
3627                    if x.max <= y.min {
3628                        return Some(true);
3629                    }
3630                    if y.max < x.min {
3631                        return Some(false);
3632                    }
3633                    None
3634                })
3635            },
3636        )
3637    }
3638    fn jle64<'a>(
3639        &mut self,
3640        context: &mut Self::Context<'a>,
3641        dst: Register,
3642        src: Source,
3643        offset: i16,
3644    ) -> Result<(), String> {
3645        self.conditional_jump(
3646            "jle",
3647            context,
3648            dst,
3649            src,
3650            offset,
3651            JumpType::Le,
3652            JumpWidth::W64,
3653            |x, y| {
3654                comp64(x, y, |x, y| {
3655                    // x <= y
3656                    if x.max <= y.min {
3657                        return Some(true);
3658                    }
3659                    if y.max < x.min {
3660                        return Some(false);
3661                    }
3662                    None
3663                })
3664            },
3665        )
3666    }
3667    fn jlt<'a>(
3668        &mut self,
3669        context: &mut Self::Context<'a>,
3670        dst: Register,
3671        src: Source,
3672        offset: i16,
3673    ) -> Result<(), String> {
3674        self.conditional_jump(
3675            "jlt32",
3676            context,
3677            dst,
3678            src,
3679            offset,
3680            JumpType::Lt,
3681            JumpWidth::W32,
3682            |x, y| {
3683                comp32(x, y, |x, y| {
3684                    // x < y
3685                    if x.max < y.min {
3686                        return Some(true);
3687                    }
3688                    if y.max <= x.min {
3689                        return Some(false);
3690                    }
3691                    None
3692                })
3693            },
3694        )
3695    }
3696    fn jlt64<'a>(
3697        &mut self,
3698        context: &mut Self::Context<'a>,
3699        dst: Register,
3700        src: Source,
3701        offset: i16,
3702    ) -> Result<(), String> {
3703        self.conditional_jump(
3704            "jlt",
3705            context,
3706            dst,
3707            src,
3708            offset,
3709            JumpType::Lt,
3710            JumpWidth::W64,
3711            |x, y| {
3712                comp64(x, y, |x, y| {
3713                    // x < y
3714                    if x.max < y.min {
3715                        return Some(true);
3716                    }
3717                    if y.max <= x.min {
3718                        return Some(false);
3719                    }
3720                    None
3721                })
3722            },
3723        )
3724    }
3725    fn jsge<'a>(
3726        &mut self,
3727        context: &mut Self::Context<'a>,
3728        dst: Register,
3729        src: Source,
3730        offset: i16,
3731    ) -> Result<(), String> {
3732        self.conditional_jump(
3733            "jsge32",
3734            context,
3735            dst,
3736            src,
3737            offset,
3738            JumpType::LooseComparaison,
3739            JumpWidth::W32,
3740            |x, y| scomp32(x, y, |x, y| x >= y),
3741        )
3742    }
3743    fn jsge64<'a>(
3744        &mut self,
3745        context: &mut Self::Context<'a>,
3746        dst: Register,
3747        src: Source,
3748        offset: i16,
3749    ) -> Result<(), String> {
3750        self.conditional_jump(
3751            "jsge",
3752            context,
3753            dst,
3754            src,
3755            offset,
3756            JumpType::LooseComparaison,
3757            JumpWidth::W64,
3758            |x, y| scomp64(x, y, |x, y| x >= y),
3759        )
3760    }
3761    fn jsgt<'a>(
3762        &mut self,
3763        context: &mut Self::Context<'a>,
3764        dst: Register,
3765        src: Source,
3766        offset: i16,
3767    ) -> Result<(), String> {
3768        self.conditional_jump(
3769            "jsgt32",
3770            context,
3771            dst,
3772            src,
3773            offset,
3774            JumpType::StrictComparaison,
3775            JumpWidth::W32,
3776            |x, y| scomp32(x, y, |x, y| x > y),
3777        )
3778    }
3779    fn jsgt64<'a>(
3780        &mut self,
3781        context: &mut Self::Context<'a>,
3782        dst: Register,
3783        src: Source,
3784        offset: i16,
3785    ) -> Result<(), String> {
3786        self.conditional_jump(
3787            "jsgt",
3788            context,
3789            dst,
3790            src,
3791            offset,
3792            JumpType::StrictComparaison,
3793            JumpWidth::W64,
3794            |x, y| scomp64(x, y, |x, y| x > y),
3795        )
3796    }
3797    fn jsle<'a>(
3798        &mut self,
3799        context: &mut Self::Context<'a>,
3800        dst: Register,
3801        src: Source,
3802        offset: i16,
3803    ) -> Result<(), String> {
3804        self.conditional_jump(
3805            "jsle32",
3806            context,
3807            dst,
3808            src,
3809            offset,
3810            JumpType::LooseComparaison,
3811            JumpWidth::W32,
3812            |x, y| scomp32(x, y, |x, y| x <= y),
3813        )
3814    }
3815    fn jsle64<'a>(
3816        &mut self,
3817        context: &mut Self::Context<'a>,
3818        dst: Register,
3819        src: Source,
3820        offset: i16,
3821    ) -> Result<(), String> {
3822        self.conditional_jump(
3823            "jsle",
3824            context,
3825            dst,
3826            src,
3827            offset,
3828            JumpType::LooseComparaison,
3829            JumpWidth::W64,
3830            |x, y| scomp64(x, y, |x, y| x <= y),
3831        )
3832    }
3833    fn jslt<'a>(
3834        &mut self,
3835        context: &mut Self::Context<'a>,
3836        dst: Register,
3837        src: Source,
3838        offset: i16,
3839    ) -> Result<(), String> {
3840        self.conditional_jump(
3841            "jslt32",
3842            context,
3843            dst,
3844            src,
3845            offset,
3846            JumpType::StrictComparaison,
3847            JumpWidth::W32,
3848            |x, y| scomp32(x, y, |x, y| x < y),
3849        )
3850    }
3851    fn jslt64<'a>(
3852        &mut self,
3853        context: &mut Self::Context<'a>,
3854        dst: Register,
3855        src: Source,
3856        offset: i16,
3857    ) -> Result<(), String> {
3858        self.conditional_jump(
3859            "jslt",
3860            context,
3861            dst,
3862            src,
3863            offset,
3864            JumpType::StrictComparaison,
3865            JumpWidth::W64,
3866            |x, y| scomp64(x, y, |x, y| x < y),
3867        )
3868    }
3869    fn jset<'a>(
3870        &mut self,
3871        context: &mut Self::Context<'a>,
3872        dst: Register,
3873        src: Source,
3874        offset: i16,
3875    ) -> Result<(), String> {
3876        self.conditional_jump(
3877            "jset32",
3878            context,
3879            dst,
3880            src,
3881            offset,
3882            JumpType::Unknown,
3883            JumpWidth::W32,
3884            |x, y| {
3885                comp32(x, y, |x, y| {
3886                    // x & y != 0
3887                    if x.min != x.max || y.min != y.max {
3888                        return None;
3889                    }
3890                    Some(x.min & y.min != 0)
3891                })
3892            },
3893        )
3894    }
3895    fn jset64<'a>(
3896        &mut self,
3897        context: &mut Self::Context<'a>,
3898        dst: Register,
3899        src: Source,
3900        offset: i16,
3901    ) -> Result<(), String> {
3902        self.conditional_jump(
3903            "jset",
3904            context,
3905            dst,
3906            src,
3907            offset,
3908            JumpType::Unknown,
3909            JumpWidth::W64,
3910            |x, y| {
3911                comp64(x, y, |x, y| {
3912                    // x & y != 0
3913                    if x.min != x.max || y.min != y.max {
3914                        return None;
3915                    }
3916                    Some(x.min & y.min != 0)
3917                })
3918            },
3919        )
3920    }
3921
3922    fn atomic_add<'a>(
3923        &mut self,
3924        context: &mut Self::Context<'a>,
3925        fetch: bool,
3926        dst: Register,
3927        offset: i16,
3928        src: Register,
3929    ) -> Result<(), String> {
3930        self.atomic_operation(
3931            "add32",
3932            context,
3933            DataWidth::U32,
3934            fetch,
3935            dst,
3936            offset,
3937            src,
3938            AluType::Add,
3939            |x, y| alu32(x, y, |x, y| x + y),
3940        )
3941    }
3942
3943    fn atomic_add64<'a>(
3944        &mut self,
3945        context: &mut Self::Context<'a>,
3946        fetch: bool,
3947        dst: Register,
3948        offset: i16,
3949        src: Register,
3950    ) -> Result<(), String> {
3951        self.atomic_operation(
3952            "add",
3953            context,
3954            DataWidth::U64,
3955            fetch,
3956            dst,
3957            offset,
3958            src,
3959            AluType::Add,
3960            |x, y| x + y,
3961        )
3962    }
3963
3964    fn atomic_and<'a>(
3965        &mut self,
3966        context: &mut Self::Context<'a>,
3967        fetch: bool,
3968        dst: Register,
3969        offset: i16,
3970        src: Register,
3971    ) -> Result<(), String> {
3972        self.atomic_operation(
3973            "and32",
3974            context,
3975            DataWidth::U32,
3976            fetch,
3977            dst,
3978            offset,
3979            src,
3980            AluType::Plain,
3981            |x, y| alu32(x, y, |x, y| x & y),
3982        )
3983    }
3984
3985    fn atomic_and64<'a>(
3986        &mut self,
3987        context: &mut Self::Context<'a>,
3988        fetch: bool,
3989        dst: Register,
3990        offset: i16,
3991        src: Register,
3992    ) -> Result<(), String> {
3993        self.atomic_operation(
3994            "and",
3995            context,
3996            DataWidth::U64,
3997            fetch,
3998            dst,
3999            offset,
4000            src,
4001            AluType::Plain,
4002            |x, y| x & y,
4003        )
4004    }
4005
4006    fn atomic_or<'a>(
4007        &mut self,
4008        context: &mut Self::Context<'a>,
4009        fetch: bool,
4010        dst: Register,
4011        offset: i16,
4012        src: Register,
4013    ) -> Result<(), String> {
4014        self.atomic_operation(
4015            "or32",
4016            context,
4017            DataWidth::U32,
4018            fetch,
4019            dst,
4020            offset,
4021            src,
4022            AluType::Plain,
4023            |x, y| alu32(x, y, |x, y| x | y),
4024        )
4025    }
4026
4027    fn atomic_or64<'a>(
4028        &mut self,
4029        context: &mut Self::Context<'a>,
4030        fetch: bool,
4031        dst: Register,
4032        offset: i16,
4033        src: Register,
4034    ) -> Result<(), String> {
4035        self.atomic_operation(
4036            "or",
4037            context,
4038            DataWidth::U64,
4039            fetch,
4040            dst,
4041            offset,
4042            src,
4043            AluType::Plain,
4044            |x, y| x | y,
4045        )
4046    }
4047
4048    fn atomic_xor<'a>(
4049        &mut self,
4050        context: &mut Self::Context<'a>,
4051        fetch: bool,
4052        dst: Register,
4053        offset: i16,
4054        src: Register,
4055    ) -> Result<(), String> {
4056        self.atomic_operation(
4057            "xor32",
4058            context,
4059            DataWidth::U32,
4060            fetch,
4061            dst,
4062            offset,
4063            src,
4064            AluType::Plain,
4065            |x, y| alu32(x, y, |x, y| x ^ y),
4066        )
4067    }
4068
4069    fn atomic_xor64<'a>(
4070        &mut self,
4071        context: &mut Self::Context<'a>,
4072        fetch: bool,
4073        dst: Register,
4074        offset: i16,
4075        src: Register,
4076    ) -> Result<(), String> {
4077        self.atomic_operation(
4078            "xor",
4079            context,
4080            DataWidth::U64,
4081            fetch,
4082            dst,
4083            offset,
4084            src,
4085            AluType::Plain,
4086            |x, y| x ^ y,
4087        )
4088    }
4089
4090    fn atomic_xchg<'a>(
4091        &mut self,
4092        context: &mut Self::Context<'a>,
4093        fetch: bool,
4094        dst: Register,
4095        offset: i16,
4096        src: Register,
4097    ) -> Result<(), String> {
4098        self.atomic_operation(
4099            "xchg32",
4100            context,
4101            DataWidth::U32,
4102            fetch,
4103            dst,
4104            offset,
4105            src,
4106            AluType::Plain,
4107            |_, x| x,
4108        )
4109    }
4110
4111    fn atomic_xchg64<'a>(
4112        &mut self,
4113        context: &mut Self::Context<'a>,
4114        fetch: bool,
4115        dst: Register,
4116        offset: i16,
4117        src: Register,
4118    ) -> Result<(), String> {
4119        self.raw_atomic_operation(
4120            "xchg",
4121            context,
4122            DataWidth::U64,
4123            fetch,
4124            dst,
4125            offset,
4126            src,
4127            |_, _, x| Ok(x),
4128        )
4129    }
4130
4131    fn atomic_cmpxchg<'a>(
4132        &mut self,
4133        context: &mut Self::Context<'a>,
4134        dst: Register,
4135        offset: i16,
4136        src: Register,
4137    ) -> Result<(), String> {
4138        self.raw_atomic_cmpxchg("cmpxchg32", context, dst, offset, src, JumpWidth::W32, |x, y| {
4139            comp32(x, y, |x, y| {
4140                // x == y
4141                if x.min == x.max && x.min == y.min && x.min == y.max {
4142                    return Some(true);
4143                }
4144                if x.max < y.min || y.max < x.min {
4145                    return Some(false);
4146                }
4147                None
4148            })
4149        })
4150    }
4151
4152    fn atomic_cmpxchg64<'a>(
4153        &mut self,
4154        context: &mut Self::Context<'a>,
4155        dst: Register,
4156        offset: i16,
4157        src: Register,
4158    ) -> Result<(), String> {
4159        self.raw_atomic_cmpxchg("cmpxchg", context, dst, offset, src, JumpWidth::W64, |x, y| {
4160            comp64(x, y, |x, y| {
4161                // x == y
4162                if x.min == x.max && x.min == y.min && x.min == y.max {
4163                    return Some(true);
4164                }
4165                if x.max < y.min || y.max < x.min {
4166                    return Some(false);
4167                }
4168                None
4169            })
4170        })
4171    }
4172
4173    fn load<'a>(
4174        &mut self,
4175        context: &mut Self::Context<'a>,
4176        dst: Register,
4177        offset: i16,
4178        src: Register,
4179        width: DataWidth,
4180    ) -> Result<(), String> {
4181        bpf_log!(
4182            self,
4183            context,
4184            "ldx{} {}, [{}{}]",
4185            width.str(),
4186            display_register(dst),
4187            display_register(src),
4188            print_offset(offset)
4189        );
4190        let addr = self.reg(src)?;
4191        let loaded_type = self.load_memory(context, &addr, Field::new(offset, width))?;
4192        let mut next = self.next()?;
4193        next.set_reg(dst, loaded_type)?;
4194        context.states.push(next);
4195        Ok(())
4196    }
4197
4198    fn load64<'a>(
4199        &mut self,
4200        context: &mut Self::Context<'a>,
4201        dst: Register,
4202        src: u8,
4203        lower: u32,
4204    ) -> Result<(), String> {
4205        // The pre-scan has already guaranteed that `pc + 1` is valid and structurally well-formed.
4206        let next_instruction = &context.code[self.pc + 1];
4207
4208        let value = match src {
4209            0 => {
4210                let value = (lower as u64) | (((next_instruction.imm() as u32) as u64) << 32);
4211                bpf_log!(self, context, "lddw {}, 0x{:x}", display_register(dst), value);
4212                Type::from(value)
4213            }
4214            BPF_PSEUDO_MAP_IDX => {
4215                let map_index = lower;
4216                bpf_log!(
4217                    self,
4218                    context,
4219                    "lddw {}, map_by_index({:x})",
4220                    display_register(dst),
4221                    map_index
4222                );
4223                context
4224                    .calling_context
4225                    .maps
4226                    .get(usize::try_from(map_index).unwrap())
4227                    .map(|schema| Type::ConstPtrToMap { id: map_index.into(), schema: *schema })
4228                    .ok_or_else(|| format!("lddw with invalid map index: {}", map_index))?
4229            }
4230            BPF_PSEUDO_MAP_IDX_VALUE => {
4231                let map_index = lower;
4232                let offset = next_instruction.imm();
4233                bpf_log!(
4234                    self,
4235                    context,
4236                    "lddw {}, map_value_by_index({:x})+{offset}",
4237                    display_register(dst),
4238                    map_index
4239                );
4240                let id = context.next_id();
4241                let map_schema = context
4242                    .calling_context
4243                    .maps
4244                    .get(usize::try_from(map_index).unwrap())
4245                    .ok_or_else(|| format!("lddw with invalid map index: {}", map_index))?;
4246
4247                if map_schema.map_type != bpf_map_type_BPF_MAP_TYPE_ARRAY {
4248                    return Err(format!(
4249                        "Invalid map type at index {map_index} for lddw. Expecting array."
4250                    ));
4251                }
4252                if map_schema.max_entries == 0 {
4253                    return Err(format!("Array has no entry."));
4254                }
4255
4256                Type::PtrToMemory {
4257                    id: MemoryId::from(id),
4258                    offset: offset.into(),
4259                    buffer_size: map_schema.value_size.into(),
4260                }
4261            }
4262            _ => {
4263                return Err(format!("invalid lddw"));
4264            }
4265        };
4266
4267        let parent = Some(Arc::new(self.clone()));
4268        let mut next = self.jump_with_offset(1, parent)?;
4269        next.set_reg(dst, value.into())?;
4270
4271        context.states.push(next);
4272        Ok(())
4273    }
4274
4275    fn load_from_packet<'a>(
4276        &mut self,
4277        context: &mut Self::Context<'a>,
4278        dst: Register,
4279        src: Register,
4280        offset: i32,
4281        register_offset: Option<Register>,
4282        width: DataWidth,
4283    ) -> Result<(), String> {
4284        bpf_log!(
4285            self,
4286            context,
4287            "ldp{} {}{}",
4288            width.str(),
4289            register_offset.map(display_register).unwrap_or_else(Default::default),
4290            print_offset(offset)
4291        );
4292
4293        // Verify that `src` refers to a packet.
4294        let src_type = self.reg(src)?;
4295        let src_is_packet = match &context.calling_context.packet_type {
4296            Some(packet_type) => src_type == *packet_type,
4297            None => false,
4298        };
4299        if !src_is_packet {
4300            return Err(format!("R{} is not a packet", src));
4301        }
4302
4303        if let Some(reg) = register_offset {
4304            let reg = self.reg(reg)?;
4305            if !reg.is_written_scalar() {
4306                return Err("access to unwritten offset".to_string());
4307            }
4308        }
4309        // Handle the case where the load succeed.
4310        let mut next = self.next()?;
4311        next.set_reg(dst, Type::UNKNOWN_SCALAR)?;
4312        for i in 1..=5 {
4313            next.set_reg(i, Type::default())?;
4314        }
4315        context.states.push(next);
4316        // Handle the case where the load fails.
4317        if !self.reg(0)?.is_written_scalar() {
4318            return Err("register 0 is incorrect at exit time".to_string());
4319        }
4320        if !self.resources.is_empty() {
4321            return Err("some resources have not been released at exit time".to_string());
4322        }
4323        self.terminated = true;
4324        Ok(())
4325    }
4326
4327    fn store<'a>(
4328        &mut self,
4329        context: &mut Self::Context<'a>,
4330        dst: Register,
4331        offset: i16,
4332        src: Source,
4333        width: DataWidth,
4334    ) -> Result<(), String> {
4335        let value = match src {
4336            Source::Reg(r) => {
4337                bpf_log!(
4338                    self,
4339                    context,
4340                    "stx{} [{}{}], {}",
4341                    width.str(),
4342                    display_register(dst),
4343                    print_offset(offset),
4344                    display_register(r),
4345                );
4346                self.reg(r)?
4347            }
4348            Source::Value(v) => {
4349                bpf_log!(
4350                    self,
4351                    context,
4352                    "st{} [{}{}], 0x{:x}",
4353                    width.str(),
4354                    display_register(dst),
4355                    print_offset(offset),
4356                    v,
4357                );
4358                Type::from(v & Type::mask(width))
4359            }
4360        };
4361        let mut next = self.next()?;
4362        let addr = self.reg(dst)?;
4363        next.store_memory(context, &addr, Field::new(offset, width), value)?;
4364        context.states.push(next);
4365        Ok(())
4366    }
4367}
4368
4369fn alu32(
4370    x: ScalarValueData,
4371    y: ScalarValueData,
4372    op: impl FnOnce(U32ScalarValueData, U32ScalarValueData) -> U32ScalarValueData,
4373) -> ScalarValueData {
4374    op(U32ScalarValueData::from(x), U32ScalarValueData::from(y)).into()
4375}
4376
4377fn comp64(
4378    x: ScalarValueData,
4379    y: ScalarValueData,
4380    op: impl FnOnce(U64Range, U64Range) -> Option<bool>,
4381) -> Result<Option<bool>, ()> {
4382    if !x.is_fully_initialized() || !y.is_fully_initialized() {
4383        return Err(());
4384    }
4385    Ok(op(x.urange, y.urange))
4386}
4387
4388fn comp32(
4389    x: ScalarValueData,
4390    y: ScalarValueData,
4391    op: impl FnOnce(U32Range, U32Range) -> Option<bool>,
4392) -> Result<Option<bool>, ()> {
4393    let x = U32ScalarValueData::from(x);
4394    let y = U32ScalarValueData::from(y);
4395    if !x.is_fully_initialized() || !y.is_fully_initialized() {
4396        return Err(());
4397    }
4398    Ok(op(x.urange, y.urange))
4399}
4400
4401fn scomp64(
4402    x: ScalarValueData,
4403    y: ScalarValueData,
4404    op: impl FnOnce(i64, i64) -> bool,
4405) -> Result<Option<bool>, ()> {
4406    if !x.is_fully_initialized() || !y.is_fully_initialized() {
4407        return Err(());
4408    }
4409    if !x.is_known() || !y.is_known() {
4410        return Ok(None);
4411    }
4412    Ok(Some(op(x.value as i64, y.value as i64)))
4413}
4414
4415fn scomp32(
4416    x: ScalarValueData,
4417    y: ScalarValueData,
4418    op: impl FnOnce(i32, i32) -> bool,
4419) -> Result<Option<bool>, ()> {
4420    let x = U32ScalarValueData::from(x);
4421    let y = U32ScalarValueData::from(y);
4422    if !x.is_fully_initialized() || !y.is_fully_initialized() {
4423        return Err(());
4424    }
4425    if !x.is_known() || !y.is_known() {
4426        return Ok(None);
4427    }
4428    Ok(Some(op(x.value as i32, y.value as i32)))
4429}
4430
4431fn print_offset<T: Into<i32>>(offset: T) -> String {
4432    let offset: i32 = offset.into();
4433    if offset == 0 {
4434        String::new()
4435    } else if offset > 0 {
4436        format!("+{offset}")
4437    } else {
4438        format!("{offset}")
4439    }
4440}
4441
4442fn run_on_stack_offset<F>(v: StackOffset, f: F) -> StackOffset
4443where
4444    F: FnOnce(ScalarValueData) -> ScalarValueData,
4445{
4446    StackOffset(f(v.reg()))
4447}
4448
4449fn error_and_log<T>(
4450    logger: &mut dyn VerifierLogger,
4451    msg: impl std::string::ToString,
4452) -> Result<T, EbpfError> {
4453    let msg = msg.to_string();
4454    logger.log(msg.as_bytes());
4455    return Err(EbpfError::ProgramVerifyError(msg));
4456}
4457
4458fn associate_orderings(o1: Ordering, o2: Ordering) -> Option<Ordering> {
4459    match (o1, o2) {
4460        (o1, o2) if o1 == o2 => Some(o1),
4461        (o, Ordering::Equal) | (Ordering::Equal, o) => Some(o),
4462        _ => None,
4463    }
4464}
4465
4466#[cfg(test)]
4467mod tests {
4468    use super::*;
4469    use std::collections::BTreeSet;
4470    use test_util::{assert_geq, assert_leq};
4471
4472    #[test]
4473    fn test_type_ordering() {
4474        let t0 = Type::from(0);
4475        let t1 = Type::from(1);
4476        let random = Type::AliasParameter { parameter_index: 8 };
4477        let unknown_written = Type::UNKNOWN_SCALAR;
4478        let unwritten = Type::default();
4479
4480        assert_eq!(t0.partial_cmp(&t0), Some(Ordering::Equal));
4481        assert_eq!(t0.partial_cmp(&t1), None);
4482        assert_eq!(t0.partial_cmp(&random), None);
4483        assert_eq!(t0.partial_cmp(&unknown_written), Some(Ordering::Less));
4484        assert_eq!(t0.partial_cmp(&unwritten), Some(Ordering::Less));
4485
4486        assert_eq!(t1.partial_cmp(&t0), None);
4487        assert_eq!(t1.partial_cmp(&t1), Some(Ordering::Equal));
4488        assert_eq!(t1.partial_cmp(&random), None);
4489        assert_eq!(t1.partial_cmp(&unknown_written), Some(Ordering::Less));
4490        assert_eq!(t1.partial_cmp(&unwritten), Some(Ordering::Less));
4491
4492        assert_eq!(random.partial_cmp(&t0), None);
4493        assert_eq!(random.partial_cmp(&t1), None);
4494        assert_eq!(random.partial_cmp(&random), Some(Ordering::Equal));
4495        assert_eq!(random.partial_cmp(&unknown_written), None);
4496        assert_eq!(random.partial_cmp(&unwritten), Some(Ordering::Less));
4497
4498        assert_eq!(unknown_written.partial_cmp(&t0), Some(Ordering::Greater));
4499        assert_eq!(unknown_written.partial_cmp(&t1), Some(Ordering::Greater));
4500        assert_eq!(unknown_written.partial_cmp(&random), None);
4501        assert_eq!(unknown_written.partial_cmp(&unknown_written), Some(Ordering::Equal));
4502        assert_eq!(unknown_written.partial_cmp(&unwritten), Some(Ordering::Less));
4503
4504        assert_eq!(unwritten.partial_cmp(&t0), Some(Ordering::Greater));
4505        assert_eq!(unwritten.partial_cmp(&t1), Some(Ordering::Greater));
4506        assert_eq!(unwritten.partial_cmp(&random), Some(Ordering::Greater));
4507        assert_eq!(unwritten.partial_cmp(&unknown_written), Some(Ordering::Greater));
4508        assert_eq!(unwritten.partial_cmp(&unwritten), Some(Ordering::Equal));
4509    }
4510
4511    #[test]
4512    fn test_stack_ordering() {
4513        let mut s1 = Stack::default();
4514        let mut s2 = Stack::default();
4515
4516        assert_eq!(s1.partial_cmp(&s2), Some(Ordering::Equal));
4517        s1.set(0, 0.into());
4518        assert_eq!(s1.partial_cmp(&s2), Some(Ordering::Less));
4519        assert_eq!(s2.partial_cmp(&s1), Some(Ordering::Greater));
4520        s2.set(1, 1.into());
4521        assert_eq!(s1.partial_cmp(&s2), None);
4522        assert_eq!(s2.partial_cmp(&s1), None);
4523    }
4524
4525    #[test]
4526    fn test_context_ordering() {
4527        let mut c1 = ComputationContext::default();
4528        let mut c2 = ComputationContext::default();
4529
4530        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Equal));
4531
4532        c1.array_bounds.insert(1.into(), 5);
4533        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Less));
4534        assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Greater));
4535
4536        c2.array_bounds.insert(1.into(), 7);
4537        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4538        assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4539
4540        c1.array_bounds.insert(2.into(), 9);
4541        assert_eq!(c1.partial_cmp(&c2), None);
4542        assert_eq!(c2.partial_cmp(&c1), None);
4543
4544        c2.array_bounds.insert(2.into(), 9);
4545        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4546        assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4547
4548        c2.array_bounds.insert(3.into(), 12);
4549        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4550        assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4551
4552        c1.pc = 8;
4553        assert_eq!(c1.partial_cmp(&c2), None);
4554        assert_eq!(c2.partial_cmp(&c1), None);
4555    }
4556
4557    #[test]
4558    fn test_stack_access() {
4559        let mut s = Stack::default();
4560
4561        // Store data in the range [8, 26) and verify that `read_data_ptr()` fails for any
4562        // reads outside of that range.
4563        assert!(s.store(StackOffset(8.into()), Type::UNKNOWN_SCALAR, DataWidth::U64).is_ok());
4564        assert!(s.store(StackOffset(16.into()), Type::UNKNOWN_SCALAR, DataWidth::U64).is_ok());
4565        assert!(s.store(StackOffset(24.into()), Type::UNKNOWN_SCALAR, DataWidth::U16).is_ok());
4566
4567        for offset in 0..32 {
4568            for end in (offset + 1)..32 {
4569                assert_eq!(
4570                    s.read_data_ptr(2, StackOffset(offset.into()), (end - offset) as u64).is_ok(),
4571                    offset >= 8 && end <= 26
4572                );
4573            }
4574        }
4575
4576        // Verify that overflows are handled properly.
4577        assert!(s.read_data_ptr(2, StackOffset(12.into()), u64::MAX - 2).is_err());
4578    }
4579
4580    #[test]
4581    fn test_compute_range_for_bytes_swap() {
4582        // Build a list of interesting values. Interesting values are all possible combination of
4583        // bytes being either 0, max value, or an intermediary value.
4584        let mut values = BTreeSet::<u64>::default();
4585        for v1 in &[0x00, 0x1, u64::MAX] {
4586            for v2 in &[0x00, 0x1, u64::MAX] {
4587                for v3 in &[0x00, 0x1, u64::MAX] {
4588                    values.insert(U64Range::assemble_slices((*v1, *v2, *v3), 1, 1));
4589                }
4590            }
4591        }
4592        // Replace the second byte of old by the first byte of new and return the result.
4593        let store = |old: u64, new: u64| (old & !0xff00) | ((new & 0xff) << 8);
4594
4595        for old in &values {
4596            for new in &values {
4597                let s = store(*old, *new);
4598                for min_old in values.iter().filter(|v| *v <= old) {
4599                    for min_new in values.iter().filter(|v| *v <= new) {
4600                        for max_old in values.iter().filter(|v| *v >= old) {
4601                            for max_new in values.iter().filter(|v| *v >= new) {
4602                                let range = U64Range::compute_range_for_bytes_swap(
4603                                    U64Range::new(*min_old, *max_old),
4604                                    U64Range::new(*min_new, *max_new),
4605                                    1,
4606                                    0,
4607                                    1,
4608                                );
4609                                assert_leq!(range.min, s);
4610                                assert_geq!(range.max, s);
4611                            }
4612                        }
4613                    }
4614                }
4615            }
4616        }
4617    }
4618
4619    #[test]
4620    fn test_type_constraint_ge_gt() {
4621        let mut context = ComputationContext::default();
4622        let (new_lhs, new_rhs) = Type::constraint(
4623            &mut context,
4624            JumpType::Ge,
4625            JumpWidth::W64,
4626            Type::UNKNOWN_SCALAR,
4627            Type::from(10),
4628        )
4629        .unwrap();
4630
4631        if let Type::ScalarValue(data1) = new_lhs {
4632            assert_eq!(data1.min(), 10);
4633            assert_eq!(data1.max(), u64::MAX);
4634        } else {
4635            panic!("Expected ScalarValue");
4636        }
4637
4638        if let Type::ScalarValue(data2) = new_rhs {
4639            assert_eq!(data2.min(), 10);
4640            assert_eq!(data2.max(), 10);
4641        } else {
4642            panic!("Expected ScalarValue");
4643        }
4644
4645        let (new_lhs, _) = Type::constraint(
4646            &mut context,
4647            JumpType::Gt,
4648            JumpWidth::W64,
4649            Type::UNKNOWN_SCALAR,
4650            Type::from(10),
4651        )
4652        .unwrap();
4653
4654        if let Type::ScalarValue(data1) = new_lhs {
4655            assert_eq!(data1.min(), 11);
4656            assert_eq!(data1.max(), u64::MAX);
4657        } else {
4658            panic!("Expected ScalarValue");
4659        }
4660    }
4661
4662    #[test]
4663    fn test_type_constraint_lt_le() {
4664        let mut context = ComputationContext::default();
4665        let (new_lhs, _) = Type::constraint(
4666            &mut context,
4667            JumpType::Lt,
4668            JumpWidth::W64,
4669            Type::UNKNOWN_SCALAR,
4670            Type::from(10),
4671        )
4672        .unwrap();
4673
4674        if let Type::ScalarValue(data1) = new_lhs {
4675            assert_eq!(data1.min(), 0);
4676            assert_eq!(data1.max(), 9);
4677        } else {
4678            panic!("Expected ScalarValue");
4679        }
4680
4681        let (new_lhs, _) = Type::constraint(
4682            &mut context,
4683            JumpType::Le,
4684            JumpWidth::W64,
4685            Type::UNKNOWN_SCALAR,
4686            Type::from(10),
4687        )
4688        .unwrap();
4689
4690        if let Type::ScalarValue(data1) = new_lhs {
4691            assert_eq!(data1.min(), 0);
4692            assert_eq!(data1.max(), 10);
4693        } else {
4694            panic!("Expected ScalarValue");
4695        }
4696    }
4697
4698    #[test]
4699    fn test_type_constraint_w32() {
4700        let mut context = ComputationContext::default();
4701        let (new_lhs, _) = Type::constraint(
4702            &mut context,
4703            JumpType::Eq,
4704            JumpWidth::W32,
4705            Type::ScalarValue(ScalarValueData::UNKNOWN_WRITTEN),
4706            Type::from(10),
4707        )
4708        .unwrap();
4709        if let Type::ScalarValue(data1) = new_lhs {
4710            assert_eq!(data1.min(), 10);
4711            assert_eq!(data1.max(), 0xffff_ffff_0000_000a);
4712        } else {
4713            panic!("Expected ScalarValue");
4714        }
4715    }
4716
4717    #[test]
4718    fn test_type_constraint_null_or() {
4719        let mut context = ComputationContext::default();
4720        let id = MemoryId::new();
4721        let null_or = Type::NullOr { id: id.clone(), inner: Box::new(Type::UNKNOWN_SCALAR) };
4722        let (new_lhs, _) =
4723            Type::constraint(&mut context, JumpType::Eq, JumpWidth::W64, null_or, Type::from(0))
4724                .unwrap();
4725        if let Type::ScalarValue(data) = new_lhs {
4726            assert_eq!(data.value, 0);
4727        } else {
4728            panic!("Expected zero ScalarValue");
4729        }
4730    }
4731
4732    #[test]
4733    fn test_type_constraint_array_bounds() {
4734        let mut context = ComputationContext::default();
4735        let id = MemoryId::new();
4736        let _ = Type::constraint(
4737            &mut context,
4738            JumpType::Le,
4739            JumpWidth::W64,
4740            Type::PtrToArray { id: id.clone(), offset: 22.into() },
4741            Type::PtrToEndArray { id: id.clone() },
4742        )
4743        .unwrap();
4744        assert_eq!(context.array_bounds.get(&id), Some(&22));
4745    }
4746}