Skip to main content

ebpf/
verifier.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#![allow(unused_variables)]
6
7use crate::scalar_value::{ScalarValueData, U32Range, U32ScalarValueData, U64Range};
8use crate::visitor::{BpfVisitor, ProgramCounter, Register, Source};
9use crate::{
10    BPF_LDDW, BPF_MAX_INSTS, BPF_PSEUDO_MAP_IDX, BPF_PSEUDO_MAP_IDX_VALUE, BPF_STACK_SIZE,
11    DataWidth, EbpfError, EbpfInstruction, GENERAL_REGISTER_COUNT, MapSchema, REGISTER_COUNT,
12};
13use byteorder::{BigEndian, ByteOrder, LittleEndian, NativeEndian};
14use fuchsia_sync::Mutex;
15use linux_uapi::{bpf_map_type, bpf_map_type_BPF_MAP_TYPE_ARRAY};
16use std::cmp::Ordering;
17use std::collections::{BTreeMap, HashMap, HashSet};
18use std::sync::Arc;
19use zerocopy::IntoBytes;
20
21const U32_MAX: u64 = u32::MAX as u64;
22
23/// A trait to receive the log from the verifier.
24pub trait VerifierLogger {
25    /// Log a line. The line is always a correct encoded ASCII string.
26    fn log(&mut self, line: &[u8]);
27}
28
29/// A `VerifierLogger` that drops all its content.
30pub struct NullVerifierLogger;
31
32impl VerifierLogger for NullVerifierLogger {
33    fn log(&mut self, line: &[u8]) {
34        debug_assert!(line.is_ascii());
35    }
36}
37
38/// An identifier for a memory buffer accessible by an ebpf program. The identifiers are built as a
39/// chain of unique identifier so that a buffer can contain multiple pointers to the same type and
40/// the verifier can distinguish between the different instances.
41/// The namespace of a `MemoryId`, used to prevent collisions between different
42/// sources of identifiers.
43#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Ord, PartialOrd)]
44pub enum Namespace {
45    /// For static structure identifiers (e.g., `SkBuff`, `BpfSock`) defined in `program_type.rs`.
46    Hardcoded,
47    /// For IDs generated by `MemoryId::new()` using the global counter.
48    Generated,
49    /// For IDs generated by `VerificationContext::next_id()` using the local counter.
50    Verification,
51}
52
53#[derive(Clone, Debug, PartialEq, Eq, Hash, Ord, PartialOrd)]
54pub struct MemoryId {
55    namespace: Namespace,
56    id: u64,
57    parent: Option<Box<MemoryId>>,
58}
59
60impl MemoryId {
61    pub fn id(&self) -> u64 {
62        self.id
63    }
64}
65
66impl From<u64> for MemoryId {
67    fn from(id: u64) -> Self {
68        Self { namespace: Namespace::Hardcoded, id, parent: None }
69    }
70}
71
72/// A counter that allows to generate new ids for parameters. The namespace is the same as for id
73/// generated for types while verifying an ebpf program. Because different sources of identifiers
74/// use different namespaces, this counter can start at 0.
75static BPF_TYPE_IDENTIFIER_COUNTER: std::sync::atomic::AtomicU64 =
76    std::sync::atomic::AtomicU64::new(0);
77
78impl MemoryId {
79    /// Build a `MemoryId` from a raw u64.
80    ///
81    /// This is supposed to be used by the `ebpf_api` crate to build static identifiers (e.g.
82    /// for hardcoded program contexts like `SkBuff`), but not elsewhere.
83    pub const fn from_raw(id: u64) -> MemoryId {
84        Self { namespace: Namespace::Hardcoded, id, parent: None }
85    }
86
87    pub fn new() -> MemoryId {
88        Self {
89            namespace: Namespace::Generated,
90            id: BPF_TYPE_IDENTIFIER_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
91            parent: None,
92        }
93    }
94
95    /// Build a new id such that `other` is prepended to the chain of parent of `self`.
96    fn prepended(&self, other: MemoryId) -> Self {
97        match &self.parent {
98            None => {
99                MemoryId { namespace: self.namespace, id: self.id, parent: Some(Box::new(other)) }
100            }
101            Some(parent) => MemoryId {
102                namespace: self.namespace,
103                id: self.id,
104                parent: Some(Box::new(parent.prepended(other))),
105            },
106        }
107    }
108
109    /// Returns whether the parent of `self` is `parent`
110    fn has_parent(&self, parent: &MemoryId) -> bool {
111        match &self.parent {
112            None => false,
113            Some(p) => p.as_ref() == parent,
114        }
115    }
116
117    /// Returns whether `self` and `other` represent the same type. This is true if both have the
118    /// same id and if they both have parent, parents must also match.
119    fn matches(&self, other: &MemoryId) -> bool {
120        if self.namespace != other.namespace || self.id != other.id {
121            return false;
122        };
123        match (&self.parent, &other.parent) {
124            (Some(p1), Some(p2)) => p1.matches(p2.as_ref()),
125            _ => true,
126        }
127    }
128}
129
130/// The type of a field in a struct pointed by `Type::PtrToStruct`.
131#[derive(Clone, Debug, PartialEq)]
132pub enum FieldType {
133    /// Read-only scalar value.
134    Scalar { size: usize },
135
136    /// Mutable scalar value.
137    MutableScalar { size: usize },
138
139    /// A pointer to the kernel memory. The full buffer is `buffer_size` bytes long.
140    PtrToMemory { is_32_bit: bool, id: MemoryId, buffer_size: usize },
141
142    /// A nullable pointer to the kernel memory. The full buffer is `buffer_size` bytes long.
143    NullablePtrToMemory { is_32_bit: bool, id: MemoryId, buffer_size: usize },
144
145    /// A pointer to the kernel memory. The full buffer size is determined by an instance of
146    /// `PtrToEndArray` with the same `id`.
147    PtrToArray { is_32_bit: bool, id: MemoryId },
148
149    /// A pointer to the kernel memory that represents the first non accessible byte of a
150    /// `PtrToArray` with the same `id`.
151    PtrToEndArray { is_32_bit: bool, id: MemoryId },
152}
153
154/// Definition of a field in a `Type::PtrToStruct`.
155#[derive(Clone, Debug, PartialEq)]
156pub struct FieldDescriptor {
157    /// The offset at which the field is located.
158    pub offset: usize,
159
160    /// The type of the pointed memory. Currently the verifier supports only `PtrToArray`,
161    /// `PtrToStruct`, `PtrToEndArray` and `ScalarValue`.
162    pub field_type: FieldType,
163}
164
165impl FieldDescriptor {
166    fn size(&self) -> usize {
167        match self.field_type {
168            FieldType::Scalar { size } | FieldType::MutableScalar { size } => size,
169            FieldType::PtrToMemory { is_32_bit, .. }
170            | FieldType::NullablePtrToMemory { is_32_bit, .. }
171            | FieldType::PtrToArray { is_32_bit, .. }
172            | FieldType::PtrToEndArray { is_32_bit, .. } => {
173                if is_32_bit {
174                    4
175                } else {
176                    8
177                }
178            }
179        }
180    }
181}
182
183/// The offset and width of a field in a struct.
184#[derive(Clone, Copy, Debug, Eq, PartialEq)]
185struct Field {
186    offset: i16,
187    width: DataWidth,
188}
189
190impl Field {
191    fn new(offset: i16, width: DataWidth) -> Self {
192        Self { offset, width }
193    }
194}
195
196/// Defines field layout in a struct pointed by `Type::PtrToStruct`.
197#[derive(Debug, PartialEq, Default)]
198pub struct StructDescriptor {
199    /// The list of fields.
200    pub fields: Vec<FieldDescriptor>,
201}
202
203impl StructDescriptor {
204    /// Return true if `self` is a subtype of `super_struct`.
205    fn is_subtype(&self, super_struct: &StructDescriptor) -> bool {
206        // Check that `super_struct.fields` is a subset of `fields`.
207        for super_field in super_struct.fields.iter() {
208            if self.fields.iter().find(|field| *field == super_field).is_none() {
209                return false;
210            }
211        }
212        true
213    }
214
215    /// Finds the field type for load/store at the specified location. None is returned if the
216    /// access is invalid and the program must be rejected.
217    fn find_field(&self, base_offset: ScalarValueData, field: Field) -> Option<&FieldDescriptor> {
218        let offset = base_offset + field.offset;
219        let field_desc = self.fields.iter().find(|f| {
220            f.offset <= offset.min() as usize && (offset.max() as usize) < f.offset + f.size()
221        })?;
222        let is_valid_load = match field_desc.field_type {
223            FieldType::Scalar { size } | FieldType::MutableScalar { size } => {
224                // For scalars check that the access is within the bounds of the field.
225                ((offset + field.width.bytes()).max() as usize) <= field_desc.offset + size
226            }
227            FieldType::PtrToMemory { is_32_bit, .. }
228            | FieldType::NullablePtrToMemory { is_32_bit, .. }
229            | FieldType::PtrToArray { is_32_bit, .. }
230            | FieldType::PtrToEndArray { is_32_bit, .. } => {
231                let expected_width = if is_32_bit { DataWidth::U32 } else { DataWidth::U64 };
232                // Pointer loads are expected to load the whole field.
233                offset.is_known()
234                    && offset.value as usize == field_desc.offset
235                    && field.width == expected_width
236            }
237        };
238
239        is_valid_load.then_some(field_desc)
240    }
241}
242
243#[derive(Clone, Debug, PartialEq)]
244pub enum MemoryParameterSize {
245    /// The memory buffer have the given size.
246    Value(u64),
247    /// The memory buffer size is given by the parameter in the given index.
248    Reference { index: u8 },
249}
250
251impl MemoryParameterSize {
252    fn size(&self, context: &ComputationContext) -> Result<u64, String> {
253        match self {
254            Self::Value(size) => Ok(*size),
255            Self::Reference { index } => {
256                let size_type = context.reg(index + 1)?;
257                match size_type {
258                    Type::ScalarValue(data) if data.is_known() => Ok(data.value),
259                    _ => Err("cannot know buffer size".to_string()),
260                }
261            }
262        }
263    }
264}
265
266#[derive(Clone, Debug, PartialEq)]
267pub enum MapTypeFilter {
268    AllowList(&'static [bpf_map_type]),
269    DenyList(&'static [bpf_map_type]),
270}
271
272impl MapTypeFilter {
273    pub fn is_allowed(&self, map_type: bpf_map_type) -> bool {
274        match self {
275            MapTypeFilter::AllowList(types) => types.contains(&map_type),
276            MapTypeFilter::DenyList(types) => !types.contains(&map_type),
277        }
278    }
279}
280
281#[derive(Clone, Debug, PartialEq)]
282pub enum Type {
283    /// A number.
284    ScalarValue(ScalarValueData),
285    /// A pointer to a map object.
286    ConstPtrToMap { id: u64, schema: MapSchema },
287    /// A pointer into the stack.
288    PtrToStack { offset: StackOffset },
289    /// A pointer to the kernel memory. The full buffer is `buffer_size` bytes long. The pointer is
290    /// situated at `offset` from the start of the buffer.
291    PtrToMemory { id: MemoryId, offset: ScalarValueData, buffer_size: u64 },
292    /// A pointer to a struct with the specified `StructDescriptor`.
293    PtrToStruct { id: MemoryId, offset: ScalarValueData, descriptor: Arc<StructDescriptor> },
294    /// A pointer to the kernel memory. The full buffer size is determined by an instance of
295    /// `PtrToEndArray` with the same `id`. The pointer is situadted at `offset` from the start of
296    /// the buffer.
297    PtrToArray { id: MemoryId, offset: ScalarValueData },
298    /// A pointer to the kernel memory that represents the first non accessible byte of a
299    /// `PtrToArray` with the same `id`.
300    PtrToEndArray { id: MemoryId },
301    /// A pointer that might be null and must be validated before being referenced.
302    NullOr { id: MemoryId, inner: Box<Type> },
303    /// An object that must be passed to a method with an associated `ReleaseParameter` before the
304    /// end of the program.
305    Releasable { id: MemoryId, inner: Box<Type> },
306    /// A function parameter that must be a `ScalarValue` when called.
307    ScalarValueParameter,
308    /// A function parameter that must be a `ConstPtrToMap` when called.
309    ConstPtrToMapParameter { filter: MapTypeFilter },
310    /// A function parameter that must be a key of a map.
311    MapKeyParameter {
312        /// The index in the arguments list that contains a `ConstPtrToMap` for the map this key is
313        /// associated with.
314        map_ptr_index: u8,
315    },
316    /// A function parameter that must be a value of a map.
317    MapValueParameter {
318        /// The index in the arguments list that contains a `ConstPtrToMap` for the map this key is
319        /// associated with.
320        map_ptr_index: u8,
321    },
322    /// A function parameter that must be a pointer to memory.
323    MemoryParameter {
324        /// The index in the arguments list that contains a scalar value containing the size of the
325        /// memory.
326        size: MemoryParameterSize,
327        /// Whether this memory is read by the function.
328        input: bool,
329        /// Whether this memory is written by the function.
330        output: bool,
331    },
332    /// A function return value that is the same type as a parameter.
333    AliasParameter {
334        /// The index in the argument list of the parameter that has the type of this return value.
335        parameter_index: u8,
336    },
337    /// A function return value that is either null, or the given type.
338    NullOrParameter(Box<Type>),
339    /// A function parameter that must be a pointer to memory with the given id.
340    StructParameter { id: MemoryId },
341    /// The parameter should be the same as the specified argument passed to the program.
342    ContextParameter { parameter_index: u8 },
343    /// A function return value that must be passed to a method with an associated
344    /// `ReleaseParameter` before the end of the program.
345    ReleasableParameter { id: MemoryId, inner: Box<Type> },
346    /// A function parameter that will release the value.
347    ReleaseParameter { id: MemoryId },
348    /// A function parameter that is not checked.
349    AnyParameter,
350}
351
352/// Defines a partial ordering on `Type` instances, capturing the notion of how "broad"
353/// a type is in terms of the set of potential values it represents.
354///
355/// The ordering is defined such that `t1 > t2` if a proof that an eBPF program terminates
356/// in a state where a register or memory location has type `t1` is also a proof that
357/// the program terminates in a state where that location has type `t2`.
358///
359/// In other words, a "broader" type represents a larger set of possible values, and
360/// proving termination with a broader type implies termination with any narrower type.
361///
362/// Examples:
363/// * `Type::ScalarValue(ScalarValueData({ unknown_mask: 0, .. }))` (a known scalar value) is less than
364///   `Type::ScalarValue(ScalarValueData({ unknown_mask: u64::MAX, .. }))` (an unknown scalar value).
365impl PartialOrd for Type {
366    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
367        // If the values are equals, return the known result.
368        if self == other {
369            return Some(Ordering::Equal);
370        }
371
372        // If one value is not initialized, the types are ordered.
373        // UNINITIALIZED is treated as the Top element (Greater than all other types).
374        // If a path is proven safe when a register is UNINITIALIZED, it means the path
375        // does not read that register. Consequently, it is safe to prune any state
376        // that has a more specific type (like a pointer) in that register, because
377        // the program will still not read it.
378        if self == &Type::UNINITIALIZED {
379            return Some(Ordering::Greater);
380        }
381        if other == &Type::UNINITIALIZED {
382            return Some(Ordering::Less);
383        }
384
385        // Otherwise, only scalars are comparables.
386        match (self, other) {
387            (Self::ScalarValue(data1), Self::ScalarValue(data2)) => data1.partial_cmp(data2),
388            _ => None,
389        }
390    }
391}
392
393impl From<ScalarValueData> for Type {
394    fn from(value: ScalarValueData) -> Self {
395        Self::ScalarValue(value)
396    }
397}
398
399impl From<u64> for Type {
400    fn from(value: u64) -> Self {
401        Self::ScalarValue(value.into())
402    }
403}
404
405impl Default for Type {
406    /// A new instance of `Type` where no bit has been written yet.
407    fn default() -> Self {
408        Self::UNINITIALIZED.clone()
409    }
410}
411
412impl Type {
413    /// An uninitialized value.
414    pub const UNINITIALIZED: Self = Self::ScalarValue(ScalarValueData::UNINITIALIZED);
415
416    /// A `Type` where the data is usable by userspace, but the value is not known by the verifier.
417    pub const UNKNOWN_SCALAR: Self = Self::ScalarValue(ScalarValueData::UNKNOWN_WRITTEN);
418
419    /// The mask associated with a data of size `width`.
420    fn mask(width: DataWidth) -> u64 {
421        if width == DataWidth::U64 { u64::MAX } else { (1 << width.bits()) - 1 }
422    }
423
424    /// Return true if `self` is a scalar with all bytes initialized.
425    fn is_written_scalar(&self) -> bool {
426        match self {
427            Self::ScalarValue(data) if data.is_fully_initialized() => true,
428            _ => false,
429        }
430    }
431
432    /// Return true if `self` has all bytes initialized.
433    fn is_initialized(&self) -> bool {
434        match self {
435            Self::ScalarValue(data) => data.is_fully_initialized(),
436            _ => true,
437        }
438    }
439
440    /// Return true if `self` is a subtype of `super_type`.
441    pub fn is_subtype(&self, super_type: &Type) -> bool {
442        match (self, super_type) {
443            // Anything can be used in place of an uninitialized value.
444            (_, Self::ScalarValue(data)) if data.is_uninitialized() => true,
445
446            (
447                Self::PtrToStruct { id: id1, offset: offset1, descriptor: descriptor1 },
448                Self::PtrToStruct { id: id2, offset: offset2, descriptor: descriptor2 },
449            ) => id1 == id2 && offset1 <= offset2 && descriptor1.is_subtype(descriptor2),
450
451            // Every type is a subtype of itself.
452            (self_type, super_type) if self_type == super_type => true,
453
454            _ => false,
455        }
456    }
457
458    /// Return true is `self` is guaranteed to be non-zero
459    pub fn is_non_zero(&self) -> bool {
460        match self {
461            Self::ScalarValue(d) => d.min() > 0,
462            Self::NullOr { .. } => false,
463            _ => true,
464        }
465    }
466
467    fn inner(&self, context: &ComputationContext) -> Result<&Type, String> {
468        match self {
469            Self::Releasable { id, inner } => {
470                if context.resources.contains(id) {
471                    Ok(&inner)
472                } else {
473                    Err("Access to released resource".to_string())
474                }
475            }
476            _ => Ok(self),
477        }
478    }
479
480    /// Constraints `type1` and `type2` for a conditional jump with the specified `jump_type` and
481    /// `jump_width`.
482    fn constraint(
483        context: &mut ComputationContext,
484        jump_type: JumpType,
485        jump_width: JumpWidth,
486        type1: Self,
487        type2: Self,
488    ) -> Result<(Self, Self), String> {
489        let result = match (jump_width, jump_type, type1.inner(context)?, type2.inner(context)?) {
490            (JumpWidth::W64, JumpType::Eq, Type::ScalarValue(data1), Type::ScalarValue(data2))
491                if data1.is_fully_initialized() && data2.is_fully_initialized() =>
492            {
493                let umin = std::cmp::max(data1.min(), data2.min());
494                let umax = std::cmp::min(data1.max(), data2.max());
495                let v = Type::ScalarValue(ScalarValueData::new(
496                    data1.value | data2.value,
497                    data1.unknown_mask & data2.unknown_mask,
498                    0,
499                    U64Range::new(umin, umax),
500                ));
501                (v.clone(), v)
502            }
503            (JumpWidth::W32, JumpType::Eq, Type::ScalarValue(data1), Type::ScalarValue(data2))
504                if data1.is_fully_initialized() && data2.is_fully_initialized() =>
505            {
506                // Refine ranges only when both operands fit in 32 bits; a range
507                // straddling 2^32 can have low bits below its min, so refining would
508                // drop reachable values.
509                let both_fit_u32 = data1.max() <= U32_MAX && data2.max() <= U32_MAX;
510                let (urange1, urange2) = if both_fit_u32 {
511                    let umin = std::cmp::max(data1.min(), data2.min());
512                    let umax = std::cmp::min(data1.max(), data2.max());
513                    let urange = U64Range::new(umin, umax);
514                    (urange, urange)
515                } else {
516                    (data1.urange, data2.urange)
517                };
518                let v1 = Type::ScalarValue(ScalarValueData::new(
519                    data1.value | (data2.value & U32_MAX),
520                    data1.unknown_mask & (data2.unknown_mask | (U32_MAX << 32)),
521                    0,
522                    urange1,
523                ));
524                let v2 = Type::ScalarValue(ScalarValueData::new(
525                    data2.value | (data1.value & U32_MAX),
526                    data2.unknown_mask & (data1.unknown_mask | (U32_MAX << 32)),
527                    0,
528                    urange2,
529                ));
530                (v1, v2)
531            }
532            (JumpWidth::W64, JumpType::Eq, Type::ScalarValue(data), Type::NullOr { id, .. })
533            | (JumpWidth::W64, JumpType::Eq, Type::NullOr { id, .. }, Type::ScalarValue(data))
534                if data.is_zero() =>
535            {
536                context.set_null(id, true);
537                let zero = Type::from(0);
538                (zero.clone(), zero)
539            }
540            (JumpWidth::W64, jump_type, Type::NullOr { id, inner }, Type::ScalarValue(data))
541                if jump_type.is_strict() && data.is_zero() =>
542            {
543                context.set_null(id, false);
544                let inner = *inner.clone();
545                inner.register_resource(context);
546                (inner, type2)
547            }
548            (JumpWidth::W64, jump_type, Type::ScalarValue(data), Type::NullOr { id, inner })
549                if jump_type.is_strict() && data.is_zero() =>
550            {
551                context.set_null(id, false);
552                let inner = *inner.clone();
553                inner.register_resource(context);
554                (type1, inner)
555            }
556
557            (JumpWidth::W64, JumpType::Lt, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
558                debug_assert!(lhs.min() < u64::MAX);
559                debug_assert!(rhs.max() > 0);
560                let new_max_lhs = std::cmp::min(lhs.max(), rhs.max() - 1);
561                debug_assert!(lhs.min() <= new_max_lhs);
562                let new_min_rhs = std::cmp::max(rhs.min(), lhs.min() + 1);
563                debug_assert!(new_min_rhs <= rhs.max());
564                let new_range_lhs = U64Range::new(lhs.min(), new_max_lhs);
565                let new_range_rhs = U64Range::new(new_min_rhs, rhs.max());
566                (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
567            }
568            (JumpWidth::W64, JumpType::Gt, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
569                debug_assert!(rhs.min() < u64::MAX);
570                debug_assert!(lhs.max() > 0);
571                let new_min_lhs = std::cmp::max(lhs.min(), rhs.min() + 1);
572                debug_assert!(new_min_lhs <= lhs.max());
573                let new_max_rhs = std::cmp::min(rhs.max(), lhs.max() - 1);
574                debug_assert!(rhs.min() <= new_max_rhs);
575                let new_range_lhs = U64Range::new(new_min_lhs, lhs.max());
576                let new_range_rhs = U64Range::new(rhs.min(), new_max_rhs);
577                (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
578            }
579
580            (JumpWidth::W64, JumpType::Le, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
581                let new_max_lhs = std::cmp::min(lhs.max(), rhs.max());
582                debug_assert!(lhs.min() <= new_max_lhs);
583                let new_min_rhs = std::cmp::max(rhs.min(), lhs.min());
584                debug_assert!(new_min_rhs <= rhs.max());
585                let new_range_lhs = U64Range::new(lhs.min(), new_max_lhs);
586                let new_range_rhs = U64Range::new(new_min_rhs, rhs.max());
587                (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
588            }
589            (JumpWidth::W64, JumpType::Ge, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
590                let new_min_lhs = std::cmp::max(lhs.min(), rhs.min());
591                debug_assert!(new_min_lhs <= lhs.max());
592                let new_max_rhs = std::cmp::min(rhs.max(), lhs.max());
593                debug_assert!(rhs.min() <= new_max_rhs);
594                let new_range_lhs = U64Range::new(new_min_lhs, lhs.max());
595                let new_range_rhs = U64Range::new(rhs.min(), new_max_rhs);
596                (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
597            }
598
599            (
600                JumpWidth::W64,
601                JumpType::Eq,
602                Type::PtrToArray { id: id1, offset },
603                Type::PtrToEndArray { id: id2 },
604            )
605            | (
606                JumpWidth::W64,
607                JumpType::Le,
608                Type::PtrToArray { id: id1, offset },
609                Type::PtrToEndArray { id: id2 },
610            )
611            | (
612                JumpWidth::W64,
613                JumpType::Ge,
614                Type::PtrToEndArray { id: id1 },
615                Type::PtrToArray { id: id2, offset },
616            ) if id1 == id2 => {
617                context.update_array_bounds(id1.clone(), *offset);
618                (type1, type2)
619            }
620            (
621                JumpWidth::W64,
622                JumpType::Lt,
623                Type::PtrToArray { id: id1, offset },
624                Type::PtrToEndArray { id: id2 },
625            )
626            | (
627                JumpWidth::W64,
628                JumpType::Gt,
629                Type::PtrToEndArray { id: id1 },
630                Type::PtrToArray { id: id2, offset },
631            ) if id1 == id2 => {
632                context.update_array_bounds(id1.clone(), *offset + 1);
633                (type1, type2)
634            }
635            (JumpWidth::W64, JumpType::Eq, _, _) => (type1.clone(), type1),
636            _ => (type1, type2),
637        };
638        Ok(result)
639    }
640
641    fn match_parameter_type(
642        &self,
643        verification_context: &VerificationContext<'_>,
644        context: &ComputationContext,
645        helper_name: &str,
646        parameter_type: &Type,
647        index: usize,
648        next: &mut ComputationContext,
649    ) -> Result<(), String> {
650        match (parameter_type, self) {
651            (Type::NullOrParameter(t), Type::ScalarValue(data))
652                if data.is_known() && data.value == 0 =>
653            {
654                Ok(())
655            }
656            (Type::NullOrParameter(t), _) => self.match_parameter_type(
657                verification_context,
658                context,
659                helper_name,
660                t,
661                index,
662                next,
663            ),
664            (Type::ScalarValueParameter, Type::ScalarValue(data))
665                if data.is_fully_initialized() =>
666            {
667                Ok(())
668            }
669            (Type::ConstPtrToMapParameter { filter }, Type::ConstPtrToMap { schema, .. }) => {
670                let map_type = schema.map_type;
671                if !filter.is_allowed(map_type) {
672                    return Err(format!("Map type {map_type} not allowed in {helper_name}"));
673                }
674                Ok(())
675            }
676            (
677                Type::MapKeyParameter { map_ptr_index },
678                Type::PtrToMemory { offset, buffer_size, .. },
679            ) => {
680                let schema = context.get_map_schema(*map_ptr_index)?;
681                context.check_memory_access(*offset, *buffer_size, 0, schema.key_size as usize)
682            }
683            (Type::MapKeyParameter { map_ptr_index }, Type::PtrToStack { offset }) => {
684                let schema = context.get_map_schema(*map_ptr_index)?;
685                context.stack.read_data_ptr(context.pc, *offset, schema.key_size as u64)
686            }
687            (
688                Type::MapValueParameter { map_ptr_index },
689                Type::PtrToMemory { offset, buffer_size, .. },
690            ) => {
691                let schema = context.get_map_schema(*map_ptr_index)?;
692                context.check_memory_access(*offset, *buffer_size, 0, schema.value_size as usize)
693            }
694            (Type::MapValueParameter { map_ptr_index }, Type::PtrToStack { offset }) => {
695                let schema = context.get_map_schema(*map_ptr_index)?;
696                context.stack.read_data_ptr(context.pc, *offset, schema.value_size as u64)
697            }
698            (Type::MemoryParameter { size, .. }, Type::PtrToMemory { offset, buffer_size, .. }) => {
699                let expected_size = size.size(context)?;
700                let offset_max = offset.max();
701                if offset_max > *buffer_size {
702                    return Err("out of bound read".to_string());
703                }
704                let size_left = *buffer_size - offset_max;
705                if expected_size > size_left {
706                    return Err("out of bound read".to_string());
707                }
708                Ok(())
709            }
710
711            (Type::MemoryParameter { size, input, output }, Type::PtrToStack { offset }) => {
712                let size = size.size(context)?;
713                let buffer_end = offset.add(size);
714                if !buffer_end.is_within_stack() {
715                    Err("out of bound access".to_string())
716                } else {
717                    if *output {
718                        next.stack.write_data_ptr(context.pc, *offset, size)?;
719                    }
720                    if *input {
721                        context.stack.read_data_ptr(context.pc, *offset, size)?;
722                    }
723                    Ok(())
724                }
725            }
726            (
727                Type::StructParameter { id: id1 },
728                Type::PtrToMemory { id: id2, offset, .. }
729                | Type::PtrToStruct { id: id2, offset, .. },
730            ) if offset.is_zero() && id1.matches(id2) => Ok(()),
731            (
732                Type::ReleasableParameter { id: id1, inner: inner1 },
733                Type::Releasable { id: id2, inner: inner2 },
734            ) if id2.has_parent(id1) => {
735                if next.resources.contains(id2) {
736                    inner2.match_parameter_type(
737                        verification_context,
738                        context,
739                        helper_name,
740                        inner1,
741                        index,
742                        next,
743                    )
744                } else {
745                    Err(format!("Resource already released for index {index}"))
746                }
747            }
748            (Type::ContextParameter { parameter_index }, arg_type) => {
749                if verification_context.calling_context.args.get(*parameter_index as usize)
750                    == Some(arg_type)
751                {
752                    Ok(())
753                } else {
754                    Err(format!("Helper expects program argument {parameter_index}"))
755                }
756            }
757            (Type::ReleaseParameter { id: id1 }, Type::Releasable { id: id2, .. })
758                if id2.has_parent(id1) =>
759            {
760                if next.resources.remove(id2) {
761                    Ok(())
762                } else {
763                    Err(format!("{id2:?} Resource already released for index {index}"))
764                }
765            }
766            (_, Type::Releasable { inner, .. }) => inner.match_parameter_type(
767                verification_context,
768                context,
769                helper_name,
770                parameter_type,
771                index,
772                next,
773            ),
774            (Type::AnyParameter, _) => Ok(()),
775
776            _ => Err(format!("incorrect parameter for index {index}")),
777        }
778    }
779
780    /// If this `Type` is an instance of NullOr with the given `null_id`, replace it wither either
781    /// 0 or the subtype depending on `is_null`
782    fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
783        match self {
784            Type::NullOr { id, inner } if id == null_id => {
785                if is_null {
786                    *self = Type::from(0);
787                } else {
788                    *self = *inner.clone();
789                }
790            }
791            _ => {}
792        }
793    }
794
795    fn register_resource(&self, context: &mut ComputationContext) {
796        match self {
797            Type::Releasable { id, .. } => {
798                context.resources.insert(id.clone());
799            }
800            _ => {}
801        }
802    }
803
804    /// Partially Compares two iterators of comparable items.
805    ///
806    /// This function iterates through both input iterators simultaneously and compares the corresponding elements.
807    /// The comparison continues until:
808    /// 1. Both iterators are exhausted and all elements were considered equal, in which case it returns `Some(Ordering::Equal)`.
809    /// 2. All pairs of corresponding elements that are not equal have the same ordering (`Ordering::Less` or `Ordering::Greater`), in which case it returns `Some(Ordering)` reflecting that consistent ordering.
810    /// 3. One iterator is exhausted before the other, or any comparison between elements yields `None`, or not all non-equal pairs have the same ordering, in which case it returns `None`.
811    fn compare_list<'a>(
812        mut l1: impl Iterator<Item = &'a Self>,
813        mut l2: impl Iterator<Item = &'a Self>,
814    ) -> Option<Ordering> {
815        let mut result = Ordering::Equal;
816        loop {
817            match (l1.next(), l2.next()) {
818                (None, None) => return Some(result),
819                (_, None) | (None, _) => return None,
820                (Some(v1), Some(v2)) => {
821                    result = associate_orderings(result, v1.partial_cmp(v2)?)?;
822                }
823            }
824        }
825    }
826}
827
828#[derive(Clone, Debug)]
829pub struct FunctionSignature {
830    pub args: Vec<Type>,
831    pub return_value: Type,
832    pub invalidate_array_bounds: bool,
833}
834
835#[derive(Clone, Debug)]
836pub struct HelperDefinition {
837    pub index: u32,
838    pub name: &'static str,
839    pub signature: FunctionSignature,
840}
841
842#[derive(Debug, Default)]
843pub struct CallingContext {
844    /// List of map schemas of the associated map. The maps can be accessed using LDDW instruction
845    /// with `src_reg=BPF_PSEUDO_MAP_IDX`.
846    pub maps: Vec<MapSchema>,
847    /// The registered external functions.
848    pub helpers: HashMap<u32, &'static HelperDefinition>,
849    /// The args of the program.
850    pub args: Vec<Type>,
851    /// Packet type. Normally it should be either `None` or `args[0]`.
852    pub packet_type: Option<Type>,
853}
854
855impl CallingContext {
856    pub fn register_map(&mut self, schema: MapSchema) -> usize {
857        let index = self.maps.len();
858        self.maps.push(schema);
859        index
860    }
861}
862
863#[derive(Debug, PartialEq, Clone)]
864pub struct StructAccess {
865    pub pc: ProgramCounter,
866
867    // Memory Id of the struct being accessed.
868    pub memory_id: MemoryId,
869
870    // Offset of the field being loaded.
871    pub field_offset: usize,
872
873    // Indicates that this is a 32-bit pointer load. These instructions must be remapped.
874    pub is_32_bit_ptr_load: bool,
875}
876
877#[derive(Debug, Clone)]
878pub struct VerifiedEbpfProgram {
879    pub(crate) code: Vec<EbpfInstruction>,
880    pub(crate) args: Vec<Type>,
881    pub(crate) struct_access_instructions: Vec<StructAccess>,
882    pub(crate) maps: Vec<MapSchema>,
883}
884
885impl VerifiedEbpfProgram {
886    // Convert the program to raw code. Can be used only when the program doesn't access any
887    // structs and maps.
888    pub fn to_code(self) -> Vec<EbpfInstruction> {
889        debug_assert!(self.struct_access_instructions.is_empty());
890        debug_assert!(self.maps.is_empty());
891        self.code
892    }
893
894    pub fn code(&self) -> &[EbpfInstruction] {
895        &self.code
896    }
897
898    pub fn struct_access_instructions(&self) -> &[StructAccess] {
899        &self.struct_access_instructions
900    }
901
902    pub fn from_verified_code(
903        code: Vec<EbpfInstruction>,
904        args: Vec<Type>,
905        struct_access_instructions: Vec<StructAccess>,
906        maps: Vec<MapSchema>,
907    ) -> Self {
908        Self { code, args, struct_access_instructions, maps }
909    }
910
911    pub fn maps(&self) -> &[MapSchema] {
912        &self.maps
913    }
914}
915
916/// Verify the given code depending on the type of the parameters and the registered external
917/// functions. Returned `VerifiedEbpfProgram` should be linked in order to execute it.
918pub fn verify_program(
919    code: Vec<EbpfInstruction>,
920    calling_context: CallingContext,
921    logger: &mut dyn VerifierLogger,
922) -> Result<VerifiedEbpfProgram, EbpfError> {
923    if code.len() > BPF_MAX_INSTS {
924        return error_and_log(logger, "ebpf program too long");
925    }
926    // Pre-scan the program to ensure all LDDW instructions are structurally well-formed,
927    // even if they are in unreachable code. This is necessary because:
928    // 1. The linker may rewrite LDDW instructions (e.g., to resolve map pointers) and
929    //    expects them to be well-formed.
930    // 2. It ensures that the second part of every LDDW has a `code` of 0 (invalid opcode).
931    //    This guarantees that any attempt to jump directly into the second part of an
932    //    LDDW will be rejected during verification as an invalid instruction.
933    let mut scan_pc = 0;
934    while scan_pc < code.len() {
935        let inst = &code[scan_pc];
936        if inst.code() == BPF_LDDW {
937            let Some(next_instruction) = code.get(scan_pc + 1) else {
938                return error_and_log(logger, "incomplete lddw");
939            };
940            if next_instruction.code() != 0
941                || next_instruction.offset() != 0
942                || next_instruction.src_reg() != 0
943                || next_instruction.dst_reg() != 0
944            {
945                return error_and_log(logger, "invalid lddw");
946            }
947            scan_pc += 2;
948        } else {
949            scan_pc += 1;
950        }
951    }
952
953    let mut context = ComputationContext::default();
954    for (i, t) in calling_context.args.iter().enumerate() {
955        // The parameter registers are r1 to r5.
956        context.set_reg((i + 1) as u8, t.clone()).map_err(EbpfError::ProgramVerifyError)?;
957    }
958    let states = vec![context];
959    let mut verification_context = VerificationContext {
960        calling_context,
961        logger,
962        states,
963        code: &code,
964        counter: 0,
965        iteration: 0,
966        terminating_contexts: Default::default(),
967        struct_access_instructions: Default::default(),
968    };
969    while let Some(mut context) = verification_context.states.pop() {
970        if let Some(terminating_contexts) =
971            verification_context.terminating_contexts.get(&context.pc)
972        {
973            // Check whether there exist a context that terminate and prove that this context does
974            // also terminate.
975            if let Some(ending_context) =
976                terminating_contexts.iter().find(|c| c.computation_context >= context)
977            {
978                // One such context has been found, this proves the current context terminates.
979                // If the context has a parent, register the data dependencies and try to terminate
980                // it.
981                if let Some(parent) = context.parent.take() {
982                    parent.dependencies.lock().push(ending_context.dependencies.clone());
983                    if let Some(parent) = Arc::into_inner(parent) {
984                        parent
985                            .terminate(&mut verification_context)
986                            .map_err(EbpfError::ProgramVerifyError)?;
987                    }
988                }
989                continue;
990            }
991        }
992        if verification_context.iteration > 10 * BPF_MAX_INSTS {
993            return error_and_log(verification_context.logger, "bpf byte code does not terminate");
994        }
995        if context.pc >= code.len() {
996            return error_and_log(verification_context.logger, "pc out of bounds");
997        }
998        if let Err(message) = context.visit(&mut verification_context, code[context.pc]) {
999            let message = format!("at PC {}: {}", context.pc, message);
1000            return error_and_log(verification_context.logger, message);
1001        }
1002        if context.terminated {
1003            context.terminate(&mut verification_context).map_err(EbpfError::ProgramVerifyError)?;
1004        }
1005        verification_context.iteration += 1;
1006    }
1007
1008    let struct_access_instructions =
1009        verification_context.struct_access_instructions.into_values().collect::<Vec<_>>();
1010    let CallingContext { maps, args, .. } = verification_context.calling_context;
1011    Ok(VerifiedEbpfProgram { code, struct_access_instructions, maps, args })
1012}
1013
1014struct VerificationContext<'a> {
1015    /// The type information for the program arguments and the registered functions.
1016    calling_context: CallingContext,
1017    /// The logger to use.
1018    logger: &'a mut dyn VerifierLogger,
1019    /// The `ComputationContext` yet to be validated.
1020    states: Vec<ComputationContext>,
1021    /// The program being analyzed.
1022    code: &'a [EbpfInstruction],
1023    /// A counter used to generated unique ids for memory buffers and maps.
1024    counter: u64,
1025    /// The current iteration of the verifier. Used to ensure termination by limiting the number of
1026    /// iteration before bailing out.
1027    iteration: usize,
1028    /// Keep track of the context that terminates at a given pc. The list of context will all be
1029    /// incomparables as each time a bigger context is computed, the smaller ones are removed from
1030    /// the list.
1031    terminating_contexts: BTreeMap<ProgramCounter, Vec<TerminatingContext>>,
1032    /// The current set of struct access instructions that will need to be updated when the
1033    /// program is linked. This is also used to ensure that a given instruction always loads the
1034    /// same field. If this is not the case, the verifier will reject the program.
1035    struct_access_instructions: HashMap<ProgramCounter, StructAccess>,
1036}
1037
1038impl<'a> VerificationContext<'a> {
1039    fn next_id(&mut self) -> MemoryId {
1040        let id = self.counter;
1041        self.counter += 1;
1042        MemoryId { namespace: Namespace::Verification, id, parent: None }
1043    }
1044
1045    /// Register an instruction that loads or stores a struct field. These instructions will need
1046    /// to updated later when the program is linked.
1047    fn register_struct_access(&mut self, struct_access: StructAccess) -> Result<(), String> {
1048        match self.struct_access_instructions.entry(struct_access.pc) {
1049            std::collections::hash_map::Entry::Vacant(entry) => {
1050                entry.insert(struct_access);
1051            }
1052            std::collections::hash_map::Entry::Occupied(entry) => {
1053                if *entry.get() != struct_access {
1054                    return Err("Inconsistent struct field access".to_string());
1055                }
1056            }
1057        }
1058        Ok(())
1059    }
1060}
1061
1062const STACK_ELEMENT_SIZE: usize = std::mem::size_of::<u64>();
1063
1064/// An offset inside the stack. The offset is from the end of the stack.
1065/// downward.
1066#[derive(Clone, Copy, Debug, PartialEq)]
1067pub struct StackOffset(ScalarValueData);
1068
1069impl Default for StackOffset {
1070    fn default() -> Self {
1071        Self(BPF_STACK_SIZE.into())
1072    }
1073}
1074
1075impl StackOffset {
1076    /// Whether the current offset is valid for element access.
1077    fn is_valid_offset(&self) -> bool {
1078        self.0.is_known() && self.0.value < (BPF_STACK_SIZE as u64)
1079    }
1080
1081    /// Whether the current offset is within the stack bounds, inclusive of `BPF_STACK_SIZE`.
1082    fn is_within_stack(&self) -> bool {
1083        self.0.is_known() && self.0.value <= (BPF_STACK_SIZE as u64)
1084    }
1085
1086    /// The value of the register.
1087    fn reg(&self) -> ScalarValueData {
1088        self.0
1089    }
1090
1091    /// The index into the stack array this offset points to. Can be called only if
1092    /// `is_within_stack()` is true.
1093    fn array_index(&self) -> usize {
1094        debug_assert!(self.is_within_stack());
1095        usize::try_from(self.0.value).unwrap() / STACK_ELEMENT_SIZE
1096    }
1097
1098    /// The offset inside the aligned u64 in the stack.
1099    fn sub_index(&self) -> usize {
1100        debug_assert!(self.is_within_stack());
1101        usize::try_from(self.0.value).unwrap() % STACK_ELEMENT_SIZE
1102    }
1103
1104    fn add<T: Into<ScalarValueData>>(self, rhs: T) -> Self {
1105        Self(self.0 + rhs)
1106    }
1107}
1108
1109/// The state of the stack
1110#[derive(Clone, Debug, Default, PartialEq)]
1111struct Stack {
1112    data: HashMap<usize, Type>,
1113}
1114
1115impl Stack {
1116    /// Replace all instances of the NullOr type with the given `null_id` to either 0 or the
1117    /// subtype depending on `is_null`
1118    fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
1119        for (_, t) in self.data.iter_mut() {
1120            t.set_null(null_id, is_null);
1121        }
1122    }
1123
1124    fn get(&self, index: usize) -> &Type {
1125        self.data.get(&index).unwrap_or(&Type::UNINITIALIZED)
1126    }
1127
1128    fn set(&mut self, index: usize, t: Type) {
1129        if t == Type::UNINITIALIZED {
1130            self.data.remove(&index);
1131        } else {
1132            self.data.insert(index, t);
1133        }
1134    }
1135
1136    fn extract_sub_value(value: u64, offset: usize, byte_count: usize) -> u64 {
1137        NativeEndian::read_uint(&value.as_bytes()[offset..], byte_count)
1138    }
1139
1140    fn insert_sub_value(mut original: u64, value: u64, width: DataWidth, offset: usize) -> u64 {
1141        let byte_count = width.bytes();
1142        let original_buf = original.as_mut_bytes();
1143        let value_buf = value.as_bytes();
1144        original_buf[offset..(byte_count + offset)].copy_from_slice(&value_buf[..byte_count]);
1145        original
1146    }
1147
1148    fn write_data_ptr(
1149        &mut self,
1150        pc: ProgramCounter,
1151        mut offset: StackOffset,
1152        bytes: u64,
1153    ) -> Result<(), String> {
1154        for i in 0..bytes {
1155            self.store(offset, Type::UNKNOWN_SCALAR, DataWidth::U8)?;
1156            offset = offset.add(1);
1157        }
1158        Ok(())
1159    }
1160
1161    fn read_data_ptr(
1162        &self,
1163        pc: ProgramCounter,
1164        offset: StackOffset,
1165        bytes: u64,
1166    ) -> Result<(), String> {
1167        let read_element =
1168            |index: usize, start_offset: usize, end_offset: usize| -> Result<(), String> {
1169                match self.get(index) {
1170                    Type::ScalarValue(data) => {
1171                        debug_assert!(end_offset > start_offset);
1172                        let unwritten_bits = Self::extract_sub_value(
1173                            data.unwritten_mask,
1174                            start_offset,
1175                            end_offset - start_offset,
1176                        );
1177                        if unwritten_bits == 0 {
1178                            Ok(())
1179                        } else {
1180                            Err("reading unwritten value from the stack".to_string())
1181                        }
1182                    }
1183                    _ => Err("invalid read from the stack".to_string()),
1184                }
1185            };
1186        if bytes == 0 {
1187            return Ok(());
1188        }
1189
1190        if bytes as usize > BPF_STACK_SIZE {
1191            return Err("stack overflow".to_string());
1192        }
1193
1194        if !offset.is_valid_offset() {
1195            return Err("invalid stack offset".to_string());
1196        }
1197
1198        let end_offset = offset.add(bytes);
1199        if !end_offset.is_within_stack() {
1200            return Err("stack overflow".to_string())?;
1201        }
1202
1203        // Handle the case where all the data is contained in a single element excluding the last
1204        // byte (the case when the read ends at an element edge, i.e. `end_offset.sub_index()==0`,
1205        // is covered by the default path below).
1206        if offset.array_index() == end_offset.array_index() {
1207            return read_element(offset.array_index(), offset.sub_index(), end_offset.sub_index());
1208        }
1209
1210        // Handle the first element, that might be partial.
1211        read_element(offset.array_index(), offset.sub_index(), STACK_ELEMENT_SIZE)?;
1212
1213        // Handle the last element, that might be partial.
1214        if end_offset.sub_index() != 0 {
1215            read_element(end_offset.array_index(), 0, end_offset.sub_index())?;
1216        }
1217
1218        // Handle the any full type between beginning and end.
1219        for i in (offset.array_index() + 1)..end_offset.array_index() {
1220            read_element(i, 0, STACK_ELEMENT_SIZE)?;
1221        }
1222
1223        Ok(())
1224    }
1225
1226    fn store(&mut self, offset: StackOffset, value: Type, width: DataWidth) -> Result<(), String> {
1227        if !offset.is_valid_offset() {
1228            return Err("out of bounds store".to_string());
1229        }
1230        if offset.sub_index() % width.bytes() != 0 {
1231            return Err("misaligned access".to_string());
1232        }
1233
1234        let index = offset.array_index();
1235        if width == DataWidth::U64 {
1236            self.set(index, value);
1237        } else {
1238            match value {
1239                Type::ScalarValue(data) => {
1240                    let old_data = match self.get(index) {
1241                        Type::ScalarValue(data) => *data,
1242                        _ => {
1243                            // The value in the stack is not a scalar. Let consider it an scalar
1244                            // value with no written bits.
1245                            ScalarValueData::UNINITIALIZED
1246                        }
1247                    };
1248                    let sub_index = offset.sub_index();
1249                    let value =
1250                        Self::insert_sub_value(old_data.value, data.value, width, sub_index);
1251                    let unknown_mask = Self::insert_sub_value(
1252                        old_data.unknown_mask,
1253                        data.unknown_mask,
1254                        width,
1255                        sub_index,
1256                    );
1257                    let unwritten_mask = Self::insert_sub_value(
1258                        old_data.unwritten_mask,
1259                        data.unwritten_mask,
1260                        width,
1261                        sub_index,
1262                    );
1263                    let urange = U64Range::compute_range_for_bytes_swap(
1264                        old_data.urange,
1265                        data.urange,
1266                        sub_index,
1267                        0,
1268                        width.bytes(),
1269                    );
1270                    self.set(
1271                        index,
1272                        Type::ScalarValue(ScalarValueData::new(
1273                            value,
1274                            unknown_mask,
1275                            unwritten_mask,
1276                            urange,
1277                        )),
1278                    );
1279                }
1280                _ => {
1281                    return Err("cannot store part of a non scalar value on the stack".to_string());
1282                }
1283            }
1284        }
1285        Ok(())
1286    }
1287
1288    fn load(&self, offset: StackOffset, width: DataWidth) -> Result<Type, String> {
1289        if !offset.is_valid_offset() {
1290            return Err("out of bounds load".to_string());
1291        }
1292        if offset.sub_index() % width.bytes() != 0 {
1293            return Err("misaligned access".to_string());
1294        }
1295
1296        let index = offset.array_index();
1297        let loaded_type = self.get(index).clone();
1298        let result = if width == DataWidth::U64 {
1299            loaded_type
1300        } else {
1301            match loaded_type {
1302                Type::ScalarValue(data) => {
1303                    let sub_index = offset.sub_index();
1304                    let value = Self::extract_sub_value(data.value, sub_index, width.bytes());
1305                    let unknown_mask =
1306                        Self::extract_sub_value(data.unknown_mask, sub_index, width.bytes());
1307                    let unwritten_mask =
1308                        Self::extract_sub_value(data.unwritten_mask, sub_index, width.bytes());
1309                    let urange = U64Range::compute_range_for_bytes_swap(
1310                        0.into(),
1311                        data.urange,
1312                        0,
1313                        sub_index,
1314                        width.bytes(),
1315                    );
1316                    Type::ScalarValue(ScalarValueData::new(
1317                        value,
1318                        unknown_mask,
1319                        unwritten_mask,
1320                        urange,
1321                    ))
1322                }
1323                _ => return Err(format!("incorrect load of {} bytes", width.bytes())),
1324            }
1325        };
1326        if !result.is_initialized() {
1327            return Err("reading unwritten value from the stack".to_string());
1328        }
1329        Ok(result)
1330    }
1331}
1332
1333/// Two types are ordered with `t1` > `t2` if a proof that a program in a state `t1` finish is also
1334/// a proof that a program in a state `t2` finish.
1335impl PartialOrd for Stack {
1336    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1337        let mut result = Ordering::Equal;
1338        let mut data_iter1 = self.data.iter().peekable();
1339        let mut data_iter2 = other.data.iter().peekable();
1340        loop {
1341            let k1 = data_iter1.peek().map(|(k, _)| *k);
1342            let k2 = data_iter2.peek().map(|(k, _)| *k);
1343            let k = match (k1, k2) {
1344                (None, None) => return Some(result),
1345                (Some(k), None) => {
1346                    data_iter1.next();
1347                    *k
1348                }
1349                (None, Some(k)) => {
1350                    data_iter2.next();
1351                    *k
1352                }
1353                (Some(k1), Some(k2)) => {
1354                    if k1 <= k2 {
1355                        data_iter1.next();
1356                    }
1357                    if k2 <= k1 {
1358                        data_iter2.next();
1359                    }
1360                    *std::cmp::min(k1, k2)
1361                }
1362            };
1363            result = associate_orderings(result, self.get(k).partial_cmp(other.get(k))?)?;
1364        }
1365    }
1366}
1367
1368macro_rules! bpf_log {
1369    ($context:ident, $verification_context:ident, $($msg:tt)*) => {
1370        let prefix = format!("{}: ({:02x})", $context.pc, $verification_context.code[$context.pc].code());
1371        let suffix = format!($($msg)*);
1372        $verification_context.logger.log(format!("{prefix} {suffix}").as_bytes());
1373    }
1374}
1375
1376/// The state of the computation as known by the verifier at a given point in time.
1377#[derive(Debug, Default)]
1378struct ComputationContext {
1379    /// The program counter.
1380    pc: ProgramCounter,
1381    /// Register 0 to 9.
1382    registers: [Type; GENERAL_REGISTER_COUNT as usize],
1383    /// The state of the stack.
1384    stack: Stack,
1385    /// The dynamically known bounds of buffers indexed by their ids.
1386    array_bounds: BTreeMap<MemoryId, u64>,
1387    /// The currently allocated resources.
1388    resources: HashSet<MemoryId>,
1389    /// The previous context in the computation.
1390    parent: Option<Arc<ComputationContext>>,
1391    /// The data dependencies of this context. This is used to broaden a known ending context to
1392    /// help cutting computation branches.
1393    dependencies: Mutex<Vec<DataDependencies>>,
1394    /// Whether this context has reached an exit instruction. The main loop uses this flag to
1395    /// call terminate() on the owned context, avoiding the extra Arc reference that would be
1396    /// created by cloning self in the exit handler.
1397    terminated: bool,
1398}
1399
1400impl Clone for ComputationContext {
1401    fn clone(&self) -> Self {
1402        Self {
1403            pc: self.pc,
1404            registers: self.registers.clone(),
1405            stack: self.stack.clone(),
1406            array_bounds: self.array_bounds.clone(),
1407            resources: self.resources.clone(),
1408            parent: self.parent.clone(),
1409            // dependencies are erased as they must always be used on the same instance of the
1410            // context.
1411            dependencies: Default::default(),
1412            terminated: false,
1413        }
1414    }
1415}
1416
1417/// parent and dependencies are ignored for the comparison, as they do not matter for termination.
1418impl PartialEq for ComputationContext {
1419    fn eq(&self, other: &Self) -> bool {
1420        self.pc == other.pc
1421            && self.registers == other.registers
1422            && self.stack == other.stack
1423            && self.array_bounds == other.array_bounds
1424    }
1425}
1426
1427impl ComputationContext {
1428    /// Replace all instances of the NullOr type with the given `null_id` to either 0 or the
1429    /// subtype depending on `is_null`
1430    fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
1431        for i in 0..self.registers.len() {
1432            self.registers[i].set_null(null_id, is_null);
1433        }
1434        self.stack.set_null(null_id, is_null);
1435    }
1436
1437    fn reg(&self, index: Register) -> Result<Type, String> {
1438        if index >= REGISTER_COUNT {
1439            return Err(format!("R{index} is invalid"));
1440        }
1441        if index < GENERAL_REGISTER_COUNT {
1442            Ok(self.registers[index as usize].clone())
1443        } else {
1444            Ok(Type::PtrToStack { offset: StackOffset::default() })
1445        }
1446    }
1447
1448    fn set_reg(&mut self, index: Register, reg_type: Type) -> Result<(), String> {
1449        if index >= GENERAL_REGISTER_COUNT {
1450            return Err(format!("R{index} is invalid"));
1451        }
1452        self.registers[index as usize] = reg_type;
1453        Ok(())
1454    }
1455
1456    fn update_array_bounds(&mut self, id: MemoryId, new_bound: ScalarValueData) {
1457        let new_bound_min = new_bound.min();
1458        self.array_bounds
1459            .entry(id)
1460            .and_modify(|v| *v = std::cmp::max(*v, new_bound_min))
1461            .or_insert(new_bound_min);
1462    }
1463
1464    fn get_map_schema(&self, argument: u8) -> Result<MapSchema, String> {
1465        match self.reg(argument + 1)? {
1466            Type::ConstPtrToMap { schema, .. } => Ok(schema),
1467            _ => Err(format!("No map found at argument {argument}")),
1468        }
1469    }
1470
1471    fn next(&self) -> Result<Self, String> {
1472        let parent = Some(Arc::new(self.clone()));
1473        self.jump_with_offset(0, parent)
1474    }
1475
1476    /// Returns a new `ComputationContext` where the pc has jump by `offset + 1`. In particular,
1477    /// the next instruction is reached with `jump_with_offset(0)`.
1478    fn jump_with_offset(&self, offset: i16, parent: Option<Arc<Self>>) -> Result<Self, String> {
1479        let pc = self
1480            .pc
1481            .checked_add_signed(offset.into())
1482            .and_then(|v| v.checked_add_signed(1))
1483            .ok_or_else(|| "jump outside of program".to_string())?;
1484        let result = Self {
1485            pc,
1486            registers: self.registers.clone(),
1487            stack: self.stack.clone(),
1488            array_bounds: self.array_bounds.clone(),
1489            resources: self.resources.clone(),
1490            parent,
1491            dependencies: Default::default(),
1492            terminated: false,
1493        };
1494        Ok(result)
1495    }
1496
1497    fn check_memory_access(
1498        &self,
1499        dst_offset: ScalarValueData,
1500        dst_buffer_size: u64,
1501        instruction_offset: i16,
1502        width: usize,
1503    ) -> Result<(), String> {
1504        let memory_range = dst_offset.urange + instruction_offset + U64Range::new(0, width as u64);
1505        if memory_range.max > dst_buffer_size {
1506            return Err("out of bound access".to_string());
1507        }
1508        Ok(())
1509    }
1510
1511    fn store_memory(
1512        &mut self,
1513        context: &mut VerificationContext<'_>,
1514        addr: &Type,
1515        field: Field,
1516        value: Type,
1517    ) -> Result<(), String> {
1518        let addr = addr.inner(self)?;
1519        match *addr {
1520            Type::PtrToStack { offset } => {
1521                let offset_sum = offset.add(field.offset);
1522                return self.stack.store(offset_sum, value, field.width);
1523            }
1524            Type::PtrToMemory { offset, buffer_size, .. } => {
1525                self.check_memory_access(offset, buffer_size, field.offset, field.width.bytes())?;
1526            }
1527            Type::PtrToStruct { ref id, offset, ref descriptor, .. } => {
1528                let field_desc = descriptor
1529                    .find_field(offset, field)
1530                    .ok_or_else(|| "incorrect store".to_string())?;
1531
1532                if !matches!(field_desc.field_type, FieldType::MutableScalar { .. }) {
1533                    return Err("store to a read-only field".to_string());
1534                }
1535
1536                context.register_struct_access(StructAccess {
1537                    pc: self.pc,
1538                    memory_id: id.clone(),
1539                    field_offset: field_desc.offset,
1540                    is_32_bit_ptr_load: false,
1541                })?;
1542            }
1543            Type::PtrToArray { ref id, offset } => {
1544                self.check_memory_access(
1545                    offset,
1546                    *self.array_bounds.get(&id).unwrap_or(&0),
1547                    field.offset,
1548                    field.width.bytes(),
1549                )?;
1550            }
1551            _ => return Err("incorrect store".to_string()),
1552        }
1553
1554        match value {
1555            Type::ScalarValue(data) if data.is_fully_initialized() => {}
1556            // Private data should not be leaked.
1557            _ => return Err("incorrect store".to_string()),
1558        }
1559        Ok(())
1560    }
1561
1562    fn load_memory(
1563        &self,
1564        context: &mut VerificationContext<'_>,
1565        addr: &Type,
1566        field: Field,
1567    ) -> Result<Type, String> {
1568        let addr = addr.inner(self)?;
1569        match *addr {
1570            Type::PtrToStack { offset } => {
1571                let offset_sum = offset.add(field.offset);
1572                self.stack.load(offset_sum, field.width)
1573            }
1574            Type::PtrToMemory { ref id, offset, buffer_size, .. } => {
1575                self.check_memory_access(offset, buffer_size, field.offset, field.width.bytes())?;
1576                Ok(Type::UNKNOWN_SCALAR)
1577            }
1578            Type::PtrToStruct { ref id, offset, ref descriptor, .. } => {
1579                let field_desc = descriptor
1580                    .find_field(offset, field)
1581                    .ok_or_else(|| "incorrect load".to_string())?;
1582
1583                let (return_type, is_32_bit_ptr_load) = match &field_desc.field_type {
1584                    FieldType::Scalar { .. } | FieldType::MutableScalar { .. } => {
1585                        (Type::UNKNOWN_SCALAR, false)
1586                    }
1587                    FieldType::PtrToArray { id: array_id, is_32_bit } => (
1588                        Type::PtrToArray { id: array_id.prepended(id.clone()), offset: 0.into() },
1589                        *is_32_bit,
1590                    ),
1591                    FieldType::PtrToEndArray { id: array_id, is_32_bit } => {
1592                        (Type::PtrToEndArray { id: array_id.prepended(id.clone()) }, *is_32_bit)
1593                    }
1594                    FieldType::PtrToMemory { id: memory_id, buffer_size, is_32_bit } => (
1595                        Type::PtrToMemory {
1596                            id: memory_id.prepended(id.clone()),
1597                            offset: 0.into(),
1598                            buffer_size: *buffer_size as u64,
1599                        },
1600                        *is_32_bit,
1601                    ),
1602                    FieldType::NullablePtrToMemory { id: memory_id, buffer_size, is_32_bit } => {
1603                        let id = memory_id.prepended(id.clone());
1604                        (
1605                            Type::NullOr {
1606                                id: id.clone(),
1607                                inner: Box::new(Type::PtrToMemory {
1608                                    id,
1609                                    offset: 0.into(),
1610                                    buffer_size: *buffer_size as u64,
1611                                }),
1612                            },
1613                            *is_32_bit,
1614                        )
1615                    }
1616                };
1617
1618                context.register_struct_access(StructAccess {
1619                    pc: self.pc,
1620                    memory_id: id.clone(),
1621                    field_offset: field_desc.offset,
1622                    is_32_bit_ptr_load,
1623                })?;
1624
1625                Ok(return_type)
1626            }
1627            Type::PtrToArray { ref id, offset } => {
1628                self.check_memory_access(
1629                    offset,
1630                    *self.array_bounds.get(&id).unwrap_or(&0),
1631                    field.offset,
1632                    field.width.bytes(),
1633                )?;
1634                Ok(Type::UNKNOWN_SCALAR)
1635            }
1636            _ => Err("incorrect load".to_string()),
1637        }
1638    }
1639
1640    /**
1641     * Given the given `return_value` in a method signature, return the concrete type to use,
1642     * updating the `next` context is needed.
1643     *
1644     * `maybe_null` is true is the computed value will be a descendant of a `NullOr` type.
1645     */
1646    fn resolve_return_value(
1647        &self,
1648        verification_context: &mut VerificationContext<'_>,
1649        return_value: &Type,
1650        next: &mut ComputationContext,
1651        maybe_null: bool,
1652    ) -> Result<Type, String> {
1653        match return_value {
1654            Type::AliasParameter { parameter_index } => self.reg(parameter_index + 1),
1655            Type::ReleasableParameter { id, inner } => {
1656                let id = verification_context.next_id().prepended(id.clone());
1657                if !maybe_null {
1658                    next.resources.insert(id.clone());
1659                }
1660                Ok(Type::Releasable {
1661                    id,
1662                    inner: Box::new(self.resolve_return_value(
1663                        verification_context,
1664                        inner,
1665                        next,
1666                        maybe_null,
1667                    )?),
1668                })
1669            }
1670            Type::NullOrParameter(t) => {
1671                let id = verification_context.next_id();
1672                Ok(Type::NullOr {
1673                    id,
1674                    inner: Box::new(self.resolve_return_value(
1675                        verification_context,
1676                        t,
1677                        next,
1678                        true,
1679                    )?),
1680                })
1681            }
1682            Type::MapValueParameter { map_ptr_index } => {
1683                let schema = self.get_map_schema(*map_ptr_index)?;
1684                let id = verification_context.next_id();
1685                Ok(Type::PtrToMemory {
1686                    id,
1687                    offset: 0.into(),
1688                    buffer_size: schema.value_size as u64,
1689                })
1690            }
1691            Type::MemoryParameter { size, .. } => {
1692                let buffer_size = size.size(self)?;
1693                let id = verification_context.next_id();
1694                Ok(Type::PtrToMemory { id, offset: 0.into(), buffer_size })
1695            }
1696            t => Ok(t.clone()),
1697        }
1698    }
1699
1700    fn compute_source(&self, src: Source) -> Result<Type, String> {
1701        match src {
1702            Source::Reg(reg) => self.reg(reg),
1703            Source::Value(v) => Ok(v.into()),
1704        }
1705    }
1706
1707    fn apply_computation(
1708        context: &ComputationContext,
1709        op1: Type,
1710        op2: Type,
1711        alu_type: AluType,
1712        op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1713    ) -> Result<Type, String> {
1714        let result: Type = match (alu_type, op1.inner(context)?, op2.inner(context)?) {
1715            (_, Type::ScalarValue(data1), Type::ScalarValue(data2)) => op(*data1, *data2).into(),
1716            (
1717                AluType::Add,
1718                Type::ScalarValue(_),
1719                Type::PtrToStack { .. } | Type::PtrToMemory { .. } | Type::PtrToStruct { .. },
1720            ) => {
1721                return Self::apply_computation(context, op2, op1, alu_type, op);
1722            }
1723            (alu_type, Type::PtrToStack { offset: x }, Type::ScalarValue(data))
1724                if alu_type.is_ptr_compatible() =>
1725            {
1726                Type::PtrToStack { offset: run_on_stack_offset(*x, |x| op(x, *data)) }
1727            }
1728            (
1729                alu_type,
1730                Type::PtrToMemory { id, offset: x, buffer_size },
1731                Type::ScalarValue(data),
1732            ) if alu_type.is_ptr_compatible() => {
1733                let offset = op(*x, *data);
1734                Type::PtrToMemory { id: id.clone(), offset, buffer_size: *buffer_size }
1735            }
1736            (
1737                alu_type,
1738                Type::PtrToStruct { id, offset: x, descriptor },
1739                Type::ScalarValue(data),
1740            ) if alu_type.is_ptr_compatible() => {
1741                let offset = op(*x, *data);
1742                Type::PtrToStruct { id: id.clone(), offset, descriptor: descriptor.clone() }
1743            }
1744            (AluType::Add, Type::PtrToArray { id, offset: x }, Type::ScalarValue(data)) => {
1745                let offset = x.checked_add(*data).ok_or_else(|| format!("XXX"))?;
1746                Type::PtrToArray { id: id.clone(), offset }
1747            }
1748            (AluType::Sub, Type::PtrToArray { id, offset: x }, Type::ScalarValue(data)) => {
1749                let offset = x.checked_sub(*data).ok_or_else(|| format!("XXX"))?;
1750                Type::PtrToArray { id: id.clone(), offset }
1751            }
1752            (
1753                AluType::Sub,
1754                Type::PtrToMemory { id: id1, offset: x1, .. },
1755                Type::PtrToMemory { id: id2, offset: x2, .. },
1756            )
1757            | (
1758                AluType::Sub,
1759                Type::PtrToStruct { id: id1, offset: x1, .. },
1760                Type::PtrToStruct { id: id2, offset: x2, .. },
1761            )
1762            | (
1763                AluType::Sub,
1764                Type::PtrToArray { id: id1, offset: x1 },
1765                Type::PtrToArray { id: id2, offset: x2 },
1766            ) if id1 == id2 => Type::from(op(*x1, *x2)),
1767            (AluType::Sub, Type::PtrToStack { offset: x1 }, Type::PtrToStack { offset: x2 }) => {
1768                Type::from(op(x1.reg(), x2.reg()))
1769            }
1770            (
1771                AluType::Sub,
1772                Type::PtrToArray { id: id1, .. },
1773                Type::PtrToEndArray { id: id2, .. },
1774            )
1775            | (
1776                AluType::Sub,
1777                Type::PtrToEndArray { id: id1, .. },
1778                Type::PtrToArray { id: id2, .. },
1779            ) if id1 == id2 => Type::UNKNOWN_SCALAR,
1780            _ => Type::default(),
1781        };
1782        Ok(result)
1783    }
1784
1785    fn alu(
1786        &mut self,
1787        op_name: Option<&str>,
1788        verification_context: &mut VerificationContext<'_>,
1789        dst: Register,
1790        src: Source,
1791        alu_type: AluType,
1792        op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1793    ) -> Result<(), String> {
1794        if let Some(op_name) = op_name {
1795            bpf_log!(
1796                self,
1797                verification_context,
1798                "{op_name} {}, {}",
1799                display_register(dst),
1800                display_source(src)
1801            );
1802        }
1803        let op1 = self.reg(dst)?;
1804        let op2 = self.compute_source(src)?;
1805        let result = Self::apply_computation(self, op1, op2, alu_type, op)?;
1806        let mut next = self.next()?;
1807        next.set_reg(dst, result)?;
1808        verification_context.states.push(next);
1809        Ok(())
1810    }
1811
1812    fn log_atomic_operation(
1813        &mut self,
1814        op_name: &str,
1815        verification_context: &mut VerificationContext<'_>,
1816        fetch: bool,
1817        dst: Register,
1818        offset: i16,
1819        src: Register,
1820    ) {
1821        bpf_log!(
1822            self,
1823            verification_context,
1824            "lock {}{} [{}{}], {}",
1825            if fetch { "fetch " } else { "" },
1826            op_name,
1827            display_register(dst),
1828            print_offset(offset),
1829            display_register(src),
1830        );
1831    }
1832
1833    fn raw_atomic_operation(
1834        &mut self,
1835        op_name: &str,
1836        verification_context: &mut VerificationContext<'_>,
1837        width: DataWidth,
1838        fetch: bool,
1839        dst: Register,
1840        offset: i16,
1841        src: Register,
1842        op: impl FnOnce(&ComputationContext, Type, Type) -> Result<Type, String>,
1843    ) -> Result<(), String> {
1844        self.log_atomic_operation(op_name, verification_context, fetch, dst, offset, src);
1845        let addr = self.reg(dst)?;
1846        let value = self.reg(src)?;
1847        let field = Field::new(offset, width);
1848        let loaded_type = self.load_memory(verification_context, &addr, field)?;
1849        let result = op(self, loaded_type.clone(), value)?;
1850        let mut next = self.next()?;
1851        next.store_memory(verification_context, &addr, field, result)?;
1852        if fetch {
1853            next.set_reg(src, loaded_type)?;
1854        }
1855        verification_context.states.push(next);
1856        Ok(())
1857    }
1858
1859    fn atomic_operation(
1860        &mut self,
1861        op_name: &str,
1862        verification_context: &mut VerificationContext<'_>,
1863        width: DataWidth,
1864        fetch: bool,
1865        dst: Register,
1866        offset: i16,
1867        src: Register,
1868        alu_type: AluType,
1869        op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1870    ) -> Result<(), String> {
1871        self.raw_atomic_operation(
1872            op_name,
1873            verification_context,
1874            width,
1875            fetch,
1876            dst,
1877            offset,
1878            src,
1879            |context: &ComputationContext, v1: Type, v2: Type| {
1880                Self::apply_computation(context, v1, v2, alu_type, op)
1881            },
1882        )
1883    }
1884
1885    fn raw_atomic_cmpxchg(
1886        &mut self,
1887        op_name: &str,
1888        verification_context: &mut VerificationContext<'_>,
1889        dst: Register,
1890        offset: i16,
1891        src: Register,
1892        jump_width: JumpWidth,
1893        op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
1894    ) -> Result<(), String> {
1895        self.log_atomic_operation(op_name, verification_context, true, dst, offset, src);
1896        let width = match jump_width {
1897            JumpWidth::W32 => DataWidth::U32,
1898            JumpWidth::W64 => DataWidth::U64,
1899        };
1900        let addr = self.reg(dst)?;
1901        let field = Field::new(offset, width);
1902        let dst = self.load_memory(verification_context, &addr, field)?;
1903        let value = self.reg(src)?;
1904        let r0 = self.reg(0)?;
1905        let branch = self.compute_branch(jump_width, &dst, &r0, op)?;
1906        // r0 = dst
1907        if branch.unwrap_or(true) {
1908            let mut next = self.next()?;
1909            let (dst, r0) =
1910                Type::constraint(&mut next, JumpType::Eq, jump_width, dst.clone(), r0.clone())?;
1911            next.set_reg(0, dst)?;
1912            next.store_memory(verification_context, &addr, field, value)?;
1913            verification_context.states.push(next);
1914        }
1915        // r0 != dst
1916        if !branch.unwrap_or(false) {
1917            let mut next = self.next()?;
1918            let (dst, r0) = Type::constraint(&mut next, JumpType::Ne, jump_width, dst, r0)?;
1919            next.set_reg(0, dst.clone())?;
1920            next.store_memory(verification_context, &addr, field, dst)?;
1921            verification_context.states.push(next);
1922        }
1923
1924        Ok(())
1925    }
1926    fn endianness<BO: ByteOrder>(
1927        &mut self,
1928        op_name: &str,
1929        verification_context: &mut VerificationContext<'_>,
1930        dst: Register,
1931        width: DataWidth,
1932    ) -> Result<(), String> {
1933        bpf_log!(self, verification_context, "{op_name}{} {}", width.bits(), display_register(dst),);
1934        let bit_op = |value: u64| match width {
1935            DataWidth::U16 => BO::read_u16((value as u16).as_bytes()) as u64,
1936            DataWidth::U32 => BO::read_u32((value as u32).as_bytes()) as u64,
1937            DataWidth::U64 => BO::read_u64(value.as_bytes()),
1938            _ => {
1939                panic!("Unexpected bit width for endianness operation");
1940            }
1941        };
1942        let value = self.reg(dst)?;
1943        let new_value = match value {
1944            Type::ScalarValue(data) => Type::ScalarValue(ScalarValueData::new(
1945                bit_op(data.value),
1946                bit_op(data.unknown_mask),
1947                bit_op(data.unwritten_mask),
1948                U64Range::max(),
1949            )),
1950            _ => Type::default(),
1951        };
1952        let mut next = self.next()?;
1953        next.set_reg(dst, new_value)?;
1954        verification_context.states.push(next);
1955        Ok(())
1956    }
1957
1958    fn compute_branch(
1959        &self,
1960        jump_width: JumpWidth,
1961        op1: &Type,
1962        op2: &Type,
1963        op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
1964    ) -> Result<Option<bool>, String> {
1965        match (jump_width, op1, op2) {
1966            (_, Type::ScalarValue(data1), Type::ScalarValue(data2)) => op(*data1, *data2),
1967            (JumpWidth::W64, Type::ScalarValue(data), Type::NullOr { .. })
1968            | (JumpWidth::W64, Type::NullOr { .. }, Type::ScalarValue(data))
1969                if data.is_zero() =>
1970            {
1971                Ok(None)
1972            }
1973
1974            (JumpWidth::W64, Type::ScalarValue(data), t) if data.is_zero() && t.is_non_zero() => {
1975                let non_zero =
1976                    ScalarValueData::UNKNOWN_WRITTEN.update_range(U64Range::new(1, u64::MAX));
1977                op(0.into(), non_zero)
1978            }
1979
1980            (JumpWidth::W64, t, Type::ScalarValue(data)) if data.is_zero() && t.is_non_zero() => {
1981                let non_zero =
1982                    ScalarValueData::UNKNOWN_WRITTEN.update_range(U64Range::new(1, u64::MAX));
1983                op(non_zero, 0.into())
1984            }
1985
1986            (JumpWidth::W64, Type::PtrToStack { offset: x }, Type::PtrToStack { offset: y }) => {
1987                op(x.reg(), y.reg())
1988            }
1989
1990            (
1991                JumpWidth::W64,
1992                Type::PtrToMemory { id: id1, offset: x, .. },
1993                Type::PtrToMemory { id: id2, offset: y, .. },
1994            )
1995            | (
1996                JumpWidth::W64,
1997                Type::PtrToStruct { id: id1, offset: x, .. },
1998                Type::PtrToStruct { id: id2, offset: y, .. },
1999            )
2000            | (
2001                JumpWidth::W64,
2002                Type::PtrToArray { id: id1, offset: x, .. },
2003                Type::PtrToArray { id: id2, offset: y, .. },
2004            ) if *id1 == *id2 => op(*x, *y),
2005
2006            (JumpWidth::W64, Type::PtrToArray { id: id1, .. }, Type::PtrToEndArray { id: id2 })
2007            | (JumpWidth::W64, Type::PtrToEndArray { id: id1 }, Type::PtrToArray { id: id2, .. })
2008                if *id1 == *id2 =>
2009            {
2010                Ok(None)
2011            }
2012
2013            _ => Err(()),
2014        }
2015        .map_err(|_| "non permitted comparison".to_string())
2016    }
2017
2018    fn conditional_jump(
2019        &mut self,
2020        op_name: &str,
2021        verification_context: &mut VerificationContext<'_>,
2022        dst: Register,
2023        src: Source,
2024        offset: i16,
2025        jump_type: JumpType,
2026        jump_width: JumpWidth,
2027        op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
2028    ) -> Result<(), String> {
2029        bpf_log!(
2030            self,
2031            verification_context,
2032            "{op_name} {}, {}, {}",
2033            display_register(dst),
2034            display_source(src),
2035            if offset == 0 { format!("0") } else { print_offset(offset) },
2036        );
2037        let op1 = self.reg(dst)?;
2038        let op2 = self.compute_source(src.clone())?;
2039        let apply_constraints_and_register = |mut next: Self,
2040                                              jump_type: JumpType|
2041         -> Result<Self, String> {
2042            if jump_type != JumpType::Unknown {
2043                let (new_op1, new_op2) =
2044                    Type::constraint(&mut next, jump_type, jump_width, op1.clone(), op2.clone())?;
2045                if dst < REGISTER_COUNT {
2046                    next.set_reg(dst, new_op1)?;
2047                }
2048                match src {
2049                    Source::Reg(r) => {
2050                        next.set_reg(r, new_op2)?;
2051                    }
2052                    _ => {
2053                        // Nothing to do
2054                    }
2055                }
2056            }
2057            Ok(next)
2058        };
2059        let branch = self.compute_branch(jump_width, &op1, &op2, op)?;
2060        let parent = Some(Arc::new(self.clone()));
2061        if branch.unwrap_or(true) {
2062            // Do the jump
2063            verification_context.states.push(apply_constraints_and_register(
2064                self.jump_with_offset(offset, parent.clone())?,
2065                jump_type,
2066            )?);
2067        }
2068        if !branch.unwrap_or(false) {
2069            // Skip the jump
2070            verification_context.states.push(apply_constraints_and_register(
2071                self.jump_with_offset(0, parent)?,
2072                jump_type.invert(),
2073            )?);
2074        }
2075        Ok(())
2076    }
2077
2078    /// Handles the termination of a `ComputationContext`, performing branch cutting optimization.
2079    ///
2080    /// This method is called when it has been proven that the current context terminates (e.g.,
2081    /// reaches an `exit` instruction).
2082    ///
2083    /// The following steps are performed:
2084    /// 1. **Dependency Calculation:** The data dependencies of the context are computed based on
2085    ///    the dependencies of its terminated children and the instruction at the current PC.
2086    /// 2. **Context Broadening:** The context's state is broadened by clearing registers and stack
2087    ///    slots that are *not* in the calculated data dependencies. This optimization assumes that
2088    ///    data not used by the terminated branch is irrelevant for future execution paths.
2089    /// 3. **Termination Registration:** The broadened context is added to the set of terminating
2090    ///    contexts if it is not less than any existing terminating context at the same PC.
2091    /// 4. **Parent Termination:** If all the children of the current context have terminated,
2092    ///    its parent context is recursively terminated.
2093    fn terminate(self, verification_context: &mut VerificationContext<'_>) -> Result<(), String> {
2094        let mut next = Some(self);
2095        // Because of the potential length of the parent chain, do not use recursion.
2096        while let Some(mut current) = next.take() {
2097            // Take the parent to process it at the end and not keep it in the terminating
2098            // contexts.
2099            let parent = current.parent.take();
2100
2101            // 1. Compute the dependencies of the context using the dependencies of its children
2102            //    and the actual operation.
2103            let mut dependencies = DataDependencies::default();
2104            for dependency in current.dependencies.get_mut().iter() {
2105                dependencies.merge(dependency);
2106            }
2107
2108            dependencies.visit(
2109                &mut DataDependenciesVisitorContext {
2110                    calling_context: &verification_context.calling_context,
2111                    computation_context: &current,
2112                },
2113                verification_context.code[current.pc],
2114            )?;
2115
2116            // 2. Clear the state depending on the dependencies states
2117            for register in 0..GENERAL_REGISTER_COUNT {
2118                if !dependencies.registers.contains(&register) {
2119                    current.set_reg(register, Default::default())?;
2120                }
2121            }
2122            current.stack.data.retain(|k, _| dependencies.stack.contains(k));
2123
2124            // 3. Add the cleared state to the set of `terminating_contexts`
2125            let terminating_contexts =
2126                verification_context.terminating_contexts.entry(current.pc).or_default();
2127            let mut is_dominated = false;
2128            terminating_contexts.retain(|c| match c.computation_context.partial_cmp(&current) {
2129                Some(Ordering::Less) => false,
2130                Some(Ordering::Equal) | Some(Ordering::Greater) => {
2131                    // If the list contains a context greater or equal to the current one, it
2132                    // should not be added.
2133                    is_dominated = true;
2134                    true
2135                }
2136                _ => true,
2137            });
2138            if !is_dominated {
2139                terminating_contexts.push(TerminatingContext {
2140                    computation_context: current,
2141                    dependencies: dependencies.clone(),
2142                });
2143            }
2144
2145            // 4. Register the computed dependencies in our parent, and terminate it if all
2146            //    dependencies has been computed.
2147            if let Some(parent) = parent {
2148                parent.dependencies.lock().push(dependencies);
2149                // To check whether all dependencies have been computed, rely on the fact that the Arc
2150                // count of the parent keep track of how many dependencies are left.
2151                next = Arc::into_inner(parent);
2152            }
2153        }
2154        Ok(())
2155    }
2156}
2157
2158impl Drop for ComputationContext {
2159    fn drop(&mut self) {
2160        let mut next = self.parent.take().and_then(Arc::into_inner);
2161        // Because of the potential length of the parent chain, do not use recursion.
2162        while let Some(mut current) = next {
2163            next = current.parent.take().and_then(Arc::into_inner);
2164        }
2165    }
2166}
2167
2168/// Two types are ordered with `t1` > `t2` if a proof that a program in a state `t1` finish is also
2169/// a proof that a program in a state `t2` finish.
2170impl PartialOrd for ComputationContext {
2171    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
2172        if self.pc != other.pc || self.resources != other.resources {
2173            return None;
2174        }
2175        let mut result = self.stack.partial_cmp(&other.stack)?;
2176        result = associate_orderings(
2177            result,
2178            Type::compare_list(self.registers.iter(), other.registers.iter())?,
2179        )?;
2180        let mut array_bound_iter1 = self.array_bounds.iter().peekable();
2181        let mut array_bound_iter2 = other.array_bounds.iter().peekable();
2182        let result = loop {
2183            match (array_bound_iter1.peek().cloned(), array_bound_iter2.peek().cloned()) {
2184                (None, None) => break result,
2185                (None, _) => break associate_orderings(result, Ordering::Greater)?,
2186                (_, None) => break associate_orderings(result, Ordering::Less)?,
2187                (Some((k1, v1)), Some((k2, v2))) => match k1.cmp(k2) {
2188                    Ordering::Equal => {
2189                        array_bound_iter1.next();
2190                        array_bound_iter2.next();
2191                        // The comparison is intentionally reversed (v2.cmp(v1) instead of
2192                        // v1.cmp(v2)). `v1` is the bound of the explored state (`self`) and `v2` is
2193                        // the bound of the current state (`other`). A smaller bound value
2194                        // represents a stronger safety guarantee (fewer assumed valid bytes,
2195                        // meaning a more restricted program). Therefore, a state with a smaller
2196                        // bound dominates (is "Greater" than) a state with a larger bound.
2197                        result = associate_orderings(result, v2.cmp(v1))?;
2198                    }
2199                    v @ Ordering::Less => {
2200                        array_bound_iter1.next();
2201                        result = associate_orderings(result, v)?;
2202                    }
2203                    v @ Ordering::Greater => {
2204                        array_bound_iter2.next();
2205                        result = associate_orderings(result, v)?;
2206                    }
2207                },
2208            }
2209        };
2210        Some(result)
2211    }
2212}
2213
2214/// Represents the read data dependencies of an eBPF program branch.
2215///
2216/// This struct tracks which registers and stack positions are *read* by the
2217/// instructions within a branch of the eBPF program.  This information is used
2218/// during branch cutting optimization to broaden terminating contexts.
2219///
2220/// The verifier assumes that data not read by a terminated branch is irrelevant
2221/// for future execution paths and can be safely cleared.
2222#[derive(Clone, Debug, Default)]
2223struct DataDependencies {
2224    /// The set of registers read by the children of a context.
2225    registers: HashSet<Register>,
2226    /// The stack positions read by the children of a context.
2227    stack: HashSet<usize>,
2228}
2229
2230impl DataDependencies {
2231    fn merge(&mut self, other: &DataDependencies) {
2232        self.registers.extend(other.registers.iter());
2233        self.stack.extend(other.stack.iter());
2234    }
2235
2236    fn alu(&mut self, dst: Register, src: Source) -> Result<(), String> {
2237        // Only do something if the dst is read, otherwise the computation doesn't matter.
2238        if self.registers.contains(&dst) {
2239            if let Source::Reg(src) = src {
2240                self.registers.insert(src);
2241            }
2242        }
2243        Ok(())
2244    }
2245
2246    fn jmp(&mut self, dst: Register, src: Source) -> Result<(), String> {
2247        self.registers.insert(dst);
2248        if let Source::Reg(src) = src {
2249            self.registers.insert(src);
2250        }
2251        Ok(())
2252    }
2253
2254    fn atomic(
2255        &mut self,
2256        context: &ComputationContext,
2257        fetch: bool,
2258        dst: Register,
2259        offset: i16,
2260        src: Register,
2261        width: DataWidth,
2262        is_cmpxchg: bool,
2263    ) -> Result<(), String> {
2264        let mut is_read = false;
2265        if is_cmpxchg && self.registers.contains(&0) {
2266            is_read = true;
2267        }
2268        if fetch && self.registers.contains(&src) {
2269            is_read = true;
2270        }
2271        let addr = context.reg(dst)?;
2272        if let Type::PtrToStack { offset: stack_offset } = addr {
2273            let stack_offset = stack_offset.add(offset);
2274            if !stack_offset.is_valid_offset() {
2275                return Err(format!("Invalid stack offset at {}", context.pc));
2276            }
2277            if is_read || self.stack.contains(&stack_offset.array_index()) {
2278                is_read = true;
2279                self.stack.insert(stack_offset.array_index());
2280            }
2281        }
2282        if is_read {
2283            self.registers.insert(0);
2284            self.registers.insert(src);
2285        }
2286        self.registers.insert(dst);
2287        Ok(())
2288    }
2289}
2290
2291struct DataDependenciesVisitorContext<'a> {
2292    calling_context: &'a CallingContext,
2293    computation_context: &'a ComputationContext,
2294}
2295
2296impl BpfVisitor for DataDependencies {
2297    type Context<'a> = DataDependenciesVisitorContext<'a>;
2298
2299    fn add<'a>(
2300        &mut self,
2301        _context: &mut Self::Context<'a>,
2302        dst: Register,
2303        src: Source,
2304    ) -> Result<(), String> {
2305        self.alu(dst, src)
2306    }
2307    fn add64<'a>(
2308        &mut self,
2309        _context: &mut Self::Context<'a>,
2310        dst: Register,
2311        src: Source,
2312    ) -> Result<(), String> {
2313        self.alu(dst, src)
2314    }
2315    fn and<'a>(
2316        &mut self,
2317        _context: &mut Self::Context<'a>,
2318        dst: Register,
2319        src: Source,
2320    ) -> Result<(), String> {
2321        self.alu(dst, src)
2322    }
2323    fn and64<'a>(
2324        &mut self,
2325        _context: &mut Self::Context<'a>,
2326        dst: Register,
2327        src: Source,
2328    ) -> Result<(), String> {
2329        self.alu(dst, src)
2330    }
2331    fn arsh<'a>(
2332        &mut self,
2333        _context: &mut Self::Context<'a>,
2334        dst: Register,
2335        src: Source,
2336    ) -> Result<(), String> {
2337        self.alu(dst, src)
2338    }
2339    fn arsh64<'a>(
2340        &mut self,
2341        _context: &mut Self::Context<'a>,
2342        dst: Register,
2343        src: Source,
2344    ) -> Result<(), String> {
2345        self.alu(dst, src)
2346    }
2347    fn div<'a>(
2348        &mut self,
2349        _context: &mut Self::Context<'a>,
2350        dst: Register,
2351        src: Source,
2352    ) -> Result<(), String> {
2353        self.alu(dst, src)
2354    }
2355    fn div64<'a>(
2356        &mut self,
2357        _context: &mut Self::Context<'a>,
2358        dst: Register,
2359        src: Source,
2360    ) -> Result<(), String> {
2361        self.alu(dst, src)
2362    }
2363    fn lsh<'a>(
2364        &mut self,
2365        _context: &mut Self::Context<'a>,
2366        dst: Register,
2367        src: Source,
2368    ) -> Result<(), String> {
2369        self.alu(dst, src)
2370    }
2371    fn lsh64<'a>(
2372        &mut self,
2373        _context: &mut Self::Context<'a>,
2374        dst: Register,
2375        src: Source,
2376    ) -> Result<(), String> {
2377        self.alu(dst, src)
2378    }
2379    fn r#mod<'a>(
2380        &mut self,
2381        _context: &mut Self::Context<'a>,
2382        dst: Register,
2383        src: Source,
2384    ) -> Result<(), String> {
2385        self.alu(dst, src)
2386    }
2387    fn mod64<'a>(
2388        &mut self,
2389        _context: &mut Self::Context<'a>,
2390        dst: Register,
2391        src: Source,
2392    ) -> Result<(), String> {
2393        self.alu(dst, src)
2394    }
2395    fn mul<'a>(
2396        &mut self,
2397        _context: &mut Self::Context<'a>,
2398        dst: Register,
2399        src: Source,
2400    ) -> Result<(), String> {
2401        self.alu(dst, src)
2402    }
2403    fn mul64<'a>(
2404        &mut self,
2405        _context: &mut Self::Context<'a>,
2406        dst: Register,
2407        src: Source,
2408    ) -> Result<(), String> {
2409        self.alu(dst, src)
2410    }
2411    fn or<'a>(
2412        &mut self,
2413        _context: &mut Self::Context<'a>,
2414        dst: Register,
2415        src: Source,
2416    ) -> Result<(), String> {
2417        self.alu(dst, src)
2418    }
2419    fn or64<'a>(
2420        &mut self,
2421        _context: &mut Self::Context<'a>,
2422        dst: Register,
2423        src: Source,
2424    ) -> Result<(), String> {
2425        self.alu(dst, src)
2426    }
2427    fn rsh<'a>(
2428        &mut self,
2429        _context: &mut Self::Context<'a>,
2430        dst: Register,
2431        src: Source,
2432    ) -> Result<(), String> {
2433        self.alu(dst, src)
2434    }
2435    fn rsh64<'a>(
2436        &mut self,
2437        _context: &mut Self::Context<'a>,
2438        dst: Register,
2439        src: Source,
2440    ) -> Result<(), String> {
2441        self.alu(dst, src)
2442    }
2443    fn sub<'a>(
2444        &mut self,
2445        _context: &mut Self::Context<'a>,
2446        dst: Register,
2447        src: Source,
2448    ) -> Result<(), String> {
2449        self.alu(dst, src)
2450    }
2451    fn sub64<'a>(
2452        &mut self,
2453        _context: &mut Self::Context<'a>,
2454        dst: Register,
2455        src: Source,
2456    ) -> Result<(), String> {
2457        self.alu(dst, src)
2458    }
2459    fn xor<'a>(
2460        &mut self,
2461        _context: &mut Self::Context<'a>,
2462        dst: Register,
2463        src: Source,
2464    ) -> Result<(), String> {
2465        self.alu(dst, src)
2466    }
2467    fn xor64<'a>(
2468        &mut self,
2469        _context: &mut Self::Context<'a>,
2470        dst: Register,
2471        src: Source,
2472    ) -> Result<(), String> {
2473        self.alu(dst, src)
2474    }
2475
2476    fn mov<'a>(
2477        &mut self,
2478        _context: &mut Self::Context<'a>,
2479        dst: Register,
2480        src: Source,
2481    ) -> Result<(), String> {
2482        if src == Source::Reg(dst) || !self.registers.contains(&dst) {
2483            return Ok(());
2484        }
2485        if let Source::Reg(src) = src {
2486            self.registers.insert(src);
2487        }
2488        self.registers.remove(&dst);
2489        Ok(())
2490    }
2491    fn mov64<'a>(
2492        &mut self,
2493        context: &mut Self::Context<'a>,
2494        dst: Register,
2495        src: Source,
2496    ) -> Result<(), String> {
2497        self.mov(context, dst, src)
2498    }
2499
2500    fn neg<'a>(&mut self, _context: &mut Self::Context<'a>, _dst: Register) -> Result<(), String> {
2501        // This is reading and writing the same value. This induces no change in dependencies.
2502        Ok(())
2503    }
2504    fn neg64<'a>(
2505        &mut self,
2506        _context: &mut Self::Context<'a>,
2507        _dst: Register,
2508    ) -> Result<(), String> {
2509        // This is reading and writing the same value. This induces no change in dependencies.
2510        Ok(())
2511    }
2512
2513    fn be<'a>(
2514        &mut self,
2515        _context: &mut Self::Context<'a>,
2516        _dst: Register,
2517        _width: DataWidth,
2518    ) -> Result<(), String> {
2519        // This is reading and writing the same value. This induces no change in dependencies.
2520        Ok(())
2521    }
2522    fn le<'a>(
2523        &mut self,
2524        _context: &mut Self::Context<'a>,
2525        _dst: Register,
2526        _width: DataWidth,
2527    ) -> Result<(), String> {
2528        // This is reading and writing the same value. This induces no change in dependencies.
2529        Ok(())
2530    }
2531
2532    fn call_external<'a>(
2533        &mut self,
2534        context: &mut Self::Context<'a>,
2535        index: u32,
2536    ) -> Result<(), String> {
2537        let Some(helper) = context.calling_context.helpers.get(&index).cloned() else {
2538            return Err(format!("unknown external function {}", index));
2539        };
2540        // A helper reads the stack region each `input` memory-parameter argument
2541        // points at, so those stack slots are data dependencies, like load/store.
2542        let comp = &context.computation_context;
2543        for (arg_index, arg) in helper.signature.args.iter().enumerate() {
2544            if let Type::MemoryParameter { size, input: true, .. } = arg {
2545                if let Type::PtrToStack { offset } = comp.reg((arg_index + 1) as Register)? {
2546                    let end = offset.add(size.size(comp)?);
2547                    if offset.is_valid_offset() && end.is_within_stack() {
2548                        for slot in offset.array_index()..end.array_index() {
2549                            self.stack.insert(slot);
2550                        }
2551                        if end.sub_index() != 0 {
2552                            self.stack.insert(end.array_index());
2553                        }
2554                    }
2555                }
2556            }
2557        }
2558        // 0 is overwritten and 1 to 5 are scratch registers
2559        for register in 0..helper.signature.args.len() + 1 {
2560            self.registers.remove(&(register as Register));
2561        }
2562        // 1 to k are parameters.
2563        for register in 0..helper.signature.args.len() {
2564            self.registers.insert((register + 1) as Register);
2565        }
2566        Ok(())
2567    }
2568
2569    fn exit<'a>(&mut self, _context: &mut Self::Context<'a>) -> Result<(), String> {
2570        // This read r0 unconditionally.
2571        self.registers.insert(0);
2572        Ok(())
2573    }
2574
2575    fn jump<'a>(&mut self, _context: &mut Self::Context<'a>, _offset: i16) -> Result<(), String> {
2576        // This doesn't do anything with values.
2577        Ok(())
2578    }
2579
2580    fn jeq<'a>(
2581        &mut self,
2582        _context: &mut Self::Context<'a>,
2583        dst: Register,
2584        src: Source,
2585        offset: i16,
2586    ) -> Result<(), String> {
2587        self.jmp(dst, src)
2588    }
2589    fn jeq64<'a>(
2590        &mut self,
2591        _context: &mut Self::Context<'a>,
2592        dst: Register,
2593        src: Source,
2594        offset: i16,
2595    ) -> Result<(), String> {
2596        self.jmp(dst, src)
2597    }
2598    fn jne<'a>(
2599        &mut self,
2600        _context: &mut Self::Context<'a>,
2601        dst: Register,
2602        src: Source,
2603        offset: i16,
2604    ) -> Result<(), String> {
2605        self.jmp(dst, src)
2606    }
2607    fn jne64<'a>(
2608        &mut self,
2609        _context: &mut Self::Context<'a>,
2610        dst: Register,
2611        src: Source,
2612        offset: i16,
2613    ) -> Result<(), String> {
2614        self.jmp(dst, src)
2615    }
2616    fn jge<'a>(
2617        &mut self,
2618        _context: &mut Self::Context<'a>,
2619        dst: Register,
2620        src: Source,
2621        offset: i16,
2622    ) -> Result<(), String> {
2623        self.jmp(dst, src)
2624    }
2625    fn jge64<'a>(
2626        &mut self,
2627        _context: &mut Self::Context<'a>,
2628        dst: Register,
2629        src: Source,
2630        offset: i16,
2631    ) -> Result<(), String> {
2632        self.jmp(dst, src)
2633    }
2634    fn jgt<'a>(
2635        &mut self,
2636        _context: &mut Self::Context<'a>,
2637        dst: Register,
2638        src: Source,
2639        offset: i16,
2640    ) -> Result<(), String> {
2641        self.jmp(dst, src)
2642    }
2643    fn jgt64<'a>(
2644        &mut self,
2645        _context: &mut Self::Context<'a>,
2646        dst: Register,
2647        src: Source,
2648        offset: i16,
2649    ) -> Result<(), String> {
2650        self.jmp(dst, src)
2651    }
2652    fn jle<'a>(
2653        &mut self,
2654        _context: &mut Self::Context<'a>,
2655        dst: Register,
2656        src: Source,
2657        offset: i16,
2658    ) -> Result<(), String> {
2659        self.jmp(dst, src)
2660    }
2661    fn jle64<'a>(
2662        &mut self,
2663        _context: &mut Self::Context<'a>,
2664        dst: Register,
2665        src: Source,
2666        offset: i16,
2667    ) -> Result<(), String> {
2668        self.jmp(dst, src)
2669    }
2670    fn jlt<'a>(
2671        &mut self,
2672        _context: &mut Self::Context<'a>,
2673        dst: Register,
2674        src: Source,
2675        offset: i16,
2676    ) -> Result<(), String> {
2677        self.jmp(dst, src)
2678    }
2679    fn jlt64<'a>(
2680        &mut self,
2681        _context: &mut Self::Context<'a>,
2682        dst: Register,
2683        src: Source,
2684        offset: i16,
2685    ) -> Result<(), String> {
2686        self.jmp(dst, src)
2687    }
2688    fn jsge<'a>(
2689        &mut self,
2690        _context: &mut Self::Context<'a>,
2691        dst: Register,
2692        src: Source,
2693        offset: i16,
2694    ) -> Result<(), String> {
2695        self.jmp(dst, src)
2696    }
2697    fn jsge64<'a>(
2698        &mut self,
2699        _context: &mut Self::Context<'a>,
2700        dst: Register,
2701        src: Source,
2702        offset: i16,
2703    ) -> Result<(), String> {
2704        self.jmp(dst, src)
2705    }
2706    fn jsgt<'a>(
2707        &mut self,
2708        _context: &mut Self::Context<'a>,
2709        dst: Register,
2710        src: Source,
2711        offset: i16,
2712    ) -> Result<(), String> {
2713        self.jmp(dst, src)
2714    }
2715    fn jsgt64<'a>(
2716        &mut self,
2717        _context: &mut Self::Context<'a>,
2718        dst: Register,
2719        src: Source,
2720        offset: i16,
2721    ) -> Result<(), String> {
2722        self.jmp(dst, src)
2723    }
2724    fn jsle<'a>(
2725        &mut self,
2726        _context: &mut Self::Context<'a>,
2727        dst: Register,
2728        src: Source,
2729        offset: i16,
2730    ) -> Result<(), String> {
2731        self.jmp(dst, src)
2732    }
2733    fn jsle64<'a>(
2734        &mut self,
2735        _context: &mut Self::Context<'a>,
2736        dst: Register,
2737        src: Source,
2738        offset: i16,
2739    ) -> Result<(), String> {
2740        self.jmp(dst, src)
2741    }
2742    fn jslt<'a>(
2743        &mut self,
2744        _context: &mut Self::Context<'a>,
2745        dst: Register,
2746        src: Source,
2747        offset: i16,
2748    ) -> Result<(), String> {
2749        self.jmp(dst, src)
2750    }
2751    fn jslt64<'a>(
2752        &mut self,
2753        _context: &mut Self::Context<'a>,
2754        dst: Register,
2755        src: Source,
2756        offset: i16,
2757    ) -> Result<(), String> {
2758        self.jmp(dst, src)
2759    }
2760    fn jset<'a>(
2761        &mut self,
2762        _context: &mut Self::Context<'a>,
2763        dst: Register,
2764        src: Source,
2765        offset: i16,
2766    ) -> Result<(), String> {
2767        self.jmp(dst, src)
2768    }
2769    fn jset64<'a>(
2770        &mut self,
2771        _context: &mut Self::Context<'a>,
2772        dst: Register,
2773        src: Source,
2774        offset: i16,
2775    ) -> Result<(), String> {
2776        self.jmp(dst, src)
2777    }
2778
2779    fn atomic_add<'a>(
2780        &mut self,
2781        context: &mut Self::Context<'a>,
2782        fetch: bool,
2783        dst: Register,
2784        offset: i16,
2785        src: Register,
2786    ) -> Result<(), String> {
2787        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2788    }
2789
2790    fn atomic_add64<'a>(
2791        &mut self,
2792        context: &mut Self::Context<'a>,
2793        fetch: bool,
2794        dst: Register,
2795        offset: i16,
2796        src: Register,
2797    ) -> Result<(), String> {
2798        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2799    }
2800
2801    fn atomic_and<'a>(
2802        &mut self,
2803        context: &mut Self::Context<'a>,
2804        fetch: bool,
2805        dst: Register,
2806        offset: i16,
2807        src: Register,
2808    ) -> Result<(), String> {
2809        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2810    }
2811
2812    fn atomic_and64<'a>(
2813        &mut self,
2814        context: &mut Self::Context<'a>,
2815        fetch: bool,
2816        dst: Register,
2817        offset: i16,
2818        src: Register,
2819    ) -> Result<(), String> {
2820        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2821    }
2822
2823    fn atomic_or<'a>(
2824        &mut self,
2825        context: &mut Self::Context<'a>,
2826        fetch: bool,
2827        dst: Register,
2828        offset: i16,
2829        src: Register,
2830    ) -> Result<(), String> {
2831        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2832    }
2833
2834    fn atomic_or64<'a>(
2835        &mut self,
2836        context: &mut Self::Context<'a>,
2837        fetch: bool,
2838        dst: Register,
2839        offset: i16,
2840        src: Register,
2841    ) -> Result<(), String> {
2842        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2843    }
2844
2845    fn atomic_xor<'a>(
2846        &mut self,
2847        context: &mut Self::Context<'a>,
2848        fetch: bool,
2849        dst: Register,
2850        offset: i16,
2851        src: Register,
2852    ) -> Result<(), String> {
2853        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2854    }
2855
2856    fn atomic_xor64<'a>(
2857        &mut self,
2858        context: &mut Self::Context<'a>,
2859        fetch: bool,
2860        dst: Register,
2861        offset: i16,
2862        src: Register,
2863    ) -> Result<(), String> {
2864        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2865    }
2866
2867    fn atomic_xchg<'a>(
2868        &mut self,
2869        context: &mut Self::Context<'a>,
2870        fetch: bool,
2871        dst: Register,
2872        offset: i16,
2873        src: Register,
2874    ) -> Result<(), String> {
2875        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2876    }
2877
2878    fn atomic_xchg64<'a>(
2879        &mut self,
2880        context: &mut Self::Context<'a>,
2881        fetch: bool,
2882        dst: Register,
2883        offset: i16,
2884        src: Register,
2885    ) -> Result<(), String> {
2886        self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2887    }
2888
2889    fn atomic_cmpxchg<'a>(
2890        &mut self,
2891        context: &mut Self::Context<'a>,
2892        dst: Register,
2893        offset: i16,
2894        src: Register,
2895    ) -> Result<(), String> {
2896        self.atomic(&context.computation_context, true, dst, offset, src, DataWidth::U32, true)
2897    }
2898
2899    fn atomic_cmpxchg64<'a>(
2900        &mut self,
2901        context: &mut Self::Context<'a>,
2902        dst: Register,
2903        offset: i16,
2904        src: Register,
2905    ) -> Result<(), String> {
2906        self.atomic(&context.computation_context, true, dst, offset, src, DataWidth::U64, true)
2907    }
2908
2909    fn load<'a>(
2910        &mut self,
2911        context: &mut Self::Context<'a>,
2912        dst: Register,
2913        offset: i16,
2914        src: Register,
2915        width: DataWidth,
2916    ) -> Result<(), String> {
2917        let context = &context.computation_context;
2918        if self.registers.contains(&dst) {
2919            let addr = context.reg(src)?;
2920            if let Type::PtrToStack { offset: stack_offset } = addr {
2921                let stack_offset = stack_offset.add(offset);
2922                if !stack_offset.is_valid_offset() {
2923                    return Err(format!("Invalid stack offset at {}", context.pc));
2924                }
2925                self.stack.insert(stack_offset.array_index());
2926            }
2927        }
2928        self.registers.insert(src);
2929        Ok(())
2930    }
2931
2932    fn load64<'a>(
2933        &mut self,
2934        _context: &mut Self::Context<'a>,
2935        dst: Register,
2936        _src: u8,
2937        _lower: u32,
2938    ) -> Result<(), String> {
2939        self.registers.remove(&dst);
2940        Ok(())
2941    }
2942
2943    fn load_from_packet<'a>(
2944        &mut self,
2945        _context: &mut Self::Context<'a>,
2946        dst: Register,
2947        src: Register,
2948        _offset: i32,
2949        register_offset: Option<Register>,
2950        _width: DataWidth,
2951    ) -> Result<(), String> {
2952        // 1 to 5 are scratch registers
2953        for register in 1..6 {
2954            self.registers.remove(&(register as Register));
2955        }
2956        // Only do something if the dst is read, otherwise the computation doesn't matter.
2957        if self.registers.remove(&dst) {
2958            self.registers.insert(src);
2959            if let Some(reg) = register_offset {
2960                self.registers.insert(reg);
2961            }
2962        }
2963        Ok(())
2964    }
2965
2966    fn store<'a>(
2967        &mut self,
2968        context: &mut Self::Context<'a>,
2969        dst: Register,
2970        offset: i16,
2971        src: Source,
2972        width: DataWidth,
2973    ) -> Result<(), String> {
2974        let context = &context.computation_context;
2975        let addr = context.reg(dst)?;
2976        if let Type::PtrToStack { offset: stack_offset } = addr {
2977            let stack_offset = stack_offset.add(offset);
2978            if !stack_offset.is_valid_offset() {
2979                return Err(format!("Invalid stack offset at {}", context.pc));
2980            }
2981            if self.stack.remove(&stack_offset.array_index()) {
2982                if let Source::Reg(src) = src {
2983                    self.registers.insert(src);
2984                }
2985            }
2986        } else {
2987            if let Source::Reg(src) = src {
2988                self.registers.insert(src);
2989            }
2990            self.registers.insert(dst);
2991        }
2992
2993        Ok(())
2994    }
2995}
2996
2997#[derive(Debug)]
2998struct TerminatingContext {
2999    computation_context: ComputationContext,
3000    dependencies: DataDependencies,
3001}
3002
3003#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3004enum AluType {
3005    Plain,
3006    Sub,
3007    Add,
3008}
3009
3010impl AluType {
3011    /// Can this operation be done one a pointer and a scalar.
3012    fn is_ptr_compatible(&self) -> bool {
3013        match self {
3014            Self::Sub | Self::Add => true,
3015            _ => false,
3016        }
3017    }
3018}
3019
3020#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3021enum JumpWidth {
3022    W32,
3023    W64,
3024}
3025
3026#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3027enum JumpType {
3028    Eq,
3029    Ge,
3030    Gt,
3031    Le,
3032    LooseComparaison,
3033    Lt,
3034    Ne,
3035    StrictComparaison,
3036    Unknown,
3037}
3038
3039impl JumpType {
3040    fn invert(&self) -> Self {
3041        match self {
3042            Self::Eq => Self::Ne,
3043            Self::Ge => Self::Lt,
3044            Self::Gt => Self::Le,
3045            Self::Le => Self::Gt,
3046            Self::LooseComparaison => Self::StrictComparaison,
3047            Self::Lt => Self::Ge,
3048            Self::Ne => Self::Eq,
3049            Self::StrictComparaison => Self::LooseComparaison,
3050            Self::Unknown => Self::Unknown,
3051        }
3052    }
3053
3054    fn is_strict(&self) -> bool {
3055        match self {
3056            Self::Gt | Self::Lt | Self::Ne | Self::StrictComparaison => true,
3057            _ => false,
3058        }
3059    }
3060}
3061
3062fn display_register(register: Register) -> String {
3063    format!("%r{register}")
3064}
3065
3066fn display_source(src: Source) -> String {
3067    match src {
3068        Source::Reg(r) => display_register(r),
3069        Source::Value(v) => format!("0x{v:x}"),
3070    }
3071}
3072
3073impl BpfVisitor for ComputationContext {
3074    type Context<'a> = VerificationContext<'a>;
3075
3076    fn add<'a>(
3077        &mut self,
3078        context: &mut Self::Context<'a>,
3079        dst: Register,
3080        src: Source,
3081    ) -> Result<(), String> {
3082        self.alu(Some("add32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x + y))
3083    }
3084    fn add64<'a>(
3085        &mut self,
3086        context: &mut Self::Context<'a>,
3087        dst: Register,
3088        src: Source,
3089    ) -> Result<(), String> {
3090        self.alu(Some("add"), context, dst, src, AluType::Add, |x, y| x + y)
3091    }
3092    fn and<'a>(
3093        &mut self,
3094        context: &mut Self::Context<'a>,
3095        dst: Register,
3096        src: Source,
3097    ) -> Result<(), String> {
3098        self.alu(Some("and32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x & y))
3099    }
3100    fn and64<'a>(
3101        &mut self,
3102        context: &mut Self::Context<'a>,
3103        dst: Register,
3104        src: Source,
3105    ) -> Result<(), String> {
3106        self.alu(Some("and"), context, dst, src, AluType::Plain, |x, y| x & y)
3107    }
3108    fn arsh<'a>(
3109        &mut self,
3110        context: &mut Self::Context<'a>,
3111        dst: Register,
3112        src: Source,
3113    ) -> Result<(), String> {
3114        self.alu(Some("arsh32"), context, dst, src, AluType::Plain, |x, y| {
3115            alu32(x, y, |x, y| x.ashr(y))
3116        })
3117    }
3118    fn arsh64<'a>(
3119        &mut self,
3120        context: &mut Self::Context<'a>,
3121        dst: Register,
3122        src: Source,
3123    ) -> Result<(), String> {
3124        self.alu(Some("arsh"), context, dst, src, AluType::Plain, |x, y| x.ashr(y))
3125    }
3126    fn div<'a>(
3127        &mut self,
3128        context: &mut Self::Context<'a>,
3129        dst: Register,
3130        src: Source,
3131    ) -> Result<(), String> {
3132        self.alu(Some("div32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x / y))
3133    }
3134    fn div64<'a>(
3135        &mut self,
3136        context: &mut Self::Context<'a>,
3137        dst: Register,
3138        src: Source,
3139    ) -> Result<(), String> {
3140        self.alu(Some("div"), context, dst, src, AluType::Plain, |x, y| x / y)
3141    }
3142    fn lsh<'a>(
3143        &mut self,
3144        context: &mut Self::Context<'a>,
3145        dst: Register,
3146        src: Source,
3147    ) -> Result<(), String> {
3148        self.alu(Some("lsh32"), context, dst, src, AluType::Plain, |x, y| {
3149            alu32(x, y, |x, y| x << y)
3150        })
3151    }
3152    fn lsh64<'a>(
3153        &mut self,
3154        context: &mut Self::Context<'a>,
3155        dst: Register,
3156        src: Source,
3157    ) -> Result<(), String> {
3158        self.alu(Some("lsh"), context, dst, src, AluType::Plain, |x, y| x << y)
3159    }
3160    fn r#mod<'a>(
3161        &mut self,
3162        context: &mut Self::Context<'a>,
3163        dst: Register,
3164        src: Source,
3165    ) -> Result<(), String> {
3166        self.alu(Some("mod32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x % y))
3167    }
3168    fn mod64<'a>(
3169        &mut self,
3170        context: &mut Self::Context<'a>,
3171        dst: Register,
3172        src: Source,
3173    ) -> Result<(), String> {
3174        self.alu(Some("mod"), context, dst, src, AluType::Plain, |x, y| x % y)
3175    }
3176    fn mov<'a>(
3177        &mut self,
3178        context: &mut Self::Context<'a>,
3179        dst: Register,
3180        src: Source,
3181    ) -> Result<(), String> {
3182        bpf_log!(self, context, "mov32 {}, {}", display_register(dst), display_source(src));
3183        let src = self.compute_source(src)?;
3184        let value = match src {
3185            Type::ScalarValue(data) => {
3186                let value = (data.value as u32) as u64;
3187                let unknown_mask = (data.unknown_mask as u32) as u64;
3188                let unwritten_mask = (data.unwritten_mask as u32) as u64;
3189                let urange = U64Range::compute_range_for_bytes_swap(0.into(), data.urange, 0, 0, 4);
3190                Type::ScalarValue(ScalarValueData::new(value, unknown_mask, unwritten_mask, urange))
3191            }
3192            _ => Type::default(),
3193        };
3194        let mut next = self.next()?;
3195        next.set_reg(dst, value)?;
3196        context.states.push(next);
3197        Ok(())
3198    }
3199    fn mov64<'a>(
3200        &mut self,
3201        context: &mut Self::Context<'a>,
3202        dst: Register,
3203        src: Source,
3204    ) -> Result<(), String> {
3205        bpf_log!(self, context, "mov {}, {}", display_register(dst), display_source(src));
3206        let src = self.compute_source(src)?;
3207        let mut next = self.next()?;
3208        next.set_reg(dst, src)?;
3209        context.states.push(next);
3210        Ok(())
3211    }
3212    fn mul<'a>(
3213        &mut self,
3214        context: &mut Self::Context<'a>,
3215        dst: Register,
3216        src: Source,
3217    ) -> Result<(), String> {
3218        self.alu(Some("mul32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x * y))
3219    }
3220    fn mul64<'a>(
3221        &mut self,
3222        context: &mut Self::Context<'a>,
3223        dst: Register,
3224        src: Source,
3225    ) -> Result<(), String> {
3226        self.alu(Some("mul"), context, dst, src, AluType::Plain, |x, y| x * y)
3227    }
3228    fn or<'a>(
3229        &mut self,
3230        context: &mut Self::Context<'a>,
3231        dst: Register,
3232        src: Source,
3233    ) -> Result<(), String> {
3234        self.alu(Some("or32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x | y))
3235    }
3236    fn or64<'a>(
3237        &mut self,
3238        context: &mut Self::Context<'a>,
3239        dst: Register,
3240        src: Source,
3241    ) -> Result<(), String> {
3242        self.alu(Some("or"), context, dst, src, AluType::Plain, |x, y| x | y)
3243    }
3244    fn rsh<'a>(
3245        &mut self,
3246        context: &mut Self::Context<'a>,
3247        dst: Register,
3248        src: Source,
3249    ) -> Result<(), String> {
3250        self.alu(Some("rsh32"), context, dst, src, AluType::Plain, |x, y| {
3251            alu32(x, y, |x, y| x >> y)
3252        })
3253    }
3254    fn rsh64<'a>(
3255        &mut self,
3256        context: &mut Self::Context<'a>,
3257        dst: Register,
3258        src: Source,
3259    ) -> Result<(), String> {
3260        self.alu(Some("rsh"), context, dst, src, AluType::Plain, |x, y| x >> y)
3261    }
3262    fn sub<'a>(
3263        &mut self,
3264        context: &mut Self::Context<'a>,
3265        dst: Register,
3266        src: Source,
3267    ) -> Result<(), String> {
3268        self.alu(Some("sub32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x - y))
3269    }
3270    fn sub64<'a>(
3271        &mut self,
3272        context: &mut Self::Context<'a>,
3273        dst: Register,
3274        src: Source,
3275    ) -> Result<(), String> {
3276        self.alu(Some("sub"), context, dst, src, AluType::Sub, |x, y| x - y)
3277    }
3278    fn xor<'a>(
3279        &mut self,
3280        context: &mut Self::Context<'a>,
3281        dst: Register,
3282        src: Source,
3283    ) -> Result<(), String> {
3284        self.alu(Some("xor32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x ^ y))
3285    }
3286    fn xor64<'a>(
3287        &mut self,
3288        context: &mut Self::Context<'a>,
3289        dst: Register,
3290        src: Source,
3291    ) -> Result<(), String> {
3292        self.alu(Some("xor"), context, dst, src, AluType::Plain, |x, y| x ^ y)
3293    }
3294
3295    fn neg<'a>(&mut self, context: &mut Self::Context<'a>, dst: Register) -> Result<(), String> {
3296        bpf_log!(self, context, "neg32 {}", display_register(dst));
3297        self.alu(None, context, dst, Source::Value(0), AluType::Plain, |x, y| {
3298            alu32(x, y, |x, _y| -x)
3299        })
3300    }
3301    fn neg64<'a>(&mut self, context: &mut Self::Context<'a>, dst: Register) -> Result<(), String> {
3302        bpf_log!(self, context, "neg {}", display_register(dst));
3303        self.alu(None, context, dst, Source::Value(0), AluType::Plain, |x, _y| -x)
3304    }
3305
3306    fn be<'a>(
3307        &mut self,
3308        context: &mut Self::Context<'a>,
3309        dst: Register,
3310        width: DataWidth,
3311    ) -> Result<(), String> {
3312        self.endianness::<BigEndian>("be", context, dst, width)
3313    }
3314
3315    fn le<'a>(
3316        &mut self,
3317        context: &mut Self::Context<'a>,
3318        dst: Register,
3319        width: DataWidth,
3320    ) -> Result<(), String> {
3321        self.endianness::<LittleEndian>("le", context, dst, width)
3322    }
3323
3324    fn call_external<'a>(
3325        &mut self,
3326        context: &mut Self::Context<'a>,
3327        index: u32,
3328    ) -> Result<(), String> {
3329        bpf_log!(self, context, "call 0x{:x}", index);
3330        let Some(helper) = context.calling_context.helpers.get(&index).cloned() else {
3331            return Err(format!("unknown external function {}", index));
3332        };
3333        let HelperDefinition { signature, name, .. } = helper;
3334        debug_assert!(signature.args.len() <= 5);
3335        let mut next = self.next()?;
3336        for (arg_index, arg) in signature.args.iter().enumerate() {
3337            let reg = (arg_index + 1) as u8;
3338            self.reg(reg)?.match_parameter_type(context, self, name, arg, arg_index, &mut next)?
3339        }
3340        // Parameters have been validated, specify the return value on return.
3341        if signature.invalidate_array_bounds {
3342            next.array_bounds.clear();
3343        }
3344        let value =
3345            self.resolve_return_value(context, &signature.return_value, &mut next, false)?;
3346        next.set_reg(0, value)?;
3347        for i in 1..=5 {
3348            next.set_reg(i, Type::default())?;
3349        }
3350        context.states.push(next);
3351        Ok(())
3352    }
3353
3354    fn exit<'a>(&mut self, context: &mut Self::Context<'a>) -> Result<(), String> {
3355        bpf_log!(self, context, "exit");
3356        if !self.reg(0)?.is_written_scalar() {
3357            return Err("register 0 is incorrect at exit time".to_string());
3358        }
3359        if !self.resources.is_empty() {
3360            return Err("some resources have not been released at exit time".to_string());
3361        }
3362        self.terminated = true;
3363        Ok(())
3364    }
3365
3366    fn jump<'a>(&mut self, context: &mut Self::Context<'a>, offset: i16) -> Result<(), String> {
3367        bpf_log!(self, context, "ja {}", offset);
3368        let parent = Some(Arc::new(self.clone()));
3369        context.states.push(self.jump_with_offset(offset, parent)?);
3370        Ok(())
3371    }
3372
3373    fn jeq<'a>(
3374        &mut self,
3375        context: &mut Self::Context<'a>,
3376        dst: Register,
3377        src: Source,
3378        offset: i16,
3379    ) -> Result<(), String> {
3380        self.conditional_jump(
3381            "jeq32",
3382            context,
3383            dst,
3384            src,
3385            offset,
3386            JumpType::Eq,
3387            JumpWidth::W32,
3388            |x, y| {
3389                comp32(x, y, |x, y| {
3390                    // x == y
3391                    if x.min == x.max && x.min == y.min && x.min == y.max {
3392                        return Some(true);
3393                    }
3394                    if x.max < y.min || y.max < x.min {
3395                        return Some(false);
3396                    }
3397                    None
3398                })
3399            },
3400        )
3401    }
3402    fn jeq64<'a>(
3403        &mut self,
3404        context: &mut Self::Context<'a>,
3405        dst: Register,
3406        src: Source,
3407        offset: i16,
3408    ) -> Result<(), String> {
3409        self.conditional_jump(
3410            "jeq",
3411            context,
3412            dst,
3413            src,
3414            offset,
3415            JumpType::Eq,
3416            JumpWidth::W64,
3417            |x, y| {
3418                comp64(x, y, |x, y| {
3419                    // x == y
3420                    if x.min == x.max && x.min == y.min && x.min == y.max {
3421                        return Some(true);
3422                    }
3423                    if x.max < y.min || y.max < x.min {
3424                        return Some(false);
3425                    }
3426                    None
3427                })
3428            },
3429        )
3430    }
3431    fn jne<'a>(
3432        &mut self,
3433        context: &mut Self::Context<'a>,
3434        dst: Register,
3435        src: Source,
3436        offset: i16,
3437    ) -> Result<(), String> {
3438        self.conditional_jump(
3439            "jne32",
3440            context,
3441            dst,
3442            src,
3443            offset,
3444            JumpType::Ne,
3445            JumpWidth::W32,
3446            |x, y| {
3447                comp32(x, y, |x, y| {
3448                    // x != y
3449                    if x.min == x.max && x.min == y.min && x.min == y.max {
3450                        return Some(false);
3451                    }
3452                    if x.max < y.min || y.max < x.min {
3453                        return Some(true);
3454                    }
3455                    None
3456                })
3457            },
3458        )
3459    }
3460    fn jne64<'a>(
3461        &mut self,
3462        context: &mut Self::Context<'a>,
3463        dst: Register,
3464        src: Source,
3465        offset: i16,
3466    ) -> Result<(), String> {
3467        self.conditional_jump(
3468            "jne",
3469            context,
3470            dst,
3471            src,
3472            offset,
3473            JumpType::Ne,
3474            JumpWidth::W64,
3475            |x, y| {
3476                comp64(x, y, |x, y| {
3477                    // x != y
3478                    if x.min == x.max && x.min == y.min && x.min == y.max {
3479                        return Some(false);
3480                    }
3481                    if x.max < y.min || y.max < x.min {
3482                        return Some(true);
3483                    }
3484                    None
3485                })
3486            },
3487        )
3488    }
3489    fn jge<'a>(
3490        &mut self,
3491        context: &mut Self::Context<'a>,
3492        dst: Register,
3493        src: Source,
3494        offset: i16,
3495    ) -> Result<(), String> {
3496        self.conditional_jump(
3497            "jge32",
3498            context,
3499            dst,
3500            src,
3501            offset,
3502            JumpType::Ge,
3503            JumpWidth::W32,
3504            |x, y| {
3505                comp32(x, y, |x, y| {
3506                    // x >= y
3507                    if x.min >= y.max {
3508                        return Some(true);
3509                    }
3510                    if y.min > x.max {
3511                        return Some(false);
3512                    }
3513                    None
3514                })
3515            },
3516        )
3517    }
3518    fn jge64<'a>(
3519        &mut self,
3520        context: &mut Self::Context<'a>,
3521        dst: Register,
3522        src: Source,
3523        offset: i16,
3524    ) -> Result<(), String> {
3525        self.conditional_jump(
3526            "jge",
3527            context,
3528            dst,
3529            src,
3530            offset,
3531            JumpType::Ge,
3532            JumpWidth::W64,
3533            |x, y| {
3534                comp64(x, y, |x, y| {
3535                    // x >= y
3536                    if x.min >= y.max {
3537                        return Some(true);
3538                    }
3539                    if y.min > x.max {
3540                        return Some(false);
3541                    }
3542                    None
3543                })
3544            },
3545        )
3546    }
3547    fn jgt<'a>(
3548        &mut self,
3549        context: &mut Self::Context<'a>,
3550        dst: Register,
3551        src: Source,
3552        offset: i16,
3553    ) -> Result<(), String> {
3554        self.conditional_jump(
3555            "jgt32",
3556            context,
3557            dst,
3558            src,
3559            offset,
3560            JumpType::Gt,
3561            JumpWidth::W32,
3562            |x, y| {
3563                comp32(x, y, |x, y| {
3564                    // x > y
3565                    if x.min > y.max {
3566                        return Some(true);
3567                    }
3568                    if y.min >= x.max {
3569                        return Some(false);
3570                    }
3571                    None
3572                })
3573            },
3574        )
3575    }
3576    fn jgt64<'a>(
3577        &mut self,
3578        context: &mut Self::Context<'a>,
3579        dst: Register,
3580        src: Source,
3581        offset: i16,
3582    ) -> Result<(), String> {
3583        self.conditional_jump(
3584            "jgt",
3585            context,
3586            dst,
3587            src,
3588            offset,
3589            JumpType::Gt,
3590            JumpWidth::W64,
3591            |x, y| {
3592                comp64(x, y, |x, y| {
3593                    // x > y
3594                    if x.min > y.max {
3595                        return Some(true);
3596                    }
3597                    if y.min >= x.max {
3598                        return Some(false);
3599                    }
3600                    None
3601                })
3602            },
3603        )
3604    }
3605    fn jle<'a>(
3606        &mut self,
3607        context: &mut Self::Context<'a>,
3608        dst: Register,
3609        src: Source,
3610        offset: i16,
3611    ) -> Result<(), String> {
3612        self.conditional_jump(
3613            "jle32",
3614            context,
3615            dst,
3616            src,
3617            offset,
3618            JumpType::Le,
3619            JumpWidth::W32,
3620            |x, y| {
3621                comp32(x, y, |x, y| {
3622                    // x <= y
3623                    if x.max <= y.min {
3624                        return Some(true);
3625                    }
3626                    if y.max < x.min {
3627                        return Some(false);
3628                    }
3629                    None
3630                })
3631            },
3632        )
3633    }
3634    fn jle64<'a>(
3635        &mut self,
3636        context: &mut Self::Context<'a>,
3637        dst: Register,
3638        src: Source,
3639        offset: i16,
3640    ) -> Result<(), String> {
3641        self.conditional_jump(
3642            "jle",
3643            context,
3644            dst,
3645            src,
3646            offset,
3647            JumpType::Le,
3648            JumpWidth::W64,
3649            |x, y| {
3650                comp64(x, y, |x, y| {
3651                    // x <= y
3652                    if x.max <= y.min {
3653                        return Some(true);
3654                    }
3655                    if y.max < x.min {
3656                        return Some(false);
3657                    }
3658                    None
3659                })
3660            },
3661        )
3662    }
3663    fn jlt<'a>(
3664        &mut self,
3665        context: &mut Self::Context<'a>,
3666        dst: Register,
3667        src: Source,
3668        offset: i16,
3669    ) -> Result<(), String> {
3670        self.conditional_jump(
3671            "jlt32",
3672            context,
3673            dst,
3674            src,
3675            offset,
3676            JumpType::Lt,
3677            JumpWidth::W32,
3678            |x, y| {
3679                comp32(x, y, |x, y| {
3680                    // x < y
3681                    if x.max < y.min {
3682                        return Some(true);
3683                    }
3684                    if y.max <= x.min {
3685                        return Some(false);
3686                    }
3687                    None
3688                })
3689            },
3690        )
3691    }
3692    fn jlt64<'a>(
3693        &mut self,
3694        context: &mut Self::Context<'a>,
3695        dst: Register,
3696        src: Source,
3697        offset: i16,
3698    ) -> Result<(), String> {
3699        self.conditional_jump(
3700            "jlt",
3701            context,
3702            dst,
3703            src,
3704            offset,
3705            JumpType::Lt,
3706            JumpWidth::W64,
3707            |x, y| {
3708                comp64(x, y, |x, y| {
3709                    // x < y
3710                    if x.max < y.min {
3711                        return Some(true);
3712                    }
3713                    if y.max <= x.min {
3714                        return Some(false);
3715                    }
3716                    None
3717                })
3718            },
3719        )
3720    }
3721    fn jsge<'a>(
3722        &mut self,
3723        context: &mut Self::Context<'a>,
3724        dst: Register,
3725        src: Source,
3726        offset: i16,
3727    ) -> Result<(), String> {
3728        self.conditional_jump(
3729            "jsge32",
3730            context,
3731            dst,
3732            src,
3733            offset,
3734            JumpType::LooseComparaison,
3735            JumpWidth::W32,
3736            |x, y| scomp32(x, y, |x, y| x >= y),
3737        )
3738    }
3739    fn jsge64<'a>(
3740        &mut self,
3741        context: &mut Self::Context<'a>,
3742        dst: Register,
3743        src: Source,
3744        offset: i16,
3745    ) -> Result<(), String> {
3746        self.conditional_jump(
3747            "jsge",
3748            context,
3749            dst,
3750            src,
3751            offset,
3752            JumpType::LooseComparaison,
3753            JumpWidth::W64,
3754            |x, y| scomp64(x, y, |x, y| x >= y),
3755        )
3756    }
3757    fn jsgt<'a>(
3758        &mut self,
3759        context: &mut Self::Context<'a>,
3760        dst: Register,
3761        src: Source,
3762        offset: i16,
3763    ) -> Result<(), String> {
3764        self.conditional_jump(
3765            "jsgt32",
3766            context,
3767            dst,
3768            src,
3769            offset,
3770            JumpType::StrictComparaison,
3771            JumpWidth::W32,
3772            |x, y| scomp32(x, y, |x, y| x > y),
3773        )
3774    }
3775    fn jsgt64<'a>(
3776        &mut self,
3777        context: &mut Self::Context<'a>,
3778        dst: Register,
3779        src: Source,
3780        offset: i16,
3781    ) -> Result<(), String> {
3782        self.conditional_jump(
3783            "jsgt",
3784            context,
3785            dst,
3786            src,
3787            offset,
3788            JumpType::StrictComparaison,
3789            JumpWidth::W64,
3790            |x, y| scomp64(x, y, |x, y| x > y),
3791        )
3792    }
3793    fn jsle<'a>(
3794        &mut self,
3795        context: &mut Self::Context<'a>,
3796        dst: Register,
3797        src: Source,
3798        offset: i16,
3799    ) -> Result<(), String> {
3800        self.conditional_jump(
3801            "jsle32",
3802            context,
3803            dst,
3804            src,
3805            offset,
3806            JumpType::LooseComparaison,
3807            JumpWidth::W32,
3808            |x, y| scomp32(x, y, |x, y| x <= y),
3809        )
3810    }
3811    fn jsle64<'a>(
3812        &mut self,
3813        context: &mut Self::Context<'a>,
3814        dst: Register,
3815        src: Source,
3816        offset: i16,
3817    ) -> Result<(), String> {
3818        self.conditional_jump(
3819            "jsle",
3820            context,
3821            dst,
3822            src,
3823            offset,
3824            JumpType::LooseComparaison,
3825            JumpWidth::W64,
3826            |x, y| scomp64(x, y, |x, y| x <= y),
3827        )
3828    }
3829    fn jslt<'a>(
3830        &mut self,
3831        context: &mut Self::Context<'a>,
3832        dst: Register,
3833        src: Source,
3834        offset: i16,
3835    ) -> Result<(), String> {
3836        self.conditional_jump(
3837            "jslt32",
3838            context,
3839            dst,
3840            src,
3841            offset,
3842            JumpType::StrictComparaison,
3843            JumpWidth::W32,
3844            |x, y| scomp32(x, y, |x, y| x < y),
3845        )
3846    }
3847    fn jslt64<'a>(
3848        &mut self,
3849        context: &mut Self::Context<'a>,
3850        dst: Register,
3851        src: Source,
3852        offset: i16,
3853    ) -> Result<(), String> {
3854        self.conditional_jump(
3855            "jslt",
3856            context,
3857            dst,
3858            src,
3859            offset,
3860            JumpType::StrictComparaison,
3861            JumpWidth::W64,
3862            |x, y| scomp64(x, y, |x, y| x < y),
3863        )
3864    }
3865    fn jset<'a>(
3866        &mut self,
3867        context: &mut Self::Context<'a>,
3868        dst: Register,
3869        src: Source,
3870        offset: i16,
3871    ) -> Result<(), String> {
3872        self.conditional_jump(
3873            "jset32",
3874            context,
3875            dst,
3876            src,
3877            offset,
3878            JumpType::Unknown,
3879            JumpWidth::W32,
3880            |x, y| {
3881                comp32(x, y, |x, y| {
3882                    // x & y != 0
3883                    if x.min != x.max || y.min != y.max {
3884                        return None;
3885                    }
3886                    Some(x.min & y.min != 0)
3887                })
3888            },
3889        )
3890    }
3891    fn jset64<'a>(
3892        &mut self,
3893        context: &mut Self::Context<'a>,
3894        dst: Register,
3895        src: Source,
3896        offset: i16,
3897    ) -> Result<(), String> {
3898        self.conditional_jump(
3899            "jset",
3900            context,
3901            dst,
3902            src,
3903            offset,
3904            JumpType::Unknown,
3905            JumpWidth::W64,
3906            |x, y| {
3907                comp64(x, y, |x, y| {
3908                    // x & y != 0
3909                    if x.min != x.max || y.min != y.max {
3910                        return None;
3911                    }
3912                    Some(x.min & y.min != 0)
3913                })
3914            },
3915        )
3916    }
3917
3918    fn atomic_add<'a>(
3919        &mut self,
3920        context: &mut Self::Context<'a>,
3921        fetch: bool,
3922        dst: Register,
3923        offset: i16,
3924        src: Register,
3925    ) -> Result<(), String> {
3926        self.atomic_operation(
3927            "add32",
3928            context,
3929            DataWidth::U32,
3930            fetch,
3931            dst,
3932            offset,
3933            src,
3934            AluType::Add,
3935            |x, y| alu32(x, y, |x, y| x + y),
3936        )
3937    }
3938
3939    fn atomic_add64<'a>(
3940        &mut self,
3941        context: &mut Self::Context<'a>,
3942        fetch: bool,
3943        dst: Register,
3944        offset: i16,
3945        src: Register,
3946    ) -> Result<(), String> {
3947        self.atomic_operation(
3948            "add",
3949            context,
3950            DataWidth::U64,
3951            fetch,
3952            dst,
3953            offset,
3954            src,
3955            AluType::Add,
3956            |x, y| x + y,
3957        )
3958    }
3959
3960    fn atomic_and<'a>(
3961        &mut self,
3962        context: &mut Self::Context<'a>,
3963        fetch: bool,
3964        dst: Register,
3965        offset: i16,
3966        src: Register,
3967    ) -> Result<(), String> {
3968        self.atomic_operation(
3969            "and32",
3970            context,
3971            DataWidth::U32,
3972            fetch,
3973            dst,
3974            offset,
3975            src,
3976            AluType::Plain,
3977            |x, y| alu32(x, y, |x, y| x & y),
3978        )
3979    }
3980
3981    fn atomic_and64<'a>(
3982        &mut self,
3983        context: &mut Self::Context<'a>,
3984        fetch: bool,
3985        dst: Register,
3986        offset: i16,
3987        src: Register,
3988    ) -> Result<(), String> {
3989        self.atomic_operation(
3990            "and",
3991            context,
3992            DataWidth::U64,
3993            fetch,
3994            dst,
3995            offset,
3996            src,
3997            AluType::Plain,
3998            |x, y| x & y,
3999        )
4000    }
4001
4002    fn atomic_or<'a>(
4003        &mut self,
4004        context: &mut Self::Context<'a>,
4005        fetch: bool,
4006        dst: Register,
4007        offset: i16,
4008        src: Register,
4009    ) -> Result<(), String> {
4010        self.atomic_operation(
4011            "or32",
4012            context,
4013            DataWidth::U32,
4014            fetch,
4015            dst,
4016            offset,
4017            src,
4018            AluType::Plain,
4019            |x, y| alu32(x, y, |x, y| x | y),
4020        )
4021    }
4022
4023    fn atomic_or64<'a>(
4024        &mut self,
4025        context: &mut Self::Context<'a>,
4026        fetch: bool,
4027        dst: Register,
4028        offset: i16,
4029        src: Register,
4030    ) -> Result<(), String> {
4031        self.atomic_operation(
4032            "or",
4033            context,
4034            DataWidth::U64,
4035            fetch,
4036            dst,
4037            offset,
4038            src,
4039            AluType::Plain,
4040            |x, y| x | y,
4041        )
4042    }
4043
4044    fn atomic_xor<'a>(
4045        &mut self,
4046        context: &mut Self::Context<'a>,
4047        fetch: bool,
4048        dst: Register,
4049        offset: i16,
4050        src: Register,
4051    ) -> Result<(), String> {
4052        self.atomic_operation(
4053            "xor32",
4054            context,
4055            DataWidth::U32,
4056            fetch,
4057            dst,
4058            offset,
4059            src,
4060            AluType::Plain,
4061            |x, y| alu32(x, y, |x, y| x ^ y),
4062        )
4063    }
4064
4065    fn atomic_xor64<'a>(
4066        &mut self,
4067        context: &mut Self::Context<'a>,
4068        fetch: bool,
4069        dst: Register,
4070        offset: i16,
4071        src: Register,
4072    ) -> Result<(), String> {
4073        self.atomic_operation(
4074            "xor",
4075            context,
4076            DataWidth::U64,
4077            fetch,
4078            dst,
4079            offset,
4080            src,
4081            AluType::Plain,
4082            |x, y| x ^ y,
4083        )
4084    }
4085
4086    fn atomic_xchg<'a>(
4087        &mut self,
4088        context: &mut Self::Context<'a>,
4089        fetch: bool,
4090        dst: Register,
4091        offset: i16,
4092        src: Register,
4093    ) -> Result<(), String> {
4094        self.atomic_operation(
4095            "xchg32",
4096            context,
4097            DataWidth::U32,
4098            fetch,
4099            dst,
4100            offset,
4101            src,
4102            AluType::Plain,
4103            |_, x| x,
4104        )
4105    }
4106
4107    fn atomic_xchg64<'a>(
4108        &mut self,
4109        context: &mut Self::Context<'a>,
4110        fetch: bool,
4111        dst: Register,
4112        offset: i16,
4113        src: Register,
4114    ) -> Result<(), String> {
4115        self.raw_atomic_operation(
4116            "xchg",
4117            context,
4118            DataWidth::U64,
4119            fetch,
4120            dst,
4121            offset,
4122            src,
4123            |_, _, x| Ok(x),
4124        )
4125    }
4126
4127    fn atomic_cmpxchg<'a>(
4128        &mut self,
4129        context: &mut Self::Context<'a>,
4130        dst: Register,
4131        offset: i16,
4132        src: Register,
4133    ) -> Result<(), String> {
4134        self.raw_atomic_cmpxchg("cmpxchg32", context, dst, offset, src, JumpWidth::W32, |x, y| {
4135            comp32(x, y, |x, y| {
4136                // x == y
4137                if x.min == x.max && x.min == y.min && x.min == y.max {
4138                    return Some(true);
4139                }
4140                if x.max < y.min || y.max < x.min {
4141                    return Some(false);
4142                }
4143                None
4144            })
4145        })
4146    }
4147
4148    fn atomic_cmpxchg64<'a>(
4149        &mut self,
4150        context: &mut Self::Context<'a>,
4151        dst: Register,
4152        offset: i16,
4153        src: Register,
4154    ) -> Result<(), String> {
4155        self.raw_atomic_cmpxchg("cmpxchg", context, dst, offset, src, JumpWidth::W64, |x, y| {
4156            comp64(x, y, |x, y| {
4157                // x == y
4158                if x.min == x.max && x.min == y.min && x.min == y.max {
4159                    return Some(true);
4160                }
4161                if x.max < y.min || y.max < x.min {
4162                    return Some(false);
4163                }
4164                None
4165            })
4166        })
4167    }
4168
4169    fn load<'a>(
4170        &mut self,
4171        context: &mut Self::Context<'a>,
4172        dst: Register,
4173        offset: i16,
4174        src: Register,
4175        width: DataWidth,
4176    ) -> Result<(), String> {
4177        bpf_log!(
4178            self,
4179            context,
4180            "ldx{} {}, [{}{}]",
4181            width.str(),
4182            display_register(dst),
4183            display_register(src),
4184            print_offset(offset)
4185        );
4186        let addr = self.reg(src)?;
4187        let loaded_type = self.load_memory(context, &addr, Field::new(offset, width))?;
4188        let mut next = self.next()?;
4189        next.set_reg(dst, loaded_type)?;
4190        context.states.push(next);
4191        Ok(())
4192    }
4193
4194    fn load64<'a>(
4195        &mut self,
4196        context: &mut Self::Context<'a>,
4197        dst: Register,
4198        src: u8,
4199        lower: u32,
4200    ) -> Result<(), String> {
4201        // The pre-scan has already guaranteed that `pc + 1` is valid and structurally well-formed.
4202        let next_instruction = &context.code[self.pc + 1];
4203
4204        let value = match src {
4205            0 => {
4206                let value = (lower as u64) | (((next_instruction.imm() as u32) as u64) << 32);
4207                bpf_log!(self, context, "lddw {}, 0x{:x}", display_register(dst), value);
4208                Type::from(value)
4209            }
4210            BPF_PSEUDO_MAP_IDX => {
4211                let map_index = lower;
4212                bpf_log!(
4213                    self,
4214                    context,
4215                    "lddw {}, map_by_index({:x})",
4216                    display_register(dst),
4217                    map_index
4218                );
4219                context
4220                    .calling_context
4221                    .maps
4222                    .get(usize::try_from(map_index).unwrap())
4223                    .map(|schema| Type::ConstPtrToMap { id: map_index.into(), schema: *schema })
4224                    .ok_or_else(|| format!("lddw with invalid map index: {}", map_index))?
4225            }
4226            BPF_PSEUDO_MAP_IDX_VALUE => {
4227                let map_index = lower;
4228                let offset = next_instruction.imm();
4229                bpf_log!(
4230                    self,
4231                    context,
4232                    "lddw {}, map_value_by_index({:x})+{offset}",
4233                    display_register(dst),
4234                    map_index
4235                );
4236                let id = context.next_id();
4237                let map_schema = context
4238                    .calling_context
4239                    .maps
4240                    .get(usize::try_from(map_index).unwrap())
4241                    .ok_or_else(|| format!("lddw with invalid map index: {}", map_index))?;
4242
4243                if map_schema.map_type != bpf_map_type_BPF_MAP_TYPE_ARRAY {
4244                    return Err(format!(
4245                        "Invalid map type at index {map_index} for lddw. Expecting array."
4246                    ));
4247                }
4248                if map_schema.max_entries == 0 {
4249                    return Err(format!("Array has no entry."));
4250                }
4251
4252                Type::PtrToMemory {
4253                    id: MemoryId::from(id),
4254                    offset: offset.into(),
4255                    buffer_size: map_schema.value_size.into(),
4256                }
4257            }
4258            _ => {
4259                return Err(format!("invalid lddw"));
4260            }
4261        };
4262
4263        let parent = Some(Arc::new(self.clone()));
4264        let mut next = self.jump_with_offset(1, parent)?;
4265        next.set_reg(dst, value.into())?;
4266
4267        context.states.push(next);
4268        Ok(())
4269    }
4270
4271    fn load_from_packet<'a>(
4272        &mut self,
4273        context: &mut Self::Context<'a>,
4274        dst: Register,
4275        src: Register,
4276        offset: i32,
4277        register_offset: Option<Register>,
4278        width: DataWidth,
4279    ) -> Result<(), String> {
4280        bpf_log!(
4281            self,
4282            context,
4283            "ldp{} {}{}",
4284            width.str(),
4285            register_offset.map(display_register).unwrap_or_else(Default::default),
4286            print_offset(offset)
4287        );
4288
4289        // Verify that `src` refers to a packet.
4290        let src_type = self.reg(src)?;
4291        let src_is_packet = match &context.calling_context.packet_type {
4292            Some(packet_type) => src_type == *packet_type,
4293            None => false,
4294        };
4295        if !src_is_packet {
4296            return Err(format!("R{} is not a packet", src));
4297        }
4298
4299        if let Some(reg) = register_offset {
4300            let reg = self.reg(reg)?;
4301            if !reg.is_written_scalar() {
4302                return Err("access to unwritten offset".to_string());
4303            }
4304        }
4305        // Handle the case where the load succeed.
4306        let mut next = self.next()?;
4307        next.set_reg(dst, Type::UNKNOWN_SCALAR)?;
4308        for i in 1..=5 {
4309            next.set_reg(i, Type::default())?;
4310        }
4311        context.states.push(next);
4312        // Handle the case where the load fails.
4313        if !self.reg(0)?.is_written_scalar() {
4314            return Err("register 0 is incorrect at exit time".to_string());
4315        }
4316        if !self.resources.is_empty() {
4317            return Err("some resources have not been released at exit time".to_string());
4318        }
4319        self.terminated = true;
4320        Ok(())
4321    }
4322
4323    fn store<'a>(
4324        &mut self,
4325        context: &mut Self::Context<'a>,
4326        dst: Register,
4327        offset: i16,
4328        src: Source,
4329        width: DataWidth,
4330    ) -> Result<(), String> {
4331        let value = match src {
4332            Source::Reg(r) => {
4333                bpf_log!(
4334                    self,
4335                    context,
4336                    "stx{} [{}{}], {}",
4337                    width.str(),
4338                    display_register(dst),
4339                    print_offset(offset),
4340                    display_register(r),
4341                );
4342                self.reg(r)?
4343            }
4344            Source::Value(v) => {
4345                bpf_log!(
4346                    self,
4347                    context,
4348                    "st{} [{}{}], 0x{:x}",
4349                    width.str(),
4350                    display_register(dst),
4351                    print_offset(offset),
4352                    v,
4353                );
4354                Type::from(v & Type::mask(width))
4355            }
4356        };
4357        let mut next = self.next()?;
4358        let addr = self.reg(dst)?;
4359        next.store_memory(context, &addr, Field::new(offset, width), value)?;
4360        context.states.push(next);
4361        Ok(())
4362    }
4363}
4364
4365fn alu32(
4366    x: ScalarValueData,
4367    y: ScalarValueData,
4368    op: impl FnOnce(U32ScalarValueData, U32ScalarValueData) -> U32ScalarValueData,
4369) -> ScalarValueData {
4370    op(U32ScalarValueData::from(x), U32ScalarValueData::from(y)).into()
4371}
4372
4373fn comp64(
4374    x: ScalarValueData,
4375    y: ScalarValueData,
4376    op: impl FnOnce(U64Range, U64Range) -> Option<bool>,
4377) -> Result<Option<bool>, ()> {
4378    if !x.is_fully_initialized() || !y.is_fully_initialized() {
4379        return Err(());
4380    }
4381    Ok(op(x.urange, y.urange))
4382}
4383
4384fn comp32(
4385    x: ScalarValueData,
4386    y: ScalarValueData,
4387    op: impl FnOnce(U32Range, U32Range) -> Option<bool>,
4388) -> Result<Option<bool>, ()> {
4389    let x = U32ScalarValueData::from(x);
4390    let y = U32ScalarValueData::from(y);
4391    if !x.is_fully_initialized() || !y.is_fully_initialized() {
4392        return Err(());
4393    }
4394    Ok(op(x.urange, y.urange))
4395}
4396
4397fn scomp64(
4398    x: ScalarValueData,
4399    y: ScalarValueData,
4400    op: impl FnOnce(i64, i64) -> bool,
4401) -> Result<Option<bool>, ()> {
4402    if !x.is_fully_initialized() || !y.is_fully_initialized() {
4403        return Err(());
4404    }
4405    if !x.is_known() || !y.is_known() {
4406        return Ok(None);
4407    }
4408    Ok(Some(op(x.value as i64, y.value as i64)))
4409}
4410
4411fn scomp32(
4412    x: ScalarValueData,
4413    y: ScalarValueData,
4414    op: impl FnOnce(i32, i32) -> bool,
4415) -> Result<Option<bool>, ()> {
4416    let x = U32ScalarValueData::from(x);
4417    let y = U32ScalarValueData::from(y);
4418    if !x.is_fully_initialized() || !y.is_fully_initialized() {
4419        return Err(());
4420    }
4421    if !x.is_known() || !y.is_known() {
4422        return Ok(None);
4423    }
4424    Ok(Some(op(x.value as i32, y.value as i32)))
4425}
4426
4427fn print_offset<T: Into<i32>>(offset: T) -> String {
4428    let offset: i32 = offset.into();
4429    if offset == 0 {
4430        String::new()
4431    } else if offset > 0 {
4432        format!("+{offset}")
4433    } else {
4434        format!("{offset}")
4435    }
4436}
4437
4438fn run_on_stack_offset<F>(v: StackOffset, f: F) -> StackOffset
4439where
4440    F: FnOnce(ScalarValueData) -> ScalarValueData,
4441{
4442    StackOffset(f(v.reg()))
4443}
4444
4445fn error_and_log<T>(
4446    logger: &mut dyn VerifierLogger,
4447    msg: impl std::string::ToString,
4448) -> Result<T, EbpfError> {
4449    let msg = msg.to_string();
4450    logger.log(msg.as_bytes());
4451    return Err(EbpfError::ProgramVerifyError(msg));
4452}
4453
4454fn associate_orderings(o1: Ordering, o2: Ordering) -> Option<Ordering> {
4455    match (o1, o2) {
4456        (o1, o2) if o1 == o2 => Some(o1),
4457        (o, Ordering::Equal) | (Ordering::Equal, o) => Some(o),
4458        _ => None,
4459    }
4460}
4461
4462#[cfg(test)]
4463mod tests {
4464    use super::*;
4465    use std::collections::BTreeSet;
4466    use test_util::{assert_geq, assert_leq};
4467
4468    #[test]
4469    fn test_type_ordering() {
4470        let t0 = Type::from(0);
4471        let t1 = Type::from(1);
4472        let random = Type::AliasParameter { parameter_index: 8 };
4473        let unknown_written = Type::UNKNOWN_SCALAR;
4474        let unwritten = Type::default();
4475
4476        assert_eq!(t0.partial_cmp(&t0), Some(Ordering::Equal));
4477        assert_eq!(t0.partial_cmp(&t1), None);
4478        assert_eq!(t0.partial_cmp(&random), None);
4479        assert_eq!(t0.partial_cmp(&unknown_written), Some(Ordering::Less));
4480        assert_eq!(t0.partial_cmp(&unwritten), Some(Ordering::Less));
4481
4482        assert_eq!(t1.partial_cmp(&t0), None);
4483        assert_eq!(t1.partial_cmp(&t1), Some(Ordering::Equal));
4484        assert_eq!(t1.partial_cmp(&random), None);
4485        assert_eq!(t1.partial_cmp(&unknown_written), Some(Ordering::Less));
4486        assert_eq!(t1.partial_cmp(&unwritten), Some(Ordering::Less));
4487
4488        assert_eq!(random.partial_cmp(&t0), None);
4489        assert_eq!(random.partial_cmp(&t1), None);
4490        assert_eq!(random.partial_cmp(&random), Some(Ordering::Equal));
4491        assert_eq!(random.partial_cmp(&unknown_written), None);
4492        assert_eq!(random.partial_cmp(&unwritten), Some(Ordering::Less));
4493
4494        assert_eq!(unknown_written.partial_cmp(&t0), Some(Ordering::Greater));
4495        assert_eq!(unknown_written.partial_cmp(&t1), Some(Ordering::Greater));
4496        assert_eq!(unknown_written.partial_cmp(&random), None);
4497        assert_eq!(unknown_written.partial_cmp(&unknown_written), Some(Ordering::Equal));
4498        assert_eq!(unknown_written.partial_cmp(&unwritten), Some(Ordering::Less));
4499
4500        assert_eq!(unwritten.partial_cmp(&t0), Some(Ordering::Greater));
4501        assert_eq!(unwritten.partial_cmp(&t1), Some(Ordering::Greater));
4502        assert_eq!(unwritten.partial_cmp(&random), Some(Ordering::Greater));
4503        assert_eq!(unwritten.partial_cmp(&unknown_written), Some(Ordering::Greater));
4504        assert_eq!(unwritten.partial_cmp(&unwritten), Some(Ordering::Equal));
4505    }
4506
4507    #[test]
4508    fn test_stack_ordering() {
4509        let mut s1 = Stack::default();
4510        let mut s2 = Stack::default();
4511
4512        assert_eq!(s1.partial_cmp(&s2), Some(Ordering::Equal));
4513        s1.set(0, 0.into());
4514        assert_eq!(s1.partial_cmp(&s2), Some(Ordering::Less));
4515        assert_eq!(s2.partial_cmp(&s1), Some(Ordering::Greater));
4516        s2.set(1, 1.into());
4517        assert_eq!(s1.partial_cmp(&s2), None);
4518        assert_eq!(s2.partial_cmp(&s1), None);
4519    }
4520
4521    #[test]
4522    fn test_context_ordering() {
4523        let mut c1 = ComputationContext::default();
4524        let mut c2 = ComputationContext::default();
4525
4526        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Equal));
4527
4528        c1.array_bounds.insert(1.into(), 5);
4529        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Less));
4530        assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Greater));
4531
4532        c2.array_bounds.insert(1.into(), 7);
4533        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4534        assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4535
4536        c1.array_bounds.insert(2.into(), 9);
4537        assert_eq!(c1.partial_cmp(&c2), None);
4538        assert_eq!(c2.partial_cmp(&c1), None);
4539
4540        c2.array_bounds.insert(2.into(), 9);
4541        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4542        assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4543
4544        c2.array_bounds.insert(3.into(), 12);
4545        assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4546        assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4547
4548        c1.pc = 8;
4549        assert_eq!(c1.partial_cmp(&c2), None);
4550        assert_eq!(c2.partial_cmp(&c1), None);
4551    }
4552
4553    #[test]
4554    fn test_stack_access() {
4555        let mut s = Stack::default();
4556
4557        // Store data in the range [8, 26) and verify that `read_data_ptr()` fails for any
4558        // reads outside of that range.
4559        assert!(s.store(StackOffset(8.into()), Type::UNKNOWN_SCALAR, DataWidth::U64).is_ok());
4560        assert!(s.store(StackOffset(16.into()), Type::UNKNOWN_SCALAR, DataWidth::U64).is_ok());
4561        assert!(s.store(StackOffset(24.into()), Type::UNKNOWN_SCALAR, DataWidth::U16).is_ok());
4562
4563        for offset in 0..32 {
4564            for end in (offset + 1)..32 {
4565                assert_eq!(
4566                    s.read_data_ptr(2, StackOffset(offset.into()), (end - offset) as u64).is_ok(),
4567                    offset >= 8 && end <= 26
4568                );
4569            }
4570        }
4571
4572        // Verify that overflows are handled properly.
4573        assert!(s.read_data_ptr(2, StackOffset(12.into()), u64::MAX - 2).is_err());
4574    }
4575
4576    #[test]
4577    fn test_compute_range_for_bytes_swap() {
4578        // Build a list of interesting values. Interesting values are all possible combination of
4579        // bytes being either 0, max value, or an intermediary value.
4580        let mut values = BTreeSet::<u64>::default();
4581        for v1 in &[0x00, 0x1, u64::MAX] {
4582            for v2 in &[0x00, 0x1, u64::MAX] {
4583                for v3 in &[0x00, 0x1, u64::MAX] {
4584                    values.insert(U64Range::assemble_slices((*v1, *v2, *v3), 1, 1));
4585                }
4586            }
4587        }
4588        // Replace the second byte of old by the first byte of new and return the result.
4589        let store = |old: u64, new: u64| (old & !0xff00) | ((new & 0xff) << 8);
4590
4591        for old in &values {
4592            for new in &values {
4593                let s = store(*old, *new);
4594                for min_old in values.iter().filter(|v| *v <= old) {
4595                    for min_new in values.iter().filter(|v| *v <= new) {
4596                        for max_old in values.iter().filter(|v| *v >= old) {
4597                            for max_new in values.iter().filter(|v| *v >= new) {
4598                                let range = U64Range::compute_range_for_bytes_swap(
4599                                    U64Range::new(*min_old, *max_old),
4600                                    U64Range::new(*min_new, *max_new),
4601                                    1,
4602                                    0,
4603                                    1,
4604                                );
4605                                assert_leq!(range.min, s);
4606                                assert_geq!(range.max, s);
4607                            }
4608                        }
4609                    }
4610                }
4611            }
4612        }
4613    }
4614
4615    #[test]
4616    fn test_type_constraint_ge_gt() {
4617        let mut context = ComputationContext::default();
4618        let (new_lhs, new_rhs) = Type::constraint(
4619            &mut context,
4620            JumpType::Ge,
4621            JumpWidth::W64,
4622            Type::UNKNOWN_SCALAR,
4623            Type::from(10),
4624        )
4625        .unwrap();
4626
4627        if let Type::ScalarValue(data1) = new_lhs {
4628            assert_eq!(data1.min(), 10);
4629            assert_eq!(data1.max(), u64::MAX);
4630        } else {
4631            panic!("Expected ScalarValue");
4632        }
4633
4634        if let Type::ScalarValue(data2) = new_rhs {
4635            assert_eq!(data2.min(), 10);
4636            assert_eq!(data2.max(), 10);
4637        } else {
4638            panic!("Expected ScalarValue");
4639        }
4640
4641        let (new_lhs, _) = Type::constraint(
4642            &mut context,
4643            JumpType::Gt,
4644            JumpWidth::W64,
4645            Type::UNKNOWN_SCALAR,
4646            Type::from(10),
4647        )
4648        .unwrap();
4649
4650        if let Type::ScalarValue(data1) = new_lhs {
4651            assert_eq!(data1.min(), 11);
4652            assert_eq!(data1.max(), u64::MAX);
4653        } else {
4654            panic!("Expected ScalarValue");
4655        }
4656    }
4657
4658    #[test]
4659    fn test_type_constraint_lt_le() {
4660        let mut context = ComputationContext::default();
4661        let (new_lhs, _) = Type::constraint(
4662            &mut context,
4663            JumpType::Lt,
4664            JumpWidth::W64,
4665            Type::UNKNOWN_SCALAR,
4666            Type::from(10),
4667        )
4668        .unwrap();
4669
4670        if let Type::ScalarValue(data1) = new_lhs {
4671            assert_eq!(data1.min(), 0);
4672            assert_eq!(data1.max(), 9);
4673        } else {
4674            panic!("Expected ScalarValue");
4675        }
4676
4677        let (new_lhs, _) = Type::constraint(
4678            &mut context,
4679            JumpType::Le,
4680            JumpWidth::W64,
4681            Type::UNKNOWN_SCALAR,
4682            Type::from(10),
4683        )
4684        .unwrap();
4685
4686        if let Type::ScalarValue(data1) = new_lhs {
4687            assert_eq!(data1.min(), 0);
4688            assert_eq!(data1.max(), 10);
4689        } else {
4690            panic!("Expected ScalarValue");
4691        }
4692    }
4693
4694    #[test]
4695    fn test_type_constraint_w32() {
4696        let mut context = ComputationContext::default();
4697        let (new_lhs, _) = Type::constraint(
4698            &mut context,
4699            JumpType::Eq,
4700            JumpWidth::W32,
4701            Type::ScalarValue(ScalarValueData::UNKNOWN_WRITTEN),
4702            Type::from(10),
4703        )
4704        .unwrap();
4705        if let Type::ScalarValue(data1) = new_lhs {
4706            assert_eq!(data1.min(), 10);
4707            assert_eq!(data1.max(), 0xffff_ffff_0000_000a);
4708        } else {
4709            panic!("Expected ScalarValue");
4710        }
4711    }
4712
4713    #[test]
4714    fn test_type_constraint_w32_straddling_min() {
4715        // A 32-bit equality only constrains the low 32 bits. When the comparison
4716        // partner's range straddles 2^32 (min <= U32_MAX < max), its low 32 bits
4717        // can be smaller than its minimum, so the other register's minimum must
4718        // not be raised. `data1` is in [0, 0x40]; `data2` is in [0x30, u64::MAX].
4719        // On the equal branch `data1` can still be 0 (e.g. when `data2 == 2^32`,
4720        // whose low 32 bits are 0), so its minimum must remain 0.
4721        let mut context = ComputationContext::default();
4722        let data1 = ScalarValueData::new(0, u64::MAX, 0, U64Range::new(0, 0x40));
4723        let data2 = ScalarValueData::new(0, u64::MAX, 0, U64Range::new(0x30, u64::MAX));
4724        let (new_lhs, _) = Type::constraint(
4725            &mut context,
4726            JumpType::Eq,
4727            JumpWidth::W32,
4728            Type::ScalarValue(data1),
4729            Type::ScalarValue(data2),
4730        )
4731        .unwrap();
4732        if let Type::ScalarValue(data) = new_lhs {
4733            assert_eq!(data.min(), 0);
4734            assert_eq!(data.max(), 0x40);
4735        } else {
4736            panic!("Expected ScalarValue");
4737        }
4738    }
4739
4740    #[test]
4741    fn test_type_constraint_null_or() {
4742        let mut context = ComputationContext::default();
4743        let id = MemoryId::new();
4744        let null_or = Type::NullOr { id: id.clone(), inner: Box::new(Type::UNKNOWN_SCALAR) };
4745        let (new_lhs, _) =
4746            Type::constraint(&mut context, JumpType::Eq, JumpWidth::W64, null_or, Type::from(0))
4747                .unwrap();
4748        if let Type::ScalarValue(data) = new_lhs {
4749            assert_eq!(data.value, 0);
4750        } else {
4751            panic!("Expected zero ScalarValue");
4752        }
4753    }
4754
4755    #[test]
4756    fn test_type_constraint_array_bounds() {
4757        let mut context = ComputationContext::default();
4758        let id = MemoryId::new();
4759        let _ = Type::constraint(
4760            &mut context,
4761            JumpType::Le,
4762            JumpWidth::W64,
4763            Type::PtrToArray { id: id.clone(), offset: 22.into() },
4764            Type::PtrToEndArray { id: id.clone() },
4765        )
4766        .unwrap();
4767        assert_eq!(context.array_bounds.get(&id), Some(&22));
4768    }
4769}