1#![allow(unused_variables)]
6
7use crate::scalar_value::{ScalarValueData, U32Range, U32ScalarValueData, U64Range};
8use crate::visitor::{BpfVisitor, ProgramCounter, Register, Source};
9use crate::{
10 BPF_LDDW, BPF_MAX_INSTS, BPF_PSEUDO_MAP_IDX, BPF_PSEUDO_MAP_IDX_VALUE, BPF_STACK_SIZE,
11 DataWidth, EbpfError, EbpfInstruction, GENERAL_REGISTER_COUNT, MapSchema, REGISTER_COUNT,
12};
13use byteorder::{BigEndian, ByteOrder, LittleEndian, NativeEndian};
14use fuchsia_sync::Mutex;
15use linux_uapi::{bpf_map_type, bpf_map_type_BPF_MAP_TYPE_ARRAY};
16use std::cmp::Ordering;
17use std::collections::{BTreeMap, HashMap, HashSet};
18use std::sync::Arc;
19use zerocopy::IntoBytes;
20
21const U32_MAX: u64 = u32::MAX as u64;
22
23pub trait VerifierLogger {
25 fn log(&mut self, line: &[u8]);
27}
28
29pub struct NullVerifierLogger;
31
32impl VerifierLogger for NullVerifierLogger {
33 fn log(&mut self, line: &[u8]) {
34 debug_assert!(line.is_ascii());
35 }
36}
37
38#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Ord, PartialOrd)]
44pub enum Namespace {
45 Hardcoded,
47 Generated,
49 Verification,
51}
52
53#[derive(Clone, Debug, PartialEq, Eq, Hash, Ord, PartialOrd)]
54pub struct MemoryId {
55 namespace: Namespace,
56 id: u64,
57 parent: Option<Box<MemoryId>>,
58}
59
60impl MemoryId {
61 pub fn id(&self) -> u64 {
62 self.id
63 }
64}
65
66impl From<u64> for MemoryId {
67 fn from(id: u64) -> Self {
68 Self { namespace: Namespace::Hardcoded, id, parent: None }
69 }
70}
71
72static BPF_TYPE_IDENTIFIER_COUNTER: std::sync::atomic::AtomicU64 =
76 std::sync::atomic::AtomicU64::new(0);
77
78impl MemoryId {
79 pub const fn from_raw(id: u64) -> MemoryId {
84 Self { namespace: Namespace::Hardcoded, id, parent: None }
85 }
86
87 pub fn new() -> MemoryId {
88 Self {
89 namespace: Namespace::Generated,
90 id: BPF_TYPE_IDENTIFIER_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
91 parent: None,
92 }
93 }
94
95 fn prepended(&self, other: MemoryId) -> Self {
97 match &self.parent {
98 None => {
99 MemoryId { namespace: self.namespace, id: self.id, parent: Some(Box::new(other)) }
100 }
101 Some(parent) => MemoryId {
102 namespace: self.namespace,
103 id: self.id,
104 parent: Some(Box::new(parent.prepended(other))),
105 },
106 }
107 }
108
109 fn has_parent(&self, parent: &MemoryId) -> bool {
111 match &self.parent {
112 None => false,
113 Some(p) => p.as_ref() == parent,
114 }
115 }
116
117 fn matches(&self, other: &MemoryId) -> bool {
120 if self.namespace != other.namespace || self.id != other.id {
121 return false;
122 };
123 match (&self.parent, &other.parent) {
124 (Some(p1), Some(p2)) => p1.matches(p2.as_ref()),
125 _ => true,
126 }
127 }
128}
129
130#[derive(Clone, Debug, PartialEq)]
132pub enum FieldType {
133 Scalar { size: usize },
135
136 MutableScalar { size: usize },
138
139 PtrToMemory { is_32_bit: bool, id: MemoryId, buffer_size: usize },
141
142 NullablePtrToMemory { is_32_bit: bool, id: MemoryId, buffer_size: usize },
144
145 PtrToArray { is_32_bit: bool, id: MemoryId },
148
149 PtrToEndArray { is_32_bit: bool, id: MemoryId },
152}
153
154#[derive(Clone, Debug, PartialEq)]
156pub struct FieldDescriptor {
157 pub offset: usize,
159
160 pub field_type: FieldType,
163}
164
165impl FieldDescriptor {
166 fn size(&self) -> usize {
167 match self.field_type {
168 FieldType::Scalar { size } | FieldType::MutableScalar { size } => size,
169 FieldType::PtrToMemory { is_32_bit, .. }
170 | FieldType::NullablePtrToMemory { is_32_bit, .. }
171 | FieldType::PtrToArray { is_32_bit, .. }
172 | FieldType::PtrToEndArray { is_32_bit, .. } => {
173 if is_32_bit {
174 4
175 } else {
176 8
177 }
178 }
179 }
180 }
181}
182
183#[derive(Clone, Copy, Debug, Eq, PartialEq)]
185struct Field {
186 offset: i16,
187 width: DataWidth,
188}
189
190impl Field {
191 fn new(offset: i16, width: DataWidth) -> Self {
192 Self { offset, width }
193 }
194}
195
196#[derive(Debug, PartialEq, Default)]
198pub struct StructDescriptor {
199 pub fields: Vec<FieldDescriptor>,
201}
202
203impl StructDescriptor {
204 fn is_subtype(&self, super_struct: &StructDescriptor) -> bool {
206 for super_field in super_struct.fields.iter() {
208 if self.fields.iter().find(|field| *field == super_field).is_none() {
209 return false;
210 }
211 }
212 true
213 }
214
215 fn find_field(&self, base_offset: ScalarValueData, field: Field) -> Option<&FieldDescriptor> {
218 let offset = base_offset + field.offset;
219 let field_desc = self.fields.iter().find(|f| {
220 f.offset <= offset.min() as usize && (offset.max() as usize) < f.offset + f.size()
221 })?;
222 let is_valid_load = match field_desc.field_type {
223 FieldType::Scalar { size } | FieldType::MutableScalar { size } => {
224 ((offset + field.width.bytes()).max() as usize) <= field_desc.offset + size
226 }
227 FieldType::PtrToMemory { is_32_bit, .. }
228 | FieldType::NullablePtrToMemory { is_32_bit, .. }
229 | FieldType::PtrToArray { is_32_bit, .. }
230 | FieldType::PtrToEndArray { is_32_bit, .. } => {
231 let expected_width = if is_32_bit { DataWidth::U32 } else { DataWidth::U64 };
232 offset.is_known()
234 && offset.value as usize == field_desc.offset
235 && field.width == expected_width
236 }
237 };
238
239 is_valid_load.then_some(field_desc)
240 }
241}
242
243#[derive(Clone, Debug, PartialEq)]
244pub enum MemoryParameterSize {
245 Value(u64),
247 Reference { index: u8 },
249}
250
251impl MemoryParameterSize {
252 fn size(&self, context: &ComputationContext) -> Result<u64, String> {
253 match self {
254 Self::Value(size) => Ok(*size),
255 Self::Reference { index } => {
256 let size_type = context.reg(index + 1)?;
257 match size_type {
258 Type::ScalarValue(data) if data.is_known() => Ok(data.value),
259 _ => Err("cannot know buffer size".to_string()),
260 }
261 }
262 }
263 }
264}
265
266#[derive(Clone, Debug, PartialEq)]
267pub enum MapTypeFilter {
268 AllowList(&'static [bpf_map_type]),
269 DenyList(&'static [bpf_map_type]),
270}
271
272impl MapTypeFilter {
273 pub fn is_allowed(&self, map_type: bpf_map_type) -> bool {
274 match self {
275 MapTypeFilter::AllowList(types) => types.contains(&map_type),
276 MapTypeFilter::DenyList(types) => !types.contains(&map_type),
277 }
278 }
279}
280
281#[derive(Clone, Debug, PartialEq)]
282pub enum Type {
283 ScalarValue(ScalarValueData),
285 ConstPtrToMap { id: u64, schema: MapSchema },
287 PtrToStack { offset: StackOffset },
289 PtrToMemory { id: MemoryId, offset: ScalarValueData, buffer_size: u64 },
292 PtrToStruct { id: MemoryId, offset: ScalarValueData, descriptor: Arc<StructDescriptor> },
294 PtrToArray { id: MemoryId, offset: ScalarValueData },
298 PtrToEndArray { id: MemoryId },
301 NullOr { id: MemoryId, inner: Box<Type> },
303 Releasable { id: MemoryId, inner: Box<Type> },
306 ScalarValueParameter,
308 ConstPtrToMapParameter { filter: MapTypeFilter },
310 MapKeyParameter {
312 map_ptr_index: u8,
315 },
316 MapValueParameter {
318 map_ptr_index: u8,
321 },
322 MemoryParameter {
324 size: MemoryParameterSize,
327 input: bool,
329 output: bool,
331 },
332 AliasParameter {
334 parameter_index: u8,
336 },
337 NullOrParameter(Box<Type>),
339 StructParameter { id: MemoryId },
341 ContextParameter { parameter_index: u8 },
343 ReleasableParameter { id: MemoryId, inner: Box<Type> },
346 ReleaseParameter { id: MemoryId },
348 AnyParameter,
350}
351
352impl PartialOrd for Type {
366 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
367 if self == other {
369 return Some(Ordering::Equal);
370 }
371
372 if self == &Type::UNINITIALIZED {
379 return Some(Ordering::Greater);
380 }
381 if other == &Type::UNINITIALIZED {
382 return Some(Ordering::Less);
383 }
384
385 match (self, other) {
387 (Self::ScalarValue(data1), Self::ScalarValue(data2)) => data1.partial_cmp(data2),
388 _ => None,
389 }
390 }
391}
392
393impl From<ScalarValueData> for Type {
394 fn from(value: ScalarValueData) -> Self {
395 Self::ScalarValue(value)
396 }
397}
398
399impl From<u64> for Type {
400 fn from(value: u64) -> Self {
401 Self::ScalarValue(value.into())
402 }
403}
404
405impl Default for Type {
406 fn default() -> Self {
408 Self::UNINITIALIZED.clone()
409 }
410}
411
412impl Type {
413 pub const UNINITIALIZED: Self = Self::ScalarValue(ScalarValueData::UNINITIALIZED);
415
416 pub const UNKNOWN_SCALAR: Self = Self::ScalarValue(ScalarValueData::UNKNOWN_WRITTEN);
418
419 fn mask(width: DataWidth) -> u64 {
421 if width == DataWidth::U64 { u64::MAX } else { (1 << width.bits()) - 1 }
422 }
423
424 fn is_written_scalar(&self) -> bool {
426 match self {
427 Self::ScalarValue(data) if data.is_fully_initialized() => true,
428 _ => false,
429 }
430 }
431
432 fn is_initialized(&self) -> bool {
434 match self {
435 Self::ScalarValue(data) => data.is_fully_initialized(),
436 _ => true,
437 }
438 }
439
440 pub fn is_subtype(&self, super_type: &Type) -> bool {
442 match (self, super_type) {
443 (_, Self::ScalarValue(data)) if data.is_uninitialized() => true,
445
446 (
447 Self::PtrToStruct { id: id1, offset: offset1, descriptor: descriptor1 },
448 Self::PtrToStruct { id: id2, offset: offset2, descriptor: descriptor2 },
449 ) => id1 == id2 && offset1 <= offset2 && descriptor1.is_subtype(descriptor2),
450
451 (self_type, super_type) if self_type == super_type => true,
453
454 _ => false,
455 }
456 }
457
458 pub fn is_non_zero(&self) -> bool {
460 match self {
461 Self::ScalarValue(d) => d.min() > 0,
462 Self::NullOr { .. } => false,
463 _ => true,
464 }
465 }
466
467 fn inner(&self, context: &ComputationContext) -> Result<&Type, String> {
468 match self {
469 Self::Releasable { id, inner } => {
470 if context.resources.contains(id) {
471 Ok(&inner)
472 } else {
473 Err("Access to released resource".to_string())
474 }
475 }
476 _ => Ok(self),
477 }
478 }
479
480 fn constraint(
483 context: &mut ComputationContext,
484 jump_type: JumpType,
485 jump_width: JumpWidth,
486 type1: Self,
487 type2: Self,
488 ) -> Result<(Self, Self), String> {
489 let result = match (jump_width, jump_type, type1.inner(context)?, type2.inner(context)?) {
490 (JumpWidth::W64, JumpType::Eq, Type::ScalarValue(data1), Type::ScalarValue(data2))
491 if data1.is_fully_initialized() && data2.is_fully_initialized() =>
492 {
493 let umin = std::cmp::max(data1.min(), data2.min());
494 let umax = std::cmp::min(data1.max(), data2.max());
495 let v = Type::ScalarValue(ScalarValueData::new(
496 data1.value | data2.value,
497 data1.unknown_mask & data2.unknown_mask,
498 0,
499 U64Range::new(umin, umax),
500 ));
501 (v.clone(), v)
502 }
503 (JumpWidth::W32, JumpType::Eq, Type::ScalarValue(data1), Type::ScalarValue(data2))
504 if data1.is_fully_initialized() && data2.is_fully_initialized() =>
505 {
506 let maybe_umin = if data1.min() <= U32_MAX && data2.min() <= U32_MAX {
507 Some(std::cmp::max(data1.min(), data2.min()))
508 } else {
509 None
510 };
511 let maybe_umax = if data1.max() <= U32_MAX && data2.max() <= U32_MAX {
512 Some(std::cmp::min(data1.max(), data2.max()))
513 } else {
514 None
515 };
516
517 let urange1 = U64Range::new(
518 maybe_umin.unwrap_or_else(|| data1.min()),
519 maybe_umax.unwrap_or_else(|| data1.max()),
520 );
521 let v1 = Type::ScalarValue(ScalarValueData::new(
522 data1.value | (data2.value & U32_MAX),
523 data1.unknown_mask & (data2.unknown_mask | (U32_MAX << 32)),
524 0,
525 urange1,
526 ));
527
528 let urange2 = U64Range::new(
529 maybe_umin.unwrap_or_else(|| data2.min()),
530 maybe_umax.unwrap_or_else(|| data2.max()),
531 );
532 let v2 = Type::ScalarValue(ScalarValueData::new(
533 data2.value | (data1.value & U32_MAX),
534 data2.unknown_mask & (data1.unknown_mask | (U32_MAX << 32)),
535 0,
536 urange2,
537 ));
538 (v1, v2)
539 }
540 (JumpWidth::W64, JumpType::Eq, Type::ScalarValue(data), Type::NullOr { id, .. })
541 | (JumpWidth::W64, JumpType::Eq, Type::NullOr { id, .. }, Type::ScalarValue(data))
542 if data.is_zero() =>
543 {
544 context.set_null(id, true);
545 let zero = Type::from(0);
546 (zero.clone(), zero)
547 }
548 (JumpWidth::W64, jump_type, Type::NullOr { id, inner }, Type::ScalarValue(data))
549 if jump_type.is_strict() && data.is_zero() =>
550 {
551 context.set_null(id, false);
552 let inner = *inner.clone();
553 inner.register_resource(context);
554 (inner, type2)
555 }
556 (JumpWidth::W64, jump_type, Type::ScalarValue(data), Type::NullOr { id, inner })
557 if jump_type.is_strict() && data.is_zero() =>
558 {
559 context.set_null(id, false);
560 let inner = *inner.clone();
561 inner.register_resource(context);
562 (type1, inner)
563 }
564
565 (JumpWidth::W64, JumpType::Lt, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
566 debug_assert!(lhs.min() < u64::MAX);
567 debug_assert!(rhs.max() > 0);
568 let new_max_lhs = std::cmp::min(lhs.max(), rhs.max() - 1);
569 debug_assert!(lhs.min() <= new_max_lhs);
570 let new_min_rhs = std::cmp::max(rhs.min(), lhs.min() + 1);
571 debug_assert!(new_min_rhs <= rhs.max());
572 let new_range_lhs = U64Range::new(lhs.min(), new_max_lhs);
573 let new_range_rhs = U64Range::new(new_min_rhs, rhs.max());
574 (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
575 }
576 (JumpWidth::W64, JumpType::Gt, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
577 debug_assert!(rhs.min() < u64::MAX);
578 debug_assert!(lhs.max() > 0);
579 let new_min_lhs = std::cmp::max(lhs.min(), rhs.min() + 1);
580 debug_assert!(new_min_lhs <= lhs.max());
581 let new_max_rhs = std::cmp::min(rhs.max(), lhs.max() - 1);
582 debug_assert!(rhs.min() <= new_max_rhs);
583 let new_range_lhs = U64Range::new(new_min_lhs, lhs.max());
584 let new_range_rhs = U64Range::new(rhs.min(), new_max_rhs);
585 (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
586 }
587
588 (JumpWidth::W64, JumpType::Le, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
589 let new_max_lhs = std::cmp::min(lhs.max(), rhs.max());
590 debug_assert!(lhs.min() <= new_max_lhs);
591 let new_min_rhs = std::cmp::max(rhs.min(), lhs.min());
592 debug_assert!(new_min_rhs <= rhs.max());
593 let new_range_lhs = U64Range::new(lhs.min(), new_max_lhs);
594 let new_range_rhs = U64Range::new(new_min_rhs, rhs.max());
595 (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
596 }
597 (JumpWidth::W64, JumpType::Ge, Type::ScalarValue(lhs), Type::ScalarValue(rhs)) => {
598 let new_min_lhs = std::cmp::max(lhs.min(), rhs.min());
599 debug_assert!(new_min_lhs <= lhs.max());
600 let new_max_rhs = std::cmp::min(rhs.max(), lhs.max());
601 debug_assert!(rhs.min() <= new_max_rhs);
602 let new_range_lhs = U64Range::new(new_min_lhs, lhs.max());
603 let new_range_rhs = U64Range::new(rhs.min(), new_max_rhs);
604 (lhs.update_range(new_range_lhs).into(), rhs.update_range(new_range_rhs).into())
605 }
606
607 (
608 JumpWidth::W64,
609 JumpType::Eq,
610 Type::PtrToArray { id: id1, offset },
611 Type::PtrToEndArray { id: id2 },
612 )
613 | (
614 JumpWidth::W64,
615 JumpType::Le,
616 Type::PtrToArray { id: id1, offset },
617 Type::PtrToEndArray { id: id2 },
618 )
619 | (
620 JumpWidth::W64,
621 JumpType::Ge,
622 Type::PtrToEndArray { id: id1 },
623 Type::PtrToArray { id: id2, offset },
624 ) if id1 == id2 => {
625 context.update_array_bounds(id1.clone(), *offset);
626 (type1, type2)
627 }
628 (
629 JumpWidth::W64,
630 JumpType::Lt,
631 Type::PtrToArray { id: id1, offset },
632 Type::PtrToEndArray { id: id2 },
633 )
634 | (
635 JumpWidth::W64,
636 JumpType::Gt,
637 Type::PtrToEndArray { id: id1 },
638 Type::PtrToArray { id: id2, offset },
639 ) if id1 == id2 => {
640 context.update_array_bounds(id1.clone(), *offset + 1);
641 (type1, type2)
642 }
643 (JumpWidth::W64, JumpType::Eq, _, _) => (type1.clone(), type1),
644 _ => (type1, type2),
645 };
646 Ok(result)
647 }
648
649 fn match_parameter_type(
650 &self,
651 verification_context: &VerificationContext<'_>,
652 context: &ComputationContext,
653 helper_name: &str,
654 parameter_type: &Type,
655 index: usize,
656 next: &mut ComputationContext,
657 ) -> Result<(), String> {
658 match (parameter_type, self) {
659 (Type::NullOrParameter(t), Type::ScalarValue(data))
660 if data.is_known() && data.value == 0 =>
661 {
662 Ok(())
663 }
664 (Type::NullOrParameter(t), _) => self.match_parameter_type(
665 verification_context,
666 context,
667 helper_name,
668 t,
669 index,
670 next,
671 ),
672 (Type::ScalarValueParameter, Type::ScalarValue(data))
673 if data.is_fully_initialized() =>
674 {
675 Ok(())
676 }
677 (Type::ConstPtrToMapParameter { filter }, Type::ConstPtrToMap { schema, .. }) => {
678 let map_type = schema.map_type;
679 if !filter.is_allowed(map_type) {
680 return Err(format!("Map type {map_type} not allowed in {helper_name}"));
681 }
682 Ok(())
683 }
684 (
685 Type::MapKeyParameter { map_ptr_index },
686 Type::PtrToMemory { offset, buffer_size, .. },
687 ) => {
688 let schema = context.get_map_schema(*map_ptr_index)?;
689 context.check_memory_access(*offset, *buffer_size, 0, schema.key_size as usize)
690 }
691 (Type::MapKeyParameter { map_ptr_index }, Type::PtrToStack { offset }) => {
692 let schema = context.get_map_schema(*map_ptr_index)?;
693 context.stack.read_data_ptr(context.pc, *offset, schema.key_size as u64)
694 }
695 (
696 Type::MapValueParameter { map_ptr_index },
697 Type::PtrToMemory { offset, buffer_size, .. },
698 ) => {
699 let schema = context.get_map_schema(*map_ptr_index)?;
700 context.check_memory_access(*offset, *buffer_size, 0, schema.value_size as usize)
701 }
702 (Type::MapValueParameter { map_ptr_index }, Type::PtrToStack { offset }) => {
703 let schema = context.get_map_schema(*map_ptr_index)?;
704 context.stack.read_data_ptr(context.pc, *offset, schema.value_size as u64)
705 }
706 (Type::MemoryParameter { size, .. }, Type::PtrToMemory { offset, buffer_size, .. }) => {
707 let expected_size = size.size(context)?;
708 let offset_max = offset.max();
709 if offset_max > *buffer_size {
710 return Err("out of bound read".to_string());
711 }
712 let size_left = *buffer_size - offset_max;
713 if expected_size > size_left {
714 return Err("out of bound read".to_string());
715 }
716 Ok(())
717 }
718
719 (Type::MemoryParameter { size, input, output }, Type::PtrToStack { offset }) => {
720 let size = size.size(context)?;
721 let buffer_end = offset.add(size);
722 if !buffer_end.is_within_stack() {
723 Err("out of bound access".to_string())
724 } else {
725 if *output {
726 next.stack.write_data_ptr(context.pc, *offset, size)?;
727 }
728 if *input {
729 context.stack.read_data_ptr(context.pc, *offset, size)?;
730 }
731 Ok(())
732 }
733 }
734 (
735 Type::StructParameter { id: id1 },
736 Type::PtrToMemory { id: id2, offset, .. }
737 | Type::PtrToStruct { id: id2, offset, .. },
738 ) if offset.is_zero() && id1.matches(id2) => Ok(()),
739 (
740 Type::ReleasableParameter { id: id1, inner: inner1 },
741 Type::Releasable { id: id2, inner: inner2 },
742 ) if id2.has_parent(id1) => {
743 if next.resources.contains(id2) {
744 inner2.match_parameter_type(
745 verification_context,
746 context,
747 helper_name,
748 inner1,
749 index,
750 next,
751 )
752 } else {
753 Err(format!("Resource already released for index {index}"))
754 }
755 }
756 (Type::ContextParameter { parameter_index }, arg_type) => {
757 if verification_context.calling_context.args.get(*parameter_index as usize)
758 == Some(arg_type)
759 {
760 Ok(())
761 } else {
762 Err(format!("Helper expects program argument {parameter_index}"))
763 }
764 }
765 (Type::ReleaseParameter { id: id1 }, Type::Releasable { id: id2, .. })
766 if id2.has_parent(id1) =>
767 {
768 if next.resources.remove(id2) {
769 Ok(())
770 } else {
771 Err(format!("{id2:?} Resource already released for index {index}"))
772 }
773 }
774 (_, Type::Releasable { inner, .. }) => inner.match_parameter_type(
775 verification_context,
776 context,
777 helper_name,
778 parameter_type,
779 index,
780 next,
781 ),
782 (Type::AnyParameter, _) => Ok(()),
783
784 _ => Err(format!("incorrect parameter for index {index}")),
785 }
786 }
787
788 fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
791 match self {
792 Type::NullOr { id, inner } if id == null_id => {
793 if is_null {
794 *self = Type::from(0);
795 } else {
796 *self = *inner.clone();
797 }
798 }
799 _ => {}
800 }
801 }
802
803 fn register_resource(&self, context: &mut ComputationContext) {
804 match self {
805 Type::Releasable { id, .. } => {
806 context.resources.insert(id.clone());
807 }
808 _ => {}
809 }
810 }
811
812 fn compare_list<'a>(
820 mut l1: impl Iterator<Item = &'a Self>,
821 mut l2: impl Iterator<Item = &'a Self>,
822 ) -> Option<Ordering> {
823 let mut result = Ordering::Equal;
824 loop {
825 match (l1.next(), l2.next()) {
826 (None, None) => return Some(result),
827 (_, None) | (None, _) => return None,
828 (Some(v1), Some(v2)) => {
829 result = associate_orderings(result, v1.partial_cmp(v2)?)?;
830 }
831 }
832 }
833 }
834}
835
836#[derive(Clone, Debug)]
837pub struct FunctionSignature {
838 pub args: Vec<Type>,
839 pub return_value: Type,
840 pub invalidate_array_bounds: bool,
841}
842
843#[derive(Clone, Debug)]
844pub struct HelperDefinition {
845 pub index: u32,
846 pub name: &'static str,
847 pub signature: FunctionSignature,
848}
849
850#[derive(Debug, Default)]
851pub struct CallingContext {
852 pub maps: Vec<MapSchema>,
855 pub helpers: HashMap<u32, &'static HelperDefinition>,
857 pub args: Vec<Type>,
859 pub packet_type: Option<Type>,
861}
862
863impl CallingContext {
864 pub fn register_map(&mut self, schema: MapSchema) -> usize {
865 let index = self.maps.len();
866 self.maps.push(schema);
867 index
868 }
869}
870
871#[derive(Debug, PartialEq, Clone)]
872pub struct StructAccess {
873 pub pc: ProgramCounter,
874
875 pub memory_id: MemoryId,
877
878 pub field_offset: usize,
880
881 pub is_32_bit_ptr_load: bool,
883}
884
885#[derive(Debug, Clone)]
886pub struct VerifiedEbpfProgram {
887 pub(crate) code: Vec<EbpfInstruction>,
888 pub(crate) args: Vec<Type>,
889 pub(crate) struct_access_instructions: Vec<StructAccess>,
890 pub(crate) maps: Vec<MapSchema>,
891}
892
893impl VerifiedEbpfProgram {
894 pub fn to_code(self) -> Vec<EbpfInstruction> {
897 debug_assert!(self.struct_access_instructions.is_empty());
898 debug_assert!(self.maps.is_empty());
899 self.code
900 }
901
902 pub fn code(&self) -> &[EbpfInstruction] {
903 &self.code
904 }
905
906 pub fn struct_access_instructions(&self) -> &[StructAccess] {
907 &self.struct_access_instructions
908 }
909
910 pub fn from_verified_code(
911 code: Vec<EbpfInstruction>,
912 args: Vec<Type>,
913 struct_access_instructions: Vec<StructAccess>,
914 maps: Vec<MapSchema>,
915 ) -> Self {
916 Self { code, args, struct_access_instructions, maps }
917 }
918
919 pub fn maps(&self) -> &[MapSchema] {
920 &self.maps
921 }
922}
923
924pub fn verify_program(
927 code: Vec<EbpfInstruction>,
928 calling_context: CallingContext,
929 logger: &mut dyn VerifierLogger,
930) -> Result<VerifiedEbpfProgram, EbpfError> {
931 if code.len() > BPF_MAX_INSTS {
932 return error_and_log(logger, "ebpf program too long");
933 }
934 let mut scan_pc = 0;
942 while scan_pc < code.len() {
943 let inst = &code[scan_pc];
944 if inst.code() == BPF_LDDW {
945 let Some(next_instruction) = code.get(scan_pc + 1) else {
946 return error_and_log(logger, "incomplete lddw");
947 };
948 if next_instruction.code() != 0
949 || next_instruction.offset() != 0
950 || next_instruction.src_reg() != 0
951 || next_instruction.dst_reg() != 0
952 {
953 return error_and_log(logger, "invalid lddw");
954 }
955 scan_pc += 2;
956 } else {
957 scan_pc += 1;
958 }
959 }
960
961 let mut context = ComputationContext::default();
962 for (i, t) in calling_context.args.iter().enumerate() {
963 context.set_reg((i + 1) as u8, t.clone()).map_err(EbpfError::ProgramVerifyError)?;
965 }
966 let states = vec![context];
967 let mut verification_context = VerificationContext {
968 calling_context,
969 logger,
970 states,
971 code: &code,
972 counter: 0,
973 iteration: 0,
974 terminating_contexts: Default::default(),
975 struct_access_instructions: Default::default(),
976 };
977 while let Some(mut context) = verification_context.states.pop() {
978 if let Some(terminating_contexts) =
979 verification_context.terminating_contexts.get(&context.pc)
980 {
981 if let Some(ending_context) =
984 terminating_contexts.iter().find(|c| c.computation_context >= context)
985 {
986 if let Some(parent) = context.parent.take() {
990 parent.dependencies.lock().push(ending_context.dependencies.clone());
991 if let Some(parent) = Arc::into_inner(parent) {
992 parent
993 .terminate(&mut verification_context)
994 .map_err(EbpfError::ProgramVerifyError)?;
995 }
996 }
997 continue;
998 }
999 }
1000 if verification_context.iteration > 10 * BPF_MAX_INSTS {
1001 return error_and_log(verification_context.logger, "bpf byte code does not terminate");
1002 }
1003 if context.pc >= code.len() {
1004 return error_and_log(verification_context.logger, "pc out of bounds");
1005 }
1006 if let Err(message) = context.visit(&mut verification_context, code[context.pc]) {
1007 let message = format!("at PC {}: {}", context.pc, message);
1008 return error_and_log(verification_context.logger, message);
1009 }
1010 if context.terminated {
1011 context.terminate(&mut verification_context).map_err(EbpfError::ProgramVerifyError)?;
1012 }
1013 verification_context.iteration += 1;
1014 }
1015
1016 let struct_access_instructions =
1017 verification_context.struct_access_instructions.into_values().collect::<Vec<_>>();
1018 let CallingContext { maps, args, .. } = verification_context.calling_context;
1019 Ok(VerifiedEbpfProgram { code, struct_access_instructions, maps, args })
1020}
1021
1022struct VerificationContext<'a> {
1023 calling_context: CallingContext,
1025 logger: &'a mut dyn VerifierLogger,
1027 states: Vec<ComputationContext>,
1029 code: &'a [EbpfInstruction],
1031 counter: u64,
1033 iteration: usize,
1036 terminating_contexts: BTreeMap<ProgramCounter, Vec<TerminatingContext>>,
1040 struct_access_instructions: HashMap<ProgramCounter, StructAccess>,
1044}
1045
1046impl<'a> VerificationContext<'a> {
1047 fn next_id(&mut self) -> MemoryId {
1048 let id = self.counter;
1049 self.counter += 1;
1050 MemoryId { namespace: Namespace::Verification, id, parent: None }
1051 }
1052
1053 fn register_struct_access(&mut self, struct_access: StructAccess) -> Result<(), String> {
1056 match self.struct_access_instructions.entry(struct_access.pc) {
1057 std::collections::hash_map::Entry::Vacant(entry) => {
1058 entry.insert(struct_access);
1059 }
1060 std::collections::hash_map::Entry::Occupied(entry) => {
1061 if *entry.get() != struct_access {
1062 return Err("Inconsistent struct field access".to_string());
1063 }
1064 }
1065 }
1066 Ok(())
1067 }
1068}
1069
1070const STACK_ELEMENT_SIZE: usize = std::mem::size_of::<u64>();
1071
1072#[derive(Clone, Copy, Debug, PartialEq)]
1075pub struct StackOffset(ScalarValueData);
1076
1077impl Default for StackOffset {
1078 fn default() -> Self {
1079 Self(BPF_STACK_SIZE.into())
1080 }
1081}
1082
1083impl StackOffset {
1084 fn is_valid_offset(&self) -> bool {
1086 self.0.is_known() && self.0.value < (BPF_STACK_SIZE as u64)
1087 }
1088
1089 fn is_within_stack(&self) -> bool {
1091 self.0.is_known() && self.0.value <= (BPF_STACK_SIZE as u64)
1092 }
1093
1094 fn reg(&self) -> ScalarValueData {
1096 self.0
1097 }
1098
1099 fn array_index(&self) -> usize {
1102 debug_assert!(self.is_within_stack());
1103 usize::try_from(self.0.value).unwrap() / STACK_ELEMENT_SIZE
1104 }
1105
1106 fn sub_index(&self) -> usize {
1108 debug_assert!(self.is_within_stack());
1109 usize::try_from(self.0.value).unwrap() % STACK_ELEMENT_SIZE
1110 }
1111
1112 fn add<T: Into<ScalarValueData>>(self, rhs: T) -> Self {
1113 Self(self.0 + rhs)
1114 }
1115}
1116
1117#[derive(Clone, Debug, Default, PartialEq)]
1119struct Stack {
1120 data: HashMap<usize, Type>,
1121}
1122
1123impl Stack {
1124 fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
1127 for (_, t) in self.data.iter_mut() {
1128 t.set_null(null_id, is_null);
1129 }
1130 }
1131
1132 fn get(&self, index: usize) -> &Type {
1133 self.data.get(&index).unwrap_or(&Type::UNINITIALIZED)
1134 }
1135
1136 fn set(&mut self, index: usize, t: Type) {
1137 if t == Type::UNINITIALIZED {
1138 self.data.remove(&index);
1139 } else {
1140 self.data.insert(index, t);
1141 }
1142 }
1143
1144 fn extract_sub_value(value: u64, offset: usize, byte_count: usize) -> u64 {
1145 NativeEndian::read_uint(&value.as_bytes()[offset..], byte_count)
1146 }
1147
1148 fn insert_sub_value(mut original: u64, value: u64, width: DataWidth, offset: usize) -> u64 {
1149 let byte_count = width.bytes();
1150 let original_buf = original.as_mut_bytes();
1151 let value_buf = value.as_bytes();
1152 original_buf[offset..(byte_count + offset)].copy_from_slice(&value_buf[..byte_count]);
1153 original
1154 }
1155
1156 fn write_data_ptr(
1157 &mut self,
1158 pc: ProgramCounter,
1159 mut offset: StackOffset,
1160 bytes: u64,
1161 ) -> Result<(), String> {
1162 for i in 0..bytes {
1163 self.store(offset, Type::UNKNOWN_SCALAR, DataWidth::U8)?;
1164 offset = offset.add(1);
1165 }
1166 Ok(())
1167 }
1168
1169 fn read_data_ptr(
1170 &self,
1171 pc: ProgramCounter,
1172 offset: StackOffset,
1173 bytes: u64,
1174 ) -> Result<(), String> {
1175 let read_element =
1176 |index: usize, start_offset: usize, end_offset: usize| -> Result<(), String> {
1177 match self.get(index) {
1178 Type::ScalarValue(data) => {
1179 debug_assert!(end_offset > start_offset);
1180 let unwritten_bits = Self::extract_sub_value(
1181 data.unwritten_mask,
1182 start_offset,
1183 end_offset - start_offset,
1184 );
1185 if unwritten_bits == 0 {
1186 Ok(())
1187 } else {
1188 Err("reading unwritten value from the stack".to_string())
1189 }
1190 }
1191 _ => Err("invalid read from the stack".to_string()),
1192 }
1193 };
1194 if bytes == 0 {
1195 return Ok(());
1196 }
1197
1198 if bytes as usize > BPF_STACK_SIZE {
1199 return Err("stack overflow".to_string());
1200 }
1201
1202 if !offset.is_valid_offset() {
1203 return Err("invalid stack offset".to_string());
1204 }
1205
1206 let end_offset = offset.add(bytes);
1207 if !end_offset.is_within_stack() {
1208 return Err("stack overflow".to_string())?;
1209 }
1210
1211 if offset.array_index() == end_offset.array_index() {
1215 return read_element(offset.array_index(), offset.sub_index(), end_offset.sub_index());
1216 }
1217
1218 read_element(offset.array_index(), offset.sub_index(), STACK_ELEMENT_SIZE)?;
1220
1221 if end_offset.sub_index() != 0 {
1223 read_element(end_offset.array_index(), 0, end_offset.sub_index())?;
1224 }
1225
1226 for i in (offset.array_index() + 1)..end_offset.array_index() {
1228 read_element(i, 0, STACK_ELEMENT_SIZE)?;
1229 }
1230
1231 Ok(())
1232 }
1233
1234 fn store(&mut self, offset: StackOffset, value: Type, width: DataWidth) -> Result<(), String> {
1235 if !offset.is_valid_offset() {
1236 return Err("out of bounds store".to_string());
1237 }
1238 if offset.sub_index() % width.bytes() != 0 {
1239 return Err("misaligned access".to_string());
1240 }
1241
1242 let index = offset.array_index();
1243 if width == DataWidth::U64 {
1244 self.set(index, value);
1245 } else {
1246 match value {
1247 Type::ScalarValue(data) => {
1248 let old_data = match self.get(index) {
1249 Type::ScalarValue(data) => *data,
1250 _ => {
1251 ScalarValueData::UNINITIALIZED
1254 }
1255 };
1256 let sub_index = offset.sub_index();
1257 let value =
1258 Self::insert_sub_value(old_data.value, data.value, width, sub_index);
1259 let unknown_mask = Self::insert_sub_value(
1260 old_data.unknown_mask,
1261 data.unknown_mask,
1262 width,
1263 sub_index,
1264 );
1265 let unwritten_mask = Self::insert_sub_value(
1266 old_data.unwritten_mask,
1267 data.unwritten_mask,
1268 width,
1269 sub_index,
1270 );
1271 let urange = U64Range::compute_range_for_bytes_swap(
1272 old_data.urange,
1273 data.urange,
1274 sub_index,
1275 0,
1276 width.bytes(),
1277 );
1278 self.set(
1279 index,
1280 Type::ScalarValue(ScalarValueData::new(
1281 value,
1282 unknown_mask,
1283 unwritten_mask,
1284 urange,
1285 )),
1286 );
1287 }
1288 _ => {
1289 return Err("cannot store part of a non scalar value on the stack".to_string());
1290 }
1291 }
1292 }
1293 Ok(())
1294 }
1295
1296 fn load(&self, offset: StackOffset, width: DataWidth) -> Result<Type, String> {
1297 if !offset.is_valid_offset() {
1298 return Err("out of bounds load".to_string());
1299 }
1300 if offset.sub_index() % width.bytes() != 0 {
1301 return Err("misaligned access".to_string());
1302 }
1303
1304 let index = offset.array_index();
1305 let loaded_type = self.get(index).clone();
1306 let result = if width == DataWidth::U64 {
1307 loaded_type
1308 } else {
1309 match loaded_type {
1310 Type::ScalarValue(data) => {
1311 let sub_index = offset.sub_index();
1312 let value = Self::extract_sub_value(data.value, sub_index, width.bytes());
1313 let unknown_mask =
1314 Self::extract_sub_value(data.unknown_mask, sub_index, width.bytes());
1315 let unwritten_mask =
1316 Self::extract_sub_value(data.unwritten_mask, sub_index, width.bytes());
1317 let urange = U64Range::compute_range_for_bytes_swap(
1318 0.into(),
1319 data.urange,
1320 0,
1321 sub_index,
1322 width.bytes(),
1323 );
1324 Type::ScalarValue(ScalarValueData::new(
1325 value,
1326 unknown_mask,
1327 unwritten_mask,
1328 urange,
1329 ))
1330 }
1331 _ => return Err(format!("incorrect load of {} bytes", width.bytes())),
1332 }
1333 };
1334 if !result.is_initialized() {
1335 return Err("reading unwritten value from the stack".to_string());
1336 }
1337 Ok(result)
1338 }
1339}
1340
1341impl PartialOrd for Stack {
1344 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
1345 let mut result = Ordering::Equal;
1346 let mut data_iter1 = self.data.iter().peekable();
1347 let mut data_iter2 = other.data.iter().peekable();
1348 loop {
1349 let k1 = data_iter1.peek().map(|(k, _)| *k);
1350 let k2 = data_iter2.peek().map(|(k, _)| *k);
1351 let k = match (k1, k2) {
1352 (None, None) => return Some(result),
1353 (Some(k), None) => {
1354 data_iter1.next();
1355 *k
1356 }
1357 (None, Some(k)) => {
1358 data_iter2.next();
1359 *k
1360 }
1361 (Some(k1), Some(k2)) => {
1362 if k1 <= k2 {
1363 data_iter1.next();
1364 }
1365 if k2 <= k1 {
1366 data_iter2.next();
1367 }
1368 *std::cmp::min(k1, k2)
1369 }
1370 };
1371 result = associate_orderings(result, self.get(k).partial_cmp(other.get(k))?)?;
1372 }
1373 }
1374}
1375
1376macro_rules! bpf_log {
1377 ($context:ident, $verification_context:ident, $($msg:tt)*) => {
1378 let prefix = format!("{}: ({:02x})", $context.pc, $verification_context.code[$context.pc].code());
1379 let suffix = format!($($msg)*);
1380 $verification_context.logger.log(format!("{prefix} {suffix}").as_bytes());
1381 }
1382}
1383
1384#[derive(Debug, Default)]
1386struct ComputationContext {
1387 pc: ProgramCounter,
1389 registers: [Type; GENERAL_REGISTER_COUNT as usize],
1391 stack: Stack,
1393 array_bounds: BTreeMap<MemoryId, u64>,
1395 resources: HashSet<MemoryId>,
1397 parent: Option<Arc<ComputationContext>>,
1399 dependencies: Mutex<Vec<DataDependencies>>,
1402 terminated: bool,
1406}
1407
1408impl Clone for ComputationContext {
1409 fn clone(&self) -> Self {
1410 Self {
1411 pc: self.pc,
1412 registers: self.registers.clone(),
1413 stack: self.stack.clone(),
1414 array_bounds: self.array_bounds.clone(),
1415 resources: self.resources.clone(),
1416 parent: self.parent.clone(),
1417 dependencies: Default::default(),
1420 terminated: false,
1421 }
1422 }
1423}
1424
1425impl PartialEq for ComputationContext {
1427 fn eq(&self, other: &Self) -> bool {
1428 self.pc == other.pc
1429 && self.registers == other.registers
1430 && self.stack == other.stack
1431 && self.array_bounds == other.array_bounds
1432 }
1433}
1434
1435impl ComputationContext {
1436 fn set_null(&mut self, null_id: &MemoryId, is_null: bool) {
1439 for i in 0..self.registers.len() {
1440 self.registers[i].set_null(null_id, is_null);
1441 }
1442 self.stack.set_null(null_id, is_null);
1443 }
1444
1445 fn reg(&self, index: Register) -> Result<Type, String> {
1446 if index >= REGISTER_COUNT {
1447 return Err(format!("R{index} is invalid"));
1448 }
1449 if index < GENERAL_REGISTER_COUNT {
1450 Ok(self.registers[index as usize].clone())
1451 } else {
1452 Ok(Type::PtrToStack { offset: StackOffset::default() })
1453 }
1454 }
1455
1456 fn set_reg(&mut self, index: Register, reg_type: Type) -> Result<(), String> {
1457 if index >= GENERAL_REGISTER_COUNT {
1458 return Err(format!("R{index} is invalid"));
1459 }
1460 self.registers[index as usize] = reg_type;
1461 Ok(())
1462 }
1463
1464 fn update_array_bounds(&mut self, id: MemoryId, new_bound: ScalarValueData) {
1465 let new_bound_min = new_bound.min();
1466 self.array_bounds
1467 .entry(id)
1468 .and_modify(|v| *v = std::cmp::max(*v, new_bound_min))
1469 .or_insert(new_bound_min);
1470 }
1471
1472 fn get_map_schema(&self, argument: u8) -> Result<MapSchema, String> {
1473 match self.reg(argument + 1)? {
1474 Type::ConstPtrToMap { schema, .. } => Ok(schema),
1475 _ => Err(format!("No map found at argument {argument}")),
1476 }
1477 }
1478
1479 fn next(&self) -> Result<Self, String> {
1480 let parent = Some(Arc::new(self.clone()));
1481 self.jump_with_offset(0, parent)
1482 }
1483
1484 fn jump_with_offset(&self, offset: i16, parent: Option<Arc<Self>>) -> Result<Self, String> {
1487 let pc = self
1488 .pc
1489 .checked_add_signed(offset.into())
1490 .and_then(|v| v.checked_add_signed(1))
1491 .ok_or_else(|| "jump outside of program".to_string())?;
1492 let result = Self {
1493 pc,
1494 registers: self.registers.clone(),
1495 stack: self.stack.clone(),
1496 array_bounds: self.array_bounds.clone(),
1497 resources: self.resources.clone(),
1498 parent,
1499 dependencies: Default::default(),
1500 terminated: false,
1501 };
1502 Ok(result)
1503 }
1504
1505 fn check_memory_access(
1506 &self,
1507 dst_offset: ScalarValueData,
1508 dst_buffer_size: u64,
1509 instruction_offset: i16,
1510 width: usize,
1511 ) -> Result<(), String> {
1512 let memory_range = dst_offset.urange + instruction_offset + U64Range::new(0, width as u64);
1513 if memory_range.max > dst_buffer_size {
1514 return Err("out of bound access".to_string());
1515 }
1516 Ok(())
1517 }
1518
1519 fn store_memory(
1520 &mut self,
1521 context: &mut VerificationContext<'_>,
1522 addr: &Type,
1523 field: Field,
1524 value: Type,
1525 ) -> Result<(), String> {
1526 let addr = addr.inner(self)?;
1527 match *addr {
1528 Type::PtrToStack { offset } => {
1529 let offset_sum = offset.add(field.offset);
1530 return self.stack.store(offset_sum, value, field.width);
1531 }
1532 Type::PtrToMemory { offset, buffer_size, .. } => {
1533 self.check_memory_access(offset, buffer_size, field.offset, field.width.bytes())?;
1534 }
1535 Type::PtrToStruct { ref id, offset, ref descriptor, .. } => {
1536 let field_desc = descriptor
1537 .find_field(offset, field)
1538 .ok_or_else(|| "incorrect store".to_string())?;
1539
1540 if !matches!(field_desc.field_type, FieldType::MutableScalar { .. }) {
1541 return Err("store to a read-only field".to_string());
1542 }
1543
1544 context.register_struct_access(StructAccess {
1545 pc: self.pc,
1546 memory_id: id.clone(),
1547 field_offset: field_desc.offset,
1548 is_32_bit_ptr_load: false,
1549 })?;
1550 }
1551 Type::PtrToArray { ref id, offset } => {
1552 self.check_memory_access(
1553 offset,
1554 *self.array_bounds.get(&id).unwrap_or(&0),
1555 field.offset,
1556 field.width.bytes(),
1557 )?;
1558 }
1559 _ => return Err("incorrect store".to_string()),
1560 }
1561
1562 match value {
1563 Type::ScalarValue(data) if data.is_fully_initialized() => {}
1564 _ => return Err("incorrect store".to_string()),
1566 }
1567 Ok(())
1568 }
1569
1570 fn load_memory(
1571 &self,
1572 context: &mut VerificationContext<'_>,
1573 addr: &Type,
1574 field: Field,
1575 ) -> Result<Type, String> {
1576 let addr = addr.inner(self)?;
1577 match *addr {
1578 Type::PtrToStack { offset } => {
1579 let offset_sum = offset.add(field.offset);
1580 self.stack.load(offset_sum, field.width)
1581 }
1582 Type::PtrToMemory { ref id, offset, buffer_size, .. } => {
1583 self.check_memory_access(offset, buffer_size, field.offset, field.width.bytes())?;
1584 Ok(Type::UNKNOWN_SCALAR)
1585 }
1586 Type::PtrToStruct { ref id, offset, ref descriptor, .. } => {
1587 let field_desc = descriptor
1588 .find_field(offset, field)
1589 .ok_or_else(|| "incorrect load".to_string())?;
1590
1591 let (return_type, is_32_bit_ptr_load) = match &field_desc.field_type {
1592 FieldType::Scalar { .. } | FieldType::MutableScalar { .. } => {
1593 (Type::UNKNOWN_SCALAR, false)
1594 }
1595 FieldType::PtrToArray { id: array_id, is_32_bit } => (
1596 Type::PtrToArray { id: array_id.prepended(id.clone()), offset: 0.into() },
1597 *is_32_bit,
1598 ),
1599 FieldType::PtrToEndArray { id: array_id, is_32_bit } => {
1600 (Type::PtrToEndArray { id: array_id.prepended(id.clone()) }, *is_32_bit)
1601 }
1602 FieldType::PtrToMemory { id: memory_id, buffer_size, is_32_bit } => (
1603 Type::PtrToMemory {
1604 id: memory_id.prepended(id.clone()),
1605 offset: 0.into(),
1606 buffer_size: *buffer_size as u64,
1607 },
1608 *is_32_bit,
1609 ),
1610 FieldType::NullablePtrToMemory { id: memory_id, buffer_size, is_32_bit } => {
1611 let id = memory_id.prepended(id.clone());
1612 (
1613 Type::NullOr {
1614 id: id.clone(),
1615 inner: Box::new(Type::PtrToMemory {
1616 id,
1617 offset: 0.into(),
1618 buffer_size: *buffer_size as u64,
1619 }),
1620 },
1621 *is_32_bit,
1622 )
1623 }
1624 };
1625
1626 context.register_struct_access(StructAccess {
1627 pc: self.pc,
1628 memory_id: id.clone(),
1629 field_offset: field_desc.offset,
1630 is_32_bit_ptr_load,
1631 })?;
1632
1633 Ok(return_type)
1634 }
1635 Type::PtrToArray { ref id, offset } => {
1636 self.check_memory_access(
1637 offset,
1638 *self.array_bounds.get(&id).unwrap_or(&0),
1639 field.offset,
1640 field.width.bytes(),
1641 )?;
1642 Ok(Type::UNKNOWN_SCALAR)
1643 }
1644 _ => Err("incorrect load".to_string()),
1645 }
1646 }
1647
1648 fn resolve_return_value(
1655 &self,
1656 verification_context: &mut VerificationContext<'_>,
1657 return_value: &Type,
1658 next: &mut ComputationContext,
1659 maybe_null: bool,
1660 ) -> Result<Type, String> {
1661 match return_value {
1662 Type::AliasParameter { parameter_index } => self.reg(parameter_index + 1),
1663 Type::ReleasableParameter { id, inner } => {
1664 let id = verification_context.next_id().prepended(id.clone());
1665 if !maybe_null {
1666 next.resources.insert(id.clone());
1667 }
1668 Ok(Type::Releasable {
1669 id,
1670 inner: Box::new(self.resolve_return_value(
1671 verification_context,
1672 inner,
1673 next,
1674 maybe_null,
1675 )?),
1676 })
1677 }
1678 Type::NullOrParameter(t) => {
1679 let id = verification_context.next_id();
1680 Ok(Type::NullOr {
1681 id,
1682 inner: Box::new(self.resolve_return_value(
1683 verification_context,
1684 t,
1685 next,
1686 true,
1687 )?),
1688 })
1689 }
1690 Type::MapValueParameter { map_ptr_index } => {
1691 let schema = self.get_map_schema(*map_ptr_index)?;
1692 let id = verification_context.next_id();
1693 Ok(Type::PtrToMemory {
1694 id,
1695 offset: 0.into(),
1696 buffer_size: schema.value_size as u64,
1697 })
1698 }
1699 Type::MemoryParameter { size, .. } => {
1700 let buffer_size = size.size(self)?;
1701 let id = verification_context.next_id();
1702 Ok(Type::PtrToMemory { id, offset: 0.into(), buffer_size })
1703 }
1704 t => Ok(t.clone()),
1705 }
1706 }
1707
1708 fn compute_source(&self, src: Source) -> Result<Type, String> {
1709 match src {
1710 Source::Reg(reg) => self.reg(reg),
1711 Source::Value(v) => Ok(v.into()),
1712 }
1713 }
1714
1715 fn apply_computation(
1716 context: &ComputationContext,
1717 op1: Type,
1718 op2: Type,
1719 alu_type: AluType,
1720 op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1721 ) -> Result<Type, String> {
1722 let result: Type = match (alu_type, op1.inner(context)?, op2.inner(context)?) {
1723 (_, Type::ScalarValue(data1), Type::ScalarValue(data2)) => op(*data1, *data2).into(),
1724 (
1725 AluType::Add,
1726 Type::ScalarValue(_),
1727 Type::PtrToStack { .. } | Type::PtrToMemory { .. } | Type::PtrToStruct { .. },
1728 ) => {
1729 return Self::apply_computation(context, op2, op1, alu_type, op);
1730 }
1731 (alu_type, Type::PtrToStack { offset: x }, Type::ScalarValue(data))
1732 if alu_type.is_ptr_compatible() =>
1733 {
1734 Type::PtrToStack { offset: run_on_stack_offset(*x, |x| op(x, *data)) }
1735 }
1736 (
1737 alu_type,
1738 Type::PtrToMemory { id, offset: x, buffer_size },
1739 Type::ScalarValue(data),
1740 ) if alu_type.is_ptr_compatible() => {
1741 let offset = op(*x, *data);
1742 Type::PtrToMemory { id: id.clone(), offset, buffer_size: *buffer_size }
1743 }
1744 (
1745 alu_type,
1746 Type::PtrToStruct { id, offset: x, descriptor },
1747 Type::ScalarValue(data),
1748 ) if alu_type.is_ptr_compatible() => {
1749 let offset = op(*x, *data);
1750 Type::PtrToStruct { id: id.clone(), offset, descriptor: descriptor.clone() }
1751 }
1752 (AluType::Add, Type::PtrToArray { id, offset: x }, Type::ScalarValue(data)) => {
1753 let offset = x.checked_add(*data).ok_or_else(|| format!("XXX"))?;
1754 Type::PtrToArray { id: id.clone(), offset }
1755 }
1756 (AluType::Sub, Type::PtrToArray { id, offset: x }, Type::ScalarValue(data)) => {
1757 let offset = x.checked_sub(*data).ok_or_else(|| format!("XXX"))?;
1758 Type::PtrToArray { id: id.clone(), offset }
1759 }
1760 (
1761 AluType::Sub,
1762 Type::PtrToMemory { id: id1, offset: x1, .. },
1763 Type::PtrToMemory { id: id2, offset: x2, .. },
1764 )
1765 | (
1766 AluType::Sub,
1767 Type::PtrToStruct { id: id1, offset: x1, .. },
1768 Type::PtrToStruct { id: id2, offset: x2, .. },
1769 )
1770 | (
1771 AluType::Sub,
1772 Type::PtrToArray { id: id1, offset: x1 },
1773 Type::PtrToArray { id: id2, offset: x2 },
1774 ) if id1 == id2 => Type::from(op(*x1, *x2)),
1775 (AluType::Sub, Type::PtrToStack { offset: x1 }, Type::PtrToStack { offset: x2 }) => {
1776 Type::from(op(x1.reg(), x2.reg()))
1777 }
1778 (
1779 AluType::Sub,
1780 Type::PtrToArray { id: id1, .. },
1781 Type::PtrToEndArray { id: id2, .. },
1782 )
1783 | (
1784 AluType::Sub,
1785 Type::PtrToEndArray { id: id1, .. },
1786 Type::PtrToArray { id: id2, .. },
1787 ) if id1 == id2 => Type::UNKNOWN_SCALAR,
1788 _ => Type::default(),
1789 };
1790 Ok(result)
1791 }
1792
1793 fn alu(
1794 &mut self,
1795 op_name: Option<&str>,
1796 verification_context: &mut VerificationContext<'_>,
1797 dst: Register,
1798 src: Source,
1799 alu_type: AluType,
1800 op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1801 ) -> Result<(), String> {
1802 if let Some(op_name) = op_name {
1803 bpf_log!(
1804 self,
1805 verification_context,
1806 "{op_name} {}, {}",
1807 display_register(dst),
1808 display_source(src)
1809 );
1810 }
1811 let op1 = self.reg(dst)?;
1812 let op2 = self.compute_source(src)?;
1813 let result = Self::apply_computation(self, op1, op2, alu_type, op)?;
1814 let mut next = self.next()?;
1815 next.set_reg(dst, result)?;
1816 verification_context.states.push(next);
1817 Ok(())
1818 }
1819
1820 fn log_atomic_operation(
1821 &mut self,
1822 op_name: &str,
1823 verification_context: &mut VerificationContext<'_>,
1824 fetch: bool,
1825 dst: Register,
1826 offset: i16,
1827 src: Register,
1828 ) {
1829 bpf_log!(
1830 self,
1831 verification_context,
1832 "lock {}{} [{}{}], {}",
1833 if fetch { "fetch " } else { "" },
1834 op_name,
1835 display_register(dst),
1836 print_offset(offset),
1837 display_register(src),
1838 );
1839 }
1840
1841 fn raw_atomic_operation(
1842 &mut self,
1843 op_name: &str,
1844 verification_context: &mut VerificationContext<'_>,
1845 width: DataWidth,
1846 fetch: bool,
1847 dst: Register,
1848 offset: i16,
1849 src: Register,
1850 op: impl FnOnce(&ComputationContext, Type, Type) -> Result<Type, String>,
1851 ) -> Result<(), String> {
1852 self.log_atomic_operation(op_name, verification_context, fetch, dst, offset, src);
1853 let addr = self.reg(dst)?;
1854 let value = self.reg(src)?;
1855 let field = Field::new(offset, width);
1856 let loaded_type = self.load_memory(verification_context, &addr, field)?;
1857 let result = op(self, loaded_type.clone(), value)?;
1858 let mut next = self.next()?;
1859 next.store_memory(verification_context, &addr, field, result)?;
1860 if fetch {
1861 next.set_reg(src, loaded_type)?;
1862 }
1863 verification_context.states.push(next);
1864 Ok(())
1865 }
1866
1867 fn atomic_operation(
1868 &mut self,
1869 op_name: &str,
1870 verification_context: &mut VerificationContext<'_>,
1871 width: DataWidth,
1872 fetch: bool,
1873 dst: Register,
1874 offset: i16,
1875 src: Register,
1876 alu_type: AluType,
1877 op: impl Fn(ScalarValueData, ScalarValueData) -> ScalarValueData,
1878 ) -> Result<(), String> {
1879 self.raw_atomic_operation(
1880 op_name,
1881 verification_context,
1882 width,
1883 fetch,
1884 dst,
1885 offset,
1886 src,
1887 |context: &ComputationContext, v1: Type, v2: Type| {
1888 Self::apply_computation(context, v1, v2, alu_type, op)
1889 },
1890 )
1891 }
1892
1893 fn raw_atomic_cmpxchg(
1894 &mut self,
1895 op_name: &str,
1896 verification_context: &mut VerificationContext<'_>,
1897 dst: Register,
1898 offset: i16,
1899 src: Register,
1900 jump_width: JumpWidth,
1901 op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
1902 ) -> Result<(), String> {
1903 self.log_atomic_operation(op_name, verification_context, true, dst, offset, src);
1904 let width = match jump_width {
1905 JumpWidth::W32 => DataWidth::U32,
1906 JumpWidth::W64 => DataWidth::U64,
1907 };
1908 let addr = self.reg(dst)?;
1909 let field = Field::new(offset, width);
1910 let dst = self.load_memory(verification_context, &addr, field)?;
1911 let value = self.reg(src)?;
1912 let r0 = self.reg(0)?;
1913 let branch = self.compute_branch(jump_width, &dst, &r0, op)?;
1914 if branch.unwrap_or(true) {
1916 let mut next = self.next()?;
1917 let (dst, r0) =
1918 Type::constraint(&mut next, JumpType::Eq, jump_width, dst.clone(), r0.clone())?;
1919 next.set_reg(0, dst)?;
1920 next.store_memory(verification_context, &addr, field, value)?;
1921 verification_context.states.push(next);
1922 }
1923 if !branch.unwrap_or(false) {
1925 let mut next = self.next()?;
1926 let (dst, r0) = Type::constraint(&mut next, JumpType::Ne, jump_width, dst, r0)?;
1927 next.set_reg(0, dst.clone())?;
1928 next.store_memory(verification_context, &addr, field, dst)?;
1929 verification_context.states.push(next);
1930 }
1931
1932 Ok(())
1933 }
1934 fn endianness<BO: ByteOrder>(
1935 &mut self,
1936 op_name: &str,
1937 verification_context: &mut VerificationContext<'_>,
1938 dst: Register,
1939 width: DataWidth,
1940 ) -> Result<(), String> {
1941 bpf_log!(self, verification_context, "{op_name}{} {}", width.bits(), display_register(dst),);
1942 let bit_op = |value: u64| match width {
1943 DataWidth::U16 => BO::read_u16((value as u16).as_bytes()) as u64,
1944 DataWidth::U32 => BO::read_u32((value as u32).as_bytes()) as u64,
1945 DataWidth::U64 => BO::read_u64(value.as_bytes()),
1946 _ => {
1947 panic!("Unexpected bit width for endianness operation");
1948 }
1949 };
1950 let value = self.reg(dst)?;
1951 let new_value = match value {
1952 Type::ScalarValue(data) => Type::ScalarValue(ScalarValueData::new(
1953 bit_op(data.value),
1954 bit_op(data.unknown_mask),
1955 bit_op(data.unwritten_mask),
1956 U64Range::max(),
1957 )),
1958 _ => Type::default(),
1959 };
1960 let mut next = self.next()?;
1961 next.set_reg(dst, new_value)?;
1962 verification_context.states.push(next);
1963 Ok(())
1964 }
1965
1966 fn compute_branch(
1967 &self,
1968 jump_width: JumpWidth,
1969 op1: &Type,
1970 op2: &Type,
1971 op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
1972 ) -> Result<Option<bool>, String> {
1973 match (jump_width, op1, op2) {
1974 (_, Type::ScalarValue(data1), Type::ScalarValue(data2)) => op(*data1, *data2),
1975 (JumpWidth::W64, Type::ScalarValue(data), Type::NullOr { .. })
1976 | (JumpWidth::W64, Type::NullOr { .. }, Type::ScalarValue(data))
1977 if data.is_zero() =>
1978 {
1979 Ok(None)
1980 }
1981
1982 (JumpWidth::W64, Type::ScalarValue(data), t) if data.is_zero() && t.is_non_zero() => {
1983 op(1.into(), 0.into())
1984 }
1985
1986 (JumpWidth::W64, t, Type::ScalarValue(data)) if data.is_zero() && t.is_non_zero() => {
1987 op(0.into(), 1.into())
1988 }
1989
1990 (JumpWidth::W64, Type::PtrToStack { offset: x }, Type::PtrToStack { offset: y }) => {
1991 op(x.reg(), y.reg())
1992 }
1993
1994 (
1995 JumpWidth::W64,
1996 Type::PtrToMemory { id: id1, offset: x, .. },
1997 Type::PtrToMemory { id: id2, offset: y, .. },
1998 )
1999 | (
2000 JumpWidth::W64,
2001 Type::PtrToStruct { id: id1, offset: x, .. },
2002 Type::PtrToStruct { id: id2, offset: y, .. },
2003 )
2004 | (
2005 JumpWidth::W64,
2006 Type::PtrToArray { id: id1, offset: x, .. },
2007 Type::PtrToArray { id: id2, offset: y, .. },
2008 ) if *id1 == *id2 => op(*x, *y),
2009
2010 (JumpWidth::W64, Type::PtrToArray { id: id1, .. }, Type::PtrToEndArray { id: id2 })
2011 | (JumpWidth::W64, Type::PtrToEndArray { id: id1 }, Type::PtrToArray { id: id2, .. })
2012 if *id1 == *id2 =>
2013 {
2014 Ok(None)
2015 }
2016
2017 _ => Err(()),
2018 }
2019 .map_err(|_| "non permitted comparison".to_string())
2020 }
2021
2022 fn conditional_jump(
2023 &mut self,
2024 op_name: &str,
2025 verification_context: &mut VerificationContext<'_>,
2026 dst: Register,
2027 src: Source,
2028 offset: i16,
2029 jump_type: JumpType,
2030 jump_width: JumpWidth,
2031 op: impl Fn(ScalarValueData, ScalarValueData) -> Result<Option<bool>, ()>,
2032 ) -> Result<(), String> {
2033 bpf_log!(
2034 self,
2035 verification_context,
2036 "{op_name} {}, {}, {}",
2037 display_register(dst),
2038 display_source(src),
2039 if offset == 0 { format!("0") } else { print_offset(offset) },
2040 );
2041 let op1 = self.reg(dst)?;
2042 let op2 = self.compute_source(src.clone())?;
2043 let apply_constraints_and_register = |mut next: Self,
2044 jump_type: JumpType|
2045 -> Result<Self, String> {
2046 if jump_type != JumpType::Unknown {
2047 let (new_op1, new_op2) =
2048 Type::constraint(&mut next, jump_type, jump_width, op1.clone(), op2.clone())?;
2049 if dst < REGISTER_COUNT {
2050 next.set_reg(dst, new_op1)?;
2051 }
2052 match src {
2053 Source::Reg(r) => {
2054 next.set_reg(r, new_op2)?;
2055 }
2056 _ => {
2057 }
2059 }
2060 }
2061 Ok(next)
2062 };
2063 let branch = self.compute_branch(jump_width, &op1, &op2, op)?;
2064 let parent = Some(Arc::new(self.clone()));
2065 if branch.unwrap_or(true) {
2066 verification_context.states.push(apply_constraints_and_register(
2068 self.jump_with_offset(offset, parent.clone())?,
2069 jump_type,
2070 )?);
2071 }
2072 if !branch.unwrap_or(false) {
2073 verification_context.states.push(apply_constraints_and_register(
2075 self.jump_with_offset(0, parent)?,
2076 jump_type.invert(),
2077 )?);
2078 }
2079 Ok(())
2080 }
2081
2082 fn terminate(self, verification_context: &mut VerificationContext<'_>) -> Result<(), String> {
2098 let mut next = Some(self);
2099 while let Some(mut current) = next.take() {
2101 let parent = current.parent.take();
2104
2105 let mut dependencies = DataDependencies::default();
2108 for dependency in current.dependencies.get_mut().iter() {
2109 dependencies.merge(dependency);
2110 }
2111
2112 dependencies.visit(
2113 &mut DataDependenciesVisitorContext {
2114 calling_context: &verification_context.calling_context,
2115 computation_context: ¤t,
2116 },
2117 verification_context.code[current.pc],
2118 )?;
2119
2120 for register in 0..GENERAL_REGISTER_COUNT {
2122 if !dependencies.registers.contains(®ister) {
2123 current.set_reg(register, Default::default())?;
2124 }
2125 }
2126 current.stack.data.retain(|k, _| dependencies.stack.contains(k));
2127
2128 let terminating_contexts =
2130 verification_context.terminating_contexts.entry(current.pc).or_default();
2131 let mut is_dominated = false;
2132 terminating_contexts.retain(|c| match c.computation_context.partial_cmp(¤t) {
2133 Some(Ordering::Less) => false,
2134 Some(Ordering::Equal) | Some(Ordering::Greater) => {
2135 is_dominated = true;
2138 true
2139 }
2140 _ => true,
2141 });
2142 if !is_dominated {
2143 terminating_contexts.push(TerminatingContext {
2144 computation_context: current,
2145 dependencies: dependencies.clone(),
2146 });
2147 }
2148
2149 if let Some(parent) = parent {
2152 parent.dependencies.lock().push(dependencies);
2153 next = Arc::into_inner(parent);
2156 }
2157 }
2158 Ok(())
2159 }
2160}
2161
2162impl Drop for ComputationContext {
2163 fn drop(&mut self) {
2164 let mut next = self.parent.take().and_then(Arc::into_inner);
2165 while let Some(mut current) = next {
2167 next = current.parent.take().and_then(Arc::into_inner);
2168 }
2169 }
2170}
2171
2172impl PartialOrd for ComputationContext {
2175 fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
2176 if self.pc != other.pc || self.resources != other.resources {
2177 return None;
2178 }
2179 let mut result = self.stack.partial_cmp(&other.stack)?;
2180 result = associate_orderings(
2181 result,
2182 Type::compare_list(self.registers.iter(), other.registers.iter())?,
2183 )?;
2184 let mut array_bound_iter1 = self.array_bounds.iter().peekable();
2185 let mut array_bound_iter2 = other.array_bounds.iter().peekable();
2186 let result = loop {
2187 match (array_bound_iter1.peek().cloned(), array_bound_iter2.peek().cloned()) {
2188 (None, None) => break result,
2189 (None, _) => break associate_orderings(result, Ordering::Greater)?,
2190 (_, None) => break associate_orderings(result, Ordering::Less)?,
2191 (Some((k1, v1)), Some((k2, v2))) => match k1.cmp(k2) {
2192 Ordering::Equal => {
2193 array_bound_iter1.next();
2194 array_bound_iter2.next();
2195 result = associate_orderings(result, v2.cmp(v1))?;
2202 }
2203 v @ Ordering::Less => {
2204 array_bound_iter1.next();
2205 result = associate_orderings(result, v)?;
2206 }
2207 v @ Ordering::Greater => {
2208 array_bound_iter2.next();
2209 result = associate_orderings(result, v)?;
2210 }
2211 },
2212 }
2213 };
2214 Some(result)
2215 }
2216}
2217
2218#[derive(Clone, Debug, Default)]
2227struct DataDependencies {
2228 registers: HashSet<Register>,
2230 stack: HashSet<usize>,
2232}
2233
2234impl DataDependencies {
2235 fn merge(&mut self, other: &DataDependencies) {
2236 self.registers.extend(other.registers.iter());
2237 self.stack.extend(other.stack.iter());
2238 }
2239
2240 fn alu(&mut self, dst: Register, src: Source) -> Result<(), String> {
2241 if self.registers.contains(&dst) {
2243 if let Source::Reg(src) = src {
2244 self.registers.insert(src);
2245 }
2246 }
2247 Ok(())
2248 }
2249
2250 fn jmp(&mut self, dst: Register, src: Source) -> Result<(), String> {
2251 self.registers.insert(dst);
2252 if let Source::Reg(src) = src {
2253 self.registers.insert(src);
2254 }
2255 Ok(())
2256 }
2257
2258 fn atomic(
2259 &mut self,
2260 context: &ComputationContext,
2261 fetch: bool,
2262 dst: Register,
2263 offset: i16,
2264 src: Register,
2265 width: DataWidth,
2266 is_cmpxchg: bool,
2267 ) -> Result<(), String> {
2268 let mut is_read = false;
2269 if is_cmpxchg && self.registers.contains(&0) {
2270 is_read = true;
2271 }
2272 if fetch && self.registers.contains(&src) {
2273 is_read = true;
2274 }
2275 let addr = context.reg(dst)?;
2276 if let Type::PtrToStack { offset: stack_offset } = addr {
2277 let stack_offset = stack_offset.add(offset);
2278 if !stack_offset.is_valid_offset() {
2279 return Err(format!("Invalid stack offset at {}", context.pc));
2280 }
2281 if is_read || self.stack.contains(&stack_offset.array_index()) {
2282 is_read = true;
2283 self.stack.insert(stack_offset.array_index());
2284 }
2285 }
2286 if is_read {
2287 self.registers.insert(0);
2288 self.registers.insert(src);
2289 }
2290 self.registers.insert(dst);
2291 Ok(())
2292 }
2293}
2294
2295struct DataDependenciesVisitorContext<'a> {
2296 calling_context: &'a CallingContext,
2297 computation_context: &'a ComputationContext,
2298}
2299
2300impl BpfVisitor for DataDependencies {
2301 type Context<'a> = DataDependenciesVisitorContext<'a>;
2302
2303 fn add<'a>(
2304 &mut self,
2305 _context: &mut Self::Context<'a>,
2306 dst: Register,
2307 src: Source,
2308 ) -> Result<(), String> {
2309 self.alu(dst, src)
2310 }
2311 fn add64<'a>(
2312 &mut self,
2313 _context: &mut Self::Context<'a>,
2314 dst: Register,
2315 src: Source,
2316 ) -> Result<(), String> {
2317 self.alu(dst, src)
2318 }
2319 fn and<'a>(
2320 &mut self,
2321 _context: &mut Self::Context<'a>,
2322 dst: Register,
2323 src: Source,
2324 ) -> Result<(), String> {
2325 self.alu(dst, src)
2326 }
2327 fn and64<'a>(
2328 &mut self,
2329 _context: &mut Self::Context<'a>,
2330 dst: Register,
2331 src: Source,
2332 ) -> Result<(), String> {
2333 self.alu(dst, src)
2334 }
2335 fn arsh<'a>(
2336 &mut self,
2337 _context: &mut Self::Context<'a>,
2338 dst: Register,
2339 src: Source,
2340 ) -> Result<(), String> {
2341 self.alu(dst, src)
2342 }
2343 fn arsh64<'a>(
2344 &mut self,
2345 _context: &mut Self::Context<'a>,
2346 dst: Register,
2347 src: Source,
2348 ) -> Result<(), String> {
2349 self.alu(dst, src)
2350 }
2351 fn div<'a>(
2352 &mut self,
2353 _context: &mut Self::Context<'a>,
2354 dst: Register,
2355 src: Source,
2356 ) -> Result<(), String> {
2357 self.alu(dst, src)
2358 }
2359 fn div64<'a>(
2360 &mut self,
2361 _context: &mut Self::Context<'a>,
2362 dst: Register,
2363 src: Source,
2364 ) -> Result<(), String> {
2365 self.alu(dst, src)
2366 }
2367 fn lsh<'a>(
2368 &mut self,
2369 _context: &mut Self::Context<'a>,
2370 dst: Register,
2371 src: Source,
2372 ) -> Result<(), String> {
2373 self.alu(dst, src)
2374 }
2375 fn lsh64<'a>(
2376 &mut self,
2377 _context: &mut Self::Context<'a>,
2378 dst: Register,
2379 src: Source,
2380 ) -> Result<(), String> {
2381 self.alu(dst, src)
2382 }
2383 fn r#mod<'a>(
2384 &mut self,
2385 _context: &mut Self::Context<'a>,
2386 dst: Register,
2387 src: Source,
2388 ) -> Result<(), String> {
2389 self.alu(dst, src)
2390 }
2391 fn mod64<'a>(
2392 &mut self,
2393 _context: &mut Self::Context<'a>,
2394 dst: Register,
2395 src: Source,
2396 ) -> Result<(), String> {
2397 self.alu(dst, src)
2398 }
2399 fn mul<'a>(
2400 &mut self,
2401 _context: &mut Self::Context<'a>,
2402 dst: Register,
2403 src: Source,
2404 ) -> Result<(), String> {
2405 self.alu(dst, src)
2406 }
2407 fn mul64<'a>(
2408 &mut self,
2409 _context: &mut Self::Context<'a>,
2410 dst: Register,
2411 src: Source,
2412 ) -> Result<(), String> {
2413 self.alu(dst, src)
2414 }
2415 fn or<'a>(
2416 &mut self,
2417 _context: &mut Self::Context<'a>,
2418 dst: Register,
2419 src: Source,
2420 ) -> Result<(), String> {
2421 self.alu(dst, src)
2422 }
2423 fn or64<'a>(
2424 &mut self,
2425 _context: &mut Self::Context<'a>,
2426 dst: Register,
2427 src: Source,
2428 ) -> Result<(), String> {
2429 self.alu(dst, src)
2430 }
2431 fn rsh<'a>(
2432 &mut self,
2433 _context: &mut Self::Context<'a>,
2434 dst: Register,
2435 src: Source,
2436 ) -> Result<(), String> {
2437 self.alu(dst, src)
2438 }
2439 fn rsh64<'a>(
2440 &mut self,
2441 _context: &mut Self::Context<'a>,
2442 dst: Register,
2443 src: Source,
2444 ) -> Result<(), String> {
2445 self.alu(dst, src)
2446 }
2447 fn sub<'a>(
2448 &mut self,
2449 _context: &mut Self::Context<'a>,
2450 dst: Register,
2451 src: Source,
2452 ) -> Result<(), String> {
2453 self.alu(dst, src)
2454 }
2455 fn sub64<'a>(
2456 &mut self,
2457 _context: &mut Self::Context<'a>,
2458 dst: Register,
2459 src: Source,
2460 ) -> Result<(), String> {
2461 self.alu(dst, src)
2462 }
2463 fn xor<'a>(
2464 &mut self,
2465 _context: &mut Self::Context<'a>,
2466 dst: Register,
2467 src: Source,
2468 ) -> Result<(), String> {
2469 self.alu(dst, src)
2470 }
2471 fn xor64<'a>(
2472 &mut self,
2473 _context: &mut Self::Context<'a>,
2474 dst: Register,
2475 src: Source,
2476 ) -> Result<(), String> {
2477 self.alu(dst, src)
2478 }
2479
2480 fn mov<'a>(
2481 &mut self,
2482 _context: &mut Self::Context<'a>,
2483 dst: Register,
2484 src: Source,
2485 ) -> Result<(), String> {
2486 if src == Source::Reg(dst) || !self.registers.contains(&dst) {
2487 return Ok(());
2488 }
2489 if let Source::Reg(src) = src {
2490 self.registers.insert(src);
2491 }
2492 self.registers.remove(&dst);
2493 Ok(())
2494 }
2495 fn mov64<'a>(
2496 &mut self,
2497 context: &mut Self::Context<'a>,
2498 dst: Register,
2499 src: Source,
2500 ) -> Result<(), String> {
2501 self.mov(context, dst, src)
2502 }
2503
2504 fn neg<'a>(&mut self, _context: &mut Self::Context<'a>, _dst: Register) -> Result<(), String> {
2505 Ok(())
2507 }
2508 fn neg64<'a>(
2509 &mut self,
2510 _context: &mut Self::Context<'a>,
2511 _dst: Register,
2512 ) -> Result<(), String> {
2513 Ok(())
2515 }
2516
2517 fn be<'a>(
2518 &mut self,
2519 _context: &mut Self::Context<'a>,
2520 _dst: Register,
2521 _width: DataWidth,
2522 ) -> Result<(), String> {
2523 Ok(())
2525 }
2526 fn le<'a>(
2527 &mut self,
2528 _context: &mut Self::Context<'a>,
2529 _dst: Register,
2530 _width: DataWidth,
2531 ) -> Result<(), String> {
2532 Ok(())
2534 }
2535
2536 fn call_external<'a>(
2537 &mut self,
2538 context: &mut Self::Context<'a>,
2539 index: u32,
2540 ) -> Result<(), String> {
2541 let Some(helper) = context.calling_context.helpers.get(&index).cloned() else {
2542 return Err(format!("unknown external function {}", index));
2543 };
2544 let comp = &context.computation_context;
2547 for (arg_index, arg) in helper.signature.args.iter().enumerate() {
2548 if let Type::MemoryParameter { size, input: true, .. } = arg {
2549 if let Type::PtrToStack { offset } = comp.reg((arg_index + 1) as Register)? {
2550 let end = offset.add(size.size(comp)?);
2551 if offset.is_valid_offset() && end.is_within_stack() {
2552 for slot in offset.array_index()..end.array_index() {
2553 self.stack.insert(slot);
2554 }
2555 if end.sub_index() != 0 {
2556 self.stack.insert(end.array_index());
2557 }
2558 }
2559 }
2560 }
2561 }
2562 for register in 0..helper.signature.args.len() + 1 {
2564 self.registers.remove(&(register as Register));
2565 }
2566 for register in 0..helper.signature.args.len() {
2568 self.registers.insert((register + 1) as Register);
2569 }
2570 Ok(())
2571 }
2572
2573 fn exit<'a>(&mut self, _context: &mut Self::Context<'a>) -> Result<(), String> {
2574 self.registers.insert(0);
2576 Ok(())
2577 }
2578
2579 fn jump<'a>(&mut self, _context: &mut Self::Context<'a>, _offset: i16) -> Result<(), String> {
2580 Ok(())
2582 }
2583
2584 fn jeq<'a>(
2585 &mut self,
2586 _context: &mut Self::Context<'a>,
2587 dst: Register,
2588 src: Source,
2589 offset: i16,
2590 ) -> Result<(), String> {
2591 self.jmp(dst, src)
2592 }
2593 fn jeq64<'a>(
2594 &mut self,
2595 _context: &mut Self::Context<'a>,
2596 dst: Register,
2597 src: Source,
2598 offset: i16,
2599 ) -> Result<(), String> {
2600 self.jmp(dst, src)
2601 }
2602 fn jne<'a>(
2603 &mut self,
2604 _context: &mut Self::Context<'a>,
2605 dst: Register,
2606 src: Source,
2607 offset: i16,
2608 ) -> Result<(), String> {
2609 self.jmp(dst, src)
2610 }
2611 fn jne64<'a>(
2612 &mut self,
2613 _context: &mut Self::Context<'a>,
2614 dst: Register,
2615 src: Source,
2616 offset: i16,
2617 ) -> Result<(), String> {
2618 self.jmp(dst, src)
2619 }
2620 fn jge<'a>(
2621 &mut self,
2622 _context: &mut Self::Context<'a>,
2623 dst: Register,
2624 src: Source,
2625 offset: i16,
2626 ) -> Result<(), String> {
2627 self.jmp(dst, src)
2628 }
2629 fn jge64<'a>(
2630 &mut self,
2631 _context: &mut Self::Context<'a>,
2632 dst: Register,
2633 src: Source,
2634 offset: i16,
2635 ) -> Result<(), String> {
2636 self.jmp(dst, src)
2637 }
2638 fn jgt<'a>(
2639 &mut self,
2640 _context: &mut Self::Context<'a>,
2641 dst: Register,
2642 src: Source,
2643 offset: i16,
2644 ) -> Result<(), String> {
2645 self.jmp(dst, src)
2646 }
2647 fn jgt64<'a>(
2648 &mut self,
2649 _context: &mut Self::Context<'a>,
2650 dst: Register,
2651 src: Source,
2652 offset: i16,
2653 ) -> Result<(), String> {
2654 self.jmp(dst, src)
2655 }
2656 fn jle<'a>(
2657 &mut self,
2658 _context: &mut Self::Context<'a>,
2659 dst: Register,
2660 src: Source,
2661 offset: i16,
2662 ) -> Result<(), String> {
2663 self.jmp(dst, src)
2664 }
2665 fn jle64<'a>(
2666 &mut self,
2667 _context: &mut Self::Context<'a>,
2668 dst: Register,
2669 src: Source,
2670 offset: i16,
2671 ) -> Result<(), String> {
2672 self.jmp(dst, src)
2673 }
2674 fn jlt<'a>(
2675 &mut self,
2676 _context: &mut Self::Context<'a>,
2677 dst: Register,
2678 src: Source,
2679 offset: i16,
2680 ) -> Result<(), String> {
2681 self.jmp(dst, src)
2682 }
2683 fn jlt64<'a>(
2684 &mut self,
2685 _context: &mut Self::Context<'a>,
2686 dst: Register,
2687 src: Source,
2688 offset: i16,
2689 ) -> Result<(), String> {
2690 self.jmp(dst, src)
2691 }
2692 fn jsge<'a>(
2693 &mut self,
2694 _context: &mut Self::Context<'a>,
2695 dst: Register,
2696 src: Source,
2697 offset: i16,
2698 ) -> Result<(), String> {
2699 self.jmp(dst, src)
2700 }
2701 fn jsge64<'a>(
2702 &mut self,
2703 _context: &mut Self::Context<'a>,
2704 dst: Register,
2705 src: Source,
2706 offset: i16,
2707 ) -> Result<(), String> {
2708 self.jmp(dst, src)
2709 }
2710 fn jsgt<'a>(
2711 &mut self,
2712 _context: &mut Self::Context<'a>,
2713 dst: Register,
2714 src: Source,
2715 offset: i16,
2716 ) -> Result<(), String> {
2717 self.jmp(dst, src)
2718 }
2719 fn jsgt64<'a>(
2720 &mut self,
2721 _context: &mut Self::Context<'a>,
2722 dst: Register,
2723 src: Source,
2724 offset: i16,
2725 ) -> Result<(), String> {
2726 self.jmp(dst, src)
2727 }
2728 fn jsle<'a>(
2729 &mut self,
2730 _context: &mut Self::Context<'a>,
2731 dst: Register,
2732 src: Source,
2733 offset: i16,
2734 ) -> Result<(), String> {
2735 self.jmp(dst, src)
2736 }
2737 fn jsle64<'a>(
2738 &mut self,
2739 _context: &mut Self::Context<'a>,
2740 dst: Register,
2741 src: Source,
2742 offset: i16,
2743 ) -> Result<(), String> {
2744 self.jmp(dst, src)
2745 }
2746 fn jslt<'a>(
2747 &mut self,
2748 _context: &mut Self::Context<'a>,
2749 dst: Register,
2750 src: Source,
2751 offset: i16,
2752 ) -> Result<(), String> {
2753 self.jmp(dst, src)
2754 }
2755 fn jslt64<'a>(
2756 &mut self,
2757 _context: &mut Self::Context<'a>,
2758 dst: Register,
2759 src: Source,
2760 offset: i16,
2761 ) -> Result<(), String> {
2762 self.jmp(dst, src)
2763 }
2764 fn jset<'a>(
2765 &mut self,
2766 _context: &mut Self::Context<'a>,
2767 dst: Register,
2768 src: Source,
2769 offset: i16,
2770 ) -> Result<(), String> {
2771 self.jmp(dst, src)
2772 }
2773 fn jset64<'a>(
2774 &mut self,
2775 _context: &mut Self::Context<'a>,
2776 dst: Register,
2777 src: Source,
2778 offset: i16,
2779 ) -> Result<(), String> {
2780 self.jmp(dst, src)
2781 }
2782
2783 fn atomic_add<'a>(
2784 &mut self,
2785 context: &mut Self::Context<'a>,
2786 fetch: bool,
2787 dst: Register,
2788 offset: i16,
2789 src: Register,
2790 ) -> Result<(), String> {
2791 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2792 }
2793
2794 fn atomic_add64<'a>(
2795 &mut self,
2796 context: &mut Self::Context<'a>,
2797 fetch: bool,
2798 dst: Register,
2799 offset: i16,
2800 src: Register,
2801 ) -> Result<(), String> {
2802 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2803 }
2804
2805 fn atomic_and<'a>(
2806 &mut self,
2807 context: &mut Self::Context<'a>,
2808 fetch: bool,
2809 dst: Register,
2810 offset: i16,
2811 src: Register,
2812 ) -> Result<(), String> {
2813 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2814 }
2815
2816 fn atomic_and64<'a>(
2817 &mut self,
2818 context: &mut Self::Context<'a>,
2819 fetch: bool,
2820 dst: Register,
2821 offset: i16,
2822 src: Register,
2823 ) -> Result<(), String> {
2824 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2825 }
2826
2827 fn atomic_or<'a>(
2828 &mut self,
2829 context: &mut Self::Context<'a>,
2830 fetch: bool,
2831 dst: Register,
2832 offset: i16,
2833 src: Register,
2834 ) -> Result<(), String> {
2835 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2836 }
2837
2838 fn atomic_or64<'a>(
2839 &mut self,
2840 context: &mut Self::Context<'a>,
2841 fetch: bool,
2842 dst: Register,
2843 offset: i16,
2844 src: Register,
2845 ) -> Result<(), String> {
2846 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2847 }
2848
2849 fn atomic_xor<'a>(
2850 &mut self,
2851 context: &mut Self::Context<'a>,
2852 fetch: bool,
2853 dst: Register,
2854 offset: i16,
2855 src: Register,
2856 ) -> Result<(), String> {
2857 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2858 }
2859
2860 fn atomic_xor64<'a>(
2861 &mut self,
2862 context: &mut Self::Context<'a>,
2863 fetch: bool,
2864 dst: Register,
2865 offset: i16,
2866 src: Register,
2867 ) -> Result<(), String> {
2868 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2869 }
2870
2871 fn atomic_xchg<'a>(
2872 &mut self,
2873 context: &mut Self::Context<'a>,
2874 fetch: bool,
2875 dst: Register,
2876 offset: i16,
2877 src: Register,
2878 ) -> Result<(), String> {
2879 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U32, false)
2880 }
2881
2882 fn atomic_xchg64<'a>(
2883 &mut self,
2884 context: &mut Self::Context<'a>,
2885 fetch: bool,
2886 dst: Register,
2887 offset: i16,
2888 src: Register,
2889 ) -> Result<(), String> {
2890 self.atomic(&context.computation_context, fetch, dst, offset, src, DataWidth::U64, false)
2891 }
2892
2893 fn atomic_cmpxchg<'a>(
2894 &mut self,
2895 context: &mut Self::Context<'a>,
2896 dst: Register,
2897 offset: i16,
2898 src: Register,
2899 ) -> Result<(), String> {
2900 self.atomic(&context.computation_context, true, dst, offset, src, DataWidth::U32, true)
2901 }
2902
2903 fn atomic_cmpxchg64<'a>(
2904 &mut self,
2905 context: &mut Self::Context<'a>,
2906 dst: Register,
2907 offset: i16,
2908 src: Register,
2909 ) -> Result<(), String> {
2910 self.atomic(&context.computation_context, true, dst, offset, src, DataWidth::U64, true)
2911 }
2912
2913 fn load<'a>(
2914 &mut self,
2915 context: &mut Self::Context<'a>,
2916 dst: Register,
2917 offset: i16,
2918 src: Register,
2919 width: DataWidth,
2920 ) -> Result<(), String> {
2921 let context = &context.computation_context;
2922 if self.registers.contains(&dst) {
2923 let addr = context.reg(src)?;
2924 if let Type::PtrToStack { offset: stack_offset } = addr {
2925 let stack_offset = stack_offset.add(offset);
2926 if !stack_offset.is_valid_offset() {
2927 return Err(format!("Invalid stack offset at {}", context.pc));
2928 }
2929 self.stack.insert(stack_offset.array_index());
2930 }
2931 }
2932 self.registers.insert(src);
2933 Ok(())
2934 }
2935
2936 fn load64<'a>(
2937 &mut self,
2938 _context: &mut Self::Context<'a>,
2939 dst: Register,
2940 _src: u8,
2941 _lower: u32,
2942 ) -> Result<(), String> {
2943 self.registers.remove(&dst);
2944 Ok(())
2945 }
2946
2947 fn load_from_packet<'a>(
2948 &mut self,
2949 _context: &mut Self::Context<'a>,
2950 dst: Register,
2951 src: Register,
2952 _offset: i32,
2953 register_offset: Option<Register>,
2954 _width: DataWidth,
2955 ) -> Result<(), String> {
2956 for register in 1..6 {
2958 self.registers.remove(&(register as Register));
2959 }
2960 if self.registers.remove(&dst) {
2962 self.registers.insert(src);
2963 if let Some(reg) = register_offset {
2964 self.registers.insert(reg);
2965 }
2966 }
2967 Ok(())
2968 }
2969
2970 fn store<'a>(
2971 &mut self,
2972 context: &mut Self::Context<'a>,
2973 dst: Register,
2974 offset: i16,
2975 src: Source,
2976 width: DataWidth,
2977 ) -> Result<(), String> {
2978 let context = &context.computation_context;
2979 let addr = context.reg(dst)?;
2980 if let Type::PtrToStack { offset: stack_offset } = addr {
2981 let stack_offset = stack_offset.add(offset);
2982 if !stack_offset.is_valid_offset() {
2983 return Err(format!("Invalid stack offset at {}", context.pc));
2984 }
2985 if self.stack.remove(&stack_offset.array_index()) {
2986 if let Source::Reg(src) = src {
2987 self.registers.insert(src);
2988 }
2989 }
2990 } else {
2991 if let Source::Reg(src) = src {
2992 self.registers.insert(src);
2993 }
2994 self.registers.insert(dst);
2995 }
2996
2997 Ok(())
2998 }
2999}
3000
3001#[derive(Debug)]
3002struct TerminatingContext {
3003 computation_context: ComputationContext,
3004 dependencies: DataDependencies,
3005}
3006
3007#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3008enum AluType {
3009 Plain,
3010 Sub,
3011 Add,
3012}
3013
3014impl AluType {
3015 fn is_ptr_compatible(&self) -> bool {
3017 match self {
3018 Self::Sub | Self::Add => true,
3019 _ => false,
3020 }
3021 }
3022}
3023
3024#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3025enum JumpWidth {
3026 W32,
3027 W64,
3028}
3029
3030#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3031enum JumpType {
3032 Eq,
3033 Ge,
3034 Gt,
3035 Le,
3036 LooseComparaison,
3037 Lt,
3038 Ne,
3039 StrictComparaison,
3040 Unknown,
3041}
3042
3043impl JumpType {
3044 fn invert(&self) -> Self {
3045 match self {
3046 Self::Eq => Self::Ne,
3047 Self::Ge => Self::Lt,
3048 Self::Gt => Self::Le,
3049 Self::Le => Self::Gt,
3050 Self::LooseComparaison => Self::StrictComparaison,
3051 Self::Lt => Self::Ge,
3052 Self::Ne => Self::Eq,
3053 Self::StrictComparaison => Self::LooseComparaison,
3054 Self::Unknown => Self::Unknown,
3055 }
3056 }
3057
3058 fn is_strict(&self) -> bool {
3059 match self {
3060 Self::Gt | Self::Lt | Self::Ne | Self::StrictComparaison => true,
3061 _ => false,
3062 }
3063 }
3064}
3065
3066fn display_register(register: Register) -> String {
3067 format!("%r{register}")
3068}
3069
3070fn display_source(src: Source) -> String {
3071 match src {
3072 Source::Reg(r) => display_register(r),
3073 Source::Value(v) => format!("0x{v:x}"),
3074 }
3075}
3076
3077impl BpfVisitor for ComputationContext {
3078 type Context<'a> = VerificationContext<'a>;
3079
3080 fn add<'a>(
3081 &mut self,
3082 context: &mut Self::Context<'a>,
3083 dst: Register,
3084 src: Source,
3085 ) -> Result<(), String> {
3086 self.alu(Some("add32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x + y))
3087 }
3088 fn add64<'a>(
3089 &mut self,
3090 context: &mut Self::Context<'a>,
3091 dst: Register,
3092 src: Source,
3093 ) -> Result<(), String> {
3094 self.alu(Some("add"), context, dst, src, AluType::Add, |x, y| x + y)
3095 }
3096 fn and<'a>(
3097 &mut self,
3098 context: &mut Self::Context<'a>,
3099 dst: Register,
3100 src: Source,
3101 ) -> Result<(), String> {
3102 self.alu(Some("and32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x & y))
3103 }
3104 fn and64<'a>(
3105 &mut self,
3106 context: &mut Self::Context<'a>,
3107 dst: Register,
3108 src: Source,
3109 ) -> Result<(), String> {
3110 self.alu(Some("and"), context, dst, src, AluType::Plain, |x, y| x & y)
3111 }
3112 fn arsh<'a>(
3113 &mut self,
3114 context: &mut Self::Context<'a>,
3115 dst: Register,
3116 src: Source,
3117 ) -> Result<(), String> {
3118 self.alu(Some("arsh32"), context, dst, src, AluType::Plain, |x, y| {
3119 alu32(x, y, |x, y| x.ashr(y))
3120 })
3121 }
3122 fn arsh64<'a>(
3123 &mut self,
3124 context: &mut Self::Context<'a>,
3125 dst: Register,
3126 src: Source,
3127 ) -> Result<(), String> {
3128 self.alu(Some("arsh"), context, dst, src, AluType::Plain, |x, y| x.ashr(y))
3129 }
3130 fn div<'a>(
3131 &mut self,
3132 context: &mut Self::Context<'a>,
3133 dst: Register,
3134 src: Source,
3135 ) -> Result<(), String> {
3136 self.alu(Some("div32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x / y))
3137 }
3138 fn div64<'a>(
3139 &mut self,
3140 context: &mut Self::Context<'a>,
3141 dst: Register,
3142 src: Source,
3143 ) -> Result<(), String> {
3144 self.alu(Some("div"), context, dst, src, AluType::Plain, |x, y| x / y)
3145 }
3146 fn lsh<'a>(
3147 &mut self,
3148 context: &mut Self::Context<'a>,
3149 dst: Register,
3150 src: Source,
3151 ) -> Result<(), String> {
3152 self.alu(Some("lsh32"), context, dst, src, AluType::Plain, |x, y| {
3153 alu32(x, y, |x, y| x << y)
3154 })
3155 }
3156 fn lsh64<'a>(
3157 &mut self,
3158 context: &mut Self::Context<'a>,
3159 dst: Register,
3160 src: Source,
3161 ) -> Result<(), String> {
3162 self.alu(Some("lsh"), context, dst, src, AluType::Plain, |x, y| x << y)
3163 }
3164 fn r#mod<'a>(
3165 &mut self,
3166 context: &mut Self::Context<'a>,
3167 dst: Register,
3168 src: Source,
3169 ) -> Result<(), String> {
3170 self.alu(Some("mod32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x % y))
3171 }
3172 fn mod64<'a>(
3173 &mut self,
3174 context: &mut Self::Context<'a>,
3175 dst: Register,
3176 src: Source,
3177 ) -> Result<(), String> {
3178 self.alu(Some("mod"), context, dst, src, AluType::Plain, |x, y| x % y)
3179 }
3180 fn mov<'a>(
3181 &mut self,
3182 context: &mut Self::Context<'a>,
3183 dst: Register,
3184 src: Source,
3185 ) -> Result<(), String> {
3186 bpf_log!(self, context, "mov32 {}, {}", display_register(dst), display_source(src));
3187 let src = self.compute_source(src)?;
3188 let value = match src {
3189 Type::ScalarValue(data) => {
3190 let value = (data.value as u32) as u64;
3191 let unknown_mask = (data.unknown_mask as u32) as u64;
3192 let unwritten_mask = (data.unwritten_mask as u32) as u64;
3193 let urange = U64Range::compute_range_for_bytes_swap(0.into(), data.urange, 0, 0, 4);
3194 Type::ScalarValue(ScalarValueData::new(value, unknown_mask, unwritten_mask, urange))
3195 }
3196 _ => Type::default(),
3197 };
3198 let mut next = self.next()?;
3199 next.set_reg(dst, value)?;
3200 context.states.push(next);
3201 Ok(())
3202 }
3203 fn mov64<'a>(
3204 &mut self,
3205 context: &mut Self::Context<'a>,
3206 dst: Register,
3207 src: Source,
3208 ) -> Result<(), String> {
3209 bpf_log!(self, context, "mov {}, {}", display_register(dst), display_source(src));
3210 let src = self.compute_source(src)?;
3211 let mut next = self.next()?;
3212 next.set_reg(dst, src)?;
3213 context.states.push(next);
3214 Ok(())
3215 }
3216 fn mul<'a>(
3217 &mut self,
3218 context: &mut Self::Context<'a>,
3219 dst: Register,
3220 src: Source,
3221 ) -> Result<(), String> {
3222 self.alu(Some("mul32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x * y))
3223 }
3224 fn mul64<'a>(
3225 &mut self,
3226 context: &mut Self::Context<'a>,
3227 dst: Register,
3228 src: Source,
3229 ) -> Result<(), String> {
3230 self.alu(Some("mul"), context, dst, src, AluType::Plain, |x, y| x * y)
3231 }
3232 fn or<'a>(
3233 &mut self,
3234 context: &mut Self::Context<'a>,
3235 dst: Register,
3236 src: Source,
3237 ) -> Result<(), String> {
3238 self.alu(Some("or32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x | y))
3239 }
3240 fn or64<'a>(
3241 &mut self,
3242 context: &mut Self::Context<'a>,
3243 dst: Register,
3244 src: Source,
3245 ) -> Result<(), String> {
3246 self.alu(Some("or"), context, dst, src, AluType::Plain, |x, y| x | y)
3247 }
3248 fn rsh<'a>(
3249 &mut self,
3250 context: &mut Self::Context<'a>,
3251 dst: Register,
3252 src: Source,
3253 ) -> Result<(), String> {
3254 self.alu(Some("rsh32"), context, dst, src, AluType::Plain, |x, y| {
3255 alu32(x, y, |x, y| x >> y)
3256 })
3257 }
3258 fn rsh64<'a>(
3259 &mut self,
3260 context: &mut Self::Context<'a>,
3261 dst: Register,
3262 src: Source,
3263 ) -> Result<(), String> {
3264 self.alu(Some("rsh"), context, dst, src, AluType::Plain, |x, y| x >> y)
3265 }
3266 fn sub<'a>(
3267 &mut self,
3268 context: &mut Self::Context<'a>,
3269 dst: Register,
3270 src: Source,
3271 ) -> Result<(), String> {
3272 self.alu(Some("sub32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x - y))
3273 }
3274 fn sub64<'a>(
3275 &mut self,
3276 context: &mut Self::Context<'a>,
3277 dst: Register,
3278 src: Source,
3279 ) -> Result<(), String> {
3280 self.alu(Some("sub"), context, dst, src, AluType::Sub, |x, y| x - y)
3281 }
3282 fn xor<'a>(
3283 &mut self,
3284 context: &mut Self::Context<'a>,
3285 dst: Register,
3286 src: Source,
3287 ) -> Result<(), String> {
3288 self.alu(Some("xor32"), context, dst, src, AluType::Plain, |x, y| alu32(x, y, |x, y| x ^ y))
3289 }
3290 fn xor64<'a>(
3291 &mut self,
3292 context: &mut Self::Context<'a>,
3293 dst: Register,
3294 src: Source,
3295 ) -> Result<(), String> {
3296 self.alu(Some("xor"), context, dst, src, AluType::Plain, |x, y| x ^ y)
3297 }
3298
3299 fn neg<'a>(&mut self, context: &mut Self::Context<'a>, dst: Register) -> Result<(), String> {
3300 bpf_log!(self, context, "neg32 {}", display_register(dst));
3301 self.alu(None, context, dst, Source::Value(0), AluType::Plain, |x, y| {
3302 alu32(x, y, |x, _y| -x)
3303 })
3304 }
3305 fn neg64<'a>(&mut self, context: &mut Self::Context<'a>, dst: Register) -> Result<(), String> {
3306 bpf_log!(self, context, "neg {}", display_register(dst));
3307 self.alu(None, context, dst, Source::Value(0), AluType::Plain, |x, _y| -x)
3308 }
3309
3310 fn be<'a>(
3311 &mut self,
3312 context: &mut Self::Context<'a>,
3313 dst: Register,
3314 width: DataWidth,
3315 ) -> Result<(), String> {
3316 self.endianness::<BigEndian>("be", context, dst, width)
3317 }
3318
3319 fn le<'a>(
3320 &mut self,
3321 context: &mut Self::Context<'a>,
3322 dst: Register,
3323 width: DataWidth,
3324 ) -> Result<(), String> {
3325 self.endianness::<LittleEndian>("le", context, dst, width)
3326 }
3327
3328 fn call_external<'a>(
3329 &mut self,
3330 context: &mut Self::Context<'a>,
3331 index: u32,
3332 ) -> Result<(), String> {
3333 bpf_log!(self, context, "call 0x{:x}", index);
3334 let Some(helper) = context.calling_context.helpers.get(&index).cloned() else {
3335 return Err(format!("unknown external function {}", index));
3336 };
3337 let HelperDefinition { signature, name, .. } = helper;
3338 debug_assert!(signature.args.len() <= 5);
3339 let mut next = self.next()?;
3340 for (arg_index, arg) in signature.args.iter().enumerate() {
3341 let reg = (arg_index + 1) as u8;
3342 self.reg(reg)?.match_parameter_type(context, self, name, arg, arg_index, &mut next)?
3343 }
3344 if signature.invalidate_array_bounds {
3346 next.array_bounds.clear();
3347 }
3348 let value =
3349 self.resolve_return_value(context, &signature.return_value, &mut next, false)?;
3350 next.set_reg(0, value)?;
3351 for i in 1..=5 {
3352 next.set_reg(i, Type::default())?;
3353 }
3354 context.states.push(next);
3355 Ok(())
3356 }
3357
3358 fn exit<'a>(&mut self, context: &mut Self::Context<'a>) -> Result<(), String> {
3359 bpf_log!(self, context, "exit");
3360 if !self.reg(0)?.is_written_scalar() {
3361 return Err("register 0 is incorrect at exit time".to_string());
3362 }
3363 if !self.resources.is_empty() {
3364 return Err("some resources have not been released at exit time".to_string());
3365 }
3366 self.terminated = true;
3367 Ok(())
3368 }
3369
3370 fn jump<'a>(&mut self, context: &mut Self::Context<'a>, offset: i16) -> Result<(), String> {
3371 bpf_log!(self, context, "ja {}", offset);
3372 let parent = Some(Arc::new(self.clone()));
3373 context.states.push(self.jump_with_offset(offset, parent)?);
3374 Ok(())
3375 }
3376
3377 fn jeq<'a>(
3378 &mut self,
3379 context: &mut Self::Context<'a>,
3380 dst: Register,
3381 src: Source,
3382 offset: i16,
3383 ) -> Result<(), String> {
3384 self.conditional_jump(
3385 "jeq32",
3386 context,
3387 dst,
3388 src,
3389 offset,
3390 JumpType::Eq,
3391 JumpWidth::W32,
3392 |x, y| {
3393 comp32(x, y, |x, y| {
3394 if x.min == x.max && x.min == y.min && x.min == y.max {
3396 return Some(true);
3397 }
3398 if x.max < y.min || y.max < x.min {
3399 return Some(false);
3400 }
3401 None
3402 })
3403 },
3404 )
3405 }
3406 fn jeq64<'a>(
3407 &mut self,
3408 context: &mut Self::Context<'a>,
3409 dst: Register,
3410 src: Source,
3411 offset: i16,
3412 ) -> Result<(), String> {
3413 self.conditional_jump(
3414 "jeq",
3415 context,
3416 dst,
3417 src,
3418 offset,
3419 JumpType::Eq,
3420 JumpWidth::W64,
3421 |x, y| {
3422 comp64(x, y, |x, y| {
3423 if x.min == x.max && x.min == y.min && x.min == y.max {
3425 return Some(true);
3426 }
3427 if x.max < y.min || y.max < x.min {
3428 return Some(false);
3429 }
3430 None
3431 })
3432 },
3433 )
3434 }
3435 fn jne<'a>(
3436 &mut self,
3437 context: &mut Self::Context<'a>,
3438 dst: Register,
3439 src: Source,
3440 offset: i16,
3441 ) -> Result<(), String> {
3442 self.conditional_jump(
3443 "jne32",
3444 context,
3445 dst,
3446 src,
3447 offset,
3448 JumpType::Ne,
3449 JumpWidth::W32,
3450 |x, y| {
3451 comp32(x, y, |x, y| {
3452 if x.min == x.max && x.min == y.min && x.min == y.max {
3454 return Some(false);
3455 }
3456 if x.max < y.min || y.max < x.min {
3457 return Some(true);
3458 }
3459 None
3460 })
3461 },
3462 )
3463 }
3464 fn jne64<'a>(
3465 &mut self,
3466 context: &mut Self::Context<'a>,
3467 dst: Register,
3468 src: Source,
3469 offset: i16,
3470 ) -> Result<(), String> {
3471 self.conditional_jump(
3472 "jne",
3473 context,
3474 dst,
3475 src,
3476 offset,
3477 JumpType::Ne,
3478 JumpWidth::W64,
3479 |x, y| {
3480 comp64(x, y, |x, y| {
3481 if x.min == x.max && x.min == y.min && x.min == y.max {
3483 return Some(false);
3484 }
3485 if x.max < y.min || y.max < x.min {
3486 return Some(true);
3487 }
3488 None
3489 })
3490 },
3491 )
3492 }
3493 fn jge<'a>(
3494 &mut self,
3495 context: &mut Self::Context<'a>,
3496 dst: Register,
3497 src: Source,
3498 offset: i16,
3499 ) -> Result<(), String> {
3500 self.conditional_jump(
3501 "jge32",
3502 context,
3503 dst,
3504 src,
3505 offset,
3506 JumpType::Ge,
3507 JumpWidth::W32,
3508 |x, y| {
3509 comp32(x, y, |x, y| {
3510 if x.min >= y.max {
3512 return Some(true);
3513 }
3514 if y.min > x.max {
3515 return Some(false);
3516 }
3517 None
3518 })
3519 },
3520 )
3521 }
3522 fn jge64<'a>(
3523 &mut self,
3524 context: &mut Self::Context<'a>,
3525 dst: Register,
3526 src: Source,
3527 offset: i16,
3528 ) -> Result<(), String> {
3529 self.conditional_jump(
3530 "jge",
3531 context,
3532 dst,
3533 src,
3534 offset,
3535 JumpType::Ge,
3536 JumpWidth::W64,
3537 |x, y| {
3538 comp64(x, y, |x, y| {
3539 if x.min >= y.max {
3541 return Some(true);
3542 }
3543 if y.min > x.max {
3544 return Some(false);
3545 }
3546 None
3547 })
3548 },
3549 )
3550 }
3551 fn jgt<'a>(
3552 &mut self,
3553 context: &mut Self::Context<'a>,
3554 dst: Register,
3555 src: Source,
3556 offset: i16,
3557 ) -> Result<(), String> {
3558 self.conditional_jump(
3559 "jgt32",
3560 context,
3561 dst,
3562 src,
3563 offset,
3564 JumpType::Gt,
3565 JumpWidth::W32,
3566 |x, y| {
3567 comp32(x, y, |x, y| {
3568 if x.min > y.max {
3570 return Some(true);
3571 }
3572 if y.min >= x.max {
3573 return Some(false);
3574 }
3575 None
3576 })
3577 },
3578 )
3579 }
3580 fn jgt64<'a>(
3581 &mut self,
3582 context: &mut Self::Context<'a>,
3583 dst: Register,
3584 src: Source,
3585 offset: i16,
3586 ) -> Result<(), String> {
3587 self.conditional_jump(
3588 "jgt",
3589 context,
3590 dst,
3591 src,
3592 offset,
3593 JumpType::Gt,
3594 JumpWidth::W64,
3595 |x, y| {
3596 comp64(x, y, |x, y| {
3597 if x.min > y.max {
3599 return Some(true);
3600 }
3601 if y.min >= x.max {
3602 return Some(false);
3603 }
3604 None
3605 })
3606 },
3607 )
3608 }
3609 fn jle<'a>(
3610 &mut self,
3611 context: &mut Self::Context<'a>,
3612 dst: Register,
3613 src: Source,
3614 offset: i16,
3615 ) -> Result<(), String> {
3616 self.conditional_jump(
3617 "jle32",
3618 context,
3619 dst,
3620 src,
3621 offset,
3622 JumpType::Le,
3623 JumpWidth::W32,
3624 |x, y| {
3625 comp32(x, y, |x, y| {
3626 if x.max <= y.min {
3628 return Some(true);
3629 }
3630 if y.max < x.min {
3631 return Some(false);
3632 }
3633 None
3634 })
3635 },
3636 )
3637 }
3638 fn jle64<'a>(
3639 &mut self,
3640 context: &mut Self::Context<'a>,
3641 dst: Register,
3642 src: Source,
3643 offset: i16,
3644 ) -> Result<(), String> {
3645 self.conditional_jump(
3646 "jle",
3647 context,
3648 dst,
3649 src,
3650 offset,
3651 JumpType::Le,
3652 JumpWidth::W64,
3653 |x, y| {
3654 comp64(x, y, |x, y| {
3655 if x.max <= y.min {
3657 return Some(true);
3658 }
3659 if y.max < x.min {
3660 return Some(false);
3661 }
3662 None
3663 })
3664 },
3665 )
3666 }
3667 fn jlt<'a>(
3668 &mut self,
3669 context: &mut Self::Context<'a>,
3670 dst: Register,
3671 src: Source,
3672 offset: i16,
3673 ) -> Result<(), String> {
3674 self.conditional_jump(
3675 "jlt32",
3676 context,
3677 dst,
3678 src,
3679 offset,
3680 JumpType::Lt,
3681 JumpWidth::W32,
3682 |x, y| {
3683 comp32(x, y, |x, y| {
3684 if x.max < y.min {
3686 return Some(true);
3687 }
3688 if y.max <= x.min {
3689 return Some(false);
3690 }
3691 None
3692 })
3693 },
3694 )
3695 }
3696 fn jlt64<'a>(
3697 &mut self,
3698 context: &mut Self::Context<'a>,
3699 dst: Register,
3700 src: Source,
3701 offset: i16,
3702 ) -> Result<(), String> {
3703 self.conditional_jump(
3704 "jlt",
3705 context,
3706 dst,
3707 src,
3708 offset,
3709 JumpType::Lt,
3710 JumpWidth::W64,
3711 |x, y| {
3712 comp64(x, y, |x, y| {
3713 if x.max < y.min {
3715 return Some(true);
3716 }
3717 if y.max <= x.min {
3718 return Some(false);
3719 }
3720 None
3721 })
3722 },
3723 )
3724 }
3725 fn jsge<'a>(
3726 &mut self,
3727 context: &mut Self::Context<'a>,
3728 dst: Register,
3729 src: Source,
3730 offset: i16,
3731 ) -> Result<(), String> {
3732 self.conditional_jump(
3733 "jsge32",
3734 context,
3735 dst,
3736 src,
3737 offset,
3738 JumpType::LooseComparaison,
3739 JumpWidth::W32,
3740 |x, y| scomp32(x, y, |x, y| x >= y),
3741 )
3742 }
3743 fn jsge64<'a>(
3744 &mut self,
3745 context: &mut Self::Context<'a>,
3746 dst: Register,
3747 src: Source,
3748 offset: i16,
3749 ) -> Result<(), String> {
3750 self.conditional_jump(
3751 "jsge",
3752 context,
3753 dst,
3754 src,
3755 offset,
3756 JumpType::LooseComparaison,
3757 JumpWidth::W64,
3758 |x, y| scomp64(x, y, |x, y| x >= y),
3759 )
3760 }
3761 fn jsgt<'a>(
3762 &mut self,
3763 context: &mut Self::Context<'a>,
3764 dst: Register,
3765 src: Source,
3766 offset: i16,
3767 ) -> Result<(), String> {
3768 self.conditional_jump(
3769 "jsgt32",
3770 context,
3771 dst,
3772 src,
3773 offset,
3774 JumpType::StrictComparaison,
3775 JumpWidth::W32,
3776 |x, y| scomp32(x, y, |x, y| x > y),
3777 )
3778 }
3779 fn jsgt64<'a>(
3780 &mut self,
3781 context: &mut Self::Context<'a>,
3782 dst: Register,
3783 src: Source,
3784 offset: i16,
3785 ) -> Result<(), String> {
3786 self.conditional_jump(
3787 "jsgt",
3788 context,
3789 dst,
3790 src,
3791 offset,
3792 JumpType::StrictComparaison,
3793 JumpWidth::W64,
3794 |x, y| scomp64(x, y, |x, y| x > y),
3795 )
3796 }
3797 fn jsle<'a>(
3798 &mut self,
3799 context: &mut Self::Context<'a>,
3800 dst: Register,
3801 src: Source,
3802 offset: i16,
3803 ) -> Result<(), String> {
3804 self.conditional_jump(
3805 "jsle32",
3806 context,
3807 dst,
3808 src,
3809 offset,
3810 JumpType::LooseComparaison,
3811 JumpWidth::W32,
3812 |x, y| scomp32(x, y, |x, y| x <= y),
3813 )
3814 }
3815 fn jsle64<'a>(
3816 &mut self,
3817 context: &mut Self::Context<'a>,
3818 dst: Register,
3819 src: Source,
3820 offset: i16,
3821 ) -> Result<(), String> {
3822 self.conditional_jump(
3823 "jsle",
3824 context,
3825 dst,
3826 src,
3827 offset,
3828 JumpType::LooseComparaison,
3829 JumpWidth::W64,
3830 |x, y| scomp64(x, y, |x, y| x <= y),
3831 )
3832 }
3833 fn jslt<'a>(
3834 &mut self,
3835 context: &mut Self::Context<'a>,
3836 dst: Register,
3837 src: Source,
3838 offset: i16,
3839 ) -> Result<(), String> {
3840 self.conditional_jump(
3841 "jslt32",
3842 context,
3843 dst,
3844 src,
3845 offset,
3846 JumpType::StrictComparaison,
3847 JumpWidth::W32,
3848 |x, y| scomp32(x, y, |x, y| x < y),
3849 )
3850 }
3851 fn jslt64<'a>(
3852 &mut self,
3853 context: &mut Self::Context<'a>,
3854 dst: Register,
3855 src: Source,
3856 offset: i16,
3857 ) -> Result<(), String> {
3858 self.conditional_jump(
3859 "jslt",
3860 context,
3861 dst,
3862 src,
3863 offset,
3864 JumpType::StrictComparaison,
3865 JumpWidth::W64,
3866 |x, y| scomp64(x, y, |x, y| x < y),
3867 )
3868 }
3869 fn jset<'a>(
3870 &mut self,
3871 context: &mut Self::Context<'a>,
3872 dst: Register,
3873 src: Source,
3874 offset: i16,
3875 ) -> Result<(), String> {
3876 self.conditional_jump(
3877 "jset32",
3878 context,
3879 dst,
3880 src,
3881 offset,
3882 JumpType::Unknown,
3883 JumpWidth::W32,
3884 |x, y| {
3885 comp32(x, y, |x, y| {
3886 if x.min != x.max || y.min != y.max {
3888 return None;
3889 }
3890 Some(x.min & y.min != 0)
3891 })
3892 },
3893 )
3894 }
3895 fn jset64<'a>(
3896 &mut self,
3897 context: &mut Self::Context<'a>,
3898 dst: Register,
3899 src: Source,
3900 offset: i16,
3901 ) -> Result<(), String> {
3902 self.conditional_jump(
3903 "jset",
3904 context,
3905 dst,
3906 src,
3907 offset,
3908 JumpType::Unknown,
3909 JumpWidth::W64,
3910 |x, y| {
3911 comp64(x, y, |x, y| {
3912 if x.min != x.max || y.min != y.max {
3914 return None;
3915 }
3916 Some(x.min & y.min != 0)
3917 })
3918 },
3919 )
3920 }
3921
3922 fn atomic_add<'a>(
3923 &mut self,
3924 context: &mut Self::Context<'a>,
3925 fetch: bool,
3926 dst: Register,
3927 offset: i16,
3928 src: Register,
3929 ) -> Result<(), String> {
3930 self.atomic_operation(
3931 "add32",
3932 context,
3933 DataWidth::U32,
3934 fetch,
3935 dst,
3936 offset,
3937 src,
3938 AluType::Add,
3939 |x, y| alu32(x, y, |x, y| x + y),
3940 )
3941 }
3942
3943 fn atomic_add64<'a>(
3944 &mut self,
3945 context: &mut Self::Context<'a>,
3946 fetch: bool,
3947 dst: Register,
3948 offset: i16,
3949 src: Register,
3950 ) -> Result<(), String> {
3951 self.atomic_operation(
3952 "add",
3953 context,
3954 DataWidth::U64,
3955 fetch,
3956 dst,
3957 offset,
3958 src,
3959 AluType::Add,
3960 |x, y| x + y,
3961 )
3962 }
3963
3964 fn atomic_and<'a>(
3965 &mut self,
3966 context: &mut Self::Context<'a>,
3967 fetch: bool,
3968 dst: Register,
3969 offset: i16,
3970 src: Register,
3971 ) -> Result<(), String> {
3972 self.atomic_operation(
3973 "and32",
3974 context,
3975 DataWidth::U32,
3976 fetch,
3977 dst,
3978 offset,
3979 src,
3980 AluType::Plain,
3981 |x, y| alu32(x, y, |x, y| x & y),
3982 )
3983 }
3984
3985 fn atomic_and64<'a>(
3986 &mut self,
3987 context: &mut Self::Context<'a>,
3988 fetch: bool,
3989 dst: Register,
3990 offset: i16,
3991 src: Register,
3992 ) -> Result<(), String> {
3993 self.atomic_operation(
3994 "and",
3995 context,
3996 DataWidth::U64,
3997 fetch,
3998 dst,
3999 offset,
4000 src,
4001 AluType::Plain,
4002 |x, y| x & y,
4003 )
4004 }
4005
4006 fn atomic_or<'a>(
4007 &mut self,
4008 context: &mut Self::Context<'a>,
4009 fetch: bool,
4010 dst: Register,
4011 offset: i16,
4012 src: Register,
4013 ) -> Result<(), String> {
4014 self.atomic_operation(
4015 "or32",
4016 context,
4017 DataWidth::U32,
4018 fetch,
4019 dst,
4020 offset,
4021 src,
4022 AluType::Plain,
4023 |x, y| alu32(x, y, |x, y| x | y),
4024 )
4025 }
4026
4027 fn atomic_or64<'a>(
4028 &mut self,
4029 context: &mut Self::Context<'a>,
4030 fetch: bool,
4031 dst: Register,
4032 offset: i16,
4033 src: Register,
4034 ) -> Result<(), String> {
4035 self.atomic_operation(
4036 "or",
4037 context,
4038 DataWidth::U64,
4039 fetch,
4040 dst,
4041 offset,
4042 src,
4043 AluType::Plain,
4044 |x, y| x | y,
4045 )
4046 }
4047
4048 fn atomic_xor<'a>(
4049 &mut self,
4050 context: &mut Self::Context<'a>,
4051 fetch: bool,
4052 dst: Register,
4053 offset: i16,
4054 src: Register,
4055 ) -> Result<(), String> {
4056 self.atomic_operation(
4057 "xor32",
4058 context,
4059 DataWidth::U32,
4060 fetch,
4061 dst,
4062 offset,
4063 src,
4064 AluType::Plain,
4065 |x, y| alu32(x, y, |x, y| x ^ y),
4066 )
4067 }
4068
4069 fn atomic_xor64<'a>(
4070 &mut self,
4071 context: &mut Self::Context<'a>,
4072 fetch: bool,
4073 dst: Register,
4074 offset: i16,
4075 src: Register,
4076 ) -> Result<(), String> {
4077 self.atomic_operation(
4078 "xor",
4079 context,
4080 DataWidth::U64,
4081 fetch,
4082 dst,
4083 offset,
4084 src,
4085 AluType::Plain,
4086 |x, y| x ^ y,
4087 )
4088 }
4089
4090 fn atomic_xchg<'a>(
4091 &mut self,
4092 context: &mut Self::Context<'a>,
4093 fetch: bool,
4094 dst: Register,
4095 offset: i16,
4096 src: Register,
4097 ) -> Result<(), String> {
4098 self.atomic_operation(
4099 "xchg32",
4100 context,
4101 DataWidth::U32,
4102 fetch,
4103 dst,
4104 offset,
4105 src,
4106 AluType::Plain,
4107 |_, x| x,
4108 )
4109 }
4110
4111 fn atomic_xchg64<'a>(
4112 &mut self,
4113 context: &mut Self::Context<'a>,
4114 fetch: bool,
4115 dst: Register,
4116 offset: i16,
4117 src: Register,
4118 ) -> Result<(), String> {
4119 self.raw_atomic_operation(
4120 "xchg",
4121 context,
4122 DataWidth::U64,
4123 fetch,
4124 dst,
4125 offset,
4126 src,
4127 |_, _, x| Ok(x),
4128 )
4129 }
4130
4131 fn atomic_cmpxchg<'a>(
4132 &mut self,
4133 context: &mut Self::Context<'a>,
4134 dst: Register,
4135 offset: i16,
4136 src: Register,
4137 ) -> Result<(), String> {
4138 self.raw_atomic_cmpxchg("cmpxchg32", context, dst, offset, src, JumpWidth::W32, |x, y| {
4139 comp32(x, y, |x, y| {
4140 if x.min == x.max && x.min == y.min && x.min == y.max {
4142 return Some(true);
4143 }
4144 if x.max < y.min || y.max < x.min {
4145 return Some(false);
4146 }
4147 None
4148 })
4149 })
4150 }
4151
4152 fn atomic_cmpxchg64<'a>(
4153 &mut self,
4154 context: &mut Self::Context<'a>,
4155 dst: Register,
4156 offset: i16,
4157 src: Register,
4158 ) -> Result<(), String> {
4159 self.raw_atomic_cmpxchg("cmpxchg", context, dst, offset, src, JumpWidth::W64, |x, y| {
4160 comp64(x, y, |x, y| {
4161 if x.min == x.max && x.min == y.min && x.min == y.max {
4163 return Some(true);
4164 }
4165 if x.max < y.min || y.max < x.min {
4166 return Some(false);
4167 }
4168 None
4169 })
4170 })
4171 }
4172
4173 fn load<'a>(
4174 &mut self,
4175 context: &mut Self::Context<'a>,
4176 dst: Register,
4177 offset: i16,
4178 src: Register,
4179 width: DataWidth,
4180 ) -> Result<(), String> {
4181 bpf_log!(
4182 self,
4183 context,
4184 "ldx{} {}, [{}{}]",
4185 width.str(),
4186 display_register(dst),
4187 display_register(src),
4188 print_offset(offset)
4189 );
4190 let addr = self.reg(src)?;
4191 let loaded_type = self.load_memory(context, &addr, Field::new(offset, width))?;
4192 let mut next = self.next()?;
4193 next.set_reg(dst, loaded_type)?;
4194 context.states.push(next);
4195 Ok(())
4196 }
4197
4198 fn load64<'a>(
4199 &mut self,
4200 context: &mut Self::Context<'a>,
4201 dst: Register,
4202 src: u8,
4203 lower: u32,
4204 ) -> Result<(), String> {
4205 let next_instruction = &context.code[self.pc + 1];
4207
4208 let value = match src {
4209 0 => {
4210 let value = (lower as u64) | (((next_instruction.imm() as u32) as u64) << 32);
4211 bpf_log!(self, context, "lddw {}, 0x{:x}", display_register(dst), value);
4212 Type::from(value)
4213 }
4214 BPF_PSEUDO_MAP_IDX => {
4215 let map_index = lower;
4216 bpf_log!(
4217 self,
4218 context,
4219 "lddw {}, map_by_index({:x})",
4220 display_register(dst),
4221 map_index
4222 );
4223 context
4224 .calling_context
4225 .maps
4226 .get(usize::try_from(map_index).unwrap())
4227 .map(|schema| Type::ConstPtrToMap { id: map_index.into(), schema: *schema })
4228 .ok_or_else(|| format!("lddw with invalid map index: {}", map_index))?
4229 }
4230 BPF_PSEUDO_MAP_IDX_VALUE => {
4231 let map_index = lower;
4232 let offset = next_instruction.imm();
4233 bpf_log!(
4234 self,
4235 context,
4236 "lddw {}, map_value_by_index({:x})+{offset}",
4237 display_register(dst),
4238 map_index
4239 );
4240 let id = context.next_id();
4241 let map_schema = context
4242 .calling_context
4243 .maps
4244 .get(usize::try_from(map_index).unwrap())
4245 .ok_or_else(|| format!("lddw with invalid map index: {}", map_index))?;
4246
4247 if map_schema.map_type != bpf_map_type_BPF_MAP_TYPE_ARRAY {
4248 return Err(format!(
4249 "Invalid map type at index {map_index} for lddw. Expecting array."
4250 ));
4251 }
4252 if map_schema.max_entries == 0 {
4253 return Err(format!("Array has no entry."));
4254 }
4255
4256 Type::PtrToMemory {
4257 id: MemoryId::from(id),
4258 offset: offset.into(),
4259 buffer_size: map_schema.value_size.into(),
4260 }
4261 }
4262 _ => {
4263 return Err(format!("invalid lddw"));
4264 }
4265 };
4266
4267 let parent = Some(Arc::new(self.clone()));
4268 let mut next = self.jump_with_offset(1, parent)?;
4269 next.set_reg(dst, value.into())?;
4270
4271 context.states.push(next);
4272 Ok(())
4273 }
4274
4275 fn load_from_packet<'a>(
4276 &mut self,
4277 context: &mut Self::Context<'a>,
4278 dst: Register,
4279 src: Register,
4280 offset: i32,
4281 register_offset: Option<Register>,
4282 width: DataWidth,
4283 ) -> Result<(), String> {
4284 bpf_log!(
4285 self,
4286 context,
4287 "ldp{} {}{}",
4288 width.str(),
4289 register_offset.map(display_register).unwrap_or_else(Default::default),
4290 print_offset(offset)
4291 );
4292
4293 let src_type = self.reg(src)?;
4295 let src_is_packet = match &context.calling_context.packet_type {
4296 Some(packet_type) => src_type == *packet_type,
4297 None => false,
4298 };
4299 if !src_is_packet {
4300 return Err(format!("R{} is not a packet", src));
4301 }
4302
4303 if let Some(reg) = register_offset {
4304 let reg = self.reg(reg)?;
4305 if !reg.is_written_scalar() {
4306 return Err("access to unwritten offset".to_string());
4307 }
4308 }
4309 let mut next = self.next()?;
4311 next.set_reg(dst, Type::UNKNOWN_SCALAR)?;
4312 for i in 1..=5 {
4313 next.set_reg(i, Type::default())?;
4314 }
4315 context.states.push(next);
4316 if !self.reg(0)?.is_written_scalar() {
4318 return Err("register 0 is incorrect at exit time".to_string());
4319 }
4320 if !self.resources.is_empty() {
4321 return Err("some resources have not been released at exit time".to_string());
4322 }
4323 self.terminated = true;
4324 Ok(())
4325 }
4326
4327 fn store<'a>(
4328 &mut self,
4329 context: &mut Self::Context<'a>,
4330 dst: Register,
4331 offset: i16,
4332 src: Source,
4333 width: DataWidth,
4334 ) -> Result<(), String> {
4335 let value = match src {
4336 Source::Reg(r) => {
4337 bpf_log!(
4338 self,
4339 context,
4340 "stx{} [{}{}], {}",
4341 width.str(),
4342 display_register(dst),
4343 print_offset(offset),
4344 display_register(r),
4345 );
4346 self.reg(r)?
4347 }
4348 Source::Value(v) => {
4349 bpf_log!(
4350 self,
4351 context,
4352 "st{} [{}{}], 0x{:x}",
4353 width.str(),
4354 display_register(dst),
4355 print_offset(offset),
4356 v,
4357 );
4358 Type::from(v & Type::mask(width))
4359 }
4360 };
4361 let mut next = self.next()?;
4362 let addr = self.reg(dst)?;
4363 next.store_memory(context, &addr, Field::new(offset, width), value)?;
4364 context.states.push(next);
4365 Ok(())
4366 }
4367}
4368
4369fn alu32(
4370 x: ScalarValueData,
4371 y: ScalarValueData,
4372 op: impl FnOnce(U32ScalarValueData, U32ScalarValueData) -> U32ScalarValueData,
4373) -> ScalarValueData {
4374 op(U32ScalarValueData::from(x), U32ScalarValueData::from(y)).into()
4375}
4376
4377fn comp64(
4378 x: ScalarValueData,
4379 y: ScalarValueData,
4380 op: impl FnOnce(U64Range, U64Range) -> Option<bool>,
4381) -> Result<Option<bool>, ()> {
4382 if !x.is_fully_initialized() || !y.is_fully_initialized() {
4383 return Err(());
4384 }
4385 Ok(op(x.urange, y.urange))
4386}
4387
4388fn comp32(
4389 x: ScalarValueData,
4390 y: ScalarValueData,
4391 op: impl FnOnce(U32Range, U32Range) -> Option<bool>,
4392) -> Result<Option<bool>, ()> {
4393 let x = U32ScalarValueData::from(x);
4394 let y = U32ScalarValueData::from(y);
4395 if !x.is_fully_initialized() || !y.is_fully_initialized() {
4396 return Err(());
4397 }
4398 Ok(op(x.urange, y.urange))
4399}
4400
4401fn scomp64(
4402 x: ScalarValueData,
4403 y: ScalarValueData,
4404 op: impl FnOnce(i64, i64) -> bool,
4405) -> Result<Option<bool>, ()> {
4406 if !x.is_fully_initialized() || !y.is_fully_initialized() {
4407 return Err(());
4408 }
4409 if !x.is_known() || !y.is_known() {
4410 return Ok(None);
4411 }
4412 Ok(Some(op(x.value as i64, y.value as i64)))
4413}
4414
4415fn scomp32(
4416 x: ScalarValueData,
4417 y: ScalarValueData,
4418 op: impl FnOnce(i32, i32) -> bool,
4419) -> Result<Option<bool>, ()> {
4420 let x = U32ScalarValueData::from(x);
4421 let y = U32ScalarValueData::from(y);
4422 if !x.is_fully_initialized() || !y.is_fully_initialized() {
4423 return Err(());
4424 }
4425 if !x.is_known() || !y.is_known() {
4426 return Ok(None);
4427 }
4428 Ok(Some(op(x.value as i32, y.value as i32)))
4429}
4430
4431fn print_offset<T: Into<i32>>(offset: T) -> String {
4432 let offset: i32 = offset.into();
4433 if offset == 0 {
4434 String::new()
4435 } else if offset > 0 {
4436 format!("+{offset}")
4437 } else {
4438 format!("{offset}")
4439 }
4440}
4441
4442fn run_on_stack_offset<F>(v: StackOffset, f: F) -> StackOffset
4443where
4444 F: FnOnce(ScalarValueData) -> ScalarValueData,
4445{
4446 StackOffset(f(v.reg()))
4447}
4448
4449fn error_and_log<T>(
4450 logger: &mut dyn VerifierLogger,
4451 msg: impl std::string::ToString,
4452) -> Result<T, EbpfError> {
4453 let msg = msg.to_string();
4454 logger.log(msg.as_bytes());
4455 return Err(EbpfError::ProgramVerifyError(msg));
4456}
4457
4458fn associate_orderings(o1: Ordering, o2: Ordering) -> Option<Ordering> {
4459 match (o1, o2) {
4460 (o1, o2) if o1 == o2 => Some(o1),
4461 (o, Ordering::Equal) | (Ordering::Equal, o) => Some(o),
4462 _ => None,
4463 }
4464}
4465
4466#[cfg(test)]
4467mod tests {
4468 use super::*;
4469 use std::collections::BTreeSet;
4470 use test_util::{assert_geq, assert_leq};
4471
4472 #[test]
4473 fn test_type_ordering() {
4474 let t0 = Type::from(0);
4475 let t1 = Type::from(1);
4476 let random = Type::AliasParameter { parameter_index: 8 };
4477 let unknown_written = Type::UNKNOWN_SCALAR;
4478 let unwritten = Type::default();
4479
4480 assert_eq!(t0.partial_cmp(&t0), Some(Ordering::Equal));
4481 assert_eq!(t0.partial_cmp(&t1), None);
4482 assert_eq!(t0.partial_cmp(&random), None);
4483 assert_eq!(t0.partial_cmp(&unknown_written), Some(Ordering::Less));
4484 assert_eq!(t0.partial_cmp(&unwritten), Some(Ordering::Less));
4485
4486 assert_eq!(t1.partial_cmp(&t0), None);
4487 assert_eq!(t1.partial_cmp(&t1), Some(Ordering::Equal));
4488 assert_eq!(t1.partial_cmp(&random), None);
4489 assert_eq!(t1.partial_cmp(&unknown_written), Some(Ordering::Less));
4490 assert_eq!(t1.partial_cmp(&unwritten), Some(Ordering::Less));
4491
4492 assert_eq!(random.partial_cmp(&t0), None);
4493 assert_eq!(random.partial_cmp(&t1), None);
4494 assert_eq!(random.partial_cmp(&random), Some(Ordering::Equal));
4495 assert_eq!(random.partial_cmp(&unknown_written), None);
4496 assert_eq!(random.partial_cmp(&unwritten), Some(Ordering::Less));
4497
4498 assert_eq!(unknown_written.partial_cmp(&t0), Some(Ordering::Greater));
4499 assert_eq!(unknown_written.partial_cmp(&t1), Some(Ordering::Greater));
4500 assert_eq!(unknown_written.partial_cmp(&random), None);
4501 assert_eq!(unknown_written.partial_cmp(&unknown_written), Some(Ordering::Equal));
4502 assert_eq!(unknown_written.partial_cmp(&unwritten), Some(Ordering::Less));
4503
4504 assert_eq!(unwritten.partial_cmp(&t0), Some(Ordering::Greater));
4505 assert_eq!(unwritten.partial_cmp(&t1), Some(Ordering::Greater));
4506 assert_eq!(unwritten.partial_cmp(&random), Some(Ordering::Greater));
4507 assert_eq!(unwritten.partial_cmp(&unknown_written), Some(Ordering::Greater));
4508 assert_eq!(unwritten.partial_cmp(&unwritten), Some(Ordering::Equal));
4509 }
4510
4511 #[test]
4512 fn test_stack_ordering() {
4513 let mut s1 = Stack::default();
4514 let mut s2 = Stack::default();
4515
4516 assert_eq!(s1.partial_cmp(&s2), Some(Ordering::Equal));
4517 s1.set(0, 0.into());
4518 assert_eq!(s1.partial_cmp(&s2), Some(Ordering::Less));
4519 assert_eq!(s2.partial_cmp(&s1), Some(Ordering::Greater));
4520 s2.set(1, 1.into());
4521 assert_eq!(s1.partial_cmp(&s2), None);
4522 assert_eq!(s2.partial_cmp(&s1), None);
4523 }
4524
4525 #[test]
4526 fn test_context_ordering() {
4527 let mut c1 = ComputationContext::default();
4528 let mut c2 = ComputationContext::default();
4529
4530 assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Equal));
4531
4532 c1.array_bounds.insert(1.into(), 5);
4533 assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Less));
4534 assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Greater));
4535
4536 c2.array_bounds.insert(1.into(), 7);
4537 assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4538 assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4539
4540 c1.array_bounds.insert(2.into(), 9);
4541 assert_eq!(c1.partial_cmp(&c2), None);
4542 assert_eq!(c2.partial_cmp(&c1), None);
4543
4544 c2.array_bounds.insert(2.into(), 9);
4545 assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4546 assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4547
4548 c2.array_bounds.insert(3.into(), 12);
4549 assert_eq!(c1.partial_cmp(&c2), Some(Ordering::Greater));
4550 assert_eq!(c2.partial_cmp(&c1), Some(Ordering::Less));
4551
4552 c1.pc = 8;
4553 assert_eq!(c1.partial_cmp(&c2), None);
4554 assert_eq!(c2.partial_cmp(&c1), None);
4555 }
4556
4557 #[test]
4558 fn test_stack_access() {
4559 let mut s = Stack::default();
4560
4561 assert!(s.store(StackOffset(8.into()), Type::UNKNOWN_SCALAR, DataWidth::U64).is_ok());
4564 assert!(s.store(StackOffset(16.into()), Type::UNKNOWN_SCALAR, DataWidth::U64).is_ok());
4565 assert!(s.store(StackOffset(24.into()), Type::UNKNOWN_SCALAR, DataWidth::U16).is_ok());
4566
4567 for offset in 0..32 {
4568 for end in (offset + 1)..32 {
4569 assert_eq!(
4570 s.read_data_ptr(2, StackOffset(offset.into()), (end - offset) as u64).is_ok(),
4571 offset >= 8 && end <= 26
4572 );
4573 }
4574 }
4575
4576 assert!(s.read_data_ptr(2, StackOffset(12.into()), u64::MAX - 2).is_err());
4578 }
4579
4580 #[test]
4581 fn test_compute_range_for_bytes_swap() {
4582 let mut values = BTreeSet::<u64>::default();
4585 for v1 in &[0x00, 0x1, u64::MAX] {
4586 for v2 in &[0x00, 0x1, u64::MAX] {
4587 for v3 in &[0x00, 0x1, u64::MAX] {
4588 values.insert(U64Range::assemble_slices((*v1, *v2, *v3), 1, 1));
4589 }
4590 }
4591 }
4592 let store = |old: u64, new: u64| (old & !0xff00) | ((new & 0xff) << 8);
4594
4595 for old in &values {
4596 for new in &values {
4597 let s = store(*old, *new);
4598 for min_old in values.iter().filter(|v| *v <= old) {
4599 for min_new in values.iter().filter(|v| *v <= new) {
4600 for max_old in values.iter().filter(|v| *v >= old) {
4601 for max_new in values.iter().filter(|v| *v >= new) {
4602 let range = U64Range::compute_range_for_bytes_swap(
4603 U64Range::new(*min_old, *max_old),
4604 U64Range::new(*min_new, *max_new),
4605 1,
4606 0,
4607 1,
4608 );
4609 assert_leq!(range.min, s);
4610 assert_geq!(range.max, s);
4611 }
4612 }
4613 }
4614 }
4615 }
4616 }
4617 }
4618
4619 #[test]
4620 fn test_type_constraint_ge_gt() {
4621 let mut context = ComputationContext::default();
4622 let (new_lhs, new_rhs) = Type::constraint(
4623 &mut context,
4624 JumpType::Ge,
4625 JumpWidth::W64,
4626 Type::UNKNOWN_SCALAR,
4627 Type::from(10),
4628 )
4629 .unwrap();
4630
4631 if let Type::ScalarValue(data1) = new_lhs {
4632 assert_eq!(data1.min(), 10);
4633 assert_eq!(data1.max(), u64::MAX);
4634 } else {
4635 panic!("Expected ScalarValue");
4636 }
4637
4638 if let Type::ScalarValue(data2) = new_rhs {
4639 assert_eq!(data2.min(), 10);
4640 assert_eq!(data2.max(), 10);
4641 } else {
4642 panic!("Expected ScalarValue");
4643 }
4644
4645 let (new_lhs, _) = Type::constraint(
4646 &mut context,
4647 JumpType::Gt,
4648 JumpWidth::W64,
4649 Type::UNKNOWN_SCALAR,
4650 Type::from(10),
4651 )
4652 .unwrap();
4653
4654 if let Type::ScalarValue(data1) = new_lhs {
4655 assert_eq!(data1.min(), 11);
4656 assert_eq!(data1.max(), u64::MAX);
4657 } else {
4658 panic!("Expected ScalarValue");
4659 }
4660 }
4661
4662 #[test]
4663 fn test_type_constraint_lt_le() {
4664 let mut context = ComputationContext::default();
4665 let (new_lhs, _) = Type::constraint(
4666 &mut context,
4667 JumpType::Lt,
4668 JumpWidth::W64,
4669 Type::UNKNOWN_SCALAR,
4670 Type::from(10),
4671 )
4672 .unwrap();
4673
4674 if let Type::ScalarValue(data1) = new_lhs {
4675 assert_eq!(data1.min(), 0);
4676 assert_eq!(data1.max(), 9);
4677 } else {
4678 panic!("Expected ScalarValue");
4679 }
4680
4681 let (new_lhs, _) = Type::constraint(
4682 &mut context,
4683 JumpType::Le,
4684 JumpWidth::W64,
4685 Type::UNKNOWN_SCALAR,
4686 Type::from(10),
4687 )
4688 .unwrap();
4689
4690 if let Type::ScalarValue(data1) = new_lhs {
4691 assert_eq!(data1.min(), 0);
4692 assert_eq!(data1.max(), 10);
4693 } else {
4694 panic!("Expected ScalarValue");
4695 }
4696 }
4697
4698 #[test]
4699 fn test_type_constraint_w32() {
4700 let mut context = ComputationContext::default();
4701 let (new_lhs, _) = Type::constraint(
4702 &mut context,
4703 JumpType::Eq,
4704 JumpWidth::W32,
4705 Type::ScalarValue(ScalarValueData::UNKNOWN_WRITTEN),
4706 Type::from(10),
4707 )
4708 .unwrap();
4709 if let Type::ScalarValue(data1) = new_lhs {
4710 assert_eq!(data1.min(), 10);
4711 assert_eq!(data1.max(), 0xffff_ffff_0000_000a);
4712 } else {
4713 panic!("Expected ScalarValue");
4714 }
4715 }
4716
4717 #[test]
4718 fn test_type_constraint_null_or() {
4719 let mut context = ComputationContext::default();
4720 let id = MemoryId::new();
4721 let null_or = Type::NullOr { id: id.clone(), inner: Box::new(Type::UNKNOWN_SCALAR) };
4722 let (new_lhs, _) =
4723 Type::constraint(&mut context, JumpType::Eq, JumpWidth::W64, null_or, Type::from(0))
4724 .unwrap();
4725 if let Type::ScalarValue(data) = new_lhs {
4726 assert_eq!(data.value, 0);
4727 } else {
4728 panic!("Expected zero ScalarValue");
4729 }
4730 }
4731
4732 #[test]
4733 fn test_type_constraint_array_bounds() {
4734 let mut context = ComputationContext::default();
4735 let id = MemoryId::new();
4736 let _ = Type::constraint(
4737 &mut context,
4738 JumpType::Le,
4739 JumpWidth::W64,
4740 Type::PtrToArray { id: id.clone(), offset: 22.into() },
4741 Type::PtrToEndArray { id: id.clone() },
4742 )
4743 .unwrap();
4744 assert_eq!(context.array_bounds.get(&id), Some(&22));
4745 }
4746}