1use super::buffer::{MapBuffer, VmoOrName};
6use super::lock::RwMapLock;
7use super::vmar::AllocatedVmar;
8use super::{MapError, MapImpl, MapKey, MapValueRef};
9use ebpf::{BpfValue, EbpfBufferPtr, MapSchema};
10use linux_uapi::{
11 BPF_RB_FORCE_WAKEUP, BPF_RB_NO_WAKEUP, BPF_RINGBUF_BUSY_BIT, BPF_RINGBUF_DISCARD_BIT,
12 BPF_RINGBUF_HDR_SZ,
13};
14use static_assertions::const_assert;
15use std::fmt::Debug;
16use std::ops::Deref;
17use std::pin::Pin;
18use std::sync::Arc;
19use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
20
21pub const RINGBUF_SIGNAL: zx::Signals = zx::Signals::USER_0;
24
25const RINGBUF_LOCK_SIGNAL: zx::Signals = zx::Signals::USER_1;
26
27#[derive(Debug)]
28struct RingBufferState {
29 base_addr: usize,
31
32 mask: u32,
36}
37
38impl RingBufferState {
39 fn data_addr(&self) -> usize {
41 self.base_addr + 3 * *MapBuffer::PAGE_SIZE
42 }
43
44 fn consumer_position(&self) -> &AtomicU64 {
47 unsafe { &*((self.base_addr + *MapBuffer::PAGE_SIZE) as *const AtomicU64) }
50 }
51
52 fn producer_position(&self) -> &AtomicU64 {
55 unsafe { &*((self.base_addr + *MapBuffer::PAGE_SIZE * 2) as *const AtomicU64) }
58 }
59
60 fn data_position(&self, position: u64) -> usize {
62 self.data_addr() + ((position & (self.mask as u64)) as usize)
63 }
64
65 fn is_consumer_position(&self, addr: usize) -> bool {
66 let Some(position) = addr.checked_sub(self.data_addr()) else {
67 return false;
68 };
69 let position = position as u32;
70 let consumer_position =
71 self.consumer_position().load(Ordering::Acquire) & (self.mask as u64);
72 u64::from(position) == consumer_position
73 }
74
75 fn header(&self, position: u64) -> &RingBufferRecordHeader {
77 unsafe { &*(self.data_position(position) as *const RingBufferRecordHeader) }
80 }
81}
82
83#[derive(Debug)]
84pub(crate) struct RingBuffer {
85 vmo: Arc<zx::Vmo>,
89
90 vmar: AllocatedVmar,
94
95 mask: u32,
98}
99
100impl RingBuffer {
101 pub fn new(schema: &MapSchema, vmo: impl Into<VmoOrName>) -> Result<Pin<Box<Self>>, MapError> {
123 if schema.key_size != 0 || schema.value_size != 0 {
124 return Err(MapError::InvalidParam);
125 }
126
127 let page_size = *MapBuffer::PAGE_SIZE;
128 let size = schema.max_entries as usize;
130 if size == 0 || size % page_size != 0 || size & (size - 1) != 0 {
131 return Err(MapError::InvalidParam);
132 }
133 let mask: u32 = (size - 1).try_into().map_err(|_| MapError::InvalidParam)?;
134
135 let technical_vmo_size = page_size;
139
140 let control_pages_size = 3 * page_size;
142 let vmo_size = control_pages_size + size;
143
144 let kernel_root_vmar = fuchsia_runtime::vmar_root_self();
145 #[allow(
151 clippy::undocumented_unsafe_blocks,
152 reason = "Force documented unsafe blocks in Starnix"
153 )]
154 let vmar = unsafe {
155 AllocatedVmar::allocate(
156 &kernel_root_vmar,
157 0,
158 technical_vmo_size + control_pages_size + 2 * size,
160 zx::VmarFlags::CAN_MAP_SPECIFIC
161 | zx::VmarFlags::CAN_MAP_READ
162 | zx::VmarFlags::CAN_MAP_WRITE,
163 )
164 .map_err(|_| MapError::Internal)?
165 };
166 let technical_vmo =
167 zx::Vmo::create(technical_vmo_size as u64).map_err(|_| MapError::Internal)?;
168 technical_vmo.set_name(&zx::Name::new_lossy("ebpf:ring_buffer_technical_vmo")).unwrap();
169 vmar.map(
170 0,
171 &technical_vmo,
172 0,
173 page_size,
174 zx::VmarFlags::SPECIFIC | zx::VmarFlags::PERM_READ | zx::VmarFlags::PERM_WRITE,
175 )
176 .map_err(|_| MapError::Internal)?;
177
178 let vmo = match vmo.into() {
179 VmoOrName::Vmo(vmo) => {
180 let actual_vmo_size = vmo.get_size().map_err(|_| MapError::InvalidVmo)? as usize;
181 if vmo_size != actual_vmo_size {
182 return Err(MapError::InvalidVmo);
183 }
184 vmo
185 }
186 VmoOrName::Name(name) => {
187 let vmo = zx::Vmo::create(vmo_size as u64).map_err(|e| match e {
188 zx::Status::NO_MEMORY | zx::Status::OUT_OF_RANGE => MapError::NoMemory,
189 _ => MapError::Internal,
190 })?;
191 let name = format!("ebpf:ring_buffer:{name}");
192 vmo.set_name(&zx::Name::new_lossy(&name)).unwrap();
193 vmo
194 }
195 };
196
197 vmar.map(
198 technical_vmo_size,
199 &vmo,
200 0,
201 vmo_size,
202 zx::VmarFlags::SPECIFIC | zx::VmarFlags::PERM_READ | zx::VmarFlags::PERM_WRITE,
203 )
204 .map_err(|_| MapError::Internal)?;
205
206 vmar.map(
208 technical_vmo_size + vmo_size,
209 &vmo,
210 control_pages_size as u64,
211 size,
212 zx::VmarFlags::SPECIFIC | zx::VmarFlags::PERM_READ | zx::VmarFlags::PERM_WRITE,
213 )
214 .map_err(|_| MapError::Internal)?;
215
216 #[allow(
221 clippy::undocumented_unsafe_blocks,
222 reason = "Force documented unsafe blocks in Starnix"
223 )]
224 let storage_position = unsafe { &mut *(vmar.base() as *mut *const Self) };
225 let storage = Box::pin(Self { vmo: Arc::new(vmo), vmar, mask });
226 *storage_position = storage.deref();
230 Ok(storage)
231 }
232
233 fn state<'a>(&'a self) -> RwMapLock<'a, RingBufferState> {
234 let page_size = *MapBuffer::PAGE_SIZE;
235
236 unsafe {
244 let lock_cell = &*((self.vmar.base() + page_size) as *const AtomicU32);
245 RwMapLock::new(
246 lock_cell,
247 self.vmo.as_handle_ref(),
248 RINGBUF_LOCK_SIGNAL,
249 RingBufferState { base_addr: self.vmar.base() + page_size, mask: self.mask },
250 )
251 }
252 }
253
254 fn commit(
257 &self,
258 header: &RingBufferRecordHeader,
259 flags: RingBufferWakeupPolicy,
260 discard: bool,
261 ) {
262 let mut new_length = header.length.load(Ordering::Acquire) & !BPF_RINGBUF_BUSY_BIT;
263 if discard {
264 new_length |= BPF_RINGBUF_DISCARD_BIT;
265 }
266 header.length.store(new_length, Ordering::Release);
267
268 let state = self.state().read();
271 if flags == RingBufferWakeupPolicy::ForceWakeup
272 || (flags == RingBufferWakeupPolicy::DefaultWakeup
273 && state.is_consumer_position(header as *const RingBufferRecordHeader as usize))
274 {
275 self.vmo
276 .as_handle_ref()
277 .signal(zx::Signals::empty(), RINGBUF_SIGNAL)
278 .expect("Failed to set signal or a ring buffer VMO");
279 }
280 }
281
282 pub unsafe fn submit(addr: u64, flags: RingBufferWakeupPolicy) {
289 let addr = addr as usize;
290 #[allow(clippy::undocumented_unsafe_blocks, reason = "2024 edition migration")]
291 let (ringbuf_storage, header) = unsafe { Self::get_ringbug_and_header_by_addr(addr) };
292 ringbuf_storage.commit(header, flags, false);
293 }
294
295 pub unsafe fn discard(addr: u64, flags: RingBufferWakeupPolicy) {
302 let addr = addr as usize;
303 #[allow(clippy::undocumented_unsafe_blocks, reason = "2024 edition migration")]
304 let (ringbuf_storage, header) = unsafe { Self::get_ringbug_and_header_by_addr(addr) };
305 ringbuf_storage.commit(header, flags, true);
306 }
307
308 unsafe fn get_ringbug_and_header_by_addr(
315 addr: usize,
316 ) -> (&'static RingBuffer, &'static RingBufferRecordHeader) {
317 let page_size = *MapBuffer::PAGE_SIZE;
318 #[allow(clippy::undocumented_unsafe_blocks, reason = "2024 edition migration")]
320 let header = unsafe {
321 &*((addr - std::mem::size_of::<RingBufferRecordHeader>())
322 as *const RingBufferRecordHeader)
323 };
324 let addr_page = addr / page_size;
325 let mapping_start_page = addr_page - header.page_count.load(Ordering::Acquire) as usize - 1;
326 let mapping_start_address = mapping_start_page * page_size;
327 #[allow(clippy::undocumented_unsafe_blocks, reason = "2024 edition migration")]
328 let ringbuf_impl = unsafe { &*(mapping_start_address as *const &RingBuffer) };
329 (ringbuf_impl, header)
330 }
331}
332
333impl MapImpl for RingBuffer {
334 fn lookup<'a>(&'a self, _key: &[u8]) -> Option<MapValueRef<'a>> {
335 None
336 }
337
338 fn update(&self, _key: &[u8], _value: EbpfBufferPtr<'_>, _flags: u64) -> Result<(), MapError> {
339 Err(MapError::InvalidParam)
340 }
341
342 fn delete(&self, _key: &[u8]) -> Result<(), MapError> {
343 Err(MapError::InvalidParam)
344 }
345
346 fn get_next_key(&self, _key: Option<&[u8]>) -> Result<MapKey, MapError> {
347 Err(MapError::InvalidParam)
348 }
349
350 fn vmo(&self) -> &Arc<zx::Vmo> {
351 &self.vmo
352 }
353
354 fn can_read(&self) -> Option<bool> {
355 let state = self.state().read();
356 let consumer_position = state.consumer_position().load(Ordering::Acquire);
357 let producer_position = state.producer_position().load(Ordering::Acquire);
358
359 if consumer_position % 8 != 0 {
360 return Some(false);
361 }
362
363 let can_read = consumer_position < producer_position
365 && ((state.header(consumer_position).length.load(Ordering::Acquire))
366 & BPF_RINGBUF_BUSY_BIT
367 == 0);
368 Some(can_read)
369 }
370
371 fn ringbuf_reserve(&self, size: u32, flags: u64) -> Result<usize, MapError> {
372 if flags != 0 {
373 return Err(MapError::InvalidParam);
374 }
375
376 if size & (BPF_RINGBUF_BUSY_BIT | BPF_RINGBUF_DISCARD_BIT) > 0 {
378 return Err(MapError::InvalidParam);
379 }
380
381 let state = self.state().write();
382 let consumer_position = state.consumer_position().load(Ordering::Acquire);
383 let producer_position = state.producer_position().load(Ordering::Acquire);
384 let max_size = (self.mask + 1) as u64;
385
386 let consumed_size = producer_position.wrapping_sub(consumer_position);
388 let available_size = max_size.checked_sub(consumed_size).ok_or(MapError::InvalidParam)?;
389
390 const HEADER_ALIGNMENT: u32 = std::mem::size_of::<u64>() as u32;
391
392 let total_size: u32 = size
395 .checked_add(BPF_RINGBUF_HDR_SZ + HEADER_ALIGNMENT - 1)
396 .ok_or(MapError::InvalidParam)?
397 / HEADER_ALIGNMENT
398 * HEADER_ALIGNMENT;
399
400 if u64::from(total_size) > available_size {
401 return Err(MapError::SizeLimit);
402 }
403 let data_position = state.data_position(producer_position) + BPF_RINGBUF_HDR_SZ as usize;
404 let data_length = size | BPF_RINGBUF_BUSY_BIT;
405 let page_count = ((data_position - state.data_addr()) / *MapBuffer::PAGE_SIZE + 3)
406 .try_into()
407 .map_err(|_| MapError::SizeLimit)?;
408 let header = state.header(producer_position);
409 header.length.store(data_length, Ordering::Relaxed);
410 header.page_count.store(page_count, Ordering::Relaxed);
411 state
412 .producer_position()
413 .store(producer_position + u64::from(total_size), Ordering::Release);
414 Ok(data_position)
415 }
416}
417
418#[repr(u32)]
419#[derive(Clone, Copy, Debug, PartialEq, Eq)]
420pub(crate) enum RingBufferWakeupPolicy {
421 DefaultWakeup = 0,
422 NoWakeup = BPF_RB_NO_WAKEUP,
423 ForceWakeup = BPF_RB_FORCE_WAKEUP,
424}
425
426impl From<BpfValue> for RingBufferWakeupPolicy {
427 fn from(v: BpfValue) -> Self {
428 let v = u32::try_from(v).unwrap_or(0);
429 match v {
430 BPF_RB_NO_WAKEUP => Self::NoWakeup,
431 BPF_RB_FORCE_WAKEUP => Self::ForceWakeup,
432 _ => Self::DefaultWakeup,
435 }
436 }
437}
438
439#[repr(C)]
440#[repr(align(8))]
441#[derive(Debug)]
442struct RingBufferRecordHeader {
443 length: AtomicU32,
444 page_count: AtomicU32,
445}
446
447const_assert!(std::mem::size_of::<RingBufferRecordHeader>() == BPF_RINGBUF_HDR_SZ as usize);
448
449#[cfg(test)]
450mod test {
451 use super::*;
452 use ebpf::MapFlags;
453
454 #[fuchsia::test]
455 fn test_ring_buffer_wakeup_policy() {
456 assert_eq!(
457 RingBufferWakeupPolicy::from(BpfValue::from(0)),
458 RingBufferWakeupPolicy::DefaultWakeup
459 );
460 assert_eq!(
461 RingBufferWakeupPolicy::from(BpfValue::from(BPF_RB_NO_WAKEUP)),
462 RingBufferWakeupPolicy::NoWakeup
463 );
464 assert_eq!(
465 RingBufferWakeupPolicy::from(BpfValue::from(BPF_RB_FORCE_WAKEUP)),
466 RingBufferWakeupPolicy::ForceWakeup
467 );
468 assert_eq!(
469 RingBufferWakeupPolicy::from(BpfValue::from(42)),
470 RingBufferWakeupPolicy::DefaultWakeup
471 );
472 }
473
474 #[fuchsia::test]
475 fn test_ring_buffer_can_read() {
476 let schema = MapSchema {
477 map_type: linux_uapi::bpf_map_type_BPF_MAP_TYPE_RINGBUF,
478 key_size: 0,
479 value_size: 0,
480 max_entries: 4096,
481 flags: MapFlags::empty(),
482 };
483
484 let ringbuf = RingBuffer::new(&schema, "test_ringbuf").unwrap();
485
486 assert_eq!(ringbuf.can_read(), Some(false));
488
489 let data_pos = ringbuf.ringbuf_reserve(16, 0).unwrap();
492 assert_eq!(ringbuf.can_read(), Some(false));
493
494 unsafe {
498 RingBuffer::submit(data_pos as u64, RingBufferWakeupPolicy::DefaultWakeup);
499 }
500 assert_eq!(ringbuf.can_read(), Some(true));
501
502 {
505 let state = ringbuf.state().write();
506 let producer_pos = state.producer_position().load(Ordering::Acquire);
507 state.consumer_position().store(producer_pos, Ordering::Release);
508 }
509 assert_eq!(ringbuf.can_read(), Some(false));
510 }
511
512 #[fuchsia::test]
513 fn test_ring_buffer_wrapping() {
514 let schema = MapSchema {
515 map_type: linux_uapi::bpf_map_type_BPF_MAP_TYPE_RINGBUF,
516 key_size: 0,
517 value_size: 0,
518 max_entries: 4096,
519 flags: MapFlags::empty(),
520 };
521
522 let ringbuf = RingBuffer::new(&schema, "test_ringbuf_wrapping").unwrap();
523
524 {
527 let state = ringbuf.state().read();
528 state.producer_position().store(0xFFFFFFFFFFFFFFF8, Ordering::Release);
529 state.consumer_position().store(0xFFFFFFFFFFFFFFF0, Ordering::Release);
530 }
531
532 {
538 let state = ringbuf.state().read();
539 state.producer_position().store(8, Ordering::Release);
540 }
541 let result = ringbuf.ringbuf_reserve(16, 0);
547 assert!(
548 result.is_ok(),
549 "Expected ringbuf_reserve to succeed despite wrapping, but got {:?}",
550 result
551 );
552 }
553
554 #[fuchsia::test]
555 fn test_ring_buffer_consumer_advance_wrap() {
556 let schema = MapSchema {
557 map_type: linux_uapi::bpf_map_type_BPF_MAP_TYPE_RINGBUF,
558 key_size: 0,
559 value_size: 0,
560 max_entries: 4096,
561 flags: MapFlags::empty(),
562 };
563
564 let ringbuf = RingBuffer::new(&schema, "test_ringbuf_advance").unwrap();
565
566 {
568 let state = ringbuf.state().read();
569 state.producer_position().store(8, Ordering::Release);
570 state.consumer_position().store(0xFFFFFFFFFFFFFFF0, Ordering::Release);
571 }
572
573 {
575 let state = ringbuf.state().read();
576 state.consumer_position().store(0, Ordering::Release);
577 }
578
579 let result = ringbuf.ringbuf_reserve(16, 0);
586 assert!(result.is_ok(), "Expected ringbuf_reserve to succeed, but got {:?}", result);
587
588 let data_pos = result.unwrap();
590 let state = ringbuf.state().read();
591 let expected_pos = state.data_position(8) + BPF_RINGBUF_HDR_SZ as usize;
592 assert_eq!(data_pos, expected_pos);
593 }
594
595 #[fuchsia::test]
596 fn test_ring_buffer_unaligned_consumer() {
597 let schema = MapSchema {
598 map_type: linux_uapi::bpf_map_type_BPF_MAP_TYPE_RINGBUF,
599 key_size: 0,
600 value_size: 0,
601 max_entries: 4096,
602 flags: MapFlags::empty(),
603 };
604
605 let ringbuf = RingBuffer::new(&schema, "test_ringbuf_unaligned").unwrap();
606
607 {
609 let state = ringbuf.state().read();
610 state.producer_position().store(8, Ordering::Release);
611 state.consumer_position().store(5, Ordering::Release);
612 }
613
614 let result = ringbuf.can_read();
616 assert_eq!(result, Some(false));
617 }
618
619 #[fuchsia::test]
620 fn test_ring_buffer_overflow() {
621 let schema = MapSchema {
622 map_type: linux_uapi::bpf_map_type_BPF_MAP_TYPE_RINGBUF,
623 key_size: 0,
624 value_size: 0,
625 max_entries: 4096,
626 flags: MapFlags::empty(),
627 };
628
629 let ringbuf = RingBuffer::new(&schema, "test_ringbuf").unwrap();
630
631 assert_eq!(ringbuf.ringbuf_reserve(0xffff_ffff, 0), Err(MapError::InvalidParam));
633 assert_eq!(ringbuf.ringbuf_reserve(0xffff_fff1, 0), Err(MapError::InvalidParam));
634 assert_eq!(ringbuf.ringbuf_reserve(0x3fff_ffff, 0), Err(MapError::SizeLimit));
635 }
636}