1use super::{ContainerId, Inner, SharedBuffer};
6use crate::identity::ComponentIdentity;
7use diagnostics_data::{LogError, LogsData};
8use diagnostics_log_encoding::parse::ParseError;
9use diagnostics_log_encoding::{Header, TRACING_FORMAT_LOG_RECORD_TYPE};
10use diagnostics_message::error::MessageError;
11use fidl_fuchsia_diagnostics::ComponentSelector;
12use fuchsia_async::condition::{ConditionGuard, WakerEntry};
13use futures::Stream;
14use futures::stream::FusedStream;
15use pin_project::pin_project;
16use selectors::matches_selectors;
17use std::cmp::Ordering;
18use std::marker::PhantomData;
19use std::ops::Range;
20use std::pin::Pin;
21use std::sync::Arc;
22use std::task::{Context, Poll, ready};
23use zerocopy::FromBytes;
24
25#[pin_project]
27pub struct FilterCursor {
28 buffer: Arc<SharedBuffer>,
29
30 message_id: u64,
32
33 end_msg_id: Option<u64>,
35
36 selectors: Vec<ComponentSelector>,
37
38 decompressed: DecompressedChunk,
40
41 chunk_cursor: ChunkCursor,
44
45 #[pin]
46 waker_entry: WakerEntry<Inner>,
47}
48
49impl FilterCursor {
50 pub fn new(
51 buffer: Arc<SharedBuffer>,
52 message_id: u64,
53 end_msg_id: Option<u64>,
54 selectors: Vec<ComponentSelector>,
55 ) -> Self {
56 let waker_entry = buffer.inner.waker_entry();
57 Self {
58 buffer,
59 message_id,
60 end_msg_id,
61 selectors,
62 decompressed: DecompressedChunk::default(),
63 chunk_cursor: ChunkCursor::default(),
64 waker_entry,
65 }
66 }
67
68 pub fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Message<'_>>> {
72 let this = self.project();
73 let mut inner = this.buffer.inner.lock();
74 let mut container_ids = ContainerIds::default();
75
76 let mut dropped = 0;
84 let compressed_tail = inner.compressed_tail_message_id();
85 if *this.message_id < compressed_tail {
86 dropped = compressed_tail - *this.message_id;
87 *this.message_id = compressed_tail;
88 this.decompressed.reset();
89 }
90
91 loop {
92 if this.end_msg_id.is_some_and(|end_msg_id| *this.message_id >= end_msg_id) {
93 *this.message_id = u64::MAX;
94 return Poll::Ready(None);
95 }
96
97 let (chunk_bytes, chunk_start, is_active) =
100 if *this.message_id < inner.active_msg_ids.start {
101 this.decompressed.load(&inner, *this.message_id);
102 (&this.decompressed.bytes[..], this.decompressed.msg_ids.start, false)
103 } else if *this.message_id < inner.active_msg_ids.end {
104 (&inner.active_chunk[..], inner.active_msg_ids.start, true)
105 } else {
106 break;
108 };
109
110 let (container_id, msg_range) =
111 this.chunk_cursor.read_record(chunk_bytes, chunk_start, *this.message_id);
112 *this.message_id += 1;
113
114 if container_ids.matches(&inner, container_id, this.selectors) {
115 let data = if is_active {
116 MessageData::ActiveChunk(msg_range)
117 } else {
118 MessageData::Slice(&this.decompressed.bytes[msg_range])
119 };
120 return Poll::Ready(Some(Message { inner, container_id, data, dropped }));
121 }
122 }
123
124 if inner.terminated {
125 *this.message_id = u64::MAX;
126 return Poll::Ready(None);
127 }
128
129 inner.add_waker(this.waker_entry, cx.waker().clone());
130 Poll::Pending
131 }
132
133 fn is_terminated(&self) -> bool {
134 self.message_id == u64::MAX
135 }
136}
137
138#[derive(Default)]
140struct DecompressedChunk {
141 bytes: Vec<u8>,
142 msg_ids: Range<u64>,
143}
144
145impl DecompressedChunk {
146 fn reset(&mut self) {
147 self.msg_ids = 0..0;
148 }
149
150 fn load(&mut self, inner: &Inner, message_id: u64) {
152 if self.msg_ids.contains(&message_id) {
153 return;
154 }
155 let idx = inner
156 .chunks
157 .binary_search_by(|c| {
158 if message_id < c.msg_ids.start {
159 Ordering::Greater
160 } else if message_id >= c.msg_ids.end {
161 Ordering::Less
162 } else {
163 Ordering::Equal
164 }
165 })
166 .expect("chunk must exist for message_id < active_msg_ids.start");
167 let chunk = &inner.chunks[idx];
168 let raw_bytes = inner.chunk_bytes(chunk);
169 let uncompressed_len = chunk.uncompressed_len as usize;
170 self.bytes.resize(uncompressed_len, 0);
171 if raw_bytes.len() < uncompressed_len {
172 let written =
173 lz4::decompress_into(raw_bytes, &mut self.bytes).expect("lz4 decompression failed");
174 assert_eq!(written, uncompressed_len);
175 } else {
176 self.bytes.copy_from_slice(raw_bytes);
177 }
178 self.msg_ids = chunk.msg_ids.clone();
179 }
180}
181
182#[derive(Default)]
185struct ChunkCursor {
186 chunk_start_msg_id: u64,
188
189 msg_id: u64,
191
192 offset: usize,
194}
195
196impl ChunkCursor {
197 fn read_record(
200 &mut self,
201 chunk: &[u8],
202 chunk_start_msg_id: u64,
203 target_msg_id: u64,
204 ) -> (ContainerId, Range<usize>) {
205 if self.chunk_start_msg_id != chunk_start_msg_id || self.msg_id > target_msg_id {
206 self.chunk_start_msg_id = chunk_start_msg_id;
207 self.msg_id = chunk_start_msg_id;
208 self.offset = 0;
209 }
210 while self.msg_id < target_msg_id {
211 let (_, msg_range) = Self::parse_record_at(chunk, self.offset);
212 self.offset = msg_range.end;
213 self.msg_id += 1;
214 }
215 let (container_id, msg_range) = Self::parse_record_at(chunk, self.offset);
216 self.offset = msg_range.end;
217 self.msg_id += 1;
218 (container_id, msg_range)
219 }
220
221 fn parse_record_at(chunk: &[u8], offset: usize) -> (ContainerId, Range<usize>) {
222 let container_id =
223 ContainerId(u32::from_ne_bytes(chunk[offset..offset + 4].try_into().unwrap()));
224 let msg_len =
225 u32::from_ne_bytes(chunk[offset + 4..offset + 8].try_into().unwrap()) as usize;
226 (container_id, (offset + 8)..(offset + 8 + msg_len))
227 }
228}
229
230struct ContainerIds {
233 ids: [ContainerId; 8],
234 result: u8,
235 pos: usize,
236}
237
238impl Default for ContainerIds {
239 fn default() -> Self {
240 Self { ids: [ContainerId(0xffff); 8], result: 0, pos: 0 }
241 }
242}
243
244impl ContainerIds {
245 fn matches(&mut self, inner: &Inner, id: ContainerId, selectors: &[ComponentSelector]) -> bool {
246 let Some(container) = inner.containers.get(id) else {
247 return false;
248 };
249 if selectors.is_empty() {
250 return true;
251 }
252 if let Some(pos) = self.ids.iter().position(|i| *i == id) {
253 self.result & (1 << pos) != 0
254 } else {
255 let matched = matches_selectors(&container.identity.moniker, selectors);
256 self.ids[self.pos] = id;
257 if matched {
258 self.result |= 1 << self.pos;
259 } else {
260 self.result &= !(1 << self.pos);
261 }
262 self.pos = (self.pos + 1) % self.ids.len();
263 matched
264 }
265 }
266}
267
268enum MessageData<'a> {
269 Slice(&'a [u8]),
270 ActiveChunk(Range<usize>),
271}
272
273pub struct Message<'a> {
274 inner: ConditionGuard<'a, Inner>,
275 container_id: ContainerId,
276 data: MessageData<'a>,
277 pub dropped: u64,
278}
279
280impl Message<'_> {
281 pub fn parse(&self) -> Result<(&Arc<ComponentIdentity>, Header, &[u8]), MessageError> {
284 let data = match &self.data {
285 MessageData::Slice(slice) => *slice,
286 MessageData::ActiveChunk(range) => &self.inner.active_chunk[range.clone()],
287 };
288 let (mut header, _) = Header::read_from_prefix(data)
289 .map_err(|_| MessageError::from(ParseError::InvalidHeader))?;
290 let msg_len = header.size_words() as usize * 8;
291 if msg_len > data.len() || msg_len < 16 {
292 return Err(ParseError::ValueOutOfValidRange.into());
293 }
294 if header.raw_type() != TRACING_FORMAT_LOG_RECORD_TYPE {
295 return Err(ParseError::InvalidRecordType.into());
296 }
297 header.set_tag(self.container_id.0);
298 let container = self.inner.containers.get(self.container_id).unwrap();
299 Ok((&container.identity, header, &data[..msg_len]))
300 }
301}
302
303impl TryFrom<Message<'_>> for LogsData {
304 type Error = MessageError;
305
306 fn try_from(value: Message<'_>) -> Result<Self, Self::Error> {
307 let (container, _header, data) = value.parse()?;
308 let mut data = diagnostics_message::from_structured(container.as_ref().into(), data)?;
309 if value.dropped > 0 {
310 data.metadata
311 .errors
312 .get_or_insert(vec![])
313 .push(LogError::RolledOutLogs { count: value.dropped });
314 }
315 Ok(data)
316 }
317}
318
319#[pin_project]
321pub struct FilterCursorStream<T> {
322 #[pin]
323 cursor: FilterCursor,
324 phantom: PhantomData<T>,
325}
326
327impl<T> Stream for FilterCursorStream<T>
328where
329 T: for<'a> TryFrom<Message<'a>>,
330{
331 type Item = T;
332
333 fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
334 let mut this = self.project();
335 loop {
336 match ready!(this.cursor.as_mut().poll_next(cx)) {
337 Some(item) => {
338 if let Ok(data) = T::try_from(item) {
339 return Poll::Ready(Some(data));
340 }
341 }
343 None => return Poll::Ready(None),
344 }
345 }
346 }
347}
348
349impl<T> FusedStream for FilterCursorStream<T>
350where
351 T: for<'a> TryFrom<Message<'a>>,
352{
353 fn is_terminated(&self) -> bool {
354 self.cursor.is_terminated()
355 }
356}
357
358impl<T> From<FilterCursor> for FilterCursorStream<T> {
359 fn from(cursor: FilterCursor) -> Self {
360 Self { cursor, phantom: PhantomData }
361 }
362}
363
364#[cfg(test)]
365mod tests {
366 use super::*;
367 use crate::logs::shared_buffer::{
368 InnerGuard, SharedBuffer, SharedBufferOptions, create_ring_buffer,
369 };
370 use crate::logs::stats::LogStreamStats;
371 use crate::logs::testing::make_message;
372 use assert_matches::assert_matches;
373 use fidl_fuchsia_diagnostics::StreamMode;
374 use fuchsia_inspect::Node;
375 use futures::{StreamExt, poll};
376 use selectors::{FastError, parse_component_selector};
377 use std::future::poll_fn;
378 use std::pin::pin;
379 use std::time::Duration;
380
381 fn test_stats() -> Arc<LogStreamStats> {
382 Arc::new(LogStreamStats::new(&Node::default(), &ComponentIdentity::unknown()))
383 }
384
385 #[fuchsia::test]
386 async fn cursor_basic() {
387 let buffer = SharedBuffer::new(
388 create_ring_buffer(65536),
389 Box::new(|_| {}),
390 Default::default(),
391 &Node::default(),
392 );
393 let container = buffer.new_container_buffer(Arc::new(vec!["a"].into()), test_stats());
394 let msg = make_message("a", None, zx::BootInstant::from_nanos(1));
395 container.push_back(msg.bytes());
396
397 let cursor = buffer.cursor(StreamMode::Snapshot, vec![]);
398 let mut stream = pin!(FilterCursorStream::<LogsData>::from(cursor));
399
400 let item = stream.next().await.unwrap();
401 assert_eq!(item.msg().unwrap(), "a");
402 assert!(stream.next().await.is_none());
403 }
404
405 #[fuchsia::test]
406 async fn cursor_filter() {
407 let buffer = SharedBuffer::new(
408 create_ring_buffer(65536),
409 Box::new(|_| {}),
410 Default::default(),
411 &Node::default(),
412 );
413 let container_a = buffer.new_container_buffer(Arc::new(vec!["a"].into()), test_stats());
414 let container_b = buffer.new_container_buffer(Arc::new(vec!["b"].into()), test_stats());
415
416 container_a.push_back(make_message("msg_a", None, zx::BootInstant::from_nanos(1)).bytes());
417 container_b.push_back(make_message("msg_b", None, zx::BootInstant::from_nanos(2)).bytes());
418
419 let selector = parse_component_selector::<FastError>("a").unwrap();
420 let cursor = buffer.cursor(StreamMode::Snapshot, vec![selector]);
421 let mut stream = pin!(FilterCursorStream::<LogsData>::from(cursor));
422
423 let item = stream.next().await.unwrap();
424 assert_eq!(item.msg().unwrap(), "msg_a");
425 assert!(stream.next().await.is_none());
426 }
427
428 #[fuchsia::test]
429 async fn cursor_subscribe() {
430 let buffer = SharedBuffer::new(
431 create_ring_buffer(65536),
432 Box::new(|_| {}),
433 Default::default(),
434 &Node::default(),
435 );
436 let container = buffer.new_container_buffer(Arc::new(vec!["a"].into()), test_stats());
437
438 let cursor = buffer.cursor(StreamMode::Subscribe, vec![]);
439 let mut stream = pin!(FilterCursorStream::<LogsData>::from(cursor));
440
441 assert!(poll!(stream.next()).is_pending());
442
443 container.push_back(make_message("msg", None, zx::BootInstant::from_nanos(1)).bytes());
444
445 let item = stream.next().await.unwrap();
446 assert_eq!(item.msg().unwrap(), "msg");
447 }
448
449 #[fuchsia::test]
450 async fn cursor_snapshot_then_subscribe() {
451 let buffer = SharedBuffer::new(
452 create_ring_buffer(65536),
453 Box::new(|_| {}),
454 Default::default(),
455 &Node::default(),
456 );
457 let container = buffer.new_container_buffer(Arc::new(vec!["a"].into()), test_stats());
458
459 container.push_back(make_message("msg1", None, zx::BootInstant::from_nanos(1)).bytes());
460
461 let cursor = buffer.cursor(StreamMode::SnapshotThenSubscribe, vec![]);
462 let mut stream = pin!(FilterCursorStream::<LogsData>::from(cursor));
463
464 let item = stream.next().await.unwrap();
465 assert_eq!(item.msg().unwrap(), "msg1");
466
467 assert!(poll!(stream.next()).is_pending());
468
469 container.push_back(make_message("msg2", None, zx::BootInstant::from_nanos(2)).bytes());
470
471 let item = stream.next().await.unwrap();
472 assert_eq!(item.msg().unwrap(), "msg2");
473 }
474
475 #[fuchsia::test]
476 async fn cursor_dropped_logs() {
477 let buffer = SharedBuffer::new(
478 create_ring_buffer(65536),
479 Box::new(|_| {}),
480 SharedBufferOptions { sleep_time: Duration::ZERO, ..Default::default() },
481 &Node::default(),
482 );
483 let container_a = buffer.new_container_buffer(Arc::new(vec!["a"].into()), test_stats());
484 let container_b = buffer.new_container_buffer(Arc::new(vec!["b"].into()), test_stats());
485
486 let msg = make_message("msg", None, zx::BootInstant::from_nanos(1));
487
488 container_a.push_back(msg.bytes()); container_a.push_back(msg.bytes()); container_a.push_back(msg.bytes()); let cursor = buffer.cursor(StreamMode::SnapshotThenSubscribe, vec![]);
493 let mut stream = pin!(FilterCursorStream::<LogsData>::from(cursor));
494
495 let item = stream.next().await.unwrap();
497 assert_eq!(item.msg().unwrap(), "msg");
498
499 let has_messages = async |component| {
500 let mut cursor = pin!(buffer.cursor(
501 StreamMode::Snapshot,
502 vec![parse_component_selector::<FastError>(component).unwrap()],
503 ));
504 poll_fn(|cx| cursor.as_mut().poll_next(cx).map(|i| i.is_some())).await
505 };
506
507 while has_messages("a").await {
509 container_b.push_back(msg.bytes());
510 {
512 let mut inner = InnerGuard::new(&buffer);
513 inner.drain_ring_buffer(inner.ring_buffer.head());
514 }
515 }
516
517 let item = stream.next().await.unwrap();
521 assert_eq!(item.metadata.errors.as_ref().unwrap().len(), 1);
522 assert_matches!(
523 item.metadata.errors.as_ref().unwrap()[0],
524 LogError::RolledOutLogs { count } if count >= 2
525 );
526 }
527
528 #[fuchsia::test]
529 async fn terminate_terminates_cursor() {
530 let buffer = SharedBuffer::new(
531 create_ring_buffer(65536),
532 Box::new(|_| {}),
533 SharedBufferOptions { sleep_time: Duration::from_secs(10), ..Default::default() },
534 &Node::default(),
535 );
536 let cursor = pin!(FilterCursorStream::<LogsData>::from(
537 buffer.cursor(StreamMode::SnapshotThenSubscribe, vec![])
538 ));
539 let container = buffer.new_container_buffer(Arc::new(vec!["a"].into()), test_stats());
540 container.push_back(make_message("msg", None, zx::BootInstant::from_nanos(1)).bytes());
541 drop(buffer.terminate());
542 assert_eq!(cursor.count().await, 1);
543 }
544
545 #[fuchsia::test]
546 fn chunk_cursor_read_and_skip() {
547 let mut chunk = Vec::new();
548 let records = [
549 (ContainerId(1), b"first".as_slice()),
550 (ContainerId(2), b"second_record".as_slice()),
551 (ContainerId(3), b"third".as_slice()),
552 (ContainerId(4), b"fourth_record_payload".as_slice()),
553 ];
554 for (id, payload) in records {
555 chunk.extend_from_slice(&id.0.to_ne_bytes());
556 chunk.extend_from_slice(&(payload.len() as u32).to_ne_bytes());
557 chunk.extend_from_slice(payload);
558 }
559
560 let mut cursor = ChunkCursor::default();
561
562 let (id, range) = cursor.read_record(&chunk, 10, 10);
564 assert_eq!(id, ContainerId(1));
565 assert_eq!(&chunk[range], b"first");
566
567 let (id, range) = cursor.read_record(&chunk, 10, 12);
569 assert_eq!(id, ContainerId(3));
570 assert_eq!(&chunk[range], b"third");
571
572 let (id, range) = cursor.read_record(&chunk, 10, 13);
574 assert_eq!(id, ContainerId(4));
575 assert_eq!(&chunk[range], b"fourth_record_payload");
576
577 let (id, range) = cursor.read_record(&chunk, 10, 11);
579 assert_eq!(id, ContainerId(2));
580 assert_eq!(&chunk[range], b"second_record");
581
582 let (id, range) = cursor.read_record(&chunk, 20, 20);
584 assert_eq!(id, ContainerId(1));
585 assert_eq!(&chunk[range], b"first");
586 }
587
588 #[fuchsia::test]
589 async fn container_ids_memoization_and_wraparound() {
590 let buffer = SharedBuffer::new(
591 create_ring_buffer(65536),
592 Box::new(|_| {}),
593 Default::default(),
594 &Node::default(),
595 );
596
597 let mut containers = Vec::new();
599 for i in 0..12 {
600 let name = if i % 2 == 0 { format!("match_{i}") } else { format!("other_{i}") };
601 containers.push(
602 buffer.new_container_buffer(Arc::new(vec![name.as_str()].into()), test_stats()),
603 );
604 }
605
606 let selectors = vec![parse_component_selector::<FastError>("match_*").unwrap()];
607 {
608 let inner = buffer.inner.lock();
609 let mut cache = ContainerIds::default();
610
611 assert!(!cache.matches(&inner, ContainerId(9999), &selectors));
613
614 assert!(cache.matches(&inner, containers[1].container_id, &[]));
616
617 for (i, c) in containers.iter().enumerate() {
619 assert_eq!(cache.matches(&inner, c.container_id, &selectors), i % 2 == 0);
620 }
621
622 for (i, c) in containers.iter().enumerate().rev() {
624 assert_eq!(cache.matches(&inner, c.container_id, &selectors), i % 2 == 0);
625 }
626 }
627 buffer.terminate().await;
628 }
629
630 #[fuchsia::test]
631 async fn decompressed_chunk_compressed_and_uncompressed() {
632 let buffer = SharedBuffer::new(
633 create_ring_buffer(65536),
634 Box::new(|_| {}),
635 Default::default(),
636 &Node::default(),
637 );
638 let container = buffer.new_container_buffer(Arc::new(vec!["a"].into()), test_stats());
639
640 let msg = make_message(&"a".repeat(200), None, zx::BootInstant::from_nanos(1));
642 container.push_back(msg.bytes());
643 container.push_back(msg.bytes());
644 buffer.inner.lock().seal_active_chunk();
645
646 let short_msg = make_message("x", None, zx::BootInstant::from_nanos(2));
649 {
650 let mut inner = buffer.inner.lock();
651 inner.append_message(container.container_id, short_msg.bytes());
652
653 let mut state = 0x1234_5678_9abc_def0u64;
655 for b in &mut inner.active_chunk[8..] {
656 state = state.wrapping_mul(6364136223846793005).wrapping_add(1);
657 *b = (state >> 32) as u8;
658 }
659 inner.seal_active_chunk();
660 }
661
662 {
663 let inner = buffer.inner.lock();
664 assert_eq!(inner.chunks.len(), 2);
665
666 assert!(
668 (inner.chunks[0].range.end - inner.chunks[0].range.start)
669 < inner.chunks[0].uncompressed_len as u64
670 );
671
672 assert_eq!(
674 inner.chunks[1].range.end - inner.chunks[1].range.start,
675 inner.chunks[1].uncompressed_len as u64
676 );
677
678 let mut decompressed = DecompressedChunk::default();
679 decompressed.load(&inner, 0);
680 assert_eq!(decompressed.msg_ids, 0..2);
681 assert_eq!(decompressed.bytes.len(), inner.chunks[0].uncompressed_len as usize);
682
683 decompressed.load(&inner, 1);
685 assert_eq!(decompressed.msg_ids, 0..2);
686
687 decompressed.load(&inner, 2);
689 assert_eq!(decompressed.msg_ids, 2..3);
690 assert_eq!(decompressed.bytes.len(), inner.chunks[1].uncompressed_len as usize);
691
692 decompressed.reset();
693 assert_eq!(decompressed.msg_ids, 0..0);
694 }
695 buffer.terminate().await;
696 }
697
698 #[fuchsia::test]
699 async fn cursor_reads_across_active_to_sealed_chunk_transition() {
700 let buffer = SharedBuffer::new(
701 create_ring_buffer(65536),
702 Box::new(|_| {}),
703 Default::default(),
704 &Node::default(),
705 );
706 let container = buffer.new_container_buffer(Arc::new(vec!["a"].into()), test_stats());
707
708 container.push_back(make_message("m0", None, zx::BootInstant::from_nanos(1)).bytes());
710 container.push_back(make_message("m1", None, zx::BootInstant::from_nanos(2)).bytes());
711
712 let mut stream = pin!(FilterCursorStream::<LogsData>::from(
714 buffer.cursor(StreamMode::Subscribe, vec![])
715 ));
716 assert!(poll!(stream.next()).is_pending());
717
718 container.push_back(make_message("m2", None, zx::BootInstant::from_nanos(3)).bytes());
720 container.push_back(make_message("m3", None, zx::BootInstant::from_nanos(4)).bytes());
721 assert_eq!(stream.next().await.unwrap().msg().unwrap(), "m2");
722
723 buffer.inner.lock().seal_active_chunk();
726 container.push_back(make_message("m4", None, zx::BootInstant::from_nanos(5)).bytes());
727
728 assert_eq!(stream.next().await.unwrap().msg().unwrap(), "m3");
729 assert_eq!(stream.next().await.unwrap().msg().unwrap(), "m4");
730 assert!(poll!(stream.next()).is_pending());
731 }
732}