diagnostics_log/fuchsia/
mod.rs

1// Copyright 2023 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
3
4use crate::PublishOptions;
5use diagnostics_log_types::Severity;
6use fidl::endpoints::ClientEnd;
7use fidl_fuchsia_logger::{
8    LogSinkEvent, LogSinkMarker, LogSinkOnInitRequest, LogSinkProxy, LogSinkSynchronousProxy,
9};
10use fuchsia_async as fasync;
11use fuchsia_component_client::connect::connect_to_protocol;
12use fuchsia_sync::Mutex;
13use futures::stream::StreamExt;
14use std::borrow::Borrow;
15use std::collections::HashSet;
16use std::fmt::Debug;
17use std::sync::Arc;
18use thiserror::Error;
19
20#[cfg(fuchsia_api_level_less_than = "27")]
21use fidl_fuchsia_diagnostics::Interest;
22#[cfg(fuchsia_api_level_at_least = "27")]
23use fidl_fuchsia_diagnostics_types::Interest;
24
25mod filter;
26mod sink;
27
28use filter::InterestFilter;
29use sink::{BufferedSink, IoBufferSink, Sink, SinkConfig};
30
31pub use diagnostics_log_encoding::Metatag;
32pub use diagnostics_log_encoding::encode::{LogEvent, TestRecord};
33pub use paste::paste;
34
35#[cfg(test)]
36use std::{
37    sync::atomic::{AtomicI64, Ordering},
38    time::Duration,
39};
40
41/// Callback for interest listeners
42pub trait OnInterestChanged {
43    /// Callback for when the interest changes
44    fn on_changed(&self, severity: Severity);
45}
46
47/// Options to configure a `Publisher`.
48#[derive(Default)]
49pub struct PublisherOptions<'t> {
50    blocking: bool,
51    pub(crate) interest: Interest,
52    listen_for_interest_updates: bool,
53    log_sink_client: Option<ClientEnd<LogSinkMarker>>,
54    pub(crate) metatags: HashSet<Metatag>,
55    pub(crate) tags: &'t [&'t str],
56    pub(crate) always_log_file_line: bool,
57    register_global_logger: bool,
58}
59
60impl Default for PublishOptions<'static> {
61    fn default() -> Self {
62        Self {
63            publisher: PublisherOptions {
64                // Default to registering the global logger and listening for interest updates for
65                // `PublishOptions` because it's used by the `initialize...` functions which are
66                // typically called at program start time.
67                listen_for_interest_updates: true,
68                register_global_logger: true,
69                ..PublisherOptions::default()
70            },
71            install_panic_hook: true,
72            panic_prefix: None,
73        }
74    }
75}
76
77macro_rules! publisher_options {
78    ($(($name:ident, $self:ident, $($self_arg:ident),*)),*) => {
79        $(
80            impl<'t> $name<'t> {
81                /// Whether or not to log file/line information regardless of severity.
82                ///
83                /// Default: false.
84                pub fn log_file_line_info(mut $self, enable: bool) -> Self {
85                    let this = &mut $self$(.$self_arg)*;
86                    this.always_log_file_line = enable;
87                    $self
88                }
89
90                /// When set, a `fuchsia_async::Task` will be spawned and held that will be
91                /// listening for interest changes. This option can only be set if
92                /// `register_global_logger` is set.
93                ///
94                /// Default: true for `PublishOptions`, false for `PublisherOptions`.
95                pub fn listen_for_interest_updates(mut $self, enable: bool) -> Self {
96                    let this = &mut $self$(.$self_arg)*;
97                    this.listen_for_interest_updates = enable;
98                    $self
99                }
100
101                /// Sets the `LogSink` that will be used.
102                ///
103                /// Default: the `fuchsia.logger.LogSink` available in the incoming namespace.
104                pub fn use_log_sink(mut $self, client: ClientEnd<LogSinkMarker>) -> Self {
105                    let this = &mut $self$(.$self_arg)*;
106                    this.log_sink_client = Some(client);
107                    $self
108                }
109
110                /// When set to true, writes to the log socket will be blocking. This is, we'll
111                /// retry every time the socket buffer is full until we are able to write the log.
112                ///
113                /// Default: false
114                pub fn blocking(mut $self, is_blocking: bool) -> Self {
115                    let this = &mut $self$(.$self_arg)*;
116                    this.blocking = is_blocking;
117                    $self
118                }
119
120                /// When set to true, the publisher will be registered as the global logger. This
121                /// can only be done once.
122                ///
123                /// Default: true for `PublishOptions`, false for `PublisherOptions`.
124                pub fn register_global_logger(mut $self, value: bool) -> Self {
125                    let this = &mut $self$(.$self_arg)*;
126                    this.register_global_logger = value;
127                    $self
128                }
129            }
130        )*
131    };
132}
133
134publisher_options!((PublisherOptions, self,), (PublishOptions, self, publisher));
135
136/// Initializes logging with the given options.
137///
138/// IMPORTANT: this should be called at most once in a program, and must be
139/// called only after an async executor has been set for the current thread,
140/// otherwise it'll return errors or panic. Therefore it's recommended to never
141/// call this from libraries and only do it from binaries.
142// Ideally this would be an async function, but fixing that is a bit of a Yak shave.
143pub fn initialize(opts: PublishOptions<'_>) -> Result<(), PublishError> {
144    let result = Publisher::new_sync_with_async_listener(opts.publisher);
145    if matches!(result, Err(PublishError::MissingOnInit)) {
146        // NOTE: We ignore missing OnInit errors as these can happen on products where the log sink
147        // connection isn't routed. If this is a mistake, then there will be warning messages from
148        // Component Manager regarding failed routing.
149        return Ok(());
150    }
151    result?;
152    if opts.install_panic_hook {
153        crate::install_panic_hook(opts.panic_prefix);
154    }
155    Ok(())
156}
157
158/// Sets the global minimum log severity.
159/// IMPORTANT: this function can panic if `initialize` wasn't called before.
160pub fn set_minimum_severity(severity: impl Into<Severity>) {
161    let severity: Severity = severity.into();
162    log::set_max_level(severity.into());
163}
164
165/// Initializes logging with the given options.
166///
167/// This must be used when working in an environment where a [`fuchsia_async::Executor`] can't be
168/// used.
169///
170/// IMPORTANT: this should be called at most once in a program, and must be
171/// called only after an async executor has been set for the current thread,
172/// otherwise it'll return errors or panic. Therefore it's recommended to never
173/// call this from libraries and only do it from binaries.
174pub fn initialize_sync(opts: PublishOptions<'_>) {
175    match Publisher::new_sync(opts.publisher) {
176        Ok(_) => {}
177        Err(PublishError::MissingOnInit) => {
178            // NOTE: We ignore missing OnInit errors as these can happen on products where the log
179            // sink connection isn't routed. If this is a mistake, then there will be warning
180            // messages from Component Manager regarding failed routing.
181            return;
182        }
183        Err(e) => panic!("Unable to initialize logging: {e:?}"),
184    }
185    if opts.install_panic_hook {
186        crate::install_panic_hook(opts.panic_prefix);
187    }
188}
189
190/// A `Publisher` acts as broker, implementing [`log::Log`] to receive log
191/// events from a component, and then forwarding that data on to a diagnostics service.
192#[derive(Clone)]
193pub struct Publisher {
194    inner: Arc<InnerPublisher>,
195}
196
197struct InnerPublisher {
198    sink: IoBufferSink,
199    filter: InterestFilter,
200}
201
202impl Publisher {
203    fn new(opts: PublisherOptions<'_>, iob: zx::Iob) -> Self {
204        Self {
205            inner: Arc::new(InnerPublisher {
206                sink: IoBufferSink::new(
207                    iob,
208                    SinkConfig {
209                        tags: opts.tags.iter().map(|s| s.to_string()).collect(),
210                        metatags: opts.metatags,
211                        always_log_file_line: opts.always_log_file_line,
212                    },
213                ),
214                filter: InterestFilter::new(opts.interest),
215            }),
216        }
217    }
218
219    /// Returns a new `Publisher`. This will connect synchronously and, if configured, run a
220    /// listener in a separate thread.
221    pub fn new_sync(opts: PublisherOptions<'_>) -> Result<Self, PublishError> {
222        let listen_for_interest_updates = opts.listen_for_interest_updates;
223        let (publisher, client) = Self::new_sync_no_listener(opts)?;
224        if listen_for_interest_updates {
225            let publisher = publisher.clone();
226            std::thread::spawn(move || {
227                fuchsia_async::LocalExecutor::default()
228                    .run_singlethreaded(publisher.listen_for_interest_updates(client.into_proxy()));
229            });
230        }
231        Ok(publisher)
232    }
233
234    /// Returns a new `Publisher`. This will connect synchronously and, if configured, run a
235    /// listener in an async task. Prefer to use `new_async`.
236    pub fn new_sync_with_async_listener(opts: PublisherOptions<'_>) -> Result<Self, PublishError> {
237        let listen_for_interest_updates = opts.listen_for_interest_updates;
238        let (publisher, client) = Self::new_sync_no_listener(opts)?;
239        if listen_for_interest_updates {
240            fasync::Task::spawn(publisher.clone().listen_for_interest_updates(client.into_proxy()))
241                .detach();
242        }
243        Ok(publisher)
244    }
245
246    /// Returns a new `Publisher`, but doesn't listen for interest updates. This will connect
247    /// synchronously.
248    fn new_sync_no_listener(
249        mut opts: PublisherOptions<'_>,
250    ) -> Result<(Self, ClientEnd<LogSinkMarker>), PublishError> {
251        let PublisherOptions { listen_for_interest_updates, register_global_logger, .. } = opts;
252
253        if listen_for_interest_updates && !register_global_logger {
254            // We can only support listening for interest updates if we are registering a global
255            // logger. This is because if we don't register, the initial interest is dropped.
256            return Err(PublishError::UnsupportedOption);
257        }
258
259        let client = match opts.log_sink_client.take() {
260            Some(log_sink) => log_sink,
261            None => connect_to_protocol()
262                .map_err(|e| e.to_string())
263                .map_err(PublishError::LogSinkConnect)?,
264        };
265
266        let proxy = zx::Unowned::<LogSinkSynchronousProxy>::new(client.channel());
267        let Ok(LogSinkEvent::OnInit {
268            payload: LogSinkOnInitRequest { buffer: Some(iob), interest, .. },
269        }) = proxy.wait_for_event(zx::MonotonicInstant::INFINITE)
270        else {
271            return Err(PublishError::MissingOnInit);
272        };
273
274        let publisher = Self::new(opts, iob);
275
276        if register_global_logger {
277            publisher.register_logger(if listen_for_interest_updates { interest } else { None })?;
278        }
279
280        Ok((publisher, client))
281    }
282
283    /// Returns a new `Publisher`. This will connect asynchronously and, if configured, run a
284    /// listener in an async task.
285    pub async fn new_async(mut opts: PublisherOptions<'_>) -> Result<Self, PublishError> {
286        let PublisherOptions { listen_for_interest_updates, register_global_logger, .. } = opts;
287
288        if listen_for_interest_updates && !register_global_logger {
289            // We can only support listening for interest updates if we are registering a global
290            // logger. This is because if we don't register, the initial interest is dropped.
291            return Err(PublishError::UnsupportedOption);
292        }
293
294        let proxy = match opts.log_sink_client.take() {
295            Some(log_sink) => log_sink.into_proxy(),
296            None => connect_to_protocol()
297                .map_err(|e| e.to_string())
298                .map_err(PublishError::LogSinkConnect)?,
299        };
300
301        let Some(Ok(LogSinkEvent::OnInit {
302            payload: LogSinkOnInitRequest { buffer: Some(iob), interest, .. },
303        })) = proxy.take_event_stream().next().await
304        else {
305            return Err(PublishError::MissingOnInit);
306        };
307
308        let publisher = Self::new(opts, iob);
309
310        if register_global_logger {
311            publisher.register_logger(if listen_for_interest_updates { interest } else { None })?;
312            fasync::Task::spawn(publisher.clone().listen_for_interest_updates(proxy)).detach();
313        }
314
315        Ok(publisher)
316    }
317
318    /// Publish the provided event for testing.
319    pub fn event_for_testing(&self, record: TestRecord<'_>) {
320        if self.inner.filter.enabled_for_testing(&record) {
321            self.inner.sink.event_for_testing(record);
322        }
323    }
324
325    /// Registers an interest listener
326    pub fn set_interest_listener<T>(&self, listener: T)
327    where
328        T: OnInterestChanged + Send + Sync + 'static,
329    {
330        self.inner.filter.set_interest_listener(listener);
331    }
332
333    /// Sets the global logger to this publisher. This function may only be called once in the
334    /// lifetime of a program.
335    pub fn register_logger(&self, interest: Option<Interest>) -> Result<(), PublishError> {
336        self.inner.filter.update_interest(interest.unwrap_or_default());
337        // SAFETY: This leaks which guarantees the publisher remains alive for the lifetime of the
338        // program.
339        unsafe {
340            let ptr = Arc::into_raw(self.inner.clone());
341            log::set_logger(&*ptr).inspect_err(|_| {
342                let _ = Arc::from_raw(ptr);
343            })?;
344        }
345        Ok(())
346    }
347
348    /// Listens for interest updates. Callers must maintain a clone to keep the publisher alive;
349    /// this function will downgrade to a weak reference.
350    async fn listen_for_interest_updates(self, proxy: LogSinkProxy) {
351        self.inner.filter.listen_for_interest_updates(proxy).await;
352    }
353}
354
355impl log::Log for InnerPublisher {
356    fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool {
357        // NOTE: we handle minimum severity directly through the log max_level. So we call,
358        // log::set_max_level, log::max_level where appropriate.
359        true
360    }
361
362    fn log(&self, record: &log::Record<'_>) {
363        self.sink.record_log(record);
364    }
365
366    fn flush(&self) {}
367}
368
369impl log::Log for Publisher {
370    #[inline]
371    fn enabled(&self, metadata: &log::Metadata<'_>) -> bool {
372        self.inner.enabled(metadata)
373    }
374
375    #[inline]
376    fn log(&self, record: &log::Record<'_>) {
377        self.inner.log(record)
378    }
379
380    #[inline]
381    fn flush(&self) {
382        self.inner.flush()
383    }
384}
385
386impl Borrow<InterestFilter> for InnerPublisher {
387    fn borrow(&self) -> &InterestFilter {
388        &self.filter
389    }
390}
391
392/// Initializes logging, but buffers logs until the connection is established. This is required for
393/// things like Component Manager, which would otherwise deadlock when starting. This carries some
394/// overhead, so should be avoided unless required.
395pub fn initialize_buffered(opts: PublishOptions<'_>) -> Result<(), PublishError> {
396    BufferedPublisher::new(opts.publisher)?;
397    if opts.install_panic_hook {
398        crate::install_panic_hook(opts.panic_prefix);
399    }
400    Ok(())
401}
402
403/// A buffered publisher will buffer log messages until the IOBuffer is received. If this is
404/// registered as the global logger, then messages will be logged at the default level until an
405/// updated level is received from Archivist.
406pub struct BufferedPublisher {
407    sink: BufferedSink,
408    filter: InterestFilter,
409    interest_listening_task: Mutex<Option<fasync::Task<()>>>,
410}
411
412impl BufferedPublisher {
413    /// Returns a publisher that will buffer messages until the IOBuffer is received. An async
414    /// executor must be established.
415    pub fn new(opts: PublisherOptions<'_>) -> Result<Arc<Self>, PublishError> {
416        if opts.listen_for_interest_updates && !opts.register_global_logger {
417            // We can only support listening for interest updates if we are registering a global
418            // logger. This is because if we don't register, the initial interest is dropped.
419            return Err(PublishError::UnsupportedOption);
420        }
421
422        let client = match opts.log_sink_client {
423            Some(log_sink) => log_sink,
424            None => connect_to_protocol()
425                .map_err(|e| e.to_string())
426                .map_err(PublishError::LogSinkConnect)?,
427        };
428
429        let this = Arc::new(Self {
430            sink: BufferedSink::new(SinkConfig {
431                tags: opts.tags.iter().map(|s| s.to_string()).collect(),
432                metatags: opts.metatags,
433                always_log_file_line: opts.always_log_file_line,
434            }),
435            filter: InterestFilter::new(opts.interest),
436            interest_listening_task: Mutex::default(),
437        });
438
439        if opts.register_global_logger {
440            // SAFETY: This leaks which guarantees the publisher remains alive for the lifetime
441            // of the program. This leaks even when there is an error (which shouldn't happen so
442            // we don't worry about it).
443            unsafe {
444                log::set_logger(&*Arc::into_raw(this.clone()))?;
445            }
446        }
447
448        // Whilst we are waiting for the OnInit event, we hold a strong reference to the publisher
449        // which will prevent the publisher from being dropped and ensure that buffered log messages
450        // are sent.
451        let this_clone = this.clone();
452        *this_clone.interest_listening_task.lock() = Some(fasync::Task::spawn(async move {
453            let proxy = client.into_proxy();
454
455            let Some(Ok(LogSinkEvent::OnInit {
456                payload: LogSinkOnInitRequest { buffer: Some(buffer), interest, .. },
457            })) = proxy.take_event_stream().next().await
458            else {
459                // There's not a lot we can do here: we haven't received the event we expected
460                // and there's no way we can log the issue.
461                return;
462            };
463
464            // Ignore the interest sent in the OnInit request if `listen_for_interest_updates`
465            // is false; it is assumed that the caller wants the interest specified in the
466            // options to stick.
467            this.filter.update_interest(
468                (if opts.listen_for_interest_updates { interest } else { None })
469                    .unwrap_or_default(),
470            );
471
472            this.sink.set_buffer(buffer);
473
474            if opts.listen_for_interest_updates {
475                this.filter.listen_for_interest_updates(proxy).await;
476            }
477        }));
478
479        Ok(this_clone)
480    }
481}
482
483impl log::Log for BufferedPublisher {
484    fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool {
485        // NOTE: we handle minimum severity directly through the log max_level. So we call,
486        // log::set_max_level, log::max_level where appropriate.
487        true
488    }
489
490    fn log(&self, record: &log::Record<'_>) {
491        self.sink.record_log(record);
492    }
493
494    fn flush(&self) {}
495}
496
497impl Borrow<InterestFilter> for BufferedPublisher {
498    fn borrow(&self) -> &InterestFilter {
499        &self.filter
500    }
501}
502
503/// Errors arising while forwarding a diagnostics stream to the environment.
504#[derive(Debug, Error)]
505pub enum PublishError {
506    /// Connection to fuchsia.logger.LogSink failed.
507    #[error("failed to connect to fuchsia.logger.LogSink ({0})")]
508    LogSinkConnect(String),
509
510    /// Couldn't create a new socket.
511    #[error("failed to create a socket for logging")]
512    MakeSocket(#[source] zx::Status),
513
514    /// An issue with the LogSink channel or socket prevented us from sending it to the `LogSink`.
515    #[error("failed to send a socket to the LogSink")]
516    SendSocket(#[source] fidl::Error),
517
518    /// Installing a Logger.
519    #[error("failed to install the loger")]
520    InitLogForward(#[from] log::SetLoggerError),
521
522    /// Unsupported publish option.
523    #[error("unsupported option")]
524    UnsupportedOption,
525
526    /// The channel was closed with no OnInit event.
527    #[error("did not receive the OnInit event")]
528    MissingOnInit,
529}
530
531#[cfg(test)]
532static CURRENT_TIME_NANOS: AtomicI64 = AtomicI64::new(Duration::from_secs(10).as_nanos() as i64);
533
534/// Increments the test clock.
535#[cfg(test)]
536pub fn increment_clock(duration: Duration) {
537    CURRENT_TIME_NANOS.fetch_add(duration.as_nanos() as i64, Ordering::SeqCst);
538}
539
540/// Gets the current monotonic time in nanoseconds.
541#[doc(hidden)]
542pub fn get_now() -> i64 {
543    #[cfg(not(test))]
544    return zx::MonotonicInstant::get().into_nanos();
545
546    #[cfg(test)]
547    CURRENT_TIME_NANOS.load(Ordering::Relaxed)
548}
549
550/// Logs every N seconds using an Atomic variable
551/// to keep track of the time. This will have a higher
552/// performance impact on ARM compared to regular logging due to the use
553/// of an atomic.
554#[macro_export]
555macro_rules! log_every_n_seconds {
556    ($seconds:expr, $severity:expr, $($arg:tt)*) => {
557        use std::{time::Duration, sync::atomic::{Ordering, AtomicI64}};
558        use $crate::{paste, fuchsia::get_now};
559
560        let now = get_now();
561
562        static LAST_LOG_TIMESTAMP: AtomicI64 = AtomicI64::new(0);
563        if now - LAST_LOG_TIMESTAMP.load(Ordering::Acquire) >= Duration::from_secs($seconds).as_nanos() as i64 {
564            paste! {
565                log::[< $severity:lower >]!($($arg)*);
566            }
567            LAST_LOG_TIMESTAMP.store(now, Ordering::Release);
568        }
569    }
570}
571
572#[cfg(test)]
573mod tests {
574    use super::*;
575    use diagnostics_reader::ArchiveReader;
576    use fidl_fuchsia_diagnostics_crasher::{CrasherMarker, CrasherProxy};
577    use fuchsia_async::TimeoutExt;
578    use fuchsia_component_test::{Capability, ChildOptions, RealmBuilder, Ref, Route};
579    use futures::{StreamExt, future};
580    use log::{debug, error, info};
581    use moniker::ExtendedMoniker;
582
583    #[fuchsia::test]
584    async fn panic_integration_test() {
585        let builder = RealmBuilder::new().await.unwrap();
586        let puppet = builder
587            .add_child("rust-crasher", "#meta/crasher.cm", ChildOptions::new())
588            .await
589            .unwrap();
590        builder
591            .add_route(
592                Route::new()
593                    .capability(Capability::protocol::<CrasherMarker>())
594                    .from(&puppet)
595                    .to(Ref::parent()),
596            )
597            .await
598            .unwrap();
599        let realm = builder.build().await.unwrap();
600        let child_name = realm.root.child_name();
601        let reader = ArchiveReader::logs();
602        let (logs, _) = reader.snapshot_then_subscribe().unwrap().split_streams();
603        let proxy: CrasherProxy = realm.root.connect_to_protocol_at_exposed_dir().unwrap();
604        let target_moniker =
605            ExtendedMoniker::parse_str(&format!("realm_builder:{}/rust-crasher", child_name))
606                .unwrap();
607        proxy.crash("This is a test panic.").await.unwrap();
608
609        let result =
610            logs.filter(|data| future::ready(target_moniker == data.moniker)).next().await.unwrap();
611        assert_eq!(result.line_number(), Some(29).as_ref());
612        assert_eq!(
613            result.file_path(),
614            Some("src/lib/diagnostics/log/rust/rust-crasher/src/main.rs")
615        );
616        assert!(
617            result
618                .payload_keys()
619                .unwrap()
620                .get_property("info")
621                .unwrap()
622                .to_string()
623                .contains("This is a test panic.")
624        );
625    }
626
627    #[fuchsia::test(logging = false)]
628    async fn verify_setting_minimum_log_severity() {
629        let reader = ArchiveReader::logs();
630        let (logs, _) = reader.snapshot_then_subscribe().unwrap().split_streams();
631        let _publisher = Publisher::new_async(PublisherOptions {
632            tags: &["verify_setting_minimum_log_severity"],
633            register_global_logger: true,
634            ..PublisherOptions::default()
635        })
636        .await
637        .expect("initialized log");
638
639        info!("I'm an info log");
640        debug!("I'm a debug log and won't show up");
641
642        set_minimum_severity(Severity::Debug);
643        debug!("I'm a debug log and I show up");
644
645        let results = logs
646            .filter(|data| {
647                future::ready(
648                    data.tags().unwrap().iter().any(|t| t == "verify_setting_minimum_log_severity"),
649                )
650            })
651            .take(2)
652            .collect::<Vec<_>>()
653            .await;
654        assert_eq!(results[0].msg().unwrap(), "I'm an info log");
655        assert_eq!(results[1].msg().unwrap(), "I'm a debug log and I show up");
656    }
657
658    #[fuchsia::test]
659    async fn log_macro_logs_are_recorded() {
660        let reader = ArchiveReader::logs();
661        let (logs, _) = reader.snapshot_then_subscribe().unwrap().split_streams();
662
663        let total_threads = 10;
664
665        for i in 0..total_threads {
666            std::thread::spawn(move || {
667                log::info!(thread=i; "log from thread {}", i);
668            });
669        }
670
671        let mut results = logs
672            .filter(|data| {
673                future::ready(
674                    data.tags().unwrap().iter().any(|t| t == "log_macro_logs_are_recorded"),
675                )
676            })
677            .take(total_threads);
678
679        let mut seen = vec![];
680        while let Some(log) = results
681            .next()
682            .on_timeout(Duration::from_secs(30), || {
683                error!("Timed out!");
684                None
685            })
686            .await
687        {
688            let hierarchy = log.payload_keys().unwrap();
689            assert_eq!(hierarchy.properties.len(), 1);
690            assert_eq!(hierarchy.properties[0].name(), "thread");
691            let thread_id = hierarchy.properties[0].uint().unwrap();
692            seen.push(thread_id as usize);
693            assert_eq!(log.msg().unwrap(), format!("log from thread {thread_id}"));
694        }
695        seen.sort();
696        assert_eq!(seen, (0..total_threads).collect::<Vec<_>>());
697    }
698}