diagnostics_message/
lib.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::error::MessageError;
6use byteorder::{ByteOrder, LittleEndian};
7use diagnostics_data::{
8    BuilderArgs, Data, ExtendedMoniker, Logs, LogsData, LogsDataBuilder, LogsField, LogsProperty,
9    Severity, Timestamp,
10};
11use diagnostics_log_encoding::{
12    ARCHIVIST_URL, Argument, Header, LOG_CONTROL_BIT, MONIKER, ROLLED_OUT, Record, URL, Value, zx,
13};
14use flyweights::FlyStr;
15use libc::{c_char, c_int};
16use moniker::Moniker;
17use std::collections::HashMap;
18use std::{mem, str};
19
20#[cfg(fuchsia_api_level_at_least = "HEAD")]
21use fidl_fuchsia_diagnostics as fdiagnostics;
22
23mod constants;
24pub mod error;
25pub mod ffi;
26pub use constants::*;
27
28#[cfg(test)]
29mod test;
30
31#[derive(Clone)]
32pub struct MonikerWithUrl {
33    pub moniker: ExtendedMoniker,
34    pub url: FlyStr,
35}
36
37/// Transforms the given legacy log message (already parsed) into a `LogsData` containing the
38/// given identity information.
39pub fn from_logger(source: MonikerWithUrl, msg: LoggerMessage) -> LogsData {
40    let (raw_severity, severity) = Severity::parse_exact(msg.raw_severity);
41    let mut builder = LogsDataBuilder::new(BuilderArgs {
42        timestamp: Timestamp::from_nanos(msg.timestamp.into_nanos()),
43        component_url: Some(source.url),
44        moniker: source.moniker,
45        severity,
46    })
47    .set_pid(msg.pid)
48    .set_tid(msg.tid)
49    .set_dropped(msg.dropped_logs)
50    .set_message(msg.message);
51    if let Some(raw_severity) = raw_severity {
52        builder = builder.set_raw_severity(raw_severity);
53    }
54    for tag in &msg.tags {
55        builder = builder.add_tag(tag.as_ref());
56    }
57    builder.build()
58}
59
60#[derive(Clone)]
61pub struct ExtendedMetadata {
62    pub moniker: ExtendedMoniker,
63    pub url: FlyStr,
64    pub rolled_out_logs: u64,
65}
66
67#[cfg(fuchsia_api_level_less_than = "HEAD")]
68fn parse_archivist_args<'a>(
69    builder: LogsDataBuilder,
70    _input: &'a Record<'a>,
71) -> Result<(LogsDataBuilder, usize), MessageError> {
72    Ok((builder, 0))
73}
74
75#[cfg(fuchsia_api_level_at_least = "HEAD")]
76fn parse_archivist_args<'a>(
77    mut builder: LogsDataBuilder,
78    input: &'a Record<'a>,
79) -> Result<(LogsDataBuilder, usize), MessageError> {
80    let mut archivist_argument_count = 0;
81    for argument in input.arguments.iter().rev() {
82        // If Archivist records are expected, they should always be at the end.
83        // If we see a non-archivist record, we can stop looking.
84        match argument {
85            Argument::Other { name, value } => {
86                if name == fdiagnostics::COMPONENT_URL_ARG_NAME {
87                    if let Value::Text(url) = value {
88                        builder = builder.set_url(Some(FlyStr::new(url.as_ref())));
89                        archivist_argument_count += 1;
90                        continue;
91                    }
92                } else if name == fdiagnostics::MONIKER_ARG_NAME {
93                    if let Value::Text(moniker) = value {
94                        builder = builder.set_moniker(ExtendedMoniker::parse_str(moniker)?);
95                        archivist_argument_count += 1;
96                        continue;
97                    }
98                } else if name == fdiagnostics::ROLLED_OUT_ARG_NAME
99                    && let Value::UnsignedInt(count) = value
100                {
101                    builder = builder.set_rolled_out(*count);
102                    archivist_argument_count += 1;
103                    continue;
104                }
105            }
106            _ => break,
107        }
108    }
109    Ok((builder, archivist_argument_count))
110}
111
112pub fn parse_logs_data<'a>(
113    input: &'a Record<'a>,
114    source: Option<ExtendedMetadata>,
115    rolled_out: u64,
116) -> Result<LogsData, MessageError> {
117    let (raw_severity, severity) = Severity::parse_exact(input.severity);
118    let has_attribution = source.is_some();
119
120    let (maybe_moniker, maybe_url) =
121        source.map(|value| (Some(value.moniker), Some(value.url))).unwrap_or((None, None));
122
123    let mut builder = LogsDataBuilder::new(BuilderArgs {
124        component_url: maybe_url,
125        moniker: maybe_moniker.unwrap_or(ExtendedMoniker::ComponentInstance(
126            Moniker::parse_str("placeholder").unwrap(),
127        )),
128        severity,
129        timestamp: Timestamp::from_nanos(input.timestamp.into_nanos()),
130    });
131
132    if rolled_out > 0 {
133        builder = builder.set_rolled_out(rolled_out);
134    }
135
136    if let Some(raw_severity) = raw_severity {
137        builder = builder.set_raw_severity(raw_severity);
138    }
139    let archivist_argument_count = if has_attribution {
140        0
141    } else {
142        let (new_builder, count) = parse_archivist_args(builder, input)?;
143        builder = new_builder;
144        count
145    };
146
147    for argument in input.arguments.iter().take(input.arguments.len() - archivist_argument_count) {
148        match argument {
149            Argument::Tag(tag) => {
150                builder = builder.add_tag(tag.as_ref());
151            }
152            Argument::Pid(pid) => {
153                builder = builder.set_pid(pid.raw_koid());
154            }
155            Argument::Tid(tid) => {
156                builder = builder.set_tid(tid.raw_koid());
157            }
158            Argument::Dropped(dropped) => {
159                builder = builder.set_dropped(*dropped);
160            }
161            Argument::File(file) => {
162                builder = builder.set_file(file.as_ref());
163            }
164            Argument::Line(line) => {
165                builder = builder.set_line(*line);
166            }
167            Argument::Message(msg) => {
168                builder = builder.set_message(msg.as_ref());
169            }
170            Argument::Other { value, name } => {
171                let name = LogsField::Other(name.to_string());
172                builder = builder.add_key(match value {
173                    Value::SignedInt(v) => LogsProperty::Int(name, *v),
174                    Value::UnsignedInt(v) => LogsProperty::Uint(name, *v),
175                    Value::Floating(v) => LogsProperty::Double(name, *v),
176                    Value::Text(v) => LogsProperty::String(name, v.to_string()),
177                    Value::Boolean(v) => LogsProperty::Bool(name, *v),
178                })
179            }
180        }
181    }
182
183    Ok(builder.build())
184}
185
186/// A stateful parser that reconstructs fully attributed `LogsData` records from a stream of
187/// FXT log packets.
188///
189/// # Background & Architecture
190///
191/// In the Fuchsia Trace Format (FXT) structured logging protocol, log attribution metadata is
192/// separated from the actual log payload to optimize transmission overhead. Instead of repeating
193/// the full moniker and component URL on every log record, the system transmits two distinct
194/// types of records:
195///
196/// 1. **Manifest/Control Records**: Sent with the `LOG_CONTROL_BIT` set. These records map a
197///    numeric base tag ID to component identity metadata (`ExtendedMoniker` and URL).
198/// 2. **Legacy Log Records**: Contain the message content, severity, timestamp, and
199///    arguments, along with a tag ID indicating which component produced the log.
200///
201/// # Stateful Parsing
202///
203/// `MessageParser` maintains an internal `tag_map` to track the active association between
204/// numeric tag IDs and their component identity (`ExtendedMetadata`).
205///
206/// - When parsing a manifest record (`is_control == true`), `MessageParser` updates its state
207///   mapping for the derived base tag. If the record also reports rolled out (dropped) logs, a
208///   `LogsData` payload representing those dropped logs is returned. Otherwise, it registers
209///   the attribution mapping and returns `Ok((None, remaining))`.
210/// - When parsing a legacy log record (`is_control == false`), `MessageParser` resolves the
211///   record's tag to retrieve the component's identity from the internal mapping, constructing
212///   a fully attributed `LogsData` containing the correct component moniker and URL.
213#[derive(Default)]
214pub struct MessageParser {
215    tag_map: HashMap<u32, ExtendedMetadata>,
216}
217
218pub trait MessageFormatter<'a> {
219    type Result;
220
221    fn format(
222        &mut self,
223        record: &Record<'a>,
224        metadata: Option<ExtendedMetadata>,
225    ) -> Result<Self::Result, MessageError>;
226}
227
228#[derive(Default)]
229pub struct RustMessageFormatter;
230
231impl<'a> MessageFormatter<'a> for RustMessageFormatter {
232    type Result = Data<Logs>;
233
234    fn format(
235        &mut self,
236        record: &Record<'a>,
237        metadata: Option<ExtendedMetadata>,
238    ) -> Result<Self::Result, MessageError> {
239        let rolled_out = metadata.as_ref().map(|value| value.rolled_out_logs).unwrap_or(0);
240        parse_logs_data(record, metadata, rolled_out)
241    }
242}
243
244impl MessageParser {
245    /// Parses the next log record from the given `bytes`.
246    ///
247    /// This function can handle both standard log records and "Archivist" manifest records.
248    /// Archivist records update an internal map used to attribute subsequent log records.
249    ///
250    /// # Arguments
251    ///
252    /// * `bytes`: A byte slice containing one or more log records.
253    ///
254    /// # Returns
255    ///
256    /// A `Result` containing:
257    /// * `Ok((Option<LogsData>, &[u8]))`: A tuple where the first element is `Some(LogsData)`
258    ///   if a log message was parsed, or `None` if it was an Archivist manifest record. The
259    ///   second element is the remaining slice of bytes after parsing the record.
260    /// * `Err(MessageError)`: An error if parsing failed.
261    pub fn parse_next<'a, F: MessageFormatter<'a>>(
262        &mut self,
263        bytes: &'a [u8],
264        mut formatter: F,
265    ) -> Result<(Option<F::Result>, &'a [u8]), MessageError> {
266        if bytes.len() < 8 {
267            return Err(MessageError::ShortRead { len: bytes.len() });
268        }
269        let header_bytes: [u8; 8] = bytes[0..8].try_into().unwrap();
270        let header_val = u64::from_le_bytes(header_bytes);
271        let header = Header(header_val);
272        let tag = header.tag();
273        let base_tag = tag & !LOG_CONTROL_BIT;
274        let is_control = (tag & LOG_CONTROL_BIT) != 0;
275
276        let (input, remaining) = diagnostics_log_encoding::parse::parse_record(bytes)?;
277
278        if is_control {
279            let mut moniker = None;
280            let mut url = None;
281            let mut rolled_out = None;
282            for arg in &input.arguments {
283                if arg.name() == MONIKER
284                    && let Value::Text(v) = arg.value()
285                {
286                    moniker = Some(v);
287                } else if arg.name() == URL
288                    && let Value::Text(v) = arg.value()
289                {
290                    url = Some(v);
291                } else if arg.name() == ROLLED_OUT
292                    && let Value::UnsignedInt(v) = arg.value()
293                {
294                    rolled_out = Some(v);
295                }
296            }
297            if let Some(count) = rolled_out {
298                let mut metadata =
299                    self.tag_map.get(&base_tag).cloned().unwrap_or_else(|| ExtendedMetadata {
300                        moniker: diagnostics_data::ExtendedMoniker::ComponentInstance(
301                            moniker::Moniker::parse_str("/UNKNOWN").unwrap(),
302                        ),
303                        url: flyweights::FlyStr::new(ARCHIVIST_URL),
304                        rolled_out_logs: count,
305                    });
306                metadata.rolled_out_logs = count;
307                let data = formatter.format(&input, Some(metadata))?;
308                return Ok((Some(data), remaining));
309            }
310            if let (Some(m), Some(u)) = (moniker, url)
311                && let Ok(extended_moniker) = ExtendedMoniker::parse_str(&m)
312            {
313                self.tag_map.insert(
314                    base_tag,
315                    ExtendedMetadata {
316                        moniker: extended_moniker,
317                        url: FlyStr::new(u),
318                        rolled_out_logs: 0,
319                    },
320                );
321            }
322            Ok((None, remaining))
323        } else {
324            let metadata = self.tag_map.get(&base_tag).cloned();
325            let data = formatter.format(&input, metadata)?;
326            Ok((Some(data), remaining))
327        }
328    }
329}
330
331/// Constructs a `LogsData` from the provided bytes, assuming the bytes
332/// are a single FXT log record with a potentially extended metadata section.
333/// [log encoding] https://fuchsia.dev/fuchsia-src/reference/platform-spec/diagnostics/logs-encoding
334pub fn from_extended_record(bytes: &[u8]) -> Result<(LogsData, &[u8]), MessageError> {
335    let (input, remaining) = diagnostics_log_encoding::parse::parse_record(bytes)?;
336    let (source, new_remaining, rolled_out_logs) = if remaining.len() >= 16 {
337        let moniker_len = u32::from_le_bytes(remaining[0..4].try_into().unwrap()) as usize;
338        let component_url_len = u32::from_le_bytes(remaining[4..8].try_into().unwrap()) as usize;
339        let rolled_out_logs = u64::from_le_bytes(remaining[8..16].try_into().unwrap());
340        let mut offset = 16;
341
342        // NOTE: In the extended metadata format, string fields (moniker and URL) are
343        // 8-byte word aligned with trailing zero padding.
344        // `(len + 7) & !7` rounds up `len` to the nearest multiple of 8.
345        // This arithmetic is overflow-safe on 64-bit platforms because moniker_len and
346        // component_url_len are originally 32-bit integers (u32).
347        let moniker_padded_len = (moniker_len + 7) & !7;
348        let component_url_padded_len = (component_url_len + 7) & !7;
349        let moniker_padded_end = offset + moniker_padded_len;
350        let url_padded_end = moniker_padded_end + component_url_padded_len;
351        if url_padded_end > remaining.len() {
352            return Err(MessageError::OutOfBounds);
353        }
354        let moniker = str::from_utf8(&remaining[offset..offset + moniker_len])?;
355        offset += moniker_padded_len;
356        let url = str::from_utf8(&remaining[offset..offset + component_url_len])?;
357        offset += component_url_padded_len;
358        (
359            Some(ExtendedMetadata {
360                moniker: ExtendedMoniker::parse_str(moniker)?,
361                url: FlyStr::new(url),
362                rolled_out_logs: 0,
363            }),
364            &remaining[offset..],
365            rolled_out_logs,
366        )
367    } else {
368        (None, remaining, 0)
369    };
370    let record = parse_logs_data(&input, source, rolled_out_logs)?;
371    Ok((record, new_remaining))
372}
373
374/// Constructs a `LogsData` from the provided bytes, assuming the bytes
375/// are in the format specified as in the [log encoding].
376///
377/// [log encoding] https://fuchsia.dev/fuchsia-src/development/logs/encodings
378pub fn from_structured(source: MonikerWithUrl, bytes: &[u8]) -> Result<LogsData, MessageError> {
379    let (input, _remaining) = diagnostics_log_encoding::parse::parse_record(bytes)?;
380    let record = parse_logs_data(
381        &input,
382        Some(ExtendedMetadata { moniker: source.moniker, url: source.url, rolled_out_logs: 0 }),
383        0,
384    )?;
385    Ok(record)
386}
387
388#[derive(Clone, Debug, Eq, PartialEq)]
389pub struct LoggerMessage {
390    pub timestamp: zx::BootInstant,
391    pub raw_severity: u8,
392    pub pid: u64,
393    pub tid: u64,
394    pub size_bytes: usize,
395    pub dropped_logs: u64,
396    pub message: Box<str>,
397    pub tags: Vec<Box<str>>,
398}
399
400/// Parse the provided buffer as if it implements the [logger/syslog wire format].
401///
402/// Note that this is distinct from the parsing we perform for the debuglog log, which also
403/// takes a `&[u8]` and is why we don't implement this as `TryFrom`.
404///
405/// [logger/syslog wire format]: https://fuchsia.googlesource.com/fuchsia/+/HEAD/zircon/system/ulib/syslog/include/lib/syslog/wire_format.h
406impl TryFrom<&[u8]> for LoggerMessage {
407    type Error = MessageError;
408
409    fn try_from(bytes: &[u8]) -> Result<Self, Self::Error> {
410        if bytes.len() < MIN_PACKET_SIZE {
411            return Err(MessageError::ShortRead { len: bytes.len() });
412        }
413
414        let terminator = bytes[bytes.len() - 1];
415        if terminator != 0 {
416            return Err(MessageError::NotNullTerminated { terminator });
417        }
418
419        let pid = LittleEndian::read_u64(&bytes[..8]);
420        let tid = LittleEndian::read_u64(&bytes[8..16]);
421        let timestamp = zx::BootInstant::from_nanos(LittleEndian::read_i64(&bytes[16..24]));
422
423        let raw_severity = LittleEndian::read_i32(&bytes[24..28]);
424        let raw_severity = if raw_severity > (u8::MAX as i32) {
425            u8::MAX
426        } else if raw_severity < 0 {
427            0
428        } else {
429            u8::try_from(raw_severity).unwrap()
430        };
431        let dropped_logs = LittleEndian::read_u32(&bytes[28..METADATA_SIZE]) as u64;
432
433        // start reading tags after the header
434        let mut cursor = METADATA_SIZE;
435        let mut tag_len = bytes[cursor] as usize;
436        let mut tags = Vec::new();
437        while tag_len != 0 {
438            if tags.len() == MAX_TAGS {
439                return Err(MessageError::TooManyTags);
440            }
441
442            if tag_len > MAX_TAG_LEN - 1 {
443                return Err(MessageError::TagTooLong { index: tags.len(), len: tag_len });
444            }
445
446            if (cursor + tag_len + 1) > bytes.len() {
447                return Err(MessageError::OutOfBounds);
448            }
449
450            let tag_start = cursor + 1;
451            let tag_end = tag_start + tag_len;
452            let tag = String::from_utf8_lossy(&bytes[tag_start..tag_end]);
453            tags.push(tag.into());
454
455            cursor = tag_end;
456            tag_len = bytes[cursor] as usize;
457        }
458
459        let msg_start = cursor + 1;
460        let mut msg_end = cursor + 1;
461        while msg_end < bytes.len() {
462            if bytes[msg_end] > 0 {
463                msg_end += 1;
464                continue;
465            }
466            let message = String::from_utf8_lossy(&bytes[msg_start..msg_end]).into_owned();
467            let message_len = message.len();
468            let result = LoggerMessage {
469                timestamp,
470                raw_severity,
471                message: message.into_boxed_str(),
472                pid,
473                tid,
474                dropped_logs,
475                tags,
476                size_bytes: cursor + message_len + 1,
477            };
478            return Ok(result);
479        }
480
481        Err(MessageError::OutOfBounds)
482    }
483}
484
485#[allow(non_camel_case_types)]
486pub type fx_log_severity_t = c_int;
487
488#[repr(C)]
489#[derive(Debug, Copy, Clone, Default, Eq, PartialEq)]
490pub struct fx_log_metadata_t {
491    pub pid: zx::sys::zx_koid_t,
492    pub tid: zx::sys::zx_koid_t,
493    pub time: zx::sys::zx_time_t,
494    pub severity: fx_log_severity_t,
495    pub dropped_logs: u32,
496}
497
498#[repr(C)]
499#[derive(Clone)]
500pub struct fx_log_packet_t {
501    pub metadata: fx_log_metadata_t,
502    // Contains concatenated tags and message and a null terminating character at
503    // the end.
504    // char(tag_len) + "tag1" + char(tag_len) + "tag2\0msg\0"
505    pub data: [c_char; MAX_DATAGRAM_LEN - METADATA_SIZE],
506}
507
508impl Default for fx_log_packet_t {
509    fn default() -> fx_log_packet_t {
510        fx_log_packet_t {
511            data: [0; MAX_DATAGRAM_LEN - METADATA_SIZE],
512            metadata: Default::default(),
513        }
514    }
515}
516
517impl fx_log_packet_t {
518    /// This struct has no padding bytes, but we can't use zerocopy because it needs const
519    /// generics to support arrays this large.
520    pub fn as_bytes(&self) -> &[u8] {
521        unsafe {
522            std::slice::from_raw_parts(
523                (self as *const Self) as *const u8,
524                mem::size_of::<fx_log_packet_t>(),
525            )
526        }
527    }
528
529    /// Fills data with a single value for defined region.
530    pub fn fill_data(&mut self, region: std::ops::Range<usize>, with: c_char) {
531        self.data[region].iter_mut().for_each(|c| *c = with);
532    }
533
534    /// Copies bytes to data at specifies offset.
535    pub fn add_data<T: std::convert::TryInto<c_char> + Copy>(&mut self, offset: usize, bytes: &[T])
536    where
537        <T as std::convert::TryInto<c_char>>::Error: std::fmt::Debug,
538    {
539        self.data[offset..(offset + bytes.len())]
540            .iter_mut()
541            .enumerate()
542            .for_each(|(i, x)| *x = bytes[i].try_into().unwrap());
543    }
544}
diagnostics_message/lib.rs

diagnostics_message/
lib.rs
