attribution_processing/
digest.rs

1// Copyright 2025 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::fplugin::Vmo;
6use crate::{ResourceEnumerator, ResourcesVisitor, ZXName};
7use anyhow::Result;
8use regex::bytes::Regex;
9use serde::de::Error;
10use serde::{Deserialize, Deserializer, Serialize};
11use std::collections::HashMap;
12use std::collections::hash_map::Entry::Occupied;
13use {fidl_fuchsia_kernel as fkernel, fidl_fuchsia_memory_attribution_plugin as fplugin};
14
15const UNDIGESTED: &str = "Undigested";
16const ORPHANED: &str = "Orphaned";
17const KERNEL: &str = "Kernel";
18const FREE: &str = "Free";
19const PAGER_TOTAL: &str = "[Addl]PagerTotal";
20const PAGER_NEWEST: &str = "[Addl]PagerNewest";
21const PAGER_OLDEST: &str = "[Addl]PagerOldest";
22const DISCARDABLE_LOCKED: &str = "[Addl]DiscardableLocked";
23const DISCARDABLE_UNLOCKED: &str = "[Addl]DiscardableUnlocked";
24const ZRAM_COMPRESSED_BYTES: &str = "[Addl]ZramCompressedBytes";
25
26/// Represents a specification for aggregating memory usage in meaningful groups.
27///
28/// `name` represents the meaningful name of the group; grouping is done based on process and VMO
29/// names.
30///
31// Note: This needs to mirror `//src/lib/assembly/memory_buckets/src/memory_buckets.rs`, but cannot
32// reuse it directly because it is an host-only library.
33#[derive(Clone, Debug, Deserialize)]
34pub struct BucketDefinition {
35    pub name: String,
36    #[serde(deserialize_with = "deserialize_regex")]
37    pub process: Option<Regex>,
38    #[serde(deserialize_with = "deserialize_regex")]
39    pub vmo: Option<Regex>,
40    pub event_code: u64,
41}
42
43impl BucketDefinition {
44    /// Tests whether a process matches this bucket's definition, based on its name.
45    fn process_match(&self, process: &ZXName) -> bool {
46        self.process.as_ref().map_or(true, |p| p.is_match(process.as_bstr()))
47    }
48
49    /// Tests whether a VMO matches this bucket's definition, based on its name.
50    fn vmo_match(&self, vmo: &ZXName) -> bool {
51        self.vmo.as_ref().map_or(true, |v| v.is_match(vmo.as_bstr()))
52    }
53}
54
55// Teach serde to deserialize an optional regex.
56fn deserialize_regex<'de, D>(d: D) -> Result<Option<Regex>, D::Error>
57where
58    D: Deserializer<'de>,
59{
60    // Deserialize as Option<&str>
61    Option::<String>::deserialize(d)
62        // If the parsing failed, return the error, otherwise transform the value
63        .and_then(|os| {
64            os
65                // If there is a value, try to parse it as a Regex.
66                .map(|s| {
67                    Regex::new(&s)
68                        // If the regex compilation failed, wrap the error in the error type expected
69                        // by serde.
70                        .map_err(D::Error::custom)
71                })
72                // If there was a value but it failed to compile, return an error, otherwise return
73                // the potentially parsed option.
74                .transpose()
75        })
76}
77
78/// Aggregates bytes in categories with human readable names.
79#[derive(Clone, Debug, PartialEq, Eq, Serialize)]
80pub struct Bucket {
81    pub name: String,
82    pub size: u64,
83}
84/// Contains a view of the system's memory usage, aggregated in groups called buckets, which are
85/// configurable.
86#[derive(Debug, Default, PartialEq, Eq, Serialize)]
87pub struct Digest {
88    pub buckets: Vec<Bucket>,
89}
90
91/// Compute a bucket digest as the Jobs->Processes->VMOs tree is traversed.
92struct DigestComputer<'a> {
93    // Ordered pair with a bucket specification, and the current bucket result.
94    buckets: Vec<(&'a BucketDefinition, Bucket)>,
95    // Set of VMOs what didn't fell in any bucket.
96    undigested_vmos: HashMap<zx_types::zx_koid_t, (Vmo, ZXName)>,
97}
98
99impl<'a> DigestComputer<'a> {
100    fn new(bucket_definitions: &'a [BucketDefinition]) -> DigestComputer<'a> {
101        DigestComputer {
102            buckets: bucket_definitions
103                .iter()
104                .map(|def| (def, Bucket { name: def.name.clone(), size: 0 }))
105                .collect(),
106            undigested_vmos: Default::default(),
107        }
108    }
109}
110
111impl ResourcesVisitor for DigestComputer<'_> {
112    fn on_job(
113        &mut self,
114        _job_koid: zx_types::zx_koid_t,
115        _job_name: &ZXName,
116        _job: fplugin::Job,
117    ) -> Result<(), zx_status::Status> {
118        Ok(())
119    }
120
121    fn on_process(
122        &mut self,
123        _process_koid: zx_types::zx_koid_t,
124        process_name: &ZXName,
125        process: fplugin::Process,
126    ) -> Result<(), zx_status::Status> {
127        for (bucket_definition, bucket) in self.buckets.iter_mut() {
128            if bucket_definition.process_match(process_name) {
129                for koid in process.vmos.iter().flatten() {
130                    bucket.size += match self.undigested_vmos.entry(*koid) {
131                        Occupied(e) => {
132                            let (_vmo, name) = e.get();
133                            if bucket_definition.vmo_match(&name) {
134                                let (_, (vmo, _name)) = e.remove_entry();
135                                vmo.scaled_committed_bytes.unwrap_or_default()
136                            } else {
137                                0
138                            }
139                        }
140                        _ => 0,
141                    };
142                }
143            }
144        }
145        Ok(())
146    }
147
148    fn on_vmo(
149        &mut self,
150        vmo_koid: zx_types::zx_koid_t,
151        vmo_name: &ZXName,
152        vmo: fplugin::Vmo,
153    ) -> Result<(), zx_status::Status> {
154        self.undigested_vmos.insert(vmo_koid, (vmo, vmo_name.clone()));
155        Ok(())
156    }
157}
158
159impl Digest {
160    /// Given means to query the system for memory usage, and a specification, this function
161    /// aggregates the current memory usage into human displayable units we call buckets.
162    pub fn compute(
163        resource_enumerator: &impl ResourceEnumerator,
164        kmem_stats: &fkernel::MemoryStats,
165        kmem_stats_compression: &fkernel::MemoryStatsCompression,
166        bucket_definitions: &[BucketDefinition],
167    ) -> Result<Digest> {
168        let mut digest_visitor = DigestComputer::new(bucket_definitions);
169        resource_enumerator.for_each_resource(&mut digest_visitor)?;
170        let mut buckets: Vec<Bucket> =
171            digest_visitor.buckets.drain(..).map(|(_, bucket)| bucket).collect();
172
173        // This bucket contains the total size of the known VMOs that have not been covered
174        // by any other bucket.
175        let undigested = Bucket {
176            name: UNDIGESTED.to_string(),
177            size: digest_visitor
178                .undigested_vmos
179                .values()
180                .filter_map(|vmo| vmo.0.scaled_committed_bytes)
181                .sum(),
182        };
183
184        let total_vmo_size: u64 =
185            undigested.size + buckets.iter().map(|Bucket { size, .. }| size).sum::<u64>();
186
187        // Extend the configured aggregation with a number of additional, occasionally useful meta
188        // aggregations.
189        buckets.extend([
190            undigested,
191            // This bucket accounts for VMO bytes that have been allocated by the kernel, but not
192            // claimed by any VMO (anymore).
193            Bucket {
194                name: ORPHANED.to_string(),
195                size: kmem_stats.vmo_bytes.unwrap_or(0).saturating_sub(total_vmo_size),
196            },
197            // This bucket aggregates overall kernel memory usage.
198            Bucket {
199                name: KERNEL.to_string(),
200                size: (|| {
201                    Some(
202                        kmem_stats.wired_bytes?
203                            + kmem_stats.total_heap_bytes?
204                            + kmem_stats.mmu_overhead_bytes?
205                            + kmem_stats.ipc_bytes?
206                            + kmem_stats.other_bytes?,
207                    )
208                })()
209                .unwrap_or(0),
210            },
211            // This bucket contains the amount of free memory in the system.
212            Bucket { name: FREE.to_string(), size: kmem_stats.free_bytes.unwrap_or(0) },
213            // Those buckets contain pager related information.
214            Bucket {
215                name: PAGER_TOTAL.to_string(),
216                size: kmem_stats.vmo_reclaim_total_bytes.unwrap_or(0),
217            },
218            Bucket {
219                name: PAGER_NEWEST.to_string(),
220                size: kmem_stats.vmo_reclaim_newest_bytes.unwrap_or(0),
221            },
222            Bucket {
223                name: PAGER_OLDEST.to_string(),
224                size: kmem_stats.vmo_reclaim_oldest_bytes.unwrap_or(0),
225            },
226            // Those buckets account for discardable memory.
227            Bucket {
228                name: DISCARDABLE_LOCKED.to_string(),
229                size: kmem_stats.vmo_discardable_locked_bytes.unwrap_or(0),
230            },
231            Bucket {
232                name: DISCARDABLE_UNLOCKED.to_string(),
233                size: kmem_stats.vmo_discardable_unlocked_bytes.unwrap_or(0),
234            },
235            // This bucket accounts for compressed memory.
236            Bucket {
237                name: ZRAM_COMPRESSED_BYTES.to_string(),
238                size: kmem_stats_compression.compressed_storage_bytes.unwrap_or(0),
239            },
240        ]);
241        Ok(Digest { buckets })
242    }
243}
244
245#[cfg(test)]
246mod tests {
247    use super::*;
248    use crate::{
249        Attribution, AttributionData, GlobalPrincipalIdentifier, Principal, PrincipalDescription,
250        PrincipalType, Resource, ResourceReference,
251    };
252    use fidl_fuchsia_memory_attribution_plugin as fplugin;
253
254    fn get_attribution_data() -> AttributionData {
255        let attribution_data = AttributionData {
256            principals_vec: vec![Principal {
257                identifier: GlobalPrincipalIdentifier::new_for_test(1),
258                description: Some(PrincipalDescription::Component("principal".to_owned())),
259                principal_type: PrincipalType::Runnable,
260                parent: None,
261            }],
262            resources_vec: vec![
263                Resource {
264                    koid: 10,
265                    name_index: 0,
266                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
267                        parent: None,
268                        private_committed_bytes: Some(1024),
269                        private_populated_bytes: Some(2048),
270                        scaled_committed_bytes: Some(512),
271                        scaled_populated_bytes: Some(2048),
272                        total_committed_bytes: Some(1024),
273                        total_populated_bytes: Some(2048),
274                        ..Default::default()
275                    }),
276                },
277                Resource {
278                    koid: 20,
279                    name_index: 1,
280                    resource_type: fplugin::ResourceType::Vmo(fplugin::Vmo {
281                        parent: None,
282                        private_committed_bytes: Some(1024),
283                        private_populated_bytes: Some(2048),
284                        scaled_committed_bytes: Some(512),
285                        scaled_populated_bytes: Some(2048),
286                        total_committed_bytes: Some(1024),
287                        total_populated_bytes: Some(2048),
288                        ..Default::default()
289                    }),
290                },
291                Resource {
292                    koid: 30,
293                    name_index: 1,
294                    resource_type: fplugin::ResourceType::Process(fplugin::Process {
295                        vmos: Some(vec![10, 20]),
296                        ..Default::default()
297                    }),
298                },
299            ],
300            resource_names: vec![
301                ZXName::try_from_bytes(b"resource").unwrap(),
302                ZXName::try_from_bytes(b"matched").unwrap(),
303            ],
304            attributions: vec![Attribution {
305                source: GlobalPrincipalIdentifier::new_for_test(1),
306                subject: GlobalPrincipalIdentifier::new_for_test(1),
307                resources: vec![ResourceReference::KernelObject(10)],
308            }],
309        };
310        attribution_data
311    }
312
313    fn get_kernel_stats() -> (fkernel::MemoryStats, fkernel::MemoryStatsCompression) {
314        (
315            fkernel::MemoryStats {
316                total_bytes: Some(1),
317                free_bytes: Some(2),
318                wired_bytes: Some(3),
319                total_heap_bytes: Some(4),
320                free_heap_bytes: Some(5),
321                vmo_bytes: Some(10000),
322                mmu_overhead_bytes: Some(7),
323                ipc_bytes: Some(8),
324                other_bytes: Some(9),
325                free_loaned_bytes: Some(10),
326                cache_bytes: Some(11),
327                slab_bytes: Some(12),
328                zram_bytes: Some(13),
329                vmo_reclaim_total_bytes: Some(14),
330                vmo_reclaim_newest_bytes: Some(15),
331                vmo_reclaim_oldest_bytes: Some(16),
332                vmo_reclaim_disabled_bytes: Some(17),
333                vmo_discardable_locked_bytes: Some(18),
334                vmo_discardable_unlocked_bytes: Some(19),
335                ..Default::default()
336            },
337            fkernel::MemoryStatsCompression {
338                uncompressed_storage_bytes: Some(20),
339                compressed_storage_bytes: Some(21),
340                compressed_fragmentation_bytes: Some(22),
341                compression_time: Some(23),
342                decompression_time: Some(24),
343                total_page_compression_attempts: Some(25),
344                failed_page_compression_attempts: Some(26),
345                total_page_decompressions: Some(27),
346                compressed_page_evictions: Some(28),
347                eager_page_compressions: Some(29),
348                memory_pressure_page_compressions: Some(30),
349                critical_memory_page_compressions: Some(31),
350                pages_decompressed_unit_ns: Some(32),
351                pages_decompressed_within_log_time: Some([40, 41, 42, 43, 44, 45, 46, 47]),
352                ..Default::default()
353            },
354        )
355    }
356
357    #[test]
358    fn test_digest_no_definitions() {
359        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
360        let digest = Digest::compute(
361            &get_attribution_data(),
362            &kernel_stats,
363            &kernel_stats_compression,
364            &vec![],
365        )
366        .unwrap();
367        let expected_buckets = vec![
368            Bucket { name: UNDIGESTED.to_string(), size: 1024 }, // The two VMOs are unmatched, 512 + 512
369            // No matched VMOs, one UNDIGESTED VMO => 10000 - 1024 = 8976
370            Bucket { name: ORPHANED.to_string(), size: 8976 },
371            // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
372            Bucket { name: KERNEL.to_string(), size: 31 },
373            Bucket { name: FREE.to_string(), size: 2 },
374            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
375            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
376            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
377            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
378            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
379            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
380        ];
381
382        assert_eq!(digest.buckets, expected_buckets);
383    }
384
385    #[test]
386    fn test_digest_with_matching_vmo() -> Result<(), anyhow::Error> {
387        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
388        let digest = Digest::compute(
389            &get_attribution_data(),
390            &kernel_stats,
391            &kernel_stats_compression,
392            &vec![BucketDefinition {
393                name: "matched".to_string(),
394                process: None,
395                vmo: Some(Regex::new("matched")?),
396                event_code: Default::default(),
397            }],
398        )
399        .unwrap();
400        let expected_buckets = vec![
401            Bucket { name: "matched".to_string(), size: 512 }, // One VMO is matched, the other is not
402            Bucket { name: UNDIGESTED.to_string(), size: 512 }, // One unmatched VMO
403            // One matched VMO, one unmatched VMO //=> 10000 - 512 - 512 = 8976
404            Bucket { name: ORPHANED.to_string(), size: 8976 },
405            // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
406            Bucket { name: KERNEL.to_string(), size: 31 },
407            Bucket { name: FREE.to_string(), size: 2 },
408            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
409            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
410            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
411            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
412            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
413            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
414        ];
415
416        assert_eq!(digest.buckets, expected_buckets);
417        Ok(())
418    }
419
420    #[test]
421    fn test_digest_with_matching_process() -> Result<(), anyhow::Error> {
422        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
423        let digest = Digest::compute(
424            &get_attribution_data(),
425            &kernel_stats,
426            &kernel_stats_compression,
427            &vec![BucketDefinition {
428                name: "matched".to_string(),
429                process: Some(Regex::new("matched")?),
430                vmo: None,
431                event_code: Default::default(),
432            }],
433        )
434        .unwrap();
435        let expected_buckets = vec![
436            Bucket { name: "matched".to_string(), size: 1024 }, // Both VMOs are matched => 512 + 512 = 1024
437            Bucket { name: UNDIGESTED.to_string(), size: 0 },   // No unmatched VMO
438            Bucket { name: ORPHANED.to_string(), size: 8976 }, // Two matched VMO => 10000 - 512 - 512 = 8976
439            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
440            Bucket { name: FREE.to_string(), size: 2 },
441            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
442            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
443            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
444            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
445            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
446            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
447        ];
448
449        assert_eq!(digest.buckets, expected_buckets);
450        Ok(())
451    }
452
453    #[test]
454    fn test_digest_with_matching_process_and_vmo() -> Result<(), anyhow::Error> {
455        let (kernel_stats, kernel_stats_compression) = get_kernel_stats();
456        let digest = Digest::compute(
457            &get_attribution_data(),
458            &kernel_stats,
459            &kernel_stats_compression,
460            &vec![BucketDefinition {
461                name: "matched".to_string(),
462                process: Some(Regex::new("matched")?),
463                vmo: Some(Regex::new("matched")?),
464                event_code: Default::default(),
465            }],
466        )
467        .unwrap();
468        let expected_buckets = vec![
469            Bucket { name: "matched".to_string(), size: 512 }, // One VMO is matched, the other is not
470            Bucket { name: UNDIGESTED.to_string(), size: 512 }, // One unmatched VMO
471            // One matched VMO, one unmatched VMO => 10000 - 512 - 512 = 8976
472            Bucket { name: ORPHANED.to_string(), size: 8976 },
473            Bucket { name: KERNEL.to_string(), size: 31 }, // wired + heap + mmu + ipc + other => 3 + 4 + 7 + 8 + 9 = 31
474            Bucket { name: FREE.to_string(), size: 2 },
475            Bucket { name: PAGER_TOTAL.to_string(), size: 14 },
476            Bucket { name: PAGER_NEWEST.to_string(), size: 15 },
477            Bucket { name: PAGER_OLDEST.to_string(), size: 16 },
478            Bucket { name: DISCARDABLE_LOCKED.to_string(), size: 18 },
479            Bucket { name: DISCARDABLE_UNLOCKED.to_string(), size: 19 },
480            Bucket { name: ZRAM_COMPRESSED_BYTES.to_string(), size: 21 },
481        ];
482
483        assert_eq!(digest.buckets, expected_buckets);
484        Ok(())
485    }
486}
attribution_processing/digest.rs

attribution_processing/
digest.rs
