fxfs/
blob_metadata.rs

1// Copyright 2026 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::errors::FxfsError;
6use crate::lsm_tree::Query;
7use crate::lsm_tree::types::{ItemRef, LayerIterator};
8use crate::object_handle::ObjectHandle;
9use crate::object_store::object_record::{AttributeKey, ObjectKey, ObjectKeyData, ObjectValue};
10use crate::object_store::{
11    BLOB_MERKLE_ATTRIBUTE_ID, BLOB_METADATA_ATTRIBUTE_ID, DataObjectHandle,
12    FSVERITY_MERKLE_ATTRIBUTE_ID, HandleOwner,
13};
14use crate::serialized_types::{Versioned, VersionedLatest};
15use anyhow::{Context, Error};
16use fprint::TypeFingerprint;
17use fuchsia_merkle::{Hash, LeafHashCollector, MerkleVerifier};
18use serde::{Deserialize, Serialize};
19
20#[derive(Serialize, Deserialize, Debug)]
21pub struct BlobMetadataUnversioned {
22    pub hashes: Vec<[u8; 32]>,
23    pub chunk_size: u64,
24    pub compressed_offsets: Vec<u64>,
25    pub uncompressed_size: u64,
26}
27
28pub type BlobMetadata = BlobMetadataV53;
29pub type BlobFormat = BlobFormatV53;
30pub type MerkleLeaves = Vec<[u8; 32]>;
31
32impl BlobMetadata {
33    /// Reads the blob metadata from an attribute on `blob_object`. If the attribute doesn't exist
34    /// then it's assumed to be `BlobMetadata::empty()`.
35    pub async fn read_from<S: HandleOwner>(
36        blob_object: &DataObjectHandle<S>,
37    ) -> Result<Self, Error> {
38        let store = blob_object.store();
39        let layer_set = store.tree().layer_set();
40        let mut merger = layer_set.merger();
41        // A blob should never have both attributes and also should never have the fs-verity
42        // attribute which is ordered between them. Querying for `BLOB_MERKLE_ATTRIBUTE_ID` will
43        // have the iterator point to that attribute if it exists. If it doesn't exist then the
44        // iterator will point the next item which will be the `BLOB_METADATA_ATTRIBUTE_ID`
45        // attribute if it exists.
46        static_assertions::const_assert!(BLOB_MERKLE_ATTRIBUTE_ID < BLOB_METADATA_ATTRIBUTE_ID);
47        let key = ObjectKey::attribute(
48            blob_object.object_id(),
49            BLOB_MERKLE_ATTRIBUTE_ID,
50            AttributeKey::Attribute,
51        );
52        let iter = merger.query(Query::FullRange(&key)).await?;
53        match iter.get() {
54            Some(ItemRef {
55                key:
56                    ObjectKey {
57                        object_id,
58                        data:
59                            ObjectKeyData::Attribute(BLOB_MERKLE_ATTRIBUTE_ID, AttributeKey::Attribute),
60                    },
61                value,
62                ..
63            }) if *object_id == blob_object.object_id() => match value {
64                ObjectValue::Attribute { .. } => {
65                    let serialized_metadata = blob_object.read_attr_from_iter(iter).await?;
66                    let old_metadata: BlobMetadataUnversioned =
67                        bincode::deserialize_from(&*serialized_metadata)?;
68                    Ok(Self::from(old_metadata))
69                }
70                _ => Err(FxfsError::Inconsistent.into()),
71            },
72            Some(ItemRef {
73                key:
74                    ObjectKey {
75                        object_id,
76                        data:
77                            ObjectKeyData::Attribute(
78                                BLOB_METADATA_ATTRIBUTE_ID,
79                                AttributeKey::Attribute,
80                            ),
81                    },
82                value,
83                ..
84            }) if *object_id == blob_object.object_id() => match value {
85                ObjectValue::Attribute { .. } => {
86                    let serialized_metadata = blob_object.read_attr_from_iter(iter).await?;
87                    Ok(Self::deserialize_with_version(&mut &*serialized_metadata)?.0)
88                }
89                _ => Err(FxfsError::Inconsistent.into()),
90            },
91            Some(ItemRef {
92                key:
93                    ObjectKey {
94                        object_id,
95                        data:
96                            ObjectKeyData::Attribute(
97                                FSVERITY_MERKLE_ATTRIBUTE_ID,
98                                AttributeKey::Attribute,
99                            ),
100                    },
101                ..
102            }) if *object_id == blob_object.object_id() => {
103                // Blobs should not have the fs-verity attribute. This is explicitly checked because
104                // the fs-verity attribute is ordered between the 2 blob metadata attributes.
105                // `BLOB_MERKLE_ATTRIBUTE_ID` was queried for with the expectation of finding either
106                // blob attribute. Finding the fs-verity attribute could be hiding the
107                // `BLOB_METADATA_ATTRIBUTE_ID` attribute.
108                Err(FxfsError::Inconsistent.into())
109            }
110            // Neither attribute exists.
111            _ => Ok(Self::empty()),
112        }
113    }
114
115    /// Writes the metadata to the `BLOB_METADATA_ATTRIBUTE_ID` attribute on `blob_object`. If the
116    /// metadata is equal to `BlobMetadata::empty()` then the attribute isn't written.
117    pub async fn write_to<S: HandleOwner>(
118        &self,
119        blob_object: &DataObjectHandle<S>,
120    ) -> Result<(), Error> {
121        // Don't write the attribute when there's no metadata.
122        if self.is_empty() {
123            return Ok(());
124        }
125        let mut buf = Vec::new();
126        self.serialize_with_version(&mut buf)?;
127        blob_object
128            .write_attr(BLOB_METADATA_ATTRIBUTE_ID, &buf)
129            .await
130            .context("Failed to write blob metadata attribute.")
131    }
132
133    /// Returns the size of the serialized metadata. If the metadata is equal to
134    /// `BlobMetadata::empty()` then the metadata won't get written, so 0 is returned.
135    pub fn serialized_size(&self) -> Result<usize, Error> {
136        if self.is_empty() {
137            return Ok(0);
138        }
139        struct CountingWriter(usize);
140        impl std::io::Write for CountingWriter {
141            fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
142                self.0 += buf.len();
143                Ok(buf.len())
144            }
145            fn flush(&mut self) -> std::io::Result<()> {
146                Ok(())
147            }
148        }
149        let mut writer = CountingWriter(0);
150        self.serialize_with_version(&mut writer)?;
151        Ok(writer.0)
152    }
153
154    /// Consumes the metadata and turns it into a `MerkleVerifier`.
155    pub fn into_merkle_verifier(self, root: Hash) -> Result<MerkleVerifier, Error> {
156        let hashes = if self.merkle_leaves.is_empty() {
157            Box::new([root])
158        } else {
159            // The below code gets optimized down to just a `Vec::into_boxed_slice` on release
160            // builds because `Hash` is just a wrapper around `[u8; 32]`. There are 2 intermediate
161            // Vecs that still exist on the stack but the usage of them is optimized away. Their
162            // Drop impls still run which is just a `free` on a null pointer.
163            self.merkle_leaves.into_iter().map(Into::into).collect::<Box<[Hash]>>()
164        };
165        Ok(MerkleVerifier::new(root, hashes)?)
166    }
167
168    /// Constructs a `BlobMetadata` that is considered to be empty. The empty metadata does not get
169    /// written out as an attribute.
170    pub fn empty() -> Self {
171        // WARNING: The empty metadata doesn't get written to an attribute so it's meaning must not
172        // be changed across versions.
173        Self { merkle_leaves: Vec::new(), format: BlobFormatV53::Uncompressed }
174    }
175
176    fn is_empty(&self) -> bool {
177        *self == Self::empty()
178    }
179}
180
181#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, TypeFingerprint)]
182pub struct BlobMetadataV53 {
183    #[serde(with = "crate::zerocopy_serialization")]
184    pub merkle_leaves: MerkleLeaves,
185    pub format: BlobFormatV53,
186}
187
188impl Versioned for BlobMetadataV53 {
189    fn max_serialized_size() -> Option<u64> {
190        // There's no restriction on the size of the blob metadata.
191        None
192    }
193}
194
195impl From<BlobMetadataUnversioned> for BlobMetadataV53 {
196    fn from(old: BlobMetadataUnversioned) -> Self {
197        if old.compressed_offsets.is_empty() {
198            Self { merkle_leaves: old.hashes, format: BlobFormat::Uncompressed }
199        } else {
200            Self {
201                merkle_leaves: old.hashes,
202                format: BlobFormat::ChunkedZstd {
203                    uncompressed_size: old.uncompressed_size,
204                    chunk_size: old.chunk_size,
205                    compressed_offsets: old.compressed_offsets,
206                },
207            }
208        }
209    }
210}
211
212#[derive(Serialize, Deserialize, Debug, PartialEq, Eq, TypeFingerprint)]
213pub enum BlobFormatV53 {
214    Uncompressed,
215    ChunkedZstd { uncompressed_size: u64, chunk_size: u64, compressed_offsets: Vec<u64> },
216    ChunkedLz4 { uncompressed_size: u64, chunk_size: u64, compressed_offsets: Vec<u64> },
217}
218
219#[derive(Default)]
220pub struct BlobMetadataLeafHashCollector(MerkleLeaves);
221
222impl BlobMetadataLeafHashCollector {
223    pub fn new() -> Self {
224        Self(Vec::new())
225    }
226}
227
228impl LeafHashCollector for BlobMetadataLeafHashCollector {
229    type Output = (Hash, MerkleLeaves);
230
231    fn add_leaf_hash(&mut self, hash: Hash) {
232        self.0.push(hash.into())
233    }
234
235    fn complete(mut self, root: Hash) -> Self::Output {
236        // If the there's only 1 hash then it's the root and doesn't get stored in the metadata.
237        if self.0.len() == 1 {
238            debug_assert!(*root == self.0[0]);
239            self.0 = Vec::new();
240        }
241        (root, self.0)
242    }
243}
244
245#[cfg(test)]
246mod tests {
247    use super::BlobMetadata;
248    use crate::blob_metadata::{
249        BlobFormat, BlobMetadataLeafHashCollector, BlobMetadataUnversioned,
250    };
251    use crate::filesystem::{FxFilesystem, OpenFxFilesystem};
252    use crate::object_store::transaction::{LockKey, Options, lock_keys};
253    use crate::object_store::{
254        BLOB_MERKLE_ATTRIBUTE_ID, BLOB_METADATA_ATTRIBUTE_ID, DataObjectHandle, Directory,
255        FSVERITY_MERKLE_ATTRIBUTE_ID, HandleOptions, ObjectStore,
256    };
257    use assert_matches::assert_matches;
258    use fuchsia_merkle::MerkleRootBuilder;
259    use storage_device::DeviceHolder;
260    use storage_device::fake_device::FakeDevice;
261
262    const TEST_DEVICE_BLOCK_SIZE: u32 = 512;
263    const TEST_DEVICE_BLOCK_COUNT: u64 = 16 * 1024;
264    const TEST_OBJECT_NAME: &str = "foo";
265
266    async fn test_filesystem() -> OpenFxFilesystem {
267        let device =
268            DeviceHolder::new(FakeDevice::new(TEST_DEVICE_BLOCK_COUNT, TEST_DEVICE_BLOCK_SIZE));
269        FxFilesystem::new_empty(device).await.expect("new_empty failed")
270    }
271
272    async fn test_filesystem_and_empty_object() -> (OpenFxFilesystem, DataObjectHandle<ObjectStore>)
273    {
274        let fs = test_filesystem().await;
275        let store = fs.root_store();
276
277        let mut transaction = fs
278            .clone()
279            .new_transaction(
280                lock_keys![LockKey::object(
281                    store.store_object_id(),
282                    store.root_directory_object_id()
283                )],
284                Options::default(),
285            )
286            .await
287            .expect("new_transaction failed");
288
289        let object =
290            ObjectStore::create_object(&store, &mut transaction, HandleOptions::default(), None)
291                .await
292                .expect("create_object failed");
293
294        let root_directory =
295            Directory::open(&store, store.root_directory_object_id()).await.expect("open failed");
296        root_directory
297            .add_child_file(&mut transaction, TEST_OBJECT_NAME, &object)
298            .await
299            .expect("add_child_file failed");
300
301        transaction.commit().await.expect("commit failed");
302
303        (fs, object)
304    }
305
306    #[fuchsia::test(threads = 3)]
307    async fn test_write_read_zstd() {
308        let (fs, object) = test_filesystem_and_empty_object().await;
309
310        let metadata = BlobMetadata {
311            merkle_leaves: vec![[1; 32], [2; 32], [3; 32], [4; 32]],
312            format: BlobFormat::ChunkedZstd {
313                uncompressed_size: 128 * 1024,
314                chunk_size: 32 * 1024,
315                compressed_offsets: vec![0, 100, 200, 400],
316            },
317        };
318        metadata.write_to(&object).await.expect("failed to write attribute");
319        let read_metadata =
320            BlobMetadata::read_from(&object).await.expect("failed to read attribute");
321        assert_eq!(read_metadata, metadata);
322
323        fs.close().await.expect("close failed");
324    }
325
326    #[fuchsia::test(threads = 3)]
327    async fn test_write_read_lz4() {
328        let (fs, object) = test_filesystem_and_empty_object().await;
329
330        let metadata = BlobMetadata {
331            merkle_leaves: vec![[1; 32], [2; 32], [3; 32], [4; 32]],
332            format: BlobFormat::ChunkedLz4 {
333                uncompressed_size: 128 * 1024,
334                chunk_size: 32 * 1024,
335                compressed_offsets: vec![0, 100, 200, 400],
336            },
337        };
338        metadata.write_to(&object).await.expect("failed to write attribute");
339        let read_metadata =
340            BlobMetadata::read_from(&object).await.expect("failed to read attribute");
341        assert_eq!(read_metadata, metadata);
342
343        fs.close().await.expect("close failed");
344    }
345
346    #[fuchsia::test(threads = 3)]
347    async fn test_empty_attribute_is_not_written() {
348        let (fs, object) = test_filesystem_and_empty_object().await;
349
350        BlobMetadata::empty().write_to(&object).await.expect("failed to write attribute");
351        let result = object
352            .read_attr(BLOB_METADATA_ATTRIBUTE_ID)
353            .await
354            .expect("reading the attribute failed");
355        assert_eq!(result, None);
356
357        fs.close().await.expect("close failed");
358    }
359
360    #[fuchsia::test(threads = 3)]
361    async fn test_read_corrupt_attribute_fails() {
362        let (fs, object) = test_filesystem_and_empty_object().await;
363
364        object
365            .write_attr(BLOB_METADATA_ATTRIBUTE_ID, b"garbage")
366            .await
367            .expect("failed to write attribute");
368        BlobMetadata::read_from(&object).await.expect_err("reading the metadata should fail");
369
370        fs.close().await.expect("close failed");
371    }
372
373    #[fuchsia::test(threads = 3)]
374    async fn test_read_unversioned_attribute() {
375        let (fs, object) = test_filesystem_and_empty_object().await;
376
377        let unversioned_metadata = BlobMetadataUnversioned {
378            hashes: vec![[1; 32], [2; 32]],
379            chunk_size: 32 * 1024,
380            compressed_offsets: vec![0],
381            uncompressed_size: 15 * 1024,
382        };
383        let mut buf = Vec::new();
384        bincode::serialize_into(&mut buf, &unversioned_metadata)
385            .expect("failed to serialize metadata");
386        object.write_attr(BLOB_MERKLE_ATTRIBUTE_ID, &buf).await.expect("failed to write attribute");
387        let metadata = BlobMetadata::read_from(&object).await.expect("failed to read attribute");
388        assert_eq!(metadata, BlobMetadata::from(unversioned_metadata));
389
390        fs.close().await.expect("close failed");
391    }
392
393    #[fuchsia::test(threads = 3)]
394    async fn test_read_corrupt_unversioned_attribute_fails() {
395        let (fs, object) = test_filesystem_and_empty_object().await;
396
397        object
398            .write_attr(BLOB_MERKLE_ATTRIBUTE_ID, b"garbage")
399            .await
400            .expect("failed to write attribute");
401        BlobMetadata::read_from(&object).await.expect_err("reading the metadata should fail");
402
403        fs.close().await.expect("close failed");
404    }
405
406    #[fuchsia::test(threads = 3)]
407    async fn test_fs_verity_hides_blob_metadata() {
408        let (fs, object) = test_filesystem_and_empty_object().await;
409
410        let metadata = BlobMetadata {
411            merkle_leaves: vec![[1; 32], [2; 32]],
412            format: BlobFormat::Uncompressed,
413        };
414        metadata.write_to(&object).await.expect("failed to write attribute");
415        object
416            .write_attr(FSVERITY_MERKLE_ATTRIBUTE_ID, b"fs-verify")
417            .await
418            .expect("failed to write fs-verity attribute");
419        BlobMetadata::read_from(&object).await.expect_err("fs-verity should have been found");
420
421        fs.close().await.expect("close failed");
422    }
423
424    #[fuchsia::test]
425    async fn test_serialized_size() {
426        assert_matches!(BlobMetadata::empty().serialized_size(), Ok(0));
427        assert_matches!(
428            BlobMetadata {
429                merkle_leaves: vec![[54; 32], [55; 32]],
430                format: BlobFormat::Uncompressed,
431            }
432            .serialized_size(),
433            // 4 bytes for the version.
434            // 1 byte for the count of merkle leaves.
435            // 64 bytes of merkle leaves.
436            // 1 byte discriminant for the format.
437            Ok(70)
438        );
439        assert_matches!(
440            BlobMetadata {
441                merkle_leaves: vec![[54; 32], [55; 32]],
442                format: BlobFormat::ChunkedZstd {
443                    uncompressed_size: 128 * 1024,
444                    chunk_size: 32 * 1024,
445                    compressed_offsets: vec![0, 100, 200, 400],
446                },
447            }
448            .serialized_size(),
449            // 4 bytes for the version.
450            // 1 byte for the count of merkle leaves.
451            // 64 bytes of merkle leaves.
452            // 1 byte discriminant for the format.
453            // 5 bytes for the uncompressed size.
454            // 3 bytes for the chunk size.
455            // 1 byte for the count of compressed offsets.
456            // 6 bytes of compressed offsets.
457            Ok(85)
458        );
459    }
460
461    #[fuchsia::test]
462    fn test_leaf_hash_collector_with_only_root() {
463        let data = vec![3; 4096];
464        let (_root, leaves) =
465            MerkleRootBuilder::new(BlobMetadataLeafHashCollector::new()).complete(&data);
466        assert!(leaves.is_empty());
467    }
468
469    #[fuchsia::test]
470    fn test_leaf_hash_collector_with_leaves() {
471        let data = vec![3; 12 * 1024];
472        let (_root, leaves) =
473            MerkleRootBuilder::new(BlobMetadataLeafHashCollector::new()).complete(&data);
474        assert_eq!(leaves.len(), 2);
475    }
476
477    #[fuchsia::test]
478    fn test_into_merkle_verifier_with_only_root() {
479        let data = vec![3; 4096];
480        let (root, leaves) =
481            MerkleRootBuilder::new(BlobMetadataLeafHashCollector::new()).complete(&data);
482        let metadata = BlobMetadata { merkle_leaves: leaves, format: BlobFormat::Uncompressed };
483        let verifier =
484            metadata.into_merkle_verifier(root).expect("failed to create merkle verifier");
485        verifier.verify(0, &data).expect("failed to verify data");
486    }
487
488    #[fuchsia::test]
489    fn test_into_merkle_verifier_with_leaves() {
490        let data = vec![3; 12 * 1024];
491        let (root, leaves) =
492            MerkleRootBuilder::new(BlobMetadataLeafHashCollector::new()).complete(&data);
493        let metadata = BlobMetadata { merkle_leaves: leaves, format: BlobFormat::Uncompressed };
494        let verifier =
495            metadata.into_merkle_verifier(root).expect("failed to create merkle verifier");
496        verifier.verify(0, &data).expect("failed to verify data");
497    }
498
499    #[fuchsia::test]
500    fn test_convert_unversioned_to_versioned() {
501        assert_eq!(
502            BlobMetadata::from(BlobMetadataUnversioned {
503                hashes: vec![[1; 32], [2; 32]],
504                chunk_size: 0,
505                compressed_offsets: vec![],
506                uncompressed_size: 15 * 1024,
507            }),
508            BlobMetadata {
509                merkle_leaves: vec![[1; 32], [2; 32]],
510                format: BlobFormat::Uncompressed,
511            }
512        );
513
514        assert_eq!(
515            BlobMetadata::from(BlobMetadataUnversioned {
516                hashes: vec![[1; 32], [2; 32], [3; 32], [4; 32]],
517                chunk_size: 32 * 1024,
518                compressed_offsets: vec![0, 100],
519                uncompressed_size: 33 * 1024,
520            }),
521            BlobMetadata {
522                merkle_leaves: vec![[1; 32], [2; 32], [3; 32], [4; 32]],
523                format: BlobFormat::ChunkedZstd {
524                    uncompressed_size: 33 * 1024,
525                    chunk_size: 32 * 1024,
526                    compressed_offsets: vec![0, 100]
527                },
528            }
529        );
530    }
531
532    #[fuchsia::test]
533    fn test_merkle_serialization() {}
534}
fxfs/blob_metadata.rs

fxfs/
blob_metadata.rs
