delivery_blob/compression/
compression_algorithm.rs

1// Copyright 2026 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! Compression algorithms supported by chunked-compression and corresponding compressors and
6//! decompressors.
7//!
8//! The compressors and decompressors are enums rather than traits with multiple implementations
9//! because the enums are small and avoid the heap allocation of `Box<dyn Decompressor>`.
10
11use super::{FormatError, ZstdError};
12use crate::compression::ChunkedArchiveError;
13use std::mem::MaybeUninit;
14use std::ptr::NonNull;
15use storage_ptr_slice::PtrByteSlice;
16use zstd::zstd_safe::zstd_sys;
17
18thread_local! {
19    static ZSTD_COMPRESSOR: std::cell::RefCell<zstd::zstd_safe::CCtx<'static>> =
20        std::cell::RefCell::new({
21            let mut cctx = zstd::zstd_safe::CCtx::create();
22            cctx.set_parameter(zstd::zstd_safe::CParameter::ChecksumFlag(true)).unwrap();
23            cctx
24        });
25    static ZSTD_DECOMPRESSOR: std::cell::RefCell<RawDCtx> = {
26        // SAFETY: Creating a ZSTD decompression context does not access
27        // invalid memory or violate invariants.
28        let raw_ptr = unsafe { zstd_sys::ZSTD_createDCtx() };
29        let ptr = NonNull::new(raw_ptr).expect("ZSTD_createDCtx failed");
30        std::cell::RefCell::new(RawDCtx(ptr))
31    };
32}
33
34struct RawDCtx(NonNull<zstd_sys::ZSTD_DCtx>);
35impl Drop for RawDCtx {
36    fn drop(&mut self) {
37        // SAFETY: `self.0` is non-null and was allocated via `ZSTD_createDCtx`.
38        unsafe {
39            zstd_sys::ZSTD_freeDCtx(self.0.as_ptr());
40        }
41    }
42}
43
44/// Decompresses ZSTD using a pointer slice and a raw destination pointer slice.
45///
46/// # Safety
47///
48/// `dst` must point to valid memory allocated for writes of at least
49/// `dst.len()` bytes.
50unsafe fn zstd_decompress_ptr(
51    src: PtrByteSlice<'_>,
52    dst: *mut [MaybeUninit<u8>],
53) -> Result<usize, usize> {
54    ZSTD_DECOMPRESSOR.with_borrow_mut(|decompressor| {
55        let dctx = decompressor.0.as_ptr();
56        // SAFETY: `PtrByteSlice` guarantees `src` is valid for reads. `dst`
57        // points to valid memory allocated for writes of at least `dst.len()`
58        // bytes (ensured by caller safety preconditions).
59        let result = unsafe {
60            zstd_sys::ZSTD_decompressDCtx(
61                dctx,
62                dst as *mut std::os::raw::c_void,
63                dst.len(),
64                src.as_raw_slice_ptr() as *const std::os::raw::c_void,
65                src.len(),
66            )
67        };
68        // SAFETY: Checking an integer return code is pure inspection.
69        if unsafe { zstd_sys::ZSTD_isError(result) } != 0 { Err(result) } else { Ok(result) }
70    })
71}
72
73unsafe extern "C" {
74    fn LZ4_decompress_safe(
75        src: *const std::os::raw::c_char,
76        dst: *mut std::os::raw::c_char,
77        compressedSize: std::os::raw::c_int,
78        dstCapacity: std::os::raw::c_int,
79    ) -> std::os::raw::c_int;
80}
81
82/// Decompresses LZ4 using a pointer slice and a raw destination pointer slice.
83///
84/// # Safety
85///
86/// `dst` must point to valid memory allocated for writes of at least
87/// `dst.len()` bytes.
88unsafe fn lz4_decompress_ptr(
89    src: PtrByteSlice<'_>,
90    dst: *mut [MaybeUninit<u8>],
91) -> Result<usize, lz4::Error> {
92    if src.is_empty() {
93        return Ok(0);
94    }
95    if dst.len() == 0 {
96        return Err(lz4::Error::DecompressionFailed);
97    }
98    // SAFETY: `PtrByteSlice` guarantees `src` is valid for reads. `dst`
99    // points to valid memory allocated for writes of at least `dst.len()`
100    // bytes (ensured by caller safety preconditions).
101    let result = unsafe {
102        LZ4_decompress_safe(
103            src.as_raw_slice_ptr() as *const std::os::raw::c_char,
104            dst as *mut std::os::raw::c_char,
105            src.len().try_into().map_err(|_| lz4::Error::InputTooLarge)?,
106            dst.len().try_into().map_err(|_| lz4::Error::InputTooLarge)?,
107        )
108    };
109    if result < 0 { Err(lz4::Error::DecompressionFailed) } else { Ok(result as usize) }
110}
111
112/// The compression algorithm used to compress the chunks.
113#[derive(Copy, Clone, Debug, Eq, PartialEq)]
114#[repr(u8)]
115pub enum CompressionAlgorithm {
116    Zstd = 0,
117    Lz4 = 1,
118}
119
120impl CompressionAlgorithm {
121    /// Returns a decompressor that can decompress a chunk compressed with this compression
122    /// algorithm.
123    pub fn decompressor(&self) -> Decompressor {
124        match self {
125            Self::Zstd => Decompressor::Zstd,
126            Self::Lz4 => Decompressor::Lz4,
127        }
128    }
129
130    /// Returns a decompressor that can decompress a chunk compressed with this compression
131    /// algorithm. Some decompressors require a large state object that is expensive to create but
132    /// can be reused for many decompressions. A thread-local decompressor stores the state object
133    /// in a thread-local variable.
134    pub fn thread_local_decompressor(&self) -> ThreadLocalDecompressor {
135        match self {
136            Self::Zstd => ThreadLocalDecompressor::Zstd,
137            Self::Lz4 => ThreadLocalDecompressor::Lz4,
138        }
139    }
140}
141
142impl From<CompressionAlgorithm> for u8 {
143    fn from(value: CompressionAlgorithm) -> Self {
144        value as u8
145    }
146}
147
148impl TryFrom<u8> for CompressionAlgorithm {
149    type Error = ChunkedArchiveError;
150    fn try_from(value: u8) -> Result<Self, Self::Error> {
151        match value {
152            0 => Ok(CompressionAlgorithm::Zstd),
153            1 => Ok(CompressionAlgorithm::Lz4),
154            _ => Err(ChunkedArchiveError::IntegrityError),
155        }
156    }
157}
158
159/// A decompressor that is capable of decompressing chunks of a compressed archive.
160pub enum Decompressor {
161    Zstd,
162    Lz4,
163}
164
165impl Decompressor {
166    /// Decompresses a chunk of a chunked-compression archive.
167    pub fn decompress<'a>(
168        &mut self,
169        data: impl Into<PtrByteSlice<'a>>,
170        uncompressed_size: usize,
171        chunk_index: usize,
172    ) -> Result<Vec<u8>, ChunkedArchiveError> {
173        let src = data.into();
174        let mut buffer = Vec::with_capacity(uncompressed_size);
175        let dst = buffer.spare_capacity_mut() as *mut [MaybeUninit<u8>];
176        let len = match self {
177            Self::Zstd => {
178                // SAFETY: `dst` points to `uncompressed_size` bytes of capacity
179                // in `buffer`.
180                unsafe { zstd_decompress_ptr(src, dst) }.map_err(|code| {
181                    ChunkedArchiveError::DecompressionError {
182                        index: chunk_index,
183                        error: FormatError::Zstd(ZstdError(code)),
184                    }
185                })?
186            }
187            Self::Lz4 => {
188                // SAFETY: `dst` points to `uncompressed_size` bytes of capacity
189                // in `buffer`.
190                unsafe { lz4_decompress_ptr(src, dst) }.map_err(|e| {
191                    ChunkedArchiveError::DecompressionError {
192                        index: chunk_index,
193                        error: FormatError::Lz4(e),
194                    }
195                })?
196            }
197        };
198        // SAFETY: Decompression wrote `len` initialized bytes into `buffer`.
199        unsafe {
200            buffer.set_len(len);
201        }
202        Ok(buffer)
203    }
204
205    /// Decompresses a chunk of a chunked-compression archive into a pre-allocated buffer.
206    pub fn decompress_into<'a>(
207        &mut self,
208        data: impl Into<PtrByteSlice<'a>>,
209        destination: &mut [u8],
210        chunk_index: usize,
211    ) -> Result<usize, ChunkedArchiveError> {
212        let src = data.into();
213        let dst = destination as *mut [u8] as *mut [MaybeUninit<u8>];
214        match self {
215            Self::Zstd => {
216                // SAFETY: `dst` points to `destination.len()` bytes of valid
217                // memory in `destination`.
218                unsafe { zstd_decompress_ptr(src, dst) }.map_err(|code| {
219                    ChunkedArchiveError::DecompressionError {
220                        index: chunk_index,
221                        error: FormatError::Zstd(ZstdError(code)),
222                    }
223                })
224            }
225            Self::Lz4 => {
226                // SAFETY: `dst` points to `destination.len()` bytes of valid
227                // memory in `destination`.
228                unsafe { lz4_decompress_ptr(src, dst) }.map_err(|e| {
229                    ChunkedArchiveError::DecompressionError {
230                        index: chunk_index,
231                        error: FormatError::Lz4(e),
232                    }
233                })
234            }
235        }
236    }
237}
238
239#[derive(Copy, Clone)]
240/// A decompressor that uses thread-local storage to avoid reallocation of large state objects.
241pub enum ThreadLocalDecompressor {
242    Zstd,
243    Lz4,
244}
245
246impl ThreadLocalDecompressor {
247    /// Decompresses a chunk of a chunked-compression archive.
248    pub fn decompress<'a>(
249        &self,
250        data: impl Into<PtrByteSlice<'a>>,
251        uncompressed_size: usize,
252        chunk_index: usize,
253    ) -> Result<Vec<u8>, ChunkedArchiveError> {
254        let src = data.into();
255        let mut buffer = Vec::with_capacity(uncompressed_size);
256        let dst = buffer.spare_capacity_mut() as *mut [MaybeUninit<u8>];
257        let len = match self {
258            Self::Zstd => {
259                // SAFETY: `dst` points to `uncompressed_size` bytes of capacity
260                // in `buffer`.
261                unsafe { zstd_decompress_ptr(src, dst) }.map_err(|code| {
262                    ChunkedArchiveError::DecompressionError {
263                        index: chunk_index,
264                        error: FormatError::Zstd(ZstdError(code)),
265                    }
266                })?
267            }
268            Self::Lz4 => {
269                // SAFETY: `dst` points to `uncompressed_size` bytes of capacity
270                // in `buffer`.
271                unsafe { lz4_decompress_ptr(src, dst) }.map_err(|e| {
272                    ChunkedArchiveError::DecompressionError {
273                        index: chunk_index,
274                        error: FormatError::Lz4(e),
275                    }
276                })?
277            }
278        };
279        // SAFETY: Decompression wrote `len` initialized bytes into `buffer`.
280        unsafe {
281            buffer.set_len(len);
282        }
283        Ok(buffer)
284    }
285
286    /// Decompresses a chunk of a chunked-compression archive into a pre-allocated buffer.
287    pub fn decompress_into<'a>(
288        &self,
289        data: impl Into<PtrByteSlice<'a>>,
290        destination: &mut [u8],
291        chunk_index: usize,
292    ) -> Result<usize, ChunkedArchiveError> {
293        let src = data.into();
294        let dst = destination as *mut [u8] as *mut [MaybeUninit<u8>];
295        match self {
296            Self::Zstd => {
297                // SAFETY: `dst` points to `destination.len()` bytes of valid
298                // memory in `destination`.
299                unsafe { zstd_decompress_ptr(src, dst) }.map_err(|code| {
300                    ChunkedArchiveError::DecompressionError {
301                        index: chunk_index,
302                        error: FormatError::Zstd(ZstdError(code)),
303                    }
304                })
305            }
306            Self::Lz4 => {
307                // SAFETY: `dst` points to `destination.len()` bytes of valid
308                // memory in `destination`.
309                unsafe { lz4_decompress_ptr(src, dst) }.map_err(|e| {
310                    ChunkedArchiveError::DecompressionError {
311                        index: chunk_index,
312                        error: FormatError::Lz4(e),
313                    }
314                })
315            }
316        }
317    }
318}
319
320/// A compressor that is capable of compressing chunks of a chunked-compression archive.
321pub enum Compressor {
322    Zstd(zstd::zstd_safe::CCtx<'static>),
323    Lz4 { compression_level: lz4::HcCompressionLevel },
324}
325
326impl Compressor {
327    /// Compresses a chunk of a chunked-compression archive.
328    pub fn compress(
329        &mut self,
330        data: &[u8],
331        chunk_index: usize,
332    ) -> Result<Vec<u8>, ChunkedArchiveError> {
333        match self {
334            Self::Zstd(cctx) => {
335                let buffer_len = zstd::zstd_safe::compress_bound(data.len());
336                let mut buffer = Vec::with_capacity(buffer_len);
337                match cctx.compress2(&mut buffer, data) {
338                    Ok(_) => Ok(buffer),
339                    Err(code) => Err(ChunkedArchiveError::CompressionError {
340                        index: chunk_index,
341                        error: FormatError::Zstd(ZstdError(code)),
342                    }),
343                }
344            }
345            Self::Lz4 { compression_level } => {
346                lz4::compress_hc(data, *compression_level).map_err(|error| {
347                    ChunkedArchiveError::CompressionError {
348                        index: chunk_index,
349                        error: FormatError::Lz4(error),
350                    }
351                })
352            }
353        }
354    }
355}
356
357#[derive(Copy, Clone)]
358/// A compressor that uses thread-local storage to avoid reallocation of large state objects.
359pub enum ThreadLocalCompressor {
360    Zstd { compression_level: i32 },
361    Lz4 { compression_level: lz4::HcCompressionLevel },
362}
363
364impl ThreadLocalCompressor {
365    /// Compresses a chunk of a chunked-compression archive.
366    pub fn compress(
367        &self,
368        data: &[u8],
369        chunk_index: usize,
370    ) -> Result<Vec<u8>, ChunkedArchiveError> {
371        match self {
372            Self::Zstd { compression_level } => ZSTD_COMPRESSOR.with_borrow_mut(|cctx| {
373                cctx.set_parameter(zstd::zstd_safe::CParameter::CompressionLevel(
374                    *compression_level,
375                ))
376                .expect("setting the compression level should never fail");
377                let buffer_len = zstd::zstd_safe::compress_bound(data.len());
378                let mut buffer = Vec::with_capacity(buffer_len);
379                match cctx.compress2(&mut buffer, data) {
380                    Ok(_) => Ok(buffer),
381                    Err(code) => Err(ChunkedArchiveError::CompressionError {
382                        index: chunk_index,
383                        error: FormatError::Zstd(ZstdError(code)),
384                    }),
385                }
386            }),
387            Self::Lz4 { compression_level } => {
388                lz4::compress_hc(data, *compression_level).map_err(|error| {
389                    ChunkedArchiveError::CompressionError {
390                        index: chunk_index,
391                        error: FormatError::Lz4(error),
392                    }
393                })
394            }
395        }
396    }
397}
398
399#[cfg(test)]
400mod tests {
401    use super::*;
402    use crate::compression::ChunkedArchiveOptions;
403
404    const TEST_DATA: &[u8] = b"hello world this is some test data to compress and decompress";
405
406    #[test]
407    fn test_zstd_roundtrip() {
408        let options =
409            ChunkedArchiveOptions::V3 { compression_algorithm: CompressionAlgorithm::Zstd };
410        let mut compressor = options.compressor();
411        let compressed = compressor.compress(TEST_DATA, 0).unwrap();
412
413        let mut decompressor = CompressionAlgorithm::Zstd.decompressor();
414        let decompressed = decompressor.decompress(&compressed, TEST_DATA.len(), 0).unwrap();
415
416        assert_eq!(decompressed, TEST_DATA);
417    }
418
419    #[test]
420    fn test_lz4_roundtrip() {
421        let options =
422            ChunkedArchiveOptions::V3 { compression_algorithm: CompressionAlgorithm::Lz4 };
423        let mut compressor = options.compressor();
424        let compressed = compressor.compress(TEST_DATA, 0).unwrap();
425
426        let mut decompressor = CompressionAlgorithm::Lz4.decompressor();
427        let decompressed = decompressor.decompress(&compressed, TEST_DATA.len(), 0).unwrap();
428
429        assert_eq!(decompressed, TEST_DATA);
430    }
431
432    #[test]
433    fn test_thread_local_zstd_roundtrip() {
434        let options =
435            ChunkedArchiveOptions::V3 { compression_algorithm: CompressionAlgorithm::Zstd };
436        let compressor = options.thread_local_compressor();
437        let compressed = compressor.compress(TEST_DATA, 0).unwrap();
438
439        let decompressor = CompressionAlgorithm::Zstd.thread_local_decompressor();
440        let decompressed = decompressor.decompress(&compressed, TEST_DATA.len(), 0).unwrap();
441
442        assert_eq!(decompressed, TEST_DATA);
443    }
444
445    #[test]
446    fn test_thread_local_lz4_roundtrip() {
447        let options =
448            ChunkedArchiveOptions::V3 { compression_algorithm: CompressionAlgorithm::Lz4 };
449        let compressor = options.thread_local_compressor();
450        let compressed = compressor.compress(TEST_DATA, 0).unwrap();
451
452        let decompressor = CompressionAlgorithm::Lz4.thread_local_decompressor();
453        let decompressed = decompressor.decompress(&compressed, TEST_DATA.len(), 0).unwrap();
454
455        assert_eq!(decompressed, TEST_DATA);
456    }
457
458    #[test]
459    fn test_decompress_into() {
460        let options = ChunkedArchiveOptions::V2 {
461            minimum_chunk_size: 0,
462            chunk_alignment: 0,
463            compression_level: 1,
464        };
465        let mut compressor = options.compressor();
466        let compressed = compressor.compress(TEST_DATA, 0).unwrap();
467
468        let mut decompressor = CompressionAlgorithm::Zstd.decompressor();
469        let mut buffer = vec![0u8; TEST_DATA.len()];
470        let len = decompressor.decompress_into(&compressed, &mut buffer, 0).unwrap();
471
472        assert_eq!(len, TEST_DATA.len());
473        assert_eq!(buffer, TEST_DATA);
474    }
475
476    #[test]
477    fn test_algorithm_conversion() {
478        assert_eq!(u8::from(CompressionAlgorithm::Zstd), 0);
479        assert_eq!(u8::from(CompressionAlgorithm::Lz4), 1);
480
481        assert_eq!(CompressionAlgorithm::try_from(0).unwrap(), CompressionAlgorithm::Zstd);
482        assert_eq!(CompressionAlgorithm::try_from(1).unwrap(), CompressionAlgorithm::Lz4);
483        assert!(CompressionAlgorithm::try_from(2).is_err());
484    }
485
486    #[test]
487    fn test_decompress_into_ptr() {
488        for algorithm in [CompressionAlgorithm::Zstd, CompressionAlgorithm::Lz4] {
489            let options = ChunkedArchiveOptions::V3 { compression_algorithm: algorithm };
490            let compressor = options.thread_local_compressor();
491            let compressed = compressor.compress(TEST_DATA, 0).unwrap();
492
493            let decompressor = algorithm.thread_local_decompressor();
494            let mut buffer = vec![0u8; TEST_DATA.len()];
495            let len = decompressor
496                .decompress_into(PtrByteSlice::from(compressed.as_slice()), &mut buffer, 0)
497                .unwrap();
498
499            assert_eq!(len, TEST_DATA.len());
500            assert_eq!(buffer, TEST_DATA);
501        }
502    }
503}
delivery_blob/compression/compression_algorithm.rs

delivery_blob/compression/
compression_algorithm.rs
