video_frame_stream/
lib.rs

Help
1// Copyright 2020 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use anyhow::format_err;
6use async_trait::async_trait;
7use fidl_fuchsia_images2 as images2;
8use fidl_fuchsia_media::*;
9use fuchsia_image_format::sysmem1_image_format_from_images2_image_format;
10use std::rc::Rc;
11use stream_processor_test::*;
12
13#[derive(Debug)]
14pub struct VideoFrame {
15    // TODO(https://fxbug.dev/42165549)
16    #[allow(unused)]
17    format: images2::ImageFormat,
18    data: Vec<u8>,
19}
20
21/// Container for raw video frame and associated format
22impl VideoFrame {
23    /// Generates a frame with specified `format`.
24    /// `step` is for diagonally shifting the checkerboard pattern.
25    pub fn create(format: images2::ImageFormat, step: usize) -> Self {
26        // For 4:2:0 YUV, the UV data is 1/2 the size of the Y data,
27        // so the size of the frame is 3/2 the size of the Y plane.
28        let width = *format.bytes_per_row.as_ref().unwrap() as usize;
29        let height = format.size.as_ref().unwrap().height as usize;
30        let frame_size = width * height * 3usize / 2usize;
31        let mut data = vec![128; frame_size];
32
33        // generate checkerboard
34        const NUM_BLOCKS: usize = 8usize;
35        let block_size = width / NUM_BLOCKS;
36        let mut y_on = true;
37        let mut x_on = true;
38        for y in 0..height {
39            if y % block_size == 0 {
40                y_on = !y_on;
41                x_on = y_on;
42            }
43            for x in 0..width {
44                if x % block_size == 0 {
45                    x_on = !x_on;
46                }
47                let luma = if x_on { 255 } else { 0 };
48                let y_s = (y + step) % height;
49                let x_s = (x + step) % width;
50                data[y_s * width + x_s] = luma;
51            }
52        }
53
54        Self { format, data }
55    }
56}
57
58/// Generates timestamps according to a timebase and rate of playback of uncompressed video.
59///
60/// Since the rate is constant, this can also be used to extrapolate timestamps.
61pub struct TimestampGenerator {
62    pub timebase: u64,
63    pub frames_per_second: usize,
64}
65
66impl TimestampGenerator {
67    pub fn timestamp_at(&self, input_index: usize) -> u64 {
68        let fps = self.frames_per_second as u64;
69        (input_index as u64) * self.timebase / fps
70    }
71}
72
73/// Validates that timestamps match the expected output from the specified generator
74pub struct TimestampValidator {
75    pub generator: Option<TimestampGenerator>,
76}
77
78#[async_trait(?Send)]
79impl OutputValidator for TimestampValidator {
80    async fn validate(&self, output: &[Output]) -> Result<()> {
81        let packets = output_packets(output);
82        for (pos, packet) in packets.enumerate() {
83            if packet.packet.timestamp_ish.is_none() {
84                return Err(format_err!("Missing timestamp"));
85            }
86
87            if let Some(generator) = &self.generator {
88                let current_ts = packet.packet.timestamp_ish.unwrap();
89                let expected_ts = generator.timestamp_at(pos);
90
91                if current_ts != expected_ts {
92                    return Err(format_err!(
93                        "Unexpected timestamp {} (expected {})",
94                        current_ts,
95                        expected_ts
96                    ));
97                }
98            }
99        }
100        Ok(())
101    }
102}
103
104fn new_video_uncompressed_format(image_format: images2::ImageFormat) -> VideoUncompressedFormat {
105    VideoUncompressedFormat {
106        image_format: sysmem1_image_format_from_images2_image_format(&image_format).unwrap(),
107        fourcc: 0,
108        primary_width_pixels: 0,
109        primary_height_pixels: 0,
110        secondary_width_pixels: 0,
111        secondary_height_pixels: 0,
112        planar: false,
113        swizzled: false,
114        primary_line_stride_bytes: 0,
115        secondary_line_stride_bytes: 0,
116        primary_start_offset: 0,
117        secondary_start_offset: 0,
118        tertiary_start_offset: 0,
119        primary_pixel_stride: 0,
120        secondary_pixel_stride: 0,
121        primary_display_width_pixels: 0,
122        primary_display_height_pixels: 0,
123        has_pixel_aspect_ratio: false,
124        pixel_aspect_ratio_width: 0,
125        pixel_aspect_ratio_height: 0,
126    }
127}
128
129/// Implements an `ElementaryStream` of raw video frames with the specified `format`, intended to be
130/// fed to an encoder StreamProcessor.
131pub struct VideoFrameStream {
132    pub num_frames: usize,
133    pub format: images2::ImageFormat,
134    pub encoder_settings: Rc<dyn Fn() -> EncoderSettings>,
135    pub frames_per_second: usize,
136    pub timebase: Option<u64>,
137    pub mime_type: String,
138}
139
140impl VideoFrameStream {
141    pub fn create(
142        format: images2::ImageFormat,
143        num_frames: usize,
144        encoder_settings: Rc<dyn Fn() -> EncoderSettings>,
145        frames_per_second: usize,
146        timebase: Option<u64>,
147        mime_type: &str,
148    ) -> Result<Self> {
149        Ok(Self {
150            num_frames,
151            format,
152            encoder_settings,
153            frames_per_second,
154            timebase,
155            mime_type: mime_type.to_string(),
156        })
157    }
158
159    pub fn timestamp_generator(&self) -> Option<TimestampGenerator> {
160        self.timebase.map(|timebase| TimestampGenerator {
161            frames_per_second: self.frames_per_second,
162            timebase,
163        })
164    }
165}
166
167impl ElementaryStream for VideoFrameStream {
168    fn format_details(&self, format_details_version_ordinal: u64) -> FormatDetails {
169        FormatDetails {
170            domain: Some(DomainFormat::Video(VideoFormat::Uncompressed(
171                new_video_uncompressed_format(self.format.clone()),
172            ))),
173            encoder_settings: Some((self.encoder_settings)()),
174            format_details_version_ordinal: Some(format_details_version_ordinal),
175            mime_type: Some(self.mime_type.to_string()),
176            oob_bytes: None,
177            pass_through_parameters: None,
178            timebase: self.timebase,
179            ..Default::default()
180        }
181    }
182
183    fn is_access_units(&self) -> bool {
184        false
185    }
186
187    fn stream<'a>(&'a self) -> Box<dyn Iterator<Item = ElementaryStreamChunk> + 'a> {
188        Box::new((0..self.num_frames).map(move |input_index| {
189            let frame = VideoFrame::create(self.format.clone(), input_index);
190            ElementaryStreamChunk {
191                start_access_unit: false,
192                known_end_access_unit: false,
193                data: frame.data,
194                significance: Significance::Video(VideoSignificance::Picture),
195                timestamp: self
196                    .timestamp_generator()
197                    .as_ref()
198                    .map(|timestamp_generator| timestamp_generator.timestamp_at(input_index)),
199            }
200        }))
201    }
202}
203
204#[cfg(test)]
205mod test {
206    use super::*;
207    use fidl_fuchsia_math::{RectU, SizeU};
208    use fuchsia_image_format::images2_image_format_from_sysmem_image_format;
209
210    #[derive(Debug, Copy, Clone)]
211    struct TestSpec {
212        pixel_format: images2::PixelFormat,
213        coded_width: usize,
214        coded_height: usize,
215        display_width: usize,
216        display_height: usize,
217        bytes_per_row: usize,
218    }
219
220    impl Into<VideoUncompressedFormat> for TestSpec {
221        fn into(self) -> VideoUncompressedFormat {
222            new_video_uncompressed_format(images2::ImageFormat {
223                pixel_format: Some(self.pixel_format),
224                size: Some(SizeU {
225                    width: self.coded_width.try_into().unwrap(),
226                    height: self.coded_height.try_into().unwrap(),
227                }),
228                bytes_per_row: Some(self.bytes_per_row as u32),
229                display_rect: Some(RectU {
230                    x: 0,
231                    y: 0,
232                    width: self.display_width as u32,
233                    height: self.display_height as u32,
234                }),
235                color_space: Some(images2::ColorSpace::Rec709),
236                ..Default::default()
237            })
238        }
239    }
240
241    #[fuchsia::test]
242    fn stream_timestamps() {
243        let test_spec = TestSpec {
244            pixel_format: images2::PixelFormat::Nv12,
245            coded_width: 16,
246            coded_height: 16,
247            display_height: 12,
248            display_width: 16,
249            bytes_per_row: 16,
250        };
251
252        let format: VideoUncompressedFormat = test_spec.into();
253        let stream = VideoFrameStream::create(
254            images2_image_format_from_sysmem_image_format(&format.image_format).unwrap(),
255            /*num_frames=*/ 2,
256            Rc::new(move || -> EncoderSettings {
257                EncoderSettings::H264(H264EncoderSettings {
258                    bit_rate: Some(200000),
259                    frame_rate: Some(60),
260                    ..Default::default()
261                })
262            }),
263            /*frames_per_second=*/ 60,
264            /*timebase=*/ Some(zx::MonotonicDuration::from_seconds(1).into_nanos() as u64),
265            /*mime_type=*/ "video/h264",
266        )
267        .expect("stream");
268        let mut chunks = stream.stream();
269
270        assert_eq!(chunks.next().and_then(|chunk| chunk.timestamp), Some(0));
271        assert_eq!(
272            chunks.next().and_then(|chunk| chunk.timestamp),
273            Some(zx::MonotonicDuration::from_seconds(1).into_nanos() as u64 / 60)
274        );
275    }
276
277    #[fuchsia::test]
278    fn pattern_check() {
279        let test_spec = TestSpec {
280            pixel_format: images2::PixelFormat::Nv12,
281            coded_width: 8,
282            coded_height: 8,
283            display_height: 8,
284            display_width: 8,
285            bytes_per_row: 8,
286        };
287
288        let format: VideoUncompressedFormat = test_spec.into();
289        let frame = VideoFrame::create(
290            images2_image_format_from_sysmem_image_format(&format.image_format).unwrap(),
291            0,
292        );
293        assert_eq!(
294            frame.data,
295            vec![
296                255, 0, 255, 0, 255, 0, 255, 0, 0, 255, 0, 255, 0, 255, 0, 255, 255, 0, 255, 0,
297                255, 0, 255, 0, 0, 255, 0, 255, 0, 255, 0, 255, 255, 0, 255, 0, 255, 0, 255, 0, 0,
298                255, 0, 255, 0, 255, 0, 255, 255, 0, 255, 0, 255, 0, 255, 0, 0, 255, 0, 255, 0,
299                255, 0, 255, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
300                128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
301                128, 128
302            ]
303        );
304    }
305}
video_frame_stream/lib.rs

video_frame_stream/
lib.rs
