fuchsia_criterion/
lib.rs

1// Copyright 2019 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! Thin wrapper crate around the [Criterion benchmark suite].
6//!
7//! In order to facilitate micro-benchmarking Rust code on Fuchsia while at the same time providing
8//! a pleasant experience locally, this crate provides a wrapper type [`FuchsiaCriterion`] that
9//! implements `Deref<Target = Criterion>` with two constructors.
10//!
11//! By default, calling [`FuchsiaCriterion::default`] will create a [`Criterion`] object whose
12//! output is saved to a command-line-provided JSON file in the [Fuchsiaperf file format].
13//!
14//! In order to benchmark locally, i.e. in an `fx shell`, simply pass `--args=local_bench='true'` to
15//! the `fx set` command which will create a CMD-configurable Criterion object that is useful for
16//! fine-tuning performance.
17//! ```no_run
18//! # use fuchsia_criterion::FuchsiaCriterion;
19//! #
20//! fn main() {
21//!     let mut _c = FuchsiaCriterion::default();
22//! }
23//! ```
24//!
25//! [Criterion benchmark suite]: https://github.com/bheisler/criterion.rs
26//! [default]: https://doc.rust-lang.org/std/default/trait.Default.html#tymethod.default
27//! [Fuchsiaperf file format]: https://fuchsia.dev/fuchsia-src/development/performance/fuchsiaperf_format
28
29#![deny(missing_docs)]
30
31use std::collections::HashMap;
32use std::fs::OpenOptions;
33use std::io::Write;
34use std::ops::{Deref, DerefMut};
35use std::path::{Path, PathBuf};
36use std::{env, process};
37
38use criterion::Criterion;
39use tempfile::TempDir;
40use walkdir::WalkDir;
41
42pub use criterion;
43
44/// Thin wrapper around [`Criterion`].
45pub struct FuchsiaCriterion {
46    criterion: Criterion,
47    output: Option<(TempDir, PathBuf)>,
48}
49
50impl FuchsiaCriterion {
51    /// Creates a new [`Criterion`] wrapper.
52    pub fn new(criterion: Criterion) -> Self {
53        Self { criterion, output: None }
54    }
55
56    /// Creates a new [`Criterion`] object whose output is tailored to Fuchsia-CI.
57    ///
58    /// It calls its [`Default::default`] constructor with 10,000 resamples, then looks for a CMD
59    /// argument providing the path for JSON file where the micro-benchmark results should be
60    /// written (in the [Fuchsiaperf file format]).
61    ///
62    /// [Fuchsiaperf file format]: https://fuchsia.dev/fuchsia-src/development/performance/fuchsiaperf_format
63    pub fn fuchsia_bench() -> Self {
64        let args: Vec<String> = env::args().collect();
65        let args: Vec<&str> = args.iter().map(|s| &**s).collect();
66        Self::fuchsia_bench_with_args(&args[..])
67    }
68
69    /// Same as `fuchsia_bench()`, but parses the specified args instead of parsing
70    /// `std::env::args`.
71    ///
72    /// This is useful when the benchmarks needs to accept some arguments specified in the CML
73    /// file.
74    pub fn fuchsia_bench_with_args(args: &[&str]) -> Self {
75        fn help_and_exit(name: &str, wrong_args: Option<String>) -> ! {
76            println!(
77                "fuchsia-criterion benchmark\n\
78                 \n\
79                 USAGE: {} [FLAGS] JSON_OUTPUT\n\
80                 \n\
81                 FLAGS:\n\
82                 --filter <string>  Only runs benchmarks with names that contain the given string\n\
83                 -h, --help         Prints help information",
84                name,
85            );
86
87            if let Some(wrong_args) = wrong_args {
88                eprintln!("error: unrecognized args: {}", wrong_args);
89                process::exit(1)
90            } else {
91                process::exit(0)
92            }
93        }
94
95        match &args[..] {
96            [_, "--filter", filter, json_output] => {
97                let output_directory =
98                    TempDir::new().expect("failed to access temporary directory");
99                let criterion = Criterion::default()
100                    .nresamples(10_000)
101                    .with_filter(*filter)
102                    .output_directory(output_directory.path());
103                Self {
104                    criterion,
105                    output: Some((output_directory, Path::new(*json_output).to_path_buf())),
106                }
107            }
108            [_, arg] if !arg.starts_with('-') => {
109                let output_directory =
110                    TempDir::new().expect("failed to access temporary directory");
111                let criterion = Criterion::default()
112                    .nresamples(10_000)
113                    .output_directory(output_directory.path());
114
115                Self {
116                    criterion,
117                    output: Some((output_directory, Path::new(&args[1]).to_path_buf())),
118                }
119            }
120            [name, "-h"] | [name, "--help"] => help_and_exit(name, None),
121            _ => help_and_exit(&args[0], Some(args[1..].join(" "))),
122        }
123    }
124
125    fn convert_csvs(output_directory: &Path, output_json: &Path) {
126        let csv_entries = WalkDir::new(output_directory).into_iter().filter_map(|res| {
127            res.ok().filter(|entry| {
128                let path = entry.path();
129                path.is_file() && path.extension().map(|ext| ext == "csv").unwrap_or(false)
130            })
131        });
132
133        let mut results: HashMap<_, Vec<f64>> = HashMap::new();
134
135        for csv_entry in csv_entries {
136            let mut reader = csv::Reader::from_path(csv_entry.path())
137                .expect("found non-CSV file in fuchsia-criterion specific temporary directory");
138
139            for record in reader.records() {
140                let record = record.expect("CSV record is not UTF-8");
141
142                let (label, test_suite, sample) = match record.len() {
143                    5 => {
144                        let label = record[1].to_string();
145                        let test_suite = record[0].to_string();
146                        let sample = match (record[3].parse::<f64>(), record[4].parse::<f64>()) {
147                            (Ok(time), Ok(iter_count)) => time / iter_count,
148                            _ => panic!("non floating point values in Criterion-generated CSV"),
149                        };
150                        (label, test_suite, sample)
151                    }
152                    // Newly introduced format with the update to criterion 0.8
153                    // in https://fxrev.dev/1730349
154                    8 => {
155                        let group = record[0].to_string();
156                        let function = &record[1];
157                        let value = &record[2];
158                        let label = match (!function.is_empty(), !value.is_empty()) {
159                            (true, true) => format!("{function}/{value}"),
160                            (true, false) => function.to_string(),
161                            (false, true) => value.to_string(),
162                            (false, false) => "".to_string(),
163                        };
164                        let sample = match (record[5].parse::<f64>(), record[7].parse::<f64>()) {
165                            (Ok(time), Ok(iter_count)) => time / iter_count,
166                            _ => panic!("non floating point values in Criterion-generated CSV"),
167                        };
168                        (label, group, sample)
169                    }
170                    _ => panic!("wrong number of records in Criterion-generated CSV"),
171                };
172
173                results
174                    .entry((label, test_suite))
175                    .and_modify(|samples| samples.push(sample))
176                    .or_insert_with(|| vec![sample]);
177            }
178        }
179
180        let entries: Vec<_> = results
181            .into_iter()
182            .map(|((label, test_suite), samples)| {
183                format!(
184                    r#"{{
185        "label": {:?},
186        "test_suite": {:?},
187        "unit": "nanoseconds",
188        "values": {:?}
189    }}"#,
190                    label, test_suite, samples,
191                )
192            })
193            .collect();
194
195        let mut f = OpenOptions::new()
196            .truncate(true)
197            .write(true)
198            .create(true)
199            .open(output_json)
200            .expect("failed to open output JSON");
201        write!(f, "[\n    {}\n]\n", entries.join(",\n"),).expect("failed to write to output JSON");
202    }
203}
204
205impl Default for FuchsiaCriterion {
206    fn default() -> Self {
207        if cfg!(feature = "local_bench") {
208            let criterion = Criterion::default()
209                .nresamples(10_000)
210                .output_directory(Path::new("/tmp/criterion"))
211                .configure_from_args();
212
213            FuchsiaCriterion::new(criterion)
214        } else {
215            FuchsiaCriterion::fuchsia_bench()
216        }
217    }
218}
219
220impl Deref for FuchsiaCriterion {
221    type Target = Criterion;
222
223    fn deref(&self) -> &Self::Target {
224        &self.criterion
225    }
226}
227
228impl DerefMut for FuchsiaCriterion {
229    fn deref_mut(&mut self) -> &mut Self::Target {
230        &mut self.criterion
231    }
232}
233
234impl Drop for FuchsiaCriterion {
235    /// Drops the [`Criterion`] wrapper.
236    ///
237    /// If initialized with [`FuchsiaCriterion::fuchsia_bench`], it will write the Fuchsia-specific
238    /// JSON file before dropping.
239    fn drop(&mut self) {
240        self.criterion.final_summary();
241
242        if let Some((output_directory, output_json)) = &self.output {
243            Self::convert_csvs(output_directory.path(), output_json);
244        }
245    }
246}
247
248#[cfg(test)]
249mod tests {
250    use super::*;
251
252    use std::fs::{self, File};
253    use std::io;
254
255    #[test]
256    fn criterion_results_conversion() -> io::Result<()> {
257        let temp = TempDir::new()?;
258        let output_directory = temp.path().join("some/where/deep");
259
260        fs::create_dir_all(output_directory.clone())?;
261
262        let mut csv = File::create(output_directory.join("criterion.csv"))?;
263        csv.write_all(
264            b"\
265            group,function,value,sample_time_nanos,iteration_count\n\
266            fib,parallel,,1000000,100000\n\
267            fib,parallel,,2000000,200000\n\
268            fib,parallel,,4000000,300000\n\
269            fib,parallel,,6000000,400000\n\
270        ",
271        )?;
272
273        let json = output_directory.join("fuchsia.json");
274        FuchsiaCriterion::convert_csvs(&output_directory, &json);
275
276        assert_eq!(
277            fs::read_to_string(json)?,
278            r#"[
279    {
280        "label": "parallel",
281        "test_suite": "fib",
282        "unit": "nanoseconds",
283        "values": [10.0, 10.0, 13.333333333333334, 15.0]
284    }
285]
286"#
287        );
288
289        Ok(())
290    }
291
292    #[test]
293    fn criterion_0_8_results_conversion() -> io::Result<()> {
294        let temp = TempDir::new()?;
295        let output_directory = temp.path().join("some/where/deep");
296
297        fs::create_dir_all(output_directory.clone())?;
298
299        let mut csv = File::create(output_directory.join("raw.csv"))?;
300        csv.write_all(
301            b"\
302group,function,value,throughput_num,throughput_type,sample_measured_value,unit,iteration_count\n\
303fib,parallel,,,,1000000.0,ns,100000\n\
304fib,parallel,,,,2000000.0,ns,200000\n\
305fib,parallel,,,,4000000.0,ns,300000\n\
306fib,parallel,,,,6000000.0,ns,400000\n\
307",
308        )?;
309
310        let json = output_directory.join("fuchsia.json");
311        FuchsiaCriterion::convert_csvs(&output_directory, &json);
312
313        assert_eq!(
314            fs::read_to_string(json)?,
315            r#"[
316    {
317        "label": "parallel",
318        "test_suite": "fib",
319        "unit": "nanoseconds",
320        "values": [10.0, 10.0, 13.333333333333334, 15.0]
321    }
322]
323"#
324        );
325
326        Ok(())
327    }
328}
fuchsia_criterion/lib.rs

fuchsia_criterion/
lib.rs
