fuchsia_criterion/
lib.rs

1// Copyright 2019 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! Thin wrapper crate around the [Criterion benchmark suite].
6//!
7//! In order to facilitate micro-benchmarking Rust code on Fuchsia while at the same time providing
8//! a pleasant experience locally, this crate provides a wrapper type [`FuchsiaCriterion`] that
9//! implements `Deref<Target = Criterion>` with two constructors.
10//!
11//! By default, calling [`FuchsiaCriterion::default`] will create a [`Criterion`] object whose
12//! output is saved to a command-line-provided JSON file in the [Fuchsiaperf file format].
13//!
14//! In order to benchmark locally, i.e. in an `fx shell`, simply pass `--args=local_bench='true'` to
15//! the `fx set` command which will create a CMD-configurable Criterion object that is useful for
16//! fine-tuning performance.
17//! ```no_run
18//! # use fuchsia_criterion::FuchsiaCriterion;
19//! #
20//! fn main() {
21//!     let mut _c = FuchsiaCriterion::default();
22//! }
23//! ```
24//!
25//! [Criterion benchmark suite]: https://github.com/bheisler/criterion.rs
26//! [default]: https://doc.rust-lang.org/std/default/trait.Default.html#tymethod.default
27//! [Fuchsiaperf file format]: https://fuchsia.dev/fuchsia-src/development/performance/fuchsiaperf_format
28
29#![deny(missing_docs)]
30
31use std::collections::HashMap;
32use std::fs::OpenOptions;
33use std::io::Write;
34use std::ops::{Deref, DerefMut};
35use std::path::{Path, PathBuf};
36use std::{env, process};
37
38use criterion::Criterion;
39use tempfile::TempDir;
40use walkdir::WalkDir;
41
42pub use criterion;
43
44/// Thin wrapper around [`Criterion`].
45pub struct FuchsiaCriterion {
46    criterion: Criterion,
47    output: Option<(TempDir, PathBuf)>,
48}
49
50impl FuchsiaCriterion {
51    /// Creates a new [`Criterion`] wrapper.
52    pub fn new(criterion: Criterion) -> Self {
53        Self { criterion, output: None }
54    }
55
56    /// Creates a new [`Criterion`] object whose output is tailored to Fuchsia-CI.
57    ///
58    /// It calls its [`Default::default`] constructor with 10,000 resamples, then looks for a CMD
59    /// argument providing the path for JSON file where the micro-benchmark results should be
60    /// written (in the [Fuchsiaperf file format]).
61    ///
62    /// [Fuchsiaperf file format]: https://fuchsia.dev/fuchsia-src/development/performance/fuchsiaperf_format
63    pub fn fuchsia_bench() -> Self {
64        let args: Vec<String> = env::args().collect();
65        let args: Vec<&str> = args.iter().map(|s| &**s).collect();
66        Self::fuchsia_bench_with_args(&args[..])
67    }
68
69    /// Same as `fuchsia_bench()`, but parses the specified args instead of parsing
70    /// `std::env::args`.
71    ///
72    /// This is useful when the benchmarks needs to accept some arguments specified in the CML
73    /// file.
74    pub fn fuchsia_bench_with_args(args: &[&str]) -> Self {
75        fn help_and_exit(name: &str, wrong_args: Option<String>) -> ! {
76            println!(
77                "fuchsia-criterion benchmark\n\
78                 \n\
79                 USAGE: {} [FLAGS] JSON_OUTPUT\n\
80                 \n\
81                 FLAGS:\n\
82                 --filter <string>  Only runs benchmarks with names that contain the given string\n\
83                 -h, --help         Prints help information",
84                name,
85            );
86
87            if let Some(wrong_args) = wrong_args {
88                eprintln!("error: unrecognized args: {}", wrong_args);
89                process::exit(1)
90            } else {
91                process::exit(0)
92            }
93        }
94
95        match &args[..] {
96            [_, "--filter", filter, json_output] => {
97                let output_directory =
98                    TempDir::new().expect("failed to access temporary directory");
99                let criterion = Criterion::default()
100                    .nresamples(10_000)
101                    .with_filter(*filter)
102                    .output_directory(output_directory.path());
103                Self {
104                    criterion,
105                    output: Some((output_directory, Path::new(*json_output).to_path_buf())),
106                }
107            }
108            [_, arg] if !arg.starts_with('-') => {
109                let output_directory =
110                    TempDir::new().expect("failed to access temporary directory");
111                let criterion = Criterion::default()
112                    .nresamples(10_000)
113                    .output_directory(output_directory.path());
114
115                Self {
116                    criterion,
117                    output: Some((output_directory, Path::new(&args[1]).to_path_buf())),
118                }
119            }
120            [name, "-h"] | [name, "--help"] => help_and_exit(name, None),
121            _ => help_and_exit(&args[0], Some(args[1..].join(" "))),
122        }
123    }
124
125    fn convert_csvs(output_directory: &Path, output_json: &Path) {
126        let csv_entries = WalkDir::new(output_directory).into_iter().filter_map(|res| {
127            res.ok().filter(|entry| {
128                let path = entry.path();
129                path.is_file() && path.extension().map(|ext| ext == "csv").unwrap_or(false)
130            })
131        });
132
133        let mut results: HashMap<_, Vec<f64>> = HashMap::new();
134
135        for csv_entry in csv_entries {
136            let mut reader = csv::Reader::from_path(csv_entry.path())
137                .expect("found non-CSV file in fuchsia-criterion specific temporary directory");
138
139            for record in reader.records() {
140                let record = record.expect("CSV record is not UTF-8");
141
142                if record.len() != 5 {
143                    panic!("wrong number of records in Criterion-generated CSV");
144                }
145
146                let label = record[1].to_string();
147                let test_suite = record[0].to_string();
148                let sample = match (record[3].parse::<f64>(), record[4].parse::<f64>()) {
149                    (Ok(time), Ok(iter_count)) => time / iter_count,
150                    _ => panic!("non floating point values in Criterion-generated CSV"),
151                };
152
153                results
154                    .entry((label, test_suite))
155                    .and_modify(|samples| samples.push(sample))
156                    .or_insert_with(|| vec![sample]);
157            }
158        }
159
160        let entries: Vec<_> = results
161            .into_iter()
162            .map(|((label, test_suite), samples)| {
163                format!(
164                    r#"{{
165        "label": {:?},
166        "test_suite": {:?},
167        "unit": "nanoseconds",
168        "values": {:?}
169    }}"#,
170                    label, test_suite, samples,
171                )
172            })
173            .collect();
174
175        let mut f = OpenOptions::new()
176            .truncate(true)
177            .write(true)
178            .create(true)
179            .open(output_json)
180            .expect("failed to open output JSON");
181        write!(f, "[\n    {}\n]\n", entries.join(",\n"),).expect("failed to write to output JSON");
182    }
183}
184
185impl Default for FuchsiaCriterion {
186    fn default() -> Self {
187        if cfg!(feature = "local_bench") {
188            let criterion = Criterion::default()
189                .nresamples(10_000)
190                .output_directory(Path::new("/tmp/criterion"))
191                .configure_from_args();
192
193            FuchsiaCriterion::new(criterion)
194        } else {
195            FuchsiaCriterion::fuchsia_bench()
196        }
197    }
198}
199
200impl Deref for FuchsiaCriterion {
201    type Target = Criterion;
202
203    fn deref(&self) -> &Self::Target {
204        &self.criterion
205    }
206}
207
208impl DerefMut for FuchsiaCriterion {
209    fn deref_mut(&mut self) -> &mut Self::Target {
210        &mut self.criterion
211    }
212}
213
214impl Drop for FuchsiaCriterion {
215    /// Drops the [`Criterion`] wrapper.
216    ///
217    /// If initialized with [`FuchsiaCriterion::fuchsia_bench`], it will write the Fuchsia-specific
218    /// JSON file before dropping.
219    fn drop(&mut self) {
220        self.criterion.final_summary();
221
222        if let Some((output_directory, output_json)) = &self.output {
223            Self::convert_csvs(output_directory.path(), output_json);
224        }
225    }
226}
227
228#[cfg(test)]
229mod tests {
230    use super::*;
231
232    use std::fs::{self, File};
233    use std::io;
234
235    #[test]
236    fn criterion_results_conversion() -> io::Result<()> {
237        let temp = TempDir::new()?;
238        let output_directory = temp.path().join("some/where/deep");
239
240        fs::create_dir_all(output_directory.clone())?;
241
242        let mut csv = File::create(output_directory.join("criterion.csv"))?;
243        csv.write_all(
244            b"\
245            group,function,value,sample_time_nanos,iteration_count\n\
246            fib,parallel,,1000000,100000\n\
247            fib,parallel,,2000000,200000\n\
248            fib,parallel,,4000000,300000\n\
249            fib,parallel,,6000000,400000\n\
250        ",
251        )?;
252
253        let json = output_directory.join("fuchsia.json");
254        FuchsiaCriterion::convert_csvs(&output_directory, &json);
255
256        assert_eq!(
257            fs::read_to_string(json)?,
258            r#"[
259    {
260        "label": "parallel",
261        "test_suite": "fib",
262        "unit": "nanoseconds",
263        "values": [10.0, 10.0, 13.333333333333334, 15.0]
264    }
265]
266"#
267        );
268
269        Ok(())
270    }
271}
fuchsia_criterion/lib.rs

fuchsia_criterion/
lib.rs
