criterion/analysis/
compare.rs

Help
1use std::collections::BTreeMap;
2
3use stats::univariate::Sample;
4use stats::univariate::{self, mixed};
5use stats::Distribution;
6
7use benchmark::BenchmarkConfig;
8use error::Result;
9use estimate::Statistic;
10use estimate::{Distributions, Estimates};
11use report::BenchmarkId;
12use {build_estimates, format, fs, Criterion};
13
14// Common comparison procedure
15#[cfg_attr(feature = "cargo-clippy", allow(clippy::type_complexity))]
16pub(crate) fn common(
17    id: &BenchmarkId,
18    avg_times: &Sample<f64>,
19    config: &BenchmarkConfig,
20    criterion: &Criterion,
21) -> Result<(
22    f64,
23    Distribution<f64>,
24    Estimates,
25    Distributions,
26    Vec<f64>,
27    Vec<f64>,
28    Vec<f64>,
29    Estimates,
30)> {
31    let sample_dir = format!(
32        "{}/{}/{}/sample.json",
33        criterion.output_directory,
34        id.as_directory_name(),
35        criterion.baseline_directory
36    );
37    let (iters, times): (Vec<f64>, Vec<f64>) = fs::load(&sample_dir)?;
38
39    let estimates_file = &format!(
40        "{}/{}/{}/estimates.json",
41        criterion.output_directory,
42        id.as_directory_name(),
43        criterion.baseline_directory
44    );
45    let base_estimates: Estimates = fs::load(&estimates_file)?;
46
47    let base_avg_times: Vec<f64> = iters
48        .iter()
49        .zip(times.iter())
50        .map(|(iters, elapsed)| elapsed / iters)
51        .collect();
52    let base_avg_time_sample = Sample::new(&base_avg_times);
53
54    fs::mkdirp(&format!(
55        "{}/{}/change",
56        criterion.output_directory,
57        id.as_directory_name()
58    ))?;
59    let (t_statistic, t_distribution) = t_test(avg_times, base_avg_time_sample, config);
60
61    let (estimates, relative_distributions) =
62        estimates(id, avg_times, base_avg_time_sample, config, criterion);
63    Ok((
64        t_statistic,
65        t_distribution,
66        estimates,
67        relative_distributions,
68        iters,
69        times,
70        base_avg_times.clone(),
71        base_estimates,
72    ))
73}
74
75// Performs a two sample t-test
76fn t_test(
77    avg_times: &Sample<f64>,
78    base_avg_times: &Sample<f64>,
79    config: &BenchmarkConfig,
80) -> (f64, Distribution<f64>) {
81    let nresamples = config.nresamples;
82
83    let t_statistic = avg_times.t(base_avg_times);
84    let t_distribution = elapsed!(
85        "Bootstrapping the T distribution",
86        mixed::bootstrap(avg_times, base_avg_times, nresamples, |a, b| (a.t(b),))
87    )
88    .0;
89
90    // HACK: Filter out non-finite numbers, which can happen sometimes when sample size is very small.
91    // Downstream code doesn't like non-finite values here.
92    let t_distribution = Distribution::from(
93        t_distribution
94            .iter()
95            .filter(|a| a.is_finite())
96            .cloned()
97            .collect::<Vec<_>>()
98            .into_boxed_slice(),
99    );
100
101    (t_statistic, t_distribution)
102}
103
104// Estimates the relative change in the statistics of the population
105fn estimates(
106    id: &BenchmarkId,
107    avg_times: &Sample<f64>,
108    base_avg_times: &Sample<f64>,
109    config: &BenchmarkConfig,
110    criterion: &Criterion,
111) -> (Estimates, Distributions) {
112    fn stats(a: &Sample<f64>, b: &Sample<f64>) -> (f64, f64) {
113        (
114            a.mean() / b.mean() - 1.,
115            a.percentiles().median() / b.percentiles().median() - 1.,
116        )
117    }
118
119    let cl = config.confidence_level;
120    let nresamples = config.nresamples;
121
122    let (dist_mean, dist_median) = elapsed!(
123        "Bootstrapping the relative statistics",
124        univariate::bootstrap(avg_times, base_avg_times, nresamples, stats)
125    );
126
127    let mut distributions = Distributions::new();
128    distributions.insert(Statistic::Mean, dist_mean);
129    distributions.insert(Statistic::Median, dist_median);
130
131    let (mean, median) = stats(avg_times, base_avg_times);
132    let mut point_estimates = BTreeMap::new();
133    point_estimates.insert(Statistic::Mean, mean);
134    point_estimates.insert(Statistic::Median, median);
135
136    let estimates = build_estimates(&distributions, &point_estimates, cl);
137
138    {
139        log_if_err!(fs::save(
140            &estimates,
141            &format!(
142                "{}/{}/change/estimates.json",
143                criterion.output_directory,
144                id.as_directory_name()
145            )
146        ));
147    }
148    (estimates, distributions)
149}
criterion/analysis/compare.rs

criterion/analysis/
compare.rs
