fuchsia_triage/
validate.rs

Help
1// Copyright 2019 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use super::act::{Action, Actions};
6use super::config::ParseResult;
7use super::metrics::fetch::{Fetcher, KeyValueFetcher, TextFetcher, TrialDataFetcher};
8use super::metrics::metric_value::MetricValue;
9use super::metrics::MetricState;
10use anyhow::{bail, format_err, Error};
11use serde::Deserialize;
12use serde_json as json;
13use std::collections::HashMap;
14
15#[derive(Clone, Deserialize, Debug)]
16pub struct Trial {
17    yes: Option<Vec<String>>,
18    no: Option<Vec<String>>,
19    now: Option<String>,
20    values: Option<HashMap<String, json::Value>>,
21    klog: Option<String>,
22    syslog: Option<String>,
23    bootlog: Option<String>,
24    annotations: Option<json::map::Map<String, json::Value>>,
25}
26
27// Outer String is namespace, inner String is trial name
28pub type Trials = HashMap<String, TrialsSchema>;
29pub type TrialsSchema = HashMap<String, Trial>;
30
31pub fn validate(parse_result: &ParseResult) -> Result<(), Error> {
32    let ParseResult { metrics, actions, tests } = parse_result;
33    let mut failed = false;
34    for (namespace, trial_map) in tests {
35        for (trial_name, trial) in trial_map {
36            if trial.yes.is_none() && trial.no.is_none() {
37                bail!("Trial {} in {} needs a yes: or no: entry", trial_name, namespace);
38            }
39            let klog_fetcher;
40            let syslog_fetcher;
41            let bootlog_fetcher;
42            let annotations_fetcher;
43            let mut fetcher = match &trial.values {
44                Some(values) => Fetcher::TrialData(TrialDataFetcher::new(values)),
45                None => Fetcher::TrialData(TrialDataFetcher::new_empty()),
46            };
47            if let Fetcher::TrialData(fetcher) = &mut fetcher {
48                if let Some(klog) = &trial.klog {
49                    klog_fetcher = TextFetcher::from(&**klog);
50                    fetcher.set_klog(&klog_fetcher);
51                }
52                if let Some(syslog) = &trial.syslog {
53                    syslog_fetcher = TextFetcher::from(&**syslog);
54                    fetcher.set_syslog(&syslog_fetcher);
55                }
56                if let Some(bootlog) = &trial.bootlog {
57                    bootlog_fetcher = TextFetcher::from(&**bootlog);
58                    fetcher.set_bootlog(&bootlog_fetcher);
59                }
60                if let Some(annotations) = &trial.annotations {
61                    annotations_fetcher = KeyValueFetcher::try_from(annotations)?;
62                    fetcher.set_annotations(&annotations_fetcher);
63                }
64            }
65            let now = if trial.now.is_none() {
66                None
67            } else {
68                match MetricState::evaluate_math(trial.now.as_ref().unwrap()) {
69                    MetricValue::Int(time) => Some(time),
70                    oops => bail!(
71                        "Trial {} in {}: 'now: {}' was not integer, it was {:?}",
72                        trial_name,
73                        namespace,
74                        trial.now.as_ref().unwrap(),
75                        oops
76                    ),
77                }
78            };
79            if let Some(action_names) = &trial.yes {
80                for action_name in action_names.iter() {
81                    let original_metrics = &(metrics.clone());
82                    let original_actions = &(actions.clone());
83                    let state = MetricState::new(original_metrics, fetcher.clone(), now);
84                    failed = check_failure(
85                        namespace,
86                        trial_name,
87                        action_name,
88                        original_actions,
89                        &state,
90                        true,
91                    ) || failed;
92                }
93            }
94            if let Some(action_names) = &trial.no {
95                for action_name in action_names.iter() {
96                    let original_metrics = &(metrics.clone());
97                    let original_actions = &(actions.clone());
98                    let state = MetricState::new(original_metrics, fetcher.clone(), now);
99                    failed = check_failure(
100                        namespace,
101                        trial_name,
102                        action_name,
103                        original_actions,
104                        &state,
105                        false,
106                    ) || failed;
107                }
108            }
109        }
110    }
111    if failed {
112        Err(format_err!("Config validation test failed"))
113    } else {
114        Ok(())
115    }
116}
117
118// Returns true iff the trial did NOT get the expected result.
119fn check_failure(
120    namespace: &String,
121    trial_name: &String,
122    action_name: &String,
123    actions: &Actions,
124    metric_state: &MetricState<'_>,
125    expected: bool,
126) -> bool {
127    match actions.get(namespace) {
128        None => {
129            println!("Namespace {} not found in trial {}", action_name, trial_name);
130            true
131        }
132        Some(action_map) => match action_map.get(action_name) {
133            None => {
134                println!("Action {} not found in trial {}", action_name, trial_name);
135                true
136            }
137            Some(action) => {
138                let trigger = match action {
139                    Action::Alert(properties) => &properties.trigger,
140                    Action::Snapshot(properties) => &properties.trigger,
141                    _ => {
142                        println!("Action {:?} cannot be tested", action);
143                        return true;
144                    }
145                };
146                match metric_state.eval_action_metric(namespace, trigger) {
147                    MetricValue::Bool(actual) if actual == expected => false,
148                    other => {
149                        println!(
150                            "Test {} failed: trigger '{}' of action {} returned {:?}, expected {}",
151                            trial_name, trigger, action_name, other, expected
152                        );
153                        true
154                    }
155                }
156            }
157        },
158    }
159}
160
161#[cfg(test)]
162mod test {
163    use super::*;
164    use crate::act::{Alert, Severity};
165    use crate::make_metrics;
166    use crate::metrics::ValueSource;
167
168    // Correct operation of the klog, syslog, and bootlog fields of TrialDataFetcher are tested
169    // in the integration test via log_tests.triage.
170
171    macro_rules! build_map {($($tuple:expr),*) => ({
172        let mut map = HashMap::new();
173        $(
174            let (key, value) = $tuple;
175            map.insert(key.to_string(), value);
176         )*
177            map
178    })}
179
180    macro_rules! create_parse_result {
181        (metrics: $metrics:expr, actions: $actions:expr, tests: $tests:expr) => {
182            ParseResult {
183                metrics: $metrics.clone(),
184                actions: $actions.clone(),
185                tests: $tests.clone(),
186            }
187        };
188    }
189
190    #[fuchsia::test]
191    fn validate_works() -> Result<(), Error> {
192        let metrics = make_metrics!({
193            "foo":{
194                eval: {
195                    "true": "1 == 1",
196                    "false": "1 == 0"
197                }
198            }
199        });
200
201        let actions = build_map!((
202            "foo",
203            build_map!(
204                (
205                    "fires",
206                    Action::Alert(Alert {
207                        trigger: ValueSource::try_from_expression_with_namespace("true", "foo")?,
208                        print: "good".to_string(),
209                        tag: None,
210                        file_bug: None,
211                        severity: Severity::Warning,
212                    })
213                ),
214                (
215                    "no_fire",
216                    Action::Alert(Alert {
217                        trigger: ValueSource::try_from_expression_with_namespace("false", "foo")?,
218                        print: "what?!?".to_string(),
219                        tag: None,
220                        file_bug: None,
221                        severity: Severity::Warning,
222                    })
223                )
224            )
225        ));
226        let good_trial = Trial {
227            yes: Some(vec!["fires".to_string()]),
228            no: Some(vec!["no_fire".to_string()]),
229            values: Some(HashMap::new()),
230            klog: None,
231            syslog: None,
232            bootlog: None,
233            annotations: None,
234            now: None,
235        };
236        assert!(validate(&create_parse_result!(
237            metrics: metrics,
238            actions: actions,
239            tests: build_map!(("foo", build_map!(("good", good_trial))))
240        ))
241        .is_ok());
242        // Make sure it objects if a trial that should fire doesn't.
243        // Also, make sure it signals failure if there's both a good and a bad trial.
244        let bad_trial = Trial {
245            yes: Some(vec!["fires".to_string(), "no_fire".to_string()]),
246            no: None, // Test that None doesn't crash
247            values: None,
248            klog: None,
249            syslog: None,
250            bootlog: None,
251            annotations: None,
252            now: None,
253        };
254        let good_trial = Trial {
255            yes: Some(vec!["fires".to_string()]),
256            no: Some(vec!["no_fire".to_string()]),
257            values: Some(HashMap::new()),
258            klog: None,
259            syslog: None,
260            bootlog: None,
261            annotations: None,
262            now: None,
263        };
264        assert!(validate(&create_parse_result!(
265            metrics: metrics,
266            actions: actions,
267            tests: build_map!(("foo", build_map!(("good", good_trial), ("bad", bad_trial))))
268        ))
269        .is_err());
270        // Make sure it objects if a trial fires when it shouldn't.
271        let bad_trial = Trial {
272            yes: Some(vec![]), // Test that empty vec works right
273            no: Some(vec!["fires".to_string(), "no_fire".to_string()]),
274            values: None,
275            klog: None,
276            syslog: None,
277            bootlog: None,
278            annotations: None,
279            now: None,
280        };
281        assert!(validate(&create_parse_result!(
282            metrics: metrics,
283            actions: actions,
284            tests: build_map!(("foo", build_map!(("bad", bad_trial))))
285        ))
286        .is_err());
287        Ok(())
288    }
289
290    #[fuchsia::test]
291    fn validate_time() -> Result<(), Error> {
292        let metrics = HashMap::new();
293        let actions = build_map!((
294            "foo",
295            build_map!(
296                (
297                    "time_quarter",
298                    Action::Alert(Alert {
299                        trigger: ValueSource::try_from_expression_with_namespace(
300                            "Now()==250000000",
301                            "foo"
302                        )?,
303                        print: "time_billion".to_string(),
304                        tag: None,
305                        file_bug: None,
306                        severity: Severity::Warning,
307                    })
308                ),
309                (
310                    "time_missing",
311                    Action::Alert(Alert {
312                        trigger: ValueSource::try_from_expression_with_namespace(
313                            "Missing(Now())",
314                            "foo"
315                        )?,
316                        print: "time_missing".to_string(),
317                        tag: None,
318                        file_bug: None,
319                        severity: Severity::Warning,
320                    })
321                )
322            )
323        ));
324        let time_trial = Trial {
325            yes: Some(vec!["time_quarter".to_string()]),
326            no: Some(vec!["time_missing".to_string()]),
327            values: Some(HashMap::new()),
328            klog: None,
329            syslog: None,
330            bootlog: None,
331            annotations: None,
332            now: Some("Seconds(0.25)".to_string()),
333        };
334        assert!(validate(&create_parse_result!(
335            metrics: metrics,
336            actions: actions,
337            tests: build_map!(("foo", build_map!(("good", time_trial))))
338        ))
339        .is_ok());
340        let missing_trial = Trial {
341            yes: Some(vec!["time_missing".to_string()]),
342            no: Some(vec![]),
343            values: Some(HashMap::new()),
344            klog: None,
345            syslog: None,
346            bootlog: None,
347            annotations: None,
348            now: None,
349        };
350        assert!(validate(&create_parse_result!(
351            metrics: metrics,
352            actions: actions,
353            tests: build_map!(("foo", build_map!(("good", missing_trial))))
354        ))
355        .is_ok());
356        let bad_trial = Trial {
357            yes: Some(vec!["time_missing".to_string()]),
358            no: Some(vec![]),
359            values: Some(HashMap::new()),
360            klog: None,
361            syslog: None,
362            bootlog: None,
363            annotations: None,
364            now: Some("this won't parse".to_string()),
365        };
366        assert!(validate(&create_parse_result!(
367            metrics: metrics,
368            actions: actions,
369            tests: build_map!(("foo", build_map!(("good", bad_trial))))
370        ))
371        .is_err());
372        Ok(())
373    }
374}
fuchsia_triage/validate.rs

fuchsia_triage/
validate.rs
