test_output_directory/
testing.rs

Help
1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::{ArtifactMetadata, MaybeUnknown, Outcome, SuiteResult, TestCaseResult, TestRunResult};
6use std::collections::{HashMap, HashSet};
7use std::ops::Deref;
8use std::path::{Path, PathBuf};
9use test_list::TestTag;
10
11enum MatchOption<T> {
12    AnyOrNone,
13    None,
14    Any,
15    Specified(T),
16}
17
18macro_rules! assert_match_option {
19    ($expected:expr, $actual:expr, $field:expr) => {
20        match $expected {
21            MatchOption::AnyOrNone => (),
22            MatchOption::None => {
23                assert_eq!(None, $actual, "Expected {} to be None but was {:?}", $field, $actual)
24            }
25            MatchOption::Any => {
26                assert!($actual.is_some(), "Expected {} to contain a value but was None", $field)
27            }
28            MatchOption::Specified(val) => assert_eq!(
29                Some(val),
30                $actual,
31                "Expected {} to be {:?} but was {:?}",
32                $field,
33                Some(val),
34                $actual
35            ),
36        }
37    };
38}
39
40/// Container that identifies the entity that is being verified in an assertion.
41#[derive(Clone, Copy)]
42enum EntityContext<'a> {
43    Run,
44    Suite(&'a ExpectedSuite),
45    Case(&'a ExpectedSuite, &'a ExpectedTestCase),
46}
47
48/// Container that identifies the artifact that is being verified in an assertion.
49#[derive(Clone, Copy)]
50struct ArtifactContext<'a, 'b> {
51    entity: &'a EntityContext<'b>,
52    metadata: &'a ArtifactMetadata,
53}
54
55impl std::fmt::Display for EntityContext<'_> {
56    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
57        match self {
58            Self::Run => write!(f, "TEST RUN"),
59            Self::Suite(suite) => write!(f, "SUITE {}", suite.name),
60            Self::Case(suite, case) => write!(f, "SUITE {}: CASE {}", suite.name, case.name),
61        }
62    }
63}
64
65impl std::fmt::Display for ArtifactContext<'_, '_> {
66    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
67        write!(f, "Entity: {}, Metadata: {:?}", self.entity, self.metadata)
68    }
69}
70
71/// A mapping from artifact metadata to assertions made on the artifact.
72type ArtifactMetadataToAssertionMap = HashMap<ArtifactMetadata, ExpectedArtifact>;
73
74/// Assert that the run results contained in `actual_run` and the directory specified by `root`
75/// contain the results and artifacts in `expected_run`.
76pub fn assert_run_result(root: &Path, expected_run: &ExpectedTestRun) {
77    let context = EntityContext::Run;
78    let actual_run = TestRunResult::from_dir(root).expect("Parse output directory");
79    let TestRunResult { common, suites } = actual_run;
80    assert_match_option!(
81        expected_run.duration_milliseconds,
82        common.deref().duration_milliseconds,
83        format!("Run duration for {}", context)
84    );
85    assert_match_option!(
86        expected_run.start_time,
87        common.deref().start_time,
88        format!("Start time for {}", context)
89    );
90    assert_eq!(common.deref().outcome, expected_run.outcome, "Outcome for {}", context);
91    assert_artifacts(
92        root,
93        &common.deref().artifact_dir.root,
94        &common.deref().artifact_dir.artifacts,
95        &expected_run.artifacts,
96        EntityContext::Run,
97    );
98    assert_suite_results(root, &suites, &expected_run.suites);
99}
100
101/// Assert that the suite results contained in `actual_suites` and the directory specified by `root`
102/// contain the suites, results, artifacts, and test cases in `expected_suite`.
103/// Note that this currently does not support duplicate suite names.
104fn assert_suite_results(
105    root: &Path,
106    actual_suites: &Vec<SuiteResult<'_>>,
107    expected_suites: &Vec<ExpectedSuite>,
108) {
109    assert_eq!(actual_suites.len(), expected_suites.len());
110    let mut expected_suites_map = HashMap::new();
111    for suite in expected_suites.iter() {
112        expected_suites_map.insert(suite.name.clone(), suite);
113    }
114    assert_eq!(
115        actual_suites.len(),
116        expected_suites_map.len(),
117        "Run contains multiple suites with the same name. \
118        This is currently unsupported by assert_suite_results"
119    );
120    for suite in actual_suites.iter() {
121        assert_suite_result(
122            root,
123            suite,
124            expected_suites_map
125                .get(&suite.common.deref().name)
126                .expect("No matching expected suite"),
127        );
128    }
129}
130
131/// Assert that the suite results contained in `actual_suite` and the directory specified by `root`
132/// contain the results, artifacts, and test cases in `expected_suite`.
133pub fn assert_suite_result(
134    root: &Path,
135    actual_suite: &SuiteResult<'_>,
136    expected_suite: &ExpectedSuite,
137) {
138    let context = EntityContext::Suite(expected_suite);
139    let &SuiteResult { common, cases, tags } = &actual_suite;
140    assert_eq!(common.deref().outcome, expected_suite.outcome, "Outcome for {}", context);
141    assert_eq!(common.deref().name, expected_suite.name, "Name for {}", context);
142    assert_match_option!(
143        expected_suite.duration_milliseconds,
144        common.deref().duration_milliseconds,
145        format!("Duration for {}", context)
146    );
147    assert_match_option!(
148        expected_suite.start_time,
149        common.deref().start_time,
150        format!("Start time for {}", context)
151    );
152
153    let mut tags: Vec<TestTag> = tags.clone().into_owned();
154    tags.sort();
155
156    let mut expected_tags = expected_suite.tags.clone();
157    expected_tags.sort();
158
159    assert_eq!(tags, expected_tags);
160
161    assert_artifacts(
162        root,
163        &common.deref().artifact_dir.root,
164        &common.deref().artifact_dir.artifacts,
165        &expected_suite.artifacts,
166        context,
167    );
168
169    assert_eq!(cases.len(), expected_suite.cases.len());
170    for case in cases.iter() {
171        let expected_case = expected_suite.cases.get(&case.common.deref().name);
172        assert!(
173            expected_case.is_some(),
174            "Found unexpected case {} in {}",
175            case.common.deref().name,
176            context
177        );
178        assert_case_result(root, case, expected_case.unwrap(), expected_suite);
179    }
180}
181
182fn assert_case_result(
183    root: &Path,
184    actual_case: &TestCaseResult<'_>,
185    expected_case: &ExpectedTestCase,
186    parent_suite: &ExpectedSuite,
187) {
188    let context = EntityContext::Case(parent_suite, expected_case);
189    assert_eq!(actual_case.common.deref().name, expected_case.name, "Name for {}", context);
190    assert_eq!(
191        actual_case.common.deref().outcome,
192        expected_case.outcome,
193        "Outcome for {}",
194        context
195    );
196    assert_match_option!(
197        expected_case.duration_milliseconds,
198        actual_case.common.deref().duration_milliseconds,
199        format!("Duration for {}", context)
200    );
201    assert_match_option!(
202        expected_case.start_time,
203        actual_case.common.deref().start_time,
204        format!("Start time for {}", context)
205    );
206    assert_artifacts(
207        root,
208        &actual_case.common.deref().artifact_dir.root,
209        &actual_case.common.deref().artifact_dir.artifacts,
210        &expected_case.artifacts,
211        context,
212    );
213}
214
215fn assert_artifacts(
216    root: &Path,
217    artifact_dir: &Path,
218    actual_artifacts: &HashMap<PathBuf, ArtifactMetadata>,
219    expected_artifacts: &ArtifactMetadataToAssertionMap,
220    entity_context: EntityContext<'_>,
221) {
222    // TODO(https://fxbug.dev/42051180): add options so that the test author can explicitly declare whether
223    // artifacts should be an exact match, should contain (and may contain more) artifacts,
224    // or any number of artifacts is accesptable.
225    // This skips artifact assertion for the typical case where verifying artifacts isn't
226    // necessary and allows the author to avoid listing out every artifact that is generated
227    // by the test.
228    if expected_artifacts.is_empty() {
229        return;
230    }
231
232    let actual_artifacts_by_metadata: HashMap<ArtifactMetadata, PathBuf> =
233        actual_artifacts.iter().map(|(key, value)| (value.clone(), key.clone())).collect();
234    // For now, artifact metadata should be unique for each artifact.
235    assert_eq!(
236        actual_artifacts_by_metadata.len(),
237        actual_artifacts.len(),
238        "Artifacts for {} do not have unique metadata. Actual artifacts: {:?}",
239        entity_context,
240        actual_artifacts
241    );
242
243    let expected_metadata: HashSet<_> = expected_artifacts.keys().collect();
244    let actual_metadata: HashSet<_> = actual_artifacts_by_metadata.keys().collect();
245
246    assert_eq!(
247        expected_metadata, actual_metadata,
248        "Artifacts for {} do not have matching metadata.",
249        entity_context,
250    );
251
252    for (expected_metadata, expected_artifact) in expected_artifacts.iter() {
253        let actual_filepath =
254            artifact_dir.join(actual_artifacts_by_metadata.get(expected_metadata).unwrap());
255        match expected_artifact {
256            ExpectedArtifact::File { name, assertion_fn } => {
257                assert_file(
258                    &root.join(&actual_filepath),
259                    name,
260                    assertion_fn,
261                    ArtifactContext { entity: &entity_context, metadata: expected_metadata },
262                );
263            }
264            ExpectedArtifact::Directory { files, name } => {
265                match name {
266                    None => (),
267                    Some(name) => assert_eq!(
268                        name.as_str(),
269                        actual_filepath.file_name().unwrap().to_str().unwrap(),
270                        "Expected filename {} for artifact matching {:?} but got {}",
271                        name,
272                        expected_metadata,
273                        actual_filepath.file_name().unwrap().to_str().unwrap()
274                    ),
275                }
276                let actual_entries: HashSet<_> = std::fs::read_dir(root.join(&actual_filepath))
277                    .expect("Failed to read directory artifact path")
278                    .map(|entry| match entry {
279                        Ok(dir_entry) if dir_entry.file_type().unwrap().is_file() => {
280                            dir_entry.file_name().to_str().unwrap().to_string()
281                        }
282                        // TODO(fxbugdev/85528) - support directory artifacts with subdirectories
283                        Ok(_) => panic!("Directory artifact with subdirectories unsupported"),
284                        Err(e) => panic!("Error reading directory artifact: {:?}", e),
285                    })
286                    .collect();
287                let expected_entries: HashSet<_> =
288                    files.iter().map(|(name, _)| name.to_string()).collect();
289                assert_eq!(
290                    actual_entries, expected_entries,
291                    "Expected files {:?} in directory artifact, got {:?}",
292                    &expected_entries, &actual_entries
293                );
294                for (name, assertion) in files {
295                    assert_file(
296                        &root.join(&actual_filepath).join(name),
297                        &None,
298                        assertion,
299                        ArtifactContext { entity: &entity_context, metadata: expected_metadata },
300                    );
301                }
302            }
303        }
304    }
305}
306
307fn assert_file(
308    file_path: &Path,
309    name: &Option<String>,
310    assertion_fn: &Box<dyn Fn(&str)>,
311    artifact_context: ArtifactContext<'_, '_>,
312) {
313    match name {
314        None => (),
315        Some(name) => assert_eq!(
316            name.as_str(),
317            file_path.file_name().unwrap().to_str().unwrap(),
318            "Got incorrect filename while checking file for artifact {}",
319            artifact_context
320        ),
321    }
322    let actual_contents = std::fs::read_to_string(&file_path);
323    (assertion_fn)(&actual_contents.unwrap());
324}
325
326/// The expected contents of an artifact.
327enum ExpectedArtifact {
328    /// An artifact contained in a single file, such as stdout.
329    File {
330        /// If given, the expected name of the file.
331        name: Option<String>,
332        /// Assertion run against the contents of the file.
333        assertion_fn: Box<dyn Fn(&str)>,
334    },
335    /// An artifact consisting of files in a directory.
336    Directory {
337        /// List of expected files, as (name, assertion) pairs. The name
338        /// is the expected name of the file, and the assertion fn is run
339        /// against the contents of the file.
340        files: Vec<(String, Box<dyn Fn(&str)>)>,
341        /// If given, the expected name of the directory.
342        name: Option<String>,
343    },
344}
345
346/// Contents of an expected directory artifact.
347pub struct ExpectedDirectory {
348    files: Vec<(String, Box<dyn Fn(&str)>)>,
349}
350
351impl ExpectedDirectory {
352    /// Create a new empty expected directory.
353    pub fn new() -> Self {
354        Self { files: vec![] }
355    }
356
357    /// Add a file with expected |contents|.
358    pub fn with_file(self, name: impl AsRef<str>, contents: impl AsRef<str>) -> Self {
359        let owned_expected = contents.as_ref().to_string();
360        let owned_name = name.as_ref().to_string();
361        self.with_matching_file(name, move |actual| {
362            assert_eq!(
363                &owned_expected, actual,
364                "Mismatch in contents of file {}. Expected: '{}', actual:'{}'",
365                owned_name, &owned_expected, actual
366            )
367        })
368    }
369
370    pub fn with_matching_file(
371        mut self,
372        name: impl AsRef<str>,
373        matcher: impl 'static + Fn(&str),
374    ) -> Self {
375        self.files.push((name.as_ref().to_string(), Box::new(matcher)));
376        self
377    }
378}
379
380/// A version of a test run result that contains all output in memory. This should only be used
381/// for making assertions in a test.
382pub struct ExpectedTestRun {
383    artifacts: ArtifactMetadataToAssertionMap,
384    outcome: MaybeUnknown<Outcome>,
385    start_time: MatchOption<u64>,
386    duration_milliseconds: MatchOption<u64>,
387    suites: Vec<ExpectedSuite>,
388}
389
390/// A version of a suite run result that contains all output in memory. This should only be used
391/// for making assertions in a test.
392pub struct ExpectedSuite {
393    artifacts: ArtifactMetadataToAssertionMap,
394    name: String,
395    outcome: MaybeUnknown<Outcome>,
396    cases: HashMap<String, ExpectedTestCase>,
397    start_time: MatchOption<u64>,
398    duration_milliseconds: MatchOption<u64>,
399    tags: Vec<TestTag>,
400}
401
402/// A version of a test case result that contains all output in memory. This should only be used
403/// for making assertions in a test.
404pub struct ExpectedTestCase {
405    artifacts: ArtifactMetadataToAssertionMap,
406    name: String,
407    outcome: MaybeUnknown<Outcome>,
408    start_time: MatchOption<u64>,
409    duration_milliseconds: MatchOption<u64>,
410}
411
412macro_rules! common_impl {
413    {} => {
414        /// Add an artifact matching the exact contents. Artifacts are checked by finding
415        /// an entry matching the given metadata, then checking the contents of the corresponding
416        /// file. If |name| is provided, the name of the file is verified. Artifacts are keyed by
417        /// metadata rather than by name as the names of files are not guaranteed to be stable.
418        pub fn with_artifact<S, T, U>(
419            self, metadata: U, name: Option<S>, contents: T
420        ) -> Self
421        where
422            S: AsRef<str>,
423            T: AsRef<str>,
424            U: Into<ArtifactMetadata>
425        {
426            let owned_expected = contents.as_ref().to_string();
427            let metadata = metadata.into();
428            let metadata_clone = metadata.clone();
429            self.with_matching_artifact(metadata, name, move |actual| {
430                assert_eq!(
431                    &owned_expected, actual,
432                    "Mismatch in artifact with metadata {:?}. Expected: '{}', actual:'{}'",
433                    metadata_clone, &owned_expected, actual
434                )
435            })
436        }
437
438        /// Add an artifact matching the exact contents. Artifacts are checked by finding
439        /// an entry matching the given metadata, then running |matcher| against the contents of
440        /// the file. If |name| is provided, the name of the file is verified. Artifacts are keyed
441        /// by metadata rather than by name as the names of files are not guaranteed to be stable.
442        pub fn with_matching_artifact<S, F, U>(
443            mut self,
444            metadata: U,
445            name: Option<S>,
446            matcher: F,
447        ) -> Self
448        where
449            S: AsRef<str>,
450            F: 'static + Fn(&str),
451            U: Into<ArtifactMetadata>
452        {
453            self.artifacts.insert(
454                metadata.into(),
455                ExpectedArtifact::File {
456                    name: name.map(|s| s.as_ref().to_string()),
457                    assertion_fn: Box::new(matcher),
458                }
459            );
460            self
461        }
462
463        /// Add a directory based artifact containing the entries described in |directory|.
464        pub fn with_directory_artifact<S, U>(
465            mut self,
466            metadata: U,
467            name: Option<S>,
468            directory: ExpectedDirectory,
469        ) -> Self
470        where
471            S: AsRef<str>,
472            U: Into<ArtifactMetadata>
473        {
474            self.artifacts.insert(
475                metadata.into(),
476                ExpectedArtifact::Directory {
477                    name: name.map(|s| s.as_ref().to_string()),
478                    files: directory.files,
479                }
480            );
481            self
482        }
483
484        /// Verify an exact start time.
485        pub fn with_start_time(mut self, millis: u64) -> Self {
486            self.start_time = MatchOption::Specified(millis);
487            self
488        }
489
490        /// Verify an exact run duration.
491        pub fn with_run_duration(mut self, millis: u64) -> Self {
492            self.duration_milliseconds = MatchOption::Specified(millis);
493            self
494        }
495
496        /// Verify that a start time is present.
497        pub fn with_any_start_time(mut self) -> Self {
498            self.start_time = MatchOption::Any;
499            self
500        }
501
502        /// Verify that a run duration is present.
503        pub fn with_any_run_duration(mut self) -> Self {
504            self.duration_milliseconds = MatchOption::Any;
505            self
506        }
507
508        /// Verify that no start time is present.
509        pub fn with_no_start_time(mut self) -> Self {
510            self.start_time = MatchOption::None;
511            self
512        }
513
514        /// Verify that no run duration is present.
515        pub fn with_no_run_duration(mut self) -> Self {
516            self.duration_milliseconds = MatchOption::None;
517            self
518        }
519    };
520}
521
522impl ExpectedTestRun {
523    /// Create a new `ExpectedTestRun` with the given `outcome`.
524    pub fn new(outcome: Outcome) -> Self {
525        Self {
526            artifacts: ArtifactMetadataToAssertionMap::new(),
527            outcome: outcome.into(),
528            start_time: MatchOption::AnyOrNone,
529            duration_milliseconds: MatchOption::AnyOrNone,
530            suites: vec![],
531        }
532    }
533
534    pub fn with_suite(mut self, suite: ExpectedSuite) -> Self {
535        self.suites.push(suite);
536        self
537    }
538
539    common_impl! {}
540}
541
542impl ExpectedSuite {
543    /// Create a new `ExpectedTestRun` with the given `name` and `outcome`.
544    pub fn new<S: AsRef<str>>(name: S, outcome: Outcome) -> Self {
545        Self {
546            artifacts: ArtifactMetadataToAssertionMap::new(),
547            name: name.as_ref().to_string(),
548            outcome: outcome.into(),
549            cases: HashMap::new(),
550            start_time: MatchOption::AnyOrNone,
551            duration_milliseconds: MatchOption::AnyOrNone,
552            tags: vec![],
553        }
554    }
555
556    /// Add a test case to the suite.
557    pub fn with_case(mut self, case: ExpectedTestCase) -> Self {
558        self.cases.insert(case.name.clone(), case);
559        self
560    }
561
562    /// Add a tag to the suite.
563    pub fn with_tag(mut self, tag: TestTag) -> Self {
564        self.tags.push(tag);
565        self
566    }
567
568    common_impl! {}
569}
570
571impl ExpectedTestCase {
572    /// Create a new `ExpectedTestCase` with the given `name` and `outcome`.
573    pub fn new<S: AsRef<str>>(name: S, outcome: Outcome) -> Self {
574        Self {
575            artifacts: ArtifactMetadataToAssertionMap::new(),
576            name: name.as_ref().to_string(),
577            outcome: outcome.into(),
578            start_time: MatchOption::AnyOrNone,
579            duration_milliseconds: MatchOption::AnyOrNone,
580        }
581    }
582
583    common_impl! {}
584}
585
586#[cfg(test)]
587mod test {
588    use super::*;
589    use crate::{ArtifactType, CommonResult, OutputDirectoryBuilder, SchemaVersion, RUN_NAME};
590    use std::borrow::Cow;
591    use std::io::Write;
592
593    fn test_with_directory<F: Fn(OutputDirectoryBuilder)>(_test_name: &str, test_fn: F) {
594        for version in SchemaVersion::all_variants() {
595            let dir = tempfile::TempDir::new().unwrap();
596            let directory_builder =
597                OutputDirectoryBuilder::new(dir.path(), version).expect("Create directory builder");
598            test_fn(directory_builder);
599        }
600    }
601
602    #[fixture::fixture(test_with_directory)]
603    #[test]
604    fn assert_run_result_check_outcome_only(output_dir: OutputDirectoryBuilder) {
605        let actual = TestRunResult {
606            common: Cow::Owned(CommonResult {
607                name: RUN_NAME.to_string(),
608                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
609                outcome: Outcome::Passed.into(),
610                start_time: Some(64),
611                duration_milliseconds: Some(128),
612            }),
613            suites: vec![],
614        };
615
616        output_dir.save_summary(&actual).expect("save summary");
617        assert_run_result(
618            output_dir.path(),
619            &ExpectedTestRun::new(Outcome::Passed).with_any_start_time().with_any_run_duration(),
620        );
621    }
622
623    #[fixture::fixture(test_with_directory)]
624    #[test]
625    fn assert_run_result_check_exact_timing(output_dir: OutputDirectoryBuilder) {
626        let actual = TestRunResult {
627            common: Cow::Owned(CommonResult {
628                name: RUN_NAME.to_string(),
629                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
630                outcome: Outcome::Passed.into(),
631                start_time: Some(64),
632                duration_milliseconds: Some(128),
633            }),
634            suites: vec![],
635        };
636
637        output_dir.save_summary(&actual).expect("save summary");
638        assert_run_result(
639            output_dir.path(),
640            &ExpectedTestRun::new(Outcome::Passed).with_start_time(64).with_run_duration(128),
641        );
642    }
643
644    #[fixture::fixture(test_with_directory)]
645    #[test]
646    fn assert_run_result_check_timing_unspecified(output_dir: OutputDirectoryBuilder) {
647        let actual = TestRunResult {
648            common: Cow::Owned(CommonResult {
649                name: RUN_NAME.to_string(),
650                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
651                outcome: Outcome::Passed.into(),
652                start_time: None,
653                duration_milliseconds: None,
654            }),
655            suites: vec![],
656        };
657
658        output_dir.save_summary(&actual).expect("save summary");
659        assert_run_result(
660            output_dir.path(),
661            &ExpectedTestRun::new(Outcome::Passed).with_no_start_time().with_no_run_duration(),
662        );
663    }
664
665    #[fixture::fixture(test_with_directory)]
666    #[test]
667    fn assert_run_result_single_artifact_unspecified_name(output_dir: OutputDirectoryBuilder) {
668        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
669        let mut artifact =
670            artifact_dir.new_artifact(ArtifactType::Syslog, "b.txt").expect("create artifact");
671        write!(artifact, "hello").expect("write to artifact");
672        drop(artifact);
673
674        let actual = TestRunResult {
675            common: Cow::Owned(CommonResult {
676                name: RUN_NAME.to_string(),
677                artifact_dir,
678                outcome: Outcome::Passed.into(),
679                start_time: None,
680                duration_milliseconds: None,
681            }),
682            suites: vec![],
683        };
684
685        output_dir.save_summary(&actual).expect("save summary");
686        assert_run_result(
687            output_dir.path(),
688            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
689                ArtifactType::Syslog,
690                Option::<&str>::None,
691                "hello",
692            ),
693        );
694    }
695
696    #[fixture::fixture(test_with_directory)]
697    #[test]
698    fn assert_run_result_single_artifact_specified_name(output_dir: OutputDirectoryBuilder) {
699        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
700        let mut artifact =
701            artifact_dir.new_artifact(ArtifactType::Syslog, "b.txt").expect("create artifact");
702        write!(artifact, "hello").expect("write to artifact");
703        drop(artifact);
704
705        let actual = TestRunResult {
706            common: Cow::Owned(CommonResult {
707                name: RUN_NAME.to_string(),
708                artifact_dir,
709                outcome: Outcome::Passed.into(),
710                start_time: None,
711                duration_milliseconds: None,
712            }),
713            suites: vec![],
714        };
715
716        output_dir.save_summary(&actual).expect("save summary");
717        assert_run_result(
718            output_dir.path(),
719            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
720                ArtifactType::Syslog,
721                "b.txt".into(),
722                "hello",
723            ),
724        );
725    }
726
727    #[fixture::fixture(test_with_directory)]
728    #[test]
729    #[should_panic(expected = "Outcome for TEST RUN")]
730    fn assert_run_outcome_mismatch(output_dir: OutputDirectoryBuilder) {
731        let actual = TestRunResult {
732            common: Cow::Owned(CommonResult {
733                name: RUN_NAME.to_string(),
734                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
735                outcome: Outcome::Failed.into(),
736                start_time: None,
737                duration_milliseconds: None,
738            }),
739            suites: vec![],
740        };
741
742        output_dir.save_summary(&actual).expect("save summary");
743        assert_run_result(output_dir.path(), &ExpectedTestRun::new(Outcome::Passed));
744    }
745
746    #[fixture::fixture(test_with_directory)]
747    #[test]
748    #[should_panic(expected = "Start time for TEST RUN")]
749    fn assert_run_start_time_mismatch(output_dir: OutputDirectoryBuilder) {
750        let actual = TestRunResult {
751            common: Cow::Owned(CommonResult {
752                name: RUN_NAME.to_string(),
753                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
754                outcome: Outcome::Failed.into(),
755                start_time: Some(64),
756                duration_milliseconds: None,
757            }),
758            suites: vec![],
759        };
760
761        output_dir.save_summary(&actual).expect("save summary");
762        assert_run_result(
763            output_dir.path(),
764            &ExpectedTestRun::new(Outcome::Passed).with_start_time(23),
765        );
766    }
767
768    #[fixture::fixture(test_with_directory)]
769    #[test]
770    #[should_panic(expected = "Run duration for TEST RUN")]
771    fn assert_run_duration_mismatch(output_dir: OutputDirectoryBuilder) {
772        let actual = TestRunResult {
773            common: Cow::Owned(CommonResult {
774                name: RUN_NAME.to_string(),
775                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
776                outcome: Outcome::Failed.into(),
777                start_time: None,
778                duration_milliseconds: None,
779            }),
780            suites: vec![],
781        };
782
783        output_dir.save_summary(&actual).expect("save summary");
784        assert_run_result(
785            output_dir.path(),
786            &ExpectedTestRun::new(Outcome::Passed).with_run_duration(23),
787        );
788    }
789
790    #[fixture::fixture(test_with_directory)]
791    #[test]
792    #[should_panic]
793    fn assert_run_artifact_mismatch(output_dir: OutputDirectoryBuilder) {
794        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
795        let mut artifact =
796            artifact_dir.new_artifact(ArtifactType::Syslog, "missing").expect("create artifact");
797        write!(artifact, "hello").expect("write to artifact");
798        drop(artifact);
799
800        let actual = TestRunResult {
801            common: Cow::Owned(CommonResult {
802                name: RUN_NAME.to_string(),
803                artifact_dir,
804                outcome: Outcome::Failed.into(),
805                start_time: None,
806                duration_milliseconds: None,
807            }),
808            suites: vec![],
809        };
810
811        output_dir.save_summary(&actual).expect("save summary");
812        assert_run_result(
813            output_dir.path(),
814            &ExpectedTestRun::new(Outcome::Failed).with_artifact(
815                ArtifactType::Stderr,
816                "stderr.txt".into(),
817                "",
818            ),
819        );
820    }
821
822    fn passing_run_with_single_suite<'a>(
823        output_dir: &OutputDirectoryBuilder,
824        suite: SuiteResult<'a>,
825    ) -> TestRunResult<'a> {
826        TestRunResult {
827            common: Cow::Owned(CommonResult {
828                name: RUN_NAME.to_string(),
829                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
830                outcome: Outcome::Passed.into(),
831                start_time: Some(64),
832                duration_milliseconds: Some(128),
833            }),
834            suites: vec![suite],
835        }
836    }
837
838    #[fixture::fixture(test_with_directory)]
839    #[test]
840    fn assert_run_result_with_suite(output_dir: OutputDirectoryBuilder) {
841        let actual = passing_run_with_single_suite(
842            &output_dir,
843            SuiteResult {
844                common: Cow::Owned(CommonResult {
845                    name: "suite".to_string(),
846                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
847                    outcome: Outcome::Passed.into(),
848                    start_time: Some(64),
849                    duration_milliseconds: Some(128),
850                }),
851                cases: vec![],
852                tags: Cow::Owned(vec![]),
853            },
854        );
855
856        output_dir.save_summary(&actual).expect("save summary");
857        assert_run_result(
858            output_dir.path(),
859            &ExpectedTestRun::new(Outcome::Passed)
860                .with_any_start_time()
861                .with_any_run_duration()
862                .with_suite(
863                    ExpectedSuite::new("suite", Outcome::Passed)
864                        .with_any_start_time()
865                        .with_any_run_duration(),
866                ),
867        );
868    }
869
870    #[fixture::fixture(test_with_directory)]
871    #[test]
872    fn assert_run_result_with_suite_exact_times(output_dir: OutputDirectoryBuilder) {
873        let actual = passing_run_with_single_suite(
874            &output_dir,
875            SuiteResult {
876                common: Cow::Owned(CommonResult {
877                    name: "suite".to_string(),
878                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
879                    outcome: Outcome::Passed.into(),
880                    start_time: Some(64),
881                    duration_milliseconds: Some(128),
882                }),
883                cases: vec![],
884                tags: Cow::Owned(vec![]),
885            },
886        );
887
888        output_dir.save_summary(&actual).expect("save summary");
889        assert_run_result(
890            output_dir.path(),
891            &ExpectedTestRun::new(Outcome::Passed)
892                .with_any_start_time()
893                .with_any_run_duration()
894                .with_suite(
895                    ExpectedSuite::new("suite", Outcome::Passed)
896                        .with_start_time(64)
897                        .with_run_duration(128),
898                ),
899        );
900    }
901
902    #[fixture::fixture(test_with_directory)]
903    #[test]
904    fn assert_run_result_with_suite_no_times(output_dir: OutputDirectoryBuilder) {
905        let actual = passing_run_with_single_suite(
906            &output_dir,
907            SuiteResult {
908                common: Cow::Owned(CommonResult {
909                    name: "suite".to_string(),
910                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
911                    outcome: Outcome::Passed.into(),
912                    start_time: None,
913                    duration_milliseconds: None,
914                }),
915                cases: vec![],
916                tags: Cow::Owned(vec![]),
917            },
918        );
919
920        output_dir.save_summary(&actual).expect("save summary");
921        assert_run_result(
922            output_dir.path(),
923            &ExpectedTestRun::new(Outcome::Passed)
924                .with_any_start_time()
925                .with_any_run_duration()
926                .with_suite(
927                    ExpectedSuite::new("suite", Outcome::Passed)
928                        .with_no_start_time()
929                        .with_no_run_duration(),
930                ),
931        );
932    }
933
934    #[fixture::fixture(test_with_directory)]
935    #[test]
936    fn assert_run_result_suite_with_artifact(output_dir: OutputDirectoryBuilder) {
937        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
938        let mut artifact =
939            artifact_dir.new_artifact(ArtifactType::Syslog, "b.txt").expect("create artifact");
940        write!(artifact, "hello").expect("write to artifact");
941        drop(artifact);
942
943        let actual = passing_run_with_single_suite(
944            &output_dir,
945            SuiteResult {
946                common: Cow::Owned(CommonResult {
947                    name: "suite".to_string(),
948                    artifact_dir,
949                    outcome: Outcome::Passed.into(),
950                    start_time: None,
951                    duration_milliseconds: None,
952                }),
953                cases: vec![],
954                tags: Cow::Owned(vec![]),
955            },
956        );
957
958        output_dir.save_summary(&actual).expect("save summary");
959        assert_run_result(
960            output_dir.path(),
961            &ExpectedTestRun::new(Outcome::Passed)
962                .with_any_start_time()
963                .with_any_run_duration()
964                .with_suite(ExpectedSuite::new("suite", Outcome::Passed).with_artifact(
965                    ArtifactType::Syslog,
966                    "b.txt".into(),
967                    "hello",
968                )),
969        );
970    }
971
972    #[fixture::fixture(test_with_directory)]
973    #[test]
974    fn assert_run_result_suite_with_case(output_dir: OutputDirectoryBuilder) {
975        let actual = passing_run_with_single_suite(
976            &output_dir,
977            SuiteResult {
978                common: Cow::Owned(CommonResult {
979                    name: "suite".to_string(),
980                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
981                    outcome: Outcome::Passed.into(),
982                    start_time: None,
983                    duration_milliseconds: None,
984                }),
985                cases: vec![TestCaseResult {
986                    common: Cow::Owned(CommonResult {
987                        name: "case".to_string(),
988                        artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
989                        outcome: Outcome::Passed.into(),
990                        start_time: None,
991                        duration_milliseconds: None,
992                    }),
993                }],
994                tags: Cow::Owned(vec![]),
995            },
996        );
997
998        output_dir.save_summary(&actual).expect("save summary");
999        assert_run_result(
1000            output_dir.path(),
1001            &ExpectedTestRun::new(Outcome::Passed)
1002                .with_any_start_time()
1003                .with_any_run_duration()
1004                .with_suite(
1005                    ExpectedSuite::new("suite", Outcome::Passed).with_case(
1006                        ExpectedTestCase::new("case", Outcome::Passed)
1007                            .with_no_run_duration()
1008                            .with_no_start_time(),
1009                    ),
1010                ),
1011        );
1012    }
1013
1014    #[fixture::fixture(test_with_directory)]
1015    #[test]
1016    fn assert_run_result_suite_with_tags(output_dir: OutputDirectoryBuilder) {
1017        let actual = passing_run_with_single_suite(
1018            &output_dir,
1019            SuiteResult {
1020                common: Cow::Owned(CommonResult {
1021                    name: "suite".to_string(),
1022                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1023                    outcome: Outcome::Passed.into(),
1024                    start_time: None,
1025                    duration_milliseconds: None,
1026                }),
1027                cases: vec![],
1028                tags: Cow::Owned(vec![
1029                    TestTag { key: "os".to_string(), value: "fuchsia".to_string() },
1030                    TestTag { key: "cpu".to_string(), value: "arm64".to_string() },
1031                ]),
1032            },
1033        );
1034
1035        output_dir.save_summary(&actual).expect("save summary");
1036        assert_run_result(
1037            output_dir.path(),
1038            &ExpectedTestRun::new(Outcome::Passed)
1039                .with_any_start_time()
1040                .with_any_run_duration()
1041                .with_suite(
1042                    ExpectedSuite::new("suite", Outcome::Passed)
1043                        .with_tag(TestTag { key: "cpu".to_string(), value: "arm64".to_string() })
1044                        .with_tag(TestTag { key: "os".to_string(), value: "fuchsia".to_string() }),
1045                ),
1046        );
1047    }
1048
1049    #[fixture::fixture(test_with_directory)]
1050    #[test]
1051    #[should_panic(expected = "Outcome for SUITE suite")]
1052    fn assert_suite_outcome_mismatch(output_dir: OutputDirectoryBuilder) {
1053        let actual = passing_run_with_single_suite(
1054            &output_dir,
1055            SuiteResult {
1056                common: Cow::Owned(CommonResult {
1057                    name: "suite".to_string(),
1058                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1059                    outcome: Outcome::Failed.into(),
1060                    start_time: None,
1061                    duration_milliseconds: None,
1062                }),
1063                cases: vec![],
1064                tags: Cow::Owned(vec![]),
1065            },
1066        );
1067
1068        output_dir.save_summary(&actual).expect("save summary");
1069        assert_run_result(
1070            output_dir.path(),
1071            &ExpectedTestRun::new(Outcome::Passed)
1072                .with_any_start_time()
1073                .with_any_run_duration()
1074                .with_suite(ExpectedSuite::new("suite", Outcome::Passed)),
1075        );
1076    }
1077
1078    #[fixture::fixture(test_with_directory)]
1079    #[test]
1080    #[should_panic(expected = "Start time for SUITE suite")]
1081    fn assert_suite_start_time_mismatch(output_dir: OutputDirectoryBuilder) {
1082        let actual = passing_run_with_single_suite(
1083            &output_dir,
1084            SuiteResult {
1085                common: Cow::Owned(CommonResult {
1086                    name: "suite".to_string(),
1087                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1088                    outcome: Outcome::Passed.into(),
1089                    start_time: None,
1090                    duration_milliseconds: Some(128),
1091                }),
1092                cases: vec![],
1093                tags: Cow::Owned(vec![]),
1094            },
1095        );
1096
1097        output_dir.save_summary(&actual).expect("save summary");
1098        assert_run_result(
1099            output_dir.path(),
1100            &ExpectedTestRun::new(Outcome::Passed)
1101                .with_any_start_time()
1102                .with_any_run_duration()
1103                .with_suite(ExpectedSuite::new("suite", Outcome::Passed).with_any_start_time()),
1104        );
1105    }
1106
1107    #[fixture::fixture(test_with_directory)]
1108    #[test]
1109    #[should_panic(expected = "Duration for SUITE suite")]
1110    fn assert_suite_duration_mismatch(output_dir: OutputDirectoryBuilder) {
1111        let actual = passing_run_with_single_suite(
1112            &output_dir,
1113            SuiteResult {
1114                common: Cow::Owned(CommonResult {
1115                    name: "suite".to_string(),
1116                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1117                    outcome: Outcome::Passed.into(),
1118                    start_time: None,
1119                    duration_milliseconds: Some(128),
1120                }),
1121                cases: vec![],
1122                tags: Cow::Owned(vec![]),
1123            },
1124        );
1125
1126        output_dir.save_summary(&actual).expect("save summary");
1127        assert_run_result(
1128            output_dir.path(),
1129            &ExpectedTestRun::new(Outcome::Passed)
1130                .with_any_start_time()
1131                .with_any_run_duration()
1132                .with_suite(ExpectedSuite::new("suite", Outcome::Passed).with_run_duration(32)),
1133        );
1134    }
1135
1136    #[fixture::fixture(test_with_directory)]
1137    #[test]
1138    #[should_panic]
1139    fn assert_suite_artifact_mismatch(output_dir: OutputDirectoryBuilder) {
1140        let actual = passing_run_with_single_suite(
1141            &output_dir,
1142            SuiteResult {
1143                common: Cow::Owned(CommonResult {
1144                    name: "suite".to_string(),
1145                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1146                    outcome: Outcome::Passed.into(),
1147                    start_time: None,
1148                    duration_milliseconds: Some(128),
1149                }),
1150                cases: vec![],
1151                tags: Cow::Owned(vec![]),
1152            },
1153        );
1154
1155        output_dir.save_summary(&actual).expect("save summary");
1156        assert_run_result(
1157            output_dir.path(),
1158            &ExpectedTestRun::new(Outcome::Passed)
1159                .with_any_start_time()
1160                .with_any_run_duration()
1161                .with_suite(ExpectedSuite::new("suite", Outcome::Passed).with_artifact(
1162                    ArtifactType::Stderr,
1163                    Option::<&str>::None,
1164                    "missing contents",
1165                )),
1166        );
1167    }
1168
1169    #[fixture::fixture(test_with_directory)]
1170    #[test]
1171    #[should_panic(expected = "Found unexpected case")]
1172    fn assert_suite_case_mismatch(output_dir: OutputDirectoryBuilder) {
1173        let actual = passing_run_with_single_suite(
1174            &output_dir,
1175            SuiteResult {
1176                common: Cow::Owned(CommonResult {
1177                    name: "suite".to_string(),
1178                    artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1179                    outcome: Outcome::Failed.into(),
1180                    start_time: None,
1181                    duration_milliseconds: None,
1182                }),
1183                cases: vec![TestCaseResult {
1184                    common: Cow::Owned(CommonResult {
1185                        name: "case".to_string(),
1186                        artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1187                        outcome: Outcome::Passed.into(),
1188                        start_time: None,
1189                        duration_milliseconds: None,
1190                    }),
1191                }],
1192                tags: Cow::Owned(vec![]),
1193            },
1194        );
1195
1196        output_dir.save_summary(&actual).expect("save summary");
1197        assert_run_result(
1198            output_dir.path(),
1199            &ExpectedTestRun::new(Outcome::Passed).with_any_start_time().with_suite(
1200                ExpectedSuite::new("suite", Outcome::Failed)
1201                    .with_case(ExpectedTestCase::new("wrong name", Outcome::Passed)),
1202            ),
1203        );
1204    }
1205
1206    #[fixture::fixture(test_with_directory)]
1207    #[test]
1208    fn assert_artifacts_empty(output_dir: OutputDirectoryBuilder) {
1209        let actual = TestRunResult {
1210            common: Cow::Owned(CommonResult {
1211                name: RUN_NAME.to_string(),
1212                artifact_dir: output_dir.new_artifact_dir().expect("new artifact dir"),
1213                outcome: Outcome::Passed.into(),
1214                start_time: None,
1215                duration_milliseconds: None,
1216            }),
1217            suites: vec![],
1218        };
1219
1220        output_dir.save_summary(&actual).expect("save summary");
1221        assert_run_result(output_dir.path(), &ExpectedTestRun::new(Outcome::Passed));
1222    }
1223
1224    #[fixture::fixture(test_with_directory)]
1225    #[test]
1226    fn assert_artifacts_exact_content(output_dir: OutputDirectoryBuilder) {
1227        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1228        let mut artifact =
1229            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("new artifact");
1230        write!(artifact, "hello").expect("write to artifact");
1231        let actual = TestRunResult {
1232            common: Cow::Owned(CommonResult {
1233                name: RUN_NAME.to_string(),
1234                artifact_dir,
1235                outcome: Outcome::Passed.into(),
1236                start_time: None,
1237                duration_milliseconds: None,
1238            }),
1239            suites: vec![],
1240        };
1241
1242        output_dir.save_summary(&actual).expect("save summary");
1243        assert_run_result(
1244            output_dir.path(),
1245            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1246                ArtifactType::Stderr,
1247                Option::<&str>::None,
1248                "hello",
1249            ),
1250        );
1251    }
1252
1253    #[fixture::fixture(test_with_directory)]
1254    #[test]
1255    fn assert_artifacts_exact_content_exact_name(output_dir: OutputDirectoryBuilder) {
1256        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1257        let mut artifact =
1258            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("new artifact");
1259        write!(artifact, "hello").expect("write to artifact");
1260        let actual = TestRunResult {
1261            common: Cow::Owned(CommonResult {
1262                name: RUN_NAME.to_string(),
1263                artifact_dir,
1264                outcome: Outcome::Passed.into(),
1265                start_time: None,
1266                duration_milliseconds: None,
1267            }),
1268            suites: vec![],
1269        };
1270
1271        output_dir.save_summary(&actual).expect("save summary");
1272        assert_run_result(
1273            output_dir.path(),
1274            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1275                ArtifactType::Stderr,
1276                Some("b.txt"),
1277                "hello",
1278            ),
1279        );
1280    }
1281
1282    #[fixture::fixture(test_with_directory)]
1283    #[test]
1284    fn assert_artifacts_matching_content(output_dir: OutputDirectoryBuilder) {
1285        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1286        let mut artifact =
1287            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("new artifact");
1288        write!(artifact, "hello").expect("write to artifact");
1289        let actual = TestRunResult {
1290            common: Cow::Owned(CommonResult {
1291                name: RUN_NAME.to_string(),
1292                artifact_dir,
1293                outcome: Outcome::Passed.into(),
1294                start_time: None,
1295                duration_milliseconds: None,
1296            }),
1297            suites: vec![],
1298        };
1299
1300        output_dir.save_summary(&actual).expect("save summary");
1301        assert_run_result(
1302            output_dir.path(),
1303            &ExpectedTestRun::new(Outcome::Passed).with_matching_artifact(
1304                ArtifactType::Stderr,
1305                Some("b.txt"),
1306                |content| assert_eq!(content, "hello"),
1307            ),
1308        );
1309    }
1310
1311    #[fixture::fixture(test_with_directory)]
1312    #[test]
1313    fn assert_artifacts_moniker_specified(output_dir: OutputDirectoryBuilder) {
1314        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1315        let mut artifact = artifact_dir
1316            .new_artifact(
1317                ArtifactMetadata {
1318                    artifact_type: ArtifactType::Syslog.into(),
1319                    component_moniker: Some("moniker".into()),
1320                },
1321                "b.txt",
1322            )
1323            .expect("new artifact");
1324        write!(artifact, "hello").expect("write to artifact");
1325        let actual = TestRunResult {
1326            common: Cow::Owned(CommonResult {
1327                name: RUN_NAME.to_string(),
1328                artifact_dir,
1329                outcome: Outcome::Passed.into(),
1330                start_time: None,
1331                duration_milliseconds: None,
1332            }),
1333            suites: vec![],
1334        };
1335
1336        output_dir.save_summary(&actual).expect("save summary");
1337        assert_run_result(
1338            output_dir.path(),
1339            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1340                ArtifactMetadata {
1341                    artifact_type: ArtifactType::Syslog.into(),
1342                    component_moniker: Some("moniker".into()),
1343                },
1344                Some("b.txt"),
1345                "hello",
1346            ),
1347        );
1348    }
1349
1350    #[fixture::fixture(test_with_directory)]
1351    #[test]
1352    fn assert_artifacts_directory_artifact(output_dir: OutputDirectoryBuilder) {
1353        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1354        let dir_artifact =
1355            artifact_dir.new_directory_artifact(ArtifactType::Custom, "b").expect("new artifact");
1356        std::fs::write(dir_artifact.join("c.txt"), "hello c").unwrap();
1357        std::fs::write(dir_artifact.join("d.txt"), "hello d").unwrap();
1358        let actual = TestRunResult {
1359            common: Cow::Owned(CommonResult {
1360                name: RUN_NAME.to_string(),
1361                artifact_dir,
1362                outcome: Outcome::Passed.into(),
1363                start_time: None,
1364                duration_milliseconds: None,
1365            }),
1366            suites: vec![],
1367        };
1368
1369        output_dir.save_summary(&actual).expect("save summary");
1370        assert_run_result(
1371            output_dir.path(),
1372            &ExpectedTestRun::new(Outcome::Passed).with_directory_artifact(
1373                ArtifactType::Custom,
1374                Some("b"),
1375                ExpectedDirectory::new()
1376                    .with_file("c.txt", "hello c")
1377                    .with_matching_file("d.txt", |contents| assert_eq!(contents, "hello d")),
1378            ),
1379        );
1380    }
1381
1382    #[fixture::fixture(test_with_directory)]
1383    #[test]
1384    #[should_panic(expected = "Artifacts for TEST RUN")]
1385    fn assert_artifacts_missing(output_dir: OutputDirectoryBuilder) {
1386        let artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1387        let actual = TestRunResult {
1388            common: Cow::Owned(CommonResult {
1389                name: RUN_NAME.to_string(),
1390                artifact_dir,
1391                outcome: Outcome::Passed.into(),
1392                start_time: None,
1393                duration_milliseconds: None,
1394            }),
1395            suites: vec![],
1396        };
1397
1398        output_dir.save_summary(&actual).expect("save summary");
1399        assert_run_result(
1400            output_dir.path(),
1401            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1402                ArtifactType::Syslog,
1403                Some("missing"),
1404                "missing contents",
1405            ),
1406        );
1407    }
1408
1409    #[fixture::fixture(test_with_directory)]
1410    #[test]
1411    #[should_panic(expected = "Artifacts for TEST RUN")]
1412    fn assert_artifacts_extra_artifact(output_dir: OutputDirectoryBuilder) {
1413        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1414        let mut file_b =
1415            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("create artifact");
1416        write!(file_b, "hello").unwrap();
1417        let mut file_c =
1418            artifact_dir.new_artifact(ArtifactType::Stdout, "c.txt").expect("create artifact");
1419        write!(file_c, "hello").unwrap();
1420        drop(file_b);
1421        drop(file_c);
1422        let actual = TestRunResult {
1423            common: Cow::Owned(CommonResult {
1424                name: RUN_NAME.to_string(),
1425                artifact_dir,
1426                outcome: Outcome::Passed.into(),
1427                start_time: None,
1428                duration_milliseconds: None,
1429            }),
1430            suites: vec![],
1431        };
1432
1433        output_dir.save_summary(&actual).expect("save summary");
1434        assert_run_result(
1435            output_dir.path(),
1436            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1437                ArtifactType::Stderr,
1438                "c.txt".into(),
1439                "hello",
1440            ),
1441        );
1442    }
1443
1444    #[fixture::fixture(test_with_directory)]
1445    #[test]
1446    #[should_panic]
1447    fn assert_artifacts_content_not_equal(output_dir: OutputDirectoryBuilder) {
1448        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1449        let mut file_b =
1450            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("create artifact");
1451        write!(file_b, "wrong content").unwrap();
1452        drop(file_b);
1453        let actual = TestRunResult {
1454            common: Cow::Owned(CommonResult {
1455                name: RUN_NAME.to_string(),
1456                artifact_dir,
1457                outcome: Outcome::Passed.into(),
1458                start_time: None,
1459                duration_milliseconds: None,
1460            }),
1461            suites: vec![],
1462        };
1463
1464        output_dir.save_summary(&actual).expect("save summary");
1465        assert_run_result(
1466            output_dir.path(),
1467            &ExpectedTestRun::new(Outcome::Passed).with_artifact(
1468                ArtifactType::Syslog,
1469                Option::<&str>::None,
1470                "expected content",
1471            ),
1472        );
1473    }
1474
1475    #[fixture::fixture(test_with_directory)]
1476    #[test]
1477    #[should_panic]
1478    fn assert_artifacts_content_does_not_match(output_dir: OutputDirectoryBuilder) {
1479        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1480        let mut file_b =
1481            artifact_dir.new_artifact(ArtifactType::Stderr, "b.txt").expect("create artifact");
1482        write!(file_b, "wrong content").unwrap();
1483        drop(file_b);
1484        let actual = TestRunResult {
1485            common: Cow::Owned(CommonResult {
1486                name: RUN_NAME.to_string(),
1487                artifact_dir,
1488                outcome: Outcome::Passed.into(),
1489                start_time: None,
1490                duration_milliseconds: None,
1491            }),
1492            suites: vec![],
1493        };
1494
1495        output_dir.save_summary(&actual).expect("save summary");
1496        assert_run_result(
1497            output_dir.path(),
1498            &ExpectedTestRun::new(Outcome::Passed).with_matching_artifact(
1499                ArtifactType::Syslog,
1500                Option::<&str>::None,
1501                |content| assert_eq!(content, "expected content"),
1502            ),
1503        );
1504    }
1505
1506    #[fixture::fixture(test_with_directory)]
1507    #[test]
1508    #[should_panic]
1509    fn assert_artifacts_directory_mismatch(output_dir: OutputDirectoryBuilder) {
1510        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1511        let dir_artifact =
1512            artifact_dir.new_directory_artifact(ArtifactType::Custom, "b").expect("new artifact");
1513        std::fs::write(dir_artifact.join("c.txt"), "unexpected file").unwrap();
1514        let actual = TestRunResult {
1515            common: Cow::Owned(CommonResult {
1516                name: RUN_NAME.to_string(),
1517                artifact_dir,
1518                outcome: Outcome::Passed.into(),
1519                start_time: None,
1520                duration_milliseconds: None,
1521            }),
1522            suites: vec![],
1523        };
1524
1525        output_dir.save_summary(&actual).expect("save summary");
1526        assert_run_result(
1527            output_dir.path(),
1528            &ExpectedTestRun::new(Outcome::Passed).with_directory_artifact(
1529                ArtifactType::Custom,
1530                Option::<&str>::None,
1531                ExpectedDirectory::new(),
1532            ),
1533        );
1534    }
1535
1536    #[fixture::fixture(test_with_directory)]
1537    #[test]
1538    fn assert_artifacts_not_checked_if_unspecified(output_dir: OutputDirectoryBuilder) {
1539        let mut artifact_dir = output_dir.new_artifact_dir().expect("new artifact dir");
1540        let mut file_c =
1541            artifact_dir.new_artifact(ArtifactType::Stderr, "c.txt").expect("create artifact");
1542        write!(file_c, "unexpected file").unwrap();
1543        drop(file_c);
1544        let actual = TestRunResult {
1545            common: Cow::Owned(CommonResult {
1546                name: RUN_NAME.to_string(),
1547                artifact_dir,
1548                outcome: Outcome::Passed.into(),
1549                start_time: None,
1550                duration_milliseconds: None,
1551            }),
1552            suites: vec![],
1553        };
1554
1555        output_dir.save_summary(&actual).expect("save summary");
1556        assert_run_result(output_dir.path(), &ExpectedTestRun::new(Outcome::Passed));
1557    }
1558}
test_output_directory/testing.rs

test_output_directory/
testing.rs
