test_output_directory/
lib.rs

Help
1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5mod macros;
6pub mod testing;
7mod v1;
8
9use serde::{Deserialize, Serialize};
10use std::borrow::Cow;
11use std::collections::hash_map::Iter;
12use std::collections::HashMap;
13use std::fs::{DirBuilder, File};
14use std::io::Error;
15use std::path::{Path, PathBuf};
16use test_list::TestTag;
17
18/// Filename of the top level summary json.
19pub const RUN_SUMMARY_NAME: &str = "run_summary.json";
20pub const RUN_NAME: &str = "run";
21
22enumerable_enum! {
23    /// Schema version.
24    #[derive(PartialEq, Eq, Debug, Clone, Copy, Hash)]
25    SchemaVersion {
26        V1,
27    }
28}
29
30enumerable_enum! {
31    /// A serializable version of a test outcome.
32    #[derive(Deserialize, Serialize, PartialEq, Eq, Debug, Clone, Copy)]
33    #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
34    Outcome {
35        NotStarted,
36        Passed,
37        Failed,
38        Inconclusive,
39        Timedout,
40        Error,
41        Skipped,
42    }
43}
44
45enumerable_enum! {
46    /// Types of artifacts known to the test framework.
47    #[derive(Deserialize, Serialize, PartialEq, Eq, Debug, Clone, Copy, Hash)]
48    #[serde(rename_all = "SCREAMING_SNAKE_CASE")]
49    ArtifactType {
50        Syslog,
51        /// Unexpected high severity logs that caused a test to fail.
52        RestrictedLog,
53        Stdout,
54        Stderr,
55        /// A directory containing custom artifacts produced by a component in the test.
56        Custom,
57        /// A human readable report generated by the test framework.
58        Report,
59        /// Debug data. For example, profraw or symbolizer output.
60        Debug,
61    }
62}
63
64/// A subdirectory of an output directory that contains artifacts for a test run,
65/// test suite, or test case.
66#[derive(PartialEq, Eq, Debug, Clone)]
67pub struct ArtifactSubDirectory {
68    version: SchemaVersion,
69    root: PathBuf,
70    artifacts: HashMap<PathBuf, ArtifactMetadata>,
71}
72
73/// Contains result information common to all results. It's useful to store
74#[derive(PartialEq, Eq, Debug, Clone)]
75pub struct CommonResult {
76    pub name: String,
77    pub artifact_dir: ArtifactSubDirectory,
78    pub outcome: MaybeUnknown<Outcome>,
79    /// Approximate start time, as milliseconds since the epoch.
80    pub start_time: Option<u64>,
81    pub duration_milliseconds: Option<u64>,
82}
83
84/// A serializable test run result.
85/// This contains overall results and artifacts scoped to a test run, and
86/// a list of filenames for finding serialized suite results.
87#[derive(PartialEq, Eq, Debug, Clone)]
88pub struct TestRunResult<'a> {
89    pub common: Cow<'a, CommonResult>,
90    pub suites: Vec<SuiteResult<'a>>,
91}
92
93/// A serializable suite run result.
94/// Contains overall results and artifacts scoped to a suite run, and
95/// results and artifacts scoped to any test run within it.
96#[derive(Debug, Clone, PartialEq, Eq)]
97pub struct SuiteResult<'a> {
98    pub common: Cow<'a, CommonResult>,
99    pub cases: Vec<TestCaseResult<'a>>,
100    pub tags: Cow<'a, Vec<TestTag>>,
101}
102
103/// A serializable test case result.
104#[derive(PartialEq, Eq, Debug, Clone)]
105pub struct TestCaseResult<'a> {
106    pub common: Cow<'a, CommonResult>,
107}
108
109impl TestRunResult<'static> {
110    pub fn from_dir(root: &Path) -> Result<Self, Error> {
111        v1::parse_from_directory(root)
112    }
113}
114
115/// Metadata associated with an artifact.
116#[derive(Deserialize, Serialize, PartialEq, Eq, Debug, Clone, Hash)]
117pub struct ArtifactMetadata {
118    /// The type of the artifact.
119    pub artifact_type: MaybeUnknown<ArtifactType>,
120    /// Moniker of the component which produced the artifact, if applicable.
121    #[serde(skip_serializing_if = "Option::is_none")]
122    pub component_moniker: Option<String>,
123}
124
125#[derive(Deserialize, Serialize, PartialEq, Eq, Debug, Hash, Clone)]
126#[serde(untagged)]
127pub enum MaybeUnknown<T> {
128    Known(T),
129    Unknown(String),
130}
131
132impl<T> From<T> for MaybeUnknown<T> {
133    fn from(other: T) -> Self {
134        Self::Known(other)
135    }
136}
137
138impl From<ArtifactType> for ArtifactMetadata {
139    fn from(other: ArtifactType) -> Self {
140        Self { artifact_type: MaybeUnknown::Known(other), component_moniker: None }
141    }
142}
143
144/// A helper for accumulating results in an output directory.
145///
146/// |OutputDirectoryBuilder| handles details specific to the format of the test output
147/// format, such as the locations of summaries and artifacts, while allowing the caller to
148/// accumulate results separately. A typical usecase might look like this:
149/// ```rust
150/// let output_directory = OutputDirectoryBuilder::new("/path", SchemaVersion::V1)?;
151/// let mut run_result = TestRunResult {
152///     common: Cow::Owned(CommonResult{
153///         name: "run".to_string(),
154///         artifact_dir: output_directory.new_artifact_dir("run-artifacts")?,
155///         outcome: Outcome::Inconclusive.into(),
156///         start_time: None,
157///         duration_milliseconds: None,
158///     }),
159///     suites: vec![],
160/// };
161///
162/// // accumulate results in run_result over time... then save the summary.
163/// output_directory.save_summary(&run_result)?;
164/// ```
165pub struct OutputDirectoryBuilder {
166    version: SchemaVersion,
167    root: PathBuf,
168    /// Creation instant exists purely to make pseudo random directory names to discourage parsing
169    /// methods that rely on unstable directory names and could be broken when internal
170    /// implementation changes.
171    creation_instant: std::time::Instant,
172}
173
174impl OutputDirectoryBuilder {
175    /// Register a directory for use as an output directory using version |version|.
176    pub fn new(dir: impl Into<PathBuf>, version: SchemaVersion) -> Result<Self, Error> {
177        let root = dir.into();
178        ensure_directory_exists(&root)?;
179        Ok(Self { version, root, creation_instant: std::time::Instant::now() })
180    }
181
182    /// Create a new artifact subdirectory.
183    ///
184    /// The new |ArtifactSubDirectory| should be referenced from either the test run, suite, or
185    /// case when a summary is saved in this OutputDirectoryBuilder with |save_summary|.
186    pub fn new_artifact_dir(&self) -> Result<ArtifactSubDirectory, Error> {
187        match self.version {
188            SchemaVersion::V1 => {
189                let subdir_root =
190                    self.root.join(format!("{:?}", self.creation_instant.elapsed().as_nanos()));
191                Ok(ArtifactSubDirectory {
192                    version: self.version,
193                    root: subdir_root,
194                    artifacts: HashMap::new(),
195                })
196            }
197        }
198    }
199
200    /// Save a summary of the test results in the directory.
201    pub fn save_summary<'a, 'b>(&'a self, result: &'a TestRunResult<'b>) -> Result<(), Error> {
202        match self.version {
203            SchemaVersion::V1 => v1::save_summary(self.root.as_path(), result),
204        }
205    }
206
207    /// Get the path to the root directory.
208    pub fn path(&self) -> &Path {
209        self.root.as_path()
210    }
211}
212
213impl ArtifactSubDirectory {
214    /// Create a new file based artifact.
215    pub fn new_artifact(
216        &mut self,
217        metadata: impl Into<ArtifactMetadata>,
218        name: impl AsRef<Path>,
219    ) -> Result<File, Error> {
220        ensure_directory_exists(self.root.as_path())?;
221        match self.version {
222            SchemaVersion::V1 => {
223                // todo validate path
224                self.artifacts.insert(name.as_ref().to_path_buf(), metadata.into());
225                File::create(self.root.join(name))
226            }
227        }
228    }
229
230    /// Create a new directory based artifact.
231    pub fn new_directory_artifact(
232        &mut self,
233        metadata: impl Into<ArtifactMetadata>,
234        name: impl AsRef<Path>,
235    ) -> Result<PathBuf, Error> {
236        match self.version {
237            SchemaVersion::V1 => {
238                // todo validate path
239                let subdir = self.root.join(name.as_ref());
240                ensure_directory_exists(subdir.as_path())?;
241                self.artifacts.insert(name.as_ref().to_path_buf(), metadata.into());
242                Ok(subdir)
243            }
244        }
245    }
246
247    /// Get the absolute path of the artifact at |name|, if present.
248    pub fn path_to_artifact(&self, name: impl AsRef<Path>) -> Option<PathBuf> {
249        match self.version {
250            SchemaVersion::V1 => match self.artifacts.contains_key(name.as_ref()) {
251                true => Some(self.root.join(name.as_ref())),
252                false => None,
253            },
254        }
255    }
256
257    /// Return a list of paths of artifacts in the directory, relative to the root of the artifact
258    /// directory.
259    pub fn contents(&self) -> Vec<PathBuf> {
260        self.artifacts.keys().cloned().collect()
261    }
262
263    /// Return an iterator over the artifacts in the directory.
264    ///
265    /// Includes paths relative to the root of the artifact directory and the associated metadata.
266    pub fn artifact_iter(&self) -> Iter<'_, PathBuf, ArtifactMetadata> {
267        self.artifacts.iter()
268    }
269}
270
271fn ensure_directory_exists(dir: &Path) -> Result<(), Error> {
272    match dir.exists() {
273        true => Ok(()),
274        false => DirBuilder::new().recursive(true).create(&dir),
275    }
276}
277
278#[cfg(test)]
279mod test {
280    use super::*;
281    use std::io::Write;
282    use tempfile::tempdir;
283
284    fn validate_against_schema(version: SchemaVersion, root: &Path) {
285        match version {
286            SchemaVersion::V1 => v1::validate_against_schema(root),
287        }
288    }
289
290    /// Run a round trip test against all known schema versions.
291    fn round_trip_test_all_versions<F>(produce_run_fn: F)
292    where
293        F: Fn(&OutputDirectoryBuilder) -> TestRunResult<'static>,
294    {
295        for version in SchemaVersion::all_variants() {
296            let dir = tempdir().expect("Create dir");
297            let output_dir = OutputDirectoryBuilder::new(dir.path(), version).expect("create dir");
298
299            let run_result = produce_run_fn(&output_dir);
300            output_dir.save_summary(&run_result).expect("save summary");
301
302            validate_against_schema(version, dir.path());
303
304            let parsed = TestRunResult::from_dir(dir.path()).expect("parse output directory");
305            assert_eq!(run_result, parsed, "version: {:?}", version);
306        }
307    }
308
309    #[test]
310    fn minimal() {
311        round_trip_test_all_versions(|dir_builder| TestRunResult {
312            common: Cow::Owned(CommonResult {
313                name: RUN_NAME.to_string(),
314                artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
315                outcome: Outcome::Passed.into(),
316                start_time: None,
317                duration_milliseconds: None,
318            }),
319            suites: vec![],
320        });
321        let _ = Outcome::all_variants();
322    }
323
324    #[test]
325    fn artifact_types() {
326        for artifact_type in ArtifactType::all_variants() {
327            round_trip_test_all_versions(|dir_builder| {
328                let mut run_artifact_dir = dir_builder.new_artifact_dir().expect("new dir");
329                let mut run_artifact =
330                    run_artifact_dir.new_artifact(artifact_type, "a.txt").expect("create artifact");
331                write!(run_artifact, "run contents").unwrap();
332
333                let mut suite_artifact_dir = dir_builder.new_artifact_dir().expect("new dir");
334                let mut suite_artifact = suite_artifact_dir
335                    .new_artifact(artifact_type, "a.txt")
336                    .expect("create artifact");
337                write!(suite_artifact, "suite contents").unwrap();
338
339                let mut case_artifact_dir = dir_builder.new_artifact_dir().expect("new dir");
340                let mut case_artifact = case_artifact_dir
341                    .new_artifact(artifact_type, "a.txt")
342                    .expect("create artifact");
343                write!(case_artifact, "case contents").unwrap();
344
345                TestRunResult {
346                    common: Cow::Owned(CommonResult {
347                        name: RUN_NAME.to_string(),
348                        artifact_dir: run_artifact_dir,
349                        outcome: Outcome::Passed.into(),
350                        start_time: None,
351                        duration_milliseconds: None,
352                    }),
353                    suites: vec![SuiteResult {
354                        common: Cow::Owned(CommonResult {
355                            name: "suite".to_string(),
356                            artifact_dir: suite_artifact_dir,
357                            outcome: Outcome::Passed.into(),
358                            start_time: None,
359                            duration_milliseconds: None,
360                        }),
361                        tags: Cow::Owned(vec![]),
362                        cases: vec![TestCaseResult {
363                            common: Cow::Owned(CommonResult {
364                                name: "case".to_string(),
365                                artifact_dir: case_artifact_dir,
366                                outcome: Outcome::Passed.into(),
367                                start_time: None,
368                                duration_milliseconds: None,
369                            }),
370                        }],
371                    }],
372                }
373            });
374        }
375    }
376
377    #[test]
378    fn artifact_types_moniker_specified() {
379        for artifact_type in ArtifactType::all_variants() {
380            round_trip_test_all_versions(|dir_builder| {
381                let mut run_artifact_dir = dir_builder.new_artifact_dir().expect("new dir");
382                let mut run_artifact = run_artifact_dir
383                    .new_artifact(
384                        ArtifactMetadata {
385                            artifact_type: artifact_type.into(),
386                            component_moniker: Some("moniker".to_string()),
387                        },
388                        "a.txt",
389                    )
390                    .expect("create artifact");
391                write!(run_artifact, "run contents").unwrap();
392
393                let mut suite_artifact_dir = dir_builder.new_artifact_dir().expect("new dir");
394                let mut suite_artifact = suite_artifact_dir
395                    .new_artifact(
396                        ArtifactMetadata {
397                            artifact_type: artifact_type.into(),
398                            component_moniker: Some("moniker".to_string()),
399                        },
400                        "a.txt",
401                    )
402                    .expect("create artifact");
403                write!(suite_artifact, "suite contents").unwrap();
404
405                let mut case_artifact_dir = dir_builder.new_artifact_dir().expect("new dir");
406                let mut case_artifact = case_artifact_dir
407                    .new_artifact(
408                        ArtifactMetadata {
409                            artifact_type: artifact_type.into(),
410                            component_moniker: Some("moniker".to_string()),
411                        },
412                        "a.txt",
413                    )
414                    .expect("create artifact");
415                write!(case_artifact, "case contents").unwrap();
416
417                TestRunResult {
418                    common: Cow::Owned(CommonResult {
419                        name: RUN_NAME.to_string(),
420                        artifact_dir: run_artifact_dir,
421                        outcome: Outcome::Passed.into(),
422                        start_time: None,
423                        duration_milliseconds: None,
424                    }),
425                    suites: vec![SuiteResult {
426                        common: Cow::Owned(CommonResult {
427                            name: "suite".to_string(),
428                            artifact_dir: suite_artifact_dir,
429                            outcome: Outcome::Passed.into(),
430                            start_time: None,
431                            duration_milliseconds: None,
432                        }),
433                        tags: Cow::Owned(vec![]),
434                        cases: vec![TestCaseResult {
435                            common: Cow::Owned(CommonResult {
436                                name: "case".to_string(),
437                                artifact_dir: case_artifact_dir,
438                                outcome: Outcome::Passed.into(),
439                                start_time: None,
440                                duration_milliseconds: None,
441                            }),
442                        }],
443                    }],
444                }
445            });
446        }
447    }
448
449    #[test]
450    fn outcome_types() {
451        for outcome_type in Outcome::all_variants() {
452            round_trip_test_all_versions(|dir_builder| TestRunResult {
453                common: Cow::Owned(CommonResult {
454                    name: RUN_NAME.to_string(),
455                    artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
456                    outcome: outcome_type.into(),
457                    start_time: None,
458                    duration_milliseconds: None,
459                }),
460                suites: vec![SuiteResult {
461                    common: Cow::Owned(CommonResult {
462                        name: "suite".to_string(),
463                        artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
464                        outcome: outcome_type.into(),
465                        start_time: None,
466                        duration_milliseconds: None,
467                    }),
468                    tags: Cow::Owned(vec![]),
469                    cases: vec![TestCaseResult {
470                        common: Cow::Owned(CommonResult {
471                            name: "case".to_string(),
472                            artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
473                            outcome: outcome_type.into(),
474                            start_time: None,
475                            duration_milliseconds: None,
476                        }),
477                    }],
478                }],
479            });
480        }
481    }
482
483    #[test]
484    fn timing_specified() {
485        round_trip_test_all_versions(|dir_builder| TestRunResult {
486            common: Cow::Owned(CommonResult {
487                name: RUN_NAME.to_string(),
488                artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
489                outcome: Outcome::Passed.into(),
490                start_time: Some(1),
491                duration_milliseconds: Some(2),
492            }),
493            suites: vec![SuiteResult {
494                common: Cow::Owned(CommonResult {
495                    name: "suite".to_string(),
496                    artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
497                    outcome: Outcome::Passed.into(),
498                    start_time: Some(3),
499                    duration_milliseconds: Some(4),
500                }),
501                tags: Cow::Owned(vec![]),
502                cases: vec![TestCaseResult {
503                    common: Cow::Owned(CommonResult {
504                        name: "case".to_string(),
505                        artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
506                        outcome: Outcome::Passed.into(),
507                        start_time: Some(5),
508                        duration_milliseconds: Some(6),
509                    }),
510                }],
511            }],
512        });
513    }
514
515    #[test]
516    fn tags_specified() {
517        round_trip_test_all_versions(|dir_builder| TestRunResult {
518            common: Cow::Owned(CommonResult {
519                name: RUN_NAME.to_string(),
520                artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
521                outcome: Outcome::Passed.into(),
522                start_time: Some(1),
523                duration_milliseconds: Some(2),
524            }),
525            suites: vec![SuiteResult {
526                common: Cow::Owned(CommonResult {
527                    name: "suite".to_string(),
528                    artifact_dir: dir_builder.new_artifact_dir().expect("new dir"),
529                    outcome: Outcome::Passed.into(),
530                    start_time: Some(3),
531                    duration_milliseconds: Some(4),
532                }),
533                tags: Cow::Owned(vec![
534                    TestTag { key: "hermetic".to_string(), value: "false".to_string() },
535                    TestTag { key: "realm".to_string(), value: "system".to_string() },
536                ]),
537                cases: vec![],
538            }],
539        });
540    }
541}
test_output_directory/lib.rs

test_output_directory/
lib.rs
