run_test_suite_lib/
outcome.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use flex_fuchsia_test_manager as ftest_manager;
6use std::fmt;
7use std::sync::Arc;
8use thiserror::Error;
9
10#[derive(Debug, Clone)]
11pub enum Outcome {
12    Passed,
13    Failed,
14    Inconclusive,
15    Timedout,
16    /// Suite was stopped prematurely due to cancellation by the user.
17    Cancelled,
18    /// Suite did not report completion.
19    // TODO(https://fxbug.dev/42171445) - this outcome indicates an internal error as test manager isn't
20    // sending expected events. We should return an error instead.
21    DidNotFinish,
22    Error {
23        origin: Arc<RunTestSuiteError>,
24    },
25}
26
27// Extends `Outcome` by adding setup/teardown results needed for ResultDB.
28#[derive(Debug, Clone, PartialEq)]
29pub struct ExtendedOutcome {
30    pub outcome: Outcome,
31    pub setup_succeeded: Option<bool>,
32    pub teardown_succeeded: Option<bool>,
33}
34
35impl Outcome {
36    pub(crate) fn error<E: Into<RunTestSuiteError>>(e: E) -> Self {
37        Self::Error { origin: Arc::new(e.into()) }
38    }
39}
40
41impl PartialEq for Outcome {
42    fn eq(&self, other: &Self) -> bool {
43        match (self, other) {
44            (Self::Passed, Self::Passed)
45            | (Self::Failed, Self::Failed)
46            | (Self::Inconclusive, Self::Inconclusive)
47            | (Self::Timedout, Self::Timedout)
48            | (Self::Cancelled, Self::Cancelled)
49            | (Self::DidNotFinish, Self::DidNotFinish) => true,
50            (Self::Error { origin }, Self::Error { origin: other_origin }) => {
51                format!("{}", origin.as_ref()) == format!("{}", other_origin.as_ref())
52            }
53            (_, _) => false,
54        }
55    }
56}
57
58impl fmt::Display for Outcome {
59    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
60        match self {
61            Outcome::Passed => write!(f, "PASSED"),
62            Outcome::Failed => write!(f, "FAILED"),
63            Outcome::Inconclusive => write!(f, "INCONCLUSIVE"),
64            Outcome::Timedout => write!(f, "TIMED OUT"),
65            Outcome::Cancelled => write!(f, "CANCELLED"),
66            Outcome::DidNotFinish => write!(f, "DID_NOT_FINISH"),
67            Outcome::Error { .. } => write!(f, "ERROR"),
68        }
69    }
70}
71
72#[derive(Error, Debug)]
73/// An enum of the different errors that may be encountered while running
74/// a test.
75pub enum RunTestSuiteError {
76    #[error("fidl error: {0:?}")]
77    Fidl(#[from] fidl::Error),
78    #[error("error launching test suite: {}", convert_launch_error_to_str(.0))]
79    Launch(ftest_manager::LaunchError),
80    #[error("error reporting test results: {0:?}")]
81    Io(#[from] std::io::Error),
82    #[error("unexpected event: {0:?}")]
83    UnexpectedEvent(#[from] UnexpectedEventError),
84    #[error("Error connecting to SuiteRunner protocol: {0:?}")]
85    Connection(#[from] ConnectionError),
86}
87
88/// An error returned when test manager reports an unexpected event.
89/// This could occur if test manager violates guarantees about event
90/// ordering.
91#[derive(Error, Debug)]
92pub enum UnexpectedEventError {
93    #[error(
94        "received a 'started' event for case with id {test_case_id:?} but no 'case_found' event"
95    )]
96    CaseStartedButNotFound { test_case_id: u32 },
97    #[error(
98        "invalid case event to '{next_state:?}' received while in state '{last_state:?}' for case {test_case_name:?} with id {test_case_id:?}"
99    )]
100    InvalidCaseEvent {
101        last_state: Lifecycle,
102        next_state: Lifecycle,
103        test_case_name: String,
104        test_case_id: u32,
105    },
106    #[error(
107        "received an 'artifact' event for case with id {test_case_id:?} but no 'case_found' event"
108    )]
109    CaseArtifactButNotFound { test_case_id: u32 },
110    #[error(
111        "received an 'artifact' event for case with id {test_case_id:?} but the case is already finished"
112    )]
113    CaseArtifactButFinished { test_case_id: u32 },
114    #[error(
115        "received a '{next_state:?}' event for case with id {test_case_id:?} but no 'case_found' event"
116    )]
117    CaseEventButNotFound { next_state: Lifecycle, test_case_id: u32 },
118    #[error("received a 'stopped' event for case with id {test_case_id:?} but no 'started' event")]
119    UnrecognizedTestCaseResult { result: ftest_manager::TestCaseResult, test_case_id: u32 },
120    #[error("server closed channel without reporting finish for cases: {cases:?}")]
121    CasesDidNotFinish { cases: Vec<String> },
122    #[error("invalid event to '{next_state:?}' received while in state '{last_state:?}'")]
123    InvalidEvent { last_state: Lifecycle, next_state: Lifecycle },
124    #[error("received an unhandled suite result: {result:?}")]
125    UnrecognizedSuiteResult { result: ftest_manager::SuiteResult },
126    #[error("server closed channel without reporting a result for the suite")]
127    SuiteDidNotReportStop,
128    #[error("received an InternalError suite result")]
129    InternalErrorSuiteResult,
130    #[error("missing required field {field} in {containing_struct}")]
131    MissingRequiredField { containing_struct: &'static str, field: &'static str },
132}
133
134#[derive(Debug, Error)]
135#[error(transparent)]
136pub struct ConnectionError(pub anyhow::Error);
137
138/// Lifecycle of a test suite or test case.
139/// This is internal implementation for ::crate::run, but is located here so it can be reported
140/// for debugging via RunTestSuiteError.
141#[derive(Clone, Copy, Debug, PartialEq)]
142pub enum Lifecycle {
143    Found,
144    Started,
145    Stopped,
146    Finished,
147}
148
149impl RunTestSuiteError {
150    /// Returns true iff the error variant indicates an internal error in
151    /// Test Manager or ffx.
152    pub fn is_internal_error(&self) -> bool {
153        match self {
154            Self::Fidl(_) => true,
155            Self::Launch(ftest_manager::LaunchError::InternalError) => true,
156            Self::Launch(_) => false,
157            Self::Io(_) => true,
158            Self::UnexpectedEvent(_) => true,
159            Self::Connection(_) => true,
160        }
161    }
162}
163
164impl From<ftest_manager::LaunchError> for RunTestSuiteError {
165    fn from(launch: ftest_manager::LaunchError) -> Self {
166        Self::Launch(launch)
167    }
168}
169
170fn convert_launch_error_to_str(e: &ftest_manager::LaunchError) -> &'static str {
171    match e {
172        ftest_manager::LaunchError::CaseEnumeration => {
173            "Cannot enumerate test. This may mean `fuchsia.test.Suite` was not configured correctly. Refer to: \
174        https://fuchsia.dev/go/components/test-errors"
175        }
176        ftest_manager::LaunchError::ResourceUnavailable => "Resource unavailable",
177        ftest_manager::LaunchError::InstanceCannotResolve => "Cannot resolve test.",
178        ftest_manager::LaunchError::InvalidArgs => {
179            "Invalid args passed to builder while adding suite. Please file bug"
180        }
181        ftest_manager::LaunchError::FailedToConnectToTestSuite => {
182            "Cannot communicate with the tests. This may mean `fuchsia.test.Suite` was not \
183            configured correctly. Refer to: \
184            https://fuchsia.dev/go/components/test-errors"
185        }
186        ftest_manager::LaunchError::InternalError => "Internal error, please file bug",
187        ftest_manager::LaunchError::NoMatchingCases =>
188        // TODO(satsukiu): Ideally, we would expose these error enums up through the library
189        // and define the error messages in the main files for each tool. This would allow each
190        // tool (ffx/fx/run-test-suite) to give the correct flags.
191        {
192            "No test cases matched the specified filters.\n\
193            If you specified a test filter, verify the available test cases with \
194            'ffx test list-cases <test suite url>'.\n\
195            If the list of available tests contains only a single test case called either \
196            'legacy_test' or 'main', the suite likely uses either the legacy_test_runner or \
197            elf_test_runner. In these cases, --test-filter will not work. Instead, \
198            you can pass test arguments directly to the test instead. Refer to: \
199            https://fuchsia.dev/go/components/test-runners"
200        }
201        ftest_manager::LaunchError::InvalidManifest => {
202            "The test manifest is invalid or has invalid facets/arguments. Please check logs for detailed error."
203        }
204        ftest_manager::LaunchErrorUnknown!() => "Unrecognized launch error",
205    }
206}
run_test_suite_lib/outcome.rs

run_test_suite_lib/
outcome.rs
