run_test_suite_lib/output/shell/
mod.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::output::noop::NoopDirectoryWriter;
6use crate::output::{
7    ArtifactType, DirectoryArtifactType, DynArtifact, DynDirectoryArtifact, EntityId, EntityInfo,
8    ReportedOutcome, Reporter, Timestamp,
9};
10use fuchsia_async as fasync;
11use fuchsia_sync::Mutex;
12use log::error;
13use std::collections::HashMap;
14use std::io::{Error, Write};
15use std::sync::Arc;
16use std::sync::atomic::AtomicU32;
17use std::time::Duration;
18
19#[cfg(feature = "fdomain")]
20use test_diagnostics_fdomain as test_diagnostics;
21
22mod writer;
23pub use writer::ShellWriterView;
24use writer::{ShellWriterHandle, ShellWriterHandleInner};
25
26/// Duration after which to emit an excessive duration log.
27const EXCESSIVE_DURATION: Duration = Duration::from_secs(60);
28/// Buffer stdout and stderr for this duration before dumping to console.
29const STDIO_BUFFERING_DURATION: Duration = Duration::from_secs(5);
30/// Dump stdout and stderr to console if it exceeds this size.
31const STDIO_BUFFER_SIZE: usize = 4096;
32
33/// A reporter that outputs results and artifacts to a single stream, usually stdout.
34/// This reporter is intended to provide "live" updates to a developer watching while
35/// tests are executed.
36pub struct ShellReporter<W: 'static + Write + Send + Sync> {
37    /// Arc around the writer and state, used to dispense more handles.
38    inner: Arc<Mutex<ShellWriterHandleInner<W>>>,
39    /// Map containing known information about each entity.
40    entity_state_map: Mutex<HashMap<EntityId, EntityState>>,
41    /// Number of completed suites, used to output
42    completed_suites: AtomicU32,
43    /// Number of suites expected in the run.
44    expected_suites: Mutex<Option<u32>>,
45    /// Size of the buffer used for stdio.
46    stdio_buffer_size: usize,
47    /// Length of time to buffer stdio before printing it to console.
48    stdio_buffer_duration: Duration,
49}
50
51/// All known state needed by a |ShellReporter| to display results.
52struct EntityState {
53    name: String,
54    excessive_duration_task: Option<fasync::Task<()>>,
55    children: Vec<EntityId>,
56    restricted_logs: Option<ShellWriterView<Vec<u8>>>,
57    run_state: EntityRunningState,
58}
59
60enum EntityRunningState {
61    NotRunning,
62    Started,
63    Finished(ReportedOutcome),
64}
65
66impl EntityState {
67    fn new<S: Into<String>>(name: S) -> Self {
68        Self {
69            name: name.into(),
70            excessive_duration_task: None,
71            children: vec![],
72            restricted_logs: None,
73            run_state: EntityRunningState::NotRunning,
74        }
75    }
76
77    fn name(&self) -> &str {
78        &self.name
79    }
80}
81
82impl ShellReporter<Vec<u8>> {
83    pub fn new_expose_writer_for_test() -> (Self, ShellWriterView<Vec<u8>>) {
84        let inner = Arc::new(Mutex::new(ShellWriterHandleInner::new(vec![])));
85        let mut entity_state_map = HashMap::new();
86        entity_state_map.insert(EntityId::TestRun, EntityState::new("TEST RUN"));
87        (
88            Self {
89                inner: inner.clone(),
90                entity_state_map: Mutex::new(entity_state_map),
91                completed_suites: AtomicU32::new(0),
92                expected_suites: Mutex::new(None),
93                // Disable buffering for most tests to simplify testing.
94                stdio_buffer_duration: Duration::ZERO,
95                stdio_buffer_size: 0,
96            },
97            ShellWriterView::new(inner),
98        )
99    }
100}
101
102impl<W: 'static + Write + Send + Sync> ShellReporter<W> {
103    pub fn new(inner: W) -> Self {
104        let inner = Arc::new(Mutex::new(ShellWriterHandleInner::new(inner)));
105        let mut entity_state_map = HashMap::new();
106        entity_state_map.insert(EntityId::TestRun, EntityState::new("TEST RUN"));
107        Self {
108            inner,
109            entity_state_map: Mutex::new(entity_state_map),
110            completed_suites: AtomicU32::new(0),
111            expected_suites: Mutex::new(None),
112            stdio_buffer_duration: STDIO_BUFFERING_DURATION,
113            stdio_buffer_size: STDIO_BUFFER_SIZE,
114        }
115    }
116
117    fn new_writer_handle(&self, prefix: Option<String>) -> ShellWriterHandle<W> {
118        ShellWriterHandle::new_handle(Arc::clone(&self.inner), prefix)
119    }
120}
121
122impl<W: 'static + Write + Send + Sync> Reporter for ShellReporter<W> {
123    fn new_entity(&self, entity: &EntityId, name: &str) -> Result<(), Error> {
124        let mut map = self.entity_state_map.lock();
125        map.insert(entity.clone(), EntityState::new(name));
126        if let EntityId::Case { .. } = entity {
127            map.get_mut(&EntityId::Suite).unwrap().children.push(entity.clone());
128        }
129        Ok(())
130    }
131
132    fn set_entity_info(&self, entity: &EntityId, info: &EntityInfo) {
133        match (entity, info.expected_children) {
134            (EntityId::TestRun, Some(children)) => {
135                self.expected_suites.lock().replace(children);
136            }
137            (_, _) => (),
138        }
139    }
140
141    fn entity_started(&self, entity: &EntityId, _: Timestamp) -> Result<(), Error> {
142        let mut writer = self.new_writer_handle(None);
143        let mut entity_map_lock = self.entity_state_map.lock();
144        let entity_entry = entity_map_lock.get_mut(entity).unwrap();
145        entity_entry.run_state = EntityRunningState::Started;
146        let name = entity_entry.name().to_string();
147        match entity {
148            EntityId::TestRun => (),
149            EntityId::Suite => writeln!(writer, "Running test '{}'", name)?,
150            EntityId::Case { .. } => {
151                writeln!(writer, "[RUNNING]\t{}", name)?;
152                entity_entry.excessive_duration_task = Some(fasync::Task::spawn(async move {
153                    fasync::Timer::new(EXCESSIVE_DURATION).await;
154                    writeln!(
155                        writer,
156                        "[duration - {}]:\tStill running after {:?} seconds",
157                        name,
158                        EXCESSIVE_DURATION.as_secs()
159                    )
160                    .unwrap_or_else(|e| error!("Failed to write: {:?}", e));
161                }));
162            }
163        }
164        Ok(())
165    }
166
167    fn entity_stopped(
168        &self,
169        entity: &EntityId,
170        outcome: &ReportedOutcome,
171        _: Timestamp,
172    ) -> Result<(), Error> {
173        let mut writer = self.new_writer_handle(None);
174        let mut entity_map_lock = self.entity_state_map.lock();
175        entity_map_lock.get_mut(entity).unwrap().run_state = EntityRunningState::Finished(*outcome);
176        let entity_entry = entity_map_lock.get_mut(entity).unwrap();
177        let name = entity_entry.name().to_string();
178
179        // cancel any tasks for reporting excessive duration
180        let _ = entity_entry.excessive_duration_task.take();
181        match entity {
182            EntityId::TestRun => (),
183            EntityId::Suite => (),
184            EntityId::Case { .. } => {
185                // We don't list error result as it indicates the test didn't finish.
186                if *outcome != ReportedOutcome::Error {
187                    writeln!(writer, "[{}]\t{}", outcome, name)?;
188                }
189            }
190        }
191        Ok(())
192    }
193
194    fn entity_finished(&self, entity: &EntityId) -> Result<(), Error> {
195        let mut writer = self.new_writer_handle(None);
196        let entity_map_lock = self.entity_state_map.lock();
197        let entity_entry = entity_map_lock.get(entity).unwrap();
198        let name = entity_entry.name().to_string();
199        let outcome = match &entity_entry.run_state {
200            EntityRunningState::Finished(outcome) => *outcome,
201            _ => ReportedOutcome::Inconclusive,
202        };
203        let children: Vec<_> = entity_entry.children.iter().cloned().collect();
204        match entity {
205            EntityId::TestRun => (),
206            EntityId::Suite => {
207                if matches!(entity_entry.run_state, EntityRunningState::NotRunning) {
208                    // no need to output a report if the test wasn't even started.
209                    return Ok(());
210                }
211
212                let cases: Vec<_> =
213                    children.iter().map(|child| entity_map_lock.get(child).unwrap()).collect();
214                let executed: Vec<_> = cases
215                    .iter()
216                    .filter(|case| match &case.run_state {
217                        EntityRunningState::Started => true,
218                        EntityRunningState::Finished(ReportedOutcome::Skipped) => false,
219                        EntityRunningState::Finished(_) => true,
220                        EntityRunningState::NotRunning => false,
221                    })
222                    .collect();
223                let mut failed: Vec<_> = cases
224                    .iter()
225                    .filter(|case| {
226                        matches!(
227                            &case.run_state,
228                            EntityRunningState::Finished(
229                                ReportedOutcome::Failed | ReportedOutcome::Timedout
230                            )
231                        )
232                    })
233                    .map(|case| case.name())
234                    .collect();
235                failed.sort();
236                let mut not_finished: Vec<_> = cases
237                    .iter()
238                    .filter(|case| {
239                        matches!(
240                            &case.run_state,
241                            EntityRunningState::Started
242                                | EntityRunningState::Finished(ReportedOutcome::Error)
243                        )
244                    })
245                    .map(|case| case.name())
246                    .collect();
247                not_finished.sort();
248                let num_passed = cases
249                    .iter()
250                    .filter(|case| {
251                        matches!(
252                            &case.run_state,
253                            EntityRunningState::Finished(ReportedOutcome::Passed)
254                        )
255                    })
256                    .count();
257                let num_skipped = cases
258                    .iter()
259                    .filter(|case| {
260                        matches!(
261                            &case.run_state,
262                            EntityRunningState::Finished(ReportedOutcome::Skipped)
263                        )
264                    })
265                    .count();
266
267                let suite_number =
268                    self.completed_suites.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
269                match &*self.expected_suites.lock() {
270                    Some(total_suites) if *total_suites > 1 => writeln!(
271                        writer,
272                        "\nTest suite count {}/{}",
273                        suite_number + 1,
274                        total_suites,
275                    )?,
276                    Some(_) | None => (),
277                }
278                writeln!(writer)?;
279                if !failed.is_empty() {
280                    writeln!(writer, "Failed tests: {}", failed.join(", "))?;
281                }
282                if !not_finished.is_empty() {
283                    writeln!(writer, "\nThe following test(s) never completed:")?;
284                    for t in not_finished {
285                        writeln!(writer, "{}", t)?;
286                    }
287                }
288                match num_skipped {
289                    0 => writeln!(
290                        writer,
291                        "{} out of {} tests passed...",
292                        num_passed,
293                        executed.len()
294                    )?,
295                    skipped => writeln!(
296                        writer,
297                        "{} out of {} attempted tests passed, {} tests skipped...",
298                        num_passed,
299                        executed.len(),
300                        skipped,
301                    )?,
302                }
303                if let Some(restricted_logs) = &entity_entry.restricted_logs {
304                    writeln!(writer, "\nTest {} produced unexpected high-severity logs:", &name)?;
305                    writeln!(writer, "----------------xxxxx----------------")?;
306                    writer.write_all(restricted_logs.lock().as_slice())?;
307                    writeln!(writer, "\n----------------xxxxx----------------")?;
308                    writeln!(
309                        writer,
310                        "Failing this test. See: https://fuchsia.dev/fuchsia-src/development/diagnostics/test_and_logs#restricting_log_severity\n"
311                    )?;
312                }
313                match outcome {
314                    ReportedOutcome::Cancelled => {
315                        writeln!(writer, "{} was cancelled before completion.", &name)?
316                    }
317                    ReportedOutcome::DidNotFinish => {
318                        writeln!(writer, "{} did not complete successfully.", &name)?
319                    }
320                    other => writeln!(writer, "{} completed with result: {}", &name, other)?,
321                }
322            }
323            EntityId::Case { .. } => (),
324        }
325        Ok(())
326    }
327
328    fn new_artifact(
329        &self,
330        entity: &EntityId,
331        artifact_type: &ArtifactType,
332    ) -> Result<Box<DynArtifact>, Error> {
333        let mut lock = self.entity_state_map.lock();
334        let entity = lock.get_mut(entity).unwrap();
335        let name = entity.name();
336
337        Ok(match artifact_type {
338            ArtifactType::Stdout => Box::new(test_diagnostics::StdoutBuffer::new(
339                self.stdio_buffer_duration,
340                self.new_writer_handle(Some(format!("[stdout - {}]\n", name))),
341                self.stdio_buffer_size,
342            )),
343            ArtifactType::Stderr => Box::new(test_diagnostics::StdoutBuffer::new(
344                self.stdio_buffer_duration,
345                self.new_writer_handle(Some(format!("[stderr - {}]\n", name))),
346                self.stdio_buffer_size,
347            )),
348            ArtifactType::Syslog => Box::new(self.new_writer_handle(None)),
349            ArtifactType::RestrictedLog => {
350                // Restricted logs are saved for reporting when the entity completes.
351                let log_buffer = Arc::new(Mutex::new(ShellWriterHandleInner::new(vec![])));
352                entity.restricted_logs = Some(ShellWriterView::new(log_buffer.clone()));
353                Box::new(ShellWriterHandle::new_handle(log_buffer, None))
354            }
355        })
356    }
357
358    fn new_directory_artifact(
359        &self,
360        _entity: &EntityId,
361        _artifact_type: &DirectoryArtifactType,
362        _component_moniker: Option<String>,
363    ) -> Result<Box<DynDirectoryArtifact>, Error> {
364        // For the purpose of live reporting we don't display anything from a directory.
365        Ok(Box::new(NoopDirectoryWriter))
366    }
367}
368
369#[cfg(test)]
370mod test {
371    use super::*;
372    use crate::output::{CaseId, RunReporter};
373
374    #[fuchsia::test]
375    async fn report_case_events() {
376        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
377        let run_reporter = RunReporter::new(shell_reporter);
378        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
379        suite_reporter.started(Timestamp::Unknown).expect("case started");
380        let mut expected = "Running test 'test-suite'\n".to_string();
381        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
382
383        let case_1_reporter = suite_reporter.new_case("case-1", &CaseId(0)).expect("create case");
384        let case_2_reporter = suite_reporter.new_case("case-2", &CaseId(1)).expect("create case");
385
386        case_1_reporter.started(Timestamp::Unknown).expect("case started");
387        expected.push_str("[RUNNING]\tcase-1\n");
388        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
389
390        case_2_reporter.started(Timestamp::Unknown).expect("case started");
391        expected.push_str("[RUNNING]\tcase-2\n");
392        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
393
394        case_1_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
395        expected.push_str("[PASSED]\tcase-1\n");
396        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
397
398        case_2_reporter.stopped(&ReportedOutcome::Failed, Timestamp::Unknown).expect("stop case");
399        expected.push_str("[FAILED]\tcase-2\n");
400        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
401
402        case_1_reporter.finished().expect("finish case");
403        case_2_reporter.finished().expect("finish case");
404        suite_reporter.stopped(&ReportedOutcome::Failed, Timestamp::Unknown).expect("stop suite");
405        suite_reporter.finished().expect("finish suite");
406
407        expected.push_str("\n");
408        expected.push_str("Failed tests: case-2\n");
409        expected.push_str("1 out of 2 tests passed...\n");
410        expected.push_str("test-suite completed with result: FAILED\n");
411        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
412    }
413
414    #[fuchsia::test]
415    async fn report_multiple_suites() {
416        const NUM_SUITES: u32 = 5;
417        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
418        let run_reporter = RunReporter::new(shell_reporter);
419        run_reporter.set_expected_suites(NUM_SUITES);
420        run_reporter.started(Timestamp::Unknown).expect("run started");
421        let mut expected = "".to_string();
422
423        for suite_number in 0..4 {
424            let suite_name = format!("test-suite-{:?}", suite_number);
425            let suite_reporter = run_reporter.new_suite(&suite_name).expect("create suite");
426            suite_reporter.started(Timestamp::Unknown).expect("case started");
427            expected.push_str(&format!("Running test '{}'\n", &suite_name));
428            assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
429
430            let case_reporter = suite_reporter.new_case("case-1", &CaseId(0)).expect("create case");
431            case_reporter.started(Timestamp::Unknown).expect("case started");
432            expected.push_str("[RUNNING]\tcase-1\n");
433            assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
434            case_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
435            expected.push_str("[PASSED]\tcase-1\n");
436            assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
437
438            case_reporter.finished().expect("finish case");
439            suite_reporter
440                .stopped(&ReportedOutcome::Passed, Timestamp::Unknown)
441                .expect("stop suite");
442            suite_reporter.finished().expect("finish suite");
443
444            expected.push_str("\n");
445            expected.push_str(&format!(
446                "Test suite count {:?}/{:?}\n\n",
447                suite_number + 1,
448                NUM_SUITES
449            ));
450            expected.push_str("1 out of 1 tests passed...\n");
451            expected.push_str(&format!("{} completed with result: PASSED\n", suite_name));
452            assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
453        }
454    }
455
456    #[fuchsia::test]
457    async fn report_case_skipped() {
458        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
459        let run_reporter = RunReporter::new(shell_reporter);
460        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
461        suite_reporter.started(Timestamp::Unknown).expect("case started");
462        let mut expected = "Running test 'test-suite'\n".to_string();
463        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
464
465        let case_1_reporter = suite_reporter.new_case("case-1", &CaseId(0)).expect("create case");
466        let case_2_reporter = suite_reporter.new_case("case-2", &CaseId(1)).expect("create case");
467
468        case_1_reporter.started(Timestamp::Unknown).expect("case started");
469        expected.push_str("[RUNNING]\tcase-1\n");
470        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
471
472        case_1_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
473        expected.push_str("[PASSED]\tcase-1\n");
474        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
475
476        case_2_reporter.stopped(&ReportedOutcome::Skipped, Timestamp::Unknown).expect("stop case");
477        expected.push_str("[SKIPPED]\tcase-2\n");
478        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
479
480        case_1_reporter.finished().expect("finish case");
481        case_2_reporter.finished().expect("finish case");
482        suite_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop suite");
483        suite_reporter.finished().expect("finish suite");
484
485        expected.push_str("\n");
486        expected.push_str("1 out of 1 attempted tests passed, 1 tests skipped...\n");
487        expected.push_str("test-suite completed with result: PASSED\n");
488        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
489    }
490
491    #[fuchsia::test]
492    async fn syslog_artifacts() {
493        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
494        let run_reporter = RunReporter::new(shell_reporter);
495        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
496        suite_reporter.started(Timestamp::Unknown).expect("case started");
497        let mut syslog_writer =
498            suite_reporter.new_artifact(&ArtifactType::Syslog).expect("create syslog");
499
500        writeln!(syslog_writer, "[log] test syslog").expect("write to syslog");
501        let mut expected = "Running test 'test-suite'\n".to_string();
502        expected.push_str("[log] test syslog\n");
503        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected);
504
505        suite_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop suite");
506        writeln!(syslog_writer, "[log] more test syslog").expect("write to syslog");
507        expected.push_str("[log] more test syslog\n");
508        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected);
509
510        suite_reporter.finished().expect("finish suite");
511        expected.push_str("\n");
512        expected.push_str("0 out of 0 tests passed...\n");
513        expected.push_str("test-suite completed with result: PASSED\n");
514        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
515    }
516
517    #[fuchsia::test]
518    async fn report_retricted_logs() {
519        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
520        let run_reporter = RunReporter::new(shell_reporter);
521        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
522        suite_reporter.started(Timestamp::Unknown).expect("case started");
523
524        let case_reporter = suite_reporter.new_case("case-0", &CaseId(0)).expect("create case");
525        case_reporter.started(Timestamp::Unknown).expect("case started");
526        case_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
527        case_reporter.finished().expect("finish case");
528
529        let mut expected = "Running test 'test-suite'\n".to_string();
530        expected.push_str("[RUNNING]\tcase-0\n");
531        expected.push_str("[PASSED]\tcase-0\n");
532        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
533
534        suite_reporter.stopped(&ReportedOutcome::Failed, Timestamp::Unknown).expect("stop suite");
535        let mut restricted_log = suite_reporter
536            .new_artifact(&ArtifactType::RestrictedLog)
537            .expect("create restricted log");
538        write!(restricted_log, "suite restricted log").expect("write to restricted log");
539        drop(restricted_log);
540
541        suite_reporter.finished().expect("finish suite");
542        expected.push_str("\n");
543        expected.push_str("1 out of 1 tests passed...\n");
544        expected.push_str("\nTest test-suite produced unexpected high-severity logs:\n");
545        expected.push_str("----------------xxxxx----------------\n");
546        expected.push_str("suite restricted log\n\n");
547        expected.push_str("----------------xxxxx----------------\n");
548        expected.push_str("Failing this test. See: https://fuchsia.dev/fuchsia-src/development/diagnostics/test_and_logs#restricting_log_severity\n");
549        expected.push_str("\ntest-suite completed with result: FAILED\n");
550        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
551    }
552
553    #[fuchsia::test]
554    async fn stdout_artifacts() {
555        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
556        let run_reporter = RunReporter::new(shell_reporter);
557        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
558        suite_reporter.started(Timestamp::Unknown).expect("case started");
559
560        let case_0_reporter = suite_reporter.new_case("case-0", &CaseId(0)).expect("create case");
561        let case_1_reporter = suite_reporter.new_case("case-1", &CaseId(1)).expect("create case");
562        case_0_reporter.started(Timestamp::Unknown).expect("start case");
563        case_1_reporter.started(Timestamp::Unknown).expect("start case");
564        let mut expected = "Running test 'test-suite'\n".to_string();
565        expected.push_str("[RUNNING]\tcase-0\n[RUNNING]\tcase-1\n");
566        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
567
568        let mut case_0_stdout =
569            case_0_reporter.new_artifact(&ArtifactType::Stdout).expect("create artifact");
570        let mut case_1_stdout =
571            case_1_reporter.new_artifact(&ArtifactType::Stdout).expect("create artifact");
572
573        writeln!(case_0_stdout, "stdout from case 0").expect("write to stdout");
574        writeln!(case_1_stdout, "stdout from case 1").expect("write to stdout");
575
576        expected.push_str("[stdout - case-0]\nstdout from case 0\n");
577        expected.push_str("[stdout - case-1]\nstdout from case 1\n");
578        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
579
580        case_0_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
581        case_1_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
582        expected.push_str("[PASSED]\tcase-0\n");
583        expected.push_str("[PASSED]\tcase-1\n");
584        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
585
586        case_0_reporter.finished().expect("finish case");
587        case_1_reporter.finished().expect("finish case");
588        suite_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop suite");
589
590        suite_reporter.finished().expect("finish suite");
591        expected.push_str("\n");
592        expected.push_str("2 out of 2 tests passed...\n");
593        expected.push_str("test-suite completed with result: PASSED\n");
594        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
595    }
596
597    #[fuchsia::test]
598    async fn report_unfinished() {
599        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
600        let run_reporter = RunReporter::new(shell_reporter);
601        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
602        suite_reporter.started(Timestamp::Unknown).expect("suite started");
603
604        let case_reporter = suite_reporter.new_case("case-0", &CaseId(0)).expect("create case");
605        case_reporter.started(Timestamp::Unknown).expect("case started");
606        case_reporter.stopped(&ReportedOutcome::Passed, Timestamp::Unknown).expect("stop case");
607        case_reporter.finished().expect("finish case");
608        let mut expected = "Running test 'test-suite'\n".to_string();
609        expected.push_str("[RUNNING]\tcase-0\n[PASSED]\tcase-0\n");
610        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
611
612        // add a case that finishes with ERROR
613        let unfinished_case_1 = suite_reporter.new_case("case-1", &CaseId(1)).expect("create case");
614        unfinished_case_1.started(Timestamp::Unknown).expect("case started");
615        unfinished_case_1.stopped(&ReportedOutcome::Error, Timestamp::Unknown).expect("stop case");
616        unfinished_case_1.finished().expect("finish case");
617        expected.push_str("[RUNNING]\tcase-1\n");
618        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
619
620        // add a case that doesn't finish at all
621        let unfinished_case_2 = suite_reporter.new_case("case-2", &CaseId(2)).expect("create case");
622        unfinished_case_2.started(Timestamp::Unknown).expect("case started");
623        unfinished_case_2.finished().expect("finish case");
624        expected.push_str("[RUNNING]\tcase-2\n");
625        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
626
627        suite_reporter.stopped(&ReportedOutcome::Failed, Timestamp::Unknown).expect("stop suite");
628        suite_reporter.finished().expect("finish suite");
629        expected.push_str("\n");
630        expected.push_str("\nThe following test(s) never completed:\n");
631        expected.push_str("case-1\n");
632        expected.push_str("case-2\n");
633        expected.push_str("1 out of 3 tests passed...\n");
634        expected.push_str("test-suite completed with result: FAILED\n");
635        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
636    }
637
638    #[fuchsia::test]
639    async fn report_cancelled_suite() {
640        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
641        let run_reporter = RunReporter::new(shell_reporter);
642        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
643        suite_reporter.started(Timestamp::Unknown).expect("suite started");
644
645        let case_reporter = suite_reporter.new_case("case", &CaseId(0)).expect("create new case");
646        case_reporter.started(Timestamp::Unknown).expect("case started");
647        case_reporter.finished().expect("case finished");
648        suite_reporter
649            .stopped(&ReportedOutcome::Cancelled, Timestamp::Unknown)
650            .expect("stop suite");
651        suite_reporter.finished().expect("case finished");
652
653        let mut expected = "Running test 'test-suite'\n".to_string();
654        expected.push_str("[RUNNING]\tcase\n");
655        expected.push_str("\n");
656        expected.push_str("\nThe following test(s) never completed:\n");
657        expected.push_str("case\n");
658        expected.push_str("0 out of 1 tests passed...\n");
659        expected.push_str("test-suite was cancelled before completion.\n");
660        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
661    }
662
663    #[fuchsia::test]
664    async fn report_suite_did_not_finish() {
665        let (shell_reporter, output) = ShellReporter::new_expose_writer_for_test();
666        let run_reporter = RunReporter::new(shell_reporter);
667        let suite_reporter = run_reporter.new_suite("test-suite").expect("create suite");
668        suite_reporter.started(Timestamp::Unknown).expect("suite started");
669
670        let case_reporter = suite_reporter.new_case("case", &CaseId(0)).expect("create new case");
671        case_reporter.started(Timestamp::Unknown).expect("case started");
672        case_reporter.finished().expect("case finished");
673        suite_reporter
674            .stopped(&ReportedOutcome::DidNotFinish, Timestamp::Unknown)
675            .expect("stop suite");
676        suite_reporter.finished().expect("case finished");
677
678        let mut expected = "Running test 'test-suite'\n".to_string();
679        expected.push_str("[RUNNING]\tcase\n");
680        expected.push_str("\n");
681        expected.push_str("\nThe following test(s) never completed:\n");
682        expected.push_str("case\n");
683        expected.push_str("0 out of 1 tests passed...\n");
684        expected.push_str("test-suite did not complete successfully.\n");
685        assert_eq!(String::from_utf8(output.lock().clone()).unwrap(), expected,);
686    }
687}
run_test_suite_lib/output/shell/mod.rs

run_test_suite_lib/output/shell/
mod.rs
