input_pipeline/
ime_handler.rs

1// Copyright 2020 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::input_handler::{InputHandlerStatus, UnhandledInputHandler};
6use crate::{input_device, keyboard_binding, metrics};
7use anyhow::Error;
8use async_trait::async_trait;
9use fidl_fuchsia_ui_input3::{self as fidl_ui_input3, LockState, Modifiers};
10use fuchsia_component::client::connect_to_protocol;
11use fuchsia_inspect::health::Reporter;
12
13use keymaps::{LockStateChecker, ModifierChecker};
14use metrics_registry::*;
15use std::rc::Rc;
16
17#[derive(Debug)]
18pub struct FrozenLockState {
19    lock_state: LockState,
20}
21
22impl From<LockState> for FrozenLockState {
23    fn from(lock_state: LockState) -> Self {
24        FrozenLockState { lock_state }
25    }
26}
27
28impl LockStateChecker for FrozenLockState {
29    fn test(&self, value: LockState) -> bool {
30        self.lock_state.contains(value)
31    }
32}
33
34/// Modifier state plus a tester method.
35#[derive(Debug)]
36pub struct FrozenModifierState {
37    state: Modifiers,
38}
39
40impl From<fidl_fuchsia_ui_input3::Modifiers> for FrozenModifierState {
41    fn from(state: Modifiers) -> Self {
42        FrozenModifierState { state }
43    }
44}
45
46impl ModifierChecker for FrozenModifierState {
47    fn test(&self, value: Modifiers) -> bool {
48        self.state.contains(value)
49    }
50}
51
52/// [`ImeHandler`] is responsible for dispatching key events to the IME service, thus making sure
53/// that key events are delivered to application runtimes (e.g., web, Flutter).
54///
55/// > NOTE: The [ImeHandler] requires [ModifierHandler] to be installed upstream to apply the keymaps correctly.
56pub struct ImeHandler {
57    /// The FIDL proxy (client-side stub) to the service for key event injection.
58    key_event_injector: fidl_ui_input3::KeyEventInjectorProxy,
59
60    /// The inventory of this handler's Inspect status.
61    pub inspect_status: InputHandlerStatus,
62
63    /// The metrics logger.
64    metrics_logger: metrics::MetricsLogger,
65}
66
67#[async_trait(?Send)]
68impl UnhandledInputHandler for ImeHandler {
69    async fn handle_unhandled_input_event(
70        self: Rc<Self>,
71        unhandled_input_event: input_device::UnhandledInputEvent,
72    ) -> Vec<input_device::InputEvent> {
73        fuchsia_trace::duration!("input", "ime_handler");
74        match unhandled_input_event {
75            input_device::UnhandledInputEvent {
76                device_event: input_device::InputDeviceEvent::Keyboard(ref keyboard_device_event),
77                device_descriptor:
78                    input_device::InputDeviceDescriptor::Keyboard(ref keyboard_description),
79                event_time,
80                trace_id,
81            } => {
82                fuchsia_trace::duration!("input", "ime_handler[processing]");
83                if let Some(trace_id) = trace_id {
84                    fuchsia_trace::flow_end!("input", "event_in_input_pipeline", trace_id.into());
85                }
86
87                self.inspect_status.count_received_event(&event_time);
88                let key_event = create_key_event(
89                    &keyboard_device_event,
90                    event_time,
91                    keyboard_description.device_id,
92                );
93                self.dispatch_key(key_event).await;
94                // Consume the input event.
95                self.inspect_status.count_handled_event();
96                vec![input_device::InputEvent::from(unhandled_input_event).into_handled()]
97            }
98            _ => {
99                // TODO: b/478249522 - add cobalt logging
100                log::warn!("Unhandled input event: {:?}", unhandled_input_event.get_event_type());
101                vec![input_device::InputEvent::from(unhandled_input_event)]
102            }
103        }
104    }
105
106    fn set_handler_healthy(self: std::rc::Rc<Self>) {
107        self.inspect_status.health_node.borrow_mut().set_ok();
108    }
109
110    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
111        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
112    }
113
114    fn get_name(&self) -> &'static str {
115        "ImeHandler"
116    }
117
118    fn interest(&self) -> Vec<input_device::InputEventType> {
119        vec![input_device::InputEventType::Keyboard]
120    }
121}
122
123impl ImeHandler {
124    /// Creates a new [`ImeHandler`] by connecting out to the key event injector.
125    pub async fn new(
126        input_handlers_node: &fuchsia_inspect::Node,
127        metrics_logger: metrics::MetricsLogger,
128    ) -> Result<Rc<Self>, Error> {
129        let key_event_injector = connect_to_protocol::<fidl_ui_input3::KeyEventInjectorMarker>()?;
130
131        Self::new_handler(key_event_injector, input_handlers_node, metrics_logger).await
132    }
133
134    /// Creates a new [`ImeHandler`].
135    ///
136    /// # Parameters
137    /// `key_event_injector`: A proxy (FIDL client-side stub) to the key event
138    /// injector FIDL service.
139    async fn new_handler(
140        key_event_injector: fidl_ui_input3::KeyEventInjectorProxy,
141        input_handlers_node: &fuchsia_inspect::Node,
142        metrics_logger: metrics::MetricsLogger,
143    ) -> Result<Rc<Self>, Error> {
144        let inspect_status = InputHandlerStatus::new(
145            input_handlers_node,
146            "ime_handler",
147            /* generates_events */ false,
148        );
149        let handler = ImeHandler { key_event_injector, inspect_status, metrics_logger };
150
151        Ok(Rc::new(handler))
152    }
153
154    /// Dispatches key events to IME and returns KeyboardEvents for unhandled events.
155    ///
156    /// # Parameters
157    /// `key_events`: The key events to dispatch.
158    /// `event_time`: The time in nanoseconds when the events were first recorded.
159    async fn dispatch_key(self: &Rc<Self>, key_event: fidl_ui_input3::KeyEvent) {
160        assert!(
161            key_event.timestamp.is_some(),
162            "dispatch_key: got a key_event without a timestamp: {:?}",
163            &key_event
164        );
165        match self.key_event_injector.inject(&key_event).await {
166            Err(err) => self.metrics_logger.log_error(
167                InputPipelineErrorMetricDimensionEvent::ImeFailedToDispatchKeyToIme,
168                std::format!("Failed to dispatch key to IME: {:?}", err),
169            ),
170            _ => {}
171        };
172    }
173}
174
175/// Returns a KeyEvent with the given parameters.
176///
177/// # Parameters
178/// * `event`: The keyboard event to process.
179/// * `event_time`: The time in nanoseconds when the event was first recorded.
180fn create_key_event(
181    event: &keyboard_binding::KeyboardEvent,
182    event_time: zx::MonotonicInstant,
183    device_id: u32,
184) -> fidl_ui_input3::KeyEvent {
185    let modifier_state: FrozenModifierState =
186        event.get_modifiers().unwrap_or_else(|| Modifiers::from_bits_allow_unknown(0)).into();
187    let lock_state: FrozenLockState =
188        event.get_lock_state().unwrap_or_else(|| LockState::from_bits_allow_unknown(0)).into();
189    log::debug!(
190        "ImeHandler::create_key_event: key:{:?}, modifier_state: {:?}, lock_state: {:?}, event_type: {:?}",
191        event.get_key(),
192        modifier_state,
193        lock_state,
194        event.get_event_type(),
195    );
196    // Don't override the key meaning if already set, e.g. by prior stage.
197    let key_meaning = event
198        .get_key_meaning()
199        .or_else(|| keymaps::US_QWERTY.apply(event.get_key(), &modifier_state, &lock_state));
200
201    // Don't insert a spurious Some(0).
202    let repeat_sequence = match event.get_repeat_sequence() {
203        0 => None,
204        s => Some(s),
205    };
206
207    fidl_ui_input3::KeyEvent {
208        timestamp: Some(event_time.into_nanos()),
209        type_: event.get_event_type().into(),
210        key: event.get_key().into(),
211        modifiers: event.get_modifiers(),
212        lock_state: event.get_lock_state(),
213        key_meaning,
214        repeat_sequence,
215        device_id: Some(device_id),
216        ..Default::default()
217    }
218}
219
220#[cfg(test)]
221mod tests {
222    use super::*;
223    use crate::input_handler::InputHandler;
224    use crate::keyboard_binding::{self, KeyboardEvent};
225    use crate::testing_utilities;
226    use assert_matches::assert_matches;
227    use futures::StreamExt;
228    use std::convert::TryFrom as _;
229    use test_case::test_case;
230    use {
231        fidl_fuchsia_input as fidl_input, fidl_fuchsia_ui_input3 as fidl_ui_input3,
232        fuchsia_async as fasync,
233    };
234
235    fn handle_events(
236        ime_handler: Rc<ImeHandler>,
237        input_events: Vec<input_device::UnhandledInputEvent>,
238    ) {
239        fasync::Task::local(async move {
240            for input_event in input_events {
241                assert_matches!(
242                    ime_handler.clone().handle_unhandled_input_event(input_event).await.as_slice(),
243                    [input_device::InputEvent { handled: input_device::Handled::Yes, .. }]
244                );
245            }
246        })
247        .detach();
248    }
249
250    async fn assert_ime_receives_events(
251        expected_events: Vec<fidl_ui_input3::KeyEvent>,
252        mut request_stream: fidl_ui_input3::KeyEventInjectorRequestStream,
253    ) {
254        let mut expected_events_iter = expected_events.iter().peekable();
255        while let Some(Ok(fidl_ui_input3::KeyEventInjectorRequest::Inject {
256            key_event,
257            responder,
258            ..
259        })) = request_stream.next().await
260        {
261            pretty_assertions::assert_eq!(&key_event, expected_events_iter.next().unwrap());
262
263            // All the expected events have been received, so make sure no more events
264            // are present before returning.
265            if expected_events_iter.peek().is_none() {
266                responder
267                    .send(fidl_ui_input3::KeyEventStatus::Handled)
268                    .expect("error responding to DispatchKey");
269                return;
270            }
271            responder
272                .send(fidl_ui_input3::KeyEventStatus::Handled)
273                .expect("error responding to DispatchKey");
274        }
275
276        assert!(false);
277    }
278
279    fn connect_to_key_event_injector()
280    -> (fidl_ui_input3::KeyEventInjectorProxy, fidl_ui_input3::KeyEventInjectorRequestStream) {
281        fidl::endpoints::create_proxy_and_stream::<fidl_ui_input3::KeyEventInjectorMarker>()
282    }
283
284    fn create_unhandled_keyboard_event(
285        key: fidl_fuchsia_input::Key,
286        event_type: fidl_fuchsia_ui_input3::KeyEventType,
287        modifiers: Option<fidl_ui_input3::Modifiers>,
288        event_time: zx::MonotonicInstant,
289        device_descriptor: &input_device::InputDeviceDescriptor,
290        keymap: Option<String>,
291    ) -> input_device::UnhandledInputEvent {
292        create_unhandled_keyboard_event_with_key_meaning(
293            key,
294            event_type,
295            modifiers,
296            event_time,
297            device_descriptor,
298            keymap,
299            /* key_meaning */ None,
300        )
301    }
302
303    fn create_unhandled_keyboard_event_with_key_meaning(
304        key: fidl_fuchsia_input::Key,
305        event_type: fidl_fuchsia_ui_input3::KeyEventType,
306        modifiers: Option<fidl_ui_input3::Modifiers>,
307        event_time: zx::MonotonicInstant,
308        device_descriptor: &input_device::InputDeviceDescriptor,
309        keymap: Option<String>,
310        key_meaning: Option<fidl_fuchsia_ui_input3::KeyMeaning>,
311    ) -> input_device::UnhandledInputEvent {
312        input_device::UnhandledInputEvent::try_from(
313            testing_utilities::create_keyboard_event_with_key_meaning(
314                key,
315                event_type,
316                modifiers,
317                event_time,
318                device_descriptor,
319                keymap,
320                key_meaning,
321            ),
322        )
323        .unwrap()
324    }
325
326    fn create_unhandled_input_event(
327        keyboard_event: keyboard_binding::KeyboardEvent,
328        device_descriptor: &input_device::InputDeviceDescriptor,
329        event_time: zx::MonotonicInstant,
330    ) -> input_device::UnhandledInputEvent {
331        input_device::UnhandledInputEvent {
332            device_event: input_device::InputDeviceEvent::Keyboard(keyboard_event),
333            device_descriptor: device_descriptor.clone(),
334            event_time,
335            trace_id: None,
336        }
337    }
338
339    /// Tests that a pressed key event is dispatched.
340    ///
341    /// > NOTE: The `device_descriptor` used in this test case and elsewhere
342    /// *must* be of type `KeyboardDeviceDescriptor` as this is required by the
343    /// pattern matching in `ImeHandler`.
344    #[fasync::run_singlethreaded(test)]
345    async fn pressed_key() {
346        let (proxy, request_stream) = connect_to_key_event_injector();
347        let inspector = fuchsia_inspect::Inspector::default();
348        let test_node = inspector.root().create_child("test_node");
349        let ime_handler =
350            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
351                .await
352                .expect("Failed to create ImeHandler.");
353
354        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
355            keyboard_binding::KeyboardDeviceDescriptor {
356                keys: vec![fidl_input::Key::A],
357                device_id: 0,
358                ..Default::default()
359            },
360        );
361        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
362        let input_events = vec![create_unhandled_keyboard_event(
363            fidl_input::Key::A,
364            fidl_fuchsia_ui_input3::KeyEventType::Pressed,
365            None,
366            event_time_u64,
367            &device_descriptor,
368            /* keymap= */ None,
369        )];
370
371        let expected_events = vec![fidl_ui_input3::KeyEvent {
372            timestamp: Some(event_time_i64),
373            type_: Some(fidl_ui_input3::KeyEventType::Pressed),
374            key: Some(fidl_input::Key::A),
375            // A key "A" without shift is a lowercase 'a'.
376            key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
377            device_id: Some(0),
378            ..Default::default()
379        }];
380
381        handle_events(ime_handler, input_events);
382        assert_ime_receives_events(expected_events, request_stream).await;
383    }
384
385    /// Tests that a released key event is dispatched.
386    #[fasync::run_singlethreaded(test)]
387    async fn released_key() {
388        let (proxy, request_stream) = connect_to_key_event_injector();
389        let inspector = fuchsia_inspect::Inspector::default();
390        let test_node = inspector.root().create_child("test_node");
391        let ime_handler =
392            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
393                .await
394                .expect("Failed to create ImeHandler.");
395
396        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
397            keyboard_binding::KeyboardDeviceDescriptor {
398                keys: vec![fidl_input::Key::A],
399                device_id: 0,
400                ..Default::default()
401            },
402        );
403        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
404        let input_events = vec![create_unhandled_keyboard_event(
405            fidl_input::Key::A,
406            fidl_fuchsia_ui_input3::KeyEventType::Released,
407            None,
408            event_time_u64,
409            &device_descriptor,
410            /* keymap= */ None,
411        )];
412
413        let expected_events = vec![fidl_ui_input3::KeyEvent {
414            timestamp: Some(event_time_i64),
415            type_: Some(fidl_ui_input3::KeyEventType::Released),
416            key: Some(fidl_input::Key::A),
417            key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
418            device_id: Some(0),
419            ..Default::default()
420        }];
421
422        handle_events(ime_handler, input_events);
423        assert_ime_receives_events(expected_events, request_stream).await;
424    }
425
426    /// Tests that both pressed and released keys are dispatched appropriately.
427    #[fasync::run_singlethreaded(test)]
428    async fn pressed_and_released_key() {
429        let (proxy, request_stream) = connect_to_key_event_injector();
430        let inspector = fuchsia_inspect::Inspector::default();
431        let test_node = inspector.root().create_child("test_node");
432        let ime_handler =
433            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
434                .await
435                .expect("Failed to create ImeHandler.");
436
437        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
438            keyboard_binding::KeyboardDeviceDescriptor {
439                keys: vec![fidl_input::Key::A, fidl_input::Key::B],
440                device_id: 0,
441                ..Default::default()
442            },
443        );
444        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
445        let input_events = vec![
446            create_unhandled_keyboard_event(
447                fidl_input::Key::A,
448                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
449                None,
450                event_time_u64,
451                &device_descriptor,
452                /* keymap= */ None,
453            ),
454            create_unhandled_keyboard_event(
455                fidl_input::Key::A,
456                fidl_fuchsia_ui_input3::KeyEventType::Released,
457                None,
458                event_time_u64,
459                &device_descriptor,
460                /* keymap= */ None,
461            ),
462            create_unhandled_keyboard_event(
463                fidl_input::Key::B,
464                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
465                None,
466                event_time_u64,
467                &device_descriptor,
468                /* keymap= */ None,
469            ),
470            create_unhandled_keyboard_event_with_key_meaning(
471                fidl_input::Key::C,
472                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
473                None,
474                event_time_u64,
475                &device_descriptor,
476                /* keymap= */ None,
477                Some(fidl_fuchsia_ui_input3::KeyMeaning::Codepoint(42)),
478            ),
479        ];
480
481        let expected_events = vec![
482            fidl_ui_input3::KeyEvent {
483                timestamp: Some(event_time_i64),
484                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
485                key: Some(fidl_input::Key::A),
486                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
487                device_id: Some(0),
488                ..Default::default()
489            },
490            fidl_ui_input3::KeyEvent {
491                timestamp: Some(event_time_i64),
492                type_: Some(fidl_ui_input3::KeyEventType::Released),
493                key: Some(fidl_input::Key::A),
494                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
495                device_id: Some(0),
496                ..Default::default()
497            },
498            fidl_ui_input3::KeyEvent {
499                timestamp: Some(event_time_i64),
500                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
501                key: Some(fidl_input::Key::B),
502                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(98)),
503                device_id: Some(0),
504                ..Default::default()
505            },
506            fidl_ui_input3::KeyEvent {
507                timestamp: Some(event_time_i64),
508                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
509                key: Some(fidl_input::Key::C),
510                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(42)),
511                device_id: Some(0),
512                ..Default::default()
513            },
514        ];
515
516        handle_events(ime_handler, input_events);
517        assert_ime_receives_events(expected_events, request_stream).await;
518    }
519
520    // Tests that modifier keys are dispatched appropriately.
521    //
522    // This test depends on the incoming event having correct modifier and lock
523    // state.  Typically you'd do this by installing a ModifierHandler upstream
524    // of this pipeline stage.
525    #[fasync::run_singlethreaded(test)]
526    async fn repeated_modifier_key() {
527        let (proxy, request_stream) = connect_to_key_event_injector();
528        let inspector = fuchsia_inspect::Inspector::default();
529        let test_node = inspector.root().create_child("test_node");
530        let ime_handler =
531            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
532                .await
533                .expect("Failed to create ImeHandler.");
534
535        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
536            keyboard_binding::KeyboardDeviceDescriptor {
537                keys: vec![fidl_input::Key::A, fidl_input::Key::CapsLock],
538                device_id: 0,
539                ..Default::default()
540            },
541        );
542        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
543        let input_events = vec![
544            create_unhandled_input_event(
545                KeyboardEvent::new(
546                    fidl_input::Key::CapsLock,
547                    fidl_fuchsia_ui_input3::KeyEventType::Pressed,
548                )
549                .into_with_modifiers(Some(fidl_ui_input3::Modifiers::CAPS_LOCK))
550                .into_with_lock_state(Some(fidl_ui_input3::LockState::CAPS_LOCK)),
551                &device_descriptor,
552                event_time_u64,
553            ),
554            create_unhandled_input_event(
555                KeyboardEvent::new(
556                    fidl_input::Key::A,
557                    fidl_fuchsia_ui_input3::KeyEventType::Pressed,
558                )
559                .into_with_modifiers(Some(fidl_ui_input3::Modifiers::CAPS_LOCK))
560                .into_with_lock_state(Some(fidl_ui_input3::LockState::CAPS_LOCK)),
561                &device_descriptor,
562                event_time_u64,
563            ),
564            create_unhandled_input_event(
565                KeyboardEvent::new(
566                    fidl_input::Key::CapsLock,
567                    fidl_fuchsia_ui_input3::KeyEventType::Released,
568                )
569                .into_with_lock_state(Some(fidl_ui_input3::LockState::CAPS_LOCK)),
570                &device_descriptor,
571                event_time_u64,
572            ),
573        ];
574
575        let expected_events = vec![
576            fidl_ui_input3::KeyEvent {
577                timestamp: Some(event_time_i64),
578                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
579                key: Some(fidl_input::Key::CapsLock),
580                modifiers: Some(fidl_ui_input3::Modifiers::CAPS_LOCK),
581                lock_state: Some(fidl_ui_input3::LockState::CAPS_LOCK),
582                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
583                    fidl_ui_input3::NonPrintableKey::CapsLock,
584                )),
585                device_id: Some(0),
586                ..Default::default()
587            },
588            fidl_ui_input3::KeyEvent {
589                timestamp: Some(event_time_i64),
590                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
591                key: Some(fidl_input::Key::A),
592                modifiers: Some(fidl_ui_input3::Modifiers::CAPS_LOCK),
593                lock_state: Some(fidl_ui_input3::LockState::CAPS_LOCK),
594                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(65)),
595                device_id: Some(0),
596                ..Default::default()
597            },
598            fidl_ui_input3::KeyEvent {
599                timestamp: Some(event_time_i64),
600                type_: Some(fidl_ui_input3::KeyEventType::Released),
601                key: Some(fidl_input::Key::CapsLock),
602                lock_state: Some(fidl_ui_input3::LockState::CAPS_LOCK),
603                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
604                    fidl_ui_input3::NonPrintableKey::CapsLock,
605                )),
606                device_id: Some(0),
607                ..Default::default()
608            },
609        ];
610
611        handle_events(ime_handler, input_events);
612        assert_ime_receives_events(expected_events, request_stream).await;
613    }
614
615    #[fasync::run_singlethreaded(test)]
616    async fn nonprintable_key_meanings_set_correctly() {
617        let (proxy, request_stream) = connect_to_key_event_injector();
618        let inspector = fuchsia_inspect::Inspector::default();
619        let test_node = inspector.root().create_child("test_node");
620        let ime_handler =
621            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
622                .await
623                .expect("Failed to create ImeHandler.");
624
625        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
626            keyboard_binding::KeyboardDeviceDescriptor {
627                keys: vec![
628                    fidl_input::Key::Enter,
629                    fidl_input::Key::Tab,
630                    fidl_input::Key::Backspace,
631                ],
632                device_id: 0,
633                ..Default::default()
634            },
635        );
636        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
637        let input_events = vec![
638            create_unhandled_keyboard_event(
639                fidl_input::Key::Enter,
640                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
641                None,
642                event_time_u64,
643                &device_descriptor,
644                /* keymap= */ None,
645            ),
646            create_unhandled_keyboard_event(
647                fidl_input::Key::Tab,
648                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
649                None,
650                event_time_u64,
651                &device_descriptor,
652                /* keymap= */ None,
653            ),
654            create_unhandled_keyboard_event(
655                fidl_input::Key::Backspace,
656                fidl_fuchsia_ui_input3::KeyEventType::Released,
657                None,
658                event_time_u64,
659                &device_descriptor,
660                /* keymap= */ None,
661            ),
662        ];
663
664        let expected_events = vec![
665            fidl_ui_input3::KeyEvent {
666                timestamp: Some(event_time_i64),
667                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
668                key: Some(fidl_input::Key::Enter),
669                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
670                    fidl_ui_input3::NonPrintableKey::Enter,
671                )),
672                device_id: Some(0),
673                ..Default::default()
674            },
675            fidl_ui_input3::KeyEvent {
676                timestamp: Some(event_time_i64),
677                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
678                key: Some(fidl_input::Key::Tab),
679                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
680                    fidl_ui_input3::NonPrintableKey::Tab,
681                )),
682                device_id: Some(0),
683                ..Default::default()
684            },
685            fidl_ui_input3::KeyEvent {
686                timestamp: Some(event_time_i64),
687                // Test that things also work when a key is released.
688                type_: Some(fidl_ui_input3::KeyEventType::Released),
689                key: Some(fidl_input::Key::Backspace),
690                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
691                    fidl_ui_input3::NonPrintableKey::Backspace,
692                )),
693                device_id: Some(0),
694                ..Default::default()
695            },
696        ];
697
698        handle_events(ime_handler, input_events);
699        assert_ime_receives_events(expected_events, request_stream).await;
700    }
701
702    #[fasync::run_singlethreaded(test)]
703    async fn tab() {
704        let (proxy, request_stream) = connect_to_key_event_injector();
705        let inspector = fuchsia_inspect::Inspector::default();
706        let test_node = inspector.root().create_child("test_node");
707        let ime_handler =
708            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
709                .await
710                .expect("Failed to create ImeHandler.");
711
712        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
713            keyboard_binding::KeyboardDeviceDescriptor {
714                keys: vec![
715                    fidl_input::Key::Enter,
716                    fidl_input::Key::Tab,
717                    fidl_input::Key::Backspace,
718                ],
719                device_id: 0,
720                ..Default::default()
721            },
722        );
723        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
724        let input_events = vec![create_unhandled_keyboard_event(
725            fidl_input::Key::Tab,
726            fidl_fuchsia_ui_input3::KeyEventType::Pressed,
727            None,
728            event_time_u64,
729            &device_descriptor,
730            /* keymap= */ None,
731        )];
732
733        let expected_events = vec![fidl_ui_input3::KeyEvent {
734            timestamp: Some(event_time_i64),
735            type_: Some(fidl_ui_input3::KeyEventType::Pressed),
736            key: Some(fidl_input::Key::Tab),
737            key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
738                fidl_ui_input3::NonPrintableKey::Tab,
739            )),
740            device_id: Some(0),
741            ..Default::default()
742        }];
743
744        handle_events(ime_handler, input_events);
745        assert_ime_receives_events(expected_events, request_stream).await;
746    }
747
748    #[fasync::run_singlethreaded(test)]
749    async fn shift_shift_a() {
750        let (proxy, request_stream) = connect_to_key_event_injector();
751        let inspector = fuchsia_inspect::Inspector::default();
752        let test_node = inspector.root().create_child("test_node");
753        let ime_handler =
754            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
755                .await
756                .expect("Failed to create ImeHandler.");
757
758        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
759            keyboard_binding::KeyboardDeviceDescriptor {
760                keys: vec![fidl_input::Key::LeftCtrl, fidl_input::Key::Tab],
761                device_id: 0,
762                ..Default::default()
763            },
764        );
765        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
766        let input_events = vec![
767            create_unhandled_keyboard_event(
768                fidl_input::Key::LeftShift,
769                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
770                Some(Modifiers::LEFT_SHIFT | Modifiers::SHIFT),
771                event_time_u64,
772                &device_descriptor,
773                /* keymap= */ None,
774            ),
775            create_unhandled_keyboard_event(
776                fidl_input::Key::RightShift,
777                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
778                Some(Modifiers::LEFT_SHIFT | Modifiers::RIGHT_SHIFT | Modifiers::SHIFT),
779                event_time_u64,
780                &device_descriptor,
781                /* keymap= */ None,
782            ),
783            create_unhandled_keyboard_event(
784                fidl_input::Key::A,
785                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
786                Some(Modifiers::LEFT_SHIFT | Modifiers::RIGHT_SHIFT | Modifiers::SHIFT),
787                event_time_u64,
788                &device_descriptor,
789                /* keymap= */ None,
790            ),
791        ];
792
793        let expected_events = vec![
794            fidl_ui_input3::KeyEvent {
795                timestamp: Some(event_time_i64),
796                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
797                key: Some(fidl_input::Key::LeftShift),
798                modifiers: Some(Modifiers::LEFT_SHIFT | Modifiers::SHIFT),
799                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
800                    fidl_ui_input3::NonPrintableKey::Shift,
801                )),
802                device_id: Some(0),
803                ..Default::default()
804            },
805            fidl_ui_input3::KeyEvent {
806                timestamp: Some(event_time_i64),
807                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
808                key: Some(fidl_input::Key::RightShift),
809                modifiers: Some(Modifiers::RIGHT_SHIFT | Modifiers::LEFT_SHIFT | Modifiers::SHIFT),
810                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
811                    fidl_ui_input3::NonPrintableKey::Shift,
812                )),
813                device_id: Some(0),
814                ..Default::default()
815            },
816            fidl_ui_input3::KeyEvent {
817                timestamp: Some(event_time_i64),
818                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
819                key: Some(fidl_input::Key::A),
820                modifiers: Some(Modifiers::RIGHT_SHIFT | Modifiers::LEFT_SHIFT | Modifiers::SHIFT),
821                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(65)), // "A"
822                device_id: Some(0),
823                ..Default::default()
824            },
825        ];
826
827        handle_events(ime_handler, input_events);
828        assert_ime_receives_events(expected_events, request_stream).await;
829    }
830
831    #[fasync::run_singlethreaded(test)]
832    async fn ctrl_tab() {
833        let (proxy, request_stream) = connect_to_key_event_injector();
834        let inspector = fuchsia_inspect::Inspector::default();
835        let test_node = inspector.root().create_child("test_node");
836        let ime_handler =
837            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
838                .await
839                .expect("Failed to create ImeHandler.");
840
841        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
842            keyboard_binding::KeyboardDeviceDescriptor {
843                keys: vec![fidl_input::Key::LeftCtrl, fidl_input::Key::Tab],
844                device_id: 0,
845                ..Default::default()
846            },
847        );
848        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
849        let input_events = vec![
850            create_unhandled_keyboard_event(
851                fidl_input::Key::LeftCtrl,
852                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
853                None,
854                event_time_u64,
855                &device_descriptor,
856                /* keymap= */ None,
857            ),
858            create_unhandled_keyboard_event(
859                fidl_input::Key::Tab,
860                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
861                None,
862                event_time_u64,
863                &device_descriptor,
864                /* keymap= */ None,
865            ),
866        ];
867
868        let expected_events = vec![
869            fidl_ui_input3::KeyEvent {
870                timestamp: Some(event_time_i64),
871                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
872                key: Some(fidl_input::Key::LeftCtrl),
873                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
874                    fidl_ui_input3::NonPrintableKey::Control,
875                )),
876                device_id: Some(0),
877                ..Default::default()
878            },
879            fidl_ui_input3::KeyEvent {
880                timestamp: Some(event_time_i64),
881                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
882                key: Some(fidl_input::Key::Tab),
883                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
884                    fidl_ui_input3::NonPrintableKey::Tab,
885                )),
886                device_id: Some(0),
887                ..Default::default()
888            },
889        ];
890
891        handle_events(ime_handler, input_events);
892        assert_ime_receives_events(expected_events, request_stream).await;
893    }
894
895    #[fasync::run_singlethreaded(test)]
896    async fn ime_handler_initialized_with_inspect_node() {
897        let (proxy, _) = connect_to_key_event_injector();
898        let inspector = fuchsia_inspect::Inspector::default();
899        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
900        let _handler =
901            ImeHandler::new_handler(proxy, &fake_handlers_node, metrics::MetricsLogger::default())
902                .await
903                .expect("Failed to create ImeHandler.");
904        diagnostics_assertions::assert_data_tree!(inspector, root: {
905            input_handlers_node: {
906                ime_handler: {
907                    events_received_count: 0u64,
908                    events_handled_count: 0u64,
909                    last_received_timestamp_ns: 0u64,
910                    "fuchsia.inspect.Health": {
911                        status: "STARTING_UP",
912                        // Timestamp value is unpredictable and not relevant in this context,
913                        // so we only assert that the property is present.
914                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
915                    },
916                }
917            }
918        });
919    }
920
921    #[fasync::run_singlethreaded(test)]
922    async fn ime_handler_inspect_counts_events() {
923        let (proxy, _) = connect_to_key_event_injector();
924        let inspector = fuchsia_inspect::Inspector::default();
925        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
926        let ime_handler =
927            ImeHandler::new_handler(proxy, &fake_handlers_node, metrics::MetricsLogger::default())
928                .await
929                .expect("Failed to create ImeHandler.");
930        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
931            keyboard_binding::KeyboardDeviceDescriptor {
932                keys: vec![fidl_input::Key::A, fidl_input::Key::B],
933                ..Default::default()
934            },
935        );
936        let (_, event_time_u64) = testing_utilities::event_times();
937        let input_events = vec![
938            testing_utilities::create_keyboard_event_with_time(
939                fidl_input::Key::A,
940                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
941                None,
942                event_time_u64,
943                &device_descriptor,
944                /* keymap= */ None,
945            ),
946            // Should not count received events that have already been handled.
947            testing_utilities::create_keyboard_event_with_handled(
948                fidl_input::Key::B,
949                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
950                None,
951                event_time_u64,
952                &device_descriptor,
953                /* keymap= */ None,
954                /* key_meaning= */ None,
955                input_device::Handled::Yes,
956            ),
957            testing_utilities::create_keyboard_event_with_time(
958                fidl_input::Key::A,
959                fidl_fuchsia_ui_input3::KeyEventType::Released,
960                None,
961                event_time_u64,
962                &device_descriptor,
963                /* keymap= */ None,
964            ),
965            // Should not count non-keyboard input events.
966            testing_utilities::create_fake_input_event(event_time_u64),
967            testing_utilities::create_keyboard_event_with_time(
968                fidl_input::Key::B,
969                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
970                None,
971                event_time_u64,
972                &device_descriptor,
973                /* keymap= */ None,
974            ),
975        ];
976
977        for input_event in input_events {
978            let _ = ime_handler.clone().handle_input_event(input_event).await;
979        }
980
981        let last_event_timestamp: u64 = event_time_u64.into_nanos().try_into().unwrap();
982
983        diagnostics_assertions::assert_data_tree!(inspector, root: {
984            input_handlers_node: {
985                ime_handler: {
986                    events_received_count: 3u64,
987                    events_handled_count: 3u64,
988                    last_received_timestamp_ns: last_event_timestamp,
989                    "fuchsia.inspect.Health": {
990                        status: "STARTING_UP",
991                        // Timestamp value is unpredictable and not relevant in this context,
992                        // so we only assert that the property is present.
993                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
994                    },
995                }
996            }
997        });
998    }
999
1000    #[test_case(
1001        keyboard_binding::KeyboardEvent::new(
1002            fidl_input::Key::A,
1003            fidl_ui_input3::KeyEventType::Pressed),
1004        zx::MonotonicInstant::from_nanos(42) => fidl_ui_input3::KeyEvent{
1005            timestamp: Some(42),
1006            type_: Some(fidl_ui_input3::KeyEventType::Pressed),
1007            key: Some(fidl_input::Key::A),
1008            key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
1009            device_id: Some(0),
1010            ..Default::default()}; "basic value copy")]
1011    #[test_case(
1012        keyboard_binding::KeyboardEvent::new(
1013            fidl_input::Key::A,
1014            fidl_ui_input3::KeyEventType::Pressed)
1015            .into_with_repeat_sequence(13),
1016        zx::MonotonicInstant::from_nanos(42) => fidl_ui_input3::KeyEvent{
1017            timestamp: Some(42),
1018            type_: Some(fidl_ui_input3::KeyEventType::Pressed),
1019            key: Some(fidl_input::Key::A),
1020            key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
1021            repeat_sequence: Some(13),
1022            device_id: Some(0),
1023            ..Default::default()}; "repeat_sequence is honored")]
1024    fn test_create_key_event(
1025        event: keyboard_binding::KeyboardEvent,
1026        event_time: zx::MonotonicInstant,
1027    ) -> fidl_ui_input3::KeyEvent {
1028        super::create_key_event(&event, event_time, 0)
1029    }
1030}
input_pipeline/ime_handler.rs

input_pipeline/
ime_handler.rs
