input_pipeline/
ime_handler.rs

1// Copyright 2020 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::input_handler::{Handler, InputHandlerStatus, UnhandledInputHandler};
6use crate::{input_device, keyboard_binding, metrics};
7use anyhow::Error;
8use async_trait::async_trait;
9use fidl_fuchsia_ui_input3::{self as fidl_ui_input3, LockState, Modifiers};
10use fuchsia_component::client::connect_to_protocol;
11use fuchsia_inspect::health::Reporter;
12
13use keymaps::{LockStateChecker, ModifierChecker};
14use metrics_registry::*;
15use std::rc::Rc;
16
17#[derive(Debug)]
18pub struct FrozenLockState {
19    lock_state: LockState,
20}
21
22impl From<LockState> for FrozenLockState {
23    fn from(lock_state: LockState) -> Self {
24        FrozenLockState { lock_state }
25    }
26}
27
28impl LockStateChecker for FrozenLockState {
29    fn test(&self, value: LockState) -> bool {
30        self.lock_state.contains(value)
31    }
32}
33
34/// Modifier state plus a tester method.
35#[derive(Debug)]
36pub struct FrozenModifierState {
37    state: Modifiers,
38}
39
40impl From<fidl_fuchsia_ui_input3::Modifiers> for FrozenModifierState {
41    fn from(state: Modifiers) -> Self {
42        FrozenModifierState { state }
43    }
44}
45
46impl ModifierChecker for FrozenModifierState {
47    fn test(&self, value: Modifiers) -> bool {
48        self.state.contains(value)
49    }
50}
51
52/// [`ImeHandler`] is responsible for dispatching key events to the IME service, thus making sure
53/// that key events are delivered to application runtimes (e.g., web, Flutter).
54///
55/// > NOTE: The [ImeHandler] requires [ModifierHandler] to be installed upstream to apply the keymaps correctly.
56pub struct ImeHandler {
57    /// The FIDL proxy (client-side stub) to the service for key event injection.
58    key_event_injector: fidl_ui_input3::KeyEventInjectorProxy,
59
60    /// The inventory of this handler's Inspect status.
61    pub inspect_status: InputHandlerStatus,
62
63    /// The metrics logger.
64    metrics_logger: metrics::MetricsLogger,
65}
66
67impl Handler for ImeHandler {
68    fn set_handler_healthy(self: std::rc::Rc<Self>) {
69        self.inspect_status.health_node.borrow_mut().set_ok();
70    }
71
72    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
73        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
74    }
75
76    fn get_name(&self) -> &'static str {
77        "ImeHandler"
78    }
79
80    fn interest(&self) -> Vec<input_device::InputEventType> {
81        vec![input_device::InputEventType::Keyboard]
82    }
83}
84
85#[async_trait(?Send)]
86impl UnhandledInputHandler for ImeHandler {
87    async fn handle_unhandled_input_event(
88        self: Rc<Self>,
89        unhandled_input_event: input_device::UnhandledInputEvent,
90    ) -> Vec<input_device::InputEvent> {
91        fuchsia_trace::duration!("input", "ime_handler");
92        match unhandled_input_event {
93            input_device::UnhandledInputEvent {
94                device_event: input_device::InputDeviceEvent::Keyboard(ref keyboard_device_event),
95                device_descriptor:
96                    input_device::InputDeviceDescriptor::Keyboard(ref keyboard_description),
97                event_time,
98                trace_id,
99            } => {
100                fuchsia_trace::duration!("input", "ime_handler[processing]");
101                if let Some(trace_id) = trace_id {
102                    fuchsia_trace::flow_step!("input", "event_in_input_pipeline", trace_id.into());
103                }
104
105                self.inspect_status.count_received_event(&event_time);
106                let key_event = create_key_event(
107                    &keyboard_device_event,
108                    event_time,
109                    keyboard_description.device_id,
110                );
111                self.dispatch_key(key_event).await;
112                // Consume the input event.
113                self.inspect_status.count_handled_event();
114                vec![input_device::InputEvent::from(unhandled_input_event).into_handled()]
115            }
116            _ => {
117                self.metrics_logger.log_error(
118                    InputPipelineErrorMetricDimensionEvent::HandlerReceivedUninterestedEvent,
119                    std::format!(
120                        "uninterested input event: {:?}",
121                        unhandled_input_event.get_event_type()
122                    ),
123                );
124                vec![input_device::InputEvent::from(unhandled_input_event)]
125            }
126        }
127    }
128}
129
130impl ImeHandler {
131    /// Creates a new [`ImeHandler`] by connecting out to the key event injector.
132    pub async fn new(
133        input_handlers_node: &fuchsia_inspect::Node,
134        metrics_logger: metrics::MetricsLogger,
135    ) -> Result<Rc<Self>, Error> {
136        let key_event_injector = connect_to_protocol::<fidl_ui_input3::KeyEventInjectorMarker>()?;
137
138        Self::new_handler(key_event_injector, input_handlers_node, metrics_logger).await
139    }
140
141    /// Creates a new [`ImeHandler`].
142    ///
143    /// # Parameters
144    /// `key_event_injector`: A proxy (FIDL client-side stub) to the key event
145    /// injector FIDL service.
146    async fn new_handler(
147        key_event_injector: fidl_ui_input3::KeyEventInjectorProxy,
148        input_handlers_node: &fuchsia_inspect::Node,
149        metrics_logger: metrics::MetricsLogger,
150    ) -> Result<Rc<Self>, Error> {
151        let inspect_status = InputHandlerStatus::new(
152            input_handlers_node,
153            "ime_handler",
154            /* generates_events */ false,
155        );
156        let handler = ImeHandler { key_event_injector, inspect_status, metrics_logger };
157
158        Ok(Rc::new(handler))
159    }
160
161    /// Dispatches key events to IME and returns KeyboardEvents for unhandled events.
162    ///
163    /// # Parameters
164    /// `key_events`: The key events to dispatch.
165    /// `event_time`: The time in nanoseconds when the events were first recorded.
166    async fn dispatch_key(self: &Rc<Self>, key_event: fidl_ui_input3::KeyEvent) {
167        assert!(
168            key_event.timestamp.is_some(),
169            "dispatch_key: got a key_event without a timestamp: {:?}",
170            &key_event
171        );
172        match self.key_event_injector.inject(&key_event).await {
173            Err(err) => self.metrics_logger.log_error(
174                InputPipelineErrorMetricDimensionEvent::ImeFailedToDispatchKeyToIme,
175                std::format!("Failed to dispatch key to IME: {:?}", err),
176            ),
177            _ => {}
178        };
179    }
180}
181
182/// Returns a KeyEvent with the given parameters.
183///
184/// # Parameters
185/// * `event`: The keyboard event to process.
186/// * `event_time`: The time in nanoseconds when the event was first recorded.
187fn create_key_event(
188    event: &keyboard_binding::KeyboardEvent,
189    event_time: zx::MonotonicInstant,
190    device_id: u32,
191) -> fidl_ui_input3::KeyEvent {
192    let modifier_state: FrozenModifierState =
193        event.get_modifiers().unwrap_or_else(|| Modifiers::from_bits_allow_unknown(0)).into();
194    let lock_state: FrozenLockState =
195        event.get_lock_state().unwrap_or_else(|| LockState::from_bits_allow_unknown(0)).into();
196    log::debug!(
197        "ImeHandler::create_key_event: key:{:?}, modifier_state: {:?}, lock_state: {:?}, event_type: {:?}",
198        event.get_key(),
199        modifier_state,
200        lock_state,
201        event.get_event_type(),
202    );
203    // Don't override the key meaning if already set, e.g. by prior stage.
204    let key_meaning = event
205        .get_key_meaning()
206        .or_else(|| keymaps::US_QWERTY.apply(event.get_key(), &modifier_state, &lock_state));
207
208    // Don't insert a spurious Some(0).
209    let repeat_sequence = match event.get_repeat_sequence() {
210        0 => None,
211        s => Some(s),
212    };
213
214    fidl_ui_input3::KeyEvent {
215        timestamp: Some(event_time.into_nanos()),
216        type_: event.get_event_type().into(),
217        key: event.get_key().into(),
218        modifiers: event.get_modifiers(),
219        lock_state: event.get_lock_state(),
220        key_meaning,
221        repeat_sequence,
222        device_id: Some(device_id),
223        ..Default::default()
224    }
225}
226
227#[cfg(test)]
228mod tests {
229    use super::*;
230    use crate::input_handler::InputHandler;
231    use crate::keyboard_binding::{self, KeyboardEvent};
232    use crate::testing_utilities;
233    use assert_matches::assert_matches;
234    use futures::StreamExt;
235    use std::convert::TryFrom as _;
236    use test_case::test_case;
237    use {
238        fidl_fuchsia_input as fidl_input, fidl_fuchsia_ui_input3 as fidl_ui_input3,
239        fuchsia_async as fasync,
240    };
241
242    fn handle_events(
243        ime_handler: Rc<ImeHandler>,
244        input_events: Vec<input_device::UnhandledInputEvent>,
245    ) {
246        fasync::Task::local(async move {
247            for input_event in input_events {
248                assert_matches!(
249                    ime_handler.clone().handle_unhandled_input_event(input_event).await.as_slice(),
250                    [input_device::InputEvent { handled: input_device::Handled::Yes, .. }]
251                );
252            }
253        })
254        .detach();
255    }
256
257    async fn assert_ime_receives_events(
258        expected_events: Vec<fidl_ui_input3::KeyEvent>,
259        mut request_stream: fidl_ui_input3::KeyEventInjectorRequestStream,
260    ) {
261        let mut expected_events_iter = expected_events.iter().peekable();
262        while let Some(Ok(fidl_ui_input3::KeyEventInjectorRequest::Inject {
263            key_event,
264            responder,
265            ..
266        })) = request_stream.next().await
267        {
268            pretty_assertions::assert_eq!(&key_event, expected_events_iter.next().unwrap());
269
270            // All the expected events have been received, so make sure no more events
271            // are present before returning.
272            if expected_events_iter.peek().is_none() {
273                responder
274                    .send(fidl_ui_input3::KeyEventStatus::Handled)
275                    .expect("error responding to DispatchKey");
276                return;
277            }
278            responder
279                .send(fidl_ui_input3::KeyEventStatus::Handled)
280                .expect("error responding to DispatchKey");
281        }
282
283        assert!(false);
284    }
285
286    fn connect_to_key_event_injector()
287    -> (fidl_ui_input3::KeyEventInjectorProxy, fidl_ui_input3::KeyEventInjectorRequestStream) {
288        fidl::endpoints::create_proxy_and_stream::<fidl_ui_input3::KeyEventInjectorMarker>()
289    }
290
291    fn create_unhandled_keyboard_event(
292        key: fidl_fuchsia_input::Key,
293        event_type: fidl_fuchsia_ui_input3::KeyEventType,
294        modifiers: Option<fidl_ui_input3::Modifiers>,
295        event_time: zx::MonotonicInstant,
296        device_descriptor: &input_device::InputDeviceDescriptor,
297        keymap: Option<String>,
298    ) -> input_device::UnhandledInputEvent {
299        create_unhandled_keyboard_event_with_key_meaning(
300            key,
301            event_type,
302            modifiers,
303            event_time,
304            device_descriptor,
305            keymap,
306            /* key_meaning */ None,
307        )
308    }
309
310    fn create_unhandled_keyboard_event_with_key_meaning(
311        key: fidl_fuchsia_input::Key,
312        event_type: fidl_fuchsia_ui_input3::KeyEventType,
313        modifiers: Option<fidl_ui_input3::Modifiers>,
314        event_time: zx::MonotonicInstant,
315        device_descriptor: &input_device::InputDeviceDescriptor,
316        keymap: Option<String>,
317        key_meaning: Option<fidl_fuchsia_ui_input3::KeyMeaning>,
318    ) -> input_device::UnhandledInputEvent {
319        input_device::UnhandledInputEvent::try_from(
320            testing_utilities::create_keyboard_event_with_key_meaning(
321                key,
322                event_type,
323                modifiers,
324                event_time,
325                device_descriptor,
326                keymap,
327                key_meaning,
328            ),
329        )
330        .unwrap()
331    }
332
333    fn create_unhandled_input_event(
334        keyboard_event: keyboard_binding::KeyboardEvent,
335        device_descriptor: &input_device::InputDeviceDescriptor,
336        event_time: zx::MonotonicInstant,
337    ) -> input_device::UnhandledInputEvent {
338        input_device::UnhandledInputEvent {
339            device_event: input_device::InputDeviceEvent::Keyboard(keyboard_event),
340            device_descriptor: device_descriptor.clone(),
341            event_time,
342            trace_id: None,
343        }
344    }
345
346    /// Tests that a pressed key event is dispatched.
347    ///
348    /// > NOTE: The `device_descriptor` used in this test case and elsewhere
349    /// *must* be of type `KeyboardDeviceDescriptor` as this is required by the
350    /// pattern matching in `ImeHandler`.
351    #[fasync::run_singlethreaded(test)]
352    async fn pressed_key() {
353        let (proxy, request_stream) = connect_to_key_event_injector();
354        let inspector = fuchsia_inspect::Inspector::default();
355        let test_node = inspector.root().create_child("test_node");
356        let ime_handler =
357            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
358                .await
359                .expect("Failed to create ImeHandler.");
360
361        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
362            keyboard_binding::KeyboardDeviceDescriptor {
363                keys: vec![fidl_input::Key::A],
364                device_id: 0,
365                ..Default::default()
366            },
367        );
368        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
369        let input_events = vec![create_unhandled_keyboard_event(
370            fidl_input::Key::A,
371            fidl_fuchsia_ui_input3::KeyEventType::Pressed,
372            None,
373            event_time_u64,
374            &device_descriptor,
375            /* keymap= */ None,
376        )];
377
378        let expected_events = vec![fidl_ui_input3::KeyEvent {
379            timestamp: Some(event_time_i64),
380            type_: Some(fidl_ui_input3::KeyEventType::Pressed),
381            key: Some(fidl_input::Key::A),
382            // A key "A" without shift is a lowercase 'a'.
383            key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
384            device_id: Some(0),
385            ..Default::default()
386        }];
387
388        handle_events(ime_handler, input_events);
389        assert_ime_receives_events(expected_events, request_stream).await;
390    }
391
392    /// Tests that a released key event is dispatched.
393    #[fasync::run_singlethreaded(test)]
394    async fn released_key() {
395        let (proxy, request_stream) = connect_to_key_event_injector();
396        let inspector = fuchsia_inspect::Inspector::default();
397        let test_node = inspector.root().create_child("test_node");
398        let ime_handler =
399            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
400                .await
401                .expect("Failed to create ImeHandler.");
402
403        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
404            keyboard_binding::KeyboardDeviceDescriptor {
405                keys: vec![fidl_input::Key::A],
406                device_id: 0,
407                ..Default::default()
408            },
409        );
410        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
411        let input_events = vec![create_unhandled_keyboard_event(
412            fidl_input::Key::A,
413            fidl_fuchsia_ui_input3::KeyEventType::Released,
414            None,
415            event_time_u64,
416            &device_descriptor,
417            /* keymap= */ None,
418        )];
419
420        let expected_events = vec![fidl_ui_input3::KeyEvent {
421            timestamp: Some(event_time_i64),
422            type_: Some(fidl_ui_input3::KeyEventType::Released),
423            key: Some(fidl_input::Key::A),
424            key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
425            device_id: Some(0),
426            ..Default::default()
427        }];
428
429        handle_events(ime_handler, input_events);
430        assert_ime_receives_events(expected_events, request_stream).await;
431    }
432
433    /// Tests that both pressed and released keys are dispatched appropriately.
434    #[fasync::run_singlethreaded(test)]
435    async fn pressed_and_released_key() {
436        let (proxy, request_stream) = connect_to_key_event_injector();
437        let inspector = fuchsia_inspect::Inspector::default();
438        let test_node = inspector.root().create_child("test_node");
439        let ime_handler =
440            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
441                .await
442                .expect("Failed to create ImeHandler.");
443
444        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
445            keyboard_binding::KeyboardDeviceDescriptor {
446                keys: vec![fidl_input::Key::A, fidl_input::Key::B],
447                device_id: 0,
448                ..Default::default()
449            },
450        );
451        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
452        let input_events = vec![
453            create_unhandled_keyboard_event(
454                fidl_input::Key::A,
455                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
456                None,
457                event_time_u64,
458                &device_descriptor,
459                /* keymap= */ None,
460            ),
461            create_unhandled_keyboard_event(
462                fidl_input::Key::A,
463                fidl_fuchsia_ui_input3::KeyEventType::Released,
464                None,
465                event_time_u64,
466                &device_descriptor,
467                /* keymap= */ None,
468            ),
469            create_unhandled_keyboard_event(
470                fidl_input::Key::B,
471                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
472                None,
473                event_time_u64,
474                &device_descriptor,
475                /* keymap= */ None,
476            ),
477            create_unhandled_keyboard_event_with_key_meaning(
478                fidl_input::Key::C,
479                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
480                None,
481                event_time_u64,
482                &device_descriptor,
483                /* keymap= */ None,
484                Some(fidl_fuchsia_ui_input3::KeyMeaning::Codepoint(42)),
485            ),
486        ];
487
488        let expected_events = vec![
489            fidl_ui_input3::KeyEvent {
490                timestamp: Some(event_time_i64),
491                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
492                key: Some(fidl_input::Key::A),
493                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
494                device_id: Some(0),
495                ..Default::default()
496            },
497            fidl_ui_input3::KeyEvent {
498                timestamp: Some(event_time_i64),
499                type_: Some(fidl_ui_input3::KeyEventType::Released),
500                key: Some(fidl_input::Key::A),
501                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
502                device_id: Some(0),
503                ..Default::default()
504            },
505            fidl_ui_input3::KeyEvent {
506                timestamp: Some(event_time_i64),
507                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
508                key: Some(fidl_input::Key::B),
509                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(98)),
510                device_id: Some(0),
511                ..Default::default()
512            },
513            fidl_ui_input3::KeyEvent {
514                timestamp: Some(event_time_i64),
515                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
516                key: Some(fidl_input::Key::C),
517                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(42)),
518                device_id: Some(0),
519                ..Default::default()
520            },
521        ];
522
523        handle_events(ime_handler, input_events);
524        assert_ime_receives_events(expected_events, request_stream).await;
525    }
526
527    // Tests that modifier keys are dispatched appropriately.
528    //
529    // This test depends on the incoming event having correct modifier and lock
530    // state.  Typically you'd do this by installing a ModifierHandler upstream
531    // of this pipeline stage.
532    #[fasync::run_singlethreaded(test)]
533    async fn repeated_modifier_key() {
534        let (proxy, request_stream) = connect_to_key_event_injector();
535        let inspector = fuchsia_inspect::Inspector::default();
536        let test_node = inspector.root().create_child("test_node");
537        let ime_handler =
538            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
539                .await
540                .expect("Failed to create ImeHandler.");
541
542        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
543            keyboard_binding::KeyboardDeviceDescriptor {
544                keys: vec![fidl_input::Key::A, fidl_input::Key::CapsLock],
545                device_id: 0,
546                ..Default::default()
547            },
548        );
549        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
550        let input_events = vec![
551            create_unhandled_input_event(
552                KeyboardEvent::new(
553                    fidl_input::Key::CapsLock,
554                    fidl_fuchsia_ui_input3::KeyEventType::Pressed,
555                )
556                .into_with_modifiers(Some(fidl_ui_input3::Modifiers::CAPS_LOCK))
557                .into_with_lock_state(Some(fidl_ui_input3::LockState::CAPS_LOCK)),
558                &device_descriptor,
559                event_time_u64,
560            ),
561            create_unhandled_input_event(
562                KeyboardEvent::new(
563                    fidl_input::Key::A,
564                    fidl_fuchsia_ui_input3::KeyEventType::Pressed,
565                )
566                .into_with_modifiers(Some(fidl_ui_input3::Modifiers::CAPS_LOCK))
567                .into_with_lock_state(Some(fidl_ui_input3::LockState::CAPS_LOCK)),
568                &device_descriptor,
569                event_time_u64,
570            ),
571            create_unhandled_input_event(
572                KeyboardEvent::new(
573                    fidl_input::Key::CapsLock,
574                    fidl_fuchsia_ui_input3::KeyEventType::Released,
575                )
576                .into_with_lock_state(Some(fidl_ui_input3::LockState::CAPS_LOCK)),
577                &device_descriptor,
578                event_time_u64,
579            ),
580        ];
581
582        let expected_events = vec![
583            fidl_ui_input3::KeyEvent {
584                timestamp: Some(event_time_i64),
585                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
586                key: Some(fidl_input::Key::CapsLock),
587                modifiers: Some(fidl_ui_input3::Modifiers::CAPS_LOCK),
588                lock_state: Some(fidl_ui_input3::LockState::CAPS_LOCK),
589                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
590                    fidl_ui_input3::NonPrintableKey::CapsLock,
591                )),
592                device_id: Some(0),
593                ..Default::default()
594            },
595            fidl_ui_input3::KeyEvent {
596                timestamp: Some(event_time_i64),
597                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
598                key: Some(fidl_input::Key::A),
599                modifiers: Some(fidl_ui_input3::Modifiers::CAPS_LOCK),
600                lock_state: Some(fidl_ui_input3::LockState::CAPS_LOCK),
601                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(65)),
602                device_id: Some(0),
603                ..Default::default()
604            },
605            fidl_ui_input3::KeyEvent {
606                timestamp: Some(event_time_i64),
607                type_: Some(fidl_ui_input3::KeyEventType::Released),
608                key: Some(fidl_input::Key::CapsLock),
609                lock_state: Some(fidl_ui_input3::LockState::CAPS_LOCK),
610                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
611                    fidl_ui_input3::NonPrintableKey::CapsLock,
612                )),
613                device_id: Some(0),
614                ..Default::default()
615            },
616        ];
617
618        handle_events(ime_handler, input_events);
619        assert_ime_receives_events(expected_events, request_stream).await;
620    }
621
622    #[fasync::run_singlethreaded(test)]
623    async fn nonprintable_key_meanings_set_correctly() {
624        let (proxy, request_stream) = connect_to_key_event_injector();
625        let inspector = fuchsia_inspect::Inspector::default();
626        let test_node = inspector.root().create_child("test_node");
627        let ime_handler =
628            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
629                .await
630                .expect("Failed to create ImeHandler.");
631
632        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
633            keyboard_binding::KeyboardDeviceDescriptor {
634                keys: vec![
635                    fidl_input::Key::Enter,
636                    fidl_input::Key::Tab,
637                    fidl_input::Key::Backspace,
638                ],
639                device_id: 0,
640                ..Default::default()
641            },
642        );
643        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
644        let input_events = vec![
645            create_unhandled_keyboard_event(
646                fidl_input::Key::Enter,
647                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
648                None,
649                event_time_u64,
650                &device_descriptor,
651                /* keymap= */ None,
652            ),
653            create_unhandled_keyboard_event(
654                fidl_input::Key::Tab,
655                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
656                None,
657                event_time_u64,
658                &device_descriptor,
659                /* keymap= */ None,
660            ),
661            create_unhandled_keyboard_event(
662                fidl_input::Key::Backspace,
663                fidl_fuchsia_ui_input3::KeyEventType::Released,
664                None,
665                event_time_u64,
666                &device_descriptor,
667                /* keymap= */ None,
668            ),
669        ];
670
671        let expected_events = vec![
672            fidl_ui_input3::KeyEvent {
673                timestamp: Some(event_time_i64),
674                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
675                key: Some(fidl_input::Key::Enter),
676                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
677                    fidl_ui_input3::NonPrintableKey::Enter,
678                )),
679                device_id: Some(0),
680                ..Default::default()
681            },
682            fidl_ui_input3::KeyEvent {
683                timestamp: Some(event_time_i64),
684                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
685                key: Some(fidl_input::Key::Tab),
686                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
687                    fidl_ui_input3::NonPrintableKey::Tab,
688                )),
689                device_id: Some(0),
690                ..Default::default()
691            },
692            fidl_ui_input3::KeyEvent {
693                timestamp: Some(event_time_i64),
694                // Test that things also work when a key is released.
695                type_: Some(fidl_ui_input3::KeyEventType::Released),
696                key: Some(fidl_input::Key::Backspace),
697                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
698                    fidl_ui_input3::NonPrintableKey::Backspace,
699                )),
700                device_id: Some(0),
701                ..Default::default()
702            },
703        ];
704
705        handle_events(ime_handler, input_events);
706        assert_ime_receives_events(expected_events, request_stream).await;
707    }
708
709    #[fasync::run_singlethreaded(test)]
710    async fn tab() {
711        let (proxy, request_stream) = connect_to_key_event_injector();
712        let inspector = fuchsia_inspect::Inspector::default();
713        let test_node = inspector.root().create_child("test_node");
714        let ime_handler =
715            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
716                .await
717                .expect("Failed to create ImeHandler.");
718
719        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
720            keyboard_binding::KeyboardDeviceDescriptor {
721                keys: vec![
722                    fidl_input::Key::Enter,
723                    fidl_input::Key::Tab,
724                    fidl_input::Key::Backspace,
725                ],
726                device_id: 0,
727                ..Default::default()
728            },
729        );
730        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
731        let input_events = vec![create_unhandled_keyboard_event(
732            fidl_input::Key::Tab,
733            fidl_fuchsia_ui_input3::KeyEventType::Pressed,
734            None,
735            event_time_u64,
736            &device_descriptor,
737            /* keymap= */ None,
738        )];
739
740        let expected_events = vec![fidl_ui_input3::KeyEvent {
741            timestamp: Some(event_time_i64),
742            type_: Some(fidl_ui_input3::KeyEventType::Pressed),
743            key: Some(fidl_input::Key::Tab),
744            key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
745                fidl_ui_input3::NonPrintableKey::Tab,
746            )),
747            device_id: Some(0),
748            ..Default::default()
749        }];
750
751        handle_events(ime_handler, input_events);
752        assert_ime_receives_events(expected_events, request_stream).await;
753    }
754
755    #[fasync::run_singlethreaded(test)]
756    async fn shift_shift_a() {
757        let (proxy, request_stream) = connect_to_key_event_injector();
758        let inspector = fuchsia_inspect::Inspector::default();
759        let test_node = inspector.root().create_child("test_node");
760        let ime_handler =
761            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
762                .await
763                .expect("Failed to create ImeHandler.");
764
765        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
766            keyboard_binding::KeyboardDeviceDescriptor {
767                keys: vec![fidl_input::Key::LeftCtrl, fidl_input::Key::Tab],
768                device_id: 0,
769                ..Default::default()
770            },
771        );
772        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
773        let input_events = vec![
774            create_unhandled_keyboard_event(
775                fidl_input::Key::LeftShift,
776                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
777                Some(Modifiers::LEFT_SHIFT | Modifiers::SHIFT),
778                event_time_u64,
779                &device_descriptor,
780                /* keymap= */ None,
781            ),
782            create_unhandled_keyboard_event(
783                fidl_input::Key::RightShift,
784                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
785                Some(Modifiers::LEFT_SHIFT | Modifiers::RIGHT_SHIFT | Modifiers::SHIFT),
786                event_time_u64,
787                &device_descriptor,
788                /* keymap= */ None,
789            ),
790            create_unhandled_keyboard_event(
791                fidl_input::Key::A,
792                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
793                Some(Modifiers::LEFT_SHIFT | Modifiers::RIGHT_SHIFT | Modifiers::SHIFT),
794                event_time_u64,
795                &device_descriptor,
796                /* keymap= */ None,
797            ),
798        ];
799
800        let expected_events = vec![
801            fidl_ui_input3::KeyEvent {
802                timestamp: Some(event_time_i64),
803                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
804                key: Some(fidl_input::Key::LeftShift),
805                modifiers: Some(Modifiers::LEFT_SHIFT | Modifiers::SHIFT),
806                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
807                    fidl_ui_input3::NonPrintableKey::Shift,
808                )),
809                device_id: Some(0),
810                ..Default::default()
811            },
812            fidl_ui_input3::KeyEvent {
813                timestamp: Some(event_time_i64),
814                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
815                key: Some(fidl_input::Key::RightShift),
816                modifiers: Some(Modifiers::RIGHT_SHIFT | Modifiers::LEFT_SHIFT | Modifiers::SHIFT),
817                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
818                    fidl_ui_input3::NonPrintableKey::Shift,
819                )),
820                device_id: Some(0),
821                ..Default::default()
822            },
823            fidl_ui_input3::KeyEvent {
824                timestamp: Some(event_time_i64),
825                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
826                key: Some(fidl_input::Key::A),
827                modifiers: Some(Modifiers::RIGHT_SHIFT | Modifiers::LEFT_SHIFT | Modifiers::SHIFT),
828                key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(65)), // "A"
829                device_id: Some(0),
830                ..Default::default()
831            },
832        ];
833
834        handle_events(ime_handler, input_events);
835        assert_ime_receives_events(expected_events, request_stream).await;
836    }
837
838    #[fasync::run_singlethreaded(test)]
839    async fn ctrl_tab() {
840        let (proxy, request_stream) = connect_to_key_event_injector();
841        let inspector = fuchsia_inspect::Inspector::default();
842        let test_node = inspector.root().create_child("test_node");
843        let ime_handler =
844            ImeHandler::new_handler(proxy, &test_node, metrics::MetricsLogger::default())
845                .await
846                .expect("Failed to create ImeHandler.");
847
848        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
849            keyboard_binding::KeyboardDeviceDescriptor {
850                keys: vec![fidl_input::Key::LeftCtrl, fidl_input::Key::Tab],
851                device_id: 0,
852                ..Default::default()
853            },
854        );
855        let (event_time_i64, event_time_u64) = testing_utilities::event_times();
856        let input_events = vec![
857            create_unhandled_keyboard_event(
858                fidl_input::Key::LeftCtrl,
859                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
860                None,
861                event_time_u64,
862                &device_descriptor,
863                /* keymap= */ None,
864            ),
865            create_unhandled_keyboard_event(
866                fidl_input::Key::Tab,
867                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
868                None,
869                event_time_u64,
870                &device_descriptor,
871                /* keymap= */ None,
872            ),
873        ];
874
875        let expected_events = vec![
876            fidl_ui_input3::KeyEvent {
877                timestamp: Some(event_time_i64),
878                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
879                key: Some(fidl_input::Key::LeftCtrl),
880                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
881                    fidl_ui_input3::NonPrintableKey::Control,
882                )),
883                device_id: Some(0),
884                ..Default::default()
885            },
886            fidl_ui_input3::KeyEvent {
887                timestamp: Some(event_time_i64),
888                type_: Some(fidl_ui_input3::KeyEventType::Pressed),
889                key: Some(fidl_input::Key::Tab),
890                key_meaning: Some(fidl_ui_input3::KeyMeaning::NonPrintableKey(
891                    fidl_ui_input3::NonPrintableKey::Tab,
892                )),
893                device_id: Some(0),
894                ..Default::default()
895            },
896        ];
897
898        handle_events(ime_handler, input_events);
899        assert_ime_receives_events(expected_events, request_stream).await;
900    }
901
902    #[fasync::run_singlethreaded(test)]
903    async fn ime_handler_initialized_with_inspect_node() {
904        let (proxy, _) = connect_to_key_event_injector();
905        let inspector = fuchsia_inspect::Inspector::default();
906        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
907        let _handler =
908            ImeHandler::new_handler(proxy, &fake_handlers_node, metrics::MetricsLogger::default())
909                .await
910                .expect("Failed to create ImeHandler.");
911        diagnostics_assertions::assert_data_tree!(inspector, root: {
912            input_handlers_node: {
913                ime_handler: {
914                    events_received_count: 0u64,
915                    events_handled_count: 0u64,
916                    last_received_timestamp_ns: 0u64,
917                    "fuchsia.inspect.Health": {
918                        status: "STARTING_UP",
919                        // Timestamp value is unpredictable and not relevant in this context,
920                        // so we only assert that the property is present.
921                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
922                    },
923                }
924            }
925        });
926    }
927
928    #[fasync::run_singlethreaded(test)]
929    async fn ime_handler_inspect_counts_events() {
930        let (proxy, _) = connect_to_key_event_injector();
931        let inspector = fuchsia_inspect::Inspector::default();
932        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
933        let ime_handler =
934            ImeHandler::new_handler(proxy, &fake_handlers_node, metrics::MetricsLogger::default())
935                .await
936                .expect("Failed to create ImeHandler.");
937        let device_descriptor = input_device::InputDeviceDescriptor::Keyboard(
938            keyboard_binding::KeyboardDeviceDescriptor {
939                keys: vec![fidl_input::Key::A, fidl_input::Key::B],
940                ..Default::default()
941            },
942        );
943        let (_, event_time_u64) = testing_utilities::event_times();
944        let input_events = vec![
945            testing_utilities::create_keyboard_event_with_time(
946                fidl_input::Key::A,
947                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
948                None,
949                event_time_u64,
950                &device_descriptor,
951                /* keymap= */ None,
952            ),
953            // Should not count received events that have already been handled.
954            testing_utilities::create_keyboard_event_with_handled(
955                fidl_input::Key::B,
956                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
957                None,
958                event_time_u64,
959                &device_descriptor,
960                /* keymap= */ None,
961                /* key_meaning= */ None,
962                input_device::Handled::Yes,
963            ),
964            testing_utilities::create_keyboard_event_with_time(
965                fidl_input::Key::A,
966                fidl_fuchsia_ui_input3::KeyEventType::Released,
967                None,
968                event_time_u64,
969                &device_descriptor,
970                /* keymap= */ None,
971            ),
972            // Should not count non-keyboard input events.
973            testing_utilities::create_fake_input_event(event_time_u64),
974            testing_utilities::create_keyboard_event_with_time(
975                fidl_input::Key::B,
976                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
977                None,
978                event_time_u64,
979                &device_descriptor,
980                /* keymap= */ None,
981            ),
982        ];
983
984        for input_event in input_events {
985            let _ = ime_handler.clone().handle_input_event(input_event).await;
986        }
987
988        let last_event_timestamp: u64 = event_time_u64.into_nanos().try_into().unwrap();
989
990        diagnostics_assertions::assert_data_tree!(inspector, root: {
991            input_handlers_node: {
992                ime_handler: {
993                    events_received_count: 3u64,
994                    events_handled_count: 3u64,
995                    last_received_timestamp_ns: last_event_timestamp,
996                    "fuchsia.inspect.Health": {
997                        status: "STARTING_UP",
998                        // Timestamp value is unpredictable and not relevant in this context,
999                        // so we only assert that the property is present.
1000                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
1001                    },
1002                }
1003            }
1004        });
1005    }
1006
1007    #[test_case(
1008        keyboard_binding::KeyboardEvent::new(
1009            fidl_input::Key::A,
1010            fidl_ui_input3::KeyEventType::Pressed),
1011        zx::MonotonicInstant::from_nanos(42) => fidl_ui_input3::KeyEvent{
1012            timestamp: Some(42),
1013            type_: Some(fidl_ui_input3::KeyEventType::Pressed),
1014            key: Some(fidl_input::Key::A),
1015            key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
1016            device_id: Some(0),
1017            ..Default::default()}; "basic value copy")]
1018    #[test_case(
1019        keyboard_binding::KeyboardEvent::new(
1020            fidl_input::Key::A,
1021            fidl_ui_input3::KeyEventType::Pressed)
1022            .into_with_repeat_sequence(13),
1023        zx::MonotonicInstant::from_nanos(42) => fidl_ui_input3::KeyEvent{
1024            timestamp: Some(42),
1025            type_: Some(fidl_ui_input3::KeyEventType::Pressed),
1026            key: Some(fidl_input::Key::A),
1027            key_meaning: Some(fidl_ui_input3::KeyMeaning::Codepoint(97)),
1028            repeat_sequence: Some(13),
1029            device_id: Some(0),
1030            ..Default::default()}; "repeat_sequence is honored")]
1031    fn test_create_key_event(
1032        event: keyboard_binding::KeyboardEvent,
1033        event_time: zx::MonotonicInstant,
1034    ) -> fidl_ui_input3::KeyEvent {
1035        super::create_key_event(&event, event_time, 0)
1036    }
1037}
input_pipeline/ime_handler.rs

input_pipeline/
ime_handler.rs
