input_pipeline/
modifier_handler.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::input_device::{
6    Handled, InputDeviceEvent, InputEvent, InputEventType, UnhandledInputEvent,
7};
8use crate::input_handler::{InputHandlerStatus, UnhandledInputHandler};
9use async_trait::async_trait;
10use fidl_fuchsia_ui_input3::{KeyMeaning, Modifiers, NonPrintableKey};
11use fuchsia_inspect::health::Reporter;
12use keymaps::{LockStateKeys, ModifierState};
13use std::cell::RefCell;
14use std::rc::Rc;
15
16/// Tracks modifier state and decorates passing events with the modifiers.
17///
18/// This handler should be installed as early as possible in the input pipeline,
19/// to ensure that all later stages have the modifiers and lock states available.
20/// This holds even for non-keyboard handlers, to allow handling `Ctrl+Click`
21/// events, for example.
22///
23/// One possible exception to this rule would be a hardware key rewriting handler for
24/// limited keyboards.
25#[derive(Debug)]
26pub struct ModifierHandler {
27    /// The tracked state of the modifiers.
28    modifier_state: RefCell<ModifierState>,
29
30    /// The tracked lock state.
31    lock_state: RefCell<LockStateKeys>,
32
33    /// The inventory of this handler's Inspect status.
34    pub inspect_status: InputHandlerStatus,
35}
36
37#[async_trait(?Send)]
38impl UnhandledInputHandler for ModifierHandler {
39    async fn handle_unhandled_input_event(
40        self: Rc<Self>,
41        unhandled_input_event: UnhandledInputEvent,
42    ) -> Vec<InputEvent> {
43        fuchsia_trace::duration!("input", "modifier_handler");
44        match unhandled_input_event {
45            UnhandledInputEvent {
46                device_event: InputDeviceEvent::Keyboard(mut event),
47                device_descriptor,
48                event_time,
49                trace_id,
50            } => {
51                fuchsia_trace::duration!("input", "modifier_handler[processing]");
52                if let Some(trace_id) = trace_id {
53                    fuchsia_trace::flow_step!(
54                        c"input",
55                        c"event_in_input_pipeline",
56                        trace_id.into()
57                    );
58                }
59
60                self.inspect_status.count_received_event(&event_time);
61                self.modifier_state.borrow_mut().update(event.get_event_type(), event.get_key());
62                self.lock_state.borrow_mut().update(event.get_event_type(), event.get_key());
63                event = event
64                    .into_with_lock_state(Some(self.lock_state.borrow().get_state()))
65                    .into_with_modifiers(Some(self.modifier_state.borrow().get_state()));
66                log::debug!("modifiers and lock state applied: {:?}", &event);
67                vec![InputEvent {
68                    device_event: InputDeviceEvent::Keyboard(event),
69                    device_descriptor,
70                    event_time,
71                    handled: Handled::No,
72                    trace_id,
73                }]
74            }
75            // Pass other events through.
76            _ => {
77                // TODO: b/478249522 - add cobalt logging
78                log::warn!("Unhandled input event: {:?}", unhandled_input_event.get_event_type());
79                vec![InputEvent::from(unhandled_input_event)]
80            }
81        }
82    }
83
84    fn set_handler_healthy(self: std::rc::Rc<Self>) {
85        self.inspect_status.health_node.borrow_mut().set_ok();
86    }
87
88    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
89        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
90    }
91
92    fn get_name(&self) -> &'static str {
93        "ModifierHandler"
94    }
95
96    fn interest(&self) -> Vec<InputEventType> {
97        vec![InputEventType::Keyboard]
98    }
99}
100
101impl ModifierHandler {
102    /// Creates a new [ModifierHandler].
103    pub fn new(input_handlers_node: &fuchsia_inspect::Node) -> Rc<Self> {
104        let inspect_status = InputHandlerStatus::new(
105            input_handlers_node,
106            "modifier_handler",
107            /* generates_events */ false,
108        );
109        Rc::new(Self {
110            modifier_state: RefCell::new(ModifierState::new()),
111            lock_state: RefCell::new(LockStateKeys::new()),
112            inspect_status,
113        })
114    }
115}
116
117/// Tracks the state of the modifiers that are tied to the key meaning (as opposed to hardware
118/// keys).
119#[derive(Debug)]
120pub struct ModifierMeaningHandler {
121    /// The tracked state of the modifiers.
122    modifier_state: RefCell<ModifierState>,
123
124    /// The inventory of this handler's Inspect status.
125    pub inspect_status: InputHandlerStatus,
126}
127
128impl ModifierMeaningHandler {
129    /// Creates a new [ModifierMeaningHandler].
130    pub fn new(input_handlers_node: &fuchsia_inspect::Node) -> Rc<Self> {
131        let inspect_status = InputHandlerStatus::new(
132            input_handlers_node,
133            "modifier_meaning_handler",
134            /* generates_events */ false,
135        );
136        Rc::new(Self { modifier_state: RefCell::new(ModifierState::new()), inspect_status })
137    }
138}
139
140#[async_trait(?Send)]
141impl UnhandledInputHandler for ModifierMeaningHandler {
142    async fn handle_unhandled_input_event(
143        self: Rc<Self>,
144        unhandled_input_event: UnhandledInputEvent,
145    ) -> Vec<InputEvent> {
146        fuchsia_trace::duration!("input", "modifier_meaning_handler");
147        match unhandled_input_event {
148            UnhandledInputEvent {
149                device_event: InputDeviceEvent::Keyboard(mut event),
150                device_descriptor,
151                event_time,
152                trace_id,
153            } if event.get_key_meaning()
154                == Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)) =>
155            {
156                fuchsia_trace::duration!("input", "modifier_meaning_handler[processing]");
157                if let Some(trace_id) = trace_id {
158                    fuchsia_trace::flow_step!(
159                        c"input",
160                        c"event_in_input_pipeline",
161                        trace_id.into()
162                    );
163                }
164                self.inspect_status.count_received_event(&event_time);
165                // The "obvious" rewrite of this if and the match guard above is
166                // unstable, so doing it this way.
167                if let Some(key_meaning) = event.get_key_meaning() {
168                    self.modifier_state
169                        .borrow_mut()
170                        .update_with_key_meaning(event.get_event_type(), key_meaning);
171                    let new_modifier = event.get_modifiers().unwrap_or(Modifiers::empty())
172                        | self.modifier_state.borrow().get_state();
173                    event = event.into_with_modifiers(Some(new_modifier));
174                    log::debug!("additinal modifiers and lock state applied: {:?}", &event);
175                }
176                vec![InputEvent {
177                    device_event: InputDeviceEvent::Keyboard(event),
178                    device_descriptor,
179                    event_time,
180                    handled: Handled::No,
181                    trace_id,
182                }]
183            }
184            // Pass other events through.
185            _ => {
186                // TODO: b/478249522 - add cobalt logging
187                log::warn!("Unhandled input event: {:?}", unhandled_input_event.get_event_type());
188                vec![InputEvent::from(unhandled_input_event)]
189            }
190        }
191    }
192
193    fn set_handler_healthy(self: std::rc::Rc<Self>) {
194        self.inspect_status.health_node.borrow_mut().set_ok();
195    }
196
197    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
198        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
199    }
200
201    fn get_name(&self) -> &'static str {
202        "ModifierHandler"
203    }
204
205    fn interest(&self) -> Vec<InputEventType> {
206        vec![InputEventType::Keyboard]
207    }
208}
209
210#[cfg(test)]
211mod tests {
212    use super::*;
213    use crate::input_device::InputDeviceDescriptor;
214    use crate::input_handler::InputHandler;
215    use crate::keyboard_binding::{self, KeyboardEvent};
216    use crate::testing_utilities;
217    use fidl_fuchsia_input::Key;
218    use fidl_fuchsia_ui_input3::{KeyEventType, LockState};
219    use fuchsia_async as fasync;
220    use pretty_assertions::assert_eq;
221
222    fn get_unhandled_input_event(event: KeyboardEvent) -> UnhandledInputEvent {
223        UnhandledInputEvent {
224            device_event: InputDeviceEvent::Keyboard(event),
225            event_time: zx::MonotonicInstant::from_nanos(42),
226            device_descriptor: InputDeviceDescriptor::Fake,
227            trace_id: None,
228        }
229    }
230
231    #[fasync::run_singlethreaded(test)]
232    async fn test_decoration() {
233        let inspector = fuchsia_inspect::Inspector::default();
234        let test_node = inspector.root().create_child("test_node");
235        let handler = ModifierHandler::new(&test_node);
236        let input_event =
237            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed));
238        let result = handler.handle_unhandled_input_event(input_event.clone()).await;
239
240        // This handler decorates, but does not handle the key. Hence,
241        // the key remains unhandled.
242        let expected = InputEvent::from(get_unhandled_input_event(
243            KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
244                .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
245                .into_with_lock_state(Some(LockState::CAPS_LOCK)),
246        ));
247        assert_eq!(vec![expected], result);
248    }
249
250    #[fasync::run_singlethreaded(test)]
251    async fn test_key_meaning_decoration() {
252        let inspector = fuchsia_inspect::Inspector::default();
253        let test_node = inspector.root().create_child("test_node");
254        let handler = ModifierMeaningHandler::new(&test_node);
255        {
256            let input_event = get_unhandled_input_event(
257                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
258                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
259                        NonPrintableKey::AltGraph,
260                    )))
261                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
262            );
263            let result = handler.clone().handle_unhandled_input_event(input_event.clone()).await;
264            let expected = InputEvent::from(get_unhandled_input_event(
265                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
266                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
267                        NonPrintableKey::AltGraph,
268                    )))
269                    .into_with_modifiers(Some(Modifiers::ALT_GRAPH | Modifiers::CAPS_LOCK)),
270            ));
271            assert_eq!(vec![expected], result);
272        }
273        {
274            let input_event = get_unhandled_input_event(
275                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
276                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
277                        NonPrintableKey::AltGraph,
278                    )))
279                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
280            );
281            let handler = handler.clone();
282            let result = handler.handle_unhandled_input_event(input_event.clone()).await;
283            let expected = InputEvent::from(get_unhandled_input_event(
284                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
285                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
286                        NonPrintableKey::AltGraph,
287                    )))
288                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
289            ));
290            assert_eq!(vec![expected], result);
291        }
292    }
293
294    // CapsLock  """"""\______/""""""""""\_______/"""
295    // Modifiers ------CCCCCCCC----------CCCCCCCCC---
296    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC---
297    //
298    // C == CapsLock
299    #[fasync::run_singlethreaded(test)]
300    async fn test_modifier_press_lock_release() {
301        let input_events = vec![
302            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
303            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
304            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
305            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
306        ];
307
308        let inspector = fuchsia_inspect::Inspector::default();
309        let test_node = inspector.root().create_child("test_node");
310        let handler = ModifierHandler::new(&test_node);
311        let clone_handler = move || handler.clone();
312        let result = futures::future::join_all(
313            input_events
314                .into_iter()
315                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
316        )
317        .await
318        .into_iter()
319        .flatten()
320        .collect::<Vec<InputEvent>>();
321
322        let expected = IntoIterator::into_iter([
323            get_unhandled_input_event(
324                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
325                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
326                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
327            ),
328            get_unhandled_input_event(
329                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
330                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
331                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
332            ),
333            get_unhandled_input_event(
334                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
335                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
336                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
337            ),
338            get_unhandled_input_event(
339                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
340                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
341                    .into_with_lock_state(Some(LockState::from_bits_allow_unknown(0))),
342            ),
343        ])
344        .map(InputEvent::from)
345        .collect::<Vec<_>>();
346
347        assert_eq!(expected, result);
348    }
349
350    // CapsLock  """"""\______/"""""""""""""""""""
351    // A         """""""""""""""""""\________/""""
352    // Modifiers ------CCCCCCCC-------------------
353    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC
354    //
355    // C == CapsLock
356    #[fasync::run_singlethreaded(test)]
357    async fn repeated_modifier_key() {
358        let input_events = vec![
359            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
360            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
361            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Pressed)),
362            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Released)),
363        ];
364
365        let inspector = fuchsia_inspect::Inspector::default();
366        let test_node = inspector.root().create_child("test_node");
367        let handler = ModifierHandler::new(&test_node);
368        let clone_handler = move || handler.clone();
369        let result = futures::future::join_all(
370            input_events
371                .into_iter()
372                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
373        )
374        .await
375        .into_iter()
376        .flatten()
377        .collect::<Vec<InputEvent>>();
378
379        let expected = IntoIterator::into_iter([
380            get_unhandled_input_event(
381                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
382                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
383                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
384            ),
385            get_unhandled_input_event(
386                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
387                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
388                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
389            ),
390            get_unhandled_input_event(
391                KeyboardEvent::new(Key::A, KeyEventType::Pressed)
392                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
393                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
394            ),
395            get_unhandled_input_event(
396                KeyboardEvent::new(Key::A, KeyEventType::Released)
397                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
398                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
399            ),
400        ])
401        .map(InputEvent::from)
402        .collect::<Vec<_>>();
403        assert_eq!(expected, result);
404    }
405
406    #[fuchsia::test]
407    async fn modifier_handlers_initialized_with_inspect_node() {
408        let inspector = fuchsia_inspect::Inspector::default();
409        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
410        let _modifier_handler = ModifierHandler::new(&fake_handlers_node);
411        let _modifier_meaning_handler = ModifierMeaningHandler::new(&fake_handlers_node);
412        diagnostics_assertions::assert_data_tree!(inspector, root: {
413            input_handlers_node: {
414                modifier_handler: {
415                    events_received_count: 0u64,
416                    events_handled_count: 0u64,
417                    last_received_timestamp_ns: 0u64,
418                    "fuchsia.inspect.Health": {
419                        status: "STARTING_UP",
420                        // Timestamp value is unpredictable and not relevant in this context,
421                        // so we only assert that the property is present.
422                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
423                    },
424                },
425                modifier_meaning_handler: {
426                    events_received_count: 0u64,
427                    events_handled_count: 0u64,
428                    last_received_timestamp_ns: 0u64,
429                    "fuchsia.inspect.Health": {
430                        status: "STARTING_UP",
431                        // Timestamp value is unpredictable and not relevant in this context,
432                        // so we only assert that the property is present.
433                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
434                    },
435                }
436            }
437        });
438    }
439
440    #[fasync::run_singlethreaded(test)]
441    async fn modifier_handler_inspect_counts_events() {
442        let inspector = fuchsia_inspect::Inspector::default();
443        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
444        let modifier_handler = ModifierHandler::new(&fake_handlers_node);
445        let modifier_meaning_handler = ModifierMeaningHandler::new(&fake_handlers_node);
446        let device_descriptor =
447            InputDeviceDescriptor::Keyboard(keyboard_binding::KeyboardDeviceDescriptor {
448                keys: vec![Key::A, Key::B, Key::RightAlt],
449                ..Default::default()
450            });
451        let (_, event_time_u64) = testing_utilities::event_times();
452        let input_events = vec![
453            testing_utilities::create_keyboard_event_with_time(
454                Key::A,
455                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
456                None,
457                event_time_u64,
458                &device_descriptor,
459                /* keymap= */ None,
460            ),
461            // Should not count received events that have already been handled.
462            testing_utilities::create_keyboard_event_with_handled(
463                Key::B,
464                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
465                None,
466                event_time_u64,
467                &device_descriptor,
468                /* keymap= */ None,
469                /* key_meaning= */ None,
470                Handled::Yes,
471            ),
472            testing_utilities::create_keyboard_event_with_time(
473                Key::A,
474                fidl_fuchsia_ui_input3::KeyEventType::Released,
475                None,
476                event_time_u64,
477                &device_descriptor,
478                /* keymap= */ None,
479            ),
480            // Should not count non-keyboard input events.
481            testing_utilities::create_fake_input_event(event_time_u64),
482            // Only event that should be counted by ModifierMeaningHandler.
483            testing_utilities::create_keyboard_event_with_key_meaning(
484                Key::RightAlt,
485                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
486                None,
487                event_time_u64,
488                &device_descriptor,
489                /* keymap= */ None,
490                /* key_meaning= */
491                Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)),
492            ),
493        ];
494
495        for input_event in input_events {
496            let _ = modifier_handler.clone().handle_input_event(input_event.clone()).await;
497            let _ = modifier_meaning_handler.clone().handle_input_event(input_event).await;
498        }
499
500        let last_event_timestamp: u64 = event_time_u64.into_nanos().try_into().unwrap();
501
502        diagnostics_assertions::assert_data_tree!(inspector, root: {
503            input_handlers_node: {
504                modifier_handler: {
505                    events_received_count: 3u64,
506                    events_handled_count: 0u64,
507                    last_received_timestamp_ns: last_event_timestamp,
508                    "fuchsia.inspect.Health": {
509                        status: "STARTING_UP",
510                        // Timestamp value is unpredictable and not relevant in this context,
511                        // so we only assert that the property is present.
512                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
513                    },
514                },
515                modifier_meaning_handler: {
516                    events_received_count: 1u64,
517                    events_handled_count: 0u64,
518                    last_received_timestamp_ns: last_event_timestamp,
519                    "fuchsia.inspect.Health": {
520                        status: "STARTING_UP",
521                        // Timestamp value is unpredictable and not relevant in this context,
522                        // so we only assert that the property is present.
523                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
524                    },
525                }
526            }
527        });
528    }
529}
input_pipeline/modifier_handler.rs

input_pipeline/
modifier_handler.rs
