input_pipeline_dso/
modifier_handler.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::input_device::{
6    Handled, InputDeviceEvent, InputEvent, InputEventType, UnhandledInputEvent,
7};
8use crate::input_handler::{Handler, InputHandlerStatus, UnhandledInputHandler};
9use crate::metrics;
10use async_trait::async_trait;
11use fidl_fuchsia_ui_input3::{KeyMeaning, Modifiers, NonPrintableKey};
12use fuchsia_inspect::health::Reporter;
13use keymaps::{LockStateKeys, ModifierState};
14use metrics_registry::InputPipelineErrorMetricDimensionEvent;
15use std::cell::RefCell;
16use std::rc::Rc;
17
18/// Tracks modifier state and decorates passing events with the modifiers.
19///
20/// This handler should be installed as early as possible in the input pipeline,
21/// to ensure that all later stages have the modifiers and lock states available.
22/// This holds even for non-keyboard handlers, to allow handling `Ctrl+Click`
23/// events, for example.
24///
25/// One possible exception to this rule would be a hardware key rewriting handler for
26/// limited keyboards.
27#[derive(Debug)]
28pub struct ModifierHandler {
29    /// The tracked state of the modifiers.
30    modifier_state: RefCell<ModifierState>,
31
32    /// The tracked lock state.
33    lock_state: RefCell<LockStateKeys>,
34
35    /// The metrics logger.
36    metrics_logger: metrics::MetricsLogger,
37
38    /// The inventory of this handler's Inspect status.
39    pub inspect_status: InputHandlerStatus,
40}
41
42impl Handler for ModifierHandler {
43    fn set_handler_healthy(self: std::rc::Rc<Self>) {
44        self.inspect_status.health_node.borrow_mut().set_ok();
45    }
46
47    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
48        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
49    }
50
51    fn get_name(&self) -> &'static str {
52        "ModifierHandler"
53    }
54
55    fn interest(&self) -> Vec<InputEventType> {
56        vec![InputEventType::Keyboard]
57    }
58}
59
60#[async_trait(?Send)]
61impl UnhandledInputHandler for ModifierHandler {
62    async fn handle_unhandled_input_event(
63        self: Rc<Self>,
64        unhandled_input_event: UnhandledInputEvent,
65    ) -> Vec<InputEvent> {
66        fuchsia_trace::duration!("input", "modifier_handler");
67        match unhandled_input_event {
68            UnhandledInputEvent {
69                device_event: InputDeviceEvent::Keyboard(mut event),
70                device_descriptor,
71                event_time,
72                trace_id,
73            } => {
74                fuchsia_trace::duration!("input", "modifier_handler[processing]");
75                if let Some(trace_id) = trace_id {
76                    fuchsia_trace::flow_step!(
77                        c"input",
78                        c"event_in_input_pipeline",
79                        trace_id.into()
80                    );
81                }
82
83                self.inspect_status.count_received_event(&event_time);
84                self.modifier_state.borrow_mut().update(event.get_event_type(), event.get_key());
85                self.lock_state.borrow_mut().update(event.get_event_type(), event.get_key());
86                event = event
87                    .into_with_lock_state(Some(self.lock_state.borrow().get_state()))
88                    .into_with_modifiers(Some(self.modifier_state.borrow().get_state()));
89                log::debug!("modifiers and lock state applied: {:?}", &event);
90                vec![InputEvent {
91                    device_event: InputDeviceEvent::Keyboard(event),
92                    device_descriptor,
93                    event_time,
94                    handled: Handled::No,
95                    trace_id,
96                }]
97            }
98            // Pass other events through.
99            _ => {
100                if InputEventType::from(&unhandled_input_event.device_event)
101                    != InputEventType::Keyboard
102                {
103                    self.metrics_logger.log_error(
104                        InputPipelineErrorMetricDimensionEvent::HandlerReceivedUninterestedEvent,
105                        std::format!(
106                            "{} uninterested input event: {:?}",
107                            self.get_name(),
108                            unhandled_input_event.get_event_type()
109                        ),
110                    );
111                }
112                vec![InputEvent::from(unhandled_input_event)]
113            }
114        }
115    }
116}
117
118impl ModifierHandler {
119    pub fn new(
120        input_handlers_node: &fuchsia_inspect::Node,
121        metrics_logger: metrics::MetricsLogger,
122    ) -> Rc<Self> {
123        let inspect_status = InputHandlerStatus::new(
124            input_handlers_node,
125            "modifier_handler",
126            /* generates_events */ false,
127        );
128        Rc::new(Self {
129            modifier_state: RefCell::new(ModifierState::new()),
130            lock_state: RefCell::new(LockStateKeys::new()),
131            metrics_logger,
132            inspect_status,
133        })
134    }
135}
136
137/// Tracks the state of the modifiers that are tied to the key meaning (as opposed to hardware
138/// keys).
139#[derive(Debug)]
140pub struct ModifierMeaningHandler {
141    /// The tracked state of the modifiers.
142    modifier_state: RefCell<ModifierState>,
143
144    /// The metrics logger.
145    metrics_logger: metrics::MetricsLogger,
146
147    /// The inventory of this handler's Inspect status.
148    pub inspect_status: InputHandlerStatus,
149}
150
151impl ModifierMeaningHandler {
152    pub fn new(
153        input_handlers_node: &fuchsia_inspect::Node,
154        metrics_logger: metrics::MetricsLogger,
155    ) -> Rc<Self> {
156        let inspect_status = InputHandlerStatus::new(
157            input_handlers_node,
158            "modifier_meaning_handler",
159            /* generates_events */ false,
160        );
161        Rc::new(Self {
162            modifier_state: RefCell::new(ModifierState::new()),
163            metrics_logger,
164            inspect_status,
165        })
166    }
167}
168
169impl Handler for ModifierMeaningHandler {
170    fn set_handler_healthy(self: std::rc::Rc<Self>) {
171        self.inspect_status.health_node.borrow_mut().set_ok();
172    }
173
174    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
175        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
176    }
177
178    fn get_name(&self) -> &'static str {
179        "ModifierHandler"
180    }
181
182    fn interest(&self) -> Vec<InputEventType> {
183        vec![InputEventType::Keyboard]
184    }
185}
186
187#[async_trait(?Send)]
188impl UnhandledInputHandler for ModifierMeaningHandler {
189    async fn handle_unhandled_input_event(
190        self: Rc<Self>,
191        unhandled_input_event: UnhandledInputEvent,
192    ) -> Vec<InputEvent> {
193        fuchsia_trace::duration!("input", "modifier_meaning_handler");
194        match unhandled_input_event {
195            UnhandledInputEvent {
196                device_event: InputDeviceEvent::Keyboard(mut event),
197                device_descriptor,
198                event_time,
199                trace_id,
200            } if event.get_key_meaning()
201                == Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)) =>
202            {
203                fuchsia_trace::duration!("input", "modifier_meaning_handler[processing]");
204                if let Some(trace_id) = trace_id {
205                    fuchsia_trace::flow_step!(
206                        c"input",
207                        c"event_in_input_pipeline",
208                        trace_id.into()
209                    );
210                }
211                self.inspect_status.count_received_event(&event_time);
212                // The "obvious" rewrite of this if and the match guard above is
213                // unstable, so doing it this way.
214                if let Some(key_meaning) = event.get_key_meaning() {
215                    self.modifier_state
216                        .borrow_mut()
217                        .update_with_key_meaning(event.get_event_type(), key_meaning);
218                    let new_modifier = event.get_modifiers().unwrap_or(Modifiers::empty())
219                        | self.modifier_state.borrow().get_state();
220                    event = event.into_with_modifiers(Some(new_modifier));
221                    log::debug!("additinal modifiers and lock state applied: {:?}", &event);
222                }
223                vec![InputEvent {
224                    device_event: InputDeviceEvent::Keyboard(event),
225                    device_descriptor,
226                    event_time,
227                    handled: Handled::No,
228                    trace_id,
229                }]
230            }
231            // Pass other events through.
232            _ => {
233                if InputEventType::from(&unhandled_input_event.device_event)
234                    != InputEventType::Keyboard
235                {
236                    self.metrics_logger.log_error(
237                        InputPipelineErrorMetricDimensionEvent::HandlerReceivedUninterestedEvent,
238                        std::format!(
239                            "{} uninterested input event: {:?}",
240                            self.get_name(),
241                            unhandled_input_event.get_event_type()
242                        ),
243                    );
244                }
245                vec![InputEvent::from(unhandled_input_event)]
246            }
247        }
248    }
249}
250
251#[cfg(test)]
252mod tests {
253    use super::*;
254    use crate::input_device::InputDeviceDescriptor;
255    use crate::input_handler::InputHandler;
256    use crate::keyboard_binding::{self, KeyboardEvent};
257    use crate::testing_utilities;
258    use fidl_fuchsia_input::Key;
259    use fidl_fuchsia_ui_input3::{KeyEventType, LockState};
260    use fuchsia_async as fasync;
261    use pretty_assertions::assert_eq;
262
263    fn get_unhandled_input_event(event: KeyboardEvent) -> UnhandledInputEvent {
264        UnhandledInputEvent {
265            device_event: InputDeviceEvent::Keyboard(event),
266            event_time: zx::MonotonicInstant::from_nanos(42),
267            device_descriptor: InputDeviceDescriptor::Fake,
268            trace_id: None,
269        }
270    }
271
272    #[fasync::run_singlethreaded(test)]
273    async fn test_decoration() {
274        let inspector = fuchsia_inspect::Inspector::default();
275        let test_node = inspector.root().create_child("test_node");
276        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
277        let input_event =
278            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed));
279        let result = handler.handle_unhandled_input_event(input_event.clone()).await;
280
281        // This handler decorates, but does not handle the key. Hence,
282        // the key remains unhandled.
283        let expected = InputEvent::from(get_unhandled_input_event(
284            KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
285                .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
286                .into_with_lock_state(Some(LockState::CAPS_LOCK)),
287        ));
288        assert_eq!(vec![expected], result);
289    }
290
291    #[fasync::run_singlethreaded(test)]
292    async fn test_key_meaning_decoration() {
293        let inspector = fuchsia_inspect::Inspector::default();
294        let test_node = inspector.root().create_child("test_node");
295        let handler = ModifierMeaningHandler::new(&test_node, metrics::MetricsLogger::default());
296        {
297            let input_event = get_unhandled_input_event(
298                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
299                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
300                        NonPrintableKey::AltGraph,
301                    )))
302                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
303            );
304            let result = handler.clone().handle_unhandled_input_event(input_event.clone()).await;
305            let expected = InputEvent::from(get_unhandled_input_event(
306                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
307                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
308                        NonPrintableKey::AltGraph,
309                    )))
310                    .into_with_modifiers(Some(Modifiers::ALT_GRAPH | Modifiers::CAPS_LOCK)),
311            ));
312            assert_eq!(vec![expected], result);
313        }
314        {
315            let input_event = get_unhandled_input_event(
316                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
317                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
318                        NonPrintableKey::AltGraph,
319                    )))
320                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
321            );
322            let handler = handler.clone();
323            let result = handler.handle_unhandled_input_event(input_event.clone()).await;
324            let expected = InputEvent::from(get_unhandled_input_event(
325                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
326                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
327                        NonPrintableKey::AltGraph,
328                    )))
329                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
330            ));
331            assert_eq!(vec![expected], result);
332        }
333    }
334
335    // CapsLock  """"""\______/""""""""""\_______/"""
336    // Modifiers ------CCCCCCCC----------CCCCCCCCC---
337    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC---
338    //
339    // C == CapsLock
340    #[fasync::run_singlethreaded(test)]
341    async fn test_modifier_press_lock_release() {
342        let input_events = vec![
343            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
344            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
345            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
346            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
347        ];
348
349        let inspector = fuchsia_inspect::Inspector::default();
350        let test_node = inspector.root().create_child("test_node");
351        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
352        let clone_handler = move || handler.clone();
353        let result = futures::future::join_all(
354            input_events
355                .into_iter()
356                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
357        )
358        .await
359        .into_iter()
360        .flatten()
361        .collect::<Vec<InputEvent>>();
362
363        let expected = IntoIterator::into_iter([
364            get_unhandled_input_event(
365                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
366                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
367                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
368            ),
369            get_unhandled_input_event(
370                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
371                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
372                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
373            ),
374            get_unhandled_input_event(
375                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
376                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
377                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
378            ),
379            get_unhandled_input_event(
380                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
381                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
382                    .into_with_lock_state(Some(LockState::from_bits_allow_unknown(0))),
383            ),
384        ])
385        .map(InputEvent::from)
386        .collect::<Vec<_>>();
387
388        assert_eq!(expected, result);
389    }
390
391    // CapsLock  """"""\______/"""""""""""""""""""
392    // A         """""""""""""""""""\________/""""
393    // Modifiers ------CCCCCCCC-------------------
394    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC
395    //
396    // C == CapsLock
397    #[fasync::run_singlethreaded(test)]
398    async fn repeated_modifier_key() {
399        let input_events = vec![
400            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
401            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
402            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Pressed)),
403            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Released)),
404        ];
405
406        let inspector = fuchsia_inspect::Inspector::default();
407        let test_node = inspector.root().create_child("test_node");
408        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
409        let clone_handler = move || handler.clone();
410        let result = futures::future::join_all(
411            input_events
412                .into_iter()
413                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
414        )
415        .await
416        .into_iter()
417        .flatten()
418        .collect::<Vec<InputEvent>>();
419
420        let expected = IntoIterator::into_iter([
421            get_unhandled_input_event(
422                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
423                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
424                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
425            ),
426            get_unhandled_input_event(
427                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
428                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
429                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
430            ),
431            get_unhandled_input_event(
432                KeyboardEvent::new(Key::A, KeyEventType::Pressed)
433                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
434                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
435            ),
436            get_unhandled_input_event(
437                KeyboardEvent::new(Key::A, KeyEventType::Released)
438                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
439                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
440            ),
441        ])
442        .map(InputEvent::from)
443        .collect::<Vec<_>>();
444        assert_eq!(expected, result);
445    }
446
447    #[fuchsia::test]
448    async fn modifier_handlers_initialized_with_inspect_node() {
449        let inspector = fuchsia_inspect::Inspector::default();
450        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
451        let _modifier_handler =
452            ModifierHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
453        let _modifier_meaning_handler =
454            ModifierMeaningHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
455        diagnostics_assertions::assert_data_tree!(inspector, root: {
456            input_handlers_node: {
457                modifier_handler: {
458                    events_received_count: 0u64,
459                    events_handled_count: 0u64,
460                    last_received_timestamp_ns: 0u64,
461                    "fuchsia.inspect.Health": {
462                        status: "STARTING_UP",
463                        // Timestamp value is unpredictable and not relevant in this context,
464                        // so we only assert that the property is present.
465                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
466                    },
467                },
468                modifier_meaning_handler: {
469                    events_received_count: 0u64,
470                    events_handled_count: 0u64,
471                    last_received_timestamp_ns: 0u64,
472                    "fuchsia.inspect.Health": {
473                        status: "STARTING_UP",
474                        // Timestamp value is unpredictable and not relevant in this context,
475                        // so we only assert that the property is present.
476                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
477                    },
478                }
479            }
480        });
481    }
482
483    #[fasync::run_singlethreaded(test)]
484    async fn modifier_handler_inspect_counts_events() {
485        let inspector = fuchsia_inspect::Inspector::default();
486        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
487        let modifier_handler =
488            ModifierHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
489        let modifier_meaning_handler =
490            ModifierMeaningHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
491        let device_descriptor =
492            InputDeviceDescriptor::Keyboard(keyboard_binding::KeyboardDeviceDescriptor {
493                keys: vec![Key::A, Key::B, Key::RightAlt],
494                ..Default::default()
495            });
496        let (_, event_time_u64) = testing_utilities::event_times();
497        let input_events = vec![
498            testing_utilities::create_keyboard_event_with_time(
499                Key::A,
500                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
501                None,
502                event_time_u64,
503                &device_descriptor,
504                /* keymap= */ None,
505            ),
506            // Should not count received events that have already been handled.
507            testing_utilities::create_keyboard_event_with_handled(
508                Key::B,
509                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
510                None,
511                event_time_u64,
512                &device_descriptor,
513                /* keymap= */ None,
514                /* key_meaning= */ None,
515                Handled::Yes,
516            ),
517            testing_utilities::create_keyboard_event_with_time(
518                Key::A,
519                fidl_fuchsia_ui_input3::KeyEventType::Released,
520                None,
521                event_time_u64,
522                &device_descriptor,
523                /* keymap= */ None,
524            ),
525            // Should not count non-keyboard input events.
526            testing_utilities::create_fake_input_event(event_time_u64),
527            // Only event that should be counted by ModifierMeaningHandler.
528            testing_utilities::create_keyboard_event_with_key_meaning(
529                Key::RightAlt,
530                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
531                None,
532                event_time_u64,
533                &device_descriptor,
534                /* keymap= */ None,
535                /* key_meaning= */
536                Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)),
537            ),
538        ];
539
540        for input_event in input_events {
541            let _ = modifier_handler.clone().handle_input_event(input_event.clone()).await;
542            let _ = modifier_meaning_handler.clone().handle_input_event(input_event).await;
543        }
544
545        let last_event_timestamp: u64 = event_time_u64.into_nanos().try_into().unwrap();
546
547        diagnostics_assertions::assert_data_tree!(inspector, root: {
548            input_handlers_node: {
549                modifier_handler: {
550                    events_received_count: 3u64,
551                    events_handled_count: 0u64,
552                    last_received_timestamp_ns: last_event_timestamp,
553                    "fuchsia.inspect.Health": {
554                        status: "STARTING_UP",
555                        // Timestamp value is unpredictable and not relevant in this context,
556                        // so we only assert that the property is present.
557                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
558                    },
559                },
560                modifier_meaning_handler: {
561                    events_received_count: 1u64,
562                    events_handled_count: 0u64,
563                    last_received_timestamp_ns: last_event_timestamp,
564                    "fuchsia.inspect.Health": {
565                        status: "STARTING_UP",
566                        // Timestamp value is unpredictable and not relevant in this context,
567                        // so we only assert that the property is present.
568                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
569                    },
570                }
571            }
572        });
573    }
574}
input_pipeline_dso/modifier_handler.rs

input_pipeline_dso/
modifier_handler.rs
