input_pipeline_dso/
modifier_handler.rs

1// Copyright 2021 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5use crate::input_device::{
6    Handled, InputDeviceEvent, InputEvent, InputEventType, UnhandledInputEvent,
7};
8use crate::input_handler::{Handler, InputHandlerStatus, UnhandledInputHandler};
9use crate::metrics;
10use async_trait::async_trait;
11use fidl_fuchsia_ui_input3::{KeyMeaning, Modifiers, NonPrintableKey};
12use fuchsia_inspect::health::Reporter;
13use keymaps::{LockStateKeys, ModifierState};
14use metrics_registry::InputPipelineErrorMetricDimensionEvent;
15use std::cell::RefCell;
16use std::rc::Rc;
17
18/// Tracks modifier state and decorates passing events with the modifiers.
19///
20/// This handler should be installed as early as possible in the input pipeline,
21/// to ensure that all later stages have the modifiers and lock states available.
22/// This holds even for non-keyboard handlers, to allow handling `Ctrl+Click`
23/// events, for example.
24///
25/// One possible exception to this rule would be a hardware key rewriting handler for
26/// limited keyboards.
27#[derive(Debug)]
28pub struct ModifierHandler {
29    /// The tracked state of the modifiers.
30    modifier_state: RefCell<ModifierState>,
31
32    /// The tracked lock state.
33    lock_state: RefCell<LockStateKeys>,
34
35    /// The metrics logger.
36    metrics_logger: metrics::MetricsLogger,
37
38    /// The inventory of this handler's Inspect status.
39    pub inspect_status: InputHandlerStatus,
40}
41
42impl Handler for ModifierHandler {
43    fn set_handler_healthy(self: std::rc::Rc<Self>) {
44        self.inspect_status.health_node.borrow_mut().set_ok();
45    }
46
47    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
48        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
49    }
50
51    fn get_name(&self) -> &'static str {
52        "ModifierHandler"
53    }
54
55    fn interest(&self) -> Vec<InputEventType> {
56        vec![InputEventType::Keyboard]
57    }
58}
59
60#[async_trait(?Send)]
61impl UnhandledInputHandler for ModifierHandler {
62    async fn handle_unhandled_input_event(
63        self: Rc<Self>,
64        unhandled_input_event: UnhandledInputEvent,
65    ) -> Vec<InputEvent> {
66        fuchsia_trace::duration!("input", "modifier_handler");
67        match unhandled_input_event {
68            UnhandledInputEvent {
69                device_event: InputDeviceEvent::Keyboard(mut event),
70                device_descriptor,
71                event_time,
72                trace_id,
73            } => {
74                fuchsia_trace::duration!("input", "modifier_handler[processing]");
75                if let Some(trace_id) = trace_id {
76                    fuchsia_trace::flow_step!(
77                        c"input",
78                        c"event_in_input_pipeline",
79                        trace_id.into()
80                    );
81                }
82
83                self.inspect_status.count_received_event(&event_time);
84                self.modifier_state.borrow_mut().update(event.get_event_type(), event.get_key());
85                self.lock_state.borrow_mut().update(event.get_event_type(), event.get_key());
86                event = event
87                    .into_with_lock_state(Some(self.lock_state.borrow().get_state()))
88                    .into_with_modifiers(Some(self.modifier_state.borrow().get_state()));
89                log::debug!("modifiers and lock state applied: {:?}", event);
90                vec![InputEvent {
91                    device_event: InputDeviceEvent::Keyboard(event),
92                    device_descriptor,
93                    event_time,
94                    handled: Handled::No,
95                    trace_id,
96                }]
97            }
98            // Pass other events through.
99            _ => {
100                if InputEventType::from(&unhandled_input_event.device_event)
101                    != InputEventType::Keyboard
102                {
103                    self.metrics_logger.log_error(
104                        InputPipelineErrorMetricDimensionEvent::HandlerReceivedUninterestedEvent,
105                        std::format!(
106                            "{} uninterested input event: {:?}",
107                            self.get_name(),
108                            unhandled_input_event.get_event_type()
109                        ),
110                    );
111                }
112                vec![InputEvent::from(unhandled_input_event)]
113            }
114        }
115    }
116}
117
118impl ModifierHandler {
119    pub fn new(
120        input_handlers_node: &fuchsia_inspect::Node,
121        metrics_logger: metrics::MetricsLogger,
122    ) -> Rc<Self> {
123        let inspect_status = InputHandlerStatus::new(
124            input_handlers_node,
125            "modifier_handler",
126            /* generates_events */ false,
127        );
128        Rc::new(Self {
129            modifier_state: RefCell::new(ModifierState::new()),
130            lock_state: RefCell::new(LockStateKeys::new()),
131            metrics_logger,
132            inspect_status,
133        })
134    }
135}
136
137/// Tracks the state of the modifiers that are tied to the key meaning (as opposed to hardware
138/// keys).
139#[derive(Debug)]
140pub struct ModifierMeaningHandler {
141    /// The tracked state of the modifiers.
142    modifier_state: RefCell<ModifierState>,
143
144    /// The metrics logger.
145    metrics_logger: metrics::MetricsLogger,
146
147    /// The inventory of this handler's Inspect status.
148    pub inspect_status: InputHandlerStatus,
149}
150
151impl ModifierMeaningHandler {
152    pub fn new(
153        input_handlers_node: &fuchsia_inspect::Node,
154        metrics_logger: metrics::MetricsLogger,
155    ) -> Rc<Self> {
156        let inspect_status = InputHandlerStatus::new(
157            input_handlers_node,
158            "modifier_meaning_handler",
159            /* generates_events */ false,
160        );
161        Rc::new(Self {
162            modifier_state: RefCell::new(ModifierState::new()),
163            metrics_logger,
164            inspect_status,
165        })
166    }
167}
168
169impl Handler for ModifierMeaningHandler {
170    fn set_handler_healthy(self: std::rc::Rc<Self>) {
171        self.inspect_status.health_node.borrow_mut().set_ok();
172    }
173
174    fn set_handler_unhealthy(self: std::rc::Rc<Self>, msg: &str) {
175        self.inspect_status.health_node.borrow_mut().set_unhealthy(msg);
176    }
177
178    fn get_name(&self) -> &'static str {
179        "ModifierHandler"
180    }
181
182    fn interest(&self) -> Vec<InputEventType> {
183        vec![InputEventType::Keyboard]
184    }
185}
186
187#[async_trait(?Send)]
188impl UnhandledInputHandler for ModifierMeaningHandler {
189    async fn handle_unhandled_input_event(
190        self: Rc<Self>,
191        unhandled_input_event: UnhandledInputEvent,
192    ) -> Vec<InputEvent> {
193        fuchsia_trace::duration!("input", "modifier_meaning_handler");
194        match unhandled_input_event {
195            UnhandledInputEvent {
196                device_event: InputDeviceEvent::Keyboard(mut event),
197                device_descriptor,
198                event_time,
199                trace_id,
200            } if event.get_key_meaning()
201                == Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)) =>
202            {
203                fuchsia_trace::duration!("input", "modifier_meaning_handler[processing]");
204                if let Some(trace_id) = trace_id {
205                    fuchsia_trace::flow_step!(
206                        c"input",
207                        c"event_in_input_pipeline",
208                        trace_id.into()
209                    );
210                }
211                self.inspect_status.count_received_event(&event_time);
212                // The "obvious" rewrite of this if and the match guard above is
213                // unstable, so doing it this way.
214                if let Some(key_meaning) = event.get_key_meaning() {
215                    self.modifier_state
216                        .borrow_mut()
217                        .update_with_key_meaning(event.get_event_type(), key_meaning);
218                    let new_modifier = event.get_modifiers().unwrap_or(Modifiers::empty())
219                        | self.modifier_state.borrow().get_state();
220                    event = event.into_with_modifiers(Some(new_modifier));
221                    log::debug!("additinal modifiers and lock state applied: {:?}", event);
222                }
223                vec![InputEvent {
224                    device_event: InputDeviceEvent::Keyboard(event),
225                    device_descriptor,
226                    event_time,
227                    handled: Handled::No,
228                    trace_id,
229                }]
230            }
231            // Pass other events through.
232            _ => {
233                if InputEventType::from(&unhandled_input_event.device_event)
234                    != InputEventType::Keyboard
235                {
236                    self.metrics_logger.log_error(
237                        InputPipelineErrorMetricDimensionEvent::HandlerReceivedUninterestedEvent,
238                        std::format!(
239                            "{} uninterested input event: {:?}",
240                            self.get_name(),
241                            unhandled_input_event.get_event_type()
242                        ),
243                    );
244                }
245                vec![InputEvent::from(unhandled_input_event)]
246            }
247        }
248    }
249}
250
251#[cfg(test)]
252mod tests {
253    use super::*;
254    use crate::input_device::InputDeviceDescriptor;
255    use crate::input_handler::InputHandler;
256    use crate::keyboard_binding::{self, KeyboardEvent};
257    use crate::testing_utilities;
258    use fidl_fuchsia_input::Key;
259    use fidl_fuchsia_ui_input3::{KeyEventType, LockState};
260    use pretty_assertions::assert_eq;
261
262    fn get_unhandled_input_event(event: KeyboardEvent) -> UnhandledInputEvent {
263        UnhandledInputEvent {
264            device_event: InputDeviceEvent::Keyboard(event),
265            event_time: zx::MonotonicInstant::from_nanos(42),
266            device_descriptor: InputDeviceDescriptor::Fake,
267            trace_id: None,
268        }
269    }
270
271    #[fuchsia::test]
272    async fn test_decoration() {
273        let inspector = fuchsia_inspect::Inspector::default();
274        let test_node = inspector.root().create_child("test_node");
275        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
276        let input_event =
277            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed));
278        let result = handler.handle_unhandled_input_event(input_event.clone()).await;
279
280        // This handler decorates, but does not handle the key. Hence,
281        // the key remains unhandled.
282        let expected = InputEvent::from(get_unhandled_input_event(
283            KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
284                .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
285                .into_with_lock_state(Some(LockState::CAPS_LOCK)),
286        ));
287        assert_eq!(vec![expected], result);
288    }
289
290    #[fuchsia::test]
291    async fn test_key_meaning_decoration() {
292        let inspector = fuchsia_inspect::Inspector::default();
293        let test_node = inspector.root().create_child("test_node");
294        let handler = ModifierMeaningHandler::new(&test_node, metrics::MetricsLogger::default());
295        {
296            let input_event = get_unhandled_input_event(
297                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
298                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
299                        NonPrintableKey::AltGraph,
300                    )))
301                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
302            );
303            let result = handler.clone().handle_unhandled_input_event(input_event.clone()).await;
304            let expected = InputEvent::from(get_unhandled_input_event(
305                KeyboardEvent::new(Key::RightAlt, KeyEventType::Pressed)
306                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
307                        NonPrintableKey::AltGraph,
308                    )))
309                    .into_with_modifiers(Some(Modifiers::ALT_GRAPH | Modifiers::CAPS_LOCK)),
310            ));
311            assert_eq!(vec![expected], result);
312        }
313        {
314            let input_event = get_unhandled_input_event(
315                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
316                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
317                        NonPrintableKey::AltGraph,
318                    )))
319                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
320            );
321            let handler = handler.clone();
322            let result = handler.handle_unhandled_input_event(input_event.clone()).await;
323            let expected = InputEvent::from(get_unhandled_input_event(
324                KeyboardEvent::new(Key::RightAlt, KeyEventType::Released)
325                    .into_with_key_meaning(Some(KeyMeaning::NonPrintableKey(
326                        NonPrintableKey::AltGraph,
327                    )))
328                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK)),
329            ));
330            assert_eq!(vec![expected], result);
331        }
332    }
333
334    // CapsLock  """"""\______/""""""""""\_______/"""
335    // Modifiers ------CCCCCCCC----------CCCCCCCCC---
336    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC---
337    //
338    // C == CapsLock
339    #[fuchsia::test]
340    async fn test_modifier_press_lock_release() {
341        let input_events = vec![
342            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
343            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
344            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
345            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
346        ];
347
348        let inspector = fuchsia_inspect::Inspector::default();
349        let test_node = inspector.root().create_child("test_node");
350        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
351        let clone_handler = move || handler.clone();
352        let result = futures::future::join_all(
353            input_events
354                .into_iter()
355                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
356        )
357        .await
358        .into_iter()
359        .flatten()
360        .collect::<Vec<InputEvent>>();
361
362        let expected = IntoIterator::into_iter([
363            get_unhandled_input_event(
364                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
365                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
366                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
367            ),
368            get_unhandled_input_event(
369                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
370                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
371                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
372            ),
373            get_unhandled_input_event(
374                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
375                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
376                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
377            ),
378            get_unhandled_input_event(
379                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
380                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
381                    .into_with_lock_state(Some(LockState::from_bits_allow_unknown(0))),
382            ),
383        ])
384        .map(InputEvent::from)
385        .collect::<Vec<_>>();
386
387        assert_eq!(expected, result);
388    }
389
390    // CapsLock  """"""\______/"""""""""""""""""""
391    // A         """""""""""""""""""\________/""""
392    // Modifiers ------CCCCCCCC-------------------
393    // LockState ------CCCCCCCCCCCCCCCCCCCCCCCCCCC
394    //
395    // C == CapsLock
396    #[fuchsia::test]
397    async fn repeated_modifier_key() {
398        let input_events = vec![
399            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)),
400            get_unhandled_input_event(KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)),
401            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Pressed)),
402            get_unhandled_input_event(KeyboardEvent::new(Key::A, KeyEventType::Released)),
403        ];
404
405        let inspector = fuchsia_inspect::Inspector::default();
406        let test_node = inspector.root().create_child("test_node");
407        let handler = ModifierHandler::new(&test_node, metrics::MetricsLogger::default());
408        let clone_handler = move || handler.clone();
409        let result = futures::future::join_all(
410            input_events
411                .into_iter()
412                .map(|e| async { clone_handler().handle_unhandled_input_event(e).await }),
413        )
414        .await
415        .into_iter()
416        .flatten()
417        .collect::<Vec<InputEvent>>();
418
419        let expected = IntoIterator::into_iter([
420            get_unhandled_input_event(
421                KeyboardEvent::new(Key::CapsLock, KeyEventType::Pressed)
422                    .into_with_modifiers(Some(Modifiers::CAPS_LOCK))
423                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
424            ),
425            get_unhandled_input_event(
426                KeyboardEvent::new(Key::CapsLock, KeyEventType::Released)
427                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
428                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
429            ),
430            get_unhandled_input_event(
431                KeyboardEvent::new(Key::A, KeyEventType::Pressed)
432                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
433                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
434            ),
435            get_unhandled_input_event(
436                KeyboardEvent::new(Key::A, KeyEventType::Released)
437                    .into_with_modifiers(Some(Modifiers::from_bits_allow_unknown(0)))
438                    .into_with_lock_state(Some(LockState::CAPS_LOCK)),
439            ),
440        ])
441        .map(InputEvent::from)
442        .collect::<Vec<_>>();
443        assert_eq!(expected, result);
444    }
445
446    #[fuchsia::test]
447    async fn modifier_handlers_initialized_with_inspect_node() {
448        let inspector = fuchsia_inspect::Inspector::default();
449        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
450        let _modifier_handler =
451            ModifierHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
452        let _modifier_meaning_handler =
453            ModifierMeaningHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
454        diagnostics_assertions::assert_data_tree!(inspector, root: {
455            input_handlers_node: {
456                modifier_handler: {
457                    events_received_count: 0u64,
458                    events_handled_count: 0u64,
459                    last_received_timestamp_ns: 0u64,
460                    "fuchsia.inspect.Health": {
461                        status: "STARTING_UP",
462                        // Timestamp value is unpredictable and not relevant in this context,
463                        // so we only assert that the property is present.
464                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
465                    },
466                },
467                modifier_meaning_handler: {
468                    events_received_count: 0u64,
469                    events_handled_count: 0u64,
470                    last_received_timestamp_ns: 0u64,
471                    "fuchsia.inspect.Health": {
472                        status: "STARTING_UP",
473                        // Timestamp value is unpredictable and not relevant in this context,
474                        // so we only assert that the property is present.
475                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
476                    },
477                }
478            }
479        });
480    }
481
482    #[fuchsia::test]
483    async fn modifier_handler_inspect_counts_events() {
484        let inspector = fuchsia_inspect::Inspector::default();
485        let fake_handlers_node = inspector.root().create_child("input_handlers_node");
486        let modifier_handler =
487            ModifierHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
488        let modifier_meaning_handler =
489            ModifierMeaningHandler::new(&fake_handlers_node, metrics::MetricsLogger::default());
490        let device_descriptor =
491            InputDeviceDescriptor::Keyboard(keyboard_binding::KeyboardDeviceDescriptor {
492                keys: vec![Key::A, Key::B, Key::RightAlt],
493                ..Default::default()
494            });
495        let (_, event_time_u64) = testing_utilities::event_times();
496        let input_events = vec![
497            testing_utilities::create_keyboard_event_with_time(
498                Key::A,
499                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
500                None,
501                event_time_u64,
502                &device_descriptor,
503                /* keymap= */ None,
504            ),
505            // Should not count received events that have already been handled.
506            testing_utilities::create_keyboard_event_with_handled(
507                Key::B,
508                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
509                None,
510                event_time_u64,
511                &device_descriptor,
512                /* keymap= */ None,
513                /* key_meaning= */ None,
514                Handled::Yes,
515            ),
516            testing_utilities::create_keyboard_event_with_time(
517                Key::A,
518                fidl_fuchsia_ui_input3::KeyEventType::Released,
519                None,
520                event_time_u64,
521                &device_descriptor,
522                /* keymap= */ None,
523            ),
524            // Should not count non-keyboard input events.
525            testing_utilities::create_fake_input_event(event_time_u64),
526            // Only event that should be counted by ModifierMeaningHandler.
527            testing_utilities::create_keyboard_event_with_key_meaning(
528                Key::RightAlt,
529                fidl_fuchsia_ui_input3::KeyEventType::Pressed,
530                None,
531                event_time_u64,
532                &device_descriptor,
533                /* keymap= */ None,
534                /* key_meaning= */
535                Some(KeyMeaning::NonPrintableKey(NonPrintableKey::AltGraph)),
536            ),
537        ];
538
539        for input_event in input_events {
540            let _ = modifier_handler.clone().handle_input_event(input_event.clone()).await;
541            let _ = modifier_meaning_handler.clone().handle_input_event(input_event).await;
542        }
543
544        let last_event_timestamp: u64 = event_time_u64.into_nanos().try_into().unwrap();
545
546        diagnostics_assertions::assert_data_tree!(inspector, root: {
547            input_handlers_node: {
548                modifier_handler: {
549                    events_received_count: 3u64,
550                    events_handled_count: 0u64,
551                    last_received_timestamp_ns: last_event_timestamp,
552                    "fuchsia.inspect.Health": {
553                        status: "STARTING_UP",
554                        // Timestamp value is unpredictable and not relevant in this context,
555                        // so we only assert that the property is present.
556                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
557                    },
558                },
559                modifier_meaning_handler: {
560                    events_received_count: 1u64,
561                    events_handled_count: 0u64,
562                    last_received_timestamp_ns: last_event_timestamp,
563                    "fuchsia.inspect.Health": {
564                        status: "STARTING_UP",
565                        // Timestamp value is unpredictable and not relevant in this context,
566                        // so we only assert that the property is present.
567                        start_timestamp_nanos: diagnostics_assertions::AnyProperty
568                    },
569                }
570            }
571        });
572    }
573}
input_pipeline_dso/modifier_handler.rs

input_pipeline_dso/
modifier_handler.rs
