fxfs_unicode/
lookup.rs

1// Copyright 2024 The Fuchsia Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5//! A set of functions for parsing generated unicode data (see generator.rs and unicode_gen.rs)
6use crate::unicode_gen::*;
7use std::cmp::Ordering;
8use types::MappingOffset;
9
10/// Lookup the CCC for a code point.
11/// This aims to minimise data set size whilst maintaining performance.
12/// Worst case is O(log2(2^16)) = O(16).
13pub fn ccc(ch: char) -> u8 {
14    // CCC data is encoded in 'planes' using LUT for top two bytes of u32 unicode char.
15    let plane_offset = CCC_PLANE_OFFSETS[ch as u32 as usize >> 16];
16    let plane_end = CCC_PLANE_OFFSETS[((ch as u32 >> 16) + 1) as usize];
17    let plane = &CCC_PLANE_DATA[plane_offset..plane_end];
18    let ch_lsb = ch as u32 as u16;
19    let ix = plane
20        .binary_search_by(|entry| match entry.low_plane_end_ix.cmp(&ch_lsb) {
21            Ordering::Equal => Ordering::Less,
22            x => x,
23        })
24        .unwrap_or_else(|x| x);
25    if ix < plane.len() {
26        let entry = &plane[ix];
27        if entry.range().contains(&ch_lsb) {
28            return entry.ccc;
29        }
30    }
31    0
32}
33
34/// Both decomposition and casefold use the same lookup mechanism
35fn string_table_lookup(
36    ch: char,
37    plane_offsets: &[usize],
38    plane_data: &[MappingOffset],
39    string_data: &'static [u8],
40) -> Option<&'static str> {
41    // Unicode has 17 planes (0..=16). we store one extra entry to calculate plane_end easily.
42    let plane_start = plane_offsets[(ch as u32 >> 16) as usize];
43    let plane_end = plane_offsets[((ch as u32 >> 16) + 1) as usize];
44    let plane = &plane_data[plane_start..plane_end];
45    // Look up the lower 16-bit in the plane using binary search.
46    let ch_lsb = ch as u32 as u16;
47    if let Ok(ix) = plane.binary_search_by(|entry| entry.low_plane_ix.cmp(&ch_lsb)) {
48        // Read the start of the string and look at the next entry to find the end of the string.
49        let offset = plane[ix].offset as usize;
50        let end = if plane.len() == ix + 1 {
51            if plane_end < plane_data.len() {
52                plane_data[plane_end].offset as usize
53            } else {
54                string_data.len()
55            }
56        } else {
57            plane[ix + 1].offset as usize
58        };
59        let out = &string_data[offset..end];
60        return Some(unsafe {
61            // Safety: This comes from official unicode decompositions.
62            std::str::from_utf8_unchecked(out)
63        });
64    }
65    return None;
66}
67
68/// Returns the unicode decomposition of a given codepoint.
69pub fn decomposition(ch: char) -> Option<&'static str> {
70    string_table_lookup(ch, DECOMP_PLANE_OFFSETS, DECOMP_PLANE_DATA, DECOMP_STRINGS)
71}
72
73/// Returns true if a codepoint is a default ignorable codepoint.
74pub fn default_ignorable(ch: char) -> bool {
75    // Unicode has 17 planes (0..=16). we store one extra entry to calculate plane_end easily.
76    let plane_offset = IGNORABLE_PLANE_OFFSETS[ch as u32 as usize >> 16];
77    let plane_end = IGNORABLE_PLANE_OFFSETS[((ch as u32 >> 16) + 1) as usize];
78    let plane = &IGNORABLE_PLANE_DATA[plane_offset..plane_end];
79    let ch_lsb = ch as u32 as u16;
80    let ix = plane
81        .binary_search_by(|entry| match entry.end.cmp(&ch_lsb) {
82            Ordering::Equal => Ordering::Less,
83            x => x,
84        })
85        .unwrap_or_else(|x| x);
86    if ix < plane.len() { plane[ix].contains(&ch_lsb) } else { false }
87}
88
89pub fn casefold(ch: char) -> Option<&'static str> {
90    string_table_lookup(ch, CASEFOLD_PLANE_OFFSETS, CASEFOLD_PLANE_DATA, CASEFOLD_STRINGS)
91}
92
93#[cfg(test)]
94mod test {
95    use super::*;
96
97    #[test]
98    fn test_ccc() {
99        assert_eq!(ccc('\u{0}'), 0);
100        assert_eq!(ccc('A'), 0);
101        assert_eq!(ccc('\u{059a}'), 222);
102        assert_eq!(ccc('\u{2cee}'), 0);
103        assert_eq!(ccc('\u{2cef}'), 230);
104        assert_eq!(ccc('\u{2cf0}'), 230);
105        assert_eq!(ccc('\u{2cf1}'), 230);
106        assert_eq!(ccc('\u{2cf2}'), 0);
107    }
108
109    #[test]
110    fn test_decomposition() {
111        assert_eq!(decomposition('\u{f900}'), Some("\u{8c48}"));
112        assert_eq!(decomposition('A'), None);
113        assert_eq!(decomposition('\u{1f82}'), Some("α\u{313}\u{300}\u{345}"));
114    }
115
116    #[test]
117    fn test_deafault_ignorable() {
118        assert!(!default_ignorable('\u{00ac}'));
119        assert!(default_ignorable('\u{00ad}'));
120        assert!(!default_ignorable('\u{00ae}'));
121        assert!(!default_ignorable('\u{115e}'));
122        assert!(default_ignorable('\u{115f}'));
123        assert!(default_ignorable('\u{1160}'));
124        assert!(!default_ignorable('\u{1161}'));
125    }
126
127    #[test]
128    fn test_casefold() {
129        assert_eq!(casefold('A'), Some("a"));
130        assert_eq!(casefold('a'), None);
131    }
132
133    #[test]
134    fn test_last_char() {
135        // We look up strings end by incrementing offset + 1 from string start so we need make sure
136        // we handle the case where offset is the last string in the plane or the last string
137        // overall.
138        assert_eq!(decomposition('\u{10000}'), None);
139        assert_eq!(decomposition('\u{1fbca}'), None);
140        assert_eq!(decomposition('\u{1fbf9}'), None);
141        assert_eq!(decomposition('\u{20000}'), None);
142        assert_eq!(decomposition('\u{2fa1d}'), Some("\u{2a600}"));
143        assert_eq!(casefold('\u{ff39}'), Some("\u{ff59}"));
144        assert_eq!(casefold('\u{ff3a}'), Some("\u{ff5a}"));
145        assert_eq!(casefold('\u{ff3b}'), None);
146        assert_eq!(casefold('\u{10400}'), Some("\u{10428}"));
147        assert_eq!(casefold('\u{1e920}'), Some("\u{1e942}"));
148        assert_eq!(casefold('\u{1e921}'), Some("\u{1e943}"));
149        assert_eq!(casefold('\u{1e922}'), None);
150    }
151
152    #[test]
153    fn test_high_char() {
154        // This is the maximum unicode codepoint.
155        assert_eq!(decomposition('\u{10FFFF}'), None);
156        assert_eq!(casefold('\u{10FFFF}'), None);
157        assert_eq!(default_ignorable('\u{10FFFF}'), false);
158    }
159}
fxfs_unicode/lookup.rs

fxfs_unicode/
lookup.rs
