encode_unicode/
utf8_iterators.rs

1/* Copyright 2018-2020 Torbjørn Birch Moltu
2 *
3 * Licensed under the Apache License, Version 2.0, <LICENSE-APACHE or
4 * http://apache.org/licenses/LICENSE-2.0> or the MIT license <LICENSE-MIT or
5 * http://opensource.org/licenses/MIT>, at your option. This file may not be
6 * copied, modified, or distributed except according to those terms.
7 */
8
9use crate::utf8_char::Utf8Char;
10use crate::errors::EmptyStrError;
11extern crate core;
12use core::{u32, u64};
13use core::ops::Not;
14use core::fmt;
15use core::borrow::Borrow;
16#[cfg(feature="std")]
17use std::io::{Read, Error as ioError};
18
19
20
21/// Read or iterate over the bytes of the UTF-8 representation of a codepoint.
22#[derive(Clone)]
23pub struct Utf8Iterator (u32);
24
25impl From<Utf8Char> for Utf8Iterator {
26    fn from(uc: Utf8Char) -> Self {
27        let used = u32::from_le_bytes(uc.to_array().0);
28        // uses u64 because shifting an u32 by 32 bits is a no-op.
29        let unused_set = (u64::MAX  <<  (uc.len() as u64*8)) as u32;
30        Utf8Iterator(used | unused_set)
31    }
32}
33impl From<char> for Utf8Iterator {
34    fn from(c: char) -> Self {
35        Self::from(Utf8Char::from(c))
36    }
37}
38impl Iterator for Utf8Iterator {
39    type Item=u8;
40    fn next(&mut self) -> Option<u8> {
41        let next = self.0 as u8;
42        if next == 0xff {
43            None
44        } else {
45            self.0 = (self.0 >> 8)  |  0xff_00_00_00;
46            Some(next)
47        }
48    }
49    fn size_hint(&self) -> (usize, Option<usize>) {
50        (self.len(),  Some(self.len()))
51    }
52}
53impl ExactSizeIterator for Utf8Iterator {
54    fn len(&self) -> usize {// not straightforward, but possible
55        let unused_bytes = self.0.not().leading_zeros() / 8;
56        4 - unused_bytes as usize
57    }
58}
59#[cfg(feature="std")]
60impl Read for Utf8Iterator {
61    /// Always returns Ok
62    fn read(&mut self,  buf: &mut[u8]) -> Result<usize, ioError> {
63        // Cannot call self.next() until I know I can write the result.
64        for (i, dst) in buf.iter_mut().enumerate() {
65            match self.next() {
66                Some(b) => *dst = b,
67                None    => return Ok(i),
68            }
69        }
70        Ok(buf.len())
71    }
72}
73impl fmt::Debug for Utf8Iterator {
74    fn fmt(&self,  fmtr: &mut fmt::Formatter) -> fmt::Result {
75        let mut content = [0; 4];
76        let mut i = 0;
77        for b in self.clone() {
78            content[i] = b;
79            i += 1;
80        }
81        write!(fmtr, "{:?}", &content[..i])
82    }
83}
84
85
86
87/// Converts an iterator of `Utf8Char` (or `&Utf8Char`)
88/// to an iterator of `u8`s.
89///
90/// Is equivalent to calling `.flatten()` or `.flat_map()` on the original iterator,
91/// but the returned iterator is ~40% faster.
92///
93/// The iterator also implements `Read` (if the `std` feature isn't disabled).
94/// Reading will never produce an error, and calls to `.read()` and `.next()`
95/// can be mixed.
96///
97/// The exact number of bytes cannot be known in advance, but `size_hint()`
98/// gives the possible range.
99/// (min: all remaining characters are ASCII, max: all require four bytes)
100///
101/// # Examples
102///
103/// From iterator of values:
104///
105/// ```
106/// use encode_unicode::{IterExt, CharExt};
107///
108/// let iterator = "foo".chars().map(|c| c.to_utf8() );
109/// let mut bytes = [0; 4];
110/// iterator.to_bytes().zip(&mut bytes).for_each(|(b,dst)| *dst = b );
111/// assert_eq!(&bytes, b"foo\0");
112/// ```
113///
114/// From iterator of references:
115///
116#[cfg_attr(feature="std", doc=" ```")]
117#[cfg_attr(not(feature="std"), doc=" ```no_compile")]
118/// use encode_unicode::{IterExt, CharExt, Utf8Char};
119///
120/// let chars: Vec<Utf8Char> = "💣 bomb 💣".chars().map(|c| c.to_utf8() ).collect();
121/// let bytes: Vec<u8> = chars.iter().to_bytes().collect();
122/// let flat_map: Vec<u8> = chars.iter().cloned().flatten().collect();
123/// assert_eq!(bytes, flat_map);
124/// ```
125///
126/// `Read`ing from it:
127///
128#[cfg_attr(feature="std", doc=" ```")]
129#[cfg_attr(not(feature="std"), doc=" ```no_compile")]
130/// use encode_unicode::{IterExt, CharExt};
131/// use std::io::Read;
132///
133/// let s = "Ååh‽";
134/// assert_eq!(s.len(), 8);
135/// let mut buf = [b'E'; 9];
136/// let mut reader = s.chars().map(|c| c.to_utf8() ).to_bytes();
137/// assert_eq!(reader.read(&mut buf[..]).unwrap(), 8);
138/// assert_eq!(reader.read(&mut buf[..]).unwrap(), 0);
139/// assert_eq!(&buf[..8], s.as_bytes());
140/// assert_eq!(buf[8], b'E');
141/// ```
142#[derive(Clone)]
143pub struct Utf8CharSplitter<U:Borrow<Utf8Char>, I:Iterator<Item=U>> {
144    inner: I,
145    prev: u32,
146}
147impl<U:Borrow<Utf8Char>, I:IntoIterator<Item=U>>
148From<I> for Utf8CharSplitter<U,I::IntoIter> {
149    fn from(iterable: I) -> Self {
150        Utf8CharSplitter { inner: iterable.into_iter(),  prev: 0 }
151    }
152}
153impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Utf8CharSplitter<U,I> {
154    /// Extracts the source iterator.
155    ///
156    /// Note that `iter.into_inner().to_bytes()` is not a no-op:  
157    /// If the last returned byte from `next()` was not an ASCII character,
158    /// the remaining bytes of that codepoint is lost.
159    pub fn into_inner(self) -> I {
160        self.inner
161    }
162}
163impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Iterator for Utf8CharSplitter<U,I> {
164    type Item = u8;
165    fn next(&mut self) -> Option<Self::Item> {
166        if self.prev == 0 {
167            self.inner.next().map(|u8c| {
168                let array = u8c.borrow().to_array().0;
169                self.prev = u32::from_le_bytes(array) >> 8;
170                array[0]
171            })
172        } else {
173            let next = self.prev as u8;
174            self.prev >>= 8;
175            Some(next)
176        }
177    }
178    fn size_hint(&self) -> (usize,Option<usize>) {
179        // Doesn't need to handle unlikely overflows correctly because
180        // size_hint() cannot be relied upon anyway. (the trait isn't unsafe)
181        let (min, max) = self.inner.size_hint();
182        let add = 4 - (self.prev.leading_zeros() / 8) as usize;
183        (min.wrapping_add(add), max.map(|max| max.wrapping_mul(4).wrapping_add(add) ))
184    }
185}
186#[cfg(feature="std")]
187impl<U:Borrow<Utf8Char>, I:Iterator<Item=U>> Read for Utf8CharSplitter<U,I> {
188    /// Always returns `Ok`
189    fn read(&mut self,  buf: &mut[u8]) -> Result<usize, ioError> {
190        let mut i = 0;
191        // write remaining bytes of previous codepoint
192        while self.prev != 0  &&  i < buf.len() {
193            buf[i] = self.prev as u8;
194            self.prev >>= 8;
195            i += 1;
196        }
197        // write whole characters
198        while i < buf.len() {
199            let bytes = match self.inner.next() {
200                Some(u8c) => u8c.borrow().to_array().0,
201                None => break
202            };
203            buf[i] = bytes[0];
204            i += 1;
205            if bytes[1] != 0 {
206                let len = bytes[0].not().leading_zeros() as usize;
207                let mut written = 1;
208                while written < len {
209                    if i < buf.len() {
210                        buf[i] = bytes[written];
211                        i += 1;
212                        written += 1;
213                    } else {
214                        let bytes_as_u32 = u32::from_le_bytes(bytes);
215                        self.prev = bytes_as_u32 >> (8*written);
216                        return Ok(i);
217                    }
218                }
219            }
220        }
221        Ok(i)
222    }
223}
224
225
226
227/// An iterator over the `Utf8Char` of a string slice, and their positions.
228///
229/// This struct is created by the `utf8char_indices()` method from [`StrExt`](../trait.StrExt.html)
230/// trait. See its documentation for more.
231#[derive(Clone)]
232pub struct Utf8CharIndices<'a>{
233    str: &'a str,
234    index: usize,
235}
236impl<'a> From<&'a str> for Utf8CharIndices<'a> {
237    fn from(s: &str) -> Utf8CharIndices {
238        Utf8CharIndices{str: s, index: 0}
239    }
240}
241impl<'a> Utf8CharIndices<'a> {
242    /// Extract the remainder of the source `str`.
243    ///
244    /// # Examples
245    ///
246    /// ```
247    /// use encode_unicode::{StrExt, Utf8Char};
248    /// let mut iter = "abc".utf8char_indices();
249    /// assert_eq!(iter.next_back(), Some((2, Utf8Char::from('c'))));
250    /// assert_eq!(iter.next(), Some((0, Utf8Char::from('a'))));
251    /// assert_eq!(iter.as_str(), "b");
252    /// ```
253    pub fn as_str(&self) -> &'a str {
254        &self.str[self.index..]
255    }
256}
257impl<'a> Iterator for Utf8CharIndices<'a> {
258    type Item = (usize,Utf8Char);
259    fn next(&mut self) -> Option<(usize,Utf8Char)> {
260        match Utf8Char::from_str_start(&self.str[self.index..]) {
261            Ok((u8c, len)) => {
262                let item = (self.index, u8c);
263                self.index += len;
264                Some(item)
265            },
266            Err(EmptyStrError) => None
267        }
268    }
269    fn size_hint(&self) -> (usize,Option<usize>) {
270        let len = self.str.len() - self.index;
271        // For len+3 to overflow, the slice must fill all but two bytes of
272        // addressable memory, and size_hint() doesn't need to be correct.
273        (len.wrapping_add(3)/4, Some(len))
274    }
275}
276impl<'a> DoubleEndedIterator for Utf8CharIndices<'a> {
277    fn next_back(&mut self) -> Option<(usize,Utf8Char)> {
278        // Cannot refactor out the unwrap without switching to ::from_slice()
279        // since slicing the str panics if not on a boundary.
280        if self.index < self.str.len() {
281            let rev = self.str.bytes().rev();
282            let len = 1 + rev.take_while(|b| b & 0b1100_0000 == 0b1000_0000 ).count();
283            let starts = self.str.len() - len;
284            let (u8c,_) = Utf8Char::from_str_start(&self.str[starts..]).unwrap();
285            self.str = &self.str[..starts];
286            Some((starts, u8c))
287        } else {
288            None
289        }
290    }
291}
292impl<'a> fmt::Debug for Utf8CharIndices<'a> {
293    fn fmt(&self,  fmtr: &mut fmt::Formatter) -> fmt::Result {
294        fmtr.debug_tuple("Utf8CharIndices")
295            .field(&self.index)
296            .field(&self.as_str())
297            .finish()
298    }
299}
300
301
302/// An iterator over the codepoints in a `str` represented as `Utf8Char`.
303#[derive(Clone)]
304pub struct Utf8Chars<'a>(Utf8CharIndices<'a>);
305impl<'a> From<&'a str> for Utf8Chars<'a> {
306    fn from(s: &str) -> Utf8Chars {
307        Utf8Chars(Utf8CharIndices::from(s))
308    }
309}
310impl<'a> Utf8Chars<'a> {
311    /// Extract the remainder of the source `str`.
312    ///
313    /// # Examples
314    ///
315    /// ```
316    /// use encode_unicode::{StrExt, Utf8Char};
317    /// let mut iter = "abc".utf8chars();
318    /// assert_eq!(iter.next(), Some(Utf8Char::from('a')));
319    /// assert_eq!(iter.next_back(), Some(Utf8Char::from('c')));
320    /// assert_eq!(iter.as_str(), "b");
321    /// ```
322    pub fn as_str(&self) -> &'a str {
323        self.0.as_str()
324    }
325}
326impl<'a> Iterator for Utf8Chars<'a> {
327    type Item = Utf8Char;
328    fn next(&mut self) -> Option<Utf8Char> {
329        self.0.next().map(|(_,u8c)| u8c )
330    }
331    fn size_hint(&self) -> (usize,Option<usize>) {
332        self.0.size_hint()
333    }
334}
335impl<'a> DoubleEndedIterator for Utf8Chars<'a> {
336    fn next_back(&mut self) -> Option<Utf8Char> {
337        self.0.next_back().map(|(_,u8c)| u8c )
338    }
339}
340impl<'a> fmt::Debug for Utf8Chars<'a> {
341    fn fmt(&self,  fmtr: &mut fmt::Formatter) -> fmt::Result {
342        fmtr.debug_tuple("Utf8CharIndices")
343            .field(&self.as_str())
344            .finish()
345    }
346}
encode_unicode/utf8_iterators.rs

encode_unicode/
utf8_iterators.rs
