unicode_segmentation/
lib.rs

1// Copyright 2012-2015 The Rust Project Developers. See the COPYRIGHT
2// file at the top-level directory of this distribution and at
3// http://rust-lang.org/COPYRIGHT.
4//
5// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8// option. This file may not be copied, modified, or distributed
9// except according to those terms.
10
11//! Iterators which split strings on Grapheme Cluster, Word or Sentence boundaries, according
12//! to the [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/) rules.
13//!
14//! ```rust
15//! extern crate unicode_segmentation;
16//!
17//! use unicode_segmentation::UnicodeSegmentation;
18//!
19//! fn main() {
20//!     let s = "a̐éö̲\r\n";
21//!     let g = UnicodeSegmentation::graphemes(s, true).collect::<Vec<&str>>();
22//!     let b: &[_] = &["a̐", "é", "ö̲", "\r\n"];
23//!     assert_eq!(g, b);
24//!
25//!     let s = "The quick (\"brown\") fox can't jump 32.3 feet, right?";
26//!     let w = s.unicode_words().collect::<Vec<&str>>();
27//!     let b: &[_] = &["The", "quick", "brown", "fox", "can't", "jump", "32.3", "feet", "right"];
28//!     assert_eq!(w, b);
29//!
30//!     let s = "The quick (\"brown\")  fox";
31//!     let w = s.split_word_bounds().collect::<Vec<&str>>();
32//!     let b: &[_] = &["The", " ", "quick", " ", "(", "\"", "brown", "\"", ")", "  ", "fox"];
33//!     assert_eq!(w, b);
34//! }
35//! ```
36//!
37//! # no_std
38//!
39//! unicode-segmentation does not depend on libstd, so it can be used in crates
40//! with the `#![no_std]` attribute.
41//!
42//! # crates.io
43//!
44//! You can use this package in your project by adding the following
45//! to your `Cargo.toml`:
46//!
47//! ```toml
48//! [dependencies]
49//! unicode-segmentation = "1.9.0"
50//! ```
51
52#![deny(missing_docs, unsafe_code)]
53#![doc(
54    html_logo_url = "https://unicode-rs.github.io/unicode-rs_sm.png",
55    html_favicon_url = "https://unicode-rs.github.io/unicode-rs_sm.png"
56)]
57#![no_std]
58
59#[cfg(test)]
60#[macro_use]
61extern crate std;
62
63#[cfg(test)]
64#[macro_use]
65extern crate quickcheck;
66
67pub use grapheme::{GraphemeCursor, GraphemeIncomplete};
68pub use grapheme::{GraphemeIndices, Graphemes};
69pub use sentence::{USentenceBoundIndices, USentenceBounds, UnicodeSentences};
70pub use tables::UNICODE_VERSION;
71pub use word::{UWordBoundIndices, UWordBounds, UnicodeWordIndices, UnicodeWords};
72
73mod grapheme;
74#[rustfmt::skip]
75mod tables;
76mod sentence;
77mod word;
78
79#[cfg(test)]
80mod test;
81#[cfg(test)]
82mod testdata;
83
84/// Methods for segmenting strings according to
85/// [Unicode Standard Annex #29](http://www.unicode.org/reports/tr29/).
86pub trait UnicodeSegmentation {
87    /// Returns an iterator over the [grapheme clusters][graphemes] of `self`.
88    ///
89    /// [graphemes]: http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries
90    ///
91    /// If `is_extended` is true, the iterator is over the
92    /// *extended grapheme clusters*;
93    /// otherwise, the iterator is over the *legacy grapheme clusters*.
94    /// [UAX#29](http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries)
95    /// recommends extended grapheme cluster boundaries for general processing.
96    ///
97    /// # Examples
98    ///
99    /// ```
100    /// # use self::unicode_segmentation::UnicodeSegmentation;
101    /// let gr1 = UnicodeSegmentation::graphemes("a\u{310}e\u{301}o\u{308}\u{332}", true)
102    ///           .collect::<Vec<&str>>();
103    /// let b: &[_] = &["a\u{310}", "e\u{301}", "o\u{308}\u{332}"];
104    ///
105    /// assert_eq!(&gr1[..], b);
106    ///
107    /// let gr2 = UnicodeSegmentation::graphemes("a\r\nb🇷🇺🇸🇹", true).collect::<Vec<&str>>();
108    /// let b: &[_] = &["a", "\r\n", "b", "🇷🇺", "🇸🇹"];
109    ///
110    /// assert_eq!(&gr2[..], b);
111    /// ```
112    fn graphemes<'a>(&'a self, is_extended: bool) -> Graphemes<'a>;
113
114    /// Returns an iterator over the grapheme clusters of `self` and their
115    /// byte offsets. See `graphemes()` for more information.
116    ///
117    /// # Examples
118    ///
119    /// ```
120    /// # use self::unicode_segmentation::UnicodeSegmentation;
121    /// let gr_inds = UnicodeSegmentation::grapheme_indices("a̐éö̲\r\n", true)
122    ///               .collect::<Vec<(usize, &str)>>();
123    /// let b: &[_] = &[(0, "a̐"), (3, "é"), (6, "ö̲"), (11, "\r\n")];
124    ///
125    /// assert_eq!(&gr_inds[..], b);
126    /// ```
127    fn grapheme_indices<'a>(&'a self, is_extended: bool) -> GraphemeIndices<'a>;
128
129    /// Returns an iterator over the words of `self`, separated on
130    /// [UAX#29 word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries).
131    ///
132    /// Here, "words" are just those substrings which, after splitting on
133    /// UAX#29 word boundaries, contain any alphanumeric characters. That is, the
134    /// substring must contain at least one character with the
135    /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic)
136    /// property, or with
137    /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values).
138    ///
139    /// # Example
140    ///
141    /// ```
142    /// # use self::unicode_segmentation::UnicodeSegmentation;
143    /// let uws = "The quick (\"brown\") fox can't jump 32.3 feet, right?";
144    /// let uw1 = uws.unicode_words().collect::<Vec<&str>>();
145    /// let b: &[_] = &["The", "quick", "brown", "fox", "can't", "jump", "32.3", "feet", "right"];
146    ///
147    /// assert_eq!(&uw1[..], b);
148    /// ```
149    fn unicode_words<'a>(&'a self) -> UnicodeWords<'a>;
150
151    /// Returns an iterator over the words of `self`, separated on
152    /// [UAX#29 word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries), and their
153    /// offsets.
154    ///
155    /// Here, "words" are just those substrings which, after splitting on
156    /// UAX#29 word boundaries, contain any alphanumeric characters. That is, the
157    /// substring must contain at least one character with the
158    /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic)
159    /// property, or with
160    /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values).
161    ///
162    /// # Example
163    ///
164    /// ```
165    /// # use self::unicode_segmentation::UnicodeSegmentation;
166    /// let uwis = "The quick (\"brown\") fox can't jump 32.3 feet, right?";
167    /// let uwi1 = uwis.unicode_word_indices().collect::<Vec<(usize, &str)>>();
168    /// let b: &[_] = &[(0, "The"), (4, "quick"), (12, "brown"), (20, "fox"), (24, "can't"),
169    ///                 (30, "jump"), (35, "32.3"), (40, "feet"), (46, "right")];
170    ///
171    /// assert_eq!(&uwi1[..], b);
172    /// ```
173    fn unicode_word_indices<'a>(&'a self) -> UnicodeWordIndices<'a>;
174
175    /// Returns an iterator over substrings of `self` separated on
176    /// [UAX#29 word boundaries](http://www.unicode.org/reports/tr29/#Word_Boundaries).
177    ///
178    /// The concatenation of the substrings returned by this function is just the original string.
179    ///
180    /// # Example
181    ///
182    /// ```
183    /// # use self::unicode_segmentation::UnicodeSegmentation;
184    /// let swu1 = "The quick (\"brown\")  fox".split_word_bounds().collect::<Vec<&str>>();
185    /// let b: &[_] = &["The", " ", "quick", " ", "(", "\"", "brown", "\"", ")", "  ", "fox"];
186    ///
187    /// assert_eq!(&swu1[..], b);
188    /// ```
189    fn split_word_bounds<'a>(&'a self) -> UWordBounds<'a>;
190
191    /// Returns an iterator over substrings of `self`, split on UAX#29 word boundaries,
192    /// and their offsets. See `split_word_bounds()` for more information.
193    ///
194    /// # Example
195    ///
196    /// ```
197    /// # use self::unicode_segmentation::UnicodeSegmentation;
198    /// let swi1 = "Brr, it's 29.3°F!".split_word_bound_indices().collect::<Vec<(usize, &str)>>();
199    /// let b: &[_] = &[(0, "Brr"), (3, ","), (4, " "), (5, "it's"), (9, " "), (10, "29.3"),
200    ///                 (14, "°"), (16, "F"), (17, "!")];
201    ///
202    /// assert_eq!(&swi1[..], b);
203    /// ```
204    fn split_word_bound_indices<'a>(&'a self) -> UWordBoundIndices<'a>;
205
206    /// Returns an iterator over substrings of `self` separated on
207    /// [UAX#29 sentence boundaries](http://www.unicode.org/reports/tr29/#Sentence_Boundaries).
208    ///
209    /// Here, "sentences" are just those substrings which, after splitting on
210    /// UAX#29 sentence boundaries, contain any alphanumeric characters. That is, the
211    /// substring must contain at least one character with the
212    /// [Alphabetic](http://unicode.org/reports/tr44/#Alphabetic)
213    /// property, or with
214    /// [General_Category=Number](http://unicode.org/reports/tr44/#General_Category_Values).
215    ///
216    /// # Example
217    ///
218    /// ```
219    /// # use self::unicode_segmentation::UnicodeSegmentation;
220    /// let uss = "Mr. Fox jumped. [...] The dog was too lazy.";
221    /// let us1 = uss.unicode_sentences().collect::<Vec<&str>>();
222    /// let b: &[_] = &["Mr. ", "Fox jumped. ", "The dog was too lazy."];
223    ///
224    /// assert_eq!(&us1[..], b);
225    /// ```
226    fn unicode_sentences<'a>(&'a self) -> UnicodeSentences<'a>;
227
228    /// Returns an iterator over substrings of `self` separated on
229    /// [UAX#29 sentence boundaries](http://www.unicode.org/reports/tr29/#Sentence_Boundaries).
230    ///
231    /// The concatenation of the substrings returned by this function is just the original string.
232    ///
233    /// # Example
234    ///
235    /// ```
236    /// # use self::unicode_segmentation::UnicodeSegmentation;
237    /// let ssbs = "Mr. Fox jumped. [...] The dog was too lazy.";
238    /// let ssb1 = ssbs.split_sentence_bounds().collect::<Vec<&str>>();
239    /// let b: &[_] = &["Mr. ", "Fox jumped. ", "[...] ", "The dog was too lazy."];
240    ///
241    /// assert_eq!(&ssb1[..], b);
242    /// ```
243    fn split_sentence_bounds<'a>(&'a self) -> USentenceBounds<'a>;
244
245    /// Returns an iterator over substrings of `self`, split on UAX#29 sentence boundaries,
246    /// and their offsets. See `split_sentence_bounds()` for more information.
247    ///
248    /// # Example
249    ///
250    /// ```
251    /// # use self::unicode_segmentation::UnicodeSegmentation;
252    /// let ssis = "Mr. Fox jumped. [...] The dog was too lazy.";
253    /// let ssi1 = ssis.split_sentence_bound_indices().collect::<Vec<(usize, &str)>>();
254    /// let b: &[_] = &[(0, "Mr. "), (4, "Fox jumped. "), (16, "[...] "),
255    ///                 (22, "The dog was too lazy.")];
256    ///
257    /// assert_eq!(&ssi1[..], b);
258    /// ```
259    fn split_sentence_bound_indices<'a>(&'a self) -> USentenceBoundIndices<'a>;
260}
261
262impl UnicodeSegmentation for str {
263    #[inline]
264    fn graphemes(&self, is_extended: bool) -> Graphemes {
265        grapheme::new_graphemes(self, is_extended)
266    }
267
268    #[inline]
269    fn grapheme_indices(&self, is_extended: bool) -> GraphemeIndices {
270        grapheme::new_grapheme_indices(self, is_extended)
271    }
272
273    #[inline]
274    fn unicode_words(&self) -> UnicodeWords {
275        word::new_unicode_words(self)
276    }
277
278    #[inline]
279    fn unicode_word_indices(&self) -> UnicodeWordIndices {
280        word::new_unicode_word_indices(self)
281    }
282
283    #[inline]
284    fn split_word_bounds(&self) -> UWordBounds {
285        word::new_word_bounds(self)
286    }
287
288    #[inline]
289    fn split_word_bound_indices(&self) -> UWordBoundIndices {
290        word::new_word_bound_indices(self)
291    }
292
293    #[inline]
294    fn unicode_sentences(&self) -> UnicodeSentences {
295        sentence::new_unicode_sentences(self)
296    }
297
298    #[inline]
299    fn split_sentence_bounds(&self) -> USentenceBounds {
300        sentence::new_sentence_bounds(self)
301    }
302
303    #[inline]
304    fn split_sentence_bound_indices(&self) -> USentenceBoundIndices {
305        sentence::new_sentence_bound_indices(self)
306    }
307}