Avoid allocations for complex segmenter runs

components/segmenter/src/complex/dictionary.rs

@@ -5,11 +5,9 @@
 use crate::grapheme::*;
 use crate::indices::Utf16Indices;
 use crate::provider::*;
-#[cfg(feature = "unstable")]
 use crate::scaffold::PotentiallyIllFormedUtf8;
 use crate::scaffold::{RuleBreakType, Utf8, Utf16};
 use icu_collections::char16trie::{Char16Trie, TrieResult};
-#[cfg(feature = "unstable")]
 use utf8_iter::Utf8CharIndices;
 
 /// Lifetimes:
@@ -121,7 +119,6 @@ impl<'data> DictionarySegmenter<'data> {
     }
 
     /// Create a dictionary based break iterator for a UTF-8 string.
-    #[cfg(feature = "unstable")]
     pub(super) fn segment_utf8<'s>(
         self,
         input: &'s [u8],

components/segmenter/src/complex/language.rs

@@ -3,6 +3,7 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 use crate::provider::Language;
+use utf8_iter::Utf8CharIndices;
 
 // TODO: Use data provider
 pub(crate) fn get_language(codepoint: u32) -> Language {
@@ -89,6 +90,33 @@ impl<'s> Iterator for LanguageIterator<'s> {
     }
 }
 
+pub(super) struct LanguageIteratorUtf8<'s> {
+    rest: &'s [u8],
+}
+
+impl<'s> LanguageIteratorUtf8<'s> {
+    pub(super) fn new(input: &'s [u8]) -> Self {
+        Self { rest: input }
+    }
+}
+
+impl<'s> Iterator for LanguageIteratorUtf8<'s> {
+    type Item = (&'s [u8], Language);
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let mut indices = Utf8CharIndices::new(self.rest);
+        let lang = get_language(indices.next()?.1 as u32);
+        match indices.find(|&(_, ch)| get_language(ch as u32) != lang) {
+            Some((i, _)) => {
+                let (result, rest) = self.rest.split_at(i);
+                self.rest = rest;
+                Some((result, lang))
+            }
+            None => Some((core::mem::take(&mut self.rest), lang)),
+        }
+    }
+}
+
 pub(super) struct LanguageIteratorUtf16<'s> {
     rest: &'s [u16],
 }

components/segmenter/src/complex/lstm/mod.rs

@@ -5,13 +5,11 @@
 use crate::grapheme::GraphemeClusterSegmenterBorrowed;
 use crate::indices::Utf16Indices;
 use crate::provider::*;
-#[cfg(feature = "unstable")]
 use crate::scaffold::PotentiallyIllFormedUtf8;
 use crate::scaffold::{RuleBreakType, Utf8, Utf16};
 use alloc::vec::Vec;
 use core::char::{REPLACEMENT_CHARACTER, decode_utf16};
 use potential_utf::PotentialUtf8;
-#[cfg(feature = "unstable")]
 use utf8_iter::{Utf8CharIndices, Utf8Chars};
 use zerovec::maps::ZeroMapBorrowed;
 
@@ -124,7 +122,6 @@ impl<'data> LstmSegmenter<'data> {
     }
 
     /// Create an LSTM based break iterator for a UTF-8 string.
-    #[cfg(feature = "unstable")]
     pub(super) fn segment_utf8<'s>(
         self,
         input: &'s [u8],

components/segmenter/src/complex/mod.rs

@@ -3,7 +3,6 @@
 // (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
 
 use crate::provider::*;
-#[cfg(feature = "unstable")]
 use crate::scaffold::PotentiallyIllFormedUtf8;
 use crate::scaffold::{RuleBreakType, Utf8, Utf16};
 use crate::{GraphemeClusterSegmenter, GraphemeClusterSegmenterBorrowed};
@@ -78,7 +77,6 @@ impl<'data> ComplexPayloadBorrowed<'data> {
         )
     }
 
-    #[cfg(feature = "unstable")]
     pub(crate) fn segment_utf8<'s>(
         self,
         input: &'s [u8],
@@ -144,6 +142,67 @@ impl ComplexPayloadBorrowed<'static> {
     }
 }
 
+pub(crate) trait ComplexRunSegmenter: RuleBreakType {
+    fn complex_language_segment(
+        complex: ComplexPayloadsBorrowed<'_>,
+        start: &Self::IterAttr<'_>,
+        past: &Self::IterAttr<'_>,
+    ) -> Vec<usize>;
+
+    fn offset(iter: &Self::IterAttr<'_>) -> usize;
+}
+
+impl ComplexRunSegmenter for Utf8 {
+    fn complex_language_segment(
+        complex: ComplexPayloadsBorrowed<'_>,
+        start: &Self::IterAttr<'_>,
+        past: &Self::IterAttr<'_>,
+    ) -> Vec<usize> {
+        let start_offset = start.offset();
+        #[allow(clippy::indexing_slicing)] // valid offsets from CharIndices
+        let input = &start.as_str()[..(past.offset() - start_offset)];
+        complex.complex_language_segment_str(input)
+    }
+
+    fn offset(iter: &Self::IterAttr<'_>) -> usize {
+        iter.offset()
+    }
+}
+
+impl ComplexRunSegmenter for PotentiallyIllFormedUtf8 {
+    fn complex_language_segment(
+        complex: ComplexPayloadsBorrowed<'_>,
+        start: &Self::IterAttr<'_>,
+        past: &Self::IterAttr<'_>,
+    ) -> Vec<usize> {
+        let start_offset = start.offset();
+        #[allow(clippy::indexing_slicing)] // valid offsets from Utf8CharIndices
+        let input = &start.as_slice()[..(past.offset() - start_offset)];
+        complex.complex_language_segment_utf8(input)
+    }
+
+    fn offset(iter: &Self::IterAttr<'_>) -> usize {
+        iter.offset()
+    }
+}
+
+impl ComplexRunSegmenter for Utf16 {
+    fn complex_language_segment(
+        complex: ComplexPayloadsBorrowed<'_>,
+        start: &Self::IterAttr<'_>,
+        past: &Self::IterAttr<'_>,
+    ) -> Vec<usize> {
+        let start_offset = start.offset();
+        #[allow(clippy::indexing_slicing)] // valid offsets from Utf16Indices
+        let input = &start.as_slice()[..(past.offset() - start_offset)];
+        complex.complex_language_segment_utf16(input)
+    }
+
+    fn offset(iter: &Self::IterAttr<'_>) -> usize {
+        iter.offset()
+    }
+}
+
 #[derive(Debug)]
 pub(crate) struct ComplexPayloads {
     grapheme: GraphemeClusterSegmenter,
@@ -220,6 +279,20 @@ impl<'data> ComplexPayloadsBorrowed<'data> {
         }
         result
     }
+
+    pub(crate) fn complex_language_segment_utf8(&self, input: &[u8]) -> Vec<usize> {
+        let mut result = Vec::new();
+        let mut offset = 0;
+        for (slice, lang) in LanguageIteratorUtf8::new(input) {
+            match self.select(lang) {
+                Some(d) => result.extend(d.segment_utf8(slice, self.grapheme, offset)),
+                None => result.push(offset + slice.len()),
+            }
+            offset += slice.len();
+        }
+        result
+    }
+
     /// Return UTF-16 segment offset array using dictionary or lstm segmenter.
     pub(crate) fn complex_language_segment_utf16(&self, input: &[u16]) -> Vec<usize> {
         let mut result = Vec::new();

components/segmenter/src/grapheme.rs

@@ -168,6 +168,8 @@ impl<'data> GraphemeClusterSegmenterBorrowed<'data> {
     pub fn segment_str<'s>(self, input: &'s str) -> GraphemeClusterBreakIterator<'data, 's, Utf8> {
         GraphemeClusterBreakIterator(RuleBreakIterator {
             iter: input.char_indices(),
+            current_pos_iter: None,
+            previous_pos_iter: None,
             len: input.len(),
             current_pos_data: None,
             result_cache: Vec::new(),
@@ -189,6 +191,8 @@ impl<'data> GraphemeClusterSegmenterBorrowed<'data> {
     ) -> GraphemeClusterBreakIterator<'data, 's, PotentiallyIllFormedUtf8> {
         GraphemeClusterBreakIterator(RuleBreakIterator {
             iter: Utf8CharIndices::new(input),
+            current_pos_iter: None,
+            previous_pos_iter: None,
             len: input.len(),
             current_pos_data: None,
             result_cache: Vec::new(),
@@ -208,6 +212,8 @@ impl<'data> GraphemeClusterSegmenterBorrowed<'data> {
     ) -> GraphemeClusterBreakIterator<'data, 's, Latin1> {
         GraphemeClusterBreakIterator(RuleBreakIterator {
             iter: Latin1Indices::new(input),
+            current_pos_iter: None,
+            previous_pos_iter: None,
             len: input.len(),
             current_pos_data: None,
             result_cache: Vec::new(),
@@ -228,6 +234,8 @@ impl<'data> GraphemeClusterSegmenterBorrowed<'data> {
     ) -> GraphemeClusterBreakIterator<'data, 's, Utf16> {
         GraphemeClusterBreakIterator(RuleBreakIterator {
             iter: Utf16Indices::new(input),
+            current_pos_iter: None,
+            previous_pos_iter: None,
             len: input.len(),
             current_pos_data: None,
             result_cache: Vec::new(),