unexpand: use utf8_chunks

Author: oech3

src/uu/unexpand/src/unexpand.rs

@@ -11,7 +11,6 @@ use std::fs::File;
 use std::io::{self, BufReader, BufWriter, Read, Stdin, Stdout, Write, stdin, stdout};
 use std::num::IntErrorKind;
 use std::path::Path;
-use std::str::from_utf8;
 use thiserror::Error;
 use uucore::display::Quotable;
 use uucore::error::{FromIo, UError, UResult, USimpleError, set_exit_code};
@@ -396,29 +395,14 @@ enum CharType {
     Other,
 }
 
-fn next_char_info(utf8: bool, buf: &[u8], byte: usize) -> (CharType, usize, usize) {
+fn char_info(c: char) -> (CharType, usize) {
     use CharType::{Backspace, Other, Space, Tab};
-    let b = buf[byte];
-    if b.is_ascii() {
-        return match b {
-            b' ' => (Space, 0, 1),
-            b'\t' => (Tab, 0, 1),
-            b'\x08' => (Backspace, 0, 1),
-            _ => (Other, 1, 1),
-        };
-    }
-
-    if utf8 {
-        let nbytes = char::from(b).len_utf8();
-        // don't overrun the buffer because of invalid UTF-8
-        if buf
-            .get(byte..byte + nbytes)
-            .is_some_and(|s| from_utf8(s).is_ok())
-        {
-            return (Other, nbytes, nbytes);
-        }
+    match c {
+        ' ' => (Space, 1),
+        '\t' => (Tab, 0),
+        '\x08' => (Backspace, 0),
+        _ => (Other, 1),
     }
-    (Other, 1, 1)
 }
 
 // This struct is used to store the current state of printing the input buf.
@@ -506,56 +490,65 @@ fn unexpand_buf(
         }
     }
 
-    while byte < buf.len() {
-        // when we have a finite number of columns, never convert past the last column
-        if lastcol > 0 && print_state.col >= lastcol {
-            write_tabs(output, tab_config, print_state, true)?;
-            output.write_all(&buf[byte..])?;
-            print_state.scol = print_state.col;
-            break;
-        }
-
-        // figure out how big the next char is, if it's UTF-8
-        let (ctype, cwidth, nbytes) = next_char_info(options.utf8, buf, byte);
+    for chunk in buf.utf8_chunks() {
+        for c in chunk.valid().chars() {
+            // when we have a finite number of columns, never convert past the last column
+            if lastcol > 0 && print_state.col >= lastcol {
+                write_tabs(output, tab_config, print_state, true)?;
+                output.write_all(&buf[byte..])?;
+                print_state.scol = print_state.col;
+                break;
+            }
 
-        // now figure out how many columns this char takes up, and maybe print it
-        let tabs_buffered = print_state.leading || options.aflag;
-        match ctype {
-            CharType::Space | CharType::Tab => {
-                // compute next col, but only write space or tab chars if not buffering
-                print_state.col += if ctype == CharType::Space {
-                    1
-                } else {
-                    next_tabstop(tab_config, print_state.col).unwrap_or(1)
-                };
+            // figure out how big the next char is, if it's UTF-8
+            let (ctype, cwidth) = char_info(c);
+            let nbytes = c.len_utf8();
+
+            // now figure out how many columns this char takes up, and maybe print it
+            let tabs_buffered = print_state.leading || options.aflag;
+            match ctype {
+                CharType::Space | CharType::Tab => {
+                    // compute next col, but only write space or tab chars if not buffering
+                    print_state.col += if ctype == CharType::Space {
+                        1
+                    } else {
+                        next_tabstop(tab_config, print_state.col).unwrap_or(1)
+                    };
 
-                if !tabs_buffered {
+                    if !tabs_buffered {
+                        output.write_all(&buf[byte..byte + nbytes])?;
+                        print_state.scol = print_state.col; // now printed up to this column
+                    }
+                }
+                CharType::Other | CharType::Backspace => {
+                    // always
+                    write_tabs(output, tab_config, print_state, options.aflag)?;
+                    print_state.leading = false; // no longer at the start of a line
+                    print_state.col = if ctype == CharType::Other {
+                        // use computed width
+                        print_state.col + cwidth
+                    } else if print_state.col > 0 {
+                        // Backspace case, but only if col > 0
+                        print_state.col - 1
+                    } else {
+                        0
+                    };
                     output.write_all(&buf[byte..byte + nbytes])?;
-                    print_state.scol = print_state.col; // now printed up to this column
+                    print_state.scol = print_state.col; // we've now printed up to this column
                 }
             }
-            CharType::Other | CharType::Backspace => {
-                // always
-                write_tabs(output, tab_config, print_state, options.aflag)?;
-                print_state.leading = false; // no longer at the start of a line
-                print_state.col = if ctype == CharType::Other {
-                    // use computed width
-                    print_state.col + cwidth
-                } else if print_state.col > 0 {
-                    // Backspace case, but only if col > 0
-                    print_state.col - 1
-                } else {
-                    0
-                };
-                output.write_all(&buf[byte..byte + nbytes])?;
-                print_state.scol = print_state.col; // we've now printed up to this column
-            }
+            print_state.pctype = ctype; // save the previous type
         }
 
-        byte += nbytes; // move on to next char
-        print_state.pctype = ctype; // save the previous type
+        for &b in chunk.invalid() {
+            write_tabs(output, tab_config, print_state, options.aflag)?;
+            print_state.leading = false;
+            print_state.col += 1;
+            output.write_all(&[b])?;
+            print_state.scol = print_state.col;
+            print_state.pctype = CharType::Other;
+        }
     }
-
     Ok(())
 }