💥 handle CR/CRLF in -T/--files-from

Author: ChanTsune

Cargo.lock

@@ -33,6 +33,7 @@ itertools = "0.14.0"
 jiff = "0.2.18"
 libc = "0.2.180"
 log = "0.4.29"
+memchr = "2.7.4"
 memmap2 = { version = "0.9.9", optional = true }
 nom = "8.0.0"
 parse_datetime = "0.13.3"

cli/Cargo.toml

@@ -1,5 +1,12 @@
+use memchr::memchr2;
 use std::io::{self, BufRead};
 
+/// Converts a byte vector to a UTF-8 String, mapping encoding errors to `io::Error`.
+#[inline]
+fn bytes_to_string(bytes: Vec<u8>) -> io::Result<String> {
+    String::from_utf8(bytes).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
+}
+
 #[derive(Clone, Debug)]
 pub(crate) struct Delimited<'d, B> {
     buf: B,
@@ -45,16 +52,82 @@ impl<'d, B: BufRead> Iterator for DelimitedString<'d, B> {
 
     #[inline]
     fn next(&mut self) -> Option<Self::Item> {
-        match self.inner.next()? {
-            Err(e) => Some(Err(e)),
-            Ok(bytes) => match String::from_utf8(bytes) {
-                Ok(s) => Some(Ok(s)),
-                Err(err) => Some(Err(io::Error::new(io::ErrorKind::InvalidData, err))),
-            },
+        Some(self.inner.next()?.and_then(bytes_to_string))
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct LineBreakDelimited<B> {
+    buf: B,
+    pending: Vec<u8>,
+    done: bool,
+}
+
+impl<B: BufRead> LineBreakDelimited<B> {
+    #[inline]
+    fn new(buf: B) -> Self {
+        Self {
+            buf,
+            pending: Vec::new(),
+            done: false,
+        }
+    }
+}
+
+impl<B: BufRead> Iterator for LineBreakDelimited<B> {
+    type Item = io::Result<Vec<u8>>;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.done {
+            return None;
+        }
+        loop {
+            let buf = match self.buf.fill_buf() {
+                Ok(buf) => buf,
+                Err(err) => return Some(Err(err)),
+            };
+
+            if buf.is_empty() {
+                self.done = true;
+                return if self.pending.is_empty() {
+                    None
+                } else {
+                    Some(Ok(std::mem::take(&mut self.pending)))
+                };
+            }
+
+            match memchr2(b'\n', b'\r', buf) {
+                Some(idx) => {
+                    self.pending.extend_from_slice(&buf[..idx]);
+                    self.buf.consume(idx + 1);
+                    return Some(Ok(std::mem::take(&mut self.pending)));
+                }
+                None => {
+                    let len = buf.len();
+                    self.pending.extend_from_slice(buf);
+                    self.buf.consume(len);
+                }
+            }
         }
     }
 }
 
+/// Adapter that wraps the line-break iterator and yields UTF-8 Strings.
+#[derive(Debug)]
+pub(crate) struct LineBreakDelimitedString<B> {
+    inner: LineBreakDelimited<B>,
+}
+
+impl<B: BufRead> Iterator for LineBreakDelimitedString<B> {
+    type Item = io::Result<String>;
+
+    #[inline]
+    fn next(&mut self) -> Option<Self::Item> {
+        Some(self.inner.next()?.and_then(bytes_to_string))
+    }
+}
+
 /// Extension trait to add same utility methods to [`BufRead`].
 pub(crate) trait BufReadExt {
     /// Splits the reader by a delimiter, yielding Vec<u8>.
@@ -79,6 +152,21 @@ pub(crate) trait BufReadExt {
             inner: self.delimit_by(delimiter.as_bytes()),
         }
     }
+
+    /// Splits lines on CR, CRLF, or LF, yielding Strings.
+    ///
+    /// Both `\r` and `\n` are treated as individual line terminators.
+    /// This means `\r\n` (CRLF) will yield an empty string between the CR and LF.
+    /// Use `.filter()` to remove empty lines if this is not desired.
+    #[inline]
+    fn split_lines(self) -> LineBreakDelimitedString<Self>
+    where
+        Self: Sized + BufRead,
+    {
+        LineBreakDelimitedString {
+            inner: LineBreakDelimited::new(self),
+        }
+    }
 }
 
 impl<B: BufRead> BufReadExt for B {}
@@ -162,4 +250,53 @@ mod tests {
         assert_eq!(delimited.next().unwrap().unwrap(), "a||");
         assert_eq!(delimited.next().unwrap().unwrap(), "|b|");
     }
+
+    #[test]
+    fn split_lines_splits_mixed_endings() {
+        let input = b"f\rd1/f1\r\nd1/d2/f4\nd1/d2/f6";
+        let got = BufReader::new(&input[..])
+            .split_lines()
+            .collect::<io::Result<Vec<_>>>()
+            .unwrap();
+        assert_eq!(got, vec!["f", "d1/f1", "", "d1/d2/f4", "d1/d2/f6"]);
+    }
+
+    #[test]
+    fn split_lines_splits_crlf() {
+        let input = b"\r\n";
+        let got = BufReader::new(&input[..])
+            .split_lines()
+            .collect::<io::Result<Vec<_>>>()
+            .unwrap();
+        assert_eq!(got, vec!["", ""]);
+    }
+
+    #[test]
+    fn split_lines_empty_input() {
+        let input = b"";
+        let got = BufReader::new(&input[..])
+            .split_lines()
+            .collect::<io::Result<Vec<_>>>()
+            .unwrap();
+        assert!(got.is_empty());
+    }
+
+    #[test]
+    fn split_lines_invalid_utf8_returns_error() {
+        let input = b"valid\n\xff\xfe\n";
+        let mut iter = BufReader::new(&input[..]).split_lines();
+        assert_eq!(iter.next().unwrap().unwrap(), "valid");
+        let err = iter.next().unwrap().unwrap_err();
+        assert_eq!(err.kind(), io::ErrorKind::InvalidData);
+    }
+
+    #[test]
+    fn split_lines_consecutive_delimiters() {
+        let input = b"\n\n\n";
+        let got = BufReader::new(&input[..])
+            .split_lines()
+            .collect::<io::Result<Vec<_>>>()
+            .unwrap();
+        assert_eq!(got, vec!["", "", ""]);
+    }
 }

cli/src/ext/read_buf.rs

@@ -9,7 +9,10 @@ pub(crate) fn is_pna<R: io::Read>(mut reader: R) -> io::Result<bool> {
 
 #[inline]
 pub(crate) fn read_to_lines<R: io::BufRead>(reader: R) -> io::Result<Vec<String>> {
-    reader.lines().collect()
+    reader
+        .split_lines()
+        .filter(|line| !line.as_ref().is_ok_and(|s| s.is_empty()))
+        .collect()
 }
 
 /// Reads a reader and splits its contents on null characters ('\0'), returning a Vec<String>.
@@ -75,4 +78,18 @@ mod tests {
         let got = read_to_nul(io::BufReader::new(&input[..])).unwrap();
         assert_eq!(got, vec!["d1/d2/f3", "d1/d2/f5"]);
     }
+
+    #[test]
+    fn read_to_lines_supports_cr_crlf_lf() {
+        let input = b"f\rd1/f1\r\nd1/d2/f4\nd1/d2/f6";
+        let got = read_to_lines(io::BufReader::new(&input[..])).unwrap();
+        assert_eq!(got, vec!["f", "d1/f1", "d1/d2/f4", "d1/d2/f6"]);
+    }
+
+    #[test]
+    fn read_to_lines_ignores_empty_lines() {
+        let input = b"\n\r\n";
+        let got = read_to_lines(io::BufReader::new(&input[..])).unwrap();
+        assert!(got.is_empty());
+    }
 }

cli/src/utils/io.rs

@@ -72,3 +72,59 @@ fn create_with_files_from() {
     )
     .unwrap();
 }
+
+#[test]
+fn create_with_files_from_crlf() {
+    setup();
+    TestResources::extract_in("raw/", "create_with_files_from_crlf/src/").unwrap();
+
+    // Use CRLF line endings (Windows-style)
+    let list_path = "create_with_files_from_crlf/files.txt";
+    fs::write(
+        list_path,
+        "create_with_files_from_crlf/src/raw/empty.txt\r\ncreate_with_files_from_crlf/src/raw/text.txt\r\n",
+    )
+    .unwrap();
+
+    cli::Cli::try_parse_from([
+        "pna",
+        "--quiet",
+        "c",
+        "create_with_files_from_crlf/test.pna",
+        "--overwrite",
+        "--files-from",
+        list_path,
+        "--unstable",
+    ])
+    .unwrap()
+    .execute()
+    .unwrap();
+}
+
+#[test]
+fn create_with_files_from_cr() {
+    setup();
+    TestResources::extract_in("raw/", "create_with_files_from_cr/src/").unwrap();
+
+    // Use CR line endings (old Mac-style)
+    let list_path = "create_with_files_from_cr/files.txt";
+    fs::write(
+        list_path,
+        "create_with_files_from_cr/src/raw/empty.txt\rcreate_with_files_from_cr/src/raw/text.txt\r",
+    )
+    .unwrap();
+
+    cli::Cli::try_parse_from([
+        "pna",
+        "--quiet",
+        "c",
+        "create_with_files_from_cr/test.pna",
+        "--overwrite",
+        "--files-from",
+        list_path,
+        "--unstable",
+    ])
+    .unwrap()
+    .execute()
+    .unwrap();
+}