fix!: reject mtree specifications on archive entry points

libarchive's mtree format handler matches arbitrary free-form text, so inputs as innocuous
as a plain gunzip'd text file were being parsed as single-entry mtree "archives". This
defeats the guarantee added alongside the raw-handler removal, where archive entry points
are supposed to reliably reject non-archive bytes.

- After each successful archive_read_next_header, check archive_format and treat an mtree
  match the same way we treat the raw handler: return an Extraction error instead of
  yielding bogus entries
- Wire the check into list_archive_files, list_archive_entries, uncompress_archive,
  uncompress_archive_file, and ArchiveIterator (covering their _with_encoding and async
  siblings via the shared helpers)
- Expose archive_format plus ARCHIVE_FORMAT_BASE_MASK / ARCHIVE_FORMAT_MTREE through the
  FFI bindings and generate-ffi script
- Extend CHANGES.md and the crate-level docs to describe the mtree rejection alongside the
  existing raw-handler note
- Add regression tests that feed tests/fixtures/file.txt.gz through list_archive_files and
  ArchiveIterator to confirm both surfaces reject the mtree interpretation

Author: otavio

CHANGES.md

@@ -5,13 +5,16 @@
 ## [Unreleased] - ReleaseDate
 
 * **Breaking:** libarchive's "raw" format handler is no longer registered on
-  the archive code paths. `list_archive_files`, `list_archive_entries`,
-  `uncompress_archive`, `uncompress_archive_file`, `ArchiveIterator`, and
-  their `_with_encoding` and async variants now return an error for input
-  that isn't a real archive, instead of treating it as a single-entry
-  archive named `data`. `uncompress_data` still supports raw streams
-  (that is its purpose). Callers that want the old iterator behavior can
-  opt back in with `ArchiveIteratorBuilder::raw_format(true)` [#77]
+  the archive code paths, and entries produced by the "mtree" handler are
+  rejected (libarchive matches mtree on arbitrary free-form text, so a plain
+  gunzip'd text file used to parse as an mtree archive). `list_archive_files`,
+  `list_archive_entries`, `uncompress_archive`, `uncompress_archive_file`,
+  `ArchiveIterator`, and their `_with_encoding` and async variants now return
+  an error for input that isn't a real archive, instead of treating it as a
+  single-entry archive named `data` or as an mtree specification.
+  `uncompress_data` still supports raw streams (that is its purpose).
+  Callers that want the old iterator behavior for non-archive bytes can opt
+  back in with `ArchiveIteratorBuilder::raw_format(true)` [#77]
 * Rewind the source reader at the start of every seekable entry point
   (`list_archive_files`, `list_archive_entries`, `uncompress_archive`,
   `uncompress_archive_file`, `ArchiveIterator`, and their `_with_encoding`

scripts/generate-ffi

@@ -40,6 +40,9 @@ bindgen \
     --allowlist-var "ARCHIVE_EXTRACT_OWNER" \
     --allowlist-var "ARCHIVE_EXTRACT_FFLAGS" \
     --allowlist-var "ARCHIVE_EXTRACT_XATTR" \
+    --allowlist-var "ARCHIVE_FORMAT_BASE_MASK" \
+    --allowlist-var "ARCHIVE_FORMAT_MTREE" \
+    --allowlist-function "archive_format" \
     --allowlist-function "archive_read_new" \
     --allowlist-function "archive_read_set_seek_callback" \
     --allowlist-function "archive_read_support_filter_all" \

src/ffi/generated.rs

@@ -15,6 +15,8 @@ pub(crate) const ARCHIVE_EXTRACT_TIME: u32 = 4;
 pub(crate) const ARCHIVE_EXTRACT_ACL: u32 = 32;
 pub(crate) const ARCHIVE_EXTRACT_FFLAGS: u32 = 64;
 pub(crate) const ARCHIVE_EXTRACT_XATTR: u32 = 128;
+pub(crate) const ARCHIVE_FORMAT_BASE_MASK: ::std::os::raw::c_int = 0xff0000;
+pub(crate) const ARCHIVE_FORMAT_MTREE: ::std::os::raw::c_int = 0x80000;
 pub(crate) type __dev_t = ::std::os::raw::c_ulong;
 pub(crate) type __uid_t = ::std::os::raw::c_uint;
 pub(crate) type __gid_t = ::std::os::raw::c_uint;
@@ -158,6 +160,9 @@ extern "C" {
 extern "C" {
     pub(crate) fn archive_errno(arg1: *mut archive) -> ::std::os::raw::c_int;
 }
+extern "C" {
+    pub(crate) fn archive_format(arg1: *mut archive) -> ::std::os::raw::c_int;
+}
 extern "C" {
     pub(crate) fn archive_error_string(arg1: *mut archive) -> *const ::std::os::raw::c_char;
 }

src/iterator.rs

@@ -380,6 +380,9 @@ impl<R: Read + Seek> ArchiveIterator<R> {
         match ffi::archive_read_next_header(self.archive_reader, &mut self.archive_entry) {
             ffi::ARCHIVE_EOF => ArchiveContents::EndOfEntry,
             ffi::ARCHIVE_OK | ffi::ARCHIVE_WARN => {
+                if let Err(e) = crate::reject_mtree_format(self.archive_reader) {
+                    return ArchiveContents::Err(e);
+                }
                 let _utf8_guard = ffi::WindowsUTF8LocaleGuard::new();
                 let cstr = CStr::from_ptr(ffi::archive_entry_pathname(self.archive_entry));
                 let file_name = match (self.decode)(cstr.to_bytes()) {

src/lib.rs

@@ -52,10 +52,10 @@
 //! Archive-listing and archive-extraction entry points (`list_archive_files`,
 //! `list_archive_entries`, `uncompress_archive`, `uncompress_archive_file`,
 //! `ArchiveIterator`, and their async/`_with_encoding` siblings) no longer
-//! register libarchive's "raw" format handler. They return an error for
-//! input that isn't a real archive instead of yielding a single entry
-//! called `data`, so callers can reliably distinguish archives from other
-//! files.
+//! accept libarchive's two promiscuous format handlers: "raw" (which
+//! matches arbitrary bytes) and "mtree" (which matches free-form text).
+//! They return an error for input that isn't a real archive, so callers
+//! can reliably distinguish archives from other files.
 //!
 //! Use [`uncompress_data`] for decompressing a single stream (gzip, xz, …)
 //! — it continues to support raw input because that is its purpose. For
@@ -241,6 +241,7 @@ where
                     ffi::ARCHIVE_EOF => return Ok(entries),
                     value => archive_result(value, archive_reader)?,
                 }
+                reject_mtree_format(archive_reader)?;
 
                 let _utf8_guard = ffi::WindowsUTF8LocaleGuard::new();
                 let cstr = libarchive_entry_pathname(entry)?;
@@ -363,6 +364,7 @@ where
                     ffi::ARCHIVE_EOF => return Ok(()),
                     value => archive_result(value, archive_reader)?,
                 }
+                reject_mtree_format(archive_reader)?;
 
                 let _utf8_guard = ffi::WindowsUTF8LocaleGuard::new();
                 let cstr = libarchive_entry_pathname(entry)?;
@@ -478,6 +480,7 @@ where
                     }
                     value => archive_result(value, archive_reader)?,
                 }
+                reject_mtree_format(archive_reader)?;
 
                 let _utf8_guard = ffi::WindowsUTF8LocaleGuard::new();
                 let cstr = libarchive_entry_pathname(entry)?;
@@ -681,6 +684,23 @@ fn sanitize_destination_path(dest: &Path) -> Result<&Path> {
         })
 }
 
+// libarchive's mtree format handler parses free-form text specifications and
+// willingly matches innocuous content (a plain gunzip'd text file, for
+// instance). Treat an mtree match the same way we treat raw: not an archive.
+// Must be called after a successful `archive_read_next_header`, which is
+// when libarchive populates the format code.
+pub(crate) unsafe fn reject_mtree_format(archive_reader: *mut ffi::archive) -> Result<()> {
+    if ffi::archive_format(archive_reader) & ffi::ARCHIVE_FORMAT_BASE_MASK
+        == ffi::ARCHIVE_FORMAT_MTREE
+    {
+        return Err(Error::Extraction {
+            code: None,
+            details: "mtree specifications are not treated as archives".to_string(),
+        });
+    }
+    Ok(())
+}
+
 fn libarchive_copy_data(
     archive_reader: *mut ffi::archive,
     archive_writer: *mut ffi::archive,

tests/integration_test.rs

@@ -1416,6 +1416,28 @@ fn iterator_default_rejects_non_archive_bytes() {
     );
 }
 
+#[test]
+fn list_archive_files_rejects_plain_gzip_as_mtree() {
+    let source = std::fs::File::open("tests/fixtures/file.txt.gz").unwrap();
+    assert!(
+        list_archive_files(source).is_err(),
+        "a plain gzip'd text stream must not be parsed as an mtree archive",
+    );
+}
+
+#[test]
+fn iterator_default_rejects_plain_gzip_as_mtree() {
+    let source = std::fs::File::open("tests/fixtures/file.txt.gz").unwrap();
+    let saw_err = match ArchiveIterator::from_read(source) {
+        Err(_) => true,
+        Ok(iter) => iter.into_iter().any(|c| matches!(c, ArchiveContents::Err(_))),
+    };
+    assert!(
+        saw_err,
+        "strict iterator must surface an error on mtree-looking input"
+    );
+}
+
 #[test]
 fn iterator_raw_format_opt_in_accepts_non_archive_bytes() {
     let source = Cursor::new(NON_ARCHIVE_BYTES);