fix(file source): handle concatenated gzip streams (#25614)

Co-authored-by: Pavlos Rontidis <pavlos.rontidis@gmail.com>

Author: thomasqueirozb

Cargo.lock

@@ -141,6 +141,7 @@ antithesis-instrumentation = { version = "0.1", default-features = false, featur
 antithesis_sdk = { version = "0.2", default-features = false, features = [] }
 anyhow = { version = "1.0.102", default-features = false, features = ["std"] }
 arc-swap = { version = "1.8.2", default-features = false }
+async-compression = { version = "0.4.42", default-features = false, features = ["tokio", "gzip"] }
 async-stream = { version = "0.3.6", default-features = false }
 async-trait = { version = "0.1.89", default-features = false }
 axum = { version = "0.6.20", default-features = false }
@@ -362,7 +363,7 @@ greptimedb-ingester = { version = "0.17.0", default-features = false, optional =
 
 # External libs
 arc-swap = { workspace = true, default-features = false, optional = true }
-async-compression = { version = "0.4.27", default-features = false, features = ["tokio", "gzip", "zstd"], optional = true }
+async-compression = { workspace = true, features = ["zstd"], optional = true }
 arrow = { version = "58.2.0", default-features = false, features = ["ipc"], optional = true }
 arrow-schema = { version = "58.2.0", default-features = false, optional = true }
 parquet = { version = "58.2.0", default-features = false, features = [

Cargo.toml

@@ -0,0 +1,3 @@
+Fixed the `file` source silently dropping all but the first member of concatenated (multi-stream) gzip files. This regression was introduced in v0.50.0.
+
+authors: thomasqueirozb

changelog.d/24280_fix_gzip_multi_stream.fix.md

@@ -6,6 +6,7 @@ allow-unwrap-in-tests = true
 disallowed-methods = [
   { path = "std::io::Write::write", reason = "This doesn't handle short writes, use `write_all` instead." },
   { path = "vrl::stdlib::all", reason = "Use `vector_vrl_functions::all()` instead for consistency across all Vector VRL functions." },
+  { path = "async_compression::tokio::bufread::GzipDecoder::new", reason = "Use `vector_common::compression::gzip_multiple_decoder` instead, which enables `multiple_members` to handle concatenated gzip streams." },
 ]
 
 disallowed-macros = [

clippy.toml

@@ -20,7 +20,7 @@ serde_json.workspace = true
 bstr = { version = "1.12", default-features = false }
 bytes = { version = "1.11.1", default-features = false, features = ["serde"] }
 dashmap = { version = "6.1", default-features = false }
-async-compression = { version = "0.4.42", features = ["tokio", "gzip"] }
+async-compression.workspace = true
 vector-common = { path = "../vector-common", default-features = false }
 vector-config = { path = "../vector-config", default-features = false }
 tokio = { workspace = true, features = ["full"] }

lib/file-source-common/Cargo.toml

@@ -5,13 +5,13 @@ use std::{
     time,
 };
 
-use async_compression::tokio::bufread::GzipDecoder;
 use crc::Crc;
 use serde::{Deserialize, Serialize};
 use tokio::{
     fs::{self, File},
     io::{AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncReadExt, AsyncSeekExt, BufReader},
 };
+use vector_common::compression::gzip_multiple_decoder;
 use vector_common::constants::GZIP_MAGIC;
 
 use crate::{
@@ -128,7 +128,7 @@ impl UncompressedReader for UncompressedReaderImpl {
         // To support new compression algorithms, add them below
         match Self::check(fp).await? {
             Some(SupportedCompressionAlgorithms::Gzip) => Ok(Box::new(BufReader::new(
-                GzipDecoder::new(BufReader::new(fp)),
+                gzip_multiple_decoder(BufReader::new(fp)),
             ))),
             // No compression, or read the raw bytes
             None => Ok(Box::new(BufReader::new(fp))),

lib/file-source-common/src/fingerprinter.rs

@@ -21,7 +21,7 @@ futures = { version = "0.3.31", default-features = false, features = ["executor"
 futures-util.workspace = true
 vector-common = { path = "../vector-common", default-features = false }
 file-source-common = { path = "../file-source-common" }
-async-compression = { version = "0.4.42", features = ["tokio", "gzip"] }
+async-compression.workspace = true
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["full"] }

lib/file-source/Cargo.toml

@@ -1,4 +1,3 @@
-use async_compression::tokio::bufread::GzipDecoder;
 use bytes::{Bytes, BytesMut};
 use chrono::{DateTime, Utc};
 use std::{
@@ -18,6 +17,7 @@ use file_source_common::{
     AsyncFileInfo, FilePosition, PortableFileExt, ReadFrom,
     buffer::{ReadResult, read_until_with_max_size},
 };
+use vector_common::compression::gzip_multiple_decoder;
 
 const EOF_READ_BACKOFF_MIN: Duration = Duration::from_millis(1);
 const EOF_READ_BACKOFF_MAX: Duration = Duration::from_millis(250);
@@ -132,7 +132,7 @@ impl FileWatcher {
                     (Box::new(null_reader()), 0)
                 }
                 (true, false, ReadFrom::Beginning) => {
-                    (Box::new(BufReader::new(GzipDecoder::new(reader))), 0)
+                    (Box::new(BufReader::new(gzip_multiple_decoder(reader))), 0)
                 }
                 (false, true, _) => {
                     let pos = reader.seek(SeekFrom::End(0)).await.unwrap();
@@ -189,7 +189,7 @@ impl FileWatcher {
                 if self.file_position != 0 {
                     Box::new(null_reader())
                 } else {
-                    Box::new(BufReader::new(GzipDecoder::new(reader)))
+                    Box::new(BufReader::new(gzip_multiple_decoder(reader)))
                 }
             } else {
                 reader.seek(io::SeekFrom::Start(self.file_position)).await?;

lib/file-source/src/file_watcher/mod.rs

@@ -175,6 +175,51 @@ impl Arbitrary for FileWatcherAction {
     }
 }
 
+#[tokio::test]
+async fn gzip_multi_stream_reads_all_members() {
+    use async_compression::tokio::bufread::GzipEncoder;
+    use std::fs;
+    use tokio::io::AsyncReadExt as _;
+
+    let dir = tempfile::TempDir::new().expect("could not create tempdir");
+    let path = dir.path().join("multi.gz");
+
+    async fn encode(data: &[u8]) -> Vec<u8> {
+        let mut out = Vec::new();
+        GzipEncoder::new(data).read_to_end(&mut out).await.unwrap();
+        out
+    }
+
+    // Write two separate gzip members into one file — the bug dropped the second.
+    let mut bytes = encode(b"first\n").await;
+    bytes.extend(encode(b"second\n").await);
+    fs::write(&path, &bytes).unwrap();
+
+    let mut fw = FileWatcher::new(
+        path,
+        file_source_common::ReadFrom::Beginning,
+        None,
+        100_000,
+        Bytes::from("\n"),
+    )
+    .await
+    .expect("FileWatcher::new failed");
+
+    let mut lines = Vec::new();
+    for _ in 0..10 {
+        fw.track_read_attempt();
+        let result = fw.read_line().await.expect("read_line error");
+        if let Some(raw) = result.raw_line {
+            lines.push(String::from_utf8(raw.bytes.to_vec()).unwrap());
+        }
+        if lines.len() == 2 {
+            break;
+        }
+    }
+
+    assert_eq!(lines, vec!["first", "second"]);
+}
+
 fn watcher_for_timing() -> FileWatcher {
     let now = Instant::now();
 

lib/file-source/src/file_watcher/tests/mod.rs

@@ -34,6 +34,7 @@ test = []
 tokenize = []
 
 [dependencies]
+async-compression.workspace = true
 async-stream = "0.3.6"
 bytes = { version = "1.11.1", default-features = false, optional = true }
 chrono.workspace = true