From e1bc78058da8899ad10beee293d6d2c8255150e8 Mon Sep 17 00:00:00 2001 From: Nikhil Bagmar Date: Fri, 5 Jun 2026 14:59:56 -0700 Subject: [PATCH] Detect zstd compressed objects in automatic compression mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CompressionOption.fromFileName() backs the "automatic" compression option for the S3 source (both SQS-notification and scan paths) and the S3 enrich processor. It only recognized ".gz" (GZIP) and ".snappy" (SNAPPY) and silently fell back to NONE for everything else, so zstd-compressed objects were passed to the codec without decompression and failed to parse — even though ZSTD is a valid CompressionOption with a working ZstdDecompressionEngine already wired in. Map the ".zst" (canonical, per the zstd CLI and RFC 8478) and ".zstd" (informal alternate spelling) extensions to CompressionOption.ZSTD so automatic detection decompresses these objects correctly. Behavior is unchanged unless a user explicitly configures compression: automatic. Add unit tests covering both extensions. Signed-off-by: Nikhil Bagmar --- .../dataprepper/plugins/codec/CompressionOption.java | 4 +++- .../plugins/codec/CompressionOptionTest.java | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/codec/CompressionOption.java b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/codec/CompressionOption.java index 14c481a19a..a9fe4a97a8 100644 --- a/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/codec/CompressionOption.java +++ b/data-prepper-plugins/common/src/main/java/org/opensearch/dataprepper/plugins/codec/CompressionOption.java @@ -51,7 +51,9 @@ public static CompressionOption fromFileName(final String fileName) { return CompressionOption.GZIP; } else if(fileName.endsWith(".snappy")){ return CompressionOption.SNAPPY; - }else { + } else if (fileName.endsWith(".zst") || fileName.endsWith(".zstd")) { + return CompressionOption.ZSTD; + } else { return CompressionOption.NONE; } } diff --git a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/codec/CompressionOptionTest.java b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/codec/CompressionOptionTest.java index 772476d905..527a8486f8 100644 --- a/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/codec/CompressionOptionTest.java +++ b/data-prepper-plugins/common/src/test/java/org/opensearch/dataprepper/plugins/codec/CompressionOptionTest.java @@ -33,6 +33,16 @@ void testFromFileName_snappy() { assertThat(CompressionOption.fromFileName("temp.snappy"), is(CompressionOption.SNAPPY)); } + @Test + void testFromFileName_zstd() { + assertThat(CompressionOption.fromFileName("temp.zst"), is(CompressionOption.ZSTD)); + } + + @Test + void testFromFileName_zstd_longExtension() { + assertThat(CompressionOption.fromFileName("temp.zstd"), is(CompressionOption.ZSTD)); + } + @Test void testFromFileName_default() { assertThat(CompressionOption.fromFileName("temp.txt"), is(CompressionOption.NONE));