fix(test): stream-compare conformance CSVs via temp files to avoid OOM (#254)

dfa1 · claude · dfa1 · commit c0ab5ee2f8b3 · 2026-07-09T21:21:37.000+02:00
StringWriter materialized multi-GB CSV Strings in heap for large corpus
files, crashing the JVM before reaching corrupted lines and silently
masking data bugs. Replace with CsvExporter.exportCsv(Path,Path,...) +
Files.newBufferedWriter for oracle output; assertFilesMatch streams both
files line-by-line via BufferedReader — constant heap regardless of size.

Also marks bi-euro2016 (gap:252) and covid-world-vaccination-progress
(gap:253) as known gaps now that the test actually reaches the bad lines.

Co-Authored-By: Claude Sonnet 4.6 &lt;noreply@anthropic.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `ScanIterator.sliceArray` now handles `NullArray`; previously threw on Rust-written files where a null-typed column used a coarser chunk grid than the other columns. ([#247](https://github.com/dfa1/vortex-java/issues/247))
 - `AlpRdEncodingDecoder` now reads `left_parts_ptype` from metadata and rejects any value other than U16 with a `VortexException`; previously the field was silently ignored and U16 was hardcoded, risking a 2-byte stride over a 1-byte buffer for spec-conformant non-Rust writers. ([#249](https://github.com/dfa1/vortex-java/issues/249))
 - `SparseEncodingDecoder` now asserts exactly two children (`patch_indices`, `patch_values`) and throws `VortexException` on any other count; previously a third child was silently accepted. ([#250](https://github.com/dfa1/vortex-java/issues/250))
+- `RaincloudConformanceIntegrationTest` now writes both the vortex-java and Parquet oracle CSVs to temp files and streams them line-by-line for comparison; previously it materialized both into heap `String` objects, causing OOM for large corpus files (≥157 MB) that were previously hidden by throwing before reaching the corrupted line. ([#254](https://github.com/dfa1/vortex-java/issues/254))
 
 ## [0.12.1] — 2026-07-08
 
diff --git a/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java b/integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java
@@ -14,9 +14,10 @@
 import org.junit.jupiter.api.TestFactory;
 import org.opentest4j.TestAbortedException;
 
+import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.StringWriter;
 import java.io.UncheckedIOException;
+import java.io.Writer;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
@@ -25,7 +26,6 @@
 import java.util.stream.Stream;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 /// Real-world cross-implementation conformance over the Raincloud corpus
@@ -83,14 +83,21 @@ Stream<DynamicTest> conformancePerSlug() throws IOException {
     }
 
     private static void assertMatchesParquetOracle(Path vortex, Path parquet) throws IOException {
-        // Given
-        List<String> oracleLines = oracleLines(parquet);
+        // Given — write both CSVs to temp files; never hold multi-GB strings in heap
+        Path oracleCsv = Files.createTempFile("oracle-", ".csv");
+        Path vortexCsv = Files.createTempFile("vortex-", ".csv");
+        try {
+            writeOracle(parquet, oracleCsv);
 
-        // When
-        List<String> result = exportVortex(vortex);
+            // When
+            CsvExporter.exportCsv(vortex, vortexCsv, ExportOptions.defaults());
 
-        // Then
-        assertLinesMatch(result, oracleLines);
+            // Then
+            assertFilesMatch(vortexCsv, oracleCsv);
+        } finally {
+            Files.deleteIfExists(oracleCsv);
+            Files.deleteIfExists(vortexCsv);
+        }
     }
 
     /// Runs the full conformance check but reports the outcome as an aborted test
@@ -108,64 +115,89 @@ private static void reportUntriaged(Path vortex, Path parquet) {
         throw new TestAbortedException("untriaged slug passes — flip its matrix entry to ok");
     }
 
-    private static void assertStillFails(Path vortex, Path parquet, String status) {
+    private static void assertStillFails(Path vortex, Path parquet, String status) throws IOException {
         // Given / When — a decode gap still reproduces when the export itself throws.
         // Only VortexException (the contractual untrusted-input failure) and, narrowly,
         // IndexOutOfBoundsException count: the latter is itself a bounds-guard bug (#215
         // throws it today) and this arm dies with that fix — a blanket RuntimeException
         // catch would green unrelated regressions (NPEs, ...). No oracle needed here
         // (the oracle may not even read this parquet, e.g. nested columns).
-        List<String> result;
+        Path vortexCsv = Files.createTempFile("vortex-", ".csv");
         try {
-            result = exportVortex(vortex);
-        } catch (VortexException | IndexOutOfBoundsException e) {
-            return;
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-
-        // Then — the export now succeeds, so the gap must still be a silent-corruption
-        // one (e.g. #208): values must still mismatch the oracle. A clean pass means the
-        // gap was fixed: flip the expected-status.csv entry to ok in the same change
-        List<String> oracleLines = oracleLines(parquet);
-        assertThatThrownBy(() -> assertLinesMatch(result, oracleLines))
-                .as("known gap %s no longer reproduces — flip its matrix entry to ok", status)
-                .isInstanceOf(AssertionError.class);
-    }
+            try {
+                CsvExporter.exportCsv(vortex, vortexCsv, ExportOptions.defaults());
+            } catch (VortexException | IndexOutOfBoundsException e) {
+                return; // gap still reproduces as a thrown exception
+            }
 
-    private static void assertLinesMatch(List<String> result, List<String> oracleLines) {
-        assertThat(result).hasSameSizeAs(oracleLines);
-        for (int i = 0; i < oracleLines.size(); i++) {
-            assertThat(result.get(i)).as("line %d", i + 1).isEqualTo(oracleLines.get(i));
+            // Then — the export now succeeds, so the gap must still be a silent-corruption
+            // one (e.g. #208): values must still mismatch the oracle. A clean pass means the
+            // gap was fixed: flip the expected-status.csv entry to ok in the same change
+            Path oracleCsv = Files.createTempFile("oracle-", ".csv");
+            try {
+                writeOracle(parquet, oracleCsv);
+                boolean stillMismatches = false;
+                try {
+                    assertFilesMatch(vortexCsv, oracleCsv);
+                } catch (AssertionError ignored) {
+                    stillMismatches = true;
+                }
+                assertThat(stillMismatches)
+                        .as("known gap %s no longer reproduces — flip its matrix entry to ok", status)
+                        .isTrue();
+            } finally {
+                Files.deleteIfExists(oracleCsv);
+            }
+        } finally {
+            Files.deleteIfExists(vortexCsv);
         }
     }
 
-    private static List<String> exportVortex(Path vortex) throws IOException {
-        StringWriter out = new StringWriter();
-        CsvExporter.exportCsv(vortex, out, ExportOptions.defaults());
-        return out.toString().lines().toList();
+    /// Stream-compares two CSV files line by line without loading either into heap.
+    ///
+    /// @param result the vortex-java export
+    /// @param oracle the parquet oracle export
+    /// @throws AssertionError  if any line differs or the files have different lengths
+    /// @throws IOException     if either file cannot be read
+    private static void assertFilesMatch(Path result, Path oracle) throws IOException {
+        try (BufferedReader r = Files.newBufferedReader(result);
+             BufferedReader o = Files.newBufferedReader(oracle)) {
+            long lineNum = 0;
+            String rLine;
+            while ((rLine = r.readLine()) != null) {
+                lineNum++;
+                String oLine = o.readLine();
+                assertThat(oLine).as("oracle ended before vortex output at line %d", lineNum).isNotNull();
+                assertThat(rLine).as("line %d", lineNum).isEqualTo(oLine);
+            }
+            assertThat(o.readLine()).as("vortex output ended before oracle at line %d", lineNum + 1).isNull();
+        }
     }
 
-    /// Reads the parquet sibling through hardwood; an oracle-side failure (nested
-    /// columns, unsupported physical type) aborts the slug rather than failing it —
-    /// it says nothing about vortex-java. Deliberate asymmetry: an `ok` slug whose
-    /// parquet the oracle cannot read stops being verified (visibly, as skipped) —
-    /// widen the oracle rather than let unverifiable entries fail the build.
-    private static List<String> oracleLines(Path parquet) {
-        StringWriter out = new StringWriter();
-        try {
-            writeOracleCsv(parquet, out);
+    /// Writes the parquet oracle to a file using the same cell rules as `CsvExporter`.
+    /// An oracle-side failure (nested columns, unsupported physical type) aborts the
+    /// slug via [TestAbortedException] rather than failing it — it says nothing about
+    /// vortex-java. An `ok` slug whose parquet cannot be read stops being verified
+    /// (visibly, as skipped) — widen the oracle rather than let unverifiable entries
+    /// fail the build.
+    ///
+    /// @param parquet the parquet sibling
+    /// @param out     the output file to write CSV into
+    /// @throws TestAbortedException if the oracle cannot read this parquet
+    /// @throws IOException          if writing fails
+    private static void writeOracle(Path parquet, Path out) throws IOException {
+        try (Writer writer = Files.newBufferedWriter(out)) {
+            writeOracleCsv(parquet, writer);
         } catch (TestAbortedException e) {
             throw e;
         } catch (Exception e) {
             throw new TestAbortedException("oracle cannot read the parquet sibling: " + e);
         }
-        return out.toString().lines().toList();
     }
 
     /// Oracle: hardwood reads the parquet sibling and emits CSV through the same
     /// fastcsv writer configuration as `CsvExporter`, using its exact cell rules.
-    private static void writeOracleCsv(Path parquet, StringWriter out) throws IOException {
+    private static void writeOracleCsv(Path parquet, Writer out) throws IOException {
         try (ParquetFileReader pfr = ParquetFileReader.open(InputFile.of(parquet));
              RowReader rows = pfr.rowReader();
              CsvWriter csv = CsvWriter.builder().fieldSeparator(',').build(out)) {
diff --git a/integration/src/test/resources/raincloud/expected-status.csv b/integration/src/test/resources/raincloud/expected-status.csv
@@ -29,7 +29,7 @@ bi-cmsprovider,untriaged
 bi-commongovernment,untriaged
 bi-corporations,ok
 bi-eixo,untriaged
-bi-euro2016,ok
+bi-euro2016,gap:252
 bi-food,ok
 bi-generico,untriaged
 bi-hashtags,ok
@@ -82,7 +82,7 @@ coig,untriaged
 coronahack-chest-xraydataset,untriaged
 cosmopedia-stanford,untriaged
 countries-of-the-world,ok
-covid-world-vaccination-progress,ok
+covid-world-vaccination-progress,gap:253
 covid19-data-from-john-hopkins-university,untriaged
 crimes-in-boston,untriaged
 databricks-dolly-15k,untriaged
