test: consolidate fixture download/cache/skip logic into LocalHttpCache

Five copy-pasted variants of "check a /tmp cache, download over HTTP,
skip on transient server error or no network" had drifted apart:
RustWritesJavaReadsIntegrationTest and PcoFixtureInspectionIntegrationTest
were exact duplicates; ParquetImportIntegrationTest's fixture download
had no version-keyed cache or 5xx-skip guard; RustJavaReaderComparison
IntegrationTest used JDK HttpClient/BodyHandlers with no cache and no
network-skip guard at all (a network blip failed the test instead of
skipping it); and taxiParquet_importedSize_vsOriginal reimplemented the
same thing inline with fully-qualified java.nio.file.Files calls.

Extract LocalHttpCache.downloadIfMissing (skip on 5xx, fail on 4xx) and
.assumeNetworkAvailable, plus .downloadIfMissingOrSkip for endpoints
that skip on ANY failure (CloudFront occasionally 403s CI runners'
egress IPs). Migrate all five call sites, standardizing on
HttpURLConnection since four of five already used it.

TaxiParquetOracleVsJavaIntegrationTest previously only ran if someone
had manually placed /tmp/yellow_tripdata_2024-01.parquet — it never
downloaded anything, so it silently skipped in every environment
without that manual step. It now shares the download-and-cache path
with taxiParquet_importedSize_vsOriginal and actually runs.

Author: dfa1

integration/src/test/java/io/github/dfa1/vortex/integration/LocalHttpCache.java

@@ -0,0 +1,81 @@
+package io.github.dfa1.vortex.integration;
+
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+/// Shared fixture-download-and-cache helper for integration tests that pull test data
+/// from S3/GitHub over HTTP: check a local cache first, download on miss, and skip
+/// (rather than fail) the test on transient server errors or no network.
+final class LocalHttpCache {
+
+    private LocalHttpCache() {
+    }
+
+    /// Downloads `url` into `tmp` if `cacheDir` doesn't already have a file named `name`,
+    /// leaving no copy in `cacheDir` (callers that want the download persisted for reuse by
+    /// later runs must pass a `cacheDir` that already exists and is writable — this method
+    /// only reads from it).
+    ///
+    /// A 5xx response skips the test via `assumeTrue` (assumed transient infrastructure
+    /// noise, not a real signal); a 4xx or other `IOException` fails it (the fixture is
+    /// genuinely missing or broken).
+    ///
+    /// @param tmp      the test's `@TempDir`, used as the download destination when the
+    ///                 fixture isn't already in `cacheDir`
+    /// @param cacheDir persistent local cache directory (e.g. `/tmp/pco-fixtures/v0.75.0`)
+    /// @param url      the fixture's download URL
+    /// @param name     the fixture's file name, used for both the cache lookup and dest file
+    /// @return the local path to the fixture: the cached copy, or the freshly downloaded one
+    /// @throws Exception if the download fails for a reason other than a transient 5xx
+    static Path downloadIfMissing(Path tmp, Path cacheDir, URI url, String name) throws Exception {
+        Path cached = cacheDir.resolve(name);
+        if (Files.exists(cached)) {
+            return cached;
+        }
+        Path dest = tmp.resolve(name);
+        var conn = (HttpURLConnection) url.toURL().openConnection();
+        int code = conn.getResponseCode();
+        assumeTrue(code < 500, () -> "transient server error " + code + " for " + name);
+        try (var in = conn.getInputStream()) {
+            Files.copy(in, dest, StandardCopyOption.REPLACE_EXISTING);
+        }
+        return dest;
+    }
+
+    /// Like [#downloadIfMissing] but skips the test (rather than failing it) on ANY download
+    /// failure, not just a 5xx — for endpoints known to rate-limit or block by egress IP
+    /// (e.g. CloudFront blocking CI runners with a 403) where any failure is infrastructure
+    /// noise rather than a genuine "fixture is broken" signal.
+    ///
+    /// @param tmp      the test's `@TempDir`, used as the download destination when the
+    ///                 fixture isn't already in `cacheDir`
+    /// @param cacheDir persistent local cache directory
+    /// @param url      the fixture's download URL
+    /// @param name     the fixture's file name, used for both the cache lookup and dest file
+    /// @return the local path to the fixture: the cached copy, or the freshly downloaded one
+    static Path downloadIfMissingOrSkip(Path tmp, Path cacheDir, URI url, String name) {
+        try {
+            return downloadIfMissing(tmp, cacheDir, url, name);
+        } catch (Exception e) {
+            assumeTrue(false, "could not download " + name + ": " + e.getMessage());
+            throw new AssertionError("unreachable");
+        }
+    }
+
+    /// Skips the running test (via `assumeTrue`) if `pingUrl` cannot be reached at all —
+    /// distinguishes "no network" (skip, infrastructure) from "fixture missing" (fail, signal).
+    ///
+    /// @param pingUrl a URL known to be reachable whenever the test network is up
+    static void assumeNetworkAvailable(URI pingUrl) {
+        try {
+            pingUrl.toURL().openStream().close();
+        } catch (Exception _) {
+            assumeTrue(false, "no network");
+        }
+    }
+}

integration/src/test/java/io/github/dfa1/vortex/integration/ParquetImportIntegrationTest.java

@@ -15,16 +15,13 @@
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.io.TempDir;
 
-import java.io.IOException;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.List;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 /// Round-trip: Parquet (via Hardwood) → Vortex (via ParquetImporter) → VortexReader.
 ///
@@ -60,25 +57,14 @@ private static List<String> parquetColumnNames(Path path) throws Exception {
     }
 
     private static Path download(Path tmp, String name) throws Exception {
-        Path cached = Path.of("/tmp/parquet-fixtures", name);
-        if (Files.exists(cached)) {
-            return cached;
-        }
-        Path dest = tmp.resolve(name);
-        try (var in = URI.create(FIXTURE_URL).toURL().openStream()) {
-            Files.copy(in, dest, StandardCopyOption.REPLACE_EXISTING);
-        }
-        return dest;
+        return LocalHttpCache.downloadIfMissing(tmp,
+                Path.of("/tmp/parquet-fixtures"), URI.create(FIXTURE_URL), name);
     }
 
     // ── helpers ──────────────────────────────────────────────────────────────
 
     private static void assumeNetworkAvailable() {
-        try {
-            URI.create("https://raw.githubusercontent.com").toURL().openStream().close();
-        } catch (Exception _) {
-            assumeTrue(false, "no network");
-        }
+        LocalHttpCache.assumeNetworkAvailable(URI.create("https://raw.githubusercontent.com"));
     }
 
     @Test
@@ -154,32 +140,20 @@ void stringColumnValuesMatch(@TempDir Path tmp) throws Exception {
     @Test
     void taxiParquet_importedSize_vsOriginal(@TempDir Path tmp) throws Exception {
         // Given — NYC Yellow Taxi 2024-01 (~3M rows, 19 cols, mix of I64 / F64 / I32 / Utf8).
-        // Uses the same parquet file as ParquetVsVortexReadBenchmark (cached in /tmp).
-        Path cached = Path.of("/tmp", "yellow_tripdata_2024-01.parquet");
-        Path src;
-        if (java.nio.file.Files.exists(cached)) {
-            src = cached;
-        } else {
-            // CloudFront rate-limits / blocks some egress IPs (notably GitHub Actions
-            // runners → 403). Skip rather than fail when the download isn't possible.
-            src = tmp.resolve("yellow_tripdata_2024-01.parquet");
-            try (var in = URI.create("https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2024-01.parquet")
-                    .toURL().openStream()) {
-                java.nio.file.Files.copy(in, src);
-            } catch (IOException e) {
-                org.junit.jupiter.api.Assumptions.assumeTrue(false,
-                        "could not download taxi parquet: " + e.getMessage());
-                return;
-            }
-        }
+        // Same fixture/cache path as TaxiParquetOracleVsJavaIntegrationTest. CloudFront
+        // rate-limits/blocks some egress IPs (notably GitHub Actions runners → 403), so any
+        // download failure skips rather than fails.
+        Path src = LocalHttpCache.downloadIfMissingOrSkip(tmp, Path.of("/tmp"),
+                URI.create("https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2024-01.parquet"),
+                "yellow_tripdata_2024-01.parquet");
         Path vortex = tmp.resolve("taxi.vortex");
 
         // When
         ParquetImporter.importParquet(src, vortex);
 
         // Then
-        long parquetSize = java.nio.file.Files.size(src);
-        long vortexSize = java.nio.file.Files.size(vortex);
+        long parquetSize = Files.size(src);
+        long vortexSize = Files.size(vortex);
         System.out.printf(
                 "[TaxiSizeComparison] Parquet=%,d bytes (%.1f MB)  Vortex=%,d bytes (%.1f MB)  Vortex/Parquet=%.2fx%n",
                 parquetSize, parquetSize / 1_048_576.0,

integration/src/test/java/io/github/dfa1/vortex/integration/PcoFixtureInspectionIntegrationTest.java

@@ -12,21 +12,17 @@
 import org.junit.jupiter.api.io.TempDir;
 
 import java.lang.foreign.MemorySegment;
-import java.net.HttpURLConnection;
 import java.net.URI;
 import java.nio.ByteBuffer;
 import java.nio.ByteOrder;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 /// Phase 0 scoping for `vortex.pco` decode port.
 ///
@@ -306,34 +302,16 @@ private static String formatDType(DType d) {
         return String.valueOf(d);
     }
 
+    // Reuse the version-keyed /tmp/pco-fixtures cache if present. The version segment
+    // matters: the Rust reference rewrites identical file names with different bytes
+    // across versions, so a version-less cache would serve stale bytes after a bump.
     private static Path downloadIfMissing(Path tmp, String name) throws Exception {
-        // Reuse the version-keyed /tmp/pco-fixtures cache if present. The version
-        // segment matters: the Rust reference rewrites identical file names with
-        // different bytes across versions, so a version-less cache would serve
-        // stale bytes after a bump.
-        Path cached = Path.of("/tmp/pco-fixtures", FIXTURE_VERSION, name);
-        if (Files.exists(cached)) {
-            return cached;
-        }
-        Path dest = tmp.resolve(name);
-        var conn = (HttpURLConnection) URI.create(BASE + name).toURL().openConnection();
-        int code = conn.getResponseCode();
-        // S3 occasionally returns a transient 5xx; that is infrastructure noise, not
-        // an interop regression, so skip rather than redden the build. A 4xx (e.g. the
-        // fixture was removed/renamed) is a genuine signal and still fails.
-        assumeTrue(code < 500, () -> "transient S3 error " + code + " for " + name);
-        try (var in = conn.getInputStream()) {
-            Files.copy(in, dest, StandardCopyOption.REPLACE_EXISTING);
-        }
-        return dest;
+        return LocalHttpCache.downloadIfMissing(tmp,
+                Path.of("/tmp/pco-fixtures", FIXTURE_VERSION), URI.create(BASE + name), name);
     }
 
     private static void assumeNetworkAvailable() {
-        try {
-            URI.create("https://vortex-compat-fixtures.s3.amazonaws.com").toURL().openStream().close();
-        } catch (Exception _) {
-            assumeTrue(false, "no network");
-        }
+        LocalHttpCache.assumeNetworkAvailable(URI.create("https://vortex-compat-fixtures.s3.amazonaws.com"));
     }
 
     @Test

integration/src/test/java/io/github/dfa1/vortex/integration/RustJavaReaderComparisonIntegrationTest.java

@@ -41,9 +41,6 @@
 import org.junit.jupiter.params.provider.ValueSource;
 
 import java.net.URI;
-import java.net.http.HttpClient;
-import java.net.http.HttpRequest;
-import java.net.http.HttpResponse;
 import java.nio.file.Path;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -59,25 +56,25 @@
 /// decode. A mismatch in any column value points to a decoding bug in the Java reader.
 class RustJavaReaderComparisonIntegrationTest {
 
+    private static final String FIXTURE_VERSION = "v0.75.0";
     private static final URI BASE =
-            URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/v0.75.0/arrays/");
+            URI.create("https://vortex-compat-fixtures.s3.amazonaws.com/" + FIXTURE_VERSION + "/arrays/");
 
     private static final Session SESSION = Session.create();
     private static final BufferAllocator ALLOCATOR = ArrowAllocation.rootAllocator();
-    private static final HttpClient HTTP = HttpClient.newHttpClient();
 
     static {
         NativeLoader.loadJni();
     }
 
-    private static Path download(URI uri, Path dir) throws Exception {
+    private static Path download(URI uri, Path tmp) throws Exception {
         String name = uri.getPath().substring(uri.getPath().lastIndexOf('/') + 1);
-        Path file = dir.resolve(name);
-        HTTP.send(
-                HttpRequest.newBuilder(uri).GET().build(),
-                HttpResponse.BodyHandlers.ofFile(file)
-        );
-        return file;
+        return LocalHttpCache.downloadIfMissing(tmp,
+                Path.of("/tmp/rust-fixtures", FIXTURE_VERSION), uri, name);
+    }
+
+    private static void assumeNetworkAvailable() {
+        LocalHttpCache.assumeNetworkAvailable(URI.create("https://vortex-compat-fixtures.s3.amazonaws.com"));
     }
 
     private static Stats rustStats(Path file) throws Exception {
@@ -403,6 +400,7 @@ private static Long stringByteLength(Array arr) {
     })
     void rust_vs_javaReader_statsMatch(String fixture, @TempDir Path tmp) throws Exception {
         // Given
+        assumeNetworkAvailable();
         Path local = download(BASE.resolve(fixture), tmp);
         String inspect = VortexInspector.inspect(VortexReader.open(local));
         System.out.println(inspect);

integration/src/test/java/io/github/dfa1/vortex/integration/RustWritesJavaReadsIntegrationTest.java

@@ -39,12 +39,9 @@
 import java.io.IOException;
 import java.lang.foreign.Arena;
 import java.lang.foreign.ValueLayout;
-import java.net.HttpURLConnection;
 import java.net.URI;
 import java.nio.ByteOrder;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -54,7 +51,6 @@
 import java.util.function.Consumer;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 /// Cross-compatibility: Rust (JNI) writer → Java reader.
 class RustWritesJavaReadsIntegrationTest {
@@ -220,33 +216,16 @@ private static long[] readJavaLongColumn(Path file, String column) throws IOExce
         }
     }
 
+    // Cache is keyed by fixture version: the Rust reference rewrites the same file names
+    // with different bytes across versions, so a version-less cache would silently serve
+    // stale bytes after a version bump.
     private static Path downloadIfMissing(Path tmp, String name) throws Exception {
-        // Cache is keyed by fixture version: the Rust reference rewrites the same
-        // file names with different bytes across versions, so a version-less cache
-        // would silently serve stale bytes after a version bump.
-        Path cached = Path.of("/tmp/pco-fixtures", FIXTURE_VERSION, name);
-        if (Files.exists(cached)) {
-            return cached;
-        }
-        Path dest = tmp.resolve(name);
-        var conn = (HttpURLConnection) URI.create(S3_BASE + name).toURL().openConnection();
-        int code = conn.getResponseCode();
-        // S3 occasionally returns a transient 5xx; that is infrastructure noise, not
-        // an interop regression, so skip rather than redden the build. A 4xx (e.g. the
-        // fixture was removed/renamed) is a genuine signal and still fails.
-        assumeTrue(code < 500, () -> "transient S3 error " + code + " for " + name);
-        try (var in = conn.getInputStream()) {
-            Files.copy(in, dest, StandardCopyOption.REPLACE_EXISTING);
-        }
-        return dest;
+        return LocalHttpCache.downloadIfMissing(tmp,
+                Path.of("/tmp/pco-fixtures", FIXTURE_VERSION), URI.create(S3_BASE + name), name);
     }
 
     private static void assumeNetworkAvailable() {
-        try {
-            URI.create("https://vortex-compat-fixtures.s3.amazonaws.com").toURL().openStream().close();
-        } catch (Exception _) {
-            assumeTrue(false, "no network");
-        }
+        LocalHttpCache.assumeNetworkAvailable(URI.create("https://vortex-compat-fixtures.s3.amazonaws.com"));
     }
 
     // ── S3 fixture round-trip: Rust-written pco → Java reader ────────────────

integration/src/test/java/io/github/dfa1/vortex/integration/TaxiParquetOracleVsJavaIntegrationTest.java

@@ -14,12 +14,12 @@
 
 import java.io.BufferedWriter;
 import java.io.IOException;
+import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.List;
 
 import static org.assertj.core.api.Assertions.assertThat;
-import static org.junit.jupiter.api.Assumptions.assumeTrue;
 
 /// Oracle vs SUT correctness test for the parquet importer.
 ///
@@ -33,22 +33,27 @@
 /// value exactly.
 class TaxiParquetOracleVsJavaIntegrationTest {
 
-    private static final Path TAXI_PARQUET = Path.of("/tmp/yellow_tripdata_2024-01.parquet");
+    // NYC Yellow Taxi 2024-01, same fixture/cache path as
+    // ParquetImportIntegrationTest.taxiParquet_importedSize_vsOriginal.
+    private static final String TAXI_URL =
+            "https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2024-01.parquet";
+    private static final String TAXI_NAME = "yellow_tripdata_2024-01.parquet";
 
     @Test
     void parquetImport_csvExport_matchesDirectParquetRead(@TempDir Path tmp) throws Exception {
         // Given
-        assumeTrue(Files.exists(TAXI_PARQUET), "taxi parquet not cached in /tmp — skip");
+        Path taxiParquet = LocalHttpCache.downloadIfMissingOrSkip(
+                tmp, Path.of("/tmp"), URI.create(TAXI_URL), TAXI_NAME);
 
         Path oracleCsv = tmp.resolve("oracle.csv");
         Path sutVortex = tmp.resolve("sut.vortex");
         Path sutCsv = tmp.resolve("sut.csv");
 
         // When — oracle: hardwood → CSV
-        writeOracleCsv(TAXI_PARQUET, oracleCsv);
+        writeOracleCsv(taxiParquet, oracleCsv);
 
         // When — SUT: parquet → vortex → CSV
-        ParquetImporter.importParquet(TAXI_PARQUET, sutVortex);
+        ParquetImporter.importParquet(taxiParquet, sutVortex);
         try (BufferedWriter writer = Files.newBufferedWriter(sutCsv)) {
             CsvExporter.exportCsv(sutVortex, writer, ExportOptions.defaults());
         }