fix: de-duplicate Parquet column names in oracle; use index-based row access (#256)

The Rust Vortex writer de-duplicates repeated field names when writing a
Vortex file (N-th occurrence of 'col' becomes 'col [N]'). The Parquet
oracle now mirrors this in the CSV header. All row access switches from
name-based to index-based so duplicate-name files read the right column.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: dfa1

CHANGELOG.md

@@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Parquet files with duplicate column names now compare correctly. The oracle header de-duplicates names with the Rust Vortex writer's algorithm (first occurrence unmodified, Nth repetition gets ` [N]` suffix) and all row access uses column index rather than name. ([#256](https://github.com/dfa1/vortex-java/issues/256))
 - Blank column names are now accepted on both the read and write paths. The Rust reference legitimately produces them (e.g. `uci-electricityloaddiagrams20112014` has one at field index 0). `ColumnName` now enforces a symmetric policy: non-null and free of control characters (`\n`, `\t`, U+0000, …); blank names such as `""` or `" "` pass through on both paths. ([#255](https://github.com/dfa1/vortex-java/issues/255))
 - `VarBinArray.DictMode` reads dict-value offsets in a ptype-uniform loop: the I32 fast path eliminates the per-row `switch(dictValOffPType)` that blocked C2 vectorization (introduced by #215). ([#243](https://github.com/dfa1/vortex-java/issues/243))
 - `ConstantEncodingDecoder` now returns `NullArray` for null-scalar constants (proto `null_value` tag); previously decoded as `0`/`false`, silently corrupting nullable columns that the Rust writer encodes as an all-null constant. ([#246](https://github.com/dfa1/vortex-java/issues/246))

integration/src/test/java/io/github/dfa1/vortex/integration/RaincloudConformanceIntegrationTest.java

@@ -21,7 +21,9 @@
 import java.io.Writer;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Stream;
@@ -204,7 +206,16 @@ private static void writeOracleCsv(Path parquet, Writer out) throws IOException
              CsvWriter csv = CsvWriter.builder().fieldSeparator(',').build(out)) {
 
             List<ColumnSchema> cols = pfr.getFileSchema().getColumns();
-            csv.writeRecord(cols.stream().map(ColumnSchema::name).toList());
+            // De-duplicate duplicate column names with the Rust Vortex writer's algorithm:
+            // the Nth (N >= 1) occurrence of a base name gets a " [N]" suffix, matching
+            // the de-duplicated names in the Vortex file (#256).
+            Map<String, Integer> seen = new LinkedHashMap<>();
+            List<String> header = new ArrayList<>(cols.size());
+            for (ColumnSchema col : cols) {
+                int count = seen.merge(col.name(), 1, Integer::sum) - 1;
+                header.add(count == 0 ? col.name() : col.name() + " [" + count + "]");
+            }
+            csv.writeRecord(header);
 
             String[] row = new String[cols.size()];
             while (rows.hasNext()) {
@@ -221,6 +232,9 @@ private static void writeOracleCsv(Path parquet, Writer out) throws IOException
     /// null rows export as an empty field, valid rows use the JDK canonical
     /// `toString` of the value.
     ///
+    /// Row access uses column index rather than name so that files with duplicate
+    /// column names (#256) read the right column.
+    ///
     /// INT32/INT64 columns with a `UINT_32`/`UINT_64` logical-type annotation are treated
     /// as unsigned so their string representation matches the U32/U64 Vortex columns that
     /// carry the same bits (#253).
@@ -229,22 +243,22 @@ private static void writeOracleCsv(Path parquet, Writer out) throws IOException
     /// @param rows the row reader positioned at the current row
     /// @return the formatted cell string
     private static String oracleCell(ColumnSchema col, RowReader rows) {
-        String name = col.name();
-        if (col.repetitionType() == RepetitionType.OPTIONAL && rows.isNull(name)) {
+        int idx = col.columnIndex();
+        if (col.repetitionType() == RepetitionType.OPTIONAL && rows.isNull(idx)) {
             return "";
         }
         boolean unsignedInt = col.logicalType() instanceof LogicalType.IntType lt && !lt.isSigned();
         return switch (col.type()) {
-            case INT32 -> unsignedInt ? Integer.toUnsignedString(rows.getInt(name)) : Integer.toString(rows.getInt(name));
-            case INT64 -> unsignedInt ? Long.toUnsignedString(rows.getLong(name)) : Long.toString(rows.getLong(name));
-            case FLOAT -> Float.toString(rows.getFloat(name));
-            case DOUBLE -> Double.toString(rows.getDouble(name));
-            case BOOLEAN -> Boolean.toString(rows.getBoolean(name));
-            case BYTE_ARRAY -> rows.getString(name);
+            case INT32 -> unsignedInt ? Integer.toUnsignedString(rows.getInt(idx)) : Integer.toString(rows.getInt(idx));
+            case INT64 -> unsignedInt ? Long.toUnsignedString(rows.getLong(idx)) : Long.toString(rows.getLong(idx));
+            case FLOAT -> Float.toString(rows.getFloat(idx));
+            case DOUBLE -> Double.toString(rows.getDouble(idx));
+            case BOOLEAN -> Boolean.toString(rows.getBoolean(idx));
+            case BYTE_ARRAY -> rows.getString(idx);
             // aborts (not fails) the slug: the oracle can't format this physical type
             // yet, which is an oracle limitation rather than a vortex-java gap
             default -> throw new TestAbortedException(
-                    "oracle cannot format parquet type " + col.type() + " (column: " + name + ")");
+                    "oracle cannot format parquet type " + col.type() + " (column: " + col.name() + ")");
         };
     }
 

integration/src/test/resources/raincloud/expected-status.csv

@@ -223,14 +223,14 @@ uci-online-retail,ok
 uci-online-retail-ii,ok
 uci-online-shoppers-purchasing-intention,ok
 uci-optical-recognition-of-handwritten-digits,ok
-uci-parkinsons,gap:256
+uci-parkinsons,ok
 uci-phishing-websites,ok
 uci-predict-students-dropout-and-academic-success,ok
 uci-real-estate-valuation-data-set,ok
 uci-seeds,ok
 uci-seoul-bike-sharing-demand,ok
 uci-sms-spam-collection,ok
-uci-spambase,gap:256
+uci-spambase,ok
 uci-statlog-german-credit-data,ok
 uci-student-performance,ok
 uci-thyroid-disease,ok