Skip to content

Commit 776843a

Browse files
dfa1claude
dfa1
and
claude
committed
fix: CsvExporter + VortexWriter support for FixedSizeList and List columns (#257)
- CsvExporter: add FixedSizeListArray/ListArray arms to cellValue() and jsonValue(); render as JSON arrays [v0,v1,...]. Add jsonArray() and offsetAt() helpers (I64/I32 offsets). - VortexWriter: register ListEncodingEncoder in DEFAULT_CODECS and the cascade codec list; FixedSizeListEncodingEncoder was already present but List was missing, making DType.List columns unwritable by default. - ListEncodingEncoder: add StructEncodingEncoder, FixedSizeListEncodingEncoder, and ListEncodingEncoder itself to FALLBACK so List[Struct], List[FixedSizeList], and List[List] element types resolve. Fix int overflow in offset write loop (for (long i = ...) instead of int). - MaskedEncodingEncoder: add ListEncodingEncoder to INNER_FALLBACK so nullable List columns encode on the flat (non-cascade) path. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 1dbc67d commit 776843a

6 files changed

Lines changed: 167 additions & 5 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
77

88
## [Unreleased]
99

10+
### Added
11+
12+
- `CsvExporter` now renders `FixedSizeList` and `List` columns as JSON array cells (`[v0,v1,...]`), matching the same element-rendering rules used for nested struct fields. Fixes export of Raincloud slugs with vector or list columns (e.g. `glove-*`, `osm-germany-relations`).
13+
- `VortexWriter` now encodes `vortex.list` columns by default (both flat and cascade paths). Previously only `vortex.fixed_size_list` was registered; `DType.List` columns threw at write time unless a custom registry was supplied.
14+
1015
### Fixed
1116

1217
- Nullable Utf8/Binary columns are now compressed through the full cascade instead of stored as raw `vortex.varbin`. `MaskedEncodingEncoder` previously encoded its non-null values with a fixed first-match encoder (primitive/varbin), so a nullable low-cardinality string column never reached Dict or FSST — producing files 10–47× larger than the Rust reference on the string-heavy Raincloud corpus (e.g. `uci-mushroom` 22×, `uci-online-retail-ii` 25×). The masked values now run through the same `CascadingCompressor` as dense columns, dropping those ratios to ~2–3×. ([#258](https://github.com/dfa1/vortex-java/issues/258))

‎csv/src/main/java/io/github/dfa1/vortex/csv/CsvExporter.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,10 @@
88
import io.github.dfa1.vortex.reader.array.BoolArray;
99
import io.github.dfa1.vortex.reader.array.ByteArray;
1010
import io.github.dfa1.vortex.reader.array.DoubleArray;
11+
import io.github.dfa1.vortex.reader.array.FixedSizeListArray;
1112
import io.github.dfa1.vortex.reader.array.FloatArray;
1213
import io.github.dfa1.vortex.reader.array.IntArray;
14+
import io.github.dfa1.vortex.reader.array.ListArray;
1315
import io.github.dfa1.vortex.reader.array.LongArray;
1416
import io.github.dfa1.vortex.reader.array.ShortArray;
1517
import io.github.dfa1.vortex.reader.array.MaskedArray;
@@ -153,6 +155,10 @@ private static void writeChunk(Chunk chunk, List<String> colNames, int colCount,
153155
/// (or null nested row) becomes a JSON `null`. Only the JSON layer is escaped here; the CSV
154156
/// writer independently quotes any cell containing the delimiter or a quote character.
155157
///
158+
/// A fixed-size list column ([FixedSizeListArray]) or variable-length list column ([ListArray]),
159+
/// either possibly wrapped in a [MaskedArray] when nullable, renders as a JSON array cell
160+
/// `[v0,v1,...]` with elements following the same rules as [#jsonValue(Array, long)].
161+
///
156162
/// @param arr the column array to read from
157163
/// @param rowIdx the zero-based row index within `arr`
158164
/// @return the rendered cell text
@@ -177,6 +183,12 @@ private static String cellValue(Array arr, long rowIdx) {
177183
// Nullable columns decode as MaskedArray: null rows export as an empty field, valid
178184
// rows defer to the inner values array.
179185
case MaskedArray ma -> ma.isValid(rowIdx) ? cellValue(ma.inner(), rowIdx) : "";
186+
case FixedSizeListArray fla -> jsonArray(fla.elements(), rowIdx * fla.fixedSize(), (rowIdx + 1L) * fla.fixedSize());
187+
case ListArray la -> {
188+
long start = offsetAt(la.offsets(), rowIdx);
189+
long end = offsetAt(la.offsets(), rowIdx + 1);
190+
yield jsonArray(la.elements(), start, end);
191+
}
180192
// All-null columns (DType.Null) hold only a row count: every cell is an empty
181193
// field, same rule as a MaskedArray null row.
182194
case NullArray ignored -> "";
@@ -215,6 +227,12 @@ private static String jsonValue(Array arr, long rowIdx) {
215227
return switch (arr) {
216228
case StructArray sa -> jsonObject(sa, rowIdx);
217229
case MaskedArray ma -> ma.isValid(rowIdx) ? jsonValue(ma.inner(), rowIdx) : "null";
230+
case FixedSizeListArray fla -> jsonArray(fla.elements(), rowIdx * fla.fixedSize(), (rowIdx + 1L) * fla.fixedSize());
231+
case ListArray la -> {
232+
long start = offsetAt(la.offsets(), rowIdx);
233+
long end = offsetAt(la.offsets(), rowIdx + 1);
234+
yield jsonArray(la.elements(), start, end);
235+
}
218236
case NullArray ignored -> "null";
219237
case VarBinArray va -> {
220238
StringBuilder sb = new StringBuilder();
@@ -253,6 +271,39 @@ private static void jsonString(StringBuilder sb, String value) {
253271
sb.append('"');
254272
}
255273

274+
/// Renders elements `[start, end)` of `elements` as a JSON array `[v0,v1,...]`,
275+
/// with each element rendered by [#jsonValue(Array, long)].
276+
///
277+
/// @param elements the flat elements array
278+
/// @param start inclusive start index
279+
/// @param end exclusive end index
280+
/// @return the rendered JSON array string
281+
private static String jsonArray(Array elements, long start, long end) {
282+
StringBuilder sb = new StringBuilder("[");
283+
for (long i = start; i < end; i++) {
284+
if (i > start) {
285+
sb.append(',');
286+
}
287+
sb.append(jsonValue(elements, i));
288+
}
289+
sb.append(']');
290+
return sb.toString();
291+
}
292+
293+
/// Reads offset `idx` from `offsets` as a non-negative long.
294+
/// The encoder always writes I64 offsets; I32 is included for forward compatibility.
295+
///
296+
/// @param offsets the offsets array
297+
/// @param idx the index to read
298+
/// @return the offset value as a non-negative `long`
299+
private static long offsetAt(Array offsets, long idx) {
300+
return switch (offsets) {
301+
case LongArray la -> la.getLong(idx);
302+
case IntArray ia -> Integer.toUnsignedLong(ia.getInt(idx));
303+
default -> throw new VortexException("unexpected list offsets type: " + offsets.getClass().getSimpleName());
304+
};
305+
}
306+
256307
/// Whether the array's dtype is an unsigned integer, so high-half values must render as
257308
/// unsigned decimal rather than the two's-complement negative that the raw signed getter
258309
/// returns. Callers only pass leaf value arrays: the [MaskedArray] arm of `cellValue`

‎csv/src/test/java/io/github/dfa1/vortex/csv/CsvExporterTest.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
import io.github.dfa1.vortex.writer.VortexWriter;
77
import io.github.dfa1.vortex.writer.WriteOptions;
88
import io.github.dfa1.vortex.writer.encode.BoolEncodingEncoder;
9+
import io.github.dfa1.vortex.writer.encode.FixedSizeListData;
10+
import io.github.dfa1.vortex.writer.encode.ListData;
911
import io.github.dfa1.vortex.writer.encode.MaskedEncodingEncoder;
1012
import io.github.dfa1.vortex.writer.encode.NullEncodingEncoder;
1113
import io.github.dfa1.vortex.writer.encode.NullableData;
@@ -420,4 +422,103 @@ void rendersStructInStructAsNestedJsonObject(@TempDir Path tmp) throws Exception
420422
"region,data",
421423
"north,\"{\"\"name\"\":\"\"origin\"\",\"\"coord\"\":{\"\"lat\"\":1.0,\"\"lon\"\":2.0}}\"");
422424
}
425+
426+
@Test
427+
void rendersFixedSizeListColumnAsJsonArray(@TempDir Path tmp) throws Exception {
428+
// Given a FixedSizeList[F32, 3] column — the glove vector shape (issue #257);
429+
// each row is 3 consecutive floats from the flat elements array.
430+
Path vortex = tmp.resolve("fsl.vortex");
431+
DType.FixedSizeList vecDtype = new DType.FixedSizeList(DType.F32, 3, false);
432+
DType.Struct schema = new DType.Struct(
433+
List.of(ColumnName.of("label"), ColumnName.of("vec")),
434+
List.of(DType.UTF8, vecDtype),
435+
false);
436+
try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
437+
VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) {
438+
writer.writeChunk(Map.of(
439+
ColumnName.of("label"), new String[]{"a", "b"},
440+
ColumnName.of("vec"), new FixedSizeListData(new float[]{1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f}, 2)));
441+
}
442+
Path csv = tmp.resolve("out.csv");
443+
444+
// When
445+
CsvExporter.exportCsv(vortex, csv);
446+
List<String> result = Files.readAllLines(csv);
447+
448+
// Then each row renders as a JSON float array in element order. The CSV layer quotes the
449+
// whole cell because the JSON array contains commas.
450+
assertThat(result).containsExactly(
451+
"label,vec",
452+
"a,\"[1.0,2.0,3.0]\"",
453+
"b,\"[4.0,5.0,6.0]\"");
454+
}
455+
456+
@Test
457+
void rendersListColumnAsJsonArray(@TempDir Path tmp) throws Exception {
458+
// Given a List[I64] column with variable-length rows — tests both the offset
459+
// reading path and the empty-list edge case (issue #257).
460+
Path vortex = tmp.resolve("list.vortex");
461+
DType.List listDtype = new DType.List(DType.I64, false);
462+
DType.Struct schema = new DType.Struct(
463+
List.of(ColumnName.of("tag"), ColumnName.of("ids")),
464+
List.of(DType.UTF8, listDtype),
465+
false);
466+
try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
467+
VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) {
468+
writer.writeChunk(Map.of(
469+
ColumnName.of("tag"), new String[]{"a", "b", "c"},
470+
ColumnName.of("ids"), new ListData(
471+
new long[]{10L, 20L, 30L},
472+
new long[]{0, 2, 3, 3}, // row 0 → [10,20], row 1 → [30], row 2 → []
473+
3)));
474+
}
475+
Path csv = tmp.resolve("out.csv");
476+
477+
// When
478+
CsvExporter.exportCsv(vortex, csv);
479+
List<String> result = Files.readAllLines(csv);
480+
481+
// Then variable lengths and empty list both render correctly. The CSV layer quotes only the
482+
// multi-element cell because its JSON array contains a comma.
483+
assertThat(result).containsExactly(
484+
"tag,ids",
485+
"a,\"[10,20]\"",
486+
"b,[30]",
487+
"c,[]");
488+
}
489+
490+
@Test
491+
void rendersListOfStructsAsJsonArrayOfObjects(@TempDir Path tmp) throws Exception {
492+
// Given a List[Struct] column — the osm-germany `members` shape (issue #257);
493+
// struct elements inside the list must recurse through jsonValue → jsonObject.
494+
Path vortex = tmp.resolve("listofstruct.vortex");
495+
DType.Struct memberDtype = new DType.Struct(
496+
List.of(ColumnName.of("ref"), ColumnName.of("role")),
497+
List.of(DType.I64, DType.UTF8),
498+
false);
499+
DType.List listDtype = new DType.List(memberDtype, false);
500+
DType.Struct schema = new DType.Struct(
501+
List.of(ColumnName.of("id"), ColumnName.of("members")),
502+
List.of(DType.I64, listDtype),
503+
false);
504+
try (FileChannel ch = FileChannel.open(vortex, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
505+
VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.cascading(3))) {
506+
writer.writeChunk(Map.of(
507+
ColumnName.of("id"), new long[]{1L},
508+
ColumnName.of("members"), new ListData(
509+
new StructData(List.of(new long[]{10L, 20L}, new String[]{"", "outer"})),
510+
new long[]{0, 2},
511+
1)));
512+
}
513+
Path csv = tmp.resolve("out.csv");
514+
515+
// When
516+
CsvExporter.exportCsv(vortex, csv);
517+
List<String> result = Files.readAllLines(csv);
518+
519+
// Then each list element renders as a JSON object; the array wraps them
520+
assertThat(result).containsExactly(
521+
"id,members",
522+
"1,\"[{\"\"ref\"\":10,\"\"role\"\":\"\"\"\"},{\"\"ref\"\":20,\"\"role\"\":\"\"outer\"\"}]\"");
523+
}
423524
}

‎writer/src/main/java/io/github/dfa1/vortex/writer/VortexWriter.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import io.github.dfa1.vortex.writer.encode.FixedSizeListEncodingEncoder;
3535
import io.github.dfa1.vortex.writer.encode.FrameOfReferenceEncodingEncoder;
3636
import io.github.dfa1.vortex.writer.encode.FsstEncodingEncoder;
37+
import io.github.dfa1.vortex.writer.encode.ListEncodingEncoder;
3738
import io.github.dfa1.vortex.writer.encode.MaskedEncodingEncoder;
3839
import io.github.dfa1.vortex.writer.encode.PrimitiveEncodingEncoder;
3940
import io.github.dfa1.vortex.writer.encode.RleEncodingEncoder;
@@ -101,7 +102,7 @@ public final class VortexWriter implements Closeable {
101102
private static final List<EncodingEncoder> DEFAULT_CODECS = List.of(
102103
new AlpEncodingEncoder(), new PrimitiveEncodingEncoder(), new BoolEncodingEncoder(),
103104
new DictEncodingEncoder(), new VarBinEncodingEncoder(), new ExtEncodingEncoder(),
104-
new FixedSizeListEncodingEncoder());
105+
new FixedSizeListEncodingEncoder(), new ListEncodingEncoder());
105106

106107
// Base cascade codec list — no Zstd. Zstd is appended (before PrimitiveEncoding) when
107108
// WriteOptions.enableZstd() is true. See WriteOptions.withZstd(boolean) for the tradeoff.
@@ -187,6 +188,7 @@ private static List<EncodingEncoder> buildCascadeCodecs(WriteOptions options) {
187188
codecs.add(new DateTimePartsEncodingEncoder());
188189
codecs.add(new ExtEncodingEncoder());
189190
codecs.add(new FixedSizeListEncodingEncoder());
191+
codecs.add(new ListEncodingEncoder());
190192
codecs.add(new ConstantEncodingEncoder());
191193
codecs.add(new AlpEncodingEncoder());
192194
codecs.add(new FrameOfReferenceEncodingEncoder());

‎writer/src/main/java/io/github/dfa1/vortex/writer/encode/ListEncodingEncoder.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616
/// Write-only encoder for `vortex.list`.
1717
public final class ListEncodingEncoder implements EncodingEncoder {
1818

19+
// Includes container encoders so nested element types (List[Struct], List[List], List[FixedSizeList]) resolve.
1920
private static final List<EncodingEncoder> FALLBACK = List.of(
2021
new PrimitiveEncodingEncoder(), new VarBinEncodingEncoder(), new BoolEncodingEncoder(),
21-
new NullEncodingEncoder(), new ByteBoolEncodingEncoder());
22+
new NullEncodingEncoder(), new ByteBoolEncodingEncoder(), new StructEncodingEncoder(),
23+
new FixedSizeListEncodingEncoder(), new ListEncodingEncoder());
2224

2325
@Override
2426
public EncodingId encodingId() {
@@ -44,8 +46,8 @@ public EncodeResult encode(DType dtype, Object data, EncodeContext ctx) {
4446

4547
long nOffsets = ld.outerLen() + 1;
4648
MemorySegment offsetsBuf = ctx.arena().allocate(nOffsets * Long.BYTES, Long.BYTES);
47-
for (int i = 0; i < nOffsets; i++) {
48-
offsetsBuf.setAtIndex(VortexFormat.LE_LONG, i, ld.offsets()[i]);
49+
for (long i = 0; i < nOffsets; i++) {
50+
offsetsBuf.setAtIndex(VortexFormat.LE_LONG, i, ld.offsets()[(int) i]);
4951
}
5052
allBuffers.add(offsetsBuf);
5153
EncodeNode offsetsNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, elemBufCount);

‎writer/src/main/java/io/github/dfa1/vortex/writer/encode/MaskedEncodingEncoder.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ public final class MaskedEncodingEncoder implements EncodingEncoder {
2222
private static final List<EncodingEncoder> INNER_FALLBACK = List.of(
2323
new PrimitiveEncodingEncoder(),
2424
new VarBinEncodingEncoder(),
25-
new FixedSizeListEncodingEncoder());
25+
new FixedSizeListEncodingEncoder(),
26+
new ListEncodingEncoder());
2627

2728
@Override
2829
public EncodingId encodingId() {

0 commit comments

Comments
 (0)