Skip to content

Commit 1b3ea20

Browse files
dfa1claude
dfa1
and
claude
committed
test(writer): kill behavioral VortexWriter survivors; parameterize dict-decision tests
Two more genuinely-behavioral kills: - isUtf8DictCandidate cardinality-at-MAX boundary (`seen.size() > MAX` must not trip at exactly MAX). - rowCountForValidation on a Collection column: a two-column chunk (List<LocalDate> extension + same-length long[]) is accepted, pinning that the collection is measured by element count, not a constant. Also restructure VortexWriterDictDecisionTest into @ParameterizedTest with proper Given/When/Then: case params are the Given, `result = helper(...)` the When, the assertion the Then. VortexWriter 89% killed (strength 92%). The remaining survivors are encoding-choice mutants that round-trip identically (dict-vs-fallback, frequency sort, zstd codec list), 64-byte segment-alignment math, dead-by-exclusion ptype switch arms, or the impractical U32 code path — not correctness gaps. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 83d3436 commit 1b3ea20

2 files changed

Lines changed: 141 additions & 79 deletions

File tree

‎writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterDictDecisionTest.java‎

Lines changed: 107 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,15 @@
11
package io.github.dfa1.vortex.writer;
22

33
import io.github.dfa1.vortex.core.PType;
4-
import org.junit.jupiter.api.Test;
4+
import org.junit.jupiter.params.ParameterizedTest;
5+
import org.junit.jupiter.params.provider.Arguments;
6+
import org.junit.jupiter.params.provider.MethodSource;
7+
8+
import java.util.stream.Stream;
59

610
import static io.github.dfa1.vortex.writer.VortexWriter.GLOBAL_DICT_MAX_CARDINALITY;
711
import static org.assertj.core.api.Assertions.assertThat;
12+
import static org.junit.jupiter.params.provider.Arguments.arguments;
813

914
/// Direct unit tests for the global-dictionary decision helpers in [VortexWriter]. These choices
1015
/// (dict vs cascade fallback, and the code width) only affect *encoding*, not the values a reader
@@ -14,107 +19,121 @@ class VortexWriterDictDecisionTest {
1419

1520
// ── isDictCandidate (primitive) ──────────────────────────────────────────────
1621

17-
@Test
18-
void isDictCandidate_excludesF16AndF32() {
19-
// Given low-cardinality float data that would otherwise qualify
20-
// When / Then — F16/F32 are excluded outright (ALP wins there)
21-
assertThat(VortexWriter.isDictCandidate(PType.F32, new float[]{1f, 1f, 2f, 2f, 2f})).isFalse();
22-
assertThat(VortexWriter.isDictCandidate(PType.F16, new short[]{1, 1, 2, 2, 2})).isFalse();
22+
static Stream<Arguments> dictCandidateCases() {
23+
return Stream.of(
24+
// exclusions: a U8/U16 code is no smaller than the value, and F16/F32 prefer ALP
25+
arguments("F32 excluded", PType.F32, new float[]{1f, 1f, 2f, 2f, 2f}, false),
26+
arguments("F16 excluded", PType.F16, new short[]{1, 1, 2, 2, 2}, false),
27+
arguments("I8 excluded", PType.I8, new byte[]{1, 2, 1, 2, 1}, false),
28+
arguments("U8 excluded", PType.U8, new byte[]{1, 2, 1, 2, 1}, false),
29+
arguments("I16 excluded", PType.I16, new short[]{1, 2, 1, 2, 1}, false),
30+
arguments("U16 excluded", PType.U16, new short[]{1, 2, 1, 2, 1}, false),
31+
// admitted carriers, 2 distinct over 5 rows (4 < 5, under the gate)
32+
arguments("I64 low cardinality", PType.I64, new long[]{1, 2, 1, 2, 1}, true),
33+
arguments("F64 low cardinality", PType.F64, new double[]{1, 2, 1, 2, 1}, true),
34+
arguments("empty", PType.I64, new long[0], false),
35+
arguments("single distinct value", PType.I64, new long[]{7, 7, 7, 7}, false),
36+
arguments("ratio exactly 50%", PType.I64, new long[]{1, 2, 1, 2}, false),
37+
arguments("ratio under 50%", PType.I64, new long[]{1, 2, 1, 2, 1}, true),
38+
arguments("cardinality at MAX", PType.I64, distinctThenRepeat(GLOBAL_DICT_MAX_CARDINALITY), true),
39+
arguments("cardinality over MAX", PType.I64, distinctThenRepeat(GLOBAL_DICT_MAX_CARDINALITY + 1), false));
2340
}
2441

25-
@Test
26-
void isDictCandidate_excludesNarrowIntegers() {
27-
// I8/U8/I16/U16 are excluded: a U8/U16 code is no smaller than the value, the Rust
28-
// compressor does not dict them, and the reader's lazy dict cannot decode a narrow-int
29-
// dictionary. Low-cardinality data that would otherwise pass the ratio gate must still
30-
// be rejected.
31-
assertThat(VortexWriter.isDictCandidate(PType.I8, new byte[]{1, 2, 1, 2, 1})).isFalse();
32-
assertThat(VortexWriter.isDictCandidate(PType.U8, new byte[]{1, 2, 1, 2, 1})).isFalse();
33-
assertThat(VortexWriter.isDictCandidate(PType.I16, new short[]{1, 2, 1, 2, 1})).isFalse();
34-
assertThat(VortexWriter.isDictCandidate(PType.U16, new short[]{1, 2, 1, 2, 1})).isFalse();
35-
}
42+
@ParameterizedTest(name = "{0}")
43+
@MethodSource("dictCandidateCases")
44+
void isDictCandidate(String name, PType ptype, Object data, boolean expected) {
45+
// Given — a column of `ptype` with the case's data
3646

37-
@Test
38-
void isDictCandidate_admitsLowCardinalityI64AndF64() {
39-
// Given — 2 distinct over 5 rows: 2*2 = 4 < 5, under the 50%-unique gate
40-
assertThat(VortexWriter.isDictCandidate(PType.I64, new long[]{1, 2, 1, 2, 1})).isTrue();
41-
assertThat(VortexWriter.isDictCandidate(PType.F64, new double[]{1, 2, 1, 2, 1})).isTrue();
42-
}
47+
// When
48+
boolean result = VortexWriter.isDictCandidate(ptype, data);
4349

44-
@Test
45-
void isDictCandidate_emptyArray_isFalse() {
46-
assertThat(VortexWriter.isDictCandidate(PType.I64, new long[0])).isFalse();
50+
// Then
51+
assertThat(result).isEqualTo(expected);
4752
}
4853

49-
@Test
50-
void isDictCandidate_singleDistinctValue_isFalse() {
51-
// One distinct value fits vortex.constant better than a dict
52-
assertThat(VortexWriter.isDictCandidate(PType.I64, new long[]{7, 7, 7, 7})).isFalse();
53-
}
54+
// ── isUtf8DictCandidate ──────────────────────────────────────────────────────
5455

55-
@Test
56-
void isDictCandidate_ratioGate_isExclusive() {
57-
// 2 distinct over 4 rows: 2*2 == 4, NOT < 4 → rejected (exactly 50% unique)
58-
assertThat(VortexWriter.isDictCandidate(PType.I64, new long[]{1, 2, 1, 2})).isFalse();
59-
// 2 distinct over 5 rows: 4 < 5 → admitted
60-
assertThat(VortexWriter.isDictCandidate(PType.I64, new long[]{1, 2, 1, 2, 1})).isTrue();
56+
static Stream<Arguments> utf8DictCandidateCases() {
57+
return Stream.of(
58+
arguments("empty", new String[0], false),
59+
arguments("ratio exactly 50%", new String[]{"a", "b", "a", "b"}, false),
60+
arguments("ratio under 50%", new String[]{"a", "b", "a", "b", "a"}, true),
61+
arguments("cardinality at MAX", distinctStrings(GLOBAL_DICT_MAX_CARDINALITY), true),
62+
arguments("cardinality over MAX", distinctStrings(GLOBAL_DICT_MAX_CARDINALITY + 1), false));
6163
}
6264

63-
@Test
64-
void isDictCandidate_cardinalityAtAndOverMax() {
65-
// At MAX distinct values (well under 50% unique) → still a candidate
66-
assertThat(VortexWriter.isDictCandidate(PType.I64, distinctThenRepeat(GLOBAL_DICT_MAX_CARDINALITY))).isTrue();
67-
// One over MAX → rejected by the cardinality guard
68-
assertThat(VortexWriter.isDictCandidate(PType.I64, distinctThenRepeat(GLOBAL_DICT_MAX_CARDINALITY + 1))).isFalse();
69-
}
65+
@ParameterizedTest(name = "{0}")
66+
@MethodSource("utf8DictCandidateCases")
67+
void isUtf8DictCandidate(String name, String[] data, boolean expected) {
68+
// Given — a string column with the case's data
7069

71-
// ── isUtf8DictCandidate ──────────────────────────────────────────────────────
70+
// When
71+
boolean result = VortexWriter.isUtf8DictCandidate(data);
7272

73-
@Test
74-
void isUtf8DictCandidate_emptyArray_isFalse() {
75-
assertThat(VortexWriter.isUtf8DictCandidate(new String[0])).isFalse();
73+
// Then
74+
assertThat(result).isEqualTo(expected);
7675
}
7776

78-
@Test
79-
void isUtf8DictCandidate_ratioGate_isExclusive() {
80-
// 2 distinct over 4: 4 == 4, not < → rejected
81-
assertThat(VortexWriter.isUtf8DictCandidate(new String[]{"a", "b", "a", "b"})).isFalse();
82-
// 2 distinct over 5: 4 < 5 → admitted
83-
assertThat(VortexWriter.isUtf8DictCandidate(new String[]{"a", "b", "a", "b", "a"})).isTrue();
84-
}
77+
// ── codePTypeForSize ─────────────────────────────────────────────────────────
8578

86-
@Test
87-
void isUtf8DictCandidate_overMaxCardinality_isFalse() {
88-
String[] data = new String[(GLOBAL_DICT_MAX_CARDINALITY + 1) * 4];
89-
for (int i = 0; i < data.length; i++) {
90-
data[i] = "s" + (i % (GLOBAL_DICT_MAX_CARDINALITY + 1));
91-
}
92-
assertThat(VortexWriter.isUtf8DictCandidate(data)).isFalse();
79+
static Stream<Arguments> codePTypeCases() {
80+
return Stream.of(
81+
arguments(1, PType.U8),
82+
arguments(256, PType.U8), // upper edge of U8
83+
arguments(257, PType.U16), // first U16
84+
arguments(65_536, PType.U16), // upper edge of U16
85+
arguments(65_537, PType.U32)); // first U32
9386
}
9487

95-
// ── codePTypeForSize ─────────────────────────────────────────────────────────
88+
@ParameterizedTest
89+
@MethodSource("codePTypeCases")
90+
void codePTypeForSize_picksNarrowestUnsignedCarrier(int dictSize, PType expected) {
91+
// Given — a dictionary of `dictSize` distinct values
92+
93+
// When
94+
PType result = VortexWriter.codePTypeForSize(dictSize);
9695

97-
@Test
98-
void codePTypeForSize_picksNarrowestUnsignedCarrier() {
99-
assertThat(VortexWriter.codePTypeForSize(1)).isEqualTo(PType.U8);
100-
assertThat(VortexWriter.codePTypeForSize(256)).isEqualTo(PType.U8); // upper edge of U8
101-
assertThat(VortexWriter.codePTypeForSize(257)).isEqualTo(PType.U16); // first U16
102-
assertThat(VortexWriter.codePTypeForSize(65_536)).isEqualTo(PType.U16); // upper edge of U16
103-
assertThat(VortexWriter.codePTypeForSize(65_537)).isEqualTo(PType.U32); // first U32
96+
// Then
97+
assertThat(result).isEqualTo(expected);
10498
}
10599

106100
// ── primitiveArrayLen / readPrimitiveElement ─────────────────────────────────
107101

108-
@Test
109-
void primitiveArrayLen_returnsActualLength() {
110-
assertThat(VortexWriter.primitiveArrayLen(new long[]{1, 2, 3}, PType.I64)).isEqualTo(3);
111-
assertThat(VortexWriter.primitiveArrayLen(new int[]{1, 2}, PType.I32)).isEqualTo(2);
102+
static Stream<Arguments> primitiveArrayLenCases() {
103+
return Stream.of(
104+
arguments(new long[]{1, 2, 3}, PType.I64, 3),
105+
arguments(new int[]{1, 2}, PType.I32, 2),
106+
arguments(new byte[]{1, 2, 3, 4}, PType.I8, 4));
112107
}
113108

114-
@Test
115-
void readPrimitiveElement_returnsElementAtIndex() {
116-
assertThat(VortexWriter.readPrimitiveElement(new long[]{7, 8, 9}, PType.I64, 1)).isEqualTo(8L);
117-
assertThat(VortexWriter.readPrimitiveElement(new int[]{7, 8, 9}, PType.I32, 2)).isEqualTo(9);
109+
@ParameterizedTest
110+
@MethodSource("primitiveArrayLenCases")
111+
void primitiveArrayLen_returnsActualLength(Object data, PType ptype, int expected) {
112+
// Given — a typed primitive array of `ptype`
113+
114+
// When
115+
int result = VortexWriter.primitiveArrayLen(data, ptype);
116+
117+
// Then
118+
assertThat(result).isEqualTo(expected);
119+
}
120+
121+
static Stream<Arguments> readPrimitiveElementCases() {
122+
return Stream.of(
123+
arguments(new long[]{7, 8, 9}, PType.I64, 1, 8L),
124+
arguments(new int[]{7, 8, 9}, PType.I32, 2, 9));
125+
}
126+
127+
@ParameterizedTest
128+
@MethodSource("readPrimitiveElementCases")
129+
void readPrimitiveElement_returnsElementAtIndex(Object data, PType ptype, int index, Object expected) {
130+
// Given — a typed primitive array of `ptype`
131+
132+
// When
133+
Object result = VortexWriter.readPrimitiveElement(data, ptype, index);
134+
135+
// Then
136+
assertThat(result).isEqualTo(expected);
118137
}
119138

120139
/// Builds a long[] with exactly `distinct` distinct values, each repeated 4× (so the array is
@@ -126,4 +145,13 @@ private static long[] distinctThenRepeat(int distinct) {
126145
}
127146
return a;
128147
}
148+
149+
/// Builds a String[] with exactly `distinct` distinct values, each repeated 4×.
150+
private static String[] distinctStrings(int distinct) {
151+
String[] a = new String[distinct * 4];
152+
for (int i = 0; i < a.length; i++) {
153+
a[i] = "s" + (i % distinct);
154+
}
155+
return a;
156+
}
129157
}

‎writer/src/test/java/io/github/dfa1/vortex/writer/VortexWriterTest.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,40 @@ void writeChunk_autoroutesExtensionCollectionViaSpecExtension(@TempDir Path tmp)
8686
}
8787
}
8888

89+
@Test
90+
void writeChunk_extensionCollectionColumn_rowCountValidatedAgainstSibling(@TempDir Path tmp)
91+
throws IOException {
92+
// Given — a two-column chunk where one column is a List<LocalDate> (extension auto-route)
93+
// and the sibling is a same-length long[]. The row-count check must measure the collection
94+
// by its element count: if it reported anything else, the two columns would look mismatched
95+
// and writeChunk would reject a perfectly valid chunk.
96+
var schema = new DType.Struct(
97+
List.of("birthdays", "id"),
98+
List.of(io.github.dfa1.vortex.writer.encode.DateExtensionEncoder.INSTANCE.dtype(false),
99+
new DType.Primitive(PType.I64, false)),
100+
false);
101+
List<java.time.LocalDate> dates = List.of(
102+
java.time.LocalDate.of(1996, 2, 12),
103+
java.time.LocalDate.of(2026, 6, 9),
104+
java.time.LocalDate.of(2030, 1, 1));
105+
long[] ids = {1L, 2L, 3L};
106+
Path file = tmp.resolve("ext_rowcount.vtx");
107+
108+
// When / Then — both columns are 3 rows, so the chunk is accepted and round-trips
109+
try (var ch = FileChannel.open(file, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
110+
var sut = VortexWriter.create(ch, schema, WriteOptions.defaults())) {
111+
sut.writeChunk(Map.of("birthdays", dates, "id", ids));
112+
}
113+
try (var vf = VortexReader.open(file, ReadRegistry.loadAll());
114+
var iter = vf.scan(ScanOptions.all())) {
115+
try (Chunk chunk = iter.next()) {
116+
assertThat(chunk.rowCount()).isEqualTo(3);
117+
assertThat(chunk.as("birthdays", java.time.LocalDate.class))
118+
.containsExactlyElementsOf(dates);
119+
}
120+
}
121+
}
122+
89123
@Test
90124
void writeChunk_map_nullablePrimitive_acceptsBoxedArray(@TempDir Path tmp) throws IOException {
91125
// Given — nullable I64 column passed to the MAP entry point as a boxed Long[] with a null.

0 commit comments

Comments
 (0)