Skip to content

Commit b65937d

Browse files
dfa1claude
dfa1
and
claude
committed
test: add unit tests for FsstEncoding, PcoEncoding, VarBinEncoding
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 05afa2c commit b65937d

3 files changed

Lines changed: 426 additions & 0 deletions

File tree

‎core/src/test/java/io/github/dfa1/vortex/encoding/FsstEncodingTest.java‎

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
import io.github.dfa1.vortex.core.PType;
5+
import io.github.dfa1.vortex.core.VortexException;
6+
import io.github.dfa1.vortex.core.array.VarBinArray;
7+
import io.github.dfa1.vortex.proto.DTypeProtos;
8+
import io.github.dfa1.vortex.proto.EncodingProtos;
9+
import org.junit.jupiter.api.Nested;
10+
import org.junit.jupiter.api.Test;
11+
12+
import java.lang.foreign.Arena;
13+
import java.lang.foreign.MemorySegment;
14+
import java.lang.foreign.ValueLayout;
15+
import java.nio.ByteBuffer;
16+
import java.nio.charset.StandardCharsets;
17+
18+
import static org.assertj.core.api.Assertions.assertThat;
19+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
20+
21+
class FsstEncodingTest {
22+
23+
private static final DType UTF8 = new DType.Utf8(false);
24+
25+
@Nested
26+
class Encode {
27+
28+
@Test
29+
void encode_throwsUnsupportedOperationException() {
30+
// Given
31+
var sut = new FsstEncoding();
32+
33+
// When / Then
34+
assertThatThrownBy(() -> sut.encode(UTF8, new String[]{"hello"}))
35+
.isInstanceOf(UnsupportedOperationException.class);
36+
}
37+
}
38+
39+
@Nested
40+
class Decode {
41+
42+
@Test
43+
void decode_singleByteSymbol_decompressesCorrectly() {
44+
// Given: symbol 0 = 'A' (LE u64 = 0x41, length 1); string "AA" → codes [0, 0]
45+
var sut = new FsstEncoding();
46+
DecodeContext ctx = buildCtx(
47+
new long[]{0x41L},
48+
new byte[]{1},
49+
new byte[]{0x00, 0x00},
50+
new int[]{2},
51+
new int[]{0, 2},
52+
1
53+
);
54+
55+
// When
56+
var result = sut.decode(ctx);
57+
58+
// Then
59+
assertThat(result).isInstanceOf(VarBinArray.class);
60+
VarBinArray vba = (VarBinArray) result;
61+
assertThat(vba.length()).isEqualTo(1);
62+
assertThat(vba.getBytes(0)).isEqualTo("AA".getBytes(StandardCharsets.UTF_8));
63+
}
64+
65+
@Test
66+
void decode_escapeByte_decompressesCorrectly() {
67+
// Given: no symbols; string "XY" → ESCAPE 'X' ESCAPE 'Y'
68+
var sut = new FsstEncoding();
69+
DecodeContext ctx = buildCtx(
70+
new long[0],
71+
new byte[0],
72+
new byte[]{(byte) 0xFF, 0x58, (byte) 0xFF, 0x59},
73+
new int[]{2},
74+
new int[]{0, 4},
75+
1
76+
);
77+
78+
// When
79+
var result = sut.decode(ctx);
80+
81+
// Then
82+
assertThat(result).isInstanceOf(VarBinArray.class);
83+
VarBinArray vba = (VarBinArray) result;
84+
assertThat(vba.length()).isEqualTo(1);
85+
assertThat(vba.getBytes(0)).isEqualTo("XY".getBytes(StandardCharsets.UTF_8));
86+
}
87+
88+
@Test
89+
void decode_multiByteSymbol_decompressesCorrectly() {
90+
// Given: symbol 0 = "ab" (LE u64 = 0x6261, length 2); string "ab" → code [0]
91+
var sut = new FsstEncoding();
92+
DecodeContext ctx = buildCtx(
93+
new long[]{0x6261L},
94+
new byte[]{2},
95+
new byte[]{0x00},
96+
new int[]{2},
97+
new int[]{0, 1},
98+
1
99+
);
100+
101+
// When
102+
var result = sut.decode(ctx);
103+
104+
// Then
105+
assertThat(result).isInstanceOf(VarBinArray.class);
106+
VarBinArray vba = (VarBinArray) result;
107+
assertThat(vba.length()).isEqualTo(1);
108+
assertThat(vba.getBytes(0)).isEqualTo("ab".getBytes(StandardCharsets.UTF_8));
109+
}
110+
111+
@Test
112+
void decode_multipleStrings_decompressesAll() {
113+
// Given: symbol 0 = 'H'; strings ["H", "HH", "!"] where "!" uses ESCAPE
114+
// compressed: [0x00] | [0x00, 0x00] | [0xFF, 0x21]
115+
var sut = new FsstEncoding();
116+
DecodeContext ctx = buildCtx(
117+
new long[]{0x48L},
118+
new byte[]{1},
119+
new byte[]{0x00, 0x00, 0x00, (byte) 0xFF, 0x21},
120+
new int[]{1, 2, 1},
121+
new int[]{0, 1, 3, 5},
122+
3
123+
);
124+
125+
// When
126+
var result = sut.decode(ctx);
127+
128+
// Then
129+
assertThat(result).isInstanceOf(VarBinArray.class);
130+
VarBinArray vba = (VarBinArray) result;
131+
assertThat(vba.length()).isEqualTo(3);
132+
assertThat(vba.getBytes(0)).isEqualTo("H".getBytes(StandardCharsets.UTF_8));
133+
assertThat(vba.getBytes(1)).isEqualTo("HH".getBytes(StandardCharsets.UTF_8));
134+
assertThat(vba.getBytes(2)).isEqualTo("!".getBytes(StandardCharsets.UTF_8));
135+
}
136+
137+
@Test
138+
void decode_missingMetadata_throwsVortexException() {
139+
// Given
140+
var sut = new FsstEncoding();
141+
ArrayNode node = new ArrayNode(EncodingId.VORTEX_FSST, null, new ArrayNode[0], new int[0], null);
142+
DecodeContext ctx = new DecodeContext(node, UTF8, 0, new MemorySegment[0],
143+
buildRegistry(), Arena.ofAuto());
144+
145+
// When / Then
146+
assertThatThrownBy(() -> sut.decode(ctx))
147+
.isInstanceOf(VortexException.class);
148+
}
149+
150+
private static DecodeContext buildCtx(
151+
long[] symbols, byte[] symLens, byte[] compressed,
152+
int[] uncompLens, int[] codesOffsets, long n
153+
) {
154+
Arena arena = Arena.ofAuto();
155+
156+
// Buffer 0: symbol table (8 bytes per symbol, LE u64)
157+
MemorySegment symBuf = arena.allocate(Math.max(symbols.length * 8L, 1), 8);
158+
for (int i = 0; i < symbols.length; i++) {
159+
symBuf.setAtIndex(PTypeIO.LE_LONG, i, symbols[i]);
160+
}
161+
162+
// Buffer 1: symbol lengths (1 byte per symbol)
163+
MemorySegment symLenBuf = arena.allocate(Math.max(symLens.length, 1));
164+
for (int i = 0; i < symLens.length; i++) {
165+
symLenBuf.set(ValueLayout.JAVA_BYTE, i, symLens[i]);
166+
}
167+
168+
// Buffer 2: compressed bytes
169+
MemorySegment compBuf = arena.allocate(Math.max(compressed.length, 1));
170+
for (int i = 0; i < compressed.length; i++) {
171+
compBuf.set(ValueLayout.JAVA_BYTE, i, compressed[i]);
172+
}
173+
174+
// Buffer 3: uncompressed_lengths (I32)
175+
MemorySegment uncompLenBuf = arena.allocate((long) uncompLens.length * Integer.BYTES, Integer.BYTES);
176+
for (int i = 0; i < uncompLens.length; i++) {
177+
uncompLenBuf.setAtIndex(PTypeIO.LE_INT, i, uncompLens[i]);
178+
}
179+
180+
// Buffer 4: codes_offsets (I32, n+1 elements)
181+
MemorySegment codesOffBuf = arena.allocate((long) codesOffsets.length * Integer.BYTES, Integer.BYTES);
182+
for (int i = 0; i < codesOffsets.length; i++) {
183+
codesOffBuf.setAtIndex(PTypeIO.LE_INT, i, codesOffsets[i]);
184+
}
185+
186+
MemorySegment[] segs = {symBuf, symLenBuf, compBuf, uncompLenBuf, codesOffBuf};
187+
188+
byte[] metaBytes = EncodingProtos.FSSTMetadata.newBuilder()
189+
.setUncompressedLengthsPtype(DTypeProtos.PType.forNumber(PType.I32.ordinal()))
190+
.setCodesOffsetsPtype(DTypeProtos.PType.forNumber(PType.I32.ordinal()))
191+
.build().toByteArray();
192+
193+
ArrayNode uncompLensNode = new ArrayNode(
194+
EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{3}, null);
195+
ArrayNode codesOffNode = new ArrayNode(
196+
EncodingId.VORTEX_PRIMITIVE, null, new ArrayNode[0], new int[]{4}, null);
197+
ArrayNode root = new ArrayNode(
198+
EncodingId.VORTEX_FSST, ByteBuffer.wrap(metaBytes),
199+
new ArrayNode[]{uncompLensNode, codesOffNode}, new int[]{0, 1, 2}, null);
200+
201+
return new DecodeContext(root, UTF8, n, segs, buildRegistry(), arena);
202+
}
203+
204+
private static EncodingRegistry buildRegistry() {
205+
EncodingRegistry registry = EncodingRegistry.empty();
206+
registry.register(new PrimitiveEncoding());
207+
return registry;
208+
}
209+
}
210+
}

‎core/src/test/java/io/github/dfa1/vortex/encoding/PcoEncodingTest.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package io.github.dfa1.vortex.encoding;
2+
3+
import io.github.dfa1.vortex.core.DType;
4+
import io.github.dfa1.vortex.core.PType;
5+
import io.github.dfa1.vortex.core.VortexException;
6+
import org.junit.jupiter.api.Nested;
7+
import org.junit.jupiter.api.Test;
8+
9+
import java.lang.foreign.Arena;
10+
import java.lang.foreign.MemorySegment;
11+
12+
import static org.assertj.core.api.Assertions.assertThat;
13+
import static org.assertj.core.api.Assertions.assertThatThrownBy;
14+
15+
class PcoEncodingTest {
16+
17+
@Nested
18+
class Encode {
19+
20+
@Test
21+
void encodingId_isVortexPco() {
22+
// Given / When / Then
23+
assertThat(new PcoEncoding().encodingId()).isEqualTo(EncodingId.VORTEX_PCO);
24+
}
25+
26+
@Test
27+
void encode_throwsVortexException() {
28+
// Given
29+
var sut = new PcoEncoding();
30+
DType dtype = new DType.Primitive(PType.I64, false);
31+
32+
// When / Then
33+
assertThatThrownBy(() -> sut.encode(dtype, new long[]{1L, 2L, 3L}))
34+
.isInstanceOf(VortexException.class)
35+
.hasMessageContaining("not implemented");
36+
}
37+
}
38+
39+
@Nested
40+
class Decode {
41+
42+
@Test
43+
void decode_throwsVortexException() {
44+
// Given
45+
var sut = new PcoEncoding();
46+
DType dtype = new DType.Primitive(PType.I64, false);
47+
ArrayNode node = new ArrayNode(EncodingId.VORTEX_PCO, null, new ArrayNode[0], new int[0], null);
48+
DecodeContext ctx = new DecodeContext(node, dtype, 3, new MemorySegment[0],
49+
EncodingRegistry.empty(), Arena.ofAuto());
50+
51+
// When / Then
52+
assertThatThrownBy(() -> sut.decode(ctx))
53+
.isInstanceOf(VortexException.class)
54+
.hasMessageContaining("not implemented");
55+
}
56+
}
57+
}

0 commit comments

Comments
 (0)