merge: ALP F32 encode + DictEncoding for Utf8; resolves conflict with cascading work

- DictEncoding.encode/encodeCascade: dispatch Utf8 → encodeUtf8 (terminal)
- VortexWriter DEFAULT_CODECS: include DictEncoding for first-match path

File size: CSV=4.6MB → Vortex=3.3MB (1.40× ratio) vs 1.08× before Utf8 dict

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: dfa1

TODO.md

@@ -2,7 +2,6 @@
 
 ## Encodings
 
-- [ ] in one case F32 and F64 are accepted but only F64 is really implemented
 - [ ] the classes are very long of complex, most likely we should group the impl detail in private static inner classes Encoder/Decoder
 
 ## Performance

core/src/main/java/io/github/dfa1/vortex/encoding/AlpEncoding.java

@@ -99,10 +99,11 @@ public boolean accepts(DType dtype) {
 	@Override
 	public EncodeResult encode(DType dtype, Object data) {
 		PType ptype = ((DType.Primitive) dtype).ptype();
-		if (ptype == PType.F64) {
-			return encodeF64((double[]) data, dtype);
-		}
-		throw new UnsupportedOperationException("ALP encode not yet supported for " + ptype);
+		return switch (ptype) {
+			case F64 -> encodeF64((double[]) data, dtype);
+			case F32 -> encodeF32((float[]) data, dtype);
+			default -> throw new UnsupportedOperationException("ALP encode not supported for " + ptype);
+		};
 	}
 
 	@Override
@@ -274,6 +275,115 @@ private static byte[] scalarF64(double v) {
 		return ScalarProtos.ScalarValue.newBuilder().setF64Value(v).build().toByteArray();
 	}
 
+	// ── F32 encode ────────────────────────────────────────────────────────────
+
+	private static int[] findExponentsF32(float[] values) {
+		int sampleLen = Math.min(SAMPLE_SIZE, values.length);
+		int bestExpE = 0, bestExpF = 0, bestExceptions = sampleLen + 1;
+
+		outer:
+		for (int expE = 0; expE < F10_F32.length; expE++) {
+			for (int expF = 0; expF < F10_F32.length; expF++) {
+				float encFactor = F10_F32[expE] * IF10_F32[expF];
+				float decFactor = F10_F32[expF] * IF10_F32[expE];
+				int exceptions = 0;
+				for (int i = 0; i < sampleLen; i++) {
+					float enc = values[i] * encFactor;
+					if (!Float.isFinite(enc) || (float) Math.round(enc) * decFactor != values[i]) {
+						exceptions++;
+					}
+				}
+				if (exceptions < bestExceptions) {
+					bestExceptions = exceptions;
+					bestExpE = expE;
+					bestExpF = expF;
+					if (bestExceptions == 0) {
+						break outer;
+					}
+				}
+			}
+		}
+		return new int[]{bestExpE, bestExpF};
+	}
+
+	private static EncodeResult encodeF32(float[] values, DType dtype) {
+		int n = values.length;
+		int[] exps = findExponentsF32(values);
+		int expE = exps[0], expF = exps[1];
+		float encFactor = F10_F32[expE] * IF10_F32[expF];
+		float decFactor = F10_F32[expF] * IF10_F32[expE];
+
+		int[] encodedArr = new int[n];
+		var patchIndices = new ArrayList<Integer>();
+		var patchValues = new ArrayList<Float>();
+
+		float min = Float.MAX_VALUE, max = -Float.MAX_VALUE;
+		for (int i = 0; i < n; i++) {
+			float v = values[i];
+			float enc = v * encFactor;
+			int encoded;
+			if (Float.isFinite(enc) && (float) (encoded = Math.round(enc)) * decFactor == v) {
+				encodedArr[i] = encoded;
+			} else {
+				encodedArr[i] = 0;
+				patchIndices.add(i);
+				patchValues.add(v);
+			}
+			if (v < min) {
+				min = v;
+			}
+			if (v > max) {
+				max = v;
+			}
+		}
+
+		byte[] statsMin = n > 0 ? scalarF32(min) : null;
+		byte[] statsMax = n > 0 ? scalarF32(max) : null;
+
+		MemorySegment encodedBuf = Arena.ofAuto().allocate((long) n * 4, 4);
+		for (int i = 0; i < n; i++) {
+			encodedBuf.setAtIndex(PTypeIO.LE_INT, i, encodedArr[i]);
+		}
+
+		EncodeNode encodedNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 0);
+
+		if (patchIndices.isEmpty()) {
+			byte[] metaBytes = EncodingProtos.ALPMetadata.newBuilder()
+					.setExpE(expE).setExpF(expF).build().toByteArray();
+			EncodeNode root = new EncodeNode(EncodingId.VORTEX_ALP,
+					ByteBuffer.wrap(metaBytes), new EncodeNode[]{encodedNode}, new int[0]);
+			return new EncodeResult(root, List.of(encodedBuf), statsMin, statsMax);
+		}
+
+		int numPatches = patchIndices.size();
+		MemorySegment idxBuf = Arena.ofAuto().allocate((long) numPatches * 4, 4);
+		MemorySegment valBuf = Arena.ofAuto().allocate((long) numPatches * 4, 4);
+		for (int i = 0; i < numPatches; i++) {
+			idxBuf.setAtIndex(PTypeIO.LE_INT, i, patchIndices.get(i));
+			valBuf.setAtIndex(PTypeIO.LE_FLOAT, i, patchValues.get(i));
+		}
+
+		EncodingProtos.PatchesMetadata patches = EncodingProtos.PatchesMetadata.newBuilder()
+				.setLen(numPatches)
+				.setOffset(0)
+				.setIndicesPtype(DTypeProtos.PType.forNumber(PType.U32.ordinal()))
+				.build();
+		byte[] metaBytes = EncodingProtos.ALPMetadata.newBuilder()
+				.setExpE(expE).setExpF(expF).setPatches(patches).build().toByteArray();
+
+		EncodeNode idxNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 1);
+		EncodeNode valNode = EncodeNode.leaf(EncodingId.VORTEX_PRIMITIVE, 2);
+		EncodeNode root = new EncodeNode(EncodingId.VORTEX_ALP,
+				ByteBuffer.wrap(metaBytes),
+				new EncodeNode[]{encodedNode, idxNode, valNode},
+				new int[0]);
+		return new EncodeResult(root, List.of(encodedBuf, idxBuf, valBuf), statsMin, statsMax);
+	}
+
+	private static byte[] scalarF32(float v) {
+		return ScalarProtos.ScalarValue.newBuilder().setF32Value(v).build().toByteArray();
+	}
+
 	@Override
 	public Array decode(DecodeContext ctx) {
 		ByteBuffer rawMeta = ctx.metadata();

core/src/main/java/io/github/dfa1/vortex/encoding/DictEncoding.java

@@ -12,19 +12,25 @@
 import io.github.dfa1.vortex.core.array.IntArray;
 import io.github.dfa1.vortex.core.array.LongArray;
 import io.github.dfa1.vortex.core.array.ShortArray;
+import io.github.dfa1.vortex.core.array.VarBinArray;
 import io.github.dfa1.vortex.core.VortexException;
 
+import java.lang.foreign.Arena;
 import java.lang.foreign.MemorySegment;
 import java.lang.foreign.ValueLayout;
 import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
 import java.util.LinkedHashMap;
 import java.util.List;
 
 /// Encoding for `vortex.dict` — dictionary encoding for low-cardinality columns.
 ///
-/// Segment layout: [valuesbuffer(uniquevalues,primitive)] [codesbuffer(per-rowindices)].
-/// Metadata (1 byte): code PType ordinal (0=U8, 1=U16, 2=U32).
+/// Primitive: [valuesbuffer(uniquevalues)] [codesbuffer]. Metadata (1 byte): code PType ordinal.
 /// Node tree: DictNode{ children=[ValuesNode{buf=0},CodesNode{buf=1}] }.
+///
+/// Utf8: buf 0 = dict bytes, buf 1 = dict offsets (I64 LE, n_unique+1), buf 2 = codes.
+/// Metadata (2 bytes): [codePType ordinal, I64 ordinal]. Flat node (no children), bufferIndices=[0,1,2].
+/// Decoded as a {@link VarBinArray} in dict mode — no string materialization at decode time.
 public final class DictEncoding implements Encoding {
 
 	private static Array decodeChildAs(DecodeContext parent, int childIdx, DType dtype, long rowCount) {
@@ -198,7 +204,7 @@ public EncodingId encodingId() {
 
 	@Override
 	public boolean accepts(DType dtype) {
-		return dtype instanceof DType.Primitive;
+		return dtype instanceof DType.Primitive || dtype instanceof DType.Utf8;
 	}
 
 	private record DictData(MemorySegment valuesBuf, Object codesArr, PType codePType, int len) {}
@@ -255,6 +261,9 @@ private static DictData buildDictData(DType dtype, Object data) {
 
 	@Override
 	public EncodeResult encode(DType dtype, Object data) {
+		if (dtype instanceof DType.Utf8) {
+			return encodeUtf8((String[]) data, dtype);
+		}
 		DictData d = buildDictData(dtype, data);
 		PType codePType = d.codePType();
 		int codeBytes = codePType.byteSize();
@@ -277,6 +286,9 @@ public EncodeResult encode(DType dtype, Object data) {
 
 	@Override
 	public CascadeStep encodeCascade(DType dtype, Object data, CompressorContext ctx) {
+		if (dtype instanceof DType.Utf8) {
+			return CascadeStep.terminal(encodeUtf8((String[]) data, dtype));
+		}
 		DictData d = buildDictData(dtype, data);
 		PType codePType = d.codePType();
 
@@ -301,6 +313,75 @@ private static int readCodeFromArr(Object arr, PType codePType, int i) {
 		};
 	}
 
+	// ── Utf8 encode ───────────────────────────────────────────────────────────
+
+	private EncodeResult encodeUtf8(String[] strings, DType dtype) {
+		int n = strings.length;
+
+		var valueMap = new LinkedHashMap<String, Integer>();
+		for (String s : strings) {
+			valueMap.computeIfAbsent(s, k -> valueMap.size());
+		}
+
+		int dictSize = valueMap.size();
+		PType codePType = codePType(dictSize);
+		int codeBytes = codePType.byteSize();
+
+		byte[][] dictByteArrays = new byte[dictSize][];
+		int j = 0;
+		long totalDictBytes = 0;
+		for (String s : valueMap.keySet()) {
+			dictByteArrays[j] = s.getBytes(StandardCharsets.UTF_8);
+			totalDictBytes += dictByteArrays[j].length;
+			j++;
+		}
+
+		Arena arena = Arena.ofAuto();
+		MemorySegment dictBytesBuf = arena.allocate(totalDictBytes > 0 ? totalDictBytes : 1);
+		MemorySegment dictOffsetsBuf = arena.allocate((long) (dictSize + 1) * Long.BYTES, Long.BYTES);
+
+		long pos = 0;
+		dictOffsetsBuf.setAtIndex(PTypeIO.LE_LONG, 0, 0L);
+		for (int i = 0; i < dictSize; i++) {
+			MemorySegment.copy(MemorySegment.ofArray(dictByteArrays[i]), 0, dictBytesBuf, pos, dictByteArrays[i].length);
+			pos += dictByteArrays[i].length;
+			dictOffsetsBuf.setAtIndex(PTypeIO.LE_LONG, i + 1, pos);
+		}
+
+		MemorySegment codesBuf = Arena.ofAuto().allocate((long) n * codeBytes);
+		for (int i = 0; i < n; i++) {
+			writeCodeToSeg(codesBuf, codePType, i, valueMap.get(strings[i]));
+		}
+
+		// 2 bytes: [codePType ordinal, I64 ordinal] — distinguishes from 1-byte primitive format
+		ByteBuffer meta = ByteBuffer.allocate(2)
+				.put(0, (byte) codePType.ordinal())
+				.put(1, (byte) PType.I64.ordinal());
+
+		EncodeNode root = new EncodeNode(
+				EncodingId.VORTEX_DICT, meta,
+				new EncodeNode[0],
+				new int[]{0, 1, 2});
+
+		return new EncodeResult(root, List.of(dictBytesBuf, dictOffsetsBuf, codesBuf), null, null);
+	}
+
+	// ── Utf8 decode ───────────────────────────────────────────────────────────
+
+	private static Array decodeUtf8Dict(DecodeContext ctx, ByteBuffer meta) {
+		PType codePType = PType.values()[Byte.toUnsignedInt(meta.get(0))];
+		long n = ctx.rowCount();
+
+		MemorySegment dictBytes   = ctx.buffer(0);
+		MemorySegment dictOffsets = ctx.buffer(1);
+		MemorySegment codes       = ctx.buffer(2);
+
+		return VarBinArray.ofDict(ctx.dtype(), n,
+				dictBytes, dictOffsets, PType.I64,
+				codes, codePType,
+				ArrayStats.empty());
+	}
+
 	@Override
 	public Array decode(DecodeContext ctx) {
 		if (ctx.metadata() == null || !ctx.metadata().hasRemaining()) {
@@ -309,6 +390,10 @@ public Array decode(DecodeContext ctx) {
 
 		ByteBuffer meta = ctx.metadata();
 
+		if (ctx.dtype() instanceof DType.Utf8) {
+			return decodeUtf8Dict(ctx, meta);
+		}
+
 		// 1-byte = legacy Java format; multi-byte = Rust proto format
 		if (meta.remaining() == 1) {
 			return decodeLegacyJava(ctx, meta.get(0));

core/src/test/java/io/github/dfa1/vortex/encoding/AlpEncodingTest.java

@@ -220,4 +220,74 @@ void decode_f32_noPatches() {
 					.as("index %d", i).isCloseTo(expected[i], within(1e-6f));
 		}
 	}
+
+	@Test
+	void encode_f32_roundTrip_noPatches() {
+		// Given
+		float[] values = {1.0f, 2.5f, 3.75f, 10.0f, 0.1f};
+		AlpEncoding sut = new AlpEncoding();
+
+		EncodingRegistry registry = EncodingRegistry.empty();
+		registry.register(sut);
+		registry.register(new PrimitiveEncoding());
+
+		// When
+		EncodeResult encoded = sut.encode(F32_DTYPE, values);
+		DecodeContext ctx = EncodeTestHelper.toDecodeContext(encoded, values.length, F32_DTYPE, registry);
+		Array result = sut.decode(ctx);
+
+		// Then
+		var layout = ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+		for (int i = 0; i < values.length; i++) {
+			assertThat(result.buffer(0).get(layout, (long) i * 4))
+					.as("index %d", i).isCloseTo(values[i], within(1e-6f));
+		}
+	}
+
+	@Test
+	void encode_f32_roundTrip_withPatches() {
+		// Given — Float.NaN and Float.POSITIVE_INFINITY can't be ALP-encoded; must become patches
+		float[] values = {1.0f, Float.NaN, 2.5f, Float.POSITIVE_INFINITY, 3.0f};
+		AlpEncoding sut = new AlpEncoding();
+
+		EncodingRegistry registry = EncodingRegistry.empty();
+		registry.register(sut);
+		registry.register(new PrimitiveEncoding());
+
+		// When
+		EncodeResult encoded = sut.encode(F32_DTYPE, values);
+		DecodeContext ctx = EncodeTestHelper.toDecodeContext(encoded, values.length, F32_DTYPE, registry);
+		Array result = sut.decode(ctx);
+
+		// Then
+		var layout = ValueLayout.JAVA_FLOAT_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+		assertThat(result.buffer(0).get(layout, 0L)).isCloseTo(1.0f, within(1e-6f));
+		assertThat(result.buffer(0).get(layout, 4L)).isNaN();
+		assertThat(result.buffer(0).get(layout, 8L)).isCloseTo(2.5f, within(1e-6f));
+		assertThat(result.buffer(0).get(layout, 12L)).isInfinite();
+		assertThat(result.buffer(0).get(layout, 16L)).isCloseTo(3.0f, within(1e-6f));
+	}
+
+	@Test
+	void encode_f64_roundTrip_noPatches() {
+		// Given
+		double[] values = {1.23, 4.56, 7.89, 0.001, 100.0};
+		AlpEncoding sut = new AlpEncoding();
+
+		EncodingRegistry registry = EncodingRegistry.empty();
+		registry.register(sut);
+		registry.register(new PrimitiveEncoding());
+
+		// When
+		EncodeResult encoded = sut.encode(F64_DTYPE, values);
+		DecodeContext ctx = EncodeTestHelper.toDecodeContext(encoded, values.length, F64_DTYPE, registry);
+		Array result = sut.decode(ctx);
+
+		// Then
+		var layout = ValueLayout.JAVA_DOUBLE_UNALIGNED.withOrder(ByteOrder.LITTLE_ENDIAN);
+		for (int i = 0; i < values.length; i++) {
+			assertThat(result.buffer(0).get(layout, (long) i * 8))
+					.as("index %d", i).isCloseTo(values[i], within(1e-9));
+		}
+	}
 }