Skip to content

Commit 8fc22d4

Browse files
dfa1claude
andcommitted
feat(scan): implement limit properly — truncate last chunk to exact row count
ScanIterator now truncates the final chunk to exactly limit-rowsReturned rows instead of returning the full chunk and stopping at the next iteration. Covers: all primitive types, BoolArray, NullArray, VarBinArray (dict and non-dict), MaskedArray. Unsupported types throw VortexException. VarBinArray.truncate(long): new public method handles both dict mode (slice codes segment) and non-dict mode (slice bytes + offsets). MaskedArray.validity(): new accessor returns the validity bitmap or null. ScanOptions.limit(long): new factory method for preview/head scans. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 83dec66 commit 8fc22d4

5 files changed

Lines changed: 144 additions & 2 deletions

File tree

core/src/main/java/io/github/dfa1/vortex/core/array/MaskedArray.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@ public Array child(int i) {
5151
};
5252
}
5353

54+
/// Returns the validity bitmap, or {@code null} if all positions are valid.
55+
public BoolArray validity() {
56+
return validity;
57+
}
58+
5459
public boolean isValid(long i) {
5560
return validity == null || validity.getBoolean(i);
5661
}

core/src/main/java/io/github/dfa1/vortex/core/array/VarBinArray.java

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -188,4 +188,24 @@ private long dictReadOff(long i) {
188188
}
189189
return dictValOffsets.getAtIndex(PTypeIO.LE_LONG, i);
190190
}
191+
192+
/// Returns a new VarBinArray containing only the first {@code rows} elements.
193+
public VarBinArray truncate(long rows) {
194+
if (rows >= length) {
195+
return this;
196+
}
197+
if (dictCodesSegs != null) {
198+
int codeBytes = dictCodesPType.byteSize();
199+
return VarBinArray.ofDict(dtype, rows, bytes, dictValOffsets, dictValOffPType,
200+
dictCodesSegs.asSlice(0, rows * codeBytes), dictCodesPType, ArrayStats.empty());
201+
}
202+
long byteEnd = readOffset(rows);
203+
int offBytes = (offsetsPtype == PType.I32 || offsetsPtype == PType.U32) ? Integer.BYTES : Long.BYTES;
204+
MemorySegment newOffsetsSeg = offsetsSeg.asSlice(0, (rows + 1) * offBytes);
205+
DType offDtype = new DType.Primitive(offsetsPtype, false);
206+
Array newOffsetsArr = (offBytes == Integer.BYTES)
207+
? new IntArray(offDtype, rows + 1, newOffsetsSeg, ArrayStats.empty())
208+
: new LongArray(offDtype, rows + 1, newOffsetsSeg, ArrayStats.empty());
209+
return new VarBinArray(dtype, rows, bytes.asSlice(0, byteEnd > 0 ? byteEnd : 0), newOffsetsArr, offsetsPtype, ArrayStats.empty());
210+
}
191211
}

reader/src/main/java/io/github/dfa1/vortex/scan/ScanIterator.java

Lines changed: 48 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,11 @@
99
import io.github.dfa1.vortex.core.Layout;
1010
import io.github.dfa1.vortex.core.PType;
1111
import io.github.dfa1.vortex.core.SegmentSpec;
12+
import io.github.dfa1.vortex.core.array.BoolArray;
13+
import io.github.dfa1.vortex.core.array.MaskedArray;
1214
import io.github.dfa1.vortex.core.array.ByteArray;
15+
import io.github.dfa1.vortex.core.array.EmptyArray;
16+
import io.github.dfa1.vortex.core.array.NullArray;
1317
import io.github.dfa1.vortex.core.array.DoubleArray;
1418
import io.github.dfa1.vortex.core.array.FloatArray;
1519
import io.github.dfa1.vortex.core.array.IntArray;
@@ -132,8 +136,15 @@ public boolean hasNext() {
132136
chunkArena.close();
133137
}
134138
chunkArena = Arena.ofConfined();
135-
current = new ScanResult(chunk.rowCount(), buildColumnMap(chunk));
136-
rowsReturned += chunk.rowCount();
139+
140+
long remaining = options.limit() - rowsReturned;
141+
long chunkRows = Math.min(chunk.rowCount(), remaining);
142+
Map<String, Array> columns = buildColumnMap(chunk);
143+
if (chunkRows < chunk.rowCount()) {
144+
columns = truncateColumns(columns, chunkRows);
145+
}
146+
current = new ScanResult(chunkRows, columns);
147+
rowsReturned += chunkRows;
137148
return true;
138149
}
139150
// Do not close here — the last chunk's arena stays open until close() is called.
@@ -397,6 +408,41 @@ private static long readUnsigned(MemorySegment seg, long idx, PType ptype) {
397408
};
398409
}
399410

411+
// ── Limit truncation ─────────────────────────────────────────────────────
412+
413+
private static Map<String, Array> truncateColumns(Map<String, Array> columns, long rows) {
414+
var result = new LinkedHashMap<String, Array>(columns.size());
415+
for (var entry : columns.entrySet()) {
416+
result.put(entry.getKey(), truncateArray(entry.getValue(), rows));
417+
}
418+
return Map.copyOf(result);
419+
}
420+
421+
private static Array truncateArray(Array arr, long rows) {
422+
if (arr.length() <= rows) {
423+
return arr;
424+
}
425+
return switch (arr) {
426+
case LongArray a -> new LongArray(a.dtype(), rows, a.buffer(0).asSlice(0, rows * Long.BYTES), ArrayStats.empty());
427+
case IntArray a -> new IntArray(a.dtype(), rows, a.buffer(0).asSlice(0, rows * Integer.BYTES), ArrayStats.empty());
428+
case DoubleArray a -> new DoubleArray(a.dtype(), rows, a.buffer(0).asSlice(0, rows * Double.BYTES), ArrayStats.empty());
429+
case FloatArray a -> new FloatArray(a.dtype(), rows, a.buffer(0).asSlice(0, rows * Float.BYTES), ArrayStats.empty());
430+
case ShortArray a -> new ShortArray(a.dtype(), rows, a.buffer(0).asSlice(0, rows * Short.BYTES), ArrayStats.empty());
431+
case ByteArray a -> new ByteArray(a.dtype(), rows, a.buffer(0).asSlice(0, rows), ArrayStats.empty());
432+
case BoolArray a -> new BoolArray(a.dtype(), rows, a.buffer(0).asSlice(0, (rows + 7) / 8), ArrayStats.empty());
433+
case NullArray a -> new NullArray(a.dtype(), rows);
434+
case VarBinArray a -> a.truncate(rows);
435+
case MaskedArray a -> {
436+
Array truncChild = truncateArray((Array) a.child(0), rows);
437+
BoolArray v = a.validity();
438+
BoolArray truncValidity = (v != null) ? (BoolArray) truncateArray(v, rows) : null;
439+
yield new MaskedArray(truncChild, truncValidity);
440+
}
441+
case EmptyArray a -> a;
442+
default -> throw new VortexException("limit: truncation not supported for " + arr.getClass().getSimpleName());
443+
};
444+
}
445+
400446
private boolean canPruneChunk(ChunkSpec chunk, RowFilter filter) {
401447
return switch (filter) {
402448
case RowFilter.And(var filters) -> {

reader/src/main/java/io/github/dfa1/vortex/scan/ScanOptions.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,10 @@ public static ScanOptions columns(String... names) {
2121
return new ScanOptions(List.of(names), null, NO_LIMIT);
2222
}
2323

24+
public static ScanOptions limit(long limit) {
25+
return new ScanOptions(List.of(), null, limit);
26+
}
27+
2428
public boolean hasProjection() {
2529
return !columns.isEmpty();
2630
}

reader/src/test/java/io/github/dfa1/vortex/io/VortexReaderTest.java

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,73 @@ void scan_chunkedFixture_producesExactlyOneLayoutChunk() throws URISyntaxExcepti
265265
assertThat(chunkCount).isEqualTo(1);
266266
}
267267

268+
@Test
269+
void scan_withLimit_returnsExactlyNRows() throws URISyntaxException, IOException {
270+
// Given — primitives.vortex has 3 rows; limit=2 forces truncation of the single chunk
271+
Path path = fixtureFile("primitives.vortex");
272+
long limit = 2;
273+
274+
// When
275+
long totalRows = 0;
276+
try (var sut = VortexReader.open(path);
277+
var iter = sut.scan(ScanOptions.limit(limit))) {
278+
while (iter.hasNext()) {
279+
ScanResult chunk = iter.next();
280+
totalRows += chunk.rowCount();
281+
for (Array col : chunk.columns().values()) {
282+
assertThat(col.length()).isLessThanOrEqualTo(limit);
283+
}
284+
}
285+
}
286+
287+
// Then
288+
assertThat(totalRows).isEqualTo(limit);
289+
}
290+
291+
@Test
292+
void scan_withLimitExceedingTotal_returnsAllRows() throws URISyntaxException, IOException {
293+
// Given
294+
Path path = fixtureFile("primitives.vortex");
295+
long totalWithoutLimit;
296+
try (var sut = VortexReader.open(path);
297+
var iter = sut.scan(ScanOptions.all())) {
298+
totalWithoutLimit = 0;
299+
while (iter.hasNext()) {
300+
totalWithoutLimit += iter.next().rowCount();
301+
}
302+
}
303+
304+
// When
305+
long totalWithLimit = 0;
306+
try (var sut = VortexReader.open(path);
307+
var iter = sut.scan(ScanOptions.limit(Long.MAX_VALUE))) {
308+
while (iter.hasNext()) {
309+
totalWithLimit += iter.next().rowCount();
310+
}
311+
}
312+
313+
// Then
314+
assertThat(totalWithLimit).isEqualTo(totalWithoutLimit);
315+
}
316+
317+
@Test
318+
void scan_withLimitZero_returnsNoRows() throws URISyntaxException, IOException {
319+
// Given
320+
Path path = fixtureFile("primitives.vortex");
321+
322+
// When
323+
long totalRows = 0;
324+
try (var sut = VortexReader.open(path);
325+
var iter = sut.scan(ScanOptions.limit(0))) {
326+
while (iter.hasNext()) {
327+
totalRows += iter.next().rowCount();
328+
}
329+
}
330+
331+
// Then
332+
assertThat(totalRows).isZero();
333+
}
334+
268335
// --- helpers ---
269336

270337
private Path fixtureFile(String name) throws URISyntaxException {

0 commit comments

Comments
 (0)