Skip to content

Commit 8636197

Browse files
dfa1claude
dfa1
and
claude
committed
feat: add Gt, Lt, Neq to RowFilter; update CLI and docs
Add RowFilter.Gt/Lt/Neq with zone-map pruning support in ScanIterator. Refactor FilterCommand to use RowFilter directly (drop Op enum + conservative Gte/Lte lowering for strict ops). Extend CLI filter regex to accept !=. Update reference.md. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
1 parent 962520a commit 8636197

6 files changed

Lines changed: 200 additions & 76 deletions

File tree

‎TODO.md‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,6 @@ Remaining 8.5 MB gap — biggest to smallest:
9797

9898
## API
9999

100-
- [ ] Add `RowFilter.Neq`, `RowFilter.Gt`, `RowFilter.Lt` to the Java API. Currently only `Gte`, `Lte`, `Eq`, `And` are exposed; the CLI lowers `>`, `<`, `!=` onto the supported set.
101-
102100
- [ ] Use domain primitives (`UInt32`, `UInt64`, etc.) as value classes via Project Valhalla instead of raw `long`/`int`
103101
- See https://dfa1.github.io/articles/rethink-domain-primitives-with-valhalla
104102
- Candidates: `PType` integer kinds, buffer offsets, row indices, byte lengths

‎cli/src/main/java/io/github/dfa1/vortex/cli/FilterCommand.java‎

Lines changed: 34 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,8 @@ final class FilterCommand {
3131
private FilterCommand() {
3232
}
3333

34-
private enum Op { GT, GTE, LT, LTE, EQ }
35-
36-
private record ParsedFilter(String column, Op op, Comparable<?> value) {}
37-
3834
private static final Pattern EXPR = Pattern.compile(
39-
"^(\\w[\\w.]*?)\\s*(>=|<=|==|>|<|=)\\s*(.+)$");
35+
"^(\\w[\\w.]*?)\\s*(!=|>=|<=|==|>|<|=)\\s*(.+)$");
4036

4137
static int run(String[] args) {
4238
if (args.length < 3) {
@@ -49,15 +45,15 @@ static int run(String[] args) {
4945
return ExitStatus.FILE_NOT_FOUND;
5046
}
5147
String expr = String.join(" ", Arrays.asList(args).subList(2, args.length));
52-
ParsedFilter pf;
48+
RowFilter filter;
5349
try {
54-
pf = parseFilter(expr);
50+
filter = parseFilter(expr);
5551
} catch (IllegalArgumentException e) {
5652
System.err.println("error: " + e.getMessage());
5753
return ExitStatus.USAGE_ERROR;
5854
}
59-
ScanOptions scanOptions = new ScanOptions(List.of(), toRowFilter(pf), ScanOptions.NO_LIMIT);
60-
RowPredicate rowPred = toRowPredicate(pf);
55+
ScanOptions scanOptions = new ScanOptions(List.of(), filter, ScanOptions.NO_LIMIT);
56+
RowPredicate rowPred = toRowPredicate(filter);
6157
try {
6258
Writer stdout = new OutputStreamWriter(System.out, StandardCharsets.UTF_8);
6359
CsvExporter.exportCsvFiltered(path, stdout, ExportOptions.defaults(), scanOptions, rowPred);
@@ -69,23 +65,23 @@ static int run(String[] args) {
6965
}
7066
}
7167

72-
private static ParsedFilter parseFilter(String expr) {
68+
private static RowFilter parseFilter(String expr) {
7369
Matcher m = EXPR.matcher(expr.trim());
7470
if (!m.matches()) {
7571
throw new IllegalArgumentException("invalid filter expression: \"" + expr
76-
+ "\" expected: col op value (op: >, >=, <, <=, =, ==)");
72+
+ "\" expected: col op value (op: >, >=, <, <=, =, ==, !=)");
7773
}
7874
String col = m.group(1);
79-
Op op = switch (m.group(2)) {
80-
case ">" -> Op.GT;
81-
case ">=" -> Op.GTE;
82-
case "<" -> Op.LT;
83-
case "<=" -> Op.LTE;
84-
case "=", "==" -> Op.EQ;
75+
Comparable<?> value = parseValue(m.group(3).trim());
76+
return switch (m.group(2)) {
77+
case ">" -> RowFilter.gt(col, value);
78+
case ">=" -> RowFilter.gte(col, value);
79+
case "<" -> RowFilter.lt(col, value);
80+
case "<=" -> RowFilter.lte(col, value);
81+
case "=", "==" -> RowFilter.eq(col, value);
82+
case "!=" -> RowFilter.neq(col, value);
8583
default -> throw new IllegalArgumentException("unknown operator: " + m.group(2));
8684
};
87-
Comparable<?> value = parseValue(m.group(3).trim());
88-
return new ParsedFilter(col, op, value);
8985
}
9086

9187
private static Comparable<?> parseValue(String raw) {
@@ -106,28 +102,25 @@ private static Comparable<?> parseValue(String raw) {
106102
return raw;
107103
}
108104

109-
/// Zone-map RowFilter for chunk pruning. Conservative for strict ops (> / <):
110-
/// uses Gte/Lte which may keep one extra chunk at the boundary, but row-level
111-
/// predicate corrects that.
112-
private static RowFilter toRowFilter(ParsedFilter pf) {
113-
return switch (pf.op()) {
114-
case GT, GTE -> RowFilter.gte(pf.column(), pf.value());
115-
case LT, LTE -> RowFilter.lte(pf.column(), pf.value());
116-
case EQ -> RowFilter.eq(pf.column(), pf.value());
117-
};
118-
}
119-
120-
private static RowPredicate toRowPredicate(ParsedFilter pf) {
121-
return (chunk, rowIdx) -> {
122-
Array arr = chunk.column(pf.column());
123-
int cmp = compareValue(arr, rowIdx, pf.value());
124-
return switch (pf.op()) {
125-
case GT -> cmp > 0;
126-
case GTE -> cmp >= 0;
127-
case LT -> cmp < 0;
128-
case LTE -> cmp <= 0;
129-
case EQ -> cmp == 0;
130-
};
105+
private static RowPredicate toRowPredicate(RowFilter filter) {
106+
return switch (filter) {
107+
case RowFilter.Gt(var col, var val) -> (chunk, rowIdx) -> compareValue(chunk.column(col), rowIdx, val) > 0;
108+
case RowFilter.Gte(var col, var val) -> (chunk, rowIdx) -> compareValue(chunk.column(col), rowIdx, val) >= 0;
109+
case RowFilter.Lt(var col, var val) -> (chunk, rowIdx) -> compareValue(chunk.column(col), rowIdx, val) < 0;
110+
case RowFilter.Lte(var col, var val) -> (chunk, rowIdx) -> compareValue(chunk.column(col), rowIdx, val) <= 0;
111+
case RowFilter.Eq(var col, var val) -> (chunk, rowIdx) -> compareValue(chunk.column(col), rowIdx, (Comparable<?>) val) == 0;
112+
case RowFilter.Neq(var col, var val) -> (chunk, rowIdx) -> compareValue(chunk.column(col), rowIdx, (Comparable<?>) val) != 0;
113+
case RowFilter.And(var filters) -> {
114+
RowPredicate[] preds = filters.stream().map(FilterCommand::toRowPredicate).toArray(RowPredicate[]::new);
115+
yield (chunk, rowIdx) -> {
116+
for (RowPredicate p : preds) {
117+
if (!p.test(chunk, rowIdx)) {
118+
return false;
119+
}
120+
}
121+
return true;
122+
};
123+
}
131124
};
132125
}
133126

‎docs/reference.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -120,15 +120,14 @@ Sealed predicate used for zone-map pruning (per-chunk min/max). Chunks that cann
120120

121121
| Record | Static factory | Builder |
122122
|---|---|---|
123+
| `RowFilter.Gt(column, value)` | `RowFilter.gt(col, val)` | — |
123124
| `RowFilter.Gte(column, value)` | `RowFilter.gte(col, val)` | — |
125+
| `RowFilter.Lt(column, value)` | `RowFilter.lt(col, val)` | — |
124126
| `RowFilter.Lte(column, value)` | `RowFilter.lte(col, val)` | — |
125127
| `RowFilter.Eq(column, value)` | `RowFilter.eq(col, val)` | — |
128+
| `RowFilter.Neq(column, value)` | `RowFilter.neq(col, val)` | — |
126129
| `RowFilter.And(filters)` | `RowFilter.and(f1, f2, …)` | `f1.and(f2)` |
127130

128-
**Supported operators (Java API):** `Gte`, `Lte`, `Eq`, `And`.
129-
`Gt`, `Lt`, `Neq` are not exposed in the Java API. The CLI `filter` subcommand accepts `>`, `<`, `!=` syntactically
130-
but lowers them onto the supported predicates.
131-
132131
### `ScanIterator` (`io.github.dfa1.vortex.scan.ScanIterator`)
133132

134133
Implements `AutoCloseable`. Drives one scan.
@@ -225,6 +224,7 @@ java -jar cli/target/vortex.jar <subcommand> [args]
225224
| `>`, `>=` | Greater than, greater-or-equal |
226225
| `<`, `<=` | Less than, less-or-equal |
227226
| `=`, `==` | Equal |
227+
| `!=` | Not equal |
228228

229229
Values are parsed as integer, double, boolean, or string (in that order).
230230

‎reader/src/main/java/io/github/dfa1/vortex/scan/RowFilter.java‎

Lines changed: 28 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
/// Predicate tree for zone-map pruning. Evaluated against per-chunk min/max statistics;
66
/// chunks where no row can satisfy the filter are skipped entirely.
77
public sealed interface RowFilter
8-
permits RowFilter.And, RowFilter.Gte, RowFilter.Lte, RowFilter.Eq {
8+
permits RowFilter.And, RowFilter.Eq, RowFilter.Neq,
9+
RowFilter.Gt, RowFilter.Gte, RowFilter.Lt, RowFilter.Lte {
910

1011
static RowFilter and(RowFilter... filters) {
1112
return new And(List.of(filters));
@@ -15,27 +16,48 @@ default RowFilter and(RowFilter other) {
1516
return new And(List.of(this, other));
1617
}
1718

19+
static RowFilter eq(String col, Object val) {
20+
return new Eq(col, val);
21+
}
22+
23+
static RowFilter neq(String col, Object val) {
24+
return new Neq(col, val);
25+
}
26+
27+
static RowFilter gt(String col, Comparable<?> val) {
28+
return new Gt(col, val);
29+
}
30+
1831
static RowFilter gte(String col, Comparable<?> val) {
1932
return new Gte(col, val);
2033
}
2134

35+
static RowFilter lt(String col, Comparable<?> val) {
36+
return new Lt(col, val);
37+
}
38+
2239
static RowFilter lte(String col, Comparable<?> val) {
2340
return new Lte(col, val);
2441
}
2542

26-
static RowFilter eq(String col, Object val) {
27-
return new Eq(col, val);
43+
record And(List<RowFilter> filters) implements RowFilter {
2844
}
2945

30-
record And(List<RowFilter> filters) implements RowFilter {
46+
record Eq(String column, Object value) implements RowFilter {
47+
}
48+
49+
record Neq(String column, Object value) implements RowFilter {
50+
}
51+
52+
record Gt(String column, Comparable<?> value) implements RowFilter {
3153
}
3254

3355
record Gte(String column, Comparable<?> value) implements RowFilter {
3456
}
3557

36-
record Lte(String column, Comparable<?> value) implements RowFilter {
58+
record Lt(String column, Comparable<?> value) implements RowFilter {
3759
}
3860

39-
record Eq(String column, Object value) implements RowFilter {
61+
record Lte(String column, Comparable<?> value) implements RowFilter {
4062
}
4163
}

‎reader/src/main/java/io/github/dfa1/vortex/scan/ScanIterator.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,14 @@ private boolean canPruneChunk(ChunkSpec chunk, RowFilter filter) {
453453
}
454454
yield false;
455455
}
456+
case RowFilter.Gt(var col, var val) -> {
457+
Layout flat = chunk.layoutFor(col);
458+
if (flat == null) {
459+
yield false;
460+
}
461+
Object max = readFlatStats(flat).max();
462+
yield max != null && compareValues(max, val) <= 0;
463+
}
456464
case RowFilter.Gte(var col, var val) -> {
457465
Layout flat = chunk.layoutFor(col);
458466
if (flat == null) {
@@ -461,6 +469,14 @@ private boolean canPruneChunk(ChunkSpec chunk, RowFilter filter) {
461469
Object max = readFlatStats(flat).max();
462470
yield max != null && compareValues(max, val) < 0;
463471
}
472+
case RowFilter.Lt(var col, var val) -> {
473+
Layout flat = chunk.layoutFor(col);
474+
if (flat == null) {
475+
yield false;
476+
}
477+
Object min = readFlatStats(flat).min();
478+
yield min != null && compareValues(min, val) >= 0;
479+
}
464480
case RowFilter.Lte(var col, var val) -> {
465481
Layout flat = chunk.layoutFor(col);
466482
if (flat == null) {
@@ -488,6 +504,25 @@ private boolean canPruneChunk(ChunkSpec chunk, RowFilter filter) {
488504
yield false;
489505
}
490506
}
507+
case RowFilter.Neq(var col, var val) -> {
508+
Layout flat = chunk.layoutFor(col);
509+
if (flat == null) {
510+
yield false;
511+
}
512+
ArrayStats stats = readFlatStats(flat);
513+
Object min = stats.min();
514+
Object max = stats.max();
515+
if (min == null || max == null) {
516+
yield false;
517+
}
518+
try {
519+
@SuppressWarnings("unchecked")
520+
Comparable<Object> cv = (Comparable<Object>) val;
521+
yield cv.compareTo(min) == 0 && cv.compareTo(max) == 0;
522+
} catch (ClassCastException e) {
523+
yield false;
524+
}
525+
}
491526
};
492527
}
493528

0 commit comments

Comments
 (0)