|
| 1 | +# How-to guides |
| 2 | + |
| 3 | +Task-oriented recipes. Each section solves one concrete goal. |
| 4 | + |
| 5 | +--- |
| 6 | + |
| 7 | +## Count rows |
| 8 | + |
| 9 | +**API:** |
| 10 | + |
| 11 | +```java |
| 12 | +long total = 0; |
| 13 | +try (VortexReader vf = VortexReader.open(Path.of("data.vortex")); |
| 14 | + var iter = vf.scan(ScanOptions.all())) { |
| 15 | + while (iter.hasNext()) { |
| 16 | + total += iter.next().rowCount(); |
| 17 | + } |
| 18 | +} |
| 19 | +System.out.println(total); |
| 20 | +``` |
| 21 | + |
| 22 | +**CLI:** |
| 23 | + |
| 24 | +```bash |
| 25 | +java -jar cli/target/vortex.jar count data.vortex |
| 26 | +``` |
| 27 | + |
| 28 | +--- |
| 29 | + |
| 30 | +## Inspect file structure |
| 31 | + |
| 32 | +**API:** |
| 33 | + |
| 34 | +```java |
| 35 | +try (VortexReader vf = VortexReader.open(Path.of("data.vortex"))) { |
| 36 | + System.out.println(vf.dtype()); // column names and types |
| 37 | + System.out.println(vf.layout()); // layout tree (Struct → Chunked → Flat …) |
| 38 | +} |
| 39 | +``` |
| 40 | + |
| 41 | +**CLI:** |
| 42 | + |
| 43 | +```bash |
| 44 | +# column names and types |
| 45 | +java -jar cli/target/vortex.jar schema data.vortex |
| 46 | + |
| 47 | +# full layout tree with encoding IDs, row counts, buffer sizes |
| 48 | +java -jar cli/target/vortex.jar inspect data.vortex |
| 49 | + |
| 50 | +# per-column min/max statistics |
| 51 | +java -jar cli/target/vortex.jar stats data.vortex |
| 52 | +``` |
| 53 | + |
| 54 | +--- |
| 55 | + |
| 56 | +## Project columns |
| 57 | + |
| 58 | +**API:** |
| 59 | + |
| 60 | +```java |
| 61 | +ScanOptions opts = ScanOptions.all().withColumns("symbol", "price"); |
| 62 | + |
| 63 | +try (VortexReader vf = VortexReader.open(Path.of("trades.vortex")); |
| 64 | + var iter = vf.scan(opts)) { |
| 65 | + while (iter.hasNext()) { |
| 66 | + var chunk = iter.next(); |
| 67 | + // chunk.columns() contains only "symbol" and "price" |
| 68 | + } |
| 69 | +} |
| 70 | +``` |
| 71 | + |
| 72 | +**CLI:** |
| 73 | + |
| 74 | +```bash |
| 75 | +java -jar cli/target/vortex.jar select trades.vortex symbol price |
| 76 | +``` |
| 77 | +
|
| 78 | +--- |
| 79 | +
|
| 80 | +## Filter rows |
| 81 | +
|
| 82 | +**API:** |
| 83 | +
|
| 84 | +```java |
| 85 | +RowFilter filter = new RowFilter.Gte("volume", 1_000_000); |
| 86 | +ScanOptions opts = ScanOptions.all().withFilter(filter); |
| 87 | + |
| 88 | +try (VortexReader vf = VortexReader.open(Path.of("trades.vortex")); |
| 89 | + var iter = vf.scan(opts)) { |
| 90 | + while (iter.hasNext()) { |
| 91 | + var chunk = iter.next(); |
| 92 | + // only rows where volume >= 1_000_000 |
| 93 | + } |
| 94 | +} |
| 95 | +``` |
| 96 | +
|
| 97 | +Combine filters with `and()`: |
| 98 | +
|
| 99 | +```java |
| 100 | +RowFilter filter = new RowFilter.Gte("volume", 1_000_000) |
| 101 | + .and(new RowFilter.Lte("price", 200.0)); |
| 102 | +``` |
| 103 | +
|
| 104 | +Supported operators: `Eq`, `Neq`, `Lt`, `Lte`, `Gt`, `Gte`. |
| 105 | +
|
| 106 | +**CLI:** |
| 107 | +
|
| 108 | +```bash |
| 109 | +java -jar cli/target/vortex.jar filter trades.vortex "volume >= 1000000" |
| 110 | +``` |
| 111 | +
|
| 112 | +Filter operators: `>`, `>=`, `<`, `<=`, `=`, `==`. Values parsed as integer, double, boolean, or string. |
| 113 | +
|
| 114 | +--- |
| 115 | +
|
| 116 | +## Preview the first N rows |
| 117 | +
|
| 118 | +**API:** |
| 119 | +
|
| 120 | +```java |
| 121 | +ScanOptions opts = ScanOptions.all().withLimit(10); |
| 122 | + |
| 123 | +try (VortexReader vf = VortexReader.open(Path.of("data.vortex")); |
| 124 | + var iter = vf.scan(opts)) { |
| 125 | + while (iter.hasNext()) { |
| 126 | + var chunk = iter.next(); |
| 127 | + // at most 10 rows total across all chunks |
| 128 | + } |
| 129 | +} |
| 130 | +``` |
| 131 | +
|
| 132 | +**CLI:** |
| 133 | +
|
| 134 | +```bash |
| 135 | +# export first 10 rows to CSV |
| 136 | +java -jar cli/target/vortex.jar export data.vortex | head -n 11 # 1 header + 10 rows |
| 137 | +``` |
| 138 | +
|
| 139 | +--- |
| 140 | +
|
| 141 | +## Convert Parquet to Vortex |
| 142 | +
|
| 143 | +**API:** |
| 144 | +
|
| 145 | +```java |
| 146 | +import io.github.dfa1.vortex.parquet.ParquetImporter; |
| 147 | + |
| 148 | +ParquetImporter.importParquet( |
| 149 | + Path.of("data.parquet"), |
| 150 | + Path.of("data.vortex") |
| 151 | +); |
| 152 | +``` |
| 153 | +
|
| 154 | +Project specific columns during conversion: |
| 155 | +
|
| 156 | +```java |
| 157 | +import io.github.dfa1.vortex.parquet.ImportOptions; |
| 158 | + |
| 159 | +ImportOptions opts = ImportOptions.defaults() |
| 160 | + .withColumns(List.of("trip_distance", "fare_amount")); |
| 161 | + |
| 162 | +ParquetImporter.importParquet(Path.of("data.parquet"), Path.of("data.vortex"), opts); |
| 163 | +``` |
| 164 | +
|
| 165 | +**CLI:** |
| 166 | +
|
| 167 | +```bash |
| 168 | +# output defaults to <input>.vortex |
| 169 | +java -jar cli/target/vortex.jar import data.parquet |
| 170 | + |
| 171 | +# explicit output path |
| 172 | +java -jar cli/target/vortex.jar import data.parquet out.vortex |
| 173 | +``` |
| 174 | +
|
| 175 | +--- |
| 176 | +
|
| 177 | +## Convert CSV to Vortex |
| 178 | +
|
| 179 | +**CLI only** (CSV has no schema — types are inferred): |
| 180 | +
|
| 181 | +```bash |
| 182 | +java -jar cli/target/vortex.jar import data.csv |
| 183 | +# writes data.vortex, prints size savings |
| 184 | +``` |
| 185 | +
|
| 186 | +--- |
| 187 | +
|
| 188 | +## Export to CSV |
| 189 | +
|
| 190 | +**CLI:** |
| 191 | +
|
| 192 | +```bash |
| 193 | +# all columns |
| 194 | +java -jar cli/target/vortex.jar export data.vortex > out.csv |
| 195 | + |
| 196 | +# specific columns |
| 197 | +java -jar cli/target/vortex.jar select data.vortex col1 col2 > out.csv |
| 198 | + |
| 199 | +# filtered rows |
| 200 | +java -jar cli/target/vortex.jar filter data.vortex "price >= 100" > out.csv |
| 201 | +``` |
| 202 | +
|
| 203 | +--- |
| 204 | +
|
| 205 | +## Read files with unknown encodings |
| 206 | +
|
| 207 | +By default, a file containing an unrecognised encoding ID throws `VortexException`. |
| 208 | +Use `allowUnknown()` to read the file anyway — columns with unknown encodings are |
| 209 | +returned as `UnknownArray` (opaque, not decodable, but the rest of the file is readable): |
| 210 | +
|
| 211 | +```java |
| 212 | +import io.github.dfa1.vortex.encoding.EncodingRegistry; |
| 213 | +import io.github.dfa1.vortex.core.array.UnknownArray; |
| 214 | + |
| 215 | +EncodingRegistry registry = EncodingRegistry.loadAll().allowUnknown(); |
| 216 | + |
| 217 | +try (VortexReader vf = VortexReader.open(Path.of("future.vortex"), registry); |
| 218 | + var iter = vf.scan(ScanOptions.all())) { |
| 219 | + while (iter.hasNext()) { |
| 220 | + var chunk = iter.next(); |
| 221 | + chunk.columns().forEach((name, arr) -> { |
| 222 | + if (arr instanceof UnknownArray u) { |
| 223 | + System.out.println(name + ": unknown encoding " + u.encodingId()); |
| 224 | + } |
| 225 | + }); |
| 226 | + } |
| 227 | +} |
| 228 | +``` |
| 229 | +
|
| 230 | +--- |
| 231 | +
|
| 232 | +## Add a custom encoding |
| 233 | +
|
| 234 | +Three touch-points required: |
| 235 | +
|
| 236 | +**1. Register the encoding ID** in `EncodingId.java`: |
| 237 | +
|
| 238 | +```java |
| 239 | +MY_ENCODING("com.example.my_encoding"), |
| 240 | +``` |
| 241 | +
|
| 242 | +**2. Implement `Encoding`:** |
| 243 | +
|
| 244 | +```java |
| 245 | +public final class MyEncoding implements Encoding { |
| 246 | + |
| 247 | + @Override |
| 248 | + public EncodingId id() { return EncodingId.MY_ENCODING; } |
| 249 | + |
| 250 | + @Override |
| 251 | + public EncodeResult encode(DType dtype, Object data) { return Encoder.encode(dtype, data); } |
| 252 | + |
| 253 | + @Override |
| 254 | + public Array decode(DecodeContext ctx) { return Decoder.decode(ctx); } |
| 255 | + |
| 256 | + private static final class Encoder { /* ... */ } |
| 257 | + private static final class Decoder { /* ... */ } |
| 258 | +} |
| 259 | +``` |
| 260 | +
|
| 261 | +**3. Register via `ServiceLoader`** — add the fully-qualified class name to: |
| 262 | +
|
| 263 | +``` |
| 264 | +META-INF/services/io.github.dfa1.vortex.encoding.Encoding |
| 265 | +``` |
| 266 | +
|
| 267 | +The encoding is then picked up automatically by `EncodingRegistry.loadAll()`. |
| 268 | +To register it only for a specific reader without touching the global registry: |
| 269 | +
|
| 270 | +```java |
| 271 | +EncodingRegistry registry = EncodingRegistry.loadAll(); |
| 272 | +registry.register(new MyEncoding()); |
| 273 | +VortexReader vf = VortexReader.open(path, registry); |
| 274 | +``` |
0 commit comments