|
| 1 | +# Tutorial: Your first Vortex file |
| 2 | + |
| 3 | +This tutorial walks you through writing and reading a Vortex file from scratch. |
| 4 | +You will end up with a working Maven project that stores time-series data in Vortex format |
| 5 | +and reads it back column by column. |
| 6 | + |
| 7 | +**Prerequisites:** Java 25+, Maven 3.9+. |
| 8 | + |
| 9 | +--- |
| 10 | + |
| 11 | +## 1. Create a Maven project |
| 12 | + |
| 13 | +```bash |
| 14 | +mvn archetype:generate \ |
| 15 | + -DgroupId=com.example \ |
| 16 | + -DartifactId=vortex-demo \ |
| 17 | + -DarchetypeArtifactId=maven-archetype-quickstart \ |
| 18 | + -DarchetypeVersion=1.5 \ |
| 19 | + -DinteractiveMode=false |
| 20 | +cd vortex-demo |
| 21 | +``` |
| 22 | + |
| 23 | +Add the dependency to `pom.xml` (inside `<dependencies>`): |
| 24 | + |
| 25 | +```xml |
| 26 | +<dependency> |
| 27 | + <groupId>io.github.dfa1</groupId> |
| 28 | + <artifactId>vortex-java</artifactId> |
| 29 | + <version>0.1.0-SNAPSHOT</version> |
| 30 | +</dependency> |
| 31 | +``` |
| 32 | + |
| 33 | +Set the compiler to Java 25: |
| 34 | + |
| 35 | +```xml |
| 36 | +<properties> |
| 37 | + <maven.compiler.release>25</maven.compiler.release> |
| 38 | +</properties> |
| 39 | +``` |
| 40 | + |
| 41 | +--- |
| 42 | + |
| 43 | +## 2. Define a schema |
| 44 | + |
| 45 | +A Vortex file is a typed struct — every column has a declared type before any data is written. |
| 46 | + |
| 47 | +```java |
| 48 | +import io.github.dfa1.vortex.core.DType; |
| 49 | +import io.github.dfa1.vortex.core.PType; |
| 50 | +import java.util.List; |
| 51 | + |
| 52 | +DType.Struct schema = new DType.Struct( |
| 53 | + List.of("timestamp", "symbol", "price", "volume"), |
| 54 | + List.of( |
| 55 | + new DType.Primitive(PType.I64, false), // unix epoch millis, non-nullable |
| 56 | + DType.UTF8, // ticker symbol |
| 57 | + new DType.Primitive(PType.F64, false), // trade price |
| 58 | + new DType.Primitive(PType.I64, false) // shares traded |
| 59 | + ), |
| 60 | + false // the struct itself is non-nullable |
| 61 | +); |
| 62 | +``` |
| 63 | + |
| 64 | +`PType` mirrors Arrow's physical types: `I8`, `I16`, `I32`, `I64`, `U8`…`U64`, `F32`, `F64`. |
| 65 | +Passing `true` as the second argument to `Primitive` makes the column nullable. |
| 66 | + |
| 67 | +--- |
| 68 | + |
| 69 | +## 3. Write data |
| 70 | + |
| 71 | +```java |
| 72 | +import io.github.dfa1.vortex.writer.VortexWriter; |
| 73 | +import io.github.dfa1.vortex.writer.WriteOptions; |
| 74 | +import java.nio.channels.FileChannel; |
| 75 | +import java.nio.file.Path; |
| 76 | +import java.util.Map; |
| 77 | +import static java.nio.file.StandardOpenOption.*; |
| 78 | + |
| 79 | +Path outPath = Path.of("trades.vortex"); |
| 80 | + |
| 81 | +try (FileChannel ch = FileChannel.open(outPath, CREATE, WRITE, TRUNCATE_EXISTING); |
| 82 | + VortexWriter writer = VortexWriter.create(ch, schema, WriteOptions.defaults())) { |
| 83 | + |
| 84 | + writer.writeChunk(Map.of( |
| 85 | + "timestamp", new long[] {1_700_000_000_000L, 1_700_000_001_000L, 1_700_000_002_000L}, |
| 86 | + "symbol", new String[] {"AAPL", "AAPL", "MSFT"}, |
| 87 | + "price", new double[] {189.95, 190.10, 374.20}, |
| 88 | + "volume", new long[] {100L, 250L, 175L} |
| 89 | + )); |
| 90 | +} |
| 91 | +``` |
| 92 | + |
| 93 | +`writeChunk` takes one batch of rows. Call it multiple times to write multiple chunks — |
| 94 | +each chunk is compressed independently and can be skipped during a scan if zone-map |
| 95 | +statistics rule it out. |
| 96 | + |
| 97 | +The file is complete and readable as soon as `VortexWriter` is closed. |
| 98 | + |
| 99 | +--- |
| 100 | + |
| 101 | +## 4. Read it back |
| 102 | + |
| 103 | +```java |
| 104 | +import io.github.dfa1.vortex.io.VortexReader; |
| 105 | +import io.github.dfa1.vortex.scan.ScanOptions; |
| 106 | +import io.github.dfa1.vortex.core.array.DoubleArray; |
| 107 | +import io.github.dfa1.vortex.core.array.LongArray; |
| 108 | + |
| 109 | +try (VortexReader vf = VortexReader.open(outPath); |
| 110 | + var iter = vf.scan(ScanOptions.all())) { |
| 111 | + |
| 112 | + while (iter.hasNext()) { |
| 113 | + var chunk = iter.next(); // advances to the next batch |
| 114 | + |
| 115 | + LongArray ts = chunk.column("timestamp"); |
| 116 | + DoubleArray price = chunk.column("price"); |
| 117 | + |
| 118 | + for (long i = 0; i < chunk.rowCount(); i++) { |
| 119 | + System.out.printf("%d %.2f%n", ts.getLong(i), price.getDouble(i)); |
| 120 | + } |
| 121 | + // ⚠ do not store references past this point — |
| 122 | + // iter.hasNext() frees the chunk's memory |
| 123 | + } |
| 124 | +} |
| 125 | +``` |
| 126 | + |
| 127 | +**Important:** every chunk lives in an off-heap `Arena`. Calling `iter.hasNext()` closes |
| 128 | +that arena and releases the memory. Read all values before advancing the iterator. |
| 129 | + |
| 130 | +Expected output: |
| 131 | + |
| 132 | +``` |
| 133 | +1700000000000 189.95 |
| 134 | +1700000001000 190.10 |
| 135 | +1700000002000 374.20 |
| 136 | +``` |
| 137 | + |
| 138 | +--- |
| 139 | + |
| 140 | +## 5. Project columns and limit rows |
| 141 | + |
| 142 | +Reading every column is wasteful when you only need two. Use `withColumns` to project, and |
| 143 | +`withLimit` to stop after `n` rows: |
| 144 | + |
| 145 | +```java |
| 146 | +ScanOptions opts = ScanOptions.all() |
| 147 | + .withColumns("symbol", "price") |
| 148 | + .withLimit(2); |
| 149 | + |
| 150 | +try (VortexReader vf = VortexReader.open(outPath); |
| 151 | + var iter = vf.scan(opts)) { |
| 152 | + |
| 153 | + while (iter.hasNext()) { |
| 154 | + var chunk = iter.next(); |
| 155 | + // chunk only contains "symbol" and "price" |
| 156 | + } |
| 157 | +} |
| 158 | +``` |
| 159 | + |
| 160 | +--- |
| 161 | + |
| 162 | +## 6. Inspect with the CLI |
| 163 | + |
| 164 | +Build the CLI fat jar once: |
| 165 | + |
| 166 | +```bash |
| 167 | +./mvnw package -pl cli -am -DskipTests |
| 168 | +``` |
| 169 | + |
| 170 | +Then use it on any file without writing code: |
| 171 | + |
| 172 | +```bash |
| 173 | +# what columns and types does the file have? |
| 174 | +java -jar cli/target/vortex.jar schema trades.vortex |
| 175 | +# → struct<timestamp: I64, symbol: utf8, price: F64, volume: I64> |
| 176 | + |
| 177 | +# how many rows? |
| 178 | +java -jar cli/target/vortex.jar count trades.vortex |
| 179 | +# → 3 |
| 180 | + |
| 181 | +# dump to CSV |
| 182 | +java -jar cli/target/vortex.jar export trades.vortex |
| 183 | +``` |
| 184 | + |
| 185 | +--- |
| 186 | + |
| 187 | +## What's next |
| 188 | + |
| 189 | +- [docs/compatibility.md](compatibility.md) — which encodings are supported |
| 190 | +- [docs/explanation.md](explanation.md) — memory model, testing strategy, benchmarks |
| 191 | +- The `ScanOptions` API supports row filters: `new RowFilter.Gte("volume", 200)` — only rows where volume ≥ 200 |
0 commit comments