diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daba1d91a87..4bb2f4ed631 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -644,6 +644,9 @@ jobs: run: | cargo run --profile ci -p xtask -- generate-fbs cargo run --profile ci -p xtask -- generate-proto + - name: "regenerate the edition registry docs" + run: | + cargo run --profile ci -p xtask -- generate-editions-docs --profile ci - name: "regenerate FFI header file" run: | cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi diff --git a/Cargo.lock b/Cargo.lock index 0747e266768..0332bb74c0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11290,6 +11290,8 @@ dependencies = [ "anyhow", "clap", "prost-build", + "serde", + "toml", "xshell", ] diff --git a/Cargo.toml b/Cargo.toml index 7581d1c9653..bcf313c5d9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -264,6 +264,7 @@ thiserror = "2.0.3" tokio = { version = "1.52" } tokio-stream = "0.1.17" tokio-util = "0.7.17" +toml = "0.9.12" vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" } # Pull these into non-public crates to support DF 58 tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" } diff --git a/docs/index.md b/docs/index.md index b75a438f913..9b49f4356dd 100644 --- a/docs/index.md +++ b/docs/index.md @@ -91,6 +91,12 @@ internals. Build and benchmark locally. - **Language bindings**: First-class support for Python (PyO3), Java (JNI + Spark/Trino connectors), and C/C++ (FFI). +## Editions + +Vortex writers target **[editions](specs/editions.md)** — named, frozen sets of encodings that +carry a forever read-compatibility guarantee. See every edition in the +[edition registry](specs/editions/index.md). + ```{toctree} --- hidden: diff --git a/docs/specs/editions.md b/docs/specs/editions.md index d76bf954f1d..165a226ef1d 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -19,7 +19,7 @@ editions is the earliest version of vortex required to read that file. ## Resolving an unknown-encoding error If a read failed with an unknown encoding ID and pointed you here, the reader met an encoding -it does not support. Find the encoding ID in the [registry](#edition-registry) below: +it does not support. Find the encoding ID in the [registry](editions/index.md): 1. **The ID is listed under an edition.** The file is newer than your Vortex build. Upgrade to at least that edition's required Vortex release and the file will read. @@ -59,4 +59,14 @@ indefinitely, so deprecation never invalidates existing files. ## Edition registry -Coming soon.. +The full registry — every edition, its encoding set, and each encoding's membership — is +generated from the declarations in `vortex/src/editions/` and published as its own pages, +one per edition: see the [edition registry](editions/index.md). + +```{toctree} +--- +hidden: +--- + +editions/index +``` diff --git a/docs/specs/editions/core2025.05.0.md b/docs/specs/editions/core2025.05.0.md new file mode 100644 index 00000000000..5aed3d9d82a --- /dev/null +++ b/docs/specs/editions/core2025.05.0.md @@ -0,0 +1,35 @@ + + +# core2025.05.0 + +An edition of the `core` family. Frozen — readable by Vortex 0.36.0 and newer. + +Adds 23 encodings: `fastlanes.bitpacked`, `fastlanes.for`, `vortex.alp`, `vortex.alprd`, `vortex.bool`, `vortex.bytebool`, `vortex.chunked`, `vortex.constant`, `vortex.datetimeparts`, `vortex.decimal`, `vortex.decimal_byte_parts`, `vortex.dict`, `vortex.ext`, `vortex.fsst`, `vortex.list`, `vortex.null`, `vortex.primitive`, `vortex.runend`, `vortex.sparse`, `vortex.struct`, `vortex.varbin`, `vortex.varbinview`, `vortex.zigzag` + +## Encodings (23) + +| Encoding | Since edition | +| --- | --- | +| `fastlanes.bitpacked` | [`core2025.05.0`](core2025.05.0.md) | +| `fastlanes.for` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.alp` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.alprd` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.bool` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.bytebool` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.chunked` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.constant` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.datetimeparts` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.decimal` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.decimal_byte_parts` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.dict` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.ext` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.fsst` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.list` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.null` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.primitive` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.runend` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.sparse` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.struct` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.varbin` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.varbinview` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.zigzag` | [`core2025.05.0`](core2025.05.0.md) | diff --git a/docs/specs/editions/core2025.06.0.md b/docs/specs/editions/core2025.06.0.md new file mode 100644 index 00000000000..28ff3ce6ec4 --- /dev/null +++ b/docs/specs/editions/core2025.06.0.md @@ -0,0 +1,38 @@ + + +# core2025.06.0 + +An edition of the `core` family. Frozen — readable by Vortex 0.40.0 and newer. + +Adds 3 encodings: `vortex.pco`, `vortex.sequence`, `vortex.zstd` + +## Encodings (26) + +| Encoding | Since edition | +| --- | --- | +| `fastlanes.bitpacked` | [`core2025.05.0`](core2025.05.0.md) | +| `fastlanes.for` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.alp` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.alprd` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.bool` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.bytebool` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.chunked` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.constant` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.datetimeparts` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.decimal` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.decimal_byte_parts` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.dict` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.ext` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.fsst` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.list` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.null` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.pco` | [`core2025.06.0`](core2025.06.0.md) | +| `vortex.primitive` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.runend` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.sequence` | [`core2025.06.0`](core2025.06.0.md) | +| `vortex.sparse` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.struct` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.varbin` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.varbinview` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.zigzag` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.zstd` | [`core2025.06.0`](core2025.06.0.md) | diff --git a/docs/specs/editions/core2025.10.0.md b/docs/specs/editions/core2025.10.0.md new file mode 100644 index 00000000000..cd487d67a5b --- /dev/null +++ b/docs/specs/editions/core2025.10.0.md @@ -0,0 +1,42 @@ + + +# core2025.10.0 + +An edition of the `core` family. Frozen — readable by Vortex 0.54.0 and newer. + +Adds 4 encodings: `fastlanes.rle`, `vortex.fixed_size_list`, `vortex.listview`, `vortex.masked` + +## Encodings (30) + +| Encoding | Since edition | +| --- | --- | +| `fastlanes.bitpacked` | [`core2025.05.0`](core2025.05.0.md) | +| `fastlanes.for` | [`core2025.05.0`](core2025.05.0.md) | +| `fastlanes.rle` | [`core2025.10.0`](core2025.10.0.md) | +| `vortex.alp` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.alprd` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.bool` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.bytebool` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.chunked` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.constant` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.datetimeparts` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.decimal` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.decimal_byte_parts` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.dict` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.ext` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.fixed_size_list` | [`core2025.10.0`](core2025.10.0.md) | +| `vortex.fsst` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.list` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.listview` | [`core2025.10.0`](core2025.10.0.md) | +| `vortex.masked` | [`core2025.10.0`](core2025.10.0.md) | +| `vortex.null` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.pco` | [`core2025.06.0`](core2025.06.0.md) | +| `vortex.primitive` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.runend` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.sequence` | [`core2025.06.0`](core2025.06.0.md) | +| `vortex.sparse` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.struct` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.varbin` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.varbinview` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.zigzag` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.zstd` | [`core2025.06.0`](core2025.06.0.md) | diff --git a/docs/specs/editions/core2026.07.0.md b/docs/specs/editions/core2026.07.0.md new file mode 100644 index 00000000000..88c5b307d14 --- /dev/null +++ b/docs/specs/editions/core2026.07.0.md @@ -0,0 +1,43 @@ + + +# core2026.07.0 + +An edition of the `core` family. Frozen — readable by Vortex 0.65.0 and newer. + +Adds 1 encoding: `vortex.variant` + +## Encodings (31) + +| Encoding | Since edition | +| --- | --- | +| `fastlanes.bitpacked` | [`core2025.05.0`](core2025.05.0.md) | +| `fastlanes.for` | [`core2025.05.0`](core2025.05.0.md) | +| `fastlanes.rle` | [`core2025.10.0`](core2025.10.0.md) | +| `vortex.alp` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.alprd` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.bool` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.bytebool` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.chunked` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.constant` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.datetimeparts` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.decimal` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.decimal_byte_parts` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.dict` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.ext` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.fixed_size_list` | [`core2025.10.0`](core2025.10.0.md) | +| `vortex.fsst` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.list` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.listview` | [`core2025.10.0`](core2025.10.0.md) | +| `vortex.masked` | [`core2025.10.0`](core2025.10.0.md) | +| `vortex.null` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.pco` | [`core2025.06.0`](core2025.06.0.md) | +| `vortex.primitive` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.runend` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.sequence` | [`core2025.06.0`](core2025.06.0.md) | +| `vortex.sparse` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.struct` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.varbin` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.varbinview` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.variant` | [`core2026.07.0`](core2026.07.0.md) | +| `vortex.zigzag` | [`core2025.05.0`](core2025.05.0.md) | +| `vortex.zstd` | [`core2025.06.0`](core2025.06.0.md) | diff --git a/docs/specs/editions/index.md b/docs/specs/editions/index.md new file mode 100644 index 00000000000..a6094e2a016 --- /dev/null +++ b/docs/specs/editions/index.md @@ -0,0 +1,82 @@ + + +# Edition registry + +Every Vortex [edition](../editions.md), with one page per edition. + +## Editions + +| Edition | Status | Minimum Vortex reader | Encodings | +| --- | --- | --- | --- | +| [`core2025.05.0`](core2025.05.0.md) | frozen | 0.36.0 | 23 | +| [`core2025.06.0`](core2025.06.0.md) | frozen | 0.40.0 | 26 | +| [`core2025.10.0`](core2025.10.0.md) | frozen | 0.54.0 | 30 | +| [`core2026.07.0`](core2026.07.0.md) | frozen | 0.65.0 | 31 | +| [`unstable2025.05.0`](unstable2025.05.0.md) | draft | — | 1 | +| [`unstable2026.02.0`](unstable2026.02.0.md) | draft | — | 2 | +| [`unstable2026.04.0`](unstable2026.04.0.md) | draft | — | 8 | +| [`unstable2026.06.0`](unstable2026.06.0.md) | draft | — | 9 | + +A draft edition is still being assembled: its encoding set may change and it carries no compatibility guarantee until it is frozen. + +## Encoding index + +Each encoding lists the edition it first joined; it is a member of every later edition of the same family. The minimum Vortex reader is the one declared by that edition. + +| Encoding | Since edition | Minimum Vortex reader | +| --- | --- | --- | +| `fastlanes.bitpacked` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `fastlanes.delta` | [`unstable2025.05.0`](unstable2025.05.0.md) | — | +| `fastlanes.for` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `fastlanes.rle` | [`core2025.10.0`](core2025.10.0.md) | 0.54.0 | +| `vortex.alp` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.alprd` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.bool` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.bytebool` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.chunked` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.constant` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.datetimeparts` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.decimal` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.decimal_byte_parts` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.dict` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.ext` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.fixed_size_list` | [`core2025.10.0`](core2025.10.0.md) | 0.54.0 | +| `vortex.fsst` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.list` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.listview` | [`core2025.10.0`](core2025.10.0.md) | 0.54.0 | +| `vortex.masked` | [`core2025.10.0`](core2025.10.0.md) | 0.54.0 | +| `vortex.null` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.onpair` | [`unstable2026.06.0`](unstable2026.06.0.md) | — | +| `vortex.parquet.variant` | [`unstable2026.04.0`](unstable2026.04.0.md) | — | +| `vortex.patched` | [`unstable2026.04.0`](unstable2026.04.0.md) | — | +| `vortex.pco` | [`core2025.06.0`](core2025.06.0.md) | 0.40.0 | +| `vortex.primitive` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.runend` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.sequence` | [`core2025.06.0`](core2025.06.0.md) | 0.40.0 | +| `vortex.sparse` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.struct` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.tensor.cosine_similarity` | [`unstable2026.04.0`](unstable2026.04.0.md) | — | +| `vortex.tensor.inner_product` | [`unstable2026.04.0`](unstable2026.04.0.md) | — | +| `vortex.tensor.l2_denorm` | [`unstable2026.04.0`](unstable2026.04.0.md) | — | +| `vortex.tensor.l2_norm` | [`unstable2026.04.0`](unstable2026.04.0.md) | — | +| `vortex.varbin` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.varbinview` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.variant` | [`core2026.07.0`](core2026.07.0.md) | 0.65.0 | +| `vortex.zigzag` | [`core2025.05.0`](core2025.05.0.md) | 0.36.0 | +| `vortex.zstd` | [`core2025.06.0`](core2025.06.0.md) | 0.40.0 | +| `vortex.zstd_buffers` | [`unstable2026.02.0`](unstable2026.02.0.md) | — | + +```{toctree} +--- +hidden: +--- + +core2025.05.0 +core2025.06.0 +core2025.10.0 +core2026.07.0 +unstable2025.05.0 +unstable2026.02.0 +unstable2026.04.0 +unstable2026.06.0 +``` diff --git a/docs/specs/editions/unstable2025.05.0.md b/docs/specs/editions/unstable2025.05.0.md new file mode 100644 index 00000000000..815d0406a03 --- /dev/null +++ b/docs/specs/editions/unstable2025.05.0.md @@ -0,0 +1,13 @@ + + +# unstable2025.05.0 + +A **draft** edition of the `unstable` family — still being assembled, it carries no compatibility guarantee yet. + +Adds 1 encoding: `fastlanes.delta` + +## Encodings (1) + +| Encoding | Since edition | +| --- | --- | +| `fastlanes.delta` | [`unstable2025.05.0`](unstable2025.05.0.md) | diff --git a/docs/specs/editions/unstable2026.02.0.md b/docs/specs/editions/unstable2026.02.0.md new file mode 100644 index 00000000000..56db3292ff6 --- /dev/null +++ b/docs/specs/editions/unstable2026.02.0.md @@ -0,0 +1,14 @@ + + +# unstable2026.02.0 + +A **draft** edition of the `unstable` family — still being assembled, it carries no compatibility guarantee yet. + +Adds 1 encoding: `vortex.zstd_buffers` + +## Encodings (2) + +| Encoding | Since edition | +| --- | --- | +| `fastlanes.delta` | [`unstable2025.05.0`](unstable2025.05.0.md) | +| `vortex.zstd_buffers` | [`unstable2026.02.0`](unstable2026.02.0.md) | diff --git a/docs/specs/editions/unstable2026.04.0.md b/docs/specs/editions/unstable2026.04.0.md new file mode 100644 index 00000000000..f6eef0777d5 --- /dev/null +++ b/docs/specs/editions/unstable2026.04.0.md @@ -0,0 +1,20 @@ + + +# unstable2026.04.0 + +A **draft** edition of the `unstable` family — still being assembled, it carries no compatibility guarantee yet. + +Adds 6 encodings: `vortex.parquet.variant`, `vortex.patched`, `vortex.tensor.cosine_similarity`, `vortex.tensor.inner_product`, `vortex.tensor.l2_denorm`, `vortex.tensor.l2_norm` + +## Encodings (8) + +| Encoding | Since edition | +| --- | --- | +| `fastlanes.delta` | [`unstable2025.05.0`](unstable2025.05.0.md) | +| `vortex.parquet.variant` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.patched` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.tensor.cosine_similarity` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.tensor.inner_product` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.tensor.l2_denorm` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.tensor.l2_norm` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.zstd_buffers` | [`unstable2026.02.0`](unstable2026.02.0.md) | diff --git a/docs/specs/editions/unstable2026.06.0.md b/docs/specs/editions/unstable2026.06.0.md new file mode 100644 index 00000000000..d8ad2043235 --- /dev/null +++ b/docs/specs/editions/unstable2026.06.0.md @@ -0,0 +1,21 @@ + + +# unstable2026.06.0 + +A **draft** edition of the `unstable` family — still being assembled, it carries no compatibility guarantee yet. + +Adds 1 encoding: `vortex.onpair` + +## Encodings (9) + +| Encoding | Since edition | +| --- | --- | +| `fastlanes.delta` | [`unstable2025.05.0`](unstable2025.05.0.md) | +| `vortex.onpair` | [`unstable2026.06.0`](unstable2026.06.0.md) | +| `vortex.parquet.variant` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.patched` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.tensor.cosine_similarity` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.tensor.inner_product` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.tensor.l2_denorm` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.tensor.l2_norm` | [`unstable2026.04.0`](unstable2026.04.0.md) | +| `vortex.zstd_buffers` | [`unstable2026.02.0`](unstable2026.02.0.md) | diff --git a/vortex/examples/editions_manifest.rs b/vortex/examples/editions_manifest.rs new file mode 100644 index 00000000000..1cb32b0685f --- /dev/null +++ b/vortex/examples/editions_manifest.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Emits a TOML manifest of the declared Vortex editions. +//! +//! This is the first stage of `cargo xtask generate-editions-docs`: it validates the +//! declarations in `vortex::editions` and dumps them (plus each edition's computed full +//! encoding set) as TOML to the path given as the first argument, conventionally under +//! `target/`. The manifest is a build artifact and is never committed; xtask renders it +//! into the registry section of `docs/specs/editions.md`, which is. +//! +//! Run with: cargo run --example editions_manifest -- + +use std::error::Error; +use std::fmt::Display; +use std::fmt::Write; +use std::fs; +use std::path::PathBuf; + +use vortex::editions::EDITION_DECLARATIONS; +use vortex::editions::EditionSession; + +fn main() -> Result<(), Box> { + let out_path: PathBuf = std::env::args_os() + .nth(1) + .ok_or("usage: editions_manifest ")? + .into(); + + let editions = EditionSession::empty(); + for declaration in EDITION_DECLARATIONS { + editions.declare(declaration)?; + } + editions.validate()?; + + let mut toml = String::new(); + toml.push_str( + "# Manifest of the Vortex edition declarations in `vortex/src/editions/`.\n\ + # Generated by `cargo run -p vortex --example editions_manifest`; not committed.\n", + ); + for edition in editions.editions() { + writeln!(toml)?; + writeln!(toml, "[[editions]]")?; + writeln!(toml, "id = \"{}\"", edition.id)?; + writeln!(toml, "family = \"{}\"", edition.id.family)?; + if let Some(version) = edition.min_vortex_version { + writeln!(toml, "min_vortex_version = \"{version}\"")?; + } + let members = editions.encodings_in(&edition.id); + let added = members + .iter() + .filter(|inclusion| inclusion.since == edition.id) + .map(|inclusion| inclusion.encoding_id); + write_string_array(&mut toml, "added", added)?; + write_string_array( + &mut toml, + "encodings", + members.iter().map(|inclusion| inclusion.encoding_id), + )?; + } + + if let Some(parent) = out_path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(&out_path, toml)?; + Ok(()) +} + +/// Write a multi-line TOML array of strings. Encoding ids are validated to contain only +/// lowercase ASCII, digits, and `._-`, so no TOML escaping is needed. +fn write_string_array( + toml: &mut String, + key: &str, + values: impl Iterator, +) -> Result<(), Box> { + writeln!(toml, "{key} = [")?; + for value in values { + writeln!(toml, " \"{value}\",")?; + } + writeln!(toml, "]")?; + Ok(()) +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index eae2db43413..40a7372078a 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,6 +23,8 @@ test = false anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } prost-build = { workspace = true } +serde = { workspace = true, features = ["derive"] } +toml = { workspace = true } xshell = { workspace = true } [lints] diff --git a/xtask/README.md b/xtask/README.md index 5e551b34d19..37feb93e75e 100644 --- a/xtask/README.md +++ b/xtask/README.md @@ -19,5 +19,16 @@ must be run every time changes are made to one of the .fbs files, or if any are This will generate the `src/generated` Rust files in the `vortex-proto` crate. This must be run every time changes are made to one of the .fbs files, or if any are added/deleted. +### `generate-editions-docs` + +This regenerates the edition registry pages under `docs/specs/editions/` — an `index.md` +listing every edition plus one page per edition — from the edition declarations in +`vortex/src/editions/`. It first dumps the declarations as a TOML manifest to +`target/editions-manifest.toml` (a build artifact, never committed) by running the +`editions_manifest` example of the `vortex` crate, then renders that manifest into the +registry pages, replacing the whole directory. The static pages (`docs/specs/editions.md` +and the docs landing page) link to the generated registry. Run it every time an edition +declaration changes; CI fails if the committed registry is stale. + diff --git a/xtask/src/generate_editions_docs.rs b/xtask/src/generate_editions_docs.rs new file mode 100644 index 00000000000..14938b1fd59 --- /dev/null +++ b/xtask/src/generate_editions_docs.rs @@ -0,0 +1,232 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::fs; +use std::path::PathBuf; + +use anyhow::Context; +use serde::Deserialize; +use xshell::Shell; +use xshell::cmd; + +/// Header comment stamped on every fully generated page. +static GENERATED_HEADER: &str = ""; + +/// The manifest emitted by `cargo run -p vortex --example editions_manifest`. +#[derive(Deserialize)] +struct Manifest { + editions: Vec, +} + +#[derive(Deserialize)] +struct ManifestEdition { + id: String, + family: String, + /// Absent for draft editions. + min_vortex_version: Option, + /// The encodings that join the family at this edition. + added: Vec, + /// The edition's full computed encoding set. + encodings: Vec, +} + +impl ManifestEdition { + fn status(&self) -> &'static str { + match self.min_vortex_version { + Some(_) => "frozen", + None => "draft", + } + } + + fn min_reader(&self) -> &str { + self.min_vortex_version.as_deref().unwrap_or("—") + } + + /// This edition's page filename within `docs/specs/editions/`. + fn file_name(&self) -> String { + format!("{}.md", self.id) + } + + /// A link to this edition's page from a sibling page in `docs/specs/editions/`. + fn sibling_link(&self) -> String { + format!("[`{}`]({})", self.id, self.file_name()) + } +} + +/// Regenerate the edition registry pages under `docs/specs/editions/` (an index page plus +/// one page per edition) from the edition declarations in the `vortex` crate, via a TOML +/// manifest written under `target/`. +pub fn generate_editions_docs(profile: Option) -> anyhow::Result<()> { + let sh = Shell::new()?; + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".."); + sh.change_dir(&root); + + let manifest_path = root.join("target/editions-manifest.toml"); + let profile_args: Vec = profile + .into_iter() + .flat_map(|p| ["--profile".to_string(), p]) + .collect(); + cmd!( + sh, + "cargo run {profile_args...} -p vortex --example editions_manifest -- {manifest_path}" + ) + .run()?; + + let manifest: Manifest = toml::from_str(&fs::read_to_string(&manifest_path)?) + .context("parsing the editions manifest")?; + + // The since-edition of every encoding, for the per-edition membership tables. + let since: BTreeMap<&String, &ManifestEdition> = manifest + .editions + .iter() + .flat_map(|edition| { + edition + .added + .iter() + .map(move |encoding| (encoding, edition)) + }) + .collect(); + + // The registry directory is wholly generated: drop stale pages before writing. + let registry_dir = root.join("docs/specs/editions"); + fs::create_dir_all(®istry_dir)?; + for entry in fs::read_dir(®istry_dir)? { + let path = entry?.path(); + if path.extension().is_some_and(|ext| ext == "md") { + fs::remove_file(path)?; + } + } + + fs::write(registry_dir.join("index.md"), render_index(&manifest)?)?; + for edition in &manifest.editions { + fs::write( + registry_dir.join(edition.file_name()), + render_edition(edition, &since)?, + )?; + } + println!( + "regenerated the edition registry in {}", + registry_dir.display() + ); + + Ok(()) +} + +/// Render the registry index page, `docs/specs/editions/index.md`. +fn render_index(manifest: &Manifest) -> anyhow::Result { + let mut md = String::new(); + writeln!(md, "{GENERATED_HEADER}\n\n# Edition registry\n")?; + writeln!( + md, + "Every Vortex [edition](../editions.md), with one page per edition.\n\n\ + ## Editions\n\n\ + | Edition | Status | Minimum Vortex reader | Encodings |\n\ + | --- | --- | --- | --- |" + )?; + for edition in &manifest.editions { + writeln!( + md, + "| {} | {} | {} | {} |", + edition.sibling_link(), + edition.status(), + edition.min_reader(), + edition.encodings.len(), + )?; + } + + writeln!( + md, + "\nA draft edition is still being assembled: its encoding set may change and it \ + carries no compatibility guarantee until it is frozen.\n\n\ + ## Encoding index\n\n\ + Each encoding lists the edition it first joined; it is a member of every later \ + edition of the same family. The minimum Vortex reader is the one declared by that \ + edition.\n\n\ + | Encoding | Since edition | Minimum Vortex reader |\n\ + | --- | --- | --- |" + )?; + let mut rows: Vec<(&String, &ManifestEdition)> = manifest + .editions + .iter() + .flat_map(|edition| { + edition + .added + .iter() + .map(move |encoding| (encoding, edition)) + }) + .collect(); + // An encoding joins exactly one edition, so its id alone is a total order. + rows.sort_by(|(a, _), (b, _)| a.cmp(b)); + for (encoding, edition) in rows { + writeln!( + md, + "| `{}` | {} | {} |", + encoding, + edition.sibling_link(), + edition.min_reader(), + )?; + } + + writeln!(md, "\n```{{toctree}}\n---\nhidden:\n---\n")?; + for edition in &manifest.editions { + writeln!(md, "{}", edition.id)?; + } + writeln!(md, "```")?; + Ok(md) +} + +/// Render one edition's page, `docs/specs/editions/.md`. +fn render_edition( + edition: &ManifestEdition, + since: &BTreeMap<&String, &ManifestEdition>, +) -> anyhow::Result { + let mut md = String::new(); + writeln!(md, "{GENERATED_HEADER}\n\n# {}\n", edition.id)?; + match &edition.min_vortex_version { + Some(version) => writeln!( + md, + "An edition of the `{}` family. Frozen — readable by Vortex {version} and newer.", + edition.family + )?, + None => writeln!( + md, + "A **draft** edition of the `{}` family — still being assembled, it carries no \ + compatibility guarantee yet.", + edition.family + )?, + } + + let added = match edition.added.len() { + 1 => "1 encoding".to_string(), + n => format!("{n} encodings"), + }; + writeln!( + md, + "\nAdds {added}: {}", + edition + .added + .iter() + .map(|encoding| format!("`{encoding}`")) + .collect::>() + .join(", ") + )?; + + writeln!( + md, + "\n## Encodings ({})\n\n\ + | Encoding | Since edition |\n\ + | --- | --- |", + edition.encodings.len() + )?; + for encoding in &edition.encodings { + let joined = since + .get(encoding) + .map(|e| e.sibling_link()) + .unwrap_or_else(|| "—".to_string()); + writeln!(md, "| `{encoding}` | {joined} |")?; + } + Ok(md) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 1155ee3246a..259a671a87b 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod generate_editions_docs; mod generate_fbs; mod generate_proto; use clap::Parser; +use crate::generate_editions_docs::generate_editions_docs; use crate::generate_fbs::generate_fbs; use crate::generate_proto::generate_proto; @@ -16,6 +18,10 @@ struct Xtask { } #[derive(clap::Subcommand)] +#[expect( + clippy::enum_variant_names, + reason = "variants mirror the generate-* subcommand names" +)] enum Commands { /// Subcommand to regenerate flatbuffers language bindings for the Rust project. #[command(name = "generate-fbs")] @@ -23,6 +29,14 @@ enum Commands { /// Subcommand to regenerate protobuf language bindings for the Rust project. #[command(name = "generate-proto")] GenerateProto, + /// Subcommand to regenerate the edition registry in `docs/specs/editions.md` from the + /// edition declarations in the `vortex` crate. + #[command(name = "generate-editions-docs")] + GenerateEditionsDocs { + /// Cargo profile to build the manifest emitter with (defaults to dev). + #[arg(long)] + profile: Option, + }, } fn main() -> anyhow::Result<()> { @@ -30,6 +44,7 @@ fn main() -> anyhow::Result<()> { match cli.command { Commands::GenerateFlatbuffers => generate_fbs()?, Commands::GenerateProto => generate_proto()?, + Commands::GenerateEditionsDocs { profile } => generate_editions_docs(profile)?, } Ok(()) }