Skip to content

Commit c368466

Browse files
committed
feat(xtask): auto-generate the edition registry docs from the Rust declarations
Add a two-stage generator for the edition registry section of docs/specs/editions.md: 1. A new `editions_manifest` example in the `vortex` crate validates the declarations in `vortex::editions` and dumps them (with each edition's computed full encoding set) as a TOML manifest under `target/`. The manifest is a build artifact and is never committed. 2. A new `cargo xtask generate-editions-docs` subcommand runs the example, parses the manifest, and renders an editions table plus a per-encoding index between BEGIN/END markers in `docs/specs/editions.md`, replacing the "Coming soon.." placeholder. The generated-files CI job now regenerates the registry and fails if the committed doc is stale, so the spec can never drift from the declarations. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XKRaAHmFNvb7JoKTJVW8Pw Signed-off-by: Claude <noreply@anthropic.com>
1 parent e6ffba9 commit c368466

9 files changed

Lines changed: 334 additions & 1 deletion

File tree

.github/workflows/ci.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -644,6 +644,9 @@ jobs:
644644
run: |
645645
cargo run --profile ci -p xtask -- generate-fbs
646646
cargo run --profile ci -p xtask -- generate-proto
647+
- name: "regenerate the edition registry docs"
648+
run: |
649+
cargo run --profile ci -p xtask -- generate-editions-docs --profile ci
647650
- name: "regenerate FFI header file"
648651
run: |
649652
cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi

Cargo.lock

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,7 @@ thiserror = "2.0.3"
264264
tokio = { version = "1.52" }
265265
tokio-stream = "0.1.17"
266266
tokio-util = "0.7.17"
267+
toml = "0.9.12"
267268
vortex-array-macros = { version = "0.1.0", path = "./vortex-array-macros" }
268269
# Pull these into non-public crates to support DF 58
269270
tpchgen = { version = "2.0.2", git = "https://github.com/clflushopt/tpchgen-rs.git", rev = "438e9c2dbc25b2fff82c0efc08b3f13b5707874f" }

docs/specs/editions.md

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,4 +59,68 @@ indefinitely, so deprecation never invalidates existing files.
5959

6060
## Edition registry
6161

62-
Coming soon..
62+
<!-- BEGIN generated edition registry: edit the declarations in `vortex/src/editions/` and run `cargo xtask generate-editions-docs` -->
63+
64+
### Editions
65+
66+
| Edition | Status | Minimum Vortex reader | Encodings |
67+
| --- | --- | --- | --- |
68+
| `core2025.05.0` | frozen | 0.36.0 | 23 |
69+
| `core2025.06.0` | frozen | 0.40.0 | 26 |
70+
| `core2025.10.0` | frozen | 0.54.0 | 30 |
71+
| `core2026.07.0` | frozen | 0.65.0 | 31 |
72+
| `unstable2025.05.0` | draft || 1 |
73+
| `unstable2026.02.0` | draft || 2 |
74+
| `unstable2026.04.0` | draft || 8 |
75+
| `unstable2026.06.0` | draft || 9 |
76+
77+
A draft edition is still being assembled: its encoding set may change and it carries no compatibility guarantee until it is frozen.
78+
79+
### Encoding index
80+
81+
Each encoding lists the edition it first joined; it is a member of every later edition of the same family. The minimum Vortex reader is the one declared by that edition.
82+
83+
| Encoding | Since edition | Minimum Vortex reader |
84+
| --- | --- | --- |
85+
| `fastlanes.bitpacked` | `core2025.05.0` | 0.36.0 |
86+
| `fastlanes.delta` | `unstable2025.05.0` ||
87+
| `fastlanes.for` | `core2025.05.0` | 0.36.0 |
88+
| `fastlanes.rle` | `core2025.10.0` | 0.54.0 |
89+
| `vortex.alp` | `core2025.05.0` | 0.36.0 |
90+
| `vortex.alprd` | `core2025.05.0` | 0.36.0 |
91+
| `vortex.bool` | `core2025.05.0` | 0.36.0 |
92+
| `vortex.bytebool` | `core2025.05.0` | 0.36.0 |
93+
| `vortex.chunked` | `core2025.05.0` | 0.36.0 |
94+
| `vortex.constant` | `core2025.05.0` | 0.36.0 |
95+
| `vortex.datetimeparts` | `core2025.05.0` | 0.36.0 |
96+
| `vortex.decimal` | `core2025.05.0` | 0.36.0 |
97+
| `vortex.decimal_byte_parts` | `core2025.05.0` | 0.36.0 |
98+
| `vortex.dict` | `core2025.05.0` | 0.36.0 |
99+
| `vortex.ext` | `core2025.05.0` | 0.36.0 |
100+
| `vortex.fixed_size_list` | `core2025.10.0` | 0.54.0 |
101+
| `vortex.fsst` | `core2025.05.0` | 0.36.0 |
102+
| `vortex.list` | `core2025.05.0` | 0.36.0 |
103+
| `vortex.listview` | `core2025.10.0` | 0.54.0 |
104+
| `vortex.masked` | `core2025.10.0` | 0.54.0 |
105+
| `vortex.null` | `core2025.05.0` | 0.36.0 |
106+
| `vortex.onpair` | `unstable2026.06.0` ||
107+
| `vortex.parquet.variant` | `unstable2026.04.0` ||
108+
| `vortex.patched` | `unstable2026.04.0` ||
109+
| `vortex.pco` | `core2025.06.0` | 0.40.0 |
110+
| `vortex.primitive` | `core2025.05.0` | 0.36.0 |
111+
| `vortex.runend` | `core2025.05.0` | 0.36.0 |
112+
| `vortex.sequence` | `core2025.06.0` | 0.40.0 |
113+
| `vortex.sparse` | `core2025.05.0` | 0.36.0 |
114+
| `vortex.struct` | `core2025.05.0` | 0.36.0 |
115+
| `vortex.tensor.cosine_similarity` | `unstable2026.04.0` ||
116+
| `vortex.tensor.inner_product` | `unstable2026.04.0` ||
117+
| `vortex.tensor.l2_denorm` | `unstable2026.04.0` ||
118+
| `vortex.tensor.l2_norm` | `unstable2026.04.0` ||
119+
| `vortex.varbin` | `core2025.05.0` | 0.36.0 |
120+
| `vortex.varbinview` | `core2025.05.0` | 0.36.0 |
121+
| `vortex.variant` | `core2026.07.0` | 0.65.0 |
122+
| `vortex.zigzag` | `core2025.05.0` | 0.36.0 |
123+
| `vortex.zstd` | `core2025.06.0` | 0.40.0 |
124+
| `vortex.zstd_buffers` | `unstable2026.02.0` ||
125+
126+
<!-- END generated edition registry -->
Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Emits a TOML manifest of the declared Vortex editions.
5+
//!
6+
//! This is the first stage of `cargo xtask generate-editions-docs`: it validates the
7+
//! declarations in `vortex::editions` and dumps them (plus each edition's computed full
8+
//! encoding set) as TOML to the path given as the first argument, conventionally under
9+
//! `target/`. The manifest is a build artifact and is never committed; xtask renders it
10+
//! into the registry section of `docs/specs/editions.md`, which is.
11+
//!
12+
//! Run with: cargo run --example editions_manifest -- <output-path>
13+
14+
use std::error::Error;
15+
use std::fmt::Display;
16+
use std::fmt::Write;
17+
use std::fs;
18+
use std::path::PathBuf;
19+
20+
use vortex::editions::EDITION_DECLARATIONS;
21+
use vortex::editions::EditionSession;
22+
23+
fn main() -> Result<(), Box<dyn Error>> {
24+
let out_path: PathBuf = std::env::args_os()
25+
.nth(1)
26+
.ok_or("usage: editions_manifest <output-path>")?
27+
.into();
28+
29+
let editions = EditionSession::empty();
30+
for declaration in EDITION_DECLARATIONS {
31+
editions.declare(declaration)?;
32+
}
33+
editions.validate()?;
34+
35+
let mut toml = String::new();
36+
toml.push_str(
37+
"# Manifest of the Vortex edition declarations in `vortex/src/editions/`.\n\
38+
# Generated by `cargo run -p vortex --example editions_manifest`; not committed.\n",
39+
);
40+
for edition in editions.editions() {
41+
writeln!(toml)?;
42+
writeln!(toml, "[[editions]]")?;
43+
writeln!(toml, "id = \"{}\"", edition.id)?;
44+
writeln!(toml, "family = \"{}\"", edition.id.family)?;
45+
if let Some(version) = edition.min_vortex_version {
46+
writeln!(toml, "min_vortex_version = \"{version}\"")?;
47+
}
48+
let members = editions.encodings_in(&edition.id);
49+
let added = members
50+
.iter()
51+
.filter(|inclusion| inclusion.since == edition.id)
52+
.map(|inclusion| inclusion.encoding_id);
53+
write_string_array(&mut toml, "added", added)?;
54+
write_string_array(
55+
&mut toml,
56+
"encodings",
57+
members.iter().map(|inclusion| inclusion.encoding_id),
58+
)?;
59+
}
60+
61+
if let Some(parent) = out_path.parent() {
62+
fs::create_dir_all(parent)?;
63+
}
64+
fs::write(&out_path, toml)?;
65+
Ok(())
66+
}
67+
68+
/// Write a multi-line TOML array of strings. Encoding ids are validated to contain only
69+
/// lowercase ASCII, digits, and `._-`, so no TOML escaping is needed.
70+
fn write_string_array(
71+
toml: &mut String,
72+
key: &str,
73+
values: impl Iterator<Item = impl Display>,
74+
) -> Result<(), Box<dyn Error>> {
75+
writeln!(toml, "{key} = [")?;
76+
for value in values {
77+
writeln!(toml, " \"{value}\",")?;
78+
}
79+
writeln!(toml, "]")?;
80+
Ok(())
81+
}

xtask/Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ test = false
2323
anyhow = { workspace = true }
2424
clap = { workspace = true, features = ["derive"] }
2525
prost-build = { workspace = true }
26+
serde = { workspace = true, features = ["derive"] }
27+
toml = { workspace = true }
2628
xshell = { workspace = true }
2729

2830
[lints]

xtask/README.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,14 @@ must be run every time changes are made to one of the .fbs files, or if any are
1919
This will generate the `src/generated` Rust files in the `vortex-proto` crate. This must
2020
be run every time changes are made to one of the .fbs files, or if any are added/deleted.
2121

22+
### `generate-editions-docs`
23+
24+
This regenerates the edition registry section of `docs/specs/editions.md` from the edition
25+
declarations in `vortex/src/editions/`. It first dumps the declarations as a TOML manifest
26+
to `target/editions-manifest.toml` (a build artifact, never committed) by running the
27+
`editions_manifest` example of the `vortex` crate, then renders that manifest into the
28+
markdown between the `BEGIN`/`END generated edition registry` markers. Run it every time
29+
an edition declaration changes; CI fails if the committed registry is stale.
30+
2231

2332

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use std::fmt::Write as _;
5+
use std::fs;
6+
use std::path::PathBuf;
7+
8+
use anyhow::Context;
9+
use anyhow::bail;
10+
use serde::Deserialize;
11+
use xshell::Shell;
12+
use xshell::cmd;
13+
14+
/// Marker lines delimiting the generated registry section of the spec. Everything between
15+
/// them is owned by this generator; hand edits there are overwritten.
16+
static BEGIN_MARKER: &str = "<!-- BEGIN generated edition registry: edit the declarations in \
17+
`vortex/src/editions/` and run `cargo xtask generate-editions-docs` -->";
18+
static END_MARKER: &str = "<!-- END generated edition registry -->";
19+
20+
/// The manifest emitted by `cargo run -p vortex --example editions_manifest`.
21+
#[derive(Deserialize)]
22+
struct Manifest {
23+
editions: Vec<ManifestEdition>,
24+
}
25+
26+
#[derive(Deserialize)]
27+
struct ManifestEdition {
28+
id: String,
29+
/// Absent for draft editions.
30+
min_vortex_version: Option<String>,
31+
/// The encodings that join the family at this edition.
32+
added: Vec<String>,
33+
/// The edition's full computed encoding set.
34+
encodings: Vec<String>,
35+
}
36+
37+
impl ManifestEdition {
38+
fn status(&self) -> &'static str {
39+
match self.min_vortex_version {
40+
Some(_) => "frozen",
41+
None => "draft",
42+
}
43+
}
44+
45+
fn min_reader(&self) -> &str {
46+
self.min_vortex_version.as_deref().unwrap_or("—")
47+
}
48+
}
49+
50+
/// Regenerate the edition registry section of `docs/specs/editions.md` from the edition
51+
/// declarations in the `vortex` crate, via a TOML manifest written under `target/`.
52+
pub fn generate_editions_docs(profile: Option<String>) -> anyhow::Result<()> {
53+
let sh = Shell::new()?;
54+
let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("..");
55+
sh.change_dir(&root);
56+
57+
let manifest_path = root.join("target/editions-manifest.toml");
58+
let profile_args: Vec<String> = profile
59+
.into_iter()
60+
.flat_map(|p| ["--profile".to_string(), p])
61+
.collect();
62+
cmd!(
63+
sh,
64+
"cargo run {profile_args...} -p vortex --example editions_manifest -- {manifest_path}"
65+
)
66+
.run()?;
67+
68+
let manifest: Manifest = toml::from_str(&fs::read_to_string(&manifest_path)?)
69+
.context("parsing the editions manifest")?;
70+
71+
let spec_path = root.join("docs/specs/editions.md");
72+
let spec = fs::read_to_string(&spec_path)?;
73+
let (Some(begin), Some(end)) = (spec.find(BEGIN_MARKER), spec.find(END_MARKER)) else {
74+
bail!(
75+
"{} must contain the generated-registry BEGIN/END marker comments",
76+
spec_path.display()
77+
);
78+
};
79+
if end < begin {
80+
bail!(
81+
"generated-registry markers are out of order in {}",
82+
spec_path.display()
83+
);
84+
}
85+
86+
let mut spliced = String::new();
87+
spliced.push_str(&spec[..begin + BEGIN_MARKER.len()]);
88+
spliced.push_str("\n\n");
89+
spliced.push_str(&render(&manifest)?);
90+
spliced.push('\n');
91+
spliced.push_str(&spec[end..]);
92+
fs::write(&spec_path, spliced)?;
93+
94+
println!(
95+
"regenerated the edition registry in {}",
96+
spec_path.display()
97+
);
98+
Ok(())
99+
}
100+
101+
/// Render the registry markdown that goes between the markers.
102+
fn render(manifest: &Manifest) -> anyhow::Result<String> {
103+
let mut md = String::new();
104+
105+
writeln!(
106+
md,
107+
"### Editions\n\n\
108+
| Edition | Status | Minimum Vortex reader | Encodings |\n\
109+
| --- | --- | --- | --- |"
110+
)?;
111+
for edition in &manifest.editions {
112+
writeln!(
113+
md,
114+
"| `{}` | {} | {} | {} |",
115+
edition.id,
116+
edition.status(),
117+
edition.min_reader(),
118+
edition.encodings.len(),
119+
)?;
120+
}
121+
122+
writeln!(
123+
md,
124+
"\nA draft edition is still being assembled: its encoding set may change and it \
125+
carries no compatibility guarantee until it is frozen.\n\n\
126+
### Encoding index\n\n\
127+
Each encoding lists the edition it first joined; it is a member of every later \
128+
edition of the same family. The minimum Vortex reader is the one declared by that \
129+
edition.\n\n\
130+
| Encoding | Since edition | Minimum Vortex reader |\n\
131+
| --- | --- | --- |"
132+
)?;
133+
let mut rows: Vec<(&String, &ManifestEdition)> = manifest
134+
.editions
135+
.iter()
136+
.flat_map(|edition| {
137+
edition
138+
.added
139+
.iter()
140+
.map(move |encoding| (encoding, edition))
141+
})
142+
.collect();
143+
// An encoding joins exactly one edition, so its id alone is a total order.
144+
rows.sort_by(|(a, _), (b, _)| a.cmp(b));
145+
for (encoding, edition) in rows {
146+
writeln!(
147+
md,
148+
"| `{}` | `{}` | {} |",
149+
encoding,
150+
edition.id,
151+
edition.min_reader(),
152+
)?;
153+
}
154+
155+
Ok(md)
156+
}

0 commit comments

Comments
 (0)