diff --git a/Cargo.lock b/Cargo.lock index e79211b5436..6e5f5110e13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10093,6 +10093,14 @@ dependencies = [ "zip", ] +[[package]] +name = "vortex-edition" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "vortex-error" version = "0.1.0" @@ -11277,6 +11285,7 @@ dependencies = [ "anyhow", "clap", "prost-build", + "vortex-edition", "xshell", ] diff --git a/Cargo.toml b/Cargo.toml index 68be0c50653..471bd7ce6fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "vortex-mask", "vortex-utils", "vortex-session", + "vortex-edition", "vortex-flatbuffers", "vortex-metrics", "vortex-io", @@ -295,6 +296,7 @@ vortex-compute = { version = "0.1.0", path = "./vortex-compute", default-feature vortex-datafusion = { version = "0.1.0", path = "./vortex-datafusion", default-features = false } vortex-datetime-parts = { version = "0.1.0", path = "./encodings/datetime-parts", default-features = false } vortex-decimal-byte-parts = { version = "0.1.0", path = "encodings/decimal-byte-parts", default-features = false } +vortex-edition = { version = "0.1.0", path = "./vortex-edition", default-features = false } vortex-error = { version = "0.1.0", path = "./vortex-error", default-features = false } vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false } vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false } diff --git a/docs/developer-guide/index.md b/docs/developer-guide/index.md index e6eec4bfb70..dd7e4c74b66 100644 --- a/docs/developer-guide/index.md +++ b/docs/developer-guide/index.md @@ -27,6 +27,7 @@ internals/execution internals/stats-pruning internals/io internals/serialization +internals/editions internals/cuda ``` diff --git a/docs/developer-guide/internals/editions.md b/docs/developer-guide/internals/editions.md new file mode 100644 index 00000000000..32378f79eeb --- /dev/null +++ b/docs/developer-guide/internals/editions.md @@ -0,0 +1,175 @@ +# Editions + +How Vortex [editions](/specs/editions) are defined, computed, and published. The public spec +describes what an edition *promises*; this page describes how the machinery works and what to +touch when stabilizing an encoding or cutting an edition. + +## Single source of truth + +Everything derives from two static data structures in the dependency-free `vortex-edition` +crate: + +1. **The edition list** — `EDITIONS`: each edition's identifier + (`EditionId { family, year, month, version }`) and a `draft` flag. Nothing else is + stored: the freeze date is carried by the identifier itself (`core2026.07.0` was frozen + in 2026-07), and everything further — status, lineage, the encoding sets, the required + Vortex release — is derived. + +2. **Per-encoding membership declarations** — `ENCODINGS`: each encoding declares the edition + it has been a member of *since*: + + ```rust + EncodingDecl { + id: "vortex.fsst", + since: CORE_2026_07_0, + required_vortex_release: None, + } + ``` + + An encoding with `since: E` is a member of `E` and every later edition of the same family. + The membership edge also carries `required_vortex_release` — the earliest release able to + read and execute this encoding, recorded from evidence such as compat-fixture history + (`None` until recorded). Encoding IDs are globally unique across everything an edition + can cover: when layout encodings join editions, their IDs must be distinct from array + encoding IDs. + The declarations currently live centrally in `vortex-edition/src/definitions.rs`; they are + expected to migrate next to each encoding's vtable once the array plugin trait grows + edition metadata. + +Encoding IDs are deliberately **plain strings**, not references to encoding vtables, so the +crate depends on nothing and can be consumed by the writer, `xtask`, the compat-test suite, +and language bindings alike. An integration test in the file crate closes the loop by +asserting that every declared ID resolves to a registered, executable encoding in the default +session. + +## Computation and validation + +`vortex_edition::manifest::compute_manifests()` derives an `EditionManifest` per edition: the +resolved, sorted encoding set, plus derived metadata — `status` (`draft` if flagged; +otherwise the newest frozen edition per family is `current`, the rest `superseded`), +`supersedes` (the previous edition in the family), the freeze year-month (from the +identifier), and the required Vortex release. The delta against the superseded edition is +itself derived: the members whose `since` equals the edition's own id. + +Computation fails loudly (it never produces silently odd output) on: duplicate encoding or +edition IDs, membership (`since`) references to unknown editions, editions out of +chronological order within a family (drafts must be newest), and malformed release strings +on membership edges. + +Families partition the encoding space — every encoding belongs to exactly one family — so +the union of targeted editions is always unambiguous. + +## Required Vortex release + +An edition's required Vortex release — the earliest release guaranteed to read and execute +the full edition — is **never declared on the edition; it is inferred**, in three layers: + +1. **The operational check needs no version number at all.** A binary supports edition `E` + if and only if its own embedded edition list contains `E` marked frozen. Containment, not + version comparison, is what the writer and any diagnostics actually test. + +2. **When every membership edge records a per-encoding release**, the edition's requirement + is derived as the **maximum over its members'** `required_vortex_release` values — the + oldest release that can read every member. Per-edge values must themselves come from + evidence (e.g. the compat-fixture history proving the encoding readable at that release), + never from memory. + +3. **While any edge is unrecorded**, the fallback is inferred from release history: + + > `required_vortex_release(E)` = the first release tag whose tree contains `E` as frozen. + + Mechanically: `git tag --contains `, filtered to release tags, minimum by + version. Release tooling computes this once, after the release exists, and records it + into the published JSON/docs artifacts — recorded inference, never a hand-authored claim. + +Why is the fallback the freeze release, and why must earlier claims come from per-edge +evidence? Most of `core2026.07.0`'s encodings have been readable for many releases, so a +smaller number is genuinely possible — but old binaries are immutable, and CI going forward +can only ever prove "the current reader reads old files", never "an old reader handles this +newly named set". So an earlier requirement is only honest when each member's edge records a +release at which that encoding was demonstrably readable (the compat-fixture store is +exactly such a demonstration). Absent that evidence, the freeze release is the only provable +answer: the session cross-check test (every declared ID resolves to a registered, executable +encoding) runs in the same tree that freezes the edition, so any release containing the +frozen edition supports all of it by construction. + +This is why the field is not stored on `Edition`: the membership edges (and, as a fallback, +the release history) carry all the information needed. + +## The generation pipeline + +``` +vortex-edition statics ──compute_manifests()──▶ EditionManifest + │ │ + │ serde_json │ + ▼ ▼ +cargo xtask generate-editions ──▶ docs/specs/editions/.json + │ + re-parsed │ (pages render from the JSON, + from JSON ▼ so page and JSON cannot disagree) + docs/specs/editions/.md + + + index block in docs/specs/editions.md + (between the editions:index markers) +``` + +Run `cargo xtask generate-editions` after any change to the definitions. The command writes +one JSON file and one Markdown page per edition (drafts included, rendered with a warning +banner), and rewrites the edition index table and hidden toctree in `docs/specs/editions.md` +between the `` / `` markers. All +generated files carry a do-not-edit header; hand edits will be overwritten. CI should check +the committed artifacts are up to date by re-running the generator and diffing. + +The JSON files are the machine-readable contract for non-Rust consumers (Python/Java bindings, +external tools): same content as the pages, stable schema (`EditionManifest`). + +## Freeze tests + +Published editions are pinned by golden tests in `vortex-edition/src/tests.rs`: the exact +computed encoding set of each frozen edition is written out as a constant +(`FROZEN_CORE_2026_07_0`). Any change that alters a published edition's computed set — +editing a `since`, deleting a declaration, a refactor with surprising effect — fails CI. The +metadata is the source; the snapshot is the freeze. + +## How to: stabilize an encoding + +1. Ensure the encoding's serialized form is final and it has backward-compat fixtures + (`vortex-test/compat-gen`). +2. Add its `EncodingDecl` to `ENCODINGS` with `since` set to the **current draft** edition + (never a frozen one — the freeze test will catch you). +3. Run `cargo xtask generate-editions` and commit the regenerated artifacts. The encoding now + appears on the draft edition's page, clearly marked as carrying no guarantee yet. + +## How to: cut an edition + +1. On the draft edition: set `draft: false`. (There is no date to record — the freeze date + is the edition identifier itself, so the identifier must match the month of the freeze.) +2. Add the next draft edition to `EDITIONS` (e.g. the following quarter's identifier). +3. Add a `FROZEN_` golden test pinning the newly frozen set. +4. Run `cargo xtask generate-editions`; commit. Once the release shipping the edition is + published, release tooling infers and records the required Vortex release (see + [above](#required-vortex-release)), and compat fixtures are published for the newly added + encodings. + +## Not yet wired up + +- **Writer enforcement**: deriving the file writer's allow-list (`ALLOWED_ENCODINGS` in + `vortex-file/src/strategy.rs`) from `Edition::current()` instead of by hand, and a + `with_edition(EditionId)` API on the write options. Compressor schemes will need to declare + which array encodings they emit so the scheme pool — including cascades — can be filtered + against the target edition automatically. (`register_default_encodings` in + `vortex-file/src/lib.rs` is the long-standing seam for this.) +- **The session cross-check test** asserting every declared encoding ID resolves in the + default session registries. +- **Release tooling** that infers `required_vortex_release` from release tags (first release + containing the frozen edition) and records it into the published artifacts. +- **Reader error messages** linking to the published registry page. +- **Layout declarations**: layouts join editions as ordinary encodings (with IDs distinct + from array encoding IDs), and deprecation metadata (deprecated-for-write in a new edition, + with read-time warnings once tooling support lands). +- **Child-encoding closure validation** — checking that an edition guaranteeing an encoding + also guarantees the children it can emit — was dropped from the minimal model; it may + return once declarations migrate to the vtables, where children are authoritatively known. +- **Footer stamping**: recording the writer's target editions in the file footer for better + diagnostics. The guarantee always derives from the encoding IDs actually present in the + file, so the stamp would be informative only. diff --git a/docs/specs/editions.md b/docs/specs/editions.md index d7f50e95037..7d49f871568 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -17,10 +17,12 @@ Editions constrain **writers**, never readers. A reader always understands the u edition ever published, so reading a Vortex file requires no edition configuration at all; the edition only surfaces when something has gone wrong. -Every edition is listed in the [registry](#edition-registry) below. Once published, an edition -is **frozen**: its set never grows or shrinks. New encodings enter the promise only by cutting -a new edition. (An edition may also exist as a **draft** — the staging area for the next -edition. Drafts carry no guarantee and may change freely until frozen.) +Every edition is listed in the [registry](#edition-registry) below, and each edition has its +own page recording exactly which encodings it guarantees, together with a machine-readable +JSON definition. Once published, an edition is **frozen**: its sets never grow or shrink. New +encodings enter the promise only by cutting a new edition. (An edition may also exist as a +**draft** — the staging area for the next edition. Drafts carry no guarantee and may change +freely until frozen.) Readers are **cumulative**: each Vortex release reads everything from every edition published before it. Note that this is a promise about readers, not about editions themselves — an @@ -85,79 +87,35 @@ conflicting definitions of the same encoding. ## Edition registry +Each edition links to its own page, which lists every encoding it guarantees and links a +machine-readable JSON definition of the same content for use by tools. + + + + | Edition | Status | Frozen on | Encodings | |---|---|---|---| -| [core2026.07.0](#core2026070) | current | 2026-07 | 32 | -| [core2026.10.0](#core2026100) | draft | — | 32 | - -### core2026.07.0 - -| | | -|---|---| -| **Status** | current | -| **Frozen on** | 2026-07 | -| **Required Vortex release** | the first release that includes this edition | -| **Supersedes** | — | -| **Encodings** | 32 | - -The first edition of the `core` family captures exactly the set of encodings the default -Vortex file writer emits today: - -| Encoding ID | Since | -|---|---| -| `fastlanes.bitpacked` | core2026.07.0 | -| `fastlanes.delta` | core2026.07.0 | -| `fastlanes.for` | core2026.07.0 | -| `fastlanes.rle` | core2026.07.0 | -| `vortex.alp` | core2026.07.0 | -| `vortex.alprd` | core2026.07.0 | -| `vortex.bool` | core2026.07.0 | -| `vortex.bytebool` | core2026.07.0 | -| `vortex.chunked` | core2026.07.0 | -| `vortex.constant` | core2026.07.0 | -| `vortex.datetimeparts` | core2026.07.0 | -| `vortex.decimal` | core2026.07.0 | -| `vortex.decimal_byte_parts` | core2026.07.0 | -| `vortex.dict` | core2026.07.0 | -| `vortex.ext` | core2026.07.0 | -| `vortex.fixed_size_list` | core2026.07.0 | -| `vortex.fsst` | core2026.07.0 | -| `vortex.list` | core2026.07.0 | -| `vortex.listview` | core2026.07.0 | -| `vortex.masked` | core2026.07.0 | -| `vortex.null` | core2026.07.0 | -| `vortex.pco` | core2026.07.0 | -| `vortex.primitive` | core2026.07.0 | -| `vortex.runend` | core2026.07.0 | -| `vortex.sequence` | core2026.07.0 | -| `vortex.sparse` | core2026.07.0 | -| `vortex.struct` | core2026.07.0 | -| `vortex.varbin` | core2026.07.0 | -| `vortex.varbinview` | core2026.07.0 | -| `vortex.variant` | core2026.07.0 | -| `vortex.zigzag` | core2026.07.0 | -| `vortex.zstd` | core2026.07.0 | - -Deliberately **not** included, because they are experimental and their serialized forms may -still change: `vortex.onpair`, `vortex.zstd_buffers`, `vortex.patched`, -`vortex.piecewise-sequence`, and everything behind the `unstable_encodings` feature flag. -Writing them requires opting out of the edition and forfeits the portability promise. - -### core2026.10.0 - -:::{warning} -`core2026.10.0` is a **draft**: it carries no compatibility guarantee, its contents may -change or be removed, and it cannot be targeted by the writer until it is frozen. -::: +| [core2026.07.0](editions/core2026.07.0.md) | current | 2026-07 | 32 | +| [core2026.10.0](editions/core2026.10.0.md) | draft | — | 32 | + +```{toctree} +--- +maxdepth: 1 +hidden: true +--- -The staging area for the next `core` edition. No changes relative to -[core2026.07.0](#core2026070) yet. +editions/core2026.07.0 +editions/core2026.10.0 +``` + ## Verification The edition promise is tested, not just declared: real `.vortex` files written by previous releases are continuously re-read and checked against known-good values, and published -editions are pinned by tests so no code change can silently alter a frozen set. +editions are pinned by tests so no code change can silently alter a frozen set. If you are +curious how editions are defined and enforced — or you maintain an encoding and want to add +it to an edition — see the [implementation notes](/developer-guide/internals/editions). ## Troubleshooting unknown encodings diff --git a/docs/specs/editions/core2026.07.0.json b/docs/specs/editions/core2026.07.0.json new file mode 100644 index 00000000000..865560082f5 --- /dev/null +++ b/docs/specs/editions/core2026.07.0.json @@ -0,0 +1,170 @@ +{ + "id": "core2026.07.0", + "family": "core", + "status": "current", + "frozen": "2026-07", + "required_vortex_release": null, + "supersedes": null, + "encodings": [ + { + "id": "fastlanes.bitpacked", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "fastlanes.delta", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "fastlanes.for", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "fastlanes.rle", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.alp", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.alprd", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.bool", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.bytebool", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.chunked", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.constant", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.datetimeparts", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.decimal", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.decimal_byte_parts", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.dict", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.ext", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.fixed_size_list", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.fsst", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.list", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.listview", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.masked", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.null", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.pco", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.primitive", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.runend", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.sequence", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.sparse", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.struct", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.varbin", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.varbinview", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.variant", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.zigzag", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.zstd", + "since": "core2026.07.0", + "required_vortex_release": null + } + ] +} diff --git a/docs/specs/editions/core2026.07.0.md b/docs/specs/editions/core2026.07.0.md new file mode 100644 index 00000000000..49a3e4eb2f9 --- /dev/null +++ b/docs/specs/editions/core2026.07.0.md @@ -0,0 +1,54 @@ + + +# core2026.07.0 + +| | | +|---|---| +| **Status** | current | +| **Frozen on** | 2026-07 | +| **Required Vortex release** | the first release that includes this edition | +| **Supersedes** | — | +| **Encodings** | 32 | + +Machine-readable definition: {download}`core2026.07.0.json `. See [Editions](../editions.md) for what an edition guarantees. + +## Encodings + +| Encoding ID | Since | Requires | +|---|---|---| +| `fastlanes.bitpacked` | core2026.07.0 | — | +| `fastlanes.delta` | core2026.07.0 | — | +| `fastlanes.for` | core2026.07.0 | — | +| `fastlanes.rle` | core2026.07.0 | — | +| `vortex.alp` | core2026.07.0 | — | +| `vortex.alprd` | core2026.07.0 | — | +| `vortex.bool` | core2026.07.0 | — | +| `vortex.bytebool` | core2026.07.0 | — | +| `vortex.chunked` | core2026.07.0 | — | +| `vortex.constant` | core2026.07.0 | — | +| `vortex.datetimeparts` | core2026.07.0 | — | +| `vortex.decimal` | core2026.07.0 | — | +| `vortex.decimal_byte_parts` | core2026.07.0 | — | +| `vortex.dict` | core2026.07.0 | — | +| `vortex.ext` | core2026.07.0 | — | +| `vortex.fixed_size_list` | core2026.07.0 | — | +| `vortex.fsst` | core2026.07.0 | — | +| `vortex.list` | core2026.07.0 | — | +| `vortex.listview` | core2026.07.0 | — | +| `vortex.masked` | core2026.07.0 | — | +| `vortex.null` | core2026.07.0 | — | +| `vortex.pco` | core2026.07.0 | — | +| `vortex.primitive` | core2026.07.0 | — | +| `vortex.runend` | core2026.07.0 | — | +| `vortex.sequence` | core2026.07.0 | — | +| `vortex.sparse` | core2026.07.0 | — | +| `vortex.struct` | core2026.07.0 | — | +| `vortex.varbin` | core2026.07.0 | — | +| `vortex.varbinview` | core2026.07.0 | — | +| `vortex.variant` | core2026.07.0 | — | +| `vortex.zigzag` | core2026.07.0 | — | +| `vortex.zstd` | core2026.07.0 | — | + +## Changes + +First edition of the `core` family — every encoding listed above is newly guaranteed. diff --git a/docs/specs/editions/core2026.10.0.json b/docs/specs/editions/core2026.10.0.json new file mode 100644 index 00000000000..cbd162e8db6 --- /dev/null +++ b/docs/specs/editions/core2026.10.0.json @@ -0,0 +1,170 @@ +{ + "id": "core2026.10.0", + "family": "core", + "status": "draft", + "frozen": null, + "required_vortex_release": null, + "supersedes": "core2026.07.0", + "encodings": [ + { + "id": "fastlanes.bitpacked", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "fastlanes.delta", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "fastlanes.for", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "fastlanes.rle", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.alp", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.alprd", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.bool", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.bytebool", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.chunked", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.constant", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.datetimeparts", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.decimal", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.decimal_byte_parts", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.dict", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.ext", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.fixed_size_list", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.fsst", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.list", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.listview", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.masked", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.null", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.pco", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.primitive", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.runend", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.sequence", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.sparse", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.struct", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.varbin", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.varbinview", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.variant", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.zigzag", + "since": "core2026.07.0", + "required_vortex_release": null + }, + { + "id": "vortex.zstd", + "since": "core2026.07.0", + "required_vortex_release": null + } + ] +} diff --git a/docs/specs/editions/core2026.10.0.md b/docs/specs/editions/core2026.10.0.md new file mode 100644 index 00000000000..ee36e8b6f99 --- /dev/null +++ b/docs/specs/editions/core2026.10.0.md @@ -0,0 +1,58 @@ + + +# core2026.10.0 + +:::{warning} +`core2026.10.0` is a **draft**: it carries no compatibility guarantee, its contents may change or be removed, and it cannot be targeted by the writer until it is frozen. +::: + +| | | +|---|---| +| **Status** | draft | +| **Frozen on** | — | +| **Required Vortex release** | — | +| **Supersedes** | [core2026.07.0](core2026.07.0.md) | +| **Encodings** | 32 | + +Machine-readable definition: {download}`core2026.10.0.json `. See [Editions](../editions.md) for what an edition guarantees. + +## Encodings + +| Encoding ID | Since | Requires | +|---|---|---| +| `fastlanes.bitpacked` | core2026.07.0 | — | +| `fastlanes.delta` | core2026.07.0 | — | +| `fastlanes.for` | core2026.07.0 | — | +| `fastlanes.rle` | core2026.07.0 | — | +| `vortex.alp` | core2026.07.0 | — | +| `vortex.alprd` | core2026.07.0 | — | +| `vortex.bool` | core2026.07.0 | — | +| `vortex.bytebool` | core2026.07.0 | — | +| `vortex.chunked` | core2026.07.0 | — | +| `vortex.constant` | core2026.07.0 | — | +| `vortex.datetimeparts` | core2026.07.0 | — | +| `vortex.decimal` | core2026.07.0 | — | +| `vortex.decimal_byte_parts` | core2026.07.0 | — | +| `vortex.dict` | core2026.07.0 | — | +| `vortex.ext` | core2026.07.0 | — | +| `vortex.fixed_size_list` | core2026.07.0 | — | +| `vortex.fsst` | core2026.07.0 | — | +| `vortex.list` | core2026.07.0 | — | +| `vortex.listview` | core2026.07.0 | — | +| `vortex.masked` | core2026.07.0 | — | +| `vortex.null` | core2026.07.0 | — | +| `vortex.pco` | core2026.07.0 | — | +| `vortex.primitive` | core2026.07.0 | — | +| `vortex.runend` | core2026.07.0 | — | +| `vortex.sequence` | core2026.07.0 | — | +| `vortex.sparse` | core2026.07.0 | — | +| `vortex.struct` | core2026.07.0 | — | +| `vortex.varbin` | core2026.07.0 | — | +| `vortex.varbinview` | core2026.07.0 | — | +| `vortex.variant` | core2026.07.0 | — | +| `vortex.zigzag` | core2026.07.0 | — | +| `vortex.zstd` | core2026.07.0 | — | + +## Changes + +No changes relative to [core2026.07.0](core2026.07.0.md) yet. diff --git a/vortex-edition/Cargo.toml b/vortex-edition/Cargo.toml new file mode 100644 index 00000000000..4929656d8d9 --- /dev/null +++ b/vortex-edition/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "vortex-edition" +authors.workspace = true +description = "Definitions of Vortex editions: named, frozen sets of encodings with a read-compatibility guarantee" +edition = { workspace = true } +homepage = { workspace = true } +categories = { workspace = true } +include = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } diff --git a/vortex-edition/src/definitions.rs b/vortex-edition/src/definitions.rs new file mode 100644 index 00000000000..b0b3cfb71c3 --- /dev/null +++ b/vortex-edition/src/definitions.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The edition list and per-encoding membership declarations. +//! +//! Published editions are frozen: the computed set of a published edition is pinned by a +//! golden test in this crate, so any change to these declarations that alters a published +//! set fails CI. New encodings are staged into the current draft edition. + +use crate::Edition; +use crate::EditionId; +use crate::EncodingDecl; + +/// The first edition of the `core` family: the encodings the default file writer emits. +pub const CORE_2026_07_0: EditionId = EditionId::new("core", 2026, 7, 0); + +/// The draft successor of [`CORE_2026_07_0`]. Carries no guarantee until frozen. +pub const CORE_2026_10_0: EditionId = EditionId::new("core", 2026, 10, 0); + +/// All editions, oldest first within each family. The newest non-draft edition of each +/// family is that family's `current` edition. +pub static EDITIONS: &[Edition] = &[ + Edition { + id: CORE_2026_07_0, + draft: false, + }, + Edition { + id: CORE_2026_10_0, + draft: true, + }, +]; + +const fn encoding(id: &'static str, since: EditionId) -> EncodingDecl { + EncodingDecl { + id, + since, + // TODO(editions): record per-encoding required releases from compat-fixture + // evidence; until then the edition's required release falls back to the first + // release containing the edition. + required_vortex_release: None, + } +} + +/// Membership declarations for every encoding covered by an edition. +pub static ENCODINGS: &[EncodingDecl] = &[ + encoding("vortex.null", CORE_2026_07_0), + encoding("vortex.bool", CORE_2026_07_0), + encoding("vortex.primitive", CORE_2026_07_0), + encoding("vortex.decimal", CORE_2026_07_0), + encoding("vortex.varbin", CORE_2026_07_0), + encoding("vortex.varbinview", CORE_2026_07_0), + encoding("vortex.list", CORE_2026_07_0), + encoding("vortex.listview", CORE_2026_07_0), + encoding("vortex.fixed_size_list", CORE_2026_07_0), + encoding("vortex.struct", CORE_2026_07_0), + encoding("vortex.variant", CORE_2026_07_0), + encoding("vortex.ext", CORE_2026_07_0), + encoding("vortex.chunked", CORE_2026_07_0), + encoding("vortex.constant", CORE_2026_07_0), + encoding("vortex.dict", CORE_2026_07_0), + encoding("vortex.masked", CORE_2026_07_0), + encoding("vortex.sparse", CORE_2026_07_0), + encoding("vortex.alp", CORE_2026_07_0), + encoding("vortex.alprd", CORE_2026_07_0), + encoding("vortex.bytebool", CORE_2026_07_0), + encoding("vortex.datetimeparts", CORE_2026_07_0), + encoding("vortex.decimal_byte_parts", CORE_2026_07_0), + encoding("vortex.fsst", CORE_2026_07_0), + encoding("vortex.pco", CORE_2026_07_0), + encoding("vortex.runend", CORE_2026_07_0), + encoding("vortex.sequence", CORE_2026_07_0), + encoding("vortex.zigzag", CORE_2026_07_0), + encoding("vortex.zstd", CORE_2026_07_0), + encoding("fastlanes.bitpacked", CORE_2026_07_0), + encoding("fastlanes.delta", CORE_2026_07_0), + encoding("fastlanes.for", CORE_2026_07_0), + encoding("fastlanes.rle", CORE_2026_07_0), +]; diff --git a/vortex-edition/src/generate.rs b/vortex-edition/src/generate.rs new file mode 100644 index 00000000000..f45fa2a56d5 --- /dev/null +++ b/vortex-edition/src/generate.rs @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Generation of the published edition artifacts from the computed manifests. +//! +//! `cargo xtask generate-editions` calls [`generate`], which produces, under +//! `docs/specs/editions/`: +//! +//! 1. one JSON file per edition — the machine-readable definition; +//! 2. one Markdown page per edition, rendered *from the JSON file* so the published page and +//! the machine-readable artifact cannot disagree; +//! +//! and rewrites the generated index block (table of editions plus the Sphinx toctree) inside +//! `docs/specs/editions.md` between the `editions:index` markers. + +use std::fs; +use std::path::Path; + +use crate::EditionError; +use crate::manifest::EditionManifest; +use crate::manifest::EditionStatus; +use crate::manifest::compute_manifests; + +/// Marker opening the generated index block in `docs/specs/editions.md`. +pub const INDEX_BEGIN: &str = ""; +/// Marker closing the generated index block in `docs/specs/editions.md`. +pub const INDEX_END: &str = ""; + +/// Generate all edition artifacts under `repo_root`. Returns the paths written. +pub fn generate(repo_root: impl AsRef) -> Result, EditionError> { + let repo_root = repo_root.as_ref(); + let editions_dir = repo_root.join("docs/specs/editions"); + fs::create_dir_all(&editions_dir) + .map_err(|e| EditionError::new(format!("creating {}: {e}", editions_dir.display())))?; + + let manifests = compute_manifests()?; + let mut written = Vec::new(); + + for manifest in &manifests { + let json_path = editions_dir.join(format!("{}.json", manifest.id)); + let json = serde_json::to_string_pretty(manifest) + .map_err(|e| EditionError::new(format!("serializing {}: {e}", manifest.id)))?; + fs::write(&json_path, format!("{json}\n")) + .map_err(|e| EditionError::new(format!("writing {}: {e}", json_path.display())))?; + written.push(json_path.display().to_string()); + + // Render the page from the JSON artifact, not the in-memory manifest, so the + // published page and the machine-readable definition cannot disagree. + let parsed: EditionManifest = serde_json::from_str(&json) + .map_err(|e| EditionError::new(format!("re-parsing {}: {e}", manifest.id)))?; + let md_path = editions_dir.join(format!("{}.md", manifest.id)); + fs::write(&md_path, render_edition_page(&parsed)) + .map_err(|e| EditionError::new(format!("writing {}: {e}", md_path.display())))?; + written.push(md_path.display().to_string()); + } + + let index_path = repo_root.join("docs/specs/editions.md"); + splice_index(&index_path, &render_index(&manifests))?; + written.push(index_path.display().to_string()); + + Ok(written) +} + +fn status_label(status: EditionStatus) -> &'static str { + match status { + EditionStatus::Draft => "draft", + EditionStatus::Current => "current", + EditionStatus::Superseded => "superseded", + } +} + +fn render_edition_page(manifest: &EditionManifest) -> String { + let mut page = String::new(); + page.push_str(&format!( + "\n\n# {id}\n\n", + id = manifest.id + )); + + if manifest.status == EditionStatus::Draft { + page.push_str(&format!( + ":::{{warning}}\n`{}` is a **draft**: it carries no compatibility guarantee, its \ + contents may change or be removed, and it cannot be targeted by the writer until \ + it is frozen.\n:::\n\n", + manifest.id + )); + } + + page.push_str("| | |\n|---|---|\n"); + page.push_str(&format!( + "| **Status** | {} |\n", + status_label(manifest.status) + )); + page.push_str(&format!( + "| **Frozen on** | {} |\n", + manifest.frozen.as_deref().unwrap_or("—") + )); + let required_release = match (&manifest.required_vortex_release, manifest.status) { + (Some(release), _) => release.as_str(), + (None, EditionStatus::Draft) => "—", + (None, _) => "the first release that includes this edition", + }; + page.push_str(&format!( + "| **Required Vortex release** | {required_release} |\n" + )); + page.push_str(&format!( + "| **Supersedes** | {} |\n", + manifest + .supersedes + .as_deref() + .map(|s| format!("[{s}]({s}.md)")) + .unwrap_or_else(|| "—".to_string()) + )); + page.push_str(&format!( + "| **Encodings** | {} |\n\n", + manifest.encodings.len() + )); + + page.push_str(&format!( + "Machine-readable definition: {{download}}`{id}.json <{id}.json>`. See \ + [Editions](../editions.md) for what an edition guarantees.\n\n", + id = manifest.id + )); + + page.push_str("## Encodings\n\n"); + page.push_str("| Encoding ID | Since | Requires |\n|---|---|---|\n"); + for encoding in &manifest.encodings { + page.push_str(&format!( + "| `{}` | {} | {} |\n", + encoding.id, + encoding.since, + encoding.required_vortex_release.as_deref().unwrap_or("—"), + )); + } + page.push('\n'); + + page.push_str("## Changes\n\n"); + let newly_added: Vec<&str> = manifest + .encodings + .iter() + .filter(|e| e.since == manifest.id) + .map(|e| e.id.as_str()) + .collect(); + match manifest.supersedes.as_deref() { + None => { + page.push_str(&format!( + "First edition of the `{}` family — every encoding listed above is newly \ + guaranteed.\n", + manifest.family + )); + } + Some(base) if newly_added.is_empty() => { + page.push_str(&format!( + "No changes relative to [{base}]({base}.md) yet.\n" + )); + } + Some(base) => { + page.push_str(&format!( + "Relative to [{base}]({base}.md), added: {}.\n", + newly_added + .iter() + .map(|id| format!("`{id}`")) + .collect::>() + .join(", ") + )); + } + } + + page +} + +fn render_index(manifests: &[EditionManifest]) -> String { + let mut index = String::new(); + index.push_str(INDEX_BEGIN); + index.push_str( + "\n\n\n", + ); + index.push_str("| Edition | Status | Frozen on | Encodings |\n|---|---|---|---|\n"); + for manifest in manifests { + index.push_str(&format!( + "| [{id}](editions/{id}.md) | {status} | {frozen} | {arrays} |\n", + id = manifest.id, + status = status_label(manifest.status), + frozen = manifest.frozen.as_deref().unwrap_or("—"), + arrays = manifest.encodings.len(), + )); + } + index.push_str("\n```{toctree}\n---\nmaxdepth: 1\nhidden: true\n---\n\n"); + for manifest in manifests { + index.push_str(&format!("editions/{}\n", manifest.id)); + } + index.push_str("```\n"); + index.push_str(INDEX_END); + index +} + +fn splice_index(index_path: &Path, rendered: &str) -> Result<(), EditionError> { + let content = fs::read_to_string(index_path) + .map_err(|e| EditionError::new(format!("reading {}: {e}", index_path.display())))?; + let start = content.find(INDEX_BEGIN).ok_or_else(|| { + EditionError::new(format!( + "{} does not contain the {INDEX_BEGIN} marker", + index_path.display() + )) + })?; + let end = content.find(INDEX_END).ok_or_else(|| { + EditionError::new(format!( + "{} does not contain the {INDEX_END} marker", + index_path.display() + )) + })? + INDEX_END.len(); + if end <= start { + return Err(EditionError::new(format!( + "malformed editions index markers in {}", + index_path.display() + ))); + } + let updated = format!("{}{}{}", &content[..start], rendered, &content[end..]); + fs::write(index_path, updated) + .map_err(|e| EditionError::new(format!("writing {}: {e}", index_path.display()))) +} diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs new file mode 100644 index 00000000000..8b21bdd2aaa --- /dev/null +++ b/vortex-edition/src/lib.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Definitions of Vortex *editions*: named, frozen sets of encodings that a writer may put in +//! a file, carrying a forever read-compatibility guarantee. +//! +//! This crate is the single source of truth for editions. It holds two pieces of data: +//! +//! 1. [`EDITIONS`] — the edition list itself: identifiers and draft flags. The freeze date +//! is carried by the identifier (`core2026.07.0` was frozen in 2026-07). +//! 2. [`ENCODINGS`] — the per-encoding membership declarations: which edition each encoding +//! has been a member of *since*, and the release required to read it. +//! +//! Everything else is computed: [`manifest::compute_manifests`] derives the full, +//! closure-validated encoding set for every edition, and [`generate`] turns those manifests +//! into machine-readable JSON files and the published documentation pages (via +//! `cargo xtask generate-editions`). +//! +//! The crate deliberately depends on nothing beyond `serde`, so it can be consumed by the +//! writer, `xtask`, the compat-test suite, and language bindings alike. Encoding IDs are plain +//! strings; tests elsewhere in the workspace assert that every declared ID resolves to a +//! registered encoding in the default session. +//! +//! See the published spec at and the internal +//! design notes in `docs/developer-guide/internals/editions.md`. + +mod definitions; +pub mod generate; +pub mod manifest; +#[cfg(test)] +mod tests; + +use std::error::Error; +use std::fmt; +use std::fmt::Display; +use std::fmt::Formatter; + +pub use definitions::CORE_2026_07_0; +pub use definitions::CORE_2026_10_0; +pub use definitions::EDITIONS; +pub use definitions::ENCODINGS; + +/// The identifier of an edition, e.g. `core2026.07.0`. +/// +/// The `family` names an independently versioned, additive group of encodings (`core` is the +/// set the default writer emits). The date components record when the edition was frozen and +/// order editions chronologically *within* a family; there is no ordering across families. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EditionId { + /// The edition family, e.g. `core`. + pub family: &'static str, + /// Year the edition was cut. + pub year: u16, + /// Month the edition was cut. + pub month: u8, + /// Distinguishes editions cut in the same month; normally `0`. + pub version: u8, +} + +impl EditionId { + /// Create an edition identifier. + pub const fn new(family: &'static str, year: u16, month: u8, version: u8) -> Self { + Self { + family, + year, + month, + version, + } + } + + /// Returns true if `self` is the same edition as `other` or an earlier edition of the + /// same family. Editions of different families are never ordered. + pub fn is_at_or_before(&self, other: &EditionId) -> bool { + self.family == other.family + && (self.year, self.month, self.version) <= (other.year, other.month, other.version) + } +} + +impl Display for EditionId { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "{}{}.{:02}.{}", + self.family, self.year, self.month, self.version + ) + } +} + +/// An edition: a named, frozen set of encodings with a read-compatibility guarantee. +/// +/// The encoding sets themselves are not stored here — they are computed from the +/// per-encoding [`EncodingDecl::since`] declarations by [`manifest::compute_manifests`]. +#[derive(Clone, Copy, Debug)] +pub struct Edition { + /// The edition identifier. Also carries the freeze date: `core2026.07.0` was frozen in + /// 2026-07. + pub id: EditionId, + /// Drafts are editions being assembled: they carry no guarantee, may change freely, and + /// are never the default write target. + pub draft: bool, +} + +/// Declares an encoding's membership of an edition family. +/// +/// An encoding declared with `since: E` is a member of `E` and of every later edition of the +/// same family (until deprecation exists). These declarations currently live centrally in +/// this crate; they are expected to migrate next to each encoding's vtable once the plugin +/// trait grows edition metadata. +#[derive(Clone, Copy, Debug)] +pub struct EncodingDecl { + /// The encoding ID, e.g. `vortex.alp`. Globally unique across everything an edition can + /// cover: when layout encodings join editions, their IDs must be distinct from array + /// encoding IDs. + pub id: &'static str, + /// The first edition this encoding is a member of. + pub since: EditionId, + /// The earliest Vortex release able to read and execute this encoding, recorded on the + /// membership edge from evidence (e.g. compat-fixture history). `None` until recorded; + /// an edition's required release is derived as the maximum over its members' recorded + /// releases, falling back to the first release containing the edition when any edge is + /// unrecorded. + pub required_vortex_release: Option<&'static str>, +} + +/// Error raised when edition definitions are inconsistent or generation fails. +#[derive(Debug)] +pub struct EditionError(String); + +impl EditionError { + pub(crate) fn new(msg: impl Into) -> Self { + Self(msg.into()) + } +} + +impl Display for EditionError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl Error for EditionError {} diff --git a/vortex-edition/src/manifest.rs b/vortex-edition/src/manifest.rs new file mode 100644 index 00000000000..c3d24605a20 --- /dev/null +++ b/vortex-edition/src/manifest.rs @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Computed edition manifests: the fully resolved, validated encoding sets for every +//! edition, in the shape serialized to the per-edition JSON files. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +use serde::Deserialize; +use serde::Serialize; + +use crate::EDITIONS; +use crate::ENCODINGS; +use crate::Edition; +use crate::EditionError; + +/// The lifecycle status of an edition, derived from the edition list. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum EditionStatus { + /// Being assembled; carries no guarantee and may change freely. + Draft, + /// The newest frozen edition of its family; the default write target for `core`. + Current, + /// Frozen and replaced by a newer edition of the same family. + Superseded, +} + +/// One encoding's entry in a computed edition manifest. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct EncodingManifest { + /// The encoding ID, e.g. `vortex.alp`. + pub id: String, + /// The first edition this encoding was a member of. + pub since: String, + /// The earliest Vortex release able to read and execute this encoding, recorded on the + /// membership edge. Absent until recorded from evidence. + pub required_vortex_release: Option, +} + +/// A fully computed edition: metadata plus the resolved encoding set. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct EditionManifest { + /// The edition identifier, e.g. `core2026.07.0`. + pub id: String, + /// The edition family, e.g. `core`. + pub family: String, + /// Derived lifecycle status. + pub status: EditionStatus, + /// The year-month the edition was frozen, inferred from the edition identifier + /// (`core2026.07.0` -> `2026-07`); absent for drafts. + pub frozen: Option, + /// The earliest Vortex release guaranteed to read and execute the full edition. + /// + /// Never declared by hand: it is *derived* as the maximum of the members' + /// per-encoding `required_vortex_release` values when every member records one. + /// Otherwise absent, in which case the required release is the first release whose + /// edition list contains this edition as frozen — recorded into the published + /// artifacts by release tooling once that release exists. Always absent for drafts. + pub required_vortex_release: Option, + /// The previous edition of the same family, if any. + pub supersedes: Option, + /// The encodings in this edition, sorted by ID. The delta against the superseded + /// edition is derivable: the members whose `since` equals this edition's id. + pub encodings: Vec, +} + +/// Compute the manifest of every declared edition, validating the definitions. +/// +/// Validation errors (rather than silently odd output) on: duplicate encoding IDs, +/// membership declarations referencing unknown editions, editions out of chronological +/// order within a family, and malformed release strings. +pub fn compute_manifests() -> Result, EditionError> { + validate_definitions()?; + + let mut manifests = Vec::with_capacity(EDITIONS.len()); + for edition in EDITIONS { + let encodings = members_of(edition); + + let supersedes = EDITIONS + .iter() + .filter(|e| e.id.family == edition.id.family && e.id != edition.id) + .filter(|e| e.id.is_at_or_before(&edition.id)) + .next_back() + .map(|e| e.id.to_string()); + + manifests.push(EditionManifest { + id: edition.id.to_string(), + family: edition.id.family.to_string(), + status: status_of(edition), + frozen: (!edition.draft) + .then(|| format!("{}-{:02}", edition.id.year, edition.id.month)), + required_vortex_release: derive_required_release(edition, &encodings)?, + supersedes, + encodings, + }); + } + + Ok(manifests) +} + +/// Derive an edition's required Vortex release: the maximum of the members' recorded +/// per-encoding releases, provided every member records one. If any edge is unrecorded (or +/// the edition is a draft) the result is `None`, and the required release falls back to the +/// first release containing the frozen edition. +fn derive_required_release( + edition: &Edition, + members: &[EncodingManifest], +) -> Result, EditionError> { + if edition.draft { + return Ok(None); + } + let mut max: Option<(Vec, &str)> = None; + for member in members { + let Some(release) = member.required_vortex_release.as_deref() else { + return Ok(None); + }; + let key = parse_release(release).ok_or_else(|| { + EditionError::new(format!( + "encoding {} declares malformed required_vortex_release {release:?}", + member.id + )) + })?; + if max.as_ref().is_none_or(|(best, _)| key > *best) { + max = Some((key, release)); + } + } + Ok(max.map(|(_, release)| release.to_string())) +} + +/// Parse a `major.minor.patch` release string into a comparable key. +fn parse_release(release: &str) -> Option> { + let parts: Vec = release + .split('.') + .map(|part| part.parse::().ok()) + .collect::>()?; + (parts.len() == 3).then_some(parts) +} + +fn status_of(edition: &Edition) -> EditionStatus { + if edition.draft { + return EditionStatus::Draft; + } + let newest_frozen = EDITIONS + .iter() + .filter(|e| e.id.family == edition.id.family && !e.draft) + .next_back(); + if newest_frozen.map(|e| e.id) == Some(edition.id) { + EditionStatus::Current + } else { + EditionStatus::Superseded + } +} + +fn members_of(edition: &Edition) -> Vec { + let mut members: Vec = ENCODINGS + .iter() + .filter(|decl| decl.since.is_at_or_before(&edition.id)) + .map(|decl| EncodingManifest { + id: decl.id.to_string(), + since: decl.since.to_string(), + required_vortex_release: decl.required_vortex_release.map(str::to_string), + }) + .collect(); + members.sort_by(|l, r| l.id.cmp(&r.id)); + members +} + +fn validate_definitions() -> Result<(), EditionError> { + // Edition IDs are unique, and chronological within each family (drafts last). + let mut seen_editions = BTreeSet::new(); + let mut newest_per_family: BTreeMap<&str, &Edition> = BTreeMap::new(); + for edition in EDITIONS { + if !seen_editions.insert(edition.id.to_string()) { + return Err(EditionError::new(format!( + "duplicate edition {}", + edition.id + ))); + } + if let Some(prev) = newest_per_family.get(edition.id.family) { + if !prev.id.is_at_or_before(&edition.id) { + return Err(EditionError::new(format!( + "edition {} is out of chronological order within family {}", + edition.id, edition.id.family, + ))); + } + if prev.draft && !edition.draft { + return Err(EditionError::new(format!( + "frozen edition {} follows draft {}; drafts must be newest in a family", + edition.id, prev.id, + ))); + } + } + newest_per_family.insert(edition.id.family, edition); + } + + // Encoding IDs are unique, and `since` references a declared edition. + let mut seen_encodings = BTreeSet::new(); + for decl in ENCODINGS { + if !seen_encodings.insert(decl.id) { + return Err(EditionError::new(format!("duplicate encoding {}", decl.id))); + } + if !EDITIONS.iter().any(|e| e.id == decl.since) { + return Err(EditionError::new(format!( + "encoding {} declares membership of unknown edition {}", + decl.id, decl.since + ))); + } + } + + Ok(()) +} diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs new file mode 100644 index 00000000000..02a63c2e801 --- /dev/null +++ b/vortex-edition/src/tests.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use crate::CORE_2026_07_0; +use crate::CORE_2026_10_0; +use crate::EditionError; +use crate::manifest::EditionManifest; +use crate::manifest::EditionStatus; +use crate::manifest::compute_manifests; + +/// The frozen encoding set of `core2026.07.0`. +/// +/// This is the freeze: published editions must never change. If this test fails, you have +/// changed a published edition's computed set — revert the change, or stage it into the +/// current draft edition instead. +const FROZEN_CORE_2026_07_0: &[&str] = &[ + "fastlanes.bitpacked", + "fastlanes.delta", + "fastlanes.for", + "fastlanes.rle", + "vortex.alp", + "vortex.alprd", + "vortex.bool", + "vortex.bytebool", + "vortex.chunked", + "vortex.constant", + "vortex.datetimeparts", + "vortex.decimal", + "vortex.decimal_byte_parts", + "vortex.dict", + "vortex.ext", + "vortex.fixed_size_list", + "vortex.fsst", + "vortex.list", + "vortex.listview", + "vortex.masked", + "vortex.null", + "vortex.pco", + "vortex.primitive", + "vortex.runend", + "vortex.sequence", + "vortex.sparse", + "vortex.struct", + "vortex.varbin", + "vortex.varbinview", + "vortex.variant", + "vortex.zigzag", + "vortex.zstd", +]; + +fn manifest_of(id: &str) -> Result { + compute_manifests()? + .into_iter() + .find(|m| m.id == id) + .ok_or_else(|| EditionError::new(format!("edition {id} not found"))) +} + +#[test] +fn frozen_core_2026_07_0() -> Result<(), EditionError> { + let manifest = manifest_of("core2026.07.0")?; + let ids: Vec<&str> = manifest.encodings.iter().map(|e| e.id.as_str()).collect(); + assert_eq!(ids, FROZEN_CORE_2026_07_0); + assert_eq!(manifest.frozen.as_deref(), Some("2026-07")); + Ok(()) +} + +#[test] +fn required_release_derives_from_membership_edges() -> Result<(), EditionError> { + // No membership edge records a required release yet, so the edition-level requirement + // stays unrecorded and falls back to "first release containing the edition". + let manifest = manifest_of("core2026.07.0")?; + assert!(manifest.required_vortex_release.is_none()); + assert!( + manifest + .encodings + .iter() + .all(|e| e.required_vortex_release.is_none()) + ); + Ok(()) +} + +#[test] +fn definitions_are_valid() -> Result<(), EditionError> { + compute_manifests().map(|_| ()) +} + +#[test] +fn exactly_one_current_edition_per_family() -> Result<(), EditionError> { + let manifests = compute_manifests()?; + let core_current: Vec<&EditionManifest> = manifests + .iter() + .filter(|m| m.family == "core" && m.status == EditionStatus::Current) + .collect(); + assert_eq!(core_current.len(), 1); + assert_eq!(core_current[0].id, "core2026.07.0"); + Ok(()) +} + +#[test] +fn draft_carries_no_freeze_metadata() -> Result<(), EditionError> { + let draft = manifest_of("core2026.10.0")?; + assert_eq!(draft.status, EditionStatus::Draft); + assert!(draft.frozen.is_none()); + assert!(draft.required_vortex_release.is_none()); + assert_eq!(draft.supersedes.as_deref(), Some("core2026.07.0")); + assert!( + draft.encodings.iter().all(|e| e.since != draft.id), + "no encodings staged in the draft yet" + ); + Ok(()) +} + +#[test] +fn manifests_round_trip_through_json() -> Result<(), EditionError> { + for manifest in compute_manifests()? { + let json = serde_json::to_string(&manifest) + .map_err(|e| EditionError::new(format!("serialize: {e}")))?; + let parsed: EditionManifest = serde_json::from_str(&json) + .map_err(|e| EditionError::new(format!("deserialize: {e}")))?; + assert_eq!(parsed.id, manifest.id); + assert_eq!(parsed.encodings.len(), manifest.encodings.len()); + } + Ok(()) +} + +#[test] +fn edition_ids_order_within_family_only() { + assert!(CORE_2026_07_0.is_at_or_before(&CORE_2026_10_0)); + assert!(!CORE_2026_10_0.is_at_or_before(&CORE_2026_07_0)); + + let geo = crate::EditionId::new("geo", 2026, 9, 0); + assert!(!CORE_2026_07_0.is_at_or_before(&geo)); + assert!(!geo.is_at_or_before(&CORE_2026_07_0)); +} + +#[test] +fn edition_id_display() { + assert_eq!(CORE_2026_07_0.to_string(), "core2026.07.0"); +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index eae2db43413..8bb8cd05125 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -23,6 +23,7 @@ test = false anyhow = { workspace = true } clap = { workspace = true, features = ["derive"] } prost-build = { workspace = true } +vortex-edition = { workspace = true } xshell = { workspace = true } [lints] diff --git a/xtask/src/generate_editions.rs b/xtask/src/generate_editions.rs new file mode 100644 index 00000000000..4f4ededd4b8 --- /dev/null +++ b/xtask/src/generate_editions.rs @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use std::path::Path; + +use anyhow::Context; + +/// Regenerate the edition artifacts under `docs/specs/editions/` (per-edition JSON files and +/// Markdown pages) and the generated index block in `docs/specs/editions.md`, from the +/// definitions in the `vortex-edition` crate. +pub(crate) fn generate_editions() -> anyhow::Result<()> { + let repo_root = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .context("xtask manifest dir has no parent")?; + let written = + vortex_edition::generate::generate(repo_root).context("generating edition artifacts")?; + for path in written { + println!("wrote {path}"); + } + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 1155ee3246a..6b2890ed32e 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,11 +1,13 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright the Vortex contributors +mod generate_editions; mod generate_fbs; mod generate_proto; use clap::Parser; +use crate::generate_editions::generate_editions; use crate::generate_fbs::generate_fbs; use crate::generate_proto::generate_proto; @@ -16,6 +18,10 @@ struct Xtask { } #[derive(clap::Subcommand)] +#[expect( + clippy::enum_variant_names, + reason = "subcommands are all generators, named after their CLI commands" +)] enum Commands { /// Subcommand to regenerate flatbuffers language bindings for the Rust project. #[command(name = "generate-fbs")] @@ -23,6 +29,9 @@ enum Commands { /// Subcommand to regenerate protobuf language bindings for the Rust project. #[command(name = "generate-proto")] GenerateProto, + /// Subcommand to regenerate edition JSON definitions and documentation pages. + #[command(name = "generate-editions")] + GenerateEditions, } fn main() -> anyhow::Result<()> { @@ -30,6 +39,7 @@ fn main() -> anyhow::Result<()> { match cli.command { Commands::GenerateFlatbuffers => generate_fbs()?, Commands::GenerateProto => generate_proto()?, + Commands::GenerateEditions => generate_editions()?, } Ok(()) }