Skip to content

Commit 49c4753

Browse files
feat[edition]: session-registered editions defitions (#8871)
## Rationale for this change Any encoding that vortex will write using built-in layout strategies or compressors will come from an edition. ## What changes are included in this PR? - Adds the `vortex-editions` crate and session variable. - Defines editions for core and unstable encodings from vortex 0.36.0 until today --------- Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
1 parent 1113b83 commit 49c4753

22 files changed

Lines changed: 1050 additions & 6 deletions

File tree

Cargo.lock

Lines changed: 9 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ members = [
88
"vortex-mask",
99
"vortex-utils",
1010
"vortex-session",
11+
"vortex-edition",
1112
"vortex-flatbuffers",
1213
"vortex-metrics",
1314
"vortex-io",
@@ -294,6 +295,7 @@ vortex-compute = { version = "0.1.0", path = "./vortex-compute", default-feature
294295
vortex-datafusion = { version = "0.1.0", path = "./vortex-datafusion", default-features = false }
295296
vortex-datetime-parts = { version = "0.1.0", path = "./encodings/datetime-parts", default-features = false }
296297
vortex-decimal-byte-parts = { version = "0.1.0", path = "encodings/decimal-byte-parts", default-features = false }
298+
vortex-edition = { version = "0.1.0", path = "./vortex-edition", default-features = false }
297299
vortex-error = { version = "0.1.0", path = "./vortex-error", default-features = false }
298300
vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false }
299301
vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false }

docs/specs/editions.md

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,14 @@ Vortex defines an evergrowing set of serializable array encodings, once written
44
version of vortex.
55
**Editions** are used to keep track of these encodings and talk about groups of encodings.
66

7-
The edition `core2026.07.0` (coming soon) is the first such edition containing all encodings currently
8-
enabled by the writer.
7+
The first edition, `core2025.05.0`, contains the stable encodings that could be written by Vortex
8+
`0.36.0`. This is the release from which the Vortex file format is considered stable. Later `core`
9+
editions add stable encodings released after that compatibility boundary.
910
Editions are additive so an edition that comes after a previous one contains all the encodings from the previous one
1011
and more.
11-
The writer can be configured with a set of different editions (e.g. `core2026.07.0` and `unstable2026.05.0` all stable
12-
encoding released before July 2026 and all unstable encodings from May 2026).
12+
The writer can be configured with a set of different editions (for example, `core2026.07.0` and
13+
`unstable2026.06.0` select stable encodings released through July 2026 and unstable encodings
14+
released through June 2026).
1315

1416
Editions can be used to constrain your minimum required vortex reader, since latest version over vortex across all
1517
editions is the earliest version of vortex required to read that file.
@@ -57,4 +59,4 @@ indefinitely, so deprecation never invalidates existing files.
5759

5860
## Edition registry
5961

60-
Coming soon..
62+
Coming soon..

vortex-array/src/arrays/decimal/vtable/mod.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,7 +164,6 @@ impl VTable for Decimal {
164164
dtype: &DType,
165165
len: usize,
166166
metadata: &[u8],
167-
168167
buffers: &[BufferHandle],
169168
children: &dyn ArrayChildren,
170169
_session: &VortexSession,

vortex-edition/Cargo.toml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[package]
2+
name = "vortex-edition"
3+
authors.workspace = true
4+
description = "Definitions of Vortex editions: named, frozen sets of encodings with a read-compatibility guarantee"
5+
edition = { workspace = true }
6+
homepage = { workspace = true }
7+
categories = { workspace = true }
8+
include = { workspace = true }
9+
keywords = { workspace = true }
10+
license = { workspace = true }
11+
readme = { workspace = true }
12+
repository = { workspace = true }
13+
rust-version = { workspace = true }
14+
version = { workspace = true }
15+
16+
[package.metadata.docs.rs]
17+
all-features = true
18+
19+
[lints]
20+
workspace = true
21+
22+
[dependencies]
23+
parking_lot = { workspace = true }
24+
vortex-session = { workspace = true }

vortex-edition/src/lib.rs

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! Definitions of Vortex *editions*: named, frozen sets of encodings that a writer may put in
5+
//! a file, carrying a forever read-compatibility guarantee.
6+
//!
7+
//! Editions live on the session, like encodings do: [`EditionSession`] is the session
8+
//! variable holding the edition registry, populated at initialization time. Declarations
9+
//! are plain constants — an [`EditionId`] plus an [`Edition`] record, and one
10+
//! [`EditionInclusion`] per encoding stating that it is a member of an edition *and every
11+
//! later edition of the same family*. Any crate can register declarations into a session,
12+
//! so inclusions can live next to the encoding they describe.
13+
//!
14+
//! An edition is a **draft** until its [`Edition::min_vortex_version`] is recorded —
15+
//! recording it is the act of freezing. The per-edition encoding sets are computed from the
16+
//! registered declarations by [`EditionSession::encodings_in`], and correctness is enforced
17+
//! by unit tests: [`EditionSession::validate`] checks a whole registry, and
18+
//! [`test_harness::validate_edition`] validates one edition's constraints — call it once in
19+
//! the `#[cfg(test)]` module of each edition definition.
20+
//!
21+
//! This crate defines only the types and the session variable; the first-party edition
22+
//! declarations live in the public `vortex` crate (`vortex::editions`), which seeds them
23+
//! into the default session. See the published spec at
24+
//! <https://docs.vortex.dev/specs/editions.html>.
25+
26+
mod session;
27+
pub mod test_harness;
28+
#[cfg(test)]
29+
mod tests;
30+
31+
use std::error::Error;
32+
use std::fmt;
33+
use std::fmt::Debug;
34+
use std::fmt::Display;
35+
use std::fmt::Formatter;
36+
37+
pub use session::EditionSession;
38+
pub use session::EditionSessionExt;
39+
use vortex_session::registry::Id;
40+
41+
/// The identifier of an edition, e.g. `core2026.07.0`.
42+
///
43+
/// The `family` names an independently versioned, additive group of encodings (`core` is the
44+
/// set the default writer emits). The date components record when the edition was frozen and
45+
/// order editions chronologically *within* a family; there is no ordering across families.
46+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
47+
pub struct EditionId {
48+
/// The edition family, e.g. `core`.
49+
pub family: &'static str,
50+
/// Year the edition was cut.
51+
pub year: u16,
52+
/// Month the edition was cut.
53+
pub month: u8,
54+
/// Distinguishes editions cut in the same month; normally `0`.
55+
pub version: u8,
56+
}
57+
58+
impl EditionId {
59+
/// Create an edition identifier. Validated by [`EditionId::validate`], which
60+
/// [`test_harness::validate_edition`] exercises
61+
/// per edition in unit tests.
62+
pub const fn new(family: &'static str, year: u16, month: u8, version: u8) -> Self {
63+
Self {
64+
family,
65+
year,
66+
month,
67+
version,
68+
}
69+
}
70+
71+
/// Returns true if `self` is the same edition as `other` or an earlier edition of the
72+
/// same family. Editions of different families are never ordered.
73+
pub fn is_at_or_before(&self, other: &EditionId) -> bool {
74+
self.family == other.family
75+
&& (self.year, self.month, self.version) <= (other.year, other.month, other.version)
76+
}
77+
78+
/// Validate the identifier's form: a non-empty lowercase family, a four-digit year,
79+
/// and a month in 01-12. Checked for every declared edition by
80+
/// [`EditionSession::validate`] and per edition by
81+
/// [`test_harness::validate_edition`].
82+
pub fn validate(&self) -> Result<(), EditionError> {
83+
if self.family.is_empty() || !self.family.chars().all(|c| c.is_ascii_lowercase()) {
84+
return Err(EditionError::new(format!(
85+
"edition {self} must have a non-empty lowercase family, e.g. `core`"
86+
)));
87+
}
88+
if !(1000..=9999).contains(&self.year) {
89+
return Err(EditionError::new(format!(
90+
"edition {self} must have a four-digit year"
91+
)));
92+
}
93+
if !(1..=12).contains(&self.month) {
94+
return Err(EditionError::new(format!(
95+
"edition {self} must have a month in 01-12"
96+
)));
97+
}
98+
Ok(())
99+
}
100+
}
101+
102+
impl Display for EditionId {
103+
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
104+
write!(
105+
f,
106+
"{}{}.{:02}.{}",
107+
self.family, self.year, self.month, self.version
108+
)
109+
}
110+
}
111+
112+
/// An edition: a named set of encodings with a read-compatibility guarantee, registered with
113+
/// [`EditionSession::declare_edition`]. The set itself is computed from the registered
114+
/// [`EditionInclusion`]s by [`EditionSession::encodings_in`].
115+
#[derive(Clone, Copy, Debug)]
116+
pub struct Edition {
117+
/// The edition identifier. Also carries the freeze date: `core2026.07.0` freezes in
118+
/// 2026-07.
119+
pub id: EditionId,
120+
/// The minimum Vortex version whose reader supports every encoding in this edition.
121+
///
122+
/// Recording this is the act of freezing: an edition with `None` is a **draft** — being
123+
/// assembled, carrying no guarantee, free to change, never the default write target.
124+
/// Validated against the members' [`EditionInclusion::required_vortex_release`] values:
125+
/// no member may require a version newer than the edition declares.
126+
pub min_vortex_version: Option<&'static str>,
127+
}
128+
129+
impl Edition {
130+
/// A draft is an edition whose `min_vortex_version` has not been recorded yet.
131+
pub fn is_draft(&self) -> bool {
132+
self.min_vortex_version.is_none()
133+
}
134+
}
135+
136+
/// Declares that an encoding is a member of an edition — and of every later edition of the
137+
/// same family. Registered with [`EditionSession::declare_inclusion`].
138+
#[derive(Clone, Copy, Debug)]
139+
pub struct EditionInclusion {
140+
/// The interned encoding id, e.g. `vortex.alp`. Globally unique across everything an
141+
/// edition can cover: when layout encodings join editions, their ids must be distinct
142+
/// from array encoding ids.
143+
pub encoding_id: Id,
144+
/// The first edition this encoding is a member of.
145+
pub since: EditionId,
146+
/// The earliest Vortex release able to read and execute this encoding, recorded from
147+
/// evidence (e.g. compat-fixture history). `None` until recorded.
148+
pub required_vortex_release: Option<&'static str>,
149+
}
150+
151+
/// A source of an encoding id for edition declarations.
152+
///
153+
/// Implemented for raw id strings (`"vortex.alp"`) and interned [`Id`]s here; encoding
154+
/// vtables implement it where they are defined, so a declaration can name the vtable
155+
/// (`&Primitive`) instead of spelling its id.
156+
pub trait AsEncodingId: Debug + Send + Sync {
157+
/// The interned encoding id.
158+
fn encoding_id(&self) -> Id;
159+
}
160+
161+
impl AsEncodingId for str {
162+
#[expect(
163+
clippy::disallowed_methods,
164+
reason = "interning a dynamic encoding id at declaration time"
165+
)]
166+
fn encoding_id(&self) -> Id {
167+
Id::new(self)
168+
}
169+
}
170+
171+
impl AsEncodingId for Id {
172+
fn encoding_id(&self) -> Id {
173+
*self
174+
}
175+
}
176+
177+
// `str` is unsized and cannot be a trait object, so declaration blocks (slices of
178+
// `&dyn AsEncodingId`) name encodings as `&"vortex.alp"` through this impl.
179+
impl AsEncodingId for &'static str {
180+
fn encoding_id(&self) -> Id {
181+
(**self).encoding_id()
182+
}
183+
}
184+
185+
/// Declares an edition together with the encodings that join the family at it, in one
186+
/// block. Registered with [`EditionSession::declare`], which derives each encoding's
187+
/// membership (`since` = the declared edition) from the block structure.
188+
#[derive(Clone, Copy, Debug)]
189+
pub struct EditionDeclaration {
190+
/// The edition being declared.
191+
pub edition: Edition,
192+
/// The encodings that join the family at this edition, named by id string or by
193+
/// vtable. Members of earlier editions are inherited and never restated.
194+
pub added: &'static [&'static dyn AsEncodingId],
195+
}
196+
197+
impl EditionInclusion {
198+
/// Declare that an encoding is a member of `since` and every later edition of the same
199+
/// family. The encoding can be named by id string or by vtable.
200+
pub fn new<E: AsEncodingId + ?Sized>(encoding: &E, since: EditionId) -> Self {
201+
Self {
202+
encoding_id: encoding.encoding_id(),
203+
since,
204+
required_vortex_release: None,
205+
}
206+
}
207+
208+
/// Validate the declaration's form: a lowercase `namespace.name` encoding id and, if
209+
/// recorded, a well-formed `major.minor.patch` release. Checked for every declared
210+
/// inclusion by [`EditionSession::validate`].
211+
pub fn validate(&self) -> Result<(), EditionError> {
212+
let id = self.encoding_id.as_str();
213+
let well_formed = !id.starts_with('.')
214+
&& !id.ends_with('.')
215+
&& id.contains('.')
216+
&& id
217+
.chars()
218+
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || "._-".contains(c));
219+
if !well_formed {
220+
return Err(EditionError::new(format!(
221+
"invalid encoding id {id:?}: expected lowercase `namespace.name`, e.g. \
222+
`vortex.alp`"
223+
)));
224+
}
225+
if let Some(release) = self.required_vortex_release
226+
&& parse_release(release).is_none()
227+
{
228+
return Err(EditionError::new(format!(
229+
"encoding {id} declares malformed required_vortex_release {release:?}"
230+
)));
231+
}
232+
Ok(())
233+
}
234+
}
235+
236+
/// Parse a `major.minor.patch` release string into a comparable key.
237+
pub(crate) fn parse_release(release: &str) -> Option<Vec<u64>> {
238+
let parts: Vec<u64> = release
239+
.split('.')
240+
.map(|part| part.parse::<u64>().ok())
241+
.collect::<Option<_>>()?;
242+
(parts.len() == 3).then_some(parts)
243+
}
244+
245+
/// Error raised when edition declarations are inconsistent.
246+
#[derive(Debug)]
247+
pub struct EditionError(String);
248+
249+
impl EditionError {
250+
/// Create an error with the given message.
251+
pub fn new(msg: impl Into<String>) -> Self {
252+
Self(msg.into())
253+
}
254+
}
255+
256+
impl Display for EditionError {
257+
fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
258+
f.write_str(&self.0)
259+
}
260+
}
261+
262+
impl Error for EditionError {}

0 commit comments

Comments
 (0)