diff --git a/Cargo.lock b/Cargo.lock index 7a2fe39342d..0747e266768 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9613,6 +9613,7 @@ dependencies = [ "vortex-bytebool", "vortex-datetime-parts", "vortex-decimal-byte-parts", + "vortex-edition", "vortex-error", "vortex-fastlanes", "vortex-file", @@ -10094,6 +10095,14 @@ dependencies = [ "zip", ] +[[package]] +name = "vortex-edition" +version = "0.1.0" +dependencies = [ + "parking_lot", + "vortex-session", +] + [[package]] name = "vortex-error" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 68be0c50653..471bd7ce6fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "vortex-mask", "vortex-utils", "vortex-session", + "vortex-edition", "vortex-flatbuffers", "vortex-metrics", "vortex-io", @@ -295,6 +296,7 @@ vortex-compute = { version = "0.1.0", path = "./vortex-compute", default-feature vortex-datafusion = { version = "0.1.0", path = "./vortex-datafusion", default-features = false } vortex-datetime-parts = { version = "0.1.0", path = "./encodings/datetime-parts", default-features = false } vortex-decimal-byte-parts = { version = "0.1.0", path = "encodings/decimal-byte-parts", default-features = false } +vortex-edition = { version = "0.1.0", path = "./vortex-edition", default-features = false } vortex-error = { version = "0.1.0", path = "./vortex-error", default-features = false } vortex-fastlanes = { version = "0.1.0", path = "./encodings/fastlanes", default-features = false } vortex-file = { version = "0.1.0", path = "./vortex-file", default-features = false } diff --git a/docs/specs/editions.md b/docs/specs/editions.md index bfe0bd38e12..d76bf954f1d 100644 --- a/docs/specs/editions.md +++ b/docs/specs/editions.md @@ -4,12 +4,14 @@ Vortex defines an evergrowing set of serializable array encodings, once written version of vortex. **Editions** are used to keep track of these encodings and talk about groups of encodings. -The edition `core2026.07.0` (coming soon) is the first such edition containing all encodings currently -enabled by the writer. +The first edition, `core2025.05.0`, contains the stable encodings that could be written by Vortex +`0.36.0`. This is the release from which the Vortex file format is considered stable. Later `core` +editions add stable encodings released after that compatibility boundary. Editions are additive so an edition that comes after a previous one contains all the encodings from the previous one and more. -The writer can be configured with a set of different editions (e.g. `core2026.07.0` and `unstable2026.05.0` all stable -encoding released before July 2026 and all unstable encodings from May 2026). +The writer can be configured with a set of different editions (for example, `core2026.07.0` and +`unstable2026.06.0` select stable encodings released through July 2026 and unstable encodings +released through June 2026). Editions can be used to constrain your minimum required vortex reader, since latest version over vortex across all editions is the earliest version of vortex required to read that file. @@ -57,4 +59,4 @@ indefinitely, so deprecation never invalidates existing files. ## Edition registry -Coming soon.. \ No newline at end of file +Coming soon.. diff --git a/vortex-array/src/arrays/decimal/vtable/mod.rs b/vortex-array/src/arrays/decimal/vtable/mod.rs index 1ed839b813e..ec157b28c66 100644 --- a/vortex-array/src/arrays/decimal/vtable/mod.rs +++ b/vortex-array/src/arrays/decimal/vtable/mod.rs @@ -164,7 +164,6 @@ impl VTable for Decimal { dtype: &DType, len: usize, metadata: &[u8], - buffers: &[BufferHandle], children: &dyn ArrayChildren, _session: &VortexSession, diff --git a/vortex-edition/Cargo.toml b/vortex-edition/Cargo.toml new file mode 100644 index 00000000000..a4ec225c5a8 --- /dev/null +++ b/vortex-edition/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "vortex-edition" +authors.workspace = true +description = "Definitions of Vortex editions: named, frozen sets of encodings with a read-compatibility guarantee" +edition = { workspace = true } +homepage = { workspace = true } +categories = { workspace = true } +include = { workspace = true } +keywords = { workspace = true } +license = { workspace = true } +readme = { workspace = true } +repository = { workspace = true } +rust-version = { workspace = true } +version = { workspace = true } + +[package.metadata.docs.rs] +all-features = true + +[lints] +workspace = true + +[dependencies] +parking_lot = { workspace = true } +vortex-session = { workspace = true } diff --git a/vortex-edition/src/lib.rs b/vortex-edition/src/lib.rs new file mode 100644 index 00000000000..c4e341450d8 --- /dev/null +++ b/vortex-edition/src/lib.rs @@ -0,0 +1,262 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Definitions of Vortex *editions*: named, frozen sets of encodings that a writer may put in +//! a file, carrying a forever read-compatibility guarantee. +//! +//! Editions live on the session, like encodings do: [`EditionSession`] is the session +//! variable holding the edition registry, populated at initialization time. Declarations +//! are plain constants — an [`EditionId`] plus an [`Edition`] record, and one +//! [`EditionInclusion`] per encoding stating that it is a member of an edition *and every +//! later edition of the same family*. Any crate can register declarations into a session, +//! so inclusions can live next to the encoding they describe. +//! +//! An edition is a **draft** until its [`Edition::min_vortex_version`] is recorded — +//! recording it is the act of freezing. The per-edition encoding sets are computed from the +//! registered declarations by [`EditionSession::encodings_in`], and correctness is enforced +//! by unit tests: [`EditionSession::validate`] checks a whole registry, and +//! [`test_harness::validate_edition`] validates one edition's constraints — call it once in +//! the `#[cfg(test)]` module of each edition definition. +//! +//! This crate defines only the types and the session variable; the first-party edition +//! declarations live in the public `vortex` crate (`vortex::editions`), which seeds them +//! into the default session. See the published spec at +//! . + +mod session; +pub mod test_harness; +#[cfg(test)] +mod tests; + +use std::error::Error; +use std::fmt; +use std::fmt::Debug; +use std::fmt::Display; +use std::fmt::Formatter; + +pub use session::EditionSession; +pub use session::EditionSessionExt; +use vortex_session::registry::Id; + +/// The identifier of an edition, e.g. `core2026.07.0`. +/// +/// The `family` names an independently versioned, additive group of encodings (`core` is the +/// set the default writer emits). The date components record when the edition was frozen and +/// order editions chronologically *within* a family; there is no ordering across families. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct EditionId { + /// The edition family, e.g. `core`. + pub family: &'static str, + /// Year the edition was cut. + pub year: u16, + /// Month the edition was cut. + pub month: u8, + /// Distinguishes editions cut in the same month; normally `0`. + pub version: u8, +} + +impl EditionId { + /// Create an edition identifier. Validated by [`EditionId::validate`], which + /// [`test_harness::validate_edition`] exercises + /// per edition in unit tests. + pub const fn new(family: &'static str, year: u16, month: u8, version: u8) -> Self { + Self { + family, + year, + month, + version, + } + } + + /// Returns true if `self` is the same edition as `other` or an earlier edition of the + /// same family. Editions of different families are never ordered. + pub fn is_at_or_before(&self, other: &EditionId) -> bool { + self.family == other.family + && (self.year, self.month, self.version) <= (other.year, other.month, other.version) + } + + /// Validate the identifier's form: a non-empty lowercase family, a four-digit year, + /// and a month in 01-12. Checked for every declared edition by + /// [`EditionSession::validate`] and per edition by + /// [`test_harness::validate_edition`]. + pub fn validate(&self) -> Result<(), EditionError> { + if self.family.is_empty() || !self.family.chars().all(|c| c.is_ascii_lowercase()) { + return Err(EditionError::new(format!( + "edition {self} must have a non-empty lowercase family, e.g. `core`" + ))); + } + if !(1000..=9999).contains(&self.year) { + return Err(EditionError::new(format!( + "edition {self} must have a four-digit year" + ))); + } + if !(1..=12).contains(&self.month) { + return Err(EditionError::new(format!( + "edition {self} must have a month in 01-12" + ))); + } + Ok(()) + } +} + +impl Display for EditionId { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!( + f, + "{}{}.{:02}.{}", + self.family, self.year, self.month, self.version + ) + } +} + +/// An edition: a named set of encodings with a read-compatibility guarantee, registered with +/// [`EditionSession::declare_edition`]. The set itself is computed from the registered +/// [`EditionInclusion`]s by [`EditionSession::encodings_in`]. +#[derive(Clone, Copy, Debug)] +pub struct Edition { + /// The edition identifier. Also carries the freeze date: `core2026.07.0` freezes in + /// 2026-07. + pub id: EditionId, + /// The minimum Vortex version whose reader supports every encoding in this edition. + /// + /// Recording this is the act of freezing: an edition with `None` is a **draft** — being + /// assembled, carrying no guarantee, free to change, never the default write target. + /// Validated against the members' [`EditionInclusion::required_vortex_release`] values: + /// no member may require a version newer than the edition declares. + pub min_vortex_version: Option<&'static str>, +} + +impl Edition { + /// A draft is an edition whose `min_vortex_version` has not been recorded yet. + pub fn is_draft(&self) -> bool { + self.min_vortex_version.is_none() + } +} + +/// Declares that an encoding is a member of an edition — and of every later edition of the +/// same family. Registered with [`EditionSession::declare_inclusion`]. +#[derive(Clone, Copy, Debug)] +pub struct EditionInclusion { + /// The interned encoding id, e.g. `vortex.alp`. Globally unique across everything an + /// edition can cover: when layout encodings join editions, their ids must be distinct + /// from array encoding ids. + pub encoding_id: Id, + /// The first edition this encoding is a member of. + pub since: EditionId, + /// The earliest Vortex release able to read and execute this encoding, recorded from + /// evidence (e.g. compat-fixture history). `None` until recorded. + pub required_vortex_release: Option<&'static str>, +} + +/// A source of an encoding id for edition declarations. +/// +/// Implemented for raw id strings (`"vortex.alp"`) and interned [`Id`]s here; encoding +/// vtables implement it where they are defined, so a declaration can name the vtable +/// (`&Primitive`) instead of spelling its id. +pub trait AsEncodingId: Debug + Send + Sync { + /// The interned encoding id. + fn encoding_id(&self) -> Id; +} + +impl AsEncodingId for str { + #[expect( + clippy::disallowed_methods, + reason = "interning a dynamic encoding id at declaration time" + )] + fn encoding_id(&self) -> Id { + Id::new(self) + } +} + +impl AsEncodingId for Id { + fn encoding_id(&self) -> Id { + *self + } +} + +// `str` is unsized and cannot be a trait object, so declaration blocks (slices of +// `&dyn AsEncodingId`) name encodings as `&"vortex.alp"` through this impl. +impl AsEncodingId for &'static str { + fn encoding_id(&self) -> Id { + (**self).encoding_id() + } +} + +/// Declares an edition together with the encodings that join the family at it, in one +/// block. Registered with [`EditionSession::declare`], which derives each encoding's +/// membership (`since` = the declared edition) from the block structure. +#[derive(Clone, Copy, Debug)] +pub struct EditionDeclaration { + /// The edition being declared. + pub edition: Edition, + /// The encodings that join the family at this edition, named by id string or by + /// vtable. Members of earlier editions are inherited and never restated. + pub added: &'static [&'static dyn AsEncodingId], +} + +impl EditionInclusion { + /// Declare that an encoding is a member of `since` and every later edition of the same + /// family. The encoding can be named by id string or by vtable. + pub fn new(encoding: &E, since: EditionId) -> Self { + Self { + encoding_id: encoding.encoding_id(), + since, + required_vortex_release: None, + } + } + + /// Validate the declaration's form: a lowercase `namespace.name` encoding id and, if + /// recorded, a well-formed `major.minor.patch` release. Checked for every declared + /// inclusion by [`EditionSession::validate`]. + pub fn validate(&self) -> Result<(), EditionError> { + let id = self.encoding_id.as_str(); + let well_formed = !id.starts_with('.') + && !id.ends_with('.') + && id.contains('.') + && id + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || "._-".contains(c)); + if !well_formed { + return Err(EditionError::new(format!( + "invalid encoding id {id:?}: expected lowercase `namespace.name`, e.g. \ + `vortex.alp`" + ))); + } + if let Some(release) = self.required_vortex_release + && parse_release(release).is_none() + { + return Err(EditionError::new(format!( + "encoding {id} declares malformed required_vortex_release {release:?}" + ))); + } + Ok(()) + } +} + +/// Parse a `major.minor.patch` release string into a comparable key. +pub(crate) fn parse_release(release: &str) -> Option> { + let parts: Vec = release + .split('.') + .map(|part| part.parse::().ok()) + .collect::>()?; + (parts.len() == 3).then_some(parts) +} + +/// Error raised when edition declarations are inconsistent. +#[derive(Debug)] +pub struct EditionError(String); + +impl EditionError { + /// Create an error with the given message. + pub fn new(msg: impl Into) -> Self { + Self(msg.into()) + } +} + +impl Display for EditionError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +impl Error for EditionError {} diff --git a/vortex-edition/src/session.rs b/vortex-edition/src/session.rs new file mode 100644 index 00000000000..a2a9b50066a --- /dev/null +++ b/vortex-edition/src/session.rs @@ -0,0 +1,201 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The [`EditionSession`] session variable: the per-session registry of editions and +//! edition inclusions. + +use std::any::Any; +use std::collections::BTreeMap; +use std::sync::Arc; + +use parking_lot::RwLock; +use vortex_session::SessionExt; +use vortex_session::SessionGuard; +use vortex_session::SessionVar; +use vortex_session::registry::Id; + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionError; +use crate::EditionId; +use crate::EditionInclusion; +use crate::parse_release; + +/// The session's registry of editions and edition inclusions. +/// +/// Starts empty and is populated at initialization time: the `vortex` facade seeds the +/// first-party declarations (`vortex::editions`), and crates registering additional +/// encodings declare their inclusions (and, for new families, editions) alongside their +/// encoding registration. Clones share the same underlying registry, matching the other +/// session registries. +#[derive(Clone, Debug, Default)] +pub struct EditionSession { + inner: Arc>, +} + +#[derive(Debug, Default)] +struct Inner { + /// Keyed by the display form of the edition id. + editions: BTreeMap, + /// Keyed by interned encoding id; ordered by the id's string form. + inclusions: BTreeMap, +} + +impl EditionSession { + /// Create a session variable with no declarations. + pub fn empty() -> Self { + Self { + inner: Arc::new(RwLock::new(Inner::default())), + } + } + + /// Declare an edition together with the encodings that join the family at it. Each + /// added encoding's membership (`since`) is the declared edition; members of earlier + /// editions are inherited and must not be restated. + pub fn declare(&self, declaration: &EditionDeclaration) -> Result<(), EditionError> { + self.declare_edition(declaration.edition)?; + for encoding in declaration.added { + self.declare_inclusion(EditionInclusion::new(*encoding, declaration.edition.id))?; + } + Ok(()) + } + + /// Declare an edition. Errors if an edition with the same id is already declared. + pub fn declare_edition(&self, edition: Edition) -> Result<(), EditionError> { + let mut inner = self.inner.write(); + let key = edition.id.to_string(); + if inner.editions.contains_key(&key) { + return Err(EditionError::new(format!("duplicate edition {key}"))); + } + inner.editions.insert(key, edition); + Ok(()) + } + + /// Declare an edition inclusion. Errors if the encoding already has one: an encoding + /// belongs to exactly one family, with one membership interval. + pub fn declare_inclusion(&self, inclusion: EditionInclusion) -> Result<(), EditionError> { + let mut inner = self.inner.write(); + if inner.inclusions.contains_key(&inclusion.encoding_id) { + return Err(EditionError::new(format!( + "duplicate edition inclusion for encoding {}", + inclusion.encoding_id + ))); + } + inner.inclusions.insert(inclusion.encoding_id, inclusion); + Ok(()) + } + + /// All declared editions, sorted by family and then chronologically. The newest frozen + /// edition of each family is that family's `current` edition; unversioned editions are + /// drafts. + pub fn editions(&self) -> Vec { + let mut editions: Vec = self.inner.read().editions.values().copied().collect(); + editions.sort_by_key(|e| (e.id.family, e.id.year, e.id.month, e.id.version)); + editions + } + + /// Find a declared edition by id. + pub fn find(&self, id: &EditionId) -> Option { + self.inner.read().editions.get(&id.to_string()).copied() + } + + /// The newest frozen edition of a family, if any. Drafts are never current. + pub fn current(&self, family: &str) -> Option { + self.editions() + .into_iter() + .filter(|e| e.id.family == family && !e.is_draft()) + .next_back() + } + + /// Compute the full encoding set of an edition: every declared inclusion of the + /// edition's family whose `since` is at or before it, sorted by encoding id. + pub fn encodings_in(&self, edition: &EditionId) -> Vec { + // The map is keyed by encoding id, so the values are already sorted by it. + self.inner + .read() + .inclusions + .values() + .filter(|inclusion| inclusion.since.is_at_or_before(edition)) + .copied() + .collect() + } + + /// Validate all registered declarations. Errors on inclusions referencing undeclared + /// editions, editions out of chronological order within a family (unversioned drafts + /// must be newest), malformed version strings, and members requiring a release newer + /// than their edition declares. + pub fn validate(&self) -> Result<(), EditionError> { + let editions = self.editions(); + + for edition in &editions { + edition.id.validate()?; + if let Some(version) = edition.min_vortex_version + && parse_release(version).is_none() + { + return Err(EditionError::new(format!( + "edition {} declares malformed min_vortex_version {version:?}", + edition.id + ))); + } + } + + // Within each family, frozen editions must precede drafts: a frozen edition after + // an unversioned one would imply the draft was skipped. + for pair in editions.windows(2) { + let (prev, next) = (&pair[0], &pair[1]); + if prev.id.family == next.id.family && prev.is_draft() && !next.is_draft() { + return Err(EditionError::new(format!( + "frozen edition {} follows draft {}; drafts must be newest in a family", + next.id, prev.id, + ))); + } + } + + let inner = self.inner.read(); + for inclusion in inner.inclusions.values() { + inclusion.validate()?; + + let Some(edition) = inner.editions.get(&inclusion.since.to_string()) else { + return Err(EditionError::new(format!( + "encoding {} is included in undeclared edition {}", + inclusion.encoding_id, inclusion.since + ))); + }; + + if let Some(required) = inclusion.required_vortex_release.and_then(parse_release) + && let Some(declared) = edition.min_vortex_version.and_then(parse_release) + && required > declared + { + return Err(EditionError::new(format!( + "encoding {} requires release {}, newer than edition {}'s declared \ + min_vortex_version", + inclusion.encoding_id, + inclusion.required_vortex_release.unwrap_or_default(), + edition.id, + ))); + } + } + + Ok(()) + } +} + +impl SessionVar for EditionSession { + fn as_any(&self) -> &dyn Any { + self + } + + fn as_any_mut(&mut self) -> &mut dyn Any { + self + } +} + +/// Session data for Vortex editions. +pub trait EditionSessionExt: SessionExt { + /// Returns the edition registry. + fn editions(&self) -> SessionGuard<'_, EditionSession> { + self.get::() + } +} + +impl EditionSessionExt for S {} diff --git a/vortex-edition/src/test_harness.rs b/vortex-edition/src/test_harness.rs new file mode 100644 index 00000000000..d98315bc240 --- /dev/null +++ b/vortex-edition/src/test_harness.rs @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! Test harness for edition declarations. +//! +//! Each edition definition should call [`validate_edition`] once from its `#[cfg(test)]` +//! module, so every declared edition has a test proving its constraints hold. + +use crate::EditionError; +use crate::EditionId; +use crate::EditionSession; + +/// Validate one edition's constraints: its identifier is well-formed, it is declared in the +/// given session, and the session's declarations as a whole validate (chronology, version +/// forms, membership constraints). +/// +/// ```ignore +/// #[cfg(test)] +/// mod tests { +/// #[test] +/// fn edition_is_valid() -> Result<(), vortex_edition::EditionError> { +/// vortex_edition::test_harness::validate_edition(&edition_session(), &CORE_2026_01_0) +/// } +/// } +/// ``` +pub fn validate_edition( + editions: &EditionSession, + edition: &EditionId, +) -> Result<(), EditionError> { + edition.validate()?; + if editions.find(edition).is_none() { + return Err(EditionError::new(format!( + "{edition} is not declared in the session" + ))); + } + editions.validate() +} diff --git a/vortex-edition/src/tests.rs b/vortex-edition/src/tests.rs new file mode 100644 index 00000000000..3791173fad8 --- /dev/null +++ b/vortex-edition/src/tests.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +use vortex_session::VortexSession; + +use crate::Edition; +use crate::EditionDeclaration; +use crate::EditionId; +use crate::EditionInclusion; +use crate::EditionSession; +use crate::EditionSessionExt; + +const FIRST: EditionId = EditionId::new("test", 2026, 1, 0); +const SECOND: EditionId = EditionId::new("test", 2026, 7, 0); + +static DECLARATIONS: &[EditionDeclaration] = &[ + EditionDeclaration { + edition: Edition { + id: FIRST, + min_vortex_version: None, + }, + added: &[&"test.alpha", &"test.beta"], + }, + EditionDeclaration { + edition: Edition { + id: SECOND, + min_vortex_version: None, + }, + added: &[&"test.gamma"], + }, +]; + +fn session() -> EditionSession { + let editions = EditionSession::empty(); + for declaration in DECLARATIONS { + editions + .declare(declaration) + .unwrap_or_else(|e| panic!("declaring test editions: {e}")); + } + editions +} + +#[test] +fn editions_pass_the_test_harness() -> Result<(), crate::EditionError> { + crate::test_harness::validate_edition(&session(), &FIRST)?; + crate::test_harness::validate_edition(&session(), &SECOND)?; + + // An undeclared edition fails the harness. + let undeclared = EditionId::new("test", 2027, 1, 0); + assert!(crate::test_harness::validate_edition(&session(), &undeclared).is_err()); + Ok(()) +} + +#[test] +fn membership_is_transitive() { + let editions = session(); + + let first = editions.encodings_in(&FIRST); + let ids: Vec<&str> = first.iter().map(|i| i.encoding_id.as_str()).collect(); + assert_eq!(ids, ["test.alpha", "test.beta"]); + + // Members of the first edition are members of the second by inheritance, with their + // `since` still recording the edition they actually joined in. + let second = editions.encodings_in(&SECOND); + let ids: Vec<&str> = second.iter().map(|i| i.encoding_id.as_str()).collect(); + assert_eq!(ids, ["test.alpha", "test.beta", "test.gamma"]); + assert!( + second + .iter() + .filter(|i| i.encoding_id.as_str() != "test.gamma") + .all(|i| i.since == FIRST) + ); + + // The second edition's delta is exactly the members declared at it. + let added: Vec<&str> = second + .iter() + .filter(|i| i.since == SECOND) + .map(|i| i.encoding_id.as_str()) + .collect(); + assert_eq!(added, ["test.gamma"]); + + // Inheritance never flows backwards, extends to later editions of the family, and + // never crosses families. + assert!(first.iter().all(|i| i.since == FIRST)); + let third = EditionId::new("test", 2026, 10, 0); + assert_eq!(editions.encodings_in(&third).len(), 3); + let other = EditionId::new("other", 2026, 10, 0); + assert!(editions.encodings_in(&other).is_empty()); +} + +#[test] +fn drafts_and_current() { + let editions = session(); + // Both editions are unversioned drafts: neither is current. + assert!(editions.find(&FIRST).is_some_and(|e| e.is_draft())); + assert!(editions.current("test").is_none()); + + // Freezing the first edition makes it current; the second stays a draft. + let editions = EditionSession::empty(); + editions + .declare_edition(Edition { + id: FIRST, + min_vortex_version: Some("0.60.0"), + }) + .unwrap(); + editions + .declare_edition(Edition { + id: SECOND, + min_vortex_version: None, + }) + .unwrap(); + assert!(editions.validate().is_ok()); + assert_eq!(editions.current("test").map(|e| e.id), Some(FIRST)); +} + +#[test] +fn session_exposes_edition_registry() { + // The session variable starts empty; declarations are seeded at initialization time + // (the `vortex` facade seeds the first-party ones). + let session = VortexSession::empty().with::(); + assert!(session.editions().find(&FIRST).is_none()); + + for declaration in DECLARATIONS { + session.editions().declare(declaration).unwrap(); + } + assert!(session.editions().find(&FIRST).is_some()); +} + +#[test] +fn duplicate_declarations_error() { + let editions = session(); + assert!( + editions + .declare_edition(Edition { + id: FIRST, + min_vortex_version: None, + }) + .is_err() + ); + assert!( + editions + .declare_inclusion(EditionInclusion::new("test.alpha", FIRST)) + .is_err() + ); +} + +#[test] +fn validate_rejects_inconsistent_declarations() -> Result<(), crate::EditionError> { + // An inclusion referencing an undeclared edition. + let editions = EditionSession::empty(); + editions.declare_inclusion(EditionInclusion::new("test.alpha", FIRST))?; + assert!(editions.validate().is_err()); + + // A member requiring a release newer than its edition declares. + let editions = EditionSession::empty(); + editions.declare_edition(Edition { + id: FIRST, + min_vortex_version: Some("0.70.0"), + })?; + editions.declare_inclusion(EditionInclusion { + required_vortex_release: Some("0.80.0"), + ..EditionInclusion::new("test.alpha", FIRST) + })?; + assert!(editions.validate().is_err()); + + // A frozen edition following a draft within the same family. + let editions = EditionSession::empty(); + editions.declare_edition(Edition { + id: FIRST, + min_vortex_version: None, + })?; + editions.declare_edition(Edition { + id: SECOND, + min_vortex_version: Some("0.70.0"), + })?; + assert!(editions.validate().is_err()); + + // A malformed edition id (family not lowercase, month out of range). + let editions = EditionSession::empty(); + editions.declare_edition(Edition { + id: EditionId::new("Test", 2026, 13, 0), + min_vortex_version: None, + })?; + assert!(editions.validate().is_err()); + + // A malformed encoding id. + let editions = EditionSession::empty(); + editions.declare_edition(Edition { + id: FIRST, + min_vortex_version: None, + })?; + editions.declare_inclusion(EditionInclusion::new("Test.ALPHA", FIRST))?; + assert!(editions.validate().is_err()); + + Ok(()) +} + +#[test] +fn edition_ids_order_within_family_only() { + assert!(FIRST.is_at_or_before(&SECOND)); + assert!(!SECOND.is_at_or_before(&FIRST)); + + let other = EditionId::new("other", 2026, 3, 0); + assert!(!FIRST.is_at_or_before(&other)); + assert!(!other.is_at_or_before(&FIRST)); +} + +#[test] +fn edition_id_display() { + assert_eq!(FIRST.to_string(), "test2026.01.0"); +} diff --git a/vortex/Cargo.toml b/vortex/Cargo.toml index 2511893d09e..3cc3e40d65b 100644 --- a/vortex/Cargo.toml +++ b/vortex/Cargo.toml @@ -31,6 +31,7 @@ vortex-buffer = { workspace = true } vortex-bytebool = { workspace = true } vortex-datetime-parts = { workspace = true } vortex-decimal-byte-parts = { workspace = true } +vortex-edition = { workspace = true } vortex-error = { workspace = true } vortex-fastlanes = { workspace = true } vortex-file = { workspace = true, optional = true, default-features = true } diff --git a/vortex/src/editions/core/mod.rs b/vortex/src/editions/core/mod.rs new file mode 100644 index 00000000000..cab74642305 --- /dev/null +++ b/vortex/src/editions/core/mod.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `core` edition family: the encodings the default file writer emits. +//! +//! One module per edition, each declaring the edition and the encodings that join the +//! family at it; members of earlier editions are inherited and never restated. + +pub mod v2025_05; +pub mod v2025_06; +pub mod v2025_10; +pub mod v2026_07; + +pub use v2025_05::CORE_2025_05_0; +pub use v2025_06::CORE_2025_06_0; +pub use v2025_10::CORE_2025_10_0; +pub use v2026_07::CORE_2026_07_0; diff --git a/vortex/src/editions/core/v2025_05.rs b/vortex/src/editions/core/v2025_05.rs new file mode 100644 index 00000000000..a25a6f971a4 --- /dev/null +++ b/vortex/src/editions/core/v2025_05.rs @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The baseline `core` edition: stable encodings writable by Vortex 0.36.0. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The first edition of the `core` family, matching the first stable Vortex file release. +pub const CORE_2025_05_0: EditionId = EditionId::new("core", 2025, 5, 0); + +/// The declaration of [`CORE_2025_05_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2025_05_0, + min_vortex_version: Some("0.36.0"), + }, + added: &[ + &"fastlanes.bitpacked", + &"fastlanes.for", + &"vortex.alp", + &"vortex.alprd", + &"vortex.bool", + &"vortex.bytebool", + &"vortex.chunked", + &"vortex.constant", + &"vortex.datetimeparts", + &"vortex.decimal", + &"vortex.decimal_byte_parts", + &"vortex.dict", + &"vortex.ext", + &"vortex.fsst", + &"vortex.list", + &"vortex.null", + &"vortex.primitive", + &"vortex.runend", + &"vortex.sparse", + &"vortex.struct", + &"vortex.varbin", + &"vortex.varbinview", + &"vortex.zigzag", + ], +}; diff --git a/vortex/src/editions/core/v2025_06.rs b/vortex/src/editions/core/v2025_06.rs new file mode 100644 index 00000000000..015325b8429 --- /dev/null +++ b/vortex/src/editions/core/v2025_06.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `core` edition adding stable encodings released through June 2025. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The June 2025 edition of the `core` family. +pub const CORE_2025_06_0: EditionId = EditionId::new("core", 2025, 6, 0); + +/// The declaration of [`CORE_2025_06_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2025_06_0, + min_vortex_version: Some("0.40.0"), + }, + added: &[&"vortex.pco", &"vortex.sequence", &"vortex.zstd"], +}; diff --git a/vortex/src/editions/core/v2025_10.rs b/vortex/src/editions/core/v2025_10.rs new file mode 100644 index 00000000000..ae21026b595 --- /dev/null +++ b/vortex/src/editions/core/v2025_10.rs @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `core` edition adding stable encodings released through October 2025. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The October 2025 edition of the `core` family. +pub const CORE_2025_10_0: EditionId = EditionId::new("core", 2025, 10, 0); + +/// The declaration of [`CORE_2025_10_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2025_10_0, + min_vortex_version: Some("0.54.0"), + }, + added: &[ + &"fastlanes.rle", + &"vortex.fixed_size_list", + &"vortex.listview", + &"vortex.masked", + ], +}; diff --git a/vortex/src/editions/core/v2026_07.rs b/vortex/src/editions/core/v2026_07.rs new file mode 100644 index 00000000000..820c68cf7dd --- /dev/null +++ b/vortex/src/editions/core/v2026_07.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `core` edition adding stable encodings released through July 2026. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The July 2026 edition of the `core` family. +pub const CORE_2026_07_0: EditionId = EditionId::new("core", 2026, 7, 0); + +/// The declaration of [`CORE_2026_07_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: CORE_2026_07_0, + min_vortex_version: Some("0.65.0"), + }, + added: &[&"vortex.variant"], +}; diff --git a/vortex/src/editions/mod.rs b/vortex/src/editions/mod.rs new file mode 100644 index 00000000000..1c3323d077f --- /dev/null +++ b/vortex/src/editions/mod.rs @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The Vortex edition declarations. +//! +//! [`vortex_edition`] provides the types, the [`crate::editions::EditionSession`] session +//! variable, and the test harness; the actual declarations live here, one module per edition +//! (`editions::::`), and are seeded into the default session by +//! [`crate::editions::register_default_editions`]. +//! +//! Each edition module declares the edition together with the encodings that join the +//! family at it; members of earlier editions are inherited and never restated. Correctness +//! is enforced by unit tests: every edition module calls +//! [`vortex_edition::test_harness::validate_edition`] once from its `#[cfg(test)]` module, +//! and the computed set of a frozen edition is pinned by a golden test, so any change to +//! these declarations that alters a frozen set fails CI. New encodings are staged into the +//! newest draft edition. +//! +//! Note this is currently unused but a future PR will make this public and gate the writer behind +//! editions. + +pub mod core; +pub mod unstable; + +use vortex_edition::EditionDeclaration; +pub use vortex_edition::EditionSession; +use vortex_edition::EditionSessionExt; +use vortex_error::VortexExpect; +use vortex_error::vortex_err; +use vortex_session::VortexSession; + +pub use self::core::CORE_2025_05_0; +pub use self::core::CORE_2025_06_0; +pub use self::core::CORE_2025_10_0; +pub use self::core::CORE_2026_07_0; +pub use self::unstable::UNSTABLE_2025_05_0; +pub use self::unstable::UNSTABLE_2026_02_0; +pub use self::unstable::UNSTABLE_2026_04_0; +pub use self::unstable::UNSTABLE_2026_06_0; + +/// The Vortex editions, each declared together with the encodings it adds. +pub static EDITION_DECLARATIONS: &[&EditionDeclaration] = &[ + &core::v2025_05::DECLARATION, + &core::v2025_06::DECLARATION, + &core::v2025_10::DECLARATION, + &core::v2026_07::DECLARATION, + &unstable::v2025_05::DECLARATION, + &unstable::v2026_02::DECLARATION, + &unstable::v2026_04::DECLARATION, + &unstable::v2026_06::DECLARATION, +]; + +/// Register the Vortex edition declarations with the session's [`EditionSession`]. +pub fn register_default_editions(session: &VortexSession) { + let editions = session.editions(); + for declaration in EDITION_DECLARATIONS { + editions + .declare(declaration) + .map_err(|e| vortex_err!("{e}")) + .vortex_expect("edition declarations are valid"); + } +} diff --git a/vortex/src/editions/unstable/mod.rs b/vortex/src/editions/unstable/mod.rs new file mode 100644 index 00000000000..5544ba45c0f --- /dev/null +++ b/vortex/src/editions/unstable/mod.rs @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The `unstable` edition family: opt-in encodings without a frozen compatibility guarantee. +//! +//! One module per draft edition, each declaring the encodings that join the family at it. +//! Members of earlier editions are inherited and never restated. + +pub mod v2025_05; +pub mod v2026_02; +pub mod v2026_04; +pub mod v2026_06; + +pub use v2025_05::UNSTABLE_2025_05_0; +pub use v2026_02::UNSTABLE_2026_02_0; +pub use v2026_04::UNSTABLE_2026_04_0; +pub use v2026_06::UNSTABLE_2026_06_0; diff --git a/vortex/src/editions/unstable/v2025_05.rs b/vortex/src/editions/unstable/v2025_05.rs new file mode 100644 index 00000000000..2cc6accdc38 --- /dev/null +++ b/vortex/src/editions/unstable/v2025_05.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The May 2025 `unstable` encoding cohort. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The May 2025 draft edition of the `unstable` family. +pub const UNSTABLE_2025_05_0: EditionId = EditionId::new("unstable", 2025, 5, 0); + +/// The declaration of [`UNSTABLE_2025_05_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE_2025_05_0, + min_vortex_version: None, + }, + added: &[&"fastlanes.delta"], +}; diff --git a/vortex/src/editions/unstable/v2026_02.rs b/vortex/src/editions/unstable/v2026_02.rs new file mode 100644 index 00000000000..e992cdee5ad --- /dev/null +++ b/vortex/src/editions/unstable/v2026_02.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The February 2026 `unstable` encoding cohort. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The February 2026 draft edition of the `unstable` family. +pub const UNSTABLE_2026_02_0: EditionId = EditionId::new("unstable", 2026, 2, 0); + +/// The declaration of [`UNSTABLE_2026_02_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE_2026_02_0, + min_vortex_version: None, + }, + added: &[&"vortex.zstd_buffers"], +}; diff --git a/vortex/src/editions/unstable/v2026_04.rs b/vortex/src/editions/unstable/v2026_04.rs new file mode 100644 index 00000000000..fcc9516dea8 --- /dev/null +++ b/vortex/src/editions/unstable/v2026_04.rs @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The April 2026 `unstable` encoding cohort. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The April 2026 draft edition of the `unstable` family. +pub const UNSTABLE_2026_04_0: EditionId = EditionId::new("unstable", 2026, 4, 0); + +/// The declaration of [`UNSTABLE_2026_04_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE_2026_04_0, + min_vortex_version: None, + }, + added: &[ + &"vortex.parquet.variant", + &"vortex.patched", + &"vortex.tensor.cosine_similarity", + &"vortex.tensor.inner_product", + &"vortex.tensor.l2_denorm", + &"vortex.tensor.l2_norm", + ], +}; diff --git a/vortex/src/editions/unstable/v2026_06.rs b/vortex/src/editions/unstable/v2026_06.rs new file mode 100644 index 00000000000..acbd739b656 --- /dev/null +++ b/vortex/src/editions/unstable/v2026_06.rs @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright the Vortex contributors + +//! The June 2026 `unstable` encoding cohort. + +use vortex_edition::Edition; +use vortex_edition::EditionDeclaration; +use vortex_edition::EditionId; + +/// The June 2026 draft edition of the `unstable` family. +pub const UNSTABLE_2026_06_0: EditionId = EditionId::new("unstable", 2026, 6, 0); + +/// The declaration of [`UNSTABLE_2026_06_0`] and the encodings that join the family at it. +pub static DECLARATION: EditionDeclaration = EditionDeclaration { + edition: Edition { + id: UNSTABLE_2026_06_0, + min_vortex_version: None, + }, + added: &[&"vortex.onpair"], +}; diff --git a/vortex/src/lib.rs b/vortex/src/lib.rs index 2703020906c..1af8484f230 100644 --- a/vortex/src/lib.rs +++ b/vortex/src/lib.rs @@ -146,6 +146,9 @@ pub mod compressor { } /// Logical Vortex data types. +/// Vortex editions: named, frozen sets of encodings with a read-compatibility guarantee. +pub mod editions; + pub mod dtype { pub use vortex_array::dtype::*; } @@ -305,6 +308,7 @@ impl VortexSessionDefault for VortexSession { .with::() .with::(); vortex_arrow::initialize(&session); + editions::register_default_editions(&session); // `MultiFileSession` holds a `moka` cache whose clock reads `std::time::Instant::now()` // when constructed. `Instant` is unsupported on `wasm32` and panics with "time not