diff --git a/docs/specs/file-format.md b/docs/specs/file-format.md index 425294f2d99..97cc0dbf694 100644 --- a/docs/specs/file-format.md +++ b/docs/specs/file-format.md @@ -52,6 +52,13 @@ The postscript contains the locations of: 2. a `layout` segment containing the root `Layout` 3. a `statistics` segment containing file-level per-field statistics (e.g., minima and maxima of each field/column, for whole-file pruning) 4. a `footer` segment containing a dictionary-encoded _segment map_, and other shared configuration such as compression and encryption schemes +5. an optional `metadata` segment holding user-defined file-level metadata (a `FileMetadata` flatbuffer of opaque byte values keyed by unique, non-empty UTF-8 strings) + +The postscript carries a single locator (offset, length, and alignment) for the metadata segment +when the file has one; a file written without user metadata (and any file predating this feature) +carries none. The segment may live anywhere in the file. Readers do not load the metadata by +default; an opt-in read resolves the one locator, fetching the segment without reading the +intervening file contents when it lies outside the initial file-tail read. :::{literalinclude} ../../vortex-flatbuffers/flatbuffers/vortex-file/footer.fbs :start-after: [postscript] diff --git a/vortex-buffer/src/alignment.rs b/vortex-buffer/src/alignment.rs index ad5c51761bd..d410c7333c1 100644 --- a/vortex-buffer/src/alignment.rs +++ b/vortex-buffer/src/alignment.rs @@ -111,13 +111,32 @@ impl Alignment { .vortex_expect("alignment is a power of 2 within usize, so its exponent fits in u8") } + /// Try to create an alignment from its log2 exponent. + /// + /// Returns an error when the exponent cannot be represented by a `usize` alignment. + #[inline] + pub fn try_from_exponent(exponent: u8) -> Result { + let alignment = 1usize.checked_shl(u32::from(exponent)).ok_or_else(|| { + vortex_err!( + "Alignment exponent {exponent} must be less than {}", + usize::BITS + ) + })?; + Ok(Self::new(alignment)) + } + /// Create from the log2 exponent of the alignment. /// /// ## Panics /// - /// Panics if `1 << exponent` overflows `usize`. + /// Panics if `1 << exponent` overflows `usize`. Use [`Self::try_from_exponent`] when parsing + /// untrusted input. #[inline] pub const fn from_exponent(exponent: u8) -> Self { + assert!( + (exponent as u32) < usize::BITS, + "Alignment exponent must fit in usize" + ); Self::new(1 << exponent) } } @@ -214,6 +233,13 @@ mod test { assert_eq!(Alignment::from_exponent(10), alignment); } + #[test] + fn invalid_alignment_exponent() { + let error = Alignment::try_from_exponent(u8::try_from(usize::BITS).unwrap()) + .expect_err("an exponent as wide as usize must be rejected"); + assert!(error.to_string().contains("must be less than")); + } + #[test] fn is_aligned_to() { assert!(Alignment::new(1).is_aligned_to(Alignment::new(1))); diff --git a/vortex-file/src/file.rs b/vortex-file/src/file.rs index c9a71f85c55..df7a8e35891 100644 --- a/vortex-file/src/file.rs +++ b/vortex-file/src/file.rs @@ -15,6 +15,7 @@ use vortex_array::ArrayRef; use vortex_array::dtype::DType; use vortex_array::dtype::FieldMask; use vortex_array::expr::Expression; +use vortex_buffer::ByteBuffer; use vortex_error::VortexResult; use vortex_layout::LayoutReader; use vortex_layout::scan::layout::LayoutReaderDataSource; @@ -23,6 +24,7 @@ use vortex_layout::scan::split_by::SplitBy; use vortex_layout::segments::SegmentSource; use vortex_scan::DataSourceRef; use vortex_session::VortexSession; +use vortex_utils::aliases::hash_map::HashMap; use crate::FileStatistics; use crate::footer::Footer; @@ -42,6 +44,8 @@ pub struct VortexFile { segment_source: Arc, /// The Vortex session used to open this file. session: VortexSession, + /// User-defined metadata values resolved for this file open. + metadata: Arc>, /// None id LayoutReader caching is turned off layout_reader_cache: Option>>, } @@ -78,10 +82,16 @@ impl VortexFile { footer, segment_source, session, + metadata: Arc::new(HashMap::new()), layout_reader_cache: None, } } + pub(crate) fn with_metadata(mut self, metadata: Arc>) -> Self { + self.metadata = metadata; + self + } + /// Enable layout reader caching. /// /// Repeated calls to [`layout_reader`](Self::layout_reader), [`scan`](Self::scan), and @@ -91,6 +101,7 @@ impl VortexFile { footer: self.footer, segment_source: self.segment_source, session: self.session, + metadata: self.metadata, layout_reader_cache: Some(OnceLock::new()), } } @@ -117,6 +128,22 @@ impl VortexFile { self.footer.statistics() } + /// Returns the user-defined metadata segments loaded for this file. + /// + /// Metadata is only loaded when requested during open. Iteration order is unspecified. + pub fn metadata_segments(&self) -> impl Iterator { + self.metadata + .iter() + .map(|(key, metadata)| (key.as_str(), metadata)) + } + + /// Returns the loaded user-defined metadata segment for the given key. + /// + /// Returns `None` when the key is absent or metadata was not loaded. + pub fn metadata_segment(&self, key: &str) -> Option<&ByteBuffer> { + self.metadata.get(key) + } + /// Create a new segment source for reading from the file. /// /// This may spawn a background I/O driver that will exit when the returned segment source diff --git a/vortex-file/src/footer/deserializer.rs b/vortex-file/src/footer/deserializer.rs index a0f12e85c8a..2b7014a3399 100644 --- a/vortex-file/src/footer/deserializer.rs +++ b/vortex-file/src/footer/deserializer.rs @@ -18,6 +18,7 @@ use crate::Footer; use crate::MAGIC_BYTES; use crate::VERSION; use crate::footer::FileStatistics; +use crate::footer::SegmentSpec; use crate::footer::postscript::Postscript; use crate::footer::postscript::PostscriptSegment; @@ -165,14 +166,47 @@ impl FooterDeserializer { ) }) .transpose()?; + // Validate the optional metadata locator here; values are resolved later, on demand. It + // never widens the required initial read. + let metadata_segment = match &postscript.metadata { + None => None, + Some(segment) => { + let spec = SegmentSpec { + offset: segment.offset, + length: segment.length, + alignment: segment.alignment, + }; + let end = spec + .offset + .checked_add(u64::from(spec.length)) + .ok_or_else(|| vortex_err!("Metadata segment range overflowed u64"))?; + if end > file_size { + vortex_bail!( + "Metadata segment range {}..{} exceeds file size {}", + spec.offset, + end, + file_size + ); + } + let offset = usize::try_from(spec.offset)?; + if !spec.alignment.is_offset_aligned(offset) { + vortex_bail!( + "Metadata segment offset {} is not aligned to {}", + spec.offset, + spec.alignment + ); + } + Some(spec) + } + }; Ok(DeserializeStep::Done(self.parse_footer( initial_offset, &self.buffer, - &postscript.footer, - &postscript.layout, + postscript, dtype, file_stats, + metadata_segment, )?)) } @@ -257,22 +291,31 @@ impl FooterDeserializer { &self, initial_offset: u64, initial_read: &[u8], - footer_segment: &PostscriptSegment, - layout_segment: &PostscriptSegment, + postscript: &Postscript, dtype: DType, file_stats: Option, + metadata_segment: Option, ) -> VortexResult