From db29e2f98ee8d40f9c72952ef7e08b666579b49a Mon Sep 17 00:00:00 2001 From: John Eckersberg Date: Fri, 6 Jun 2025 10:31:00 -0400 Subject: [PATCH 1/3] ostree-ext/tar/export: add unmap_path and rework map_path to share inner fn In preparation for (un)mapping tar stream data back to ostree data so we can look up the correct metadata for a given tar entry --- ostree-ext/src/tar/export.rs | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/ostree-ext/src/tar/export.rs b/ostree-ext/src/tar/export.rs index 13f4f4ac5..bc0487b65 100644 --- a/ostree-ext/src/tar/export.rs +++ b/ostree-ext/src/tar/export.rs @@ -56,14 +56,35 @@ mode=bare-split-xattrs /// System calls are expensive. const BUF_CAPACITY: usize = 131072; -/// Convert /usr/etc back to /etc -fn map_path(p: &Utf8Path) -> std::borrow::Cow { - match p.strip_prefix("./usr/etc") { - Ok(r) => Cow::Owned(Utf8Path::new("./etc").join(r)), +/// Convert `from` to `to` +fn map_path_inner<'p>( + p: &'p Utf8Path, + from: &'_ str, + to: &'_ str, +) -> std::borrow::Cow<'p, Utf8Path> { + match p.strip_prefix(from) { + Ok(r) => { + if r.components().count() > 0 { + Cow::Owned(Utf8Path::new(to).join(r)) + } else { + Cow::Owned(Utf8PathBuf::from(to)) + } + } _ => Cow::Borrowed(p), } } +/// Convert /usr/etc back to /etc +fn map_path(p: &Utf8Path) -> std::borrow::Cow { + map_path_inner(p, "./usr/etc", "./etc") +} + +/// Convert etc to usr/etc +/// Note: no leading '/' or './' +fn unmap_path(p: &Utf8Path) -> std::borrow::Cow { + map_path_inner(p, "etc", "usr/etc") +} + /// Convert usr/etc back to etc for the tar stream. fn map_path_v1(p: &Utf8Path) -> &Utf8Path { debug_assert!(!p.starts_with("/") && !p.starts_with(".")); From 1f0d118755976c0b9dce1dff589bdf83b6dbc436 Mon Sep 17 00:00:00 2001 From: John Eckersberg Date: Fri, 6 Jun 2025 10:42:36 -0400 Subject: [PATCH 2/3] ostree-ext/tar/export: add test_unmap_path and update test_map_path to use `as_os_str()` We want to compare the actual os_str here, because `Eq` on Paths canonicalizes and removes any trailing '/', but ostree makes a distinction between trailing '/' or not. For example: ``` let p1 = Path::new("/foo"); let p2 = Path::new("/foo/"); assert_eq!(p1, p2); println!("{:?} {:?}", p1.as_os_str(), p2.as_os_str()); ``` Will print: "/foo" "/foo/" Signed-off-by: John Eckersberg --- ostree-ext/src/tar/export.rs | 48 ++++++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/ostree-ext/src/tar/export.rs b/ostree-ext/src/tar/export.rs index bc0487b65..a5aa4bff0 100644 --- a/ostree-ext/src/tar/export.rs +++ b/ostree-ext/src/tar/export.rs @@ -820,19 +820,53 @@ mod tests { #[test] fn test_map_path() { - assert_eq!(map_path("/".into()), Utf8Path::new("/")); assert_eq!( - map_path("./usr/etc/blah".into()), - Utf8Path::new("./etc/blah") + map_path("/".into()).as_os_str(), + Utf8Path::new("/").as_os_str() + ); + assert_eq!( + map_path("./usr/etc/blah".into()).as_os_str(), + Utf8Path::new("./etc/blah").as_os_str() ); for unchanged in ["boot", "usr/bin", "usr/lib/foo"].iter().map(Utf8Path::new) { - assert_eq!(unchanged, map_path_v1(unchanged)); + assert_eq!(unchanged.as_os_str(), map_path_v1(unchanged).as_os_str()); } - assert_eq!(Utf8Path::new("etc"), map_path_v1(Utf8Path::new("usr/etc"))); assert_eq!( - Utf8Path::new("etc/foo"), - map_path_v1(Utf8Path::new("usr/etc/foo")) + Utf8Path::new("etc").as_os_str(), + map_path_v1(Utf8Path::new("usr/etc")).as_os_str() + ); + assert_eq!( + Utf8Path::new("etc/foo").as_os_str(), + map_path_v1(Utf8Path::new("usr/etc/foo")).as_os_str() + ); + } + + #[test] + fn test_unmap_path() { + assert_eq!( + unmap_path("/".into()).as_os_str(), + Utf8Path::new("/").as_os_str() + ); + assert_eq!( + unmap_path("/etc".into()).as_os_str(), + Utf8Path::new("/etc").as_os_str() + ); + assert_eq!( + unmap_path("/usr/etc".into()).as_os_str(), + Utf8Path::new("/usr/etc").as_os_str() + ); + assert_eq!( + unmap_path("usr/etc".into()).as_os_str(), + Utf8Path::new("usr/etc").as_os_str() + ); + assert_eq!( + unmap_path("etc".into()).as_os_str(), + Utf8Path::new("usr/etc").as_os_str() + ); + assert_eq!( + unmap_path("etc/blah".into()).as_os_str(), + Utf8Path::new("usr/etc/blah").as_os_str() ); } From a448355326bb51c5d047f225c7cf4aaa3678d88b Mon Sep 17 00:00:00 2001 From: John Eckersberg Date: Fri, 6 Jun 2025 16:19:50 -0400 Subject: [PATCH 3/3] ostree-ext: Add `tar_create_parent_dirs` to `container::ExportOpts` Part of https://github.com/coreos/rpm-ostree/issues/5416 Default to previous behavior and do *not* create parent directories, however for reproducible builds we need to ensure we create them with consistent metadata. See also https://github.com/containers/composefs-rs/issues/132 Signed-off-by: John Eckersberg --- ostree-ext/src/container/encapsulate.rs | 20 +++++++-- ostree-ext/src/tar/export.rs | 57 ++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/ostree-ext/src/container/encapsulate.rs b/ostree-ext/src/container/encapsulate.rs index 21e814248..39a0ee030 100644 --- a/ostree-ext/src/container/encapsulate.rs +++ b/ostree-ext/src/container/encapsulate.rs @@ -93,8 +93,14 @@ fn export_chunks( .enumerate() .map(|(i, chunk)| -> Result<_> { let mut w = ociw.create_layer(Some(opts.compression()))?; - ostree_tar::export_chunk(repo, commit, chunk.content, &mut w) - .with_context(|| format!("Exporting chunk {i}"))?; + ostree_tar::export_chunk( + repo, + commit, + chunk.content, + &mut w, + opts.tar_create_parent_dirs, + ) + .with_context(|| format!("Exporting chunk {i}"))?; let w = w.into_inner()?; Ok((w.complete()?, chunk.name, chunk.packages)) }) @@ -120,7 +126,13 @@ pub(crate) fn export_chunked( // In V1, the ostree layer comes first let mut w = ociw.create_layer(compression)?; - ostree_tar::export_final_chunk(repo, commit, chunking.remainder, &mut w)?; + ostree_tar::export_final_chunk( + repo, + commit, + chunking.remainder, + &mut w, + opts.tar_create_parent_dirs, + )?; let w = w.into_inner()?; let ostree_layer = w.complete()?; @@ -418,6 +430,8 @@ pub struct ExportOpts<'m, 'o> { pub contentmeta: Option<&'o ObjectMetaSized>, /// Sets the created tag in the image manifest. pub created: Option, + /// Whether to explicitly create all parent directories in the tar layers. + pub tar_create_parent_dirs: bool, } impl ExportOpts<'_, '_> { diff --git a/ostree-ext/src/tar/export.rs b/ostree-ext/src/tar/export.rs index a5aa4bff0..744c42188 100644 --- a/ostree-ext/src/tar/export.rs +++ b/ostree-ext/src/tar/export.rs @@ -636,6 +636,52 @@ impl<'a, W: std::io::Write> OstreeTarWriter<'a, W> { .append_data(&mut header, "var/tmp", std::io::empty())?; Ok(()) } + + fn write_parents_of( + &mut self, + path: &Utf8Path, + cache: &mut HashSet, + ) -> Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; + + if parent.components().count() == 0 { + return Ok(()); + } + + if cache.contains(parent) { + return Ok(()); + } + + self.write_parents_of(parent, cache)?; + + let inserted = cache.insert(parent.to_owned()); + debug_assert!(inserted); + + let root = self + .repo + .read_commit(&self.commit_checksum, gio::Cancellable::NONE)? + .0; + let parent_file = root.resolve_relative_path(unmap_path(parent).as_ref()); + let queryattrs = "unix::*"; + let queryflags = gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS; + let stat = parent_file.query_info(&queryattrs, queryflags, gio::Cancellable::NONE)?; + let uid = stat.attribute_uint32(gio::FILE_ATTRIBUTE_UNIX_UID); + let gid = stat.attribute_uint32(gio::FILE_ATTRIBUTE_UNIX_GID); + let orig_mode = stat.attribute_uint32(gio::FILE_ATTRIBUTE_UNIX_MODE); + let mode = self.filter_mode(orig_mode); + + let mut header = tar::Header::new_gnu(); + header.set_entry_type(tar::EntryType::Directory); + header.set_size(0); + header.set_uid(uid as u64); + header.set_gid(gid as u64); + header.set_mode(mode); + self.out + .append_data(&mut header, parent, std::io::empty())?; + Ok(()) + } } /// Recursively walk an OSTree commit and generate data into a `[tar::Builder]` @@ -684,12 +730,17 @@ fn path_for_tar_v1(p: &Utf8Path) -> &Utf8Path { fn write_chunk( writer: &mut OstreeTarWriter, chunk: chunking::ChunkMapping, + create_parent_dirs: bool, ) -> Result<()> { + let mut cache = std::collections::HashSet::new(); for (checksum, (_size, paths)) in chunk.into_iter() { let (objpath, h) = writer.append_content(checksum.borrow())?; for path in paths.iter() { let path = path_for_tar_v1(path); let h = h.clone(); + if create_parent_dirs { + writer.write_parents_of(&path, &mut cache)?; + } writer.append_content_hardlink(&objpath, h, path)?; } } @@ -702,13 +753,14 @@ pub(crate) fn export_chunk( commit: &str, chunk: chunking::ChunkMapping, out: &mut tar::Builder, + create_parent_dirs: bool, ) -> Result<()> { // For chunking, we default to format version 1 #[allow(clippy::needless_update)] let opts = ExportOptions; let writer = &mut OstreeTarWriter::new(repo, commit, out, opts)?; writer.write_repo_structure()?; - write_chunk(writer, chunk) + write_chunk(writer, chunk, create_parent_dirs) } /// Output the last chunk in a chunking. @@ -718,6 +770,7 @@ pub(crate) fn export_final_chunk( commit_checksum: &str, remainder: chunking::Chunk, out: &mut tar::Builder, + create_parent_dirs: bool, ) -> Result<()> { let options = ExportOptions; let writer = &mut OstreeTarWriter::new(repo, commit_checksum, out, options)?; @@ -726,7 +779,7 @@ pub(crate) fn export_final_chunk( writer.structure_only = true; writer.write_commit()?; writer.structure_only = false; - write_chunk(writer, remainder.content) + write_chunk(writer, remainder.content, create_parent_dirs) } /// Process an exported tar stream, and update the detached metadata.