storage: preserve metadata for implicit dirs

Author: ProAdubois

storage/drivers/driver.go

@@ -269,6 +269,9 @@ type DriverWithDifferOutput struct {
 	TOCDigest digest.Digest
 	// RootDirMode is the mode of the root directory of the layer, if specified.
 	RootDirMode *os.FileMode
+	// ImplicitDirs lists directories that had to be synthesized while staging
+	// the layer contents and were never explicitly described by the layer.
+	ImplicitDirs []string
 	// Artifacts is a collection of additional artifacts
 	// generated by the differ that the storage driver can use.
 	Artifacts map[string]any

storage/drivers/overlay/overlay.go

@@ -27,6 +27,7 @@ import (
 	graphdriver "go.podman.io/storage/drivers"
 	"go.podman.io/storage/drivers/overlayutils"
 	"go.podman.io/storage/drivers/quota"
+	"go.podman.io/storage/drivers/unionbackfill"
 	"go.podman.io/storage/internal/dedup"
 	"go.podman.io/storage/internal/driver"
 	"go.podman.io/storage/internal/staging_lockfile"
@@ -40,6 +41,7 @@ import (
 	"go.podman.io/storage/pkg/idtools"
 	"go.podman.io/storage/pkg/mount"
 	"go.podman.io/storage/pkg/system"
+	"go.podman.io/storage/pkg/tarbackfill"
 	"go.podman.io/storage/pkg/unshare"
 	"golang.org/x/sys/unix"
 )
@@ -2392,6 +2394,51 @@ func (d *Driver) ApplyDiffFromStagingDirectory(id, parent string, diffOutput *gr
 			return err
 		}
 	}
+	if d.options.forceMask == nil && parent != "" && len(diffOutput.ImplicitDirs) > 0 {
+		lowerDiffDirs, err := d.getLowerDiffPaths(id)
+		if err != nil {
+			return err
+		}
+		if len(lowerDiffDirs) > 0 {
+			backfiller := unionbackfill.NewBackfiller(options.Mappings, lowerDiffDirs)
+			for _, implicitDir := range diffOutput.ImplicitDirs {
+				hdr, err := backfiller.Backfill(implicitDir)
+				if err != nil {
+					return err
+				}
+				if hdr == nil {
+					continue
+				}
+
+				path := filepath.Join(stagingDirectory, implicitDir)
+				idPair := idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}
+				if options.Mappings != nil {
+					if mapped, err := options.Mappings.ToHost(idPair); err == nil {
+						idPair = mapped
+					}
+				}
+				if err := os.Chown(path, idPair.UID, idPair.GID); err != nil {
+					return err
+				}
+				for xattr, xval := range hdr.Xattrs {
+					if err := system.Lsetxattr(path, xattr, []byte(xval), 0); err != nil {
+						return err
+					}
+				}
+				if err := os.Chmod(path, os.FileMode(hdr.Mode)&os.ModePerm); err != nil {
+					return err
+				}
+				atime := hdr.AccessTime
+				mtime := hdr.ModTime
+				if atime.IsZero() {
+					atime = mtime
+				}
+				if err := os.Chtimes(path, atime, mtime); err != nil {
+					return err
+				}
+			}
+		}
+	}
 
 	if d.usingComposefs {
 		toc := diffOutput.Artifacts[tocArtifact]
@@ -2498,8 +2545,21 @@ func (d *Driver) applyDiff(target string, options graphdriver.ApplyDiffOpts) (si
 	}
 
 	logrus.Debugf("Applying tar in %s", target)
-	// Overlay doesn't need the parent id to apply the diff
-	if err := untar(options.Diff, target, &archive.TarOptions{
+	diff := options.Diff
+	if id := filepath.Base(filepath.Dir(target)); id != "" {
+		lowerDiffDirs, err := d.getLowerDiffPaths(id)
+		if err != nil {
+			return 0, err
+		}
+		if len(lowerDiffDirs) > 0 {
+			backfiller := unionbackfill.NewBackfiller(idMappings, lowerDiffDirs)
+			rc := tarbackfill.NewIOReaderWithBackfiller(diff, backfiller)
+			defer rc.Close()
+			diff = rc
+		}
+	}
+
+	if err := untar(diff, target, &archive.TarOptions{
 		UIDMaps:           idMappings.UIDs(),
 		GIDMaps:           idMappings.GIDs(),
 		IgnoreChownErrors: d.options.ignoreChownErrors,

storage/drivers/unionbackfill/backfill.go

@@ -0,0 +1,102 @@
+package unionbackfill
+
+import (
+	"archive/tar"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+
+	"go.podman.io/storage/pkg/archive"
+	"go.podman.io/storage/pkg/idtools"
+	"go.podman.io/storage/pkg/system"
+)
+
+// NewBackfiller returns a helper that can synthesize metadata for directories
+// which exist only in lower layers.
+func NewBackfiller(idmap *idtools.IDMappings, lowerDiffDirs []string) *backfiller {
+	if idmap != nil {
+		uidMaps, gidMaps := idmap.UIDs(), idmap.GIDs()
+		if len(uidMaps) > 0 || len(gidMaps) > 0 {
+			idmap = idtools.NewIDMappingsFromMaps(append([]idtools.IDMap{}, uidMaps...), append([]idtools.IDMap{}, gidMaps...))
+		}
+	}
+	return &backfiller{
+		idmap:         idmap,
+		lowerDiffDirs: append([]string{}, lowerDiffDirs...),
+	}
+}
+
+type backfiller struct {
+	idmap         *idtools.IDMappings
+	lowerDiffDirs []string
+}
+
+// Backfill returns a tar header for pathname if it exists in a lower layer.
+func (b *backfiller) Backfill(pathname string) (*tar.Header, error) {
+	for _, lowerDiffDir := range b.lowerDiffDirs {
+		candidate := filepath.Join(lowerDiffDir, pathname)
+		if st, err := os.Lstat(candidate); err == nil {
+			var linkTarget string
+			if st.Mode()&fs.ModeType == fs.ModeSymlink {
+				target, err := os.Readlink(candidate)
+				if err != nil {
+					return nil, err
+				}
+				linkTarget = target
+			}
+			hdr, err := tar.FileInfoHeader(st, linkTarget)
+			if err != nil {
+				return nil, err
+			}
+			hdr.Name = strings.Trim(filepath.ToSlash(pathname), "/")
+			if st.Mode()&fs.ModeType == fs.ModeDir {
+				hdr.Name += "/"
+			}
+			if b.idmap != nil && !b.idmap.Empty() {
+				if uid, gid, err := b.idmap.ToContainer(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}); err == nil {
+					hdr.Uid, hdr.Gid = uid, gid
+				}
+			}
+			return hdr, nil
+		}
+
+		// If this path is hidden by an opaque directory in this lower, stop here.
+		p := strings.Trim(pathname, "/")
+		subpathname := ""
+		for {
+			dir, subdir := filepath.Split(p)
+			dir = strings.Trim(dir, "/")
+			if dir == p {
+				break
+			}
+
+			xval, err := system.Lgetxattr(filepath.Join(lowerDiffDir, dir), archive.GetOverlayXattrName("opaque"))
+			if err == nil && len(xval) == 1 && xval[0] == 'y' {
+				return nil, nil
+			}
+			if _, err := os.Stat(filepath.Join(lowerDiffDir, dir, archive.WhiteoutOpaqueDir)); err == nil {
+				return nil, nil
+			}
+
+			subpathname = strings.Trim(path.Join(subdir, subpathname), "/")
+			xval, err = system.Lgetxattr(filepath.Join(lowerDiffDir, dir), archive.GetOverlayXattrName("redirect"))
+			if err == nil && len(xval) > 0 {
+				subdir := string(xval)
+				if path.IsAbs(subdir) {
+					pathname = path.Join(subdir, subpathname)
+				} else {
+					parent, _ := filepath.Split(dir)
+					parent = strings.Trim(parent, "/")
+					pathname = path.Join(parent, subdir, subpathname)
+				}
+				break
+			}
+
+			p = dir
+		}
+	}
+
+	return nil, nil
+}