feat(shim): choose guest rootfs after inner task create

Run ChooseRootfs in the shim after inner task Create, once the bundle
rootfs is mounted (#684). Persist JSON RootfsParams in config.json as
com.urunc.internal.rootfs.params so Exec reuses the choice; read state
then spec annotations because reexec may not yet mirror config.json.

Export ChooseRootfs from unikontainers. When the annotation is absent,
Exec runs the same selection as the former u.chooseRootfs() (podman and
urunc CLI). Ensure MonRootfs exists before rootfs setup in Exec.

Fixes: #684

Signed-off-by: sidneychang <2190206983@qq.com>

Author: sidneychang

pkg/containerd-shim/guest_rootfs.go

@@ -0,0 +1,68 @@
+// Copyright (c) 2023-2026, Nubificus LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package containerdshim
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"path/filepath"
+
+	taskAPI "github.com/containerd/containerd/api/runtime/task/v2"
+	"github.com/urunc-dev/urunc/pkg/unikontainers"
+)
+
+const annotRootfsParams = "com.urunc.internal.rootfs.params"
+
+var errGuestRootfsChoiceSkipped = errors.New("guest rootfs choice skipped")
+
+// chooseGuestRootfs runs the same ChooseRootfs logic as runtime Exec after inner
+// task Create (#684) and records the result in annotRootfsParams so Exec knows
+// selection already happened.
+func chooseGuestRootfs(r *taskAPI.CreateTaskRequest) error {
+	spec, mode, err := loadSpec(r.Bundle)
+	if err != nil {
+		return err
+	}
+
+	config, err := unikontainers.GetUnikernelConfig(filepath.Clean(r.Bundle), spec)
+	if err != nil {
+		return fmt.Errorf("%w: %w", errGuestRootfsChoiceSkipped, err)
+	}
+
+	annotations := config.Map()
+	uruncCfg, _ := unikontainers.LoadUruncConfig(unikontainers.UruncConfigPath)
+
+	rootfsParams, err := unikontainers.ChooseRootfs(
+		filepath.Clean(r.Bundle),
+		spec.Root.Path,
+		annotations,
+		uruncCfg,
+	)
+	if err != nil {
+		return err
+	}
+
+	encoded, err := json.Marshal(rootfsParams)
+	if err != nil {
+		return err
+	}
+	if spec.Annotations == nil {
+		spec.Annotations = make(map[string]string)
+	}
+	spec.Annotations[annotRootfsParams] = string(encoded)
+
+	return saveSpec(r.Bundle, spec, mode)
+}

pkg/containerd-shim/task_service.go

@@ -16,6 +16,7 @@ package containerdshim
 
 import (
 	"context"
+	"errors"
 
 	taskAPI "github.com/containerd/containerd/api/runtime/task/v2"
 	"github.com/containerd/log"
@@ -49,7 +50,23 @@ func (s *taskService) Create(ctx context.Context, r *taskAPI.CreateTaskRequest)
 		}
 	}
 
-	return s.TaskService.Create(ctx, r)
+	resp, err := s.TaskService.Create(ctx, r)
+	if err != nil {
+		return resp, err
+	}
+
+	// #684: ChooseRootfs after inner task Create so bundle rootfs is mounted;
+	// params are persisted in bundle config.json for runtime Exec.
+	if err := chooseGuestRootfs(r); err != nil {
+		if errors.Is(err, errGuestRootfsChoiceSkipped) {
+			log.G(ctx).WithError(err).Debug("urunc(shim): guest rootfs choice skipped")
+			return resp, nil
+		}
+		log.G(ctx).WithError(err).Warn("urunc(shim): failed to choose guest rootfs")
+		return nil, err
+	}
+
+	return resp, nil
 }
 
 func (s *taskService) Delete(ctx context.Context, r *taskAPI.DeleteRequest) (*taskAPI.DeleteResponse, error) {

pkg/unikontainers/rootfs.go

@@ -28,6 +28,10 @@ import (
 // TODO: Find and set the correct size for the tmpfs in the host
 const tmpfsSizeForNoRootfs = "65536k"
 
+// annotRootfsParams holds JSON RootfsParams after shim chooseGuestRootfs.
+// When present in bundle config.json, Exec reuses it; otherwise Exec runs ChooseRootfs.
+const annotRootfsParams = "com.urunc.internal.rootfs.params"
+
 type rootfsBuilder interface {
 	preSetup() error
 	postSetup() error
@@ -235,15 +239,10 @@ func (rs *rootfsSelector) tryContainerRootfs() (types.RootfsParams, bool) {
 	return types.RootfsParams{}, false
 }
 
-func switchMonRootfs(res types.RootfsParams, bundle string) (types.RootfsParams, error) {
-	monRootfs := filepath.Join(bundle, monitorRootfsDirName)
-	err := os.MkdirAll(monRootfs, 0o755)
-	if err != nil {
-		return types.RootfsParams{}, fmt.Errorf("failed to create monitor rootfs directory %s: %w", monRootfs, err)
-	}
-	res.MonRootfs = monRootfs
-
-	return res, nil
+// switchMonRootfs sets MonRootfs under the bundle; creation happens in Exec.
+func switchMonRootfs(res types.RootfsParams, bundle string) types.RootfsParams {
+	res.MonRootfs = filepath.Join(bundle, monitorRootfsDirName)
+	return res
 }
 
 // pivotRootfs changes rootfs with pivot

pkg/unikontainers/unikontainers.go

@@ -238,33 +238,37 @@ func (u *Unikontainer) SetupNet() (types.NetDevParams, error) {
 //  3. Container rootfs as block device (if MountRootfs=true and supported)
 //  4. Container rootfs as shared-fs: virtiofs > 9pfs (if MountRootfs=true and supported)
 //  5. No rootfs
-func (u *Unikontainer) chooseRootfs() (types.RootfsParams, error) {
-	bundleDir := filepath.Clean(u.State.Bundle)
-	rootfsDir := filepath.Clean(u.Spec.Root.Path)
+func ChooseRootfs(bundle, specRoot string, annot map[string]string, cfg *UruncConfig) (types.RootfsParams, error) {
+	bundleDir := filepath.Clean(bundle)
+	rootfsDir := filepath.Clean(specRoot)
 	rootfsDir, err := resolveAgainstBase(bundleDir, rootfsDir)
 	if err != nil {
 		uniklog.Errorf("could not resolve rootfs directory %s: %v", rootfsDir, err)
 		return types.RootfsParams{}, err
 	}
 
-	unikernelType := u.State.Annotations[annotType]
+	if cfg == nil {
+		return types.RootfsParams{}, fmt.Errorf("urunc config is required for guest rootfs selection")
+	}
+
+	unikernelType := annot[annotType]
 	unikernel, err := unikernels.New(unikernelType)
 	if err != nil {
 		return types.RootfsParams{}, err
 	}
 
-	vmmType := u.State.Annotations[annotHypervisor]
-	vmm, err := hypervisors.NewVMM(hypervisors.VmmType(vmmType), u.UruncCfg.Monitors)
+	vmmType := annot[annotHypervisor]
+	vmm, err := hypervisors.NewVMM(hypervisors.VmmType(vmmType), cfg.Monitors)
 	if err != nil {
 		return types.RootfsParams{}, err
 	}
 
-	virtiofsdConfig := u.UruncCfg.ExtraBins["virtiofsd"]
+	virtiofsdConfig := cfg.ExtraBins["virtiofsd"]
 
 	selector := &rootfsSelector{
 		bundle:     bundleDir,
 		cntrRootfs: rootfsDir,
-		annot:      u.State.Annotations,
+		annot:      annot,
 		unikernel:  unikernel,
 		vmm:        vmm,
 		vfsdPath:   virtiofsdConfig.Path,
@@ -285,7 +289,7 @@ func (u *Unikontainer) chooseRootfs() (types.RootfsParams, error) {
 	// Priority 3 & 4: Container rootfs (block or shared-fs)
 	result, ok = selector.tryContainerRootfs()
 	if ok {
-		return switchMonRootfs(result, bundleDir)
+		return switchMonRootfs(result, bundleDir), nil
 	}
 
 	if selector.shouldMountContainerRootfs() {
@@ -426,11 +430,39 @@ func (u *Unikontainer) Exec(metrics m.Writer) error {
 	// if the respective annotation is set then, depending on the guest
 	// (supports block or 9pfs), it will use the supported option. In case
 	// both ae supported, then the block option will be used by default.
-	rootfsParams, err := u.chooseRootfs()
-	if err != nil {
-		uniklog.Errorf("could not choose guest rootfs: %v", err)
-		return err
+	var rootfsParams types.RootfsParams
+	// Read annotRootfsParams: shim writes it to config.json after inner Create;
+	// state.json may not have it yet during reexec, so fall back to spec.
+	rootfsParamsJSON := ""
+	if u.State.Annotations != nil {
+		rootfsParamsJSON = u.State.Annotations[annotRootfsParams]
+	}
+	if rootfsParamsJSON == "" && u.Spec.Annotations != nil {
+		rootfsParamsJSON = u.Spec.Annotations[annotRootfsParams]
+	}
+	if rootfsParamsJSON != "" {
+		// Shim (or a prior run) already called ChooseRootfs; decode and use as-is.
+		if err := json.Unmarshal([]byte(rootfsParamsJSON), &rootfsParams); err != nil {
+			return fmt.Errorf("could not decode guest rootfs params: %w", err)
+		}
+	} else {
+		// No annotRootfsParams (e.g. podman, urunc CLI): same path as legacy
+		// u.chooseRootfs(), including no-guest-rootfs when mountRootfs is unset.
+		specRoot := ""
+		if u.Spec.Root != nil {
+			specRoot = u.Spec.Root.Path
+		}
+		rootfsParams, err = ChooseRootfs(u.State.Bundle, specRoot, u.State.Annotations, u.UruncCfg)
+		if err != nil {
+			uniklog.Errorf("could not choose guest rootfs: %v", err)
+			return err
+		}
 	}
+	uniklog.WithFields(logrus.Fields{
+		"rootfs_type": rootfsParams.Type,
+		"rootfs_path": rootfsParams.Path,
+		"mon_rootfs":  rootfsParams.MonRootfs,
+	}).Debug("guest rootfs params")
 
 	// TODO: Add support for using both an existing
 	// block based snapshot of the container's rootfs
@@ -479,6 +511,15 @@ func (u *Unikontainer) Exec(metrics m.Writer) error {
 		}
 	}
 
+	if _, err = os.Stat(rootfsParams.MonRootfs); err != nil {
+		if !os.IsNotExist(err) {
+			return fmt.Errorf("failed to stat monitor rootfs directory %s: %w", rootfsParams.MonRootfs, err)
+		}
+		if err = os.MkdirAll(rootfsParams.MonRootfs, 0o755); err != nil {
+			return fmt.Errorf("failed to create monitor rootfs directory %s: %w", rootfsParams.MonRootfs, err)
+		}
+	}
+
 	err = rfsBuilder.preSetup()
 	if err != nil {
 		return fmt.Errorf("pre setup step for rootfs failed: %w", err)