Expose CF home dir via rw bind mount of container instance on podcvd

Author: 0405ysj

container/src/podcvd/internal/cvd.go

@@ -17,7 +17,7 @@ package internal
 import (
 	"encoding/json"
 	"fmt"
-	"path/filepath"
+	"regexp"
 	"strings"
 
 	"github.com/go-playground/validator/v10"
@@ -70,33 +70,35 @@ func ParseInstanceGroups(jsonStr, groupName string) (*InstanceGroup, error) {
 	return &instanceGroup, nil
 }
 
-func UpdateCvdGroupJsonRaw(data any, containerName, ipAddr string) {
+func UpdateCvdGroupJsonRaw(data any, podcvdHomeDir, ipAddr string) {
 	switch v := data.(type) {
 	case map[string]any:
 		for k, val := range v {
 			if s, ok := val.(string); ok {
-				v[k] = updateStringOnCvdGroupJsonRaw(s, containerName, ipAddr)
+				v[k] = updateStringOnCvdGroupJsonRaw(s, podcvdHomeDir, ipAddr)
 			} else {
-				UpdateCvdGroupJsonRaw(val, containerName, ipAddr)
+				UpdateCvdGroupJsonRaw(val, podcvdHomeDir, ipAddr)
 			}
 		}
 	case []any:
 		for k, val := range v {
 			if s, ok := val.(string); ok {
-				v[k] = updateStringOnCvdGroupJsonRaw(s, containerName, ipAddr)
+				v[k] = updateStringOnCvdGroupJsonRaw(s, podcvdHomeDir, ipAddr)
 			} else {
-				UpdateCvdGroupJsonRaw(val, containerName, ipAddr)
+				UpdateCvdGroupJsonRaw(val, podcvdHomeDir, ipAddr)
 			}
 		}
 	}
 }
 
-func updateStringOnCvdGroupJsonRaw(data, containerName, ipAddr string) string {
+var cvdPathRegex = regexp.MustCompile(`^/var/tmp/cvd/[0-9]+/[0-9]+/home`)
+
+func updateStringOnCvdGroupJsonRaw(data, podcvdHomeDir, ipAddr string) string {
 	data = strings.ReplaceAll(data, "0.0.0.0", ipAddr)
 	data = strings.ReplaceAll(data, "localhost", ipAddr)
 	data = strings.ReplaceAll(data, "127.0.0.1", ipAddr)
-	if filepath.IsAbs(data) {
-		data = fmt.Sprintf("%s:%s", containerName, data)
+	if cvdPathRegex.MatchString(data) {
+		data = cvdPathRegex.ReplaceAllString(data, podcvdHomeDir)
 	}
 	return data
 }

container/src/podcvd/internal/host.go

@@ -264,6 +264,7 @@ func createAndStartContainer(ccm libcfcontainer.CuttlefishContainerManager, comm
 		Env: []string{
 			"ANDROID_HOST_OUT=/host_out",
 			"ANDROID_PRODUCT_OUT=/product_out",
+			"HOME=/podcvd_home",
 		},
 		Image: imageName,
 		Labels: map[string]string{
@@ -294,13 +295,22 @@ func createAndStartContainer(ccm libcfcontainer.CuttlefishContainerManager, comm
 	if productOut == "" {
 		productOut = currentDir
 	}
+	podcvdRootDir := "/var/tmp/podcvd"
+	if err := os.MkdirAll(podcvdRootDir, 0777); err != nil {
+		return "", fmt.Errorf("failed to create podcvd root dir: %w", err)
+	}
+	podcvdHomeDir := filepath.Join(podcvdRootDir, strconv.Itoa(os.Getuid()), attemptID)
+	if err := os.MkdirAll(podcvdHomeDir, 0755); err != nil {
+		return "", fmt.Errorf("failed to create podcvd home dir: %w", err)
+	}
 	pidsLimit := int64(8192)
 	containerHostCfg := &container.HostConfig{
 		Annotations: map[string]string{"run.oci.keep_original_groups": "1"},
 		Binds: []string{
 			fmt.Sprintf("%s:/host_out:O", hostOut),
 			fmt.Sprintf("%s:/product_out:O", productOut),
 			fmt.Sprintf("%s:/root/.local/share/cvd:ro", cvdDataHome),
+			fmt.Sprintf("%s:/podcvd_home:rw", podcvdHomeDir),
 		},
 		CapAdd: []string{"NET_RAW"},
 		Resources: container.Resources{

container/src/podcvd/internal/main.go

@@ -21,6 +21,8 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
+	"strconv"
 	"sync"
 
 	"github.com/google/android-cuttlefish/container/src/libcfcontainer"
@@ -154,7 +156,13 @@ func handleSubcommandsForSingleInstanceGroup(ccm libcfcontainer.CuttlefishContai
 		if !exists {
 			return fmt.Errorf("failed to find IPv4 address for group name %q", cvdArgs.CommonArgs.GroupName)
 		}
-		UpdateCvdGroupJsonRaw(res, ContainerName(cvdArgs.CommonArgs.GroupName), ip)
+		inspectRes, err := ccm.GetClient().ContainerInspect(context.Background(), ContainerName(cvdArgs.CommonArgs.GroupName))
+		if err != nil {
+			return fmt.Errorf("failed to inspect container: %w", err)
+		}
+		attemptID := inspectRes.Config.Labels["attempt_id"]
+		podcvdHomeDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()), attemptID)
+		UpdateCvdGroupJsonRaw(res, podcvdHomeDir, ip)
 		stdout, err := json.MarshalIndent(res, "", "        ")
 		if err != nil {
 			return fmt.Errorf("failed to marshal json: %w", err)
@@ -201,6 +209,10 @@ func clearAllCuttlefishHosts(ccm libcfcontainer.CuttlefishContainerManager) erro
 	for err := range errCh {
 		errs = append(errs, err)
 	}
+	uidDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()))
+	if err := os.RemoveAll(uidDir); err != nil {
+		errs = append(errs, fmt.Errorf("failed to remove uid dir: %w", err))
+	}
 	return errors.Join(errs...)
 }
 
@@ -231,8 +243,15 @@ func fleetAllCuttlefishHosts(ccm libcfcontainer.CuttlefishContainerManager) erro
 				errCh <- err
 				return
 			}
+			inspectRes, err := ccm.GetClient().ContainerInspect(context.Background(), containerName)
+			if err != nil {
+				errCh <- err
+				return
+			}
+			attemptID := inspectRes.Config.Labels["attempt_id"]
+			podcvdHomeDir := filepath.Join("/var/tmp/podcvd", strconv.Itoa(os.Getuid()), attemptID)
 			for idx := range res.Groups {
-				UpdateCvdGroupJsonRaw(res.Groups[idx], containerName, ip)
+				UpdateCvdGroupJsonRaw(res.Groups[idx], podcvdHomeDir, ip)
 			}
 			resCh <- res
 		}(groupName, ip)