Merge pull request #12785 from dmcgowan/pass-socket-address

Make shim socket directory use configured directory

Author: AkihiroSuda

api/next.txtpb

@@ -69,6 +69,13 @@ message BootstrapParams {
   // Each section can contain arbitrary structured data identified by type URL
   // Examples: CRI config, NRI config, sandbox config, etc.
   repeated Extension extensions = 8;
+
+  // Optional directory for the shim to place its unix socket.
+  // If empty, the shim defaults to a short, well-known path
+  // (e.g., /run/containerd/s). The path must be kept short because
+  // the socket filename is a 64-character SHA256 hash and unix
+  // socket paths are limited to 104-108 bytes depending on platform.
+  optional string socket_dir = 9;
 }
 
 // Extension provides extensibility for new configuration types

api/runtime/bootstrap/v1/bootstrap.pb.go

@@ -40,6 +40,7 @@ import (
 	"github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/process"
 	"github.com/containerd/containerd/v2/cmd/containerd-shim-runc-v2/runc"
 	"github.com/containerd/containerd/v2/core/mount"
+	"github.com/containerd/containerd/v2/defaults"
 	"github.com/containerd/containerd/v2/pkg/namespaces"
 	"github.com/containerd/containerd/v2/pkg/schedcore"
 	"github.com/containerd/containerd/v2/pkg/shim"
@@ -105,6 +106,7 @@ func newCommand(ctx context.Context, id, containerdAddress, containerdTTRPCAddre
 	cmd.Dir = cwd
 	cmd.Env = append(os.Environ(), "GOMAXPROCS=4")
 	cmd.Env = append(cmd.Env, "OTEL_SERVICE_NAME=containerd-shim-"+id)
+
 	cmd.SysProcAttr = &syscall.SysProcAttr{
 		Setpgid: true,
 	}
@@ -145,8 +147,8 @@ func (s *shimSocket) Close() {
 	_ = shim.RemoveSocket(s.addr)
 }
 
-func newShimSocket(ctx context.Context, path, id string, debug bool) (*shimSocket, error) {
-	address, err := shim.SocketAddress(ctx, path, id, debug)
+func newShimSocket(ctx context.Context, root, path, id string, debug bool) (*shimSocket, error) {
+	address, err := shim.CreateSocketAddress(ctx, root, path, id, debug)
 	if err != nil {
 		return nil, err
 	}
@@ -217,7 +219,11 @@ func (manager) Start(ctx context.Context, opts *bootapi.BootstrapParams) (_ *boo
 		}
 	}()
 
-	s, err := newShimSocket(ctx, opts.GetContainerdGrpcAddress(), grouping, false)
+	socketDir := opts.GetSocketDir()
+	if socketDir == "" {
+		socketDir = filepath.Join(defaults.DefaultStateDir, "s")
+	}
+	s, err := newShimSocket(ctx, socketDir, opts.GetContainerdGrpcAddress(), grouping, false)
 	if err != nil {
 		if errdefs.IsAlreadyExists(err) {
 			params.Address = s.addr
@@ -229,7 +235,7 @@ func (manager) Start(ctx context.Context, opts *bootapi.BootstrapParams) (_ *boo
 	cmd.ExtraFiles = append(cmd.ExtraFiles, s.f)
 
 	if debugLog {
-		s, err = newShimSocket(ctx, opts.GetContainerdGrpcAddress(), grouping, true)
+		s, err = newShimSocket(ctx, socketDir, opts.GetContainerdGrpcAddress(), grouping, true)
 		if err != nil {
 			return nil, err
 		}

api/runtime/bootstrap/v1/bootstrap.proto

@@ -93,15 +93,6 @@ func CreateTopLevelDirectories(config *srvconfig.Config) error {
 	if err := sys.MkdirAllWithACL(config.State, 0o711); err != nil {
 		return err
 	}
-	if config.State != defaults.DefaultStateDir {
-		// XXX: socketRoot in pkg/shim is hard-coded to the default state directory.
-		// See https://github.com/containerd/containerd/issues/10502#issuecomment-2249268582 for why it's set up that way.
-		// The default fifo directory in pkg/cio is also configured separately and defaults to the default state directory instead of the configured state directory.
-		// Make sure the default state directory is created with the correct permissions.
-		if err := sys.MkdirAllWithACL(defaults.DefaultStateDir, 0o711); err != nil {
-			return err
-		}
-	}
 
 	if config.TempDir != "" {
 		if err := sys.MkdirAllWithACL(config.TempDir, 0o700); err != nil {

cmd/containerd-shim-runc-v2/manager/manager_linux.go

@@ -39,6 +39,7 @@ type shimBinaryConfig struct {
 	runtime      string
 	address      string
 	ttrpcAddress string
+	socketDir    string
 	env          []string
 }
 
@@ -48,6 +49,7 @@ func shimBinary(bundle *Bundle, config shimBinaryConfig) *binary {
 		runtime:                config.runtime,
 		containerdAddress:      config.address,
 		containerdTTRPCAddress: config.ttrpcAddress,
+		socketDir:              config.socketDir,
 		env:                    config.env,
 	}
 }
@@ -56,6 +58,7 @@ type binary struct {
 	runtime                string
 	containerdAddress      string
 	containerdTTRPCAddress string
+	socketDir              string
 	bundle                 *Bundle
 	env                    []string
 }
@@ -73,6 +76,7 @@ func (b *binary) Start(ctx context.Context, opts *types.Any, onClose func()) (_
 			Env:          b.env,
 			LogLevel:     log.GetLevel(),
 			Action:       "start",
+			SocketDir:    b.socketDir,
 		})
 	if err != nil {
 		return nil, err

cmd/containerd/server/server.go

@@ -162,6 +162,7 @@ func (m *ShimManager) loadShim(ctx context.Context, bundle *Bundle) error {
 			runtime:      runtime,
 			address:      m.containerdAddress,
 			ttrpcAddress: m.containerdTTRPCAddress,
+			socketDir:    m.socketDir,
 			env:          m.env,
 		})
 	// TODO: It seems we can only call loadShim here if it is a sandbox shim?

core/runtime/v2/binary.go

@@ -50,6 +50,13 @@ import (
 type ShimConfig struct {
 	// Env is environment variables added to shim processes
 	Env []string `toml:"env"`
+
+	// SocketDir is the directory to place shim sockets. The path must be
+	// short enough to fit within the platform's unix socket path limit.
+	// Defaults:
+	//  Linux (UID 0):  /run/containerd/s
+	//  Linux (UID >0): /run/$UID/containerd/s or /tmp/containerd-s-$(UID)
+	SocketDir string `toml:"socket_dir"`
 }
 
 func init() {
@@ -78,9 +85,27 @@ func init() {
 			events := ep.(*exchange.Exchange)
 			cs := metadata.NewContainerStore(m.(*metadata.DB))
 			ss := metadata.NewSandboxStore(m.(*metadata.DB))
+
+			// Allow configurable directory
+			if config.SocketDir != "" {
+				if !filepath.IsAbs(config.SocketDir) {
+					return nil, fmt.Errorf("socket_dir must be an absolute path: %q", config.SocketDir)
+				}
+				config.SocketDir = filepath.Clean(config.SocketDir)
+				if len(config.SocketDir) > maxSocketDirLen {
+					return nil, fmt.Errorf("socket_dir length must be no longer than %d characters", maxSocketDirLen)
+				}
+			} else {
+				config.SocketDir = defaultSocketDir()
+				if config.SocketDir == "" {
+					return nil, fmt.Errorf("failed to find a suitable socket directory for shim, please configure one")
+				}
+			}
+
 			return NewShimManager(&ManagerConfig{
 				Address:      ic.Properties[plugins.PropertyGRPCAddress],
 				TTRPCAddress: ic.Properties[plugins.PropertyTTRPCAddress],
+				SocketDir:    config.SocketDir,
 				Events:       events,
 				Store:        cs,
 				ShimEnv:      config.Env,
@@ -124,6 +149,7 @@ type ManagerConfig struct {
 	Events       *exchange.Exchange
 	Address      string
 	TTRPCAddress string
+	SocketDir    string
 	SandboxStore sandbox.Store
 	ShimEnv      []string
 }
@@ -133,6 +159,7 @@ func NewShimManager(config *ManagerConfig) (*ShimManager, error) {
 	m := &ShimManager{
 		containerdAddress:      config.Address,
 		containerdTTRPCAddress: config.TTRPCAddress,
+		socketDir:              config.SocketDir,
 		shims:                  runtime.NewNSMap[ShimInstance](),
 		events:                 config.Events,
 		containers:             config.Store,
@@ -154,6 +181,7 @@ type ShimManager struct {
 	shims                  *runtime.NSMap[ShimInstance]
 	events                 *exchange.Exchange
 	containers             containers.Store
+	socketDir              string
 	// runtimePaths is a cache of `runtime names` -> `resolved fs path`
 	runtimePaths sync.Map
 	sandboxStore sandbox.Store
@@ -289,6 +317,7 @@ func (m *ShimManager) startShim(ctx context.Context, bundle *Bundle, id string,
 		runtime:      runtimePath,
 		address:      m.containerdAddress,
 		ttrpcAddress: m.containerdTTRPCAddress,
+		socketDir:    m.socketDir,
 		env:          m.env,
 	})
 	shim, err := b.Start(ctx, typeurl.MarshalProto(topts), func() {