Skip to content

Commit aa6257e

Browse files
authored
fix: make strict replay sandbox survive docker-compose service starts (#228)
1 parent 5799643 commit aa6257e

13 files changed

Lines changed: 449 additions & 202 deletions

go.mod

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ module github.com/Use-Tusk/tusk-cli
33
go 1.25.0
44

55
require (
6-
github.com/Use-Tusk/fence v0.1.36
6+
github.com/Use-Tusk/fence v0.1.51
77
github.com/Use-Tusk/tusk-drift-schemas v0.1.36
88
github.com/agnivade/levenshtein v1.0.3
99
github.com/aymanbagabas/go-osc52/v2 v2.0.1
@@ -28,7 +28,7 @@ require (
2828
github.com/stretchr/testify v1.11.1
2929
github.com/zricethezav/gitleaks/v8 v8.30.1
3030
golang.org/x/mod v0.29.0
31-
golang.org/x/term v0.41.0
31+
golang.org/x/term v0.42.0
3232
google.golang.org/protobuf v1.36.9
3333
gopkg.in/yaml.v3 v3.0.1
3434
mvdan.cc/sh/v3 v3.12.0
@@ -119,7 +119,7 @@ require (
119119
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa // indirect
120120
golang.org/x/net v0.47.0 // indirect
121121
golang.org/x/sync v0.18.0 // indirect
122-
golang.org/x/sys v0.42.0 // indirect
122+
golang.org/x/sys v0.43.0 // indirect
123123
golang.org/x/text v0.31.0 // indirect
124124
gopkg.in/ini.v1 v1.67.0 // indirect
125125
)

go.sum

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,8 @@ github.com/Masterminds/sprig/v3 v3.3.0 h1:mQh0Yrg1XPo6vjYXgtf5OtijNAKJRNcTdOOGZe
3131
github.com/Masterminds/sprig/v3 v3.3.0/go.mod h1:Zy1iXRYNqNLUolqCpL4uhk6SHUMAOSCzdgBfDb35Lz0=
3232
github.com/STARRY-S/zip v0.2.3 h1:luE4dMvRPDOWQdeDdUxUoZkzUIpTccdKdhHHsQJ1fm4=
3333
github.com/STARRY-S/zip v0.2.3/go.mod h1:lqJ9JdeRipyOQJrYSOtpNAiaesFO6zVDsE8GIGFaoSk=
34-
github.com/Use-Tusk/fence v0.1.36 h1:8S15y8cp3X+xXukx6AN0Ky/aX9/dZyW3fLw5XOQ8YtE=
35-
github.com/Use-Tusk/fence v0.1.36/go.mod h1:YkowBDzXioVKJE16vg9z3gSVC6vhzkIZZw2dFf7MW/o=
34+
github.com/Use-Tusk/fence v0.1.51 h1:GGr4bx/eFYYA3WNNIIE7RAkJJu5zlW6nsTdrAqEzTQc=
35+
github.com/Use-Tusk/fence v0.1.51/go.mod h1:ADX3cEerqZumoA+RXDtLC1p+8vUqcNaaaXEK33vHnVs=
3636
github.com/Use-Tusk/tusk-drift-schemas v0.1.36 h1:baojaWiEFEdRU61CLYAbFievXxDLlWTFW/ijL4IpdiE=
3737
github.com/Use-Tusk/tusk-drift-schemas v0.1.36/go.mod h1:pa3EvTj9kKxl9f904RVFkj9YK1zB75QogboKi70zalM=
3838
github.com/agnivade/levenshtein v1.0.3 h1:M5ZnqLOoZR8ygVq0FfkXsNOKzMCk0xRiow0R5+5VkQ0=
@@ -437,13 +437,13 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
437437
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
438438
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
439439
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
440-
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
441-
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
440+
golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
441+
golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
442442
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
443443
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
444444
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
445-
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
446-
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
445+
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
446+
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
447447
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
448448
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
449449
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

internal/runner/compose_replay.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,12 @@ func createReplayComposeOverrideFile(envVars map[string]string, groupName string
6767
if safeGroup == "" {
6868
safeGroup = "default"
6969
}
70+
// The override file lives in the OS temp dir (/tmp on Linux). Fence
71+
// tmpfs-overmounts /tmp inside its Linux sandbox, so a naive `docker
72+
// compose -f /tmp/...` inside the sandbox can't see this file. Callers
73+
// that pass this path into a sandboxed command must register it via
74+
// fence.Manager.ExposeHostPath before launching the sandbox — see
75+
// StartService in service.go, which does this automatically.
7076
tempFile, err := os.CreateTemp("", fmt.Sprintf("tusk-replay-env-override-%s-*.yml", safeGroup))
7177
if err != nil {
7278
return "", fmt.Errorf("failed to create temporary replay compose override file: %w", err)

internal/runner/executor.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515
"sync"
1616
"time"
1717

18-
"github.com/Use-Tusk/fence/pkg/fence"
1918
"github.com/Use-Tusk/tusk-cli/internal/config"
2019
"github.com/Use-Tusk/tusk-cli/internal/log"
2120
"github.com/Use-Tusk/tusk-cli/internal/utils"
@@ -87,7 +86,7 @@ type Executor struct {
8786
sandboxMode string
8887
lastServiceSandboxed bool
8988
debug bool
90-
fenceManager *fence.Manager
89+
sandbox sandboxManager
9190
requireInboundReplay bool
9291
replayComposeOverride string
9392
replayEnvVars map[string]string
@@ -142,7 +141,7 @@ func (e *Executor) GetEffectiveSandboxMode() string {
142141
if e.sandboxMode != "" {
143142
return e.sandboxMode
144143
}
145-
if fence.IsSupported() {
144+
if isSandboxSupported() {
146145
return SandboxModeStrict
147146
}
148147
return SandboxModeAuto

internal/runner/sandbox.go

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
package runner
2+
3+
// Platform-split sandbox adapter. Real implementation lives in sandbox_unix.go
4+
// (fence-backed); Windows gets a no-op stub in sandbox_windows.go because
5+
// fence doesn't cross-compile there.
6+
7+
// sandboxManager wraps whatever sandbox backs replay isolation on the current
8+
// platform. Nil means no sandbox configured.
9+
type sandboxManager interface {
10+
WrapCommand(command string) (string, error)
11+
Cleanup()
12+
}
13+
14+
// sandboxConfigError marks errors that stem from invalid user sandbox config
15+
// (bad JSON, denied localhost, missing file). These are always fatal
16+
// regardless of sandbox mode — a user who supplied a broken config asked for
17+
// sandboxing and shouldn't silently get unisolated execution. Distinct from
18+
// runtime-availability errors (missing bwrap/socat, Initialize failure), which
19+
// auto mode treats as "fall back to no sandbox".
20+
type sandboxConfigError struct{ err error }
21+
22+
func (e *sandboxConfigError) Error() string { return e.err.Error() }
23+
func (e *sandboxConfigError) Unwrap() error { return e.err }
24+
25+
type replaySandboxOptions struct {
26+
UserConfigPath string // optional fence config override (e.g. .tusk/replay.fence.json)
27+
Debug bool
28+
ExposedPort int
29+
// BindsOnHost signals that an external daemon (docker, podman) binds
30+
// ExposedPort outside the sandbox netns; skips the reverse bridge.
31+
BindsOnHost bool
32+
ExposedHostPaths []exposedHostPath
33+
}
34+
35+
type exposedHostPath struct {
36+
Path string
37+
Writable bool
38+
}

internal/runner/sandbox_test.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@ package runner
22

33
import (
44
"testing"
5-
6-
"github.com/Use-Tusk/fence/pkg/fence"
75
)
86

97
// newExecutorForServiceLifecycleTests keeps generic lifecycle tests focused on
@@ -16,7 +14,7 @@ func newExecutorForServiceLifecycleTests() *Executor {
1614

1715
func TestGetEffectiveSandboxMode(t *testing.T) {
1816
e := NewExecutor()
19-
if fence.IsSupported() {
17+
if isSandboxSupported() {
2018
if got := e.GetEffectiveSandboxMode(); got != SandboxModeStrict {
2119
t.Fatalf("expected default sandbox mode %q on supported platform, got %q", SandboxModeStrict, got)
2220
}

internal/runner/sandbox_unix.go

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
//go:build darwin || linux || freebsd
2+
3+
package runner
4+
5+
import (
6+
"fmt"
7+
"strings"
8+
9+
"github.com/Use-Tusk/fence/pkg/fence"
10+
"github.com/Use-Tusk/tusk-cli/internal/utils"
11+
)
12+
13+
// isSandboxSupported reports whether the current platform can actually
14+
// isolate replay service startup (i.e. fence is available).
15+
func isSandboxSupported() bool {
16+
return fence.IsSupported()
17+
}
18+
19+
// fenceSandbox is the Unix-platform implementation of sandboxManager,
20+
// backed by github.com/Use-Tusk/fence.
21+
type fenceSandbox struct {
22+
mgr *fence.Manager
23+
}
24+
25+
// WrapCommand delegates to the underlying fence.Manager.
26+
func (s *fenceSandbox) WrapCommand(command string) (string, error) {
27+
return s.mgr.WrapCommand(command)
28+
}
29+
30+
// Cleanup releases fence's socat bridges, proxies, and temp sockets.
31+
func (s *fenceSandbox) Cleanup() {
32+
if s.mgr != nil {
33+
s.mgr.Cleanup()
34+
}
35+
}
36+
37+
// newReplaySandboxManager builds the effective fence config for replay
38+
// mode, creates the fence.Manager, applies the requested service
39+
// execution model + exposed host paths, and initializes the manager.
40+
// On any error after fence.NewManager succeeds, the manager's Cleanup is
41+
// invoked before returning so no fence-allocated resources leak.
42+
func newReplaySandboxManager(opts replaySandboxOptions) (sandboxManager, error) {
43+
fenceCfg, err := createReplayFenceConfig(opts.UserConfigPath)
44+
if err != nil {
45+
// No prefix here: service.go adds the user-facing
46+
// "failed to prepare replay sandbox config:" framing.
47+
return nil, &sandboxConfigError{err: err}
48+
}
49+
50+
mgr := fence.NewManager(fenceCfg, opts.Debug, false)
51+
// Defensive: Cleanup is idempotent and fence's Initialize already
52+
// unwinds its own partial state on failure, but this guards against
53+
// future fence changes that add allocating steps between NewManager
54+
// and Initialize (or between error returns inside Initialize).
55+
success := false
56+
defer func() {
57+
if !success {
58+
mgr.Cleanup()
59+
}
60+
}()
61+
62+
executionModel := fence.ServiceBindsInSandbox
63+
if opts.BindsOnHost {
64+
executionModel = fence.ServiceBindsOnHost
65+
}
66+
mgr.SetService(fence.ServiceOptions{
67+
ExposedPorts: []int{opts.ExposedPort},
68+
ExecutionModel: executionModel,
69+
})
70+
71+
for _, ehp := range opts.ExposedHostPaths {
72+
if err := mgr.ExposeHostPath(ehp.Path, ehp.Writable); err != nil {
73+
return nil, fmt.Errorf("expose host path %q to sandbox: %w", ehp.Path, err)
74+
}
75+
}
76+
77+
if err := mgr.Initialize(); err != nil {
78+
return nil, fmt.Errorf("initialize replay sandbox: %w", err)
79+
}
80+
81+
success = true
82+
return &fenceSandbox{mgr: mgr}, nil
83+
}
84+
85+
// createReplayFenceConfig creates the effective fence config for replay mode.
86+
// This blocks localhost outbound connections to force the service to use SDK
87+
// mocks.
88+
//
89+
// Exposed (lowercase) for the Unix-only service_test.go tests that verify
90+
// user-config merging behavior. Not part of the package's cross-platform
91+
// surface.
92+
func createReplayFenceConfig(userConfigPath string) (*fence.Config, error) {
93+
cfg := baseReplayFenceConfig()
94+
if userConfigPath == "" {
95+
return cfg, nil
96+
}
97+
98+
resolvedPath := utils.ResolveTuskPath(userConfigPath)
99+
userCfg, err := fence.LoadConfigResolved(resolvedPath)
100+
if err != nil {
101+
return nil, fmt.Errorf("load custom fence config %q: %w", resolvedPath, err)
102+
}
103+
if userCfg == nil {
104+
return nil, fmt.Errorf("custom fence config not found: %s", resolvedPath)
105+
}
106+
if err := validateReplayFenceConfig(userCfg); err != nil {
107+
return nil, err
108+
}
109+
110+
merged := fence.MergeConfigs(cfg, userCfg)
111+
applyReplayFenceInvariants(merged)
112+
return merged, nil
113+
}
114+
115+
func baseReplayFenceConfig() *fence.Config {
116+
f := false
117+
return &fence.Config{
118+
Network: fence.NetworkConfig{
119+
AllowedDomains: []string{
120+
// Allow localhost for the service's own health checks
121+
"localhost",
122+
"127.0.0.1",
123+
},
124+
AllowLocalBinding: true, // Allow service to bind to its port
125+
AllowLocalOutbound: &f, // Block outbound to localhost (Postgres, Redis, etc.)
126+
AllowAllUnixSockets: true, // Allow SDK to connect to mock server via Unix socket
127+
},
128+
Filesystem: fence.FilesystemConfig{
129+
AllowWrite: getAllowedWriteDirs(),
130+
},
131+
}
132+
}
133+
134+
func validateReplayFenceConfig(cfg *fence.Config) error {
135+
if cfg == nil {
136+
return nil
137+
}
138+
139+
requiredDomains := []string{"localhost", "127.0.0.1"}
140+
for _, deniedDomain := range cfg.Network.DeniedDomains {
141+
for _, requiredDomain := range requiredDomains {
142+
if strings.EqualFold(deniedDomain, requiredDomain) {
143+
return fmt.Errorf("custom replay fence config cannot deny %q because replay health checks require it", requiredDomain)
144+
}
145+
}
146+
}
147+
148+
return nil
149+
}
150+
151+
func applyReplayFenceInvariants(cfg *fence.Config) {
152+
if cfg == nil {
153+
return
154+
}
155+
156+
f := false
157+
cfg.Network.AllowedDomains = mergeUniqueStrings(
158+
cfg.Network.AllowedDomains,
159+
[]string{"localhost", "127.0.0.1"},
160+
)
161+
cfg.Network.AllowLocalBinding = true
162+
cfg.Network.AllowLocalOutbound = &f
163+
cfg.Network.AllowAllUnixSockets = true
164+
cfg.Filesystem.AllowWrite = mergeUniqueStrings(cfg.Filesystem.AllowWrite, getAllowedWriteDirs())
165+
}
166+
167+
func mergeUniqueStrings(existing, required []string) []string {
168+
if len(required) == 0 {
169+
return existing
170+
}
171+
172+
seen := make(map[string]struct{}, len(existing)+len(required))
173+
merged := make([]string, 0, len(existing)+len(required))
174+
for _, value := range existing {
175+
if _, ok := seen[value]; ok {
176+
continue
177+
}
178+
seen[value] = struct{}{}
179+
merged = append(merged, value)
180+
}
181+
for _, value := range required {
182+
if _, ok := seen[value]; ok {
183+
continue
184+
}
185+
seen[value] = struct{}{}
186+
merged = append(merged, value)
187+
}
188+
return merged
189+
}
190+
191+
// getAllowedWriteDirs returns the default writable paths for replay mode.
192+
// We allow broad local writes by default. Note that Fence still enforces
193+
// mandatory dangerous-path protections (see
194+
// https://github.com/Use-Tusk/fence/blob/main/internal/sandbox/dangerous.go).
195+
func getAllowedWriteDirs() []string {
196+
return []string{
197+
"/",
198+
}
199+
}

0 commit comments

Comments
 (0)