Skip to content

Commit e1d0d84

Browse files
authored
fix(tools): wait for pipe readers before reaping detached bash shells (#716)
## Summary Detaching a streaming bash command to the background — chat `ctrl+b`, or directexec `!cmd &` — could truncate its trailing output. The pipe-reader goroutines drain the command's `stdout`/`stderr` into the shared shell buffer, but the live `*exec.Cmd` was handed to the `jobs.Supervisor`, whose `shellJob.Run` called `cmd.Wait()` in a separate goroutine with no coordination. Per `os/exec`, `Wait` closes the pipes on process exit, so output still buffered in the pipe at exit was lost. The `time.Sleep(20ms)` band-aid ran at detach time, unrelated to exit time, so it never addressed the race. ## Fix `shellJob.Run` now waits for the pipe readers to reach EOF before `cmd.Wait()`, mirroring the already-correct foreground path (`wg.Wait()` then `cmd.Wait()`). The wait `select`s on `ctx.Done()` so `WindStop`/`Stop()` can't wedge if a grandchild holds the pipe open (a detached shell has no timeout). Both detach callers thread a readers-done channel through `DetachToBackground`. - `internal/domain/shell.go` — `ReadersDone` on `BackgroundShell`; widened `DetachToBackground` - `internal/services/background_shell_service.go` — store `readersDone` on the shell - `internal/services/jobs/shell_job.go` — cancellable readers-done wait before `cmd.Wait()` - `internal/agent/tools/bash.go`, `internal/services/directexec/bash.go` — pass the signal; drop the 20ms sleep - regenerated `fake_background_shell_service.go` ## Testing - Two regression tests in `shell_job_test.go`: ordering (Run parks until the readers drain, even after the process has exited) and the cancellable escape (`WindStop` still reaps when the readers never signal). Confirmed the latter deadlocks without the `ctx.Done()` case. - `task build`, `go test -race` on the affected packages, full `task test`, and `task lint` (0 issues) — all clean. Closes #696
1 parent 990803b commit e1d0d84

7 files changed

Lines changed: 102 additions & 15 deletions

File tree

internal/agent/tools/bash.go

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -292,9 +292,7 @@ func (t *BashTool) executeBashWithStreaming(ctx context.Context, cmd *exec.Cmd,
292292
detached = true
293293
detachedMux.Unlock()
294294

295-
time.Sleep(20 * time.Millisecond)
296-
297-
shellID, err := t.backgroundShellService.DetachToBackground(ctx, cmd, result.Command, outputBuffer)
295+
shellID, err := t.backgroundShellService.DetachToBackground(ctx, cmd, result.Command, outputBuffer, done)
298296
if err != nil {
299297
logger.Debug("bash: DetachToBackground failed", "error", err)
300298
result.ExitCode = -1

internal/domain/shell.go

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ type BackgroundShell struct {
3838
OutputBuffer OutputRingBuffer
3939
CancelFunc context.CancelFunc
4040
ReadOffset int64
41+
42+
ReadersDone <-chan struct{}
4143
}
4244

4345
// OutputRingBuffer defines the interface for the circular output buffer.
@@ -110,8 +112,10 @@ func NewShellInfo(shell *BackgroundShell) *ShellInfo {
110112

111113
// BackgroundShellService defines the interface for managing background shells
112114
type BackgroundShellService interface {
113-
// DetachToBackground moves a running command to background
114-
DetachToBackground(ctx context.Context, cmd *exec.Cmd, command string, outputBuffer OutputRingBuffer) (string, error)
115+
// DetachToBackground moves a running command to background. readersDone, when
116+
// non-nil, is closed once the caller's pipe readers reach EOF; the supervisor
117+
// waits on it before reaping so trailing output is not truncated.
118+
DetachToBackground(ctx context.Context, cmd *exec.Cmd, command string, outputBuffer OutputRingBuffer, readersDone <-chan struct{}) (string, error)
115119

116120
// GetShellOutput retrieves output from a shell
117121
GetShellOutput(shellID string, fromOffset int64) (string, int64, ShellState, error)

internal/services/background_shell_service.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ func (s *BackgroundShellService) DetachToBackground(
5151
cmd *exec.Cmd,
5252
command string,
5353
outputBuffer domain.OutputRingBuffer,
54+
readersDone <-chan struct{},
5455
) (string, error) {
5556
if !s.config.Tools.Bash.BackgroundShells.Enabled {
5657
return "", fmt.Errorf("background shells are disabled in configuration")
@@ -66,6 +67,7 @@ func (s *BackgroundShellService) DetachToBackground(
6667
State: domain.ShellStateRunning,
6768
OutputBuffer: outputBuffer,
6869
ReadOffset: 0,
70+
ReadersDone: readersDone,
6971
}
7072

7173
if err := s.shellTracker.Add(shell); err != nil {

internal/services/directexec/bash.go

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -367,18 +367,25 @@ func (s *Service) executeBashCommandInBackground(commandText, command string) te
367367
}
368368
}()
369369

370+
done := make(chan struct{})
371+
go func() {
372+
wg.Wait()
373+
close(done)
374+
}()
375+
370376
shellID, err := s.backgroundShellService.DetachToBackground(
371377
ctx,
372378
cmd,
373379
command,
374380
outputBuffer,
381+
done,
375382
)
376383

377384
if err != nil {
378385
return
379386
}
380387

381-
wg.Wait()
388+
<-done
382389

383390
assistantEntry := domain.ConversationEntry{
384391
Message: sdk.Message{

internal/services/jobs/shell_job.go

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,17 @@ func (j *shellJob) Meta() domain.JobMeta {
4040
}
4141

4242
// Run waits for the shell process to exit and records the outcome on the shell.
43+
// It first waits for the pipe readers to drain (ReadersDone) so Cmd.Wait doesn't
44+
// close the pipes mid-read and lose output; the wait honours ctx so a kill or
45+
// shutdown can't wedge when a grandchild is holding the pipe open.
4346
func (j *shellJob) Run(ctx context.Context, _ func(domain.JobSignal)) domain.ToolExecutionResult {
47+
if j.shell.ReadersDone != nil {
48+
select {
49+
case <-j.shell.ReadersDone:
50+
case <-ctx.Done():
51+
}
52+
}
53+
4454
waitErr := j.shell.Cmd.Wait()
4555

4656
now := time.Now()
@@ -49,7 +59,6 @@ func (j *shellJob) Run(ctx context.Context, _ func(domain.JobSignal)) domain.Too
4959

5060
switch {
5161
case ctx.Err() != nil:
52-
// Terminated by Wind(WindStop) / supervisor shutdown rather than finishing.
5362
code := -1
5463
j.shell.ExitCode = &code
5564
j.shell.State = domain.ShellStateCancelled

internal/services/jobs/shell_job_test.go

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,71 @@ func waitTerminal(t *testing.T, sup *Supervisor, id string) domain.JobStatus {
4545
return ""
4646
}
4747

48+
// isTerminal reports whether the job is currently in a terminal state without
49+
// blocking, for asserting that a job has NOT yet finished.
50+
func isTerminal(sup *Supervisor, id string) bool {
51+
for _, j := range sup.Snapshot() {
52+
if j.Meta.ID == id {
53+
return j.Status.IsTerminal()
54+
}
55+
}
56+
return false
57+
}
58+
59+
// TestShellJob_WaitsForReadersDoneBeforeReaping asserts Run does not call
60+
// Cmd.Wait until the pipe readers have drained (ReadersDone closed), even after
61+
// the process has already exited — otherwise Wait closes the pipes mid-drain and
62+
// truncates trailing output.
63+
func TestShellJob_WaitsForReadersDoneBeforeReaping(t *testing.T) {
64+
tracker := utils.NewShellTracker(10)
65+
sup := NewSupervisor(&domainmocks.FakeMessageQueue{}, &domainmocks.FakeConversationRepository{}, nil)
66+
defer sup.Stop()
67+
68+
// `true` exits immediately; the job must still park on ReadersDone.
69+
shell := startShell(t, "s-drain", "true")
70+
readersDone := make(chan struct{})
71+
shell.ReadersDone = readersDone
72+
_ = tracker.Add(shell)
73+
sup.Submit(NewShellJob(shell, tracker))
74+
75+
time.Sleep(50 * time.Millisecond)
76+
if isTerminal(sup, "s-drain") {
77+
t.Fatal("shell reaped before the pipe readers signalled done")
78+
}
79+
80+
close(readersDone)
81+
if got := waitTerminal(t, sup, "s-drain"); got != domain.JobCompleted {
82+
t.Fatalf("status = %s, want completed", got)
83+
}
84+
if shell.State != domain.ShellStateCompleted {
85+
t.Fatalf("shell state = %s, want completed", shell.State)
86+
}
87+
}
88+
89+
// TestShellJob_WindStopUnblocksReadersWait guards the cancellable-wait escape:
90+
// when ReadersDone never closes (e.g. a grandchild holds the pipe open),
91+
// Wind(WindStop) must still reap the job via ctx cancellation rather than
92+
// wedging Run — which would also hang Supervisor.Stop().
93+
func TestShellJob_WindStopUnblocksReadersWait(t *testing.T) {
94+
tracker := utils.NewShellTracker(10)
95+
sup := NewSupervisor(&domainmocks.FakeMessageQueue{}, &domainmocks.FakeConversationRepository{}, nil)
96+
defer sup.Stop()
97+
98+
shell := startShell(t, "s-stuck", "sleep", "60")
99+
shell.ReadersDone = make(chan struct{}) // never closed
100+
_ = tracker.Add(shell)
101+
sup.Submit(NewShellJob(shell, tracker))
102+
103+
time.Sleep(20 * time.Millisecond)
104+
if err := sup.Wind("s-stuck", domain.WindStop); err != nil {
105+
t.Fatalf("Wind: %v", err)
106+
}
107+
waitTerminal(t, sup, "s-stuck")
108+
if shell.State != domain.ShellStateCancelled {
109+
t.Fatalf("shell state = %s, want cancelled", shell.State)
110+
}
111+
}
112+
48113
func TestShellJob_CompletesAndNotifies(t *testing.T) {
49114
tracker := utils.NewShellTracker(10)
50115
queue := &domainmocks.FakeMessageQueue{}

tests/mocks/domain/fake_background_shell_service.go

Lines changed: 10 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)