Skip to content

Commit 84ac30e

Browse files
jkyberneeesclaude
andauthored
fix(serve): break approval deadlock + race in wsApprover (#32)
* fix(serve): break approval deadlock + race in wsApprover Browser approvals were dead. handleWS ran the agent loop synchronously in the only goroutine reading the socket; when a tool needed approval the wsApprover blocked in PromptCommand waiting for the browser's approval_response — which could only be delivered by that same, now-blocked, receive loop. Every approval hit the 60s timeout and was denied, making the Web UI safety prompt unusable. - Move socket reading to a dedicated goroutine. It handles approval_response inline (delivering to the blocked PromptCommand via HandleResponse) and forwards other messages to a buffered promptCh the processor consumes serially. The forward is non-blocking so the reader can never stall and re-introduce the deadlock; on socket close it cancels the approver so a blocked prompt returns immediately instead of waiting out the timeout. - Fix the concurrent map read/write in wsApprover: approveAll/trustAll were read unlocked at the top of PromptCommand while the "trust" path and SetTrustAll wrote them — a data race that can fatally crash serve under parallel tool calls. Both accesses now hold the mutex. Regression tests (both verified to fail against the unpatched code): - TestServe_E2E_ApprovalRoundTrip: real handleWS + mock LLM tool call; approves via WebSocket and asserts completion within 10s (vs the 60s deadlock timeout). - TestWSApprover_ConcurrentTrust_NoRace: concurrent trust round-trips; crashes with "concurrent map writes" / DATA RACE before the fix. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix(serve): abort prompts on disconnect via connection-scoped context vprotocol axis 2.5: the dedicated reader eagerly drains the socket into promptCh, so prompts queued behind a running one were read before a disconnect (unlike the old code, which left them unread in the socket buffer). On disconnect they would then run with a still-live context — real LLM calls streaming to a dead socket. Derive prompt contexts from a connection-scoped context the reader cancels on exit, so a disconnect aborts the in-flight prompt and drains queued ones without LLM calls. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent 40839ed commit 84ac30e

3 files changed

Lines changed: 270 additions & 18 deletions

File tree

cmd/odek/serve.go

Lines changed: 74 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -485,19 +485,84 @@ func handleWS(store *session.Store, resources *resource.Registry, resolved confi
485485
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
486486
defer cancel()
487487

488+
// Connection-scoped context, cancelled the moment the socket reader exits
489+
// (disconnect or shutdown). Prompts run under this context, so a prompt
490+
// in flight when the client disconnects is aborted promptly, and any
491+
// prompts already buffered in promptCh run against a cancelled context
492+
// instead of making real LLM calls whose output streams to a dead socket.
493+
connCtx, connCancel := context.WithCancel(ctx)
494+
defer connCancel()
495+
496+
// Dedicated socket reader.
497+
//
498+
// The agent loop (handlePrompt → RunWithMessages) runs synchronously in the
499+
// processor loop below and BLOCKS whenever a tool needs approval — the
500+
// wsApprover waits for the browser's approval_response. If the same
501+
// goroutine both ran the agent and read the socket, it could never read
502+
// that response: every approval would dead-block until the 60s timeout and
503+
// be denied, making the Web UI's safety prompt unusable.
504+
//
505+
// So a separate goroutine owns the socket. It handles approval responses
506+
// INLINE (delivering them to the blocked PromptCommand via HandleResponse)
507+
// and forwards every other message to promptCh for serial processing. The
508+
// forward is non-blocking: the reader must never stall, or a full queue
509+
// would re-introduce the deadlock by stopping approval delivery. A buffered
510+
// queue absorbs the request/reply UI's normal pacing; an overflow only
511+
// happens under a client flooding prompts, which is reported and dropped.
512+
promptCh := make(chan []byte, 8)
513+
go func() {
514+
defer close(promptCh)
515+
for {
516+
var data []byte
517+
if err := golangws.Message.Receive(conn, &data); err != nil {
518+
// Socket gone: cancel in-flight/queued prompt work and release
519+
// any pending approval so a blocked handlePrompt returns now
520+
// instead of waiting out the 60s timeout.
521+
connCancel()
522+
if approver != nil {
523+
approver.Cancel()
524+
}
525+
return
526+
}
527+
528+
var msgType struct {
529+
Type string `json:"type"`
530+
}
531+
if err := json.Unmarshal(data, &msgType); err != nil {
532+
continue
533+
}
534+
535+
// Approval responses are handled here, off the processor goroutine,
536+
// so a prompt blocked awaiting approval can be unblocked. This is
537+
// the crux of the deadlock fix.
538+
if msgType.Type == "approval_response" {
539+
var resp approvalResponse
540+
if err := json.Unmarshal(data, &resp); err == nil {
541+
approver.HandleResponse(resp.ID, resp.Action)
542+
}
543+
continue
544+
}
545+
546+
select {
547+
case promptCh <- data:
548+
default:
549+
// Processor is busy and the queue is full — only reachable by a
550+
// client sending prompts faster than they can run.
551+
if msgType.Type == "prompt" {
552+
writeWSError(conn, "busy: a prompt is already running")
553+
}
554+
}
555+
}
556+
}()
557+
488558
// Track the current session and model across WebSocket messages
489559
var currentSession *session.Session
490560
currentModel := resolved.Model // start with configured model
491561

492562
// Session-level token economics (cumulative across all turns)
493563
var sessionInputTokens, sessionOutputTokens int
494564

495-
for {
496-
var data []byte
497-
if err := golangws.Message.Receive(conn, &data); err != nil {
498-
break
499-
}
500-
565+
for data := range promptCh {
501566
// Peek at the message type without full unmarshal
502567
var msgType struct {
503568
Type string `json:"type"`
@@ -506,15 +571,6 @@ func handleWS(store *session.Store, resources *resource.Registry, resolved confi
506571
continue
507572
}
508573

509-
// Handle approval responses separately (non-blocking, from the browser)
510-
if msgType.Type == "approval_response" {
511-
var resp approvalResponse
512-
if err := json.Unmarshal(data, &resp); err == nil {
513-
approver.HandleResponse(resp.ID, resp.Action)
514-
}
515-
continue
516-
}
517-
518574
// Handle skill prompt responses (Save/Skip from skill suggestions)
519575
if msgType.Type == "skill_prompt_response" {
520576
var resp struct {
@@ -578,8 +634,9 @@ func handleWS(store *session.Store, resources *resource.Registry, resolved confi
578634
}
579635

580636
// Run prompt — passes the persistent agent for buffer continuity
581-
// Create a cancelable context for this prompt (so POST /api/cancel can abort it)
582-
promptCtx, promptCancel := context.WithCancel(ctx)
637+
// Create a cancelable context for this prompt (so POST /api/cancel can abort it).
638+
// Derived from connCtx so a disconnect also aborts the running prompt.
639+
promptCtx, promptCancel := context.WithCancel(connCtx)
583640
currentPromptCancel.Store(promptCancel)
584641

585642
currentSession = handlePrompt(promptCtx, conn, store, resources, resolved, agent, currentSession, msg.Content, msg.SessionID, &sessionInputTokens, &sessionOutputTokens)

cmd/odek/serve_approval_test.go

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
package main
2+
3+
import (
4+
"encoding/json"
5+
"fmt"
6+
"net"
7+
"net/http"
8+
"os"
9+
"path/filepath"
10+
"sync"
11+
"testing"
12+
"time"
13+
14+
"github.com/BackendStack21/odek/internal/config"
15+
"github.com/BackendStack21/odek/internal/danger"
16+
"github.com/BackendStack21/odek/internal/resource"
17+
"github.com/BackendStack21/odek/internal/session"
18+
golangws "golang.org/x/net/websocket"
19+
)
20+
21+
// buildServeMuxPromptAll is buildServeMux with the danger policy forced to
22+
// "prompt" for every risk class, so even a benign `echo` shell command routes
23+
// through the wsApprover. This lets the test drive a real browser-approval
24+
// round trip without running a genuinely dangerous command.
25+
func buildServeMuxPromptAll(t *testing.T, store *session.Store) (net.Listener, *http.ServeMux) {
26+
t.Helper()
27+
ln, err := net.Listen("tcp", "127.0.0.1:0")
28+
if err != nil {
29+
t.Fatalf("listen: %v", err)
30+
}
31+
32+
resolved := config.LoadConfig(config.CLIFlags{})
33+
systemMessage := resolved.System
34+
if systemMessage == "" {
35+
systemMessage = defaultSystem
36+
}
37+
prompt := "prompt"
38+
resolved.Dangerous.DefaultAction = &prompt
39+
40+
cwd, _ := os.Getwd()
41+
home, _ := os.UserHomeDir()
42+
resourceReg := resource.NewRegistry(
43+
resource.NewFileResolver(cwd),
44+
resource.NewSessionResolver(filepath.Join(home, ".odek", "sessions")),
45+
)
46+
47+
mux := http.NewServeMux()
48+
mux.HandleFunc("/", handleStatic())
49+
mux.Handle("/ws", &golangws.Server{
50+
Handshake: func(*golangws.Config, *http.Request) error { return nil },
51+
Handler: func(conn *golangws.Conn) {
52+
handleWS(store, resourceReg, resolved, systemMessage, conn)
53+
},
54+
})
55+
mux.HandleFunc("/api/cancel", handleCancel)
56+
57+
return ln, mux
58+
}
59+
60+
// TestServe_E2E_ApprovalRoundTrip is the regression test for the serve-mode
61+
// approval deadlock. The agent calls a tool that requires approval and blocks
62+
// in PromptCommand; the browser's approval_response must be read and delivered
63+
// while that prompt is in flight. Before the fix the socket reader and the
64+
// agent loop were the same goroutine, so the response could never be read and
65+
// every approval hit the 60s timeout. The 10s read deadline here is well under
66+
// that timeout: a regression would blow the deadline, the fix completes in ms.
67+
func TestServe_E2E_ApprovalRoundTrip(t *testing.T) {
68+
llmSrv := mockLLM(t, func(w http.ResponseWriter, callCount int) {
69+
w.Header().Set("Content-Type", "application/json")
70+
if callCount <= 1 {
71+
fmt.Fprint(w, `{"choices":[{"message":{"content":"Running.","tool_calls":[{"id":"c_1","function":{"name":"shell","arguments":"{\"command\":\"echo approved-ok\"}"}}]}}]}`)
72+
} else {
73+
w.Write([]byte(`{"choices":[{"message":{"content":"All done."}}]}`))
74+
}
75+
})
76+
defer llmSrv.Close()
77+
78+
envCleanup := setTestEnv(t, llmSrv.URL)
79+
defer envCleanup()
80+
81+
store := newTestSessionStore(t)
82+
ln, mux := buildServeMuxPromptAll(t, store)
83+
defer ln.Close()
84+
85+
go func() { _ = serveOnListener(ln, mux) }()
86+
waitForHTTP(t, ln.Addr().String())
87+
88+
wsURL := "ws://" + ln.Addr().String() + "/ws"
89+
conn, err := golangws.Dial(wsURL, "", "http://localhost")
90+
if err != nil {
91+
t.Fatalf("Dial(%q): %v", wsURL, err)
92+
}
93+
defer conn.Close()
94+
95+
prompt := map[string]string{"type": "prompt", "content": "run a command"}
96+
payload, _ := json.Marshal(prompt)
97+
if err := golangws.Message.Send(conn, string(payload)); err != nil {
98+
t.Fatalf("Send: %v", err)
99+
}
100+
101+
// 10s ceiling — far below the 60s approval timeout a deadlock would hit.
102+
conn.SetReadDeadline(time.Now().Add(10 * time.Second))
103+
104+
var sawApprovalRequest, sawToolResult, sawDone bool
105+
for i := 0; i < 30; i++ {
106+
var raw []byte
107+
if err := golangws.Message.Receive(conn, &raw); err != nil {
108+
t.Fatalf("Receive event %d: %v (approval_request=%v tool_result=%v)", i, err, sawApprovalRequest, sawToolResult)
109+
}
110+
var evt map[string]any
111+
if err := json.Unmarshal(raw, &evt); err != nil {
112+
continue
113+
}
114+
switch evt["type"] {
115+
case "approval_request":
116+
sawApprovalRequest = true
117+
id, _ := evt["id"].(string)
118+
resp, _ := json.Marshal(map[string]string{
119+
"type": "approval_response",
120+
"id": id,
121+
"action": "approve",
122+
})
123+
if err := golangws.Message.Send(conn, string(resp)); err != nil {
124+
t.Fatalf("send approval_response: %v", err)
125+
}
126+
case "tool_result":
127+
sawToolResult = true
128+
case "done":
129+
sawDone = true
130+
goto finished
131+
case "error":
132+
t.Fatalf("unexpected error event: %v", evt["message"])
133+
}
134+
}
135+
finished:
136+
if !sawApprovalRequest {
137+
t.Fatal("never received an approval_request — the tool did not prompt")
138+
}
139+
if !sawToolResult {
140+
t.Error("approval was processed but the tool never produced a result")
141+
}
142+
if !sawDone {
143+
t.Fatal("never reached done — approval round trip did not complete (deadlock regression?)")
144+
}
145+
}
146+
147+
// TestWSApprover_ConcurrentTrust_NoRace drives many PromptCommand calls
148+
// concurrently, each answered with "trust", so the trust-cache write
149+
// (approveAll[cls]=true) races against the trust-cache read at the top of
150+
// PromptCommand. Before the fix these map accesses were unsynchronised: this
151+
// test crashes with "concurrent map writes" (and is flagged under -race).
152+
func TestWSApprover_ConcurrentTrust_NoRace(t *testing.T) {
153+
a := newWSApprover(nil)
154+
// Answer every prompt with an immediate "trust" from a separate goroutine,
155+
// forcing the approveAll write path under concurrency.
156+
a.sendFn = func(v any) error {
157+
if req, ok := v.(approvalRequest); ok {
158+
go a.HandleResponse(req.ID, "trust")
159+
}
160+
return nil
161+
}
162+
163+
classes := []danger.RiskClass{
164+
danger.Safe, danger.LocalWrite, danger.SystemWrite,
165+
danger.NetworkEgress, danger.CodeExecution, danger.Install,
166+
}
167+
168+
var wg sync.WaitGroup
169+
for i := 0; i < 200; i++ {
170+
for _, cls := range classes {
171+
wg.Add(1)
172+
go func(c danger.RiskClass) {
173+
defer wg.Done()
174+
_ = a.PromptCommand(c, "cmd", "desc")
175+
}(cls)
176+
}
177+
// Interleave SetTrustAll to also exercise the trustAll field race.
178+
wg.Add(1)
179+
go func() {
180+
defer wg.Done()
181+
a.SetTrustAll(false)
182+
}()
183+
}
184+
wg.Wait()
185+
}

cmd/odek/wsapprover.go

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,15 @@ func (a *wsApprover) shouldFriction(cls danger.RiskClass) (bool, int) {
114114
}
115115

116116
func (a *wsApprover) PromptCommand(cls danger.RiskClass, cmd, description string) error {
117-
if a.approveAll[cls] || a.trustAll {
117+
// Read the trust caches under the lock: PromptCommand runs concurrently
118+
// across parallel tool calls, while HandleResponse's "trust" path and
119+
// SetTrustAll write these same fields. An unsynchronised read here against
120+
// the map write below (or SetTrustAll) is a data race that can fatally
121+
// crash serve with "concurrent map read and map write".
122+
a.mu.Lock()
123+
trusted := a.approveAll[cls] || a.trustAll
124+
a.mu.Unlock()
125+
if trusted {
118126
return nil
119127
}
120128

@@ -172,7 +180,9 @@ func (a *wsApprover) PromptCommand(cls danger.RiskClass, cmd, description string
172180
a.recordApproval(cls)
173181
return nil
174182
}
183+
a.mu.Lock()
175184
a.approveAll[cls] = true
185+
a.mu.Unlock()
176186
return nil
177187
default:
178188
return fmt.Errorf("operation denied by user: %s", cmd)

0 commit comments

Comments
 (0)