Skip to content

Commit fa274ef

Browse files
committed
test(mesh-conn): deterministic loopback harness for the ICE auth-race bug
Two-peer in-process harness with a minimal /publish + /poll broker (matches signaling/main.go's wire format; inlined because signaling/ is its own go.mod). Tests run against the existing main.go retry path with no behaviour mock-out: real pion/ice agents, real QUIC over the ice.Conn, real allowlist binds on 127.50.0.<vip>. Three tests: - TestSingleHandshake: green-path sanity check. Converges in ~80ms on loopback, so a regression in the dial/accept plumbing fails fast. - TestAuthRaceConvergence: the regression test for the bug. Forges a fresh-auth message from peer-B to peer-A after both reach "ice state: Checking" — this triggers peer-A's pollLoop supersession with a known-bad ufrag, peer-A retries and republishes, peer-B supersedes in turn, and the symmetric flap is determined entirely by runPeerLink's retry timing. On the current (broken) code this hangs to the 30s deadline 50/50 times with the exact "fresh auth ... supersedes consumed" signature from production. - TestRestartConvergence: real Mesh restart mid-link; same symmetric- restart pattern that triggered the live-cluster bug on 2026-05-13. Harness pieces in harness_test.go: testBroker (HTTPtest server), testPeer (Mesh + onLinkUp hook + ctx-bounded lifecycle), testLogSink (captures the global logger so a test can grep "supersedes consumed" on failure), waitBothLinksUp (the convergence predicate). The fix lands separately in the next commit.
1 parent 2e329e4 commit fa274ef

2 files changed

Lines changed: 459 additions & 0 deletions

File tree

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"encoding/json"
6+
"fmt"
7+
"log"
8+
"net/http"
9+
"net/http/httptest"
10+
"strings"
11+
"sync"
12+
"testing"
13+
"time"
14+
)
15+
16+
// =============================================================================
17+
// In-test signalling broker — minimal /publish + /poll implementation
18+
// matching signaling/main.go's wire format. Inlined rather than imported
19+
// because signaling/ is its own go.mod.
20+
// =============================================================================
21+
22+
type testBroker struct {
23+
server *httptest.Server
24+
25+
mu sync.Mutex
26+
queues map[string][]Message
27+
}
28+
29+
func newTestBroker() *testBroker {
30+
b := &testBroker{queues: map[string][]Message{}}
31+
mux := http.NewServeMux()
32+
mux.HandleFunc("/publish", b.handlePublish)
33+
mux.HandleFunc("/poll", b.handlePoll)
34+
b.server = httptest.NewServer(mux)
35+
return b
36+
}
37+
38+
func (b *testBroker) URL() string { return b.server.URL }
39+
func (b *testBroker) Close() { b.server.Close() }
40+
41+
// push mirrors signaling/main.go: a new auth from a sender drops any
42+
// of that sender's prior messages from the recipient's queue. The
43+
// production broker relies on this to keep stale candidates from
44+
// being delivered after a peer rolls credentials; the test broker
45+
// must match because mesh-conn's pollLoop is calibrated against it.
46+
func (b *testBroker) push(to string, msg Message) {
47+
b.mu.Lock()
48+
defer b.mu.Unlock()
49+
if msg.Type == "auth" {
50+
kept := b.queues[to][:0]
51+
for _, m := range b.queues[to] {
52+
if m.From != msg.From {
53+
kept = append(kept, m)
54+
}
55+
}
56+
b.queues[to] = kept
57+
}
58+
b.queues[to] = append(b.queues[to], msg)
59+
}
60+
61+
func (b *testBroker) drain(peer string) []Message {
62+
b.mu.Lock()
63+
defer b.mu.Unlock()
64+
q := b.queues[peer]
65+
delete(b.queues, peer)
66+
return q
67+
}
68+
69+
func (b *testBroker) handlePublish(w http.ResponseWriter, r *http.Request) {
70+
to := r.URL.Query().Get("to")
71+
if to == "" {
72+
http.Error(w, "missing ?to=", http.StatusBadRequest)
73+
return
74+
}
75+
var msg Message
76+
if err := json.NewDecoder(r.Body).Decode(&msg); err != nil {
77+
http.Error(w, err.Error(), http.StatusBadRequest)
78+
return
79+
}
80+
b.push(to, msg)
81+
w.WriteHeader(http.StatusNoContent)
82+
}
83+
84+
// handlePoll short-polls. mesh-conn's pollLoop hammers /poll in a tight
85+
// loop and re-enters on every empty reply — that's fine for a unit test
86+
// (a few hundred polls/sec on loopback is cheap). We sleep briefly to
87+
// avoid hot-spinning while waiting for the first message.
88+
func (b *testBroker) handlePoll(w http.ResponseWriter, r *http.Request) {
89+
peer := r.URL.Query().Get("peer")
90+
if peer == "" {
91+
http.Error(w, "missing ?peer=", http.StatusBadRequest)
92+
return
93+
}
94+
deadline := time.Now().Add(500 * time.Millisecond)
95+
for time.Now().Before(deadline) {
96+
if r.Context().Err() != nil {
97+
return
98+
}
99+
b.mu.Lock()
100+
if len(b.queues[peer]) > 0 {
101+
msgs := b.queues[peer]
102+
delete(b.queues, peer)
103+
b.mu.Unlock()
104+
_ = json.NewEncoder(w).Encode(msgs)
105+
return
106+
}
107+
b.mu.Unlock()
108+
time.Sleep(5 * time.Millisecond)
109+
}
110+
w.Header().Set("Content-Type", "application/json")
111+
w.Write([]byte("[]\n"))
112+
}
113+
114+
// =============================================================================
115+
// Test-peer factory: build a Mesh with onLinkUp hook + capture goroutines.
116+
// =============================================================================
117+
118+
type testPeer struct {
119+
id string
120+
mesh *Mesh
121+
cancel context.CancelFunc
122+
linkUp chan string // peer-ID of the OTHER side, sent once per "link up"
123+
done chan struct{}
124+
logPrefix string
125+
}
126+
127+
// newTestPeer builds a Mesh for one side. peers is the full PEERS_JSON
128+
// (every peer including self), selfID picks which one this Mesh is.
129+
// allowlistPort is a single TCP-only allowlist entry; tests bind real
130+
// listeners on 127.50.0.<peer.vip>:port, which is harmless on Linux
131+
// (the whole 127.0.0.0/8 is loopback). Pass distinct ports across
132+
// concurrent tests so two harnesses don't conflict.
133+
func newTestPeer(t *testing.T, brokerURL, selfID string, peers []Peer, allowlistPort int) *testPeer {
134+
t.Helper()
135+
cfg := &Config{
136+
SelfID: selfID,
137+
Peers: peers,
138+
SignalingURL: brokerURL,
139+
Allowlist: []InfraPort{{Port: allowlistPort, UDP: false}},
140+
// No TurnHost — ICE will use host candidates only, which is
141+
// what we want for a loopback test.
142+
}
143+
if err := validateConfig(cfg); err != nil {
144+
t.Fatalf("bad test config for %s: %v", selfID, err)
145+
}
146+
m := newMesh(cfg)
147+
tp := &testPeer{
148+
id: selfID,
149+
mesh: m,
150+
linkUp: make(chan string, 64),
151+
done: make(chan struct{}),
152+
logPrefix: "[" + selfID + "] ",
153+
}
154+
m.onLinkUp = func(peerID string) {
155+
select {
156+
case tp.linkUp <- peerID:
157+
default:
158+
}
159+
}
160+
return tp
161+
}
162+
163+
func (tp *testPeer) start(ctx context.Context) {
164+
ctx, cancel := context.WithCancel(ctx)
165+
tp.cancel = cancel
166+
go func() {
167+
defer close(tp.done)
168+
tp.mesh.Run(ctx)
169+
}()
170+
}
171+
172+
func (tp *testPeer) stop() {
173+
if tp.cancel != nil {
174+
tp.cancel()
175+
}
176+
select {
177+
case <-tp.done:
178+
case <-time.After(15 * time.Second):
179+
// don't block test cleanup forever; pion goroutines may take
180+
// a moment to wind down after agent.Close.
181+
}
182+
}
183+
184+
// =============================================================================
185+
// Log capture: route the standard logger into a per-test buffer so we
186+
// can grep for diagnostic strings (e.g. "supersedes") and surface them
187+
// on failure without flooding the test output on success.
188+
// =============================================================================
189+
190+
type testLogSink struct {
191+
t *testing.T
192+
mu sync.Mutex
193+
buf strings.Builder
194+
}
195+
196+
func (s *testLogSink) Write(p []byte) (int, error) {
197+
s.mu.Lock()
198+
defer s.mu.Unlock()
199+
s.buf.Write(p)
200+
return len(p), nil
201+
}
202+
203+
func (s *testLogSink) String() string {
204+
s.mu.Lock()
205+
defer s.mu.Unlock()
206+
return s.buf.String()
207+
}
208+
209+
func (s *testLogSink) countLines(needle string) int {
210+
s.mu.Lock()
211+
defer s.mu.Unlock()
212+
return strings.Count(s.buf.String(), needle)
213+
}
214+
215+
// captureLogs redirects the default logger to a sink for the duration
216+
// of the test. Restored in cleanup. Note: log package is global, so
217+
// tests using this can't run in parallel — they don't, we don't pass
218+
// t.Parallel() in any of the harness tests.
219+
func captureLogs(t *testing.T) *testLogSink {
220+
t.Helper()
221+
sink := &testLogSink{t: t}
222+
prev := log.Writer()
223+
log.SetOutput(sink)
224+
t.Cleanup(func() {
225+
log.SetOutput(prev)
226+
})
227+
return sink
228+
}
229+
230+
// =============================================================================
231+
// waitBothLinksUp blocks until both peers see their counterpart come up,
232+
// or deadline expires. Returns the time it took, or an error explaining
233+
// what was still pending.
234+
// =============================================================================
235+
236+
func waitBothLinksUp(t *testing.T, a, b *testPeer, deadline time.Duration) (time.Duration, error) {
237+
t.Helper()
238+
start := time.Now()
239+
aUp := false
240+
bUp := false
241+
timeout := time.After(deadline)
242+
for !(aUp && bUp) {
243+
select {
244+
case peerID := <-a.linkUp:
245+
if peerID == b.id {
246+
aUp = true
247+
}
248+
case peerID := <-b.linkUp:
249+
if peerID == a.id {
250+
bUp = true
251+
}
252+
case <-timeout:
253+
return time.Since(start),
254+
fmt.Errorf("link-up deadline exceeded: %s↔%s aUp=%v bUp=%v",
255+
a.id, b.id, aUp, bUp)
256+
}
257+
}
258+
return time.Since(start), nil
259+
}
260+
261+
// =============================================================================
262+
// Shared peer-pair / port plan for the harness tests. peer-A < peer-B
263+
// lex, so peer-A is the controlling (Dial) side; this matches how
264+
// mesh-conn picks Dial vs Accept everywhere else (cfg.SelfID < remoteID).
265+
// =============================================================================
266+
267+
func twoPeers() []Peer {
268+
return []Peer{
269+
{ID: "peer-A", Vip: 1},
270+
{ID: "peer-B", Vip: 2},
271+
}
272+
}

0 commit comments

Comments
 (0)