Skip to content

Commit 9eb7e6b

Browse files
stephanosclaude
stephanos
and
claude
authored
Add goroutine-leak regression test (#10762)
## What Changed? Adds a goroutine-leak regression test ## Why? First step in goroutine leak regression testing effort; establishing baseline. --------- Co-authored-by: Claude Sonnet 4.6 (1M context) <noreply@anthropic.com>
1 parent 0319516 commit 9eb7e6b

4 files changed

Lines changed: 181 additions & 0 deletions

File tree

‎.github/workflows/run-tests.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -306,6 +306,34 @@ jobs:
306306
job_name: Unit test
307307
artifact_name_suffix: unit-test
308308

309+
leak-test:
310+
name: Goroutine leak test
311+
needs: [pre-build]
312+
runs-on: ubuntu-24.04-arm
313+
env:
314+
GITHUB_TIMEOUT: 10
315+
LEAK_OUTPUT_DIR: ${{ github.workspace }}/.testoutput/leakcheck
316+
steps:
317+
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
318+
with:
319+
token: ${{ secrets.GITHUB_TOKEN }}
320+
ref: ${{ env.COMMIT }}
321+
322+
- name: Prepare Go
323+
uses: ./.github/actions/prepare-go
324+
325+
- name: Run goroutine leak test
326+
timeout-minutes: ${{ fromJSON(env.GITHUB_TIMEOUT) }}
327+
run: make LEAK_TIMEOUT="$(("$GITHUB_TIMEOUT" - 1))m" leak-test
328+
329+
- name: Upload diagnostics
330+
if: failure()
331+
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
332+
with:
333+
name: leak-diagnostics
334+
path: ${{ env.LEAK_OUTPUT_DIR }}
335+
if-no-files-found: ignore
336+
309337
integration-test:
310338
name: Integration test
311339
needs: [pre-build, test-setup]

‎Makefile‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,20 @@ mixed-brain-test: clean-test-output
509509
@CGO_ENABLED=1 TEST_OUTPUT_ROOT=$(CURDIR)/$(TEST_OUTPUT_ROOT) go test -v $(MIXED_BRAIN_TEST_ROOT) $(COMPILED_TEST_ARGS) 2>&1 | tee -a test.log
510510
@$(MAKE) verify-test-log
511511

512+
LEAK_OUTPUT_DIR ?= $(TEST_OUTPUT_ROOT)/leakcheck
513+
LEAK_ITERS ?= 15
514+
LEAK_ITERS_WARMUP ?= 3
515+
LEAK_TIMEOUT ?= 5m
516+
leak-test:
517+
@printf $(COLOR) "Run goroutine-leak regression test..."
518+
@mkdir -p $(LEAK_OUTPUT_DIR)
519+
LEAK_ITERS=$(LEAK_ITERS) \
520+
LEAK_ITERS_WARMUP=$(LEAK_ITERS_WARMUP) \
521+
LEAK_OUTPUT_DIR=$(LEAK_OUTPUT_DIR) \
522+
go test -run TestClusterShutdownLeak -count=1 -v \
523+
-timeout $(LEAK_TIMEOUT) $(TEST_TAG_FLAG) \
524+
./tests/leakcheck/ -args -persistenceType=sql -persistenceDriver=sqlite
525+
512526
verify-test-log:
513527
@test -s test.log || (echo "TEST FAILURE: test.log is missing or empty" && exit 1)
514528
@grep -q "^ok" test.log || (echo "TEST FAILURE: no passing test found in test.log" && exit 1)

‎go.mod‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ require (
6868
go.temporal.io/auto-scaled-workers v0.0.0-20260407181057-edd947d743d2
6969
go.temporal.io/sdk v1.41.1
7070
go.uber.org/fx v1.24.0
71+
go.uber.org/goleak v1.3.0
7172
go.uber.org/mock v0.6.0
7273
go.uber.org/multierr v1.11.0
7374
go.uber.org/zap v1.27.1

‎tests/leakcheck/leak_test.go‎

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
package leakcheck
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"os"
7+
"path/filepath"
8+
"runtime"
9+
"runtime/pprof"
10+
"strconv"
11+
"testing"
12+
13+
"github.com/stretchr/testify/require"
14+
sdkclient "go.temporal.io/sdk/client"
15+
"go.temporal.io/sdk/workflow"
16+
"go.temporal.io/server/tests/testcore"
17+
"go.uber.org/goleak"
18+
)
19+
20+
// goleakOpts are the goleak options applied to every Find/VerifyNone call.
21+
// TODO entries are known leaks to be fixed; remove each ignore once fixed.
22+
var goleakOpts = []goleak.Option{
23+
// By design: sqlite keeps one *sql.DB per file DSN for the process lifetime.
24+
goleak.IgnoreTopFunction("database/sql.(*DB).connectionOpener"),
25+
26+
// TODO: gRPC connection goroutines leaked because history/matching
27+
// connection pools are not closed on cluster shutdown.
28+
goleak.IgnoreTopFunction("google.golang.org/grpc/internal/grpcsync.(*CallbackSerializer).run"),
29+
goleak.IgnoreTopFunction("google.golang.org/grpc.(*addrConn).resetTransportAndUnlock"),
30+
goleak.IgnoreTopFunction("google.golang.org/grpc/internal/balancer/gracefulswitch.(*Balancer).updateSubConnState"),
31+
goleak.IgnoreTopFunction("go.temporal.io/server/client/history.watchMembershipForClose[...]"),
32+
goleak.IgnoreTopFunction("go.temporal.io/server/client/matching.(*partitionCache).Start.func1"),
33+
goleak.IgnoreTopFunction("go.temporal.io/server/client/matching.watchMembershipForEviction"),
34+
goleak.IgnoreTopFunction("go.temporal.io/server/common/membership.(*grpcResolver).listen"),
35+
36+
// TODO: worker-service and persistence goroutine leaks.
37+
goleak.IgnoreTopFunction("go.temporal.io/server/common/persistence.(*healthSignalAggregatorImpl).emitMetricsLoop"),
38+
goleak.IgnoreTopFunction("go.temporal.io/server/common/quotas.(*MapRequestRateLimiterImpl[...]).cleanupLoop"),
39+
goleak.IgnoreTopFunction("go.temporal.io/server/service/worker.(*PerNamespaceWorkerManager).periodicRefresh"),
40+
goleak.IgnoreTopFunction("net/http.(*persistConn).readLoop"),
41+
goleak.IgnoreTopFunction("net/http.(*persistConn).writeLoop"),
42+
43+
// TODO: SDK worker goroutines not fully stopped on cluster shutdown.
44+
goleak.IgnoreTopFunction("go.temporal.io/sdk/internal.(*baseWorker).runEagerTaskDispatcher"),
45+
goleak.IgnoreTopFunction("go.temporal.io/sdk/internal.(*baseWorker).runTaskDispatcher"),
46+
goleak.IgnoreTopFunction("go.temporal.io/sdk/internal.(*localActivityTunnel).getTask"),
47+
goleak.IgnoreTopFunction("go.temporal.io/sdk/internal.(*sharedNamespaceWorker).run"),
48+
goleak.IgnoreTopFunction("go.temporal.io/sdk/internal/common/backoff.(*ConcurrentRetrier).throttleInternal"),
49+
}
50+
51+
// TestClusterShutdownLeak is a goroutine-leak regression test for the functional
52+
// test infrastructure. It detects per-cluster goroutines not being released when
53+
// a test cluster shuts down.
54+
//
55+
// Tunable via env:
56+
//
57+
// LEAK_ITERS clusters built after warmup
58+
// LEAK_ITERS_WARMUP warmup clusters before snapshotting the baseline
59+
// LEAK_OUTPUT_DIR directory for diagnostics on failure
60+
func TestClusterShutdownLeak(t *testing.T) {
61+
iters, err := strconv.Atoi(os.Getenv("LEAK_ITERS"))
62+
if err != nil {
63+
t.Fatal("LEAK_ITERS must be set to a positive integer")
64+
}
65+
warmupIters, err := strconv.Atoi(os.Getenv("LEAK_ITERS_WARMUP"))
66+
if err != nil {
67+
t.Fatal("LEAK_ITERS_WARMUP must be set to a positive integer")
68+
}
69+
outputDir := os.Getenv("LEAK_OUTPUT_DIR")
70+
if outputDir == "" {
71+
t.Fatal("LEAK_OUTPUT_DIR must be set")
72+
}
73+
if err := os.MkdirAll(outputDir, 0o755); err != nil {
74+
t.Fatalf("LEAK_OUTPUT_DIR: %v", err)
75+
}
76+
77+
// Warm up with a few clusters so process-lifetime singletons (gRPC resolver
78+
// init, proto registries, ...) are created before we snapshot the baseline.
79+
for range warmupIters {
80+
buildRunTeardownCluster(t)
81+
}
82+
83+
// Wait for warmup goroutines to drain before snapshotting the baseline.
84+
_ = goleak.Find(goleakOpts...)
85+
baseline := goleak.IgnoreCurrent()
86+
87+
// Run the leak test: build, run, and tear down a cluster per iteration.
88+
for i := range iters {
89+
buildRunTeardownCluster(t)
90+
t.Logf("cluster %2d: goroutines=%d", i, runtime.NumGoroutine())
91+
}
92+
93+
// Verify that no goroutines leaked beyond the baseline.
94+
if err := goleak.Find(append(goleakOpts, baseline)...); err != nil {
95+
t.Error(err)
96+
97+
// Write the goroutine report to the output directory.
98+
reportPath := filepath.Join(outputDir, "goleak_report.txt")
99+
if err := os.WriteFile(reportPath, []byte(err.Error()+"\n"), 0o644); err != nil {
100+
t.Logf("failed to write goroutine dump: %v", err)
101+
} else {
102+
t.Logf("goroutine dump written to %s", reportPath)
103+
}
104+
105+
// Write the full goroutine profile to the output directory.
106+
profilePath := filepath.Join(outputDir, "goroutines_all.txt")
107+
var pprofDump bytes.Buffer
108+
if err := pprof.Lookup("goroutine").WriteTo(&pprofDump, 2); err != nil {
109+
t.Logf("failed to capture goroutine pprof dump: %v", err)
110+
} else if err := os.WriteFile(profilePath, pprofDump.Bytes(), 0o644); err != nil {
111+
t.Logf("failed to write goroutine pprof dump: %v", err)
112+
} else {
113+
t.Logf("goroutine pprof dump written to %s", profilePath)
114+
}
115+
}
116+
}
117+
118+
// buildRunTeardownCluster creates a dedicated cluster, runs a trivial
119+
// workflow on it to exercise the full server path, then tears it down.
120+
func buildRunTeardownCluster(t *testing.T) {
121+
// The subtest ensures all env cleanups complete before this returns.
122+
t.Run("cluster", func(t *testing.T) {
123+
env := testcore.NewEnv(t,
124+
testcore.WithDedicatedCluster(),
125+
testcore.WithWorkerService("leak regression test"))
126+
127+
env.SdkWorker().RegisterWorkflow(smokeWorkflow)
128+
run, err := env.SdkClient().ExecuteWorkflow(
129+
context.Background(),
130+
sdkclient.StartWorkflowOptions{TaskQueue: env.WorkerTaskQueue()},
131+
smokeWorkflow,
132+
)
133+
require.NoError(t, err)
134+
require.NoError(t, run.Get(context.Background(), nil))
135+
})
136+
}
137+
138+
func smokeWorkflow(workflow.Context) error { return nil }

0 commit comments

Comments
 (0)