Skip to content

Commit b5a24c9

Browse files
authored
feat(ci): tag-blocking release QA gate — server-type matrix + invariants (Spec 081 T1) (#819)
Adds a reusable release-qa-gate workflow that gates all publish jobs in release.yml and prerelease.yml behind a single machine-readable verdict. Runs the existing suites (test-api-e2e, race, server-edition race, scan-eval --gate) plus a new 5-cell server-type matrix (stdio/streamable-http/sse/docker-isolated/oauth) with connect→list→call→kill/reconnect per cell, and invariant checks (activity-log request-id correlation, token/telemetry counter deltas, quarantine flow, upgrade-in-place from the previous stable release). Report enumerates every cell with no silent skips; reserved slots left for the T2/T3/T4 Web-UI/macOS/consistency stories. Also fixes a real product bug found during verification: POST /api/v1/servers dropped the per-server isolation override. Closes the release-qa-gate-matrix roadmap task.
1 parent 6c14e74 commit b5a24c9

31 files changed

Lines changed: 6113 additions & 7 deletions

.github/workflows/prerelease.yml

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -757,8 +757,19 @@ jobs:
757757
name: installers-${{ matrix.goos }}-${{ matrix.goarch }}
758758
path: installers-artifact/*
759759

760+
# Spec 081: release qualification gate (same reusable workflow as release.yml).
761+
# Guarded to prerelease TAG refs only — skipped on `next` branch pushes,
762+
# consistent with the `release` job's own tag condition, so branch CI is not
763+
# burdened with the full gate. A red gate blocks the prerelease publish
764+
# (FR-002); a skipped gate (branch push) skips `release` too (never un-gated).
765+
qa-gate:
766+
uses: ./.github/workflows/release-qa-gate.yml
767+
permissions:
768+
contents: read
769+
if: startsWith(github.ref, 'refs/tags/v') && (contains(github.ref, '-rc.') || contains(github.ref, '-next.'))
770+
760771
release:
761-
needs: build
772+
needs: [build, qa-gate]
762773
runs-on: ubuntu-latest
763774
environment: staging
764775
# Only create releases for tag pushes, not branch pushes

.github/workflows/release-qa-gate.yml

Lines changed: 430 additions & 0 deletions
Large diffs are not rendered by default.

.github/workflows/release.yml

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1061,8 +1061,21 @@ jobs:
10611061
ghcr.io/smart-mcp-proxy/mcpproxy-server:${{ github.ref_name }}
10621062
ghcr.io/smart-mcp-proxy/mcpproxy-server:latest
10631063
1064+
# Spec 081: release qualification gate. Runs the server-type matrix +
1065+
# invariants + assembled suites and produces one pass/fail verdict. The
1066+
# `release` job (and therefore every public-facing job that cascades from it)
1067+
# depends on this, so a red gate publishes nothing (FR-002). Guarded to the
1068+
# same stable-only condition as `release` so the expensive gate does not also
1069+
# run on RC tags here — prerelease.yml owns the RC gate. If the gate is
1070+
# skipped (non-stable ref), `release` is skipped too (never un-gated).
1071+
qa-gate:
1072+
uses: ./.github/workflows/release-qa-gate.yml
1073+
permissions:
1074+
contents: read
1075+
if: startsWith(github.ref, 'refs/tags/v') && !contains(github.ref_name, '-')
1076+
10641077
release:
1065-
needs: [build, sign-windows, generate-notes] # add build-docker when server MVP is ready
1078+
needs: [build, sign-windows, generate-notes, qa-gate] # add build-docker when server MVP is ready
10661079
runs-on: ubuntu-latest
10671080
environment: production
10681081
# Explicit guard (belt-and-suspenders on top of the cascade from needs:) — RC tags

cmd/mcpfixture/Dockerfile

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Dockerfile for the release QA gate's docker-isolated stdio fixture cell
2+
# (Spec 081, FR-006/FR-009). Built by scripts/gate/build-fixture-image.sh,
3+
# which cross-compiles a fully static linux binary (CGO_ENABLED=0, pure Go)
4+
# BEFORE the docker build, so this image needs no toolchain and no third-party
5+
# pulls at all — FROM scratch.
6+
#
7+
# IMPORTANT — no ENTRYPOINT, on purpose. mcpproxy's Docker isolation
8+
# (internal/upstream/core/isolation.go BuildDockerArgs +
9+
# connection_docker.go setupDockerIsolation) always appends the container
10+
# command explicitly after the image:
11+
#
12+
# docker run --rm -i --name mcpproxy-<server>-<rand> <labels> ... \
13+
# docker.io/library/mcpfixture:gate /mcpfixture --transport stdio
14+
#
15+
# An ENTRYPOINT would prefix that argv and break it. CMD only provides the
16+
# default for a bare `docker run mcpfixture:gate` smoke test.
17+
FROM scratch
18+
COPY mcpfixture /mcpfixture
19+
CMD ["/mcpfixture", "--transport", "stdio"]

cmd/mcpfixture/main.go

Lines changed: 258 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,258 @@
1+
// Command mcpfixture is a deterministic MCP upstream fixture for the release
2+
// QA gate (Spec 081, FR-006/FR-007). One binary serves the same two tools over
3+
// three transports so gate matrix cells never depend on third-party services:
4+
//
5+
// mcpfixture --transport stdio # newline-delimited JSON-RPC on stdin/stdout
6+
// mcpfixture --transport http --port 18080 # streamable-http at http://127.0.0.1:18080/mcp
7+
// mcpfixture --transport sse --port 18081 # legacy SSE at http://127.0.0.1:18081/sse (+ POST /message)
8+
//
9+
// Transports are served by mark3labs/mcp-go's server package — the same
10+
// library whose CLIENT side mcpproxy pins (internal/upstream/core uses
11+
// transport.CreateSSEClient / NewStreamableHTTP / NewStdio), so the wire
12+
// contract is protocol-correct by construction instead of hand-rolled:
13+
// - sse: GET /sse opens the event stream and announces the message endpoint
14+
// ("endpoint" event, relative URL resolved against the client's base URL);
15+
// requests are POSTed to /message?sessionId=… and responses stream back as
16+
// SSE "message" events.
17+
// - http: POST /mcp single-shot JSON responses (StreamableHTTPServer).
18+
// - stdio: newline-delimited JSON-RPC frames (StdioServer).
19+
//
20+
// Tools (deterministic, FR-007c):
21+
// - echo — returns the call arguments back as JSON text:
22+
// {"echo": {...arguments...}}
23+
// - ping — returns {"message":"pong","counter":N,"instance_id":"…"}; the
24+
// counter increases monotonically per process and the instance_id is
25+
// random per process start, so kill/restart tests (FR-007d) can prove the
26+
// reconnected upstream is a NEW instance (counter reset, new id).
27+
//
28+
// SIGTERM/SIGINT exits cleanly with code 0 on every transport (docker stop
29+
// sends SIGTERM to PID 1, and the gate kills fixtures between matrix steps).
30+
//
31+
// Docker cell (FR-009) argv composition, verified against
32+
// internal/upstream/core/isolation.go + connection_docker.go:
33+
//
34+
// server config: command="/mcpfixture", args=["--transport","stdio"],
35+
// isolation.image="mcpfixture:gate"
36+
// DetectRuntimeType(filepath.Base("/mcpfixture")) → "binary" (default case)
37+
// TransformCommandForContainer(binary) → command+args passed through unchanged
38+
// GetDockerImage → buildFullImageName("mcpfixture:gate") → "docker.io/library/mcpfixture:gate"
39+
// (no slash ⇒ registry+library prefix; Docker normalizes local short tags
40+
// the same way, so a locally built `mcpfixture:gate` resolves without a pull)
41+
// final argv: docker run --rm -i --name mcpproxy-<server>-<rand> <labels>
42+
// [--log-opt …] [--network <mode>] [limits] [-e K=V …]
43+
// docker.io/library/mcpfixture:gate /mcpfixture --transport stdio
44+
//
45+
// Because the container command is always appended explicitly after the image,
46+
// the image must NOT set an ENTRYPOINT (it would prefix the argv and break
47+
// `/mcpfixture --transport stdio`); it ships CMD ["/mcpfixture","--transport","stdio"]
48+
// as a default for bare `docker run` instead. See cmd/mcpfixture/Dockerfile and
49+
// scripts/gate/build-fixture-image.sh.
50+
package main
51+
52+
import (
53+
"context"
54+
"crypto/rand"
55+
"encoding/hex"
56+
"encoding/json"
57+
"errors"
58+
"flag"
59+
"fmt"
60+
"net/http"
61+
"os"
62+
"os/signal"
63+
"sync/atomic"
64+
"syscall"
65+
"time"
66+
67+
"github.com/mark3labs/mcp-go/mcp"
68+
"github.com/mark3labs/mcp-go/server"
69+
)
70+
71+
const (
72+
fixtureName = "mcpfixture"
73+
fixtureVersion = "0.1.0"
74+
75+
transportStdio = "stdio"
76+
transportHTTP = "http"
77+
transportSSE = "sse"
78+
79+
shutdownTimeout = 5 * time.Second
80+
)
81+
82+
// pingCounter increases monotonically per tools/call of `ping` within one
83+
// process. It intentionally resets on restart so reconnect tests can detect a
84+
// fresh instance.
85+
var pingCounter atomic.Int64
86+
87+
// instanceID is random per process start — the second discriminator (besides
88+
// the counter reset) that kill/restart tests use to distinguish instances.
89+
var instanceID = newInstanceID()
90+
91+
func newInstanceID() string {
92+
buf := make([]byte, 8)
93+
if _, err := rand.Read(buf); err != nil {
94+
// Deterministic fallback still unique enough across restarts.
95+
return fmt.Sprintf("pid-%d-%d", os.Getpid(), time.Now().UnixNano())
96+
}
97+
return hex.EncodeToString(buf)
98+
}
99+
100+
// pingResult is the JSON payload returned by the ping tool.
101+
type pingResult struct {
102+
Message string `json:"message"`
103+
Counter int64 `json:"counter"`
104+
InstanceID string `json:"instance_id"`
105+
}
106+
107+
func main() {
108+
transportFlag := flag.String("transport", transportStdio, "Transport to serve: stdio|http|sse")
109+
port := flag.Int("port", 0, "TCP port to bind (required for http/sse)")
110+
addr := flag.String("addr", "127.0.0.1", "Bind address (http/sse)")
111+
flag.Parse()
112+
113+
mcpSrv := newFixtureServer()
114+
115+
// A stdio parent (mcpproxy, or mcp-go's stdio client) may close our stderr
116+
// pipe before we write the final shutdown line; without this, Go's runtime
117+
// treats EPIPE on fd 1/2 as a fatal SIGPIPE and the fixture would die with
118+
// a non-zero "signal: broken pipe" status instead of exiting cleanly.
119+
signal.Ignore(syscall.SIGPIPE)
120+
121+
// All operational logging goes to stderr: stdout is the protocol channel
122+
// for the stdio transport.
123+
fmt.Fprintf(os.Stderr, "[mcpfixture] transport=%s instance_id=%s pid=%d\n", *transportFlag, instanceID, os.Getpid())
124+
125+
switch *transportFlag {
126+
case transportStdio:
127+
// server.ServeStdio installs its own SIGTERM/SIGINT handler and
128+
// returns context.Canceled after a signal — treat that as clean.
129+
if err := server.ServeStdio(mcpSrv); err != nil && !errors.Is(err, context.Canceled) {
130+
fmt.Fprintf(os.Stderr, "[mcpfixture] stdio serve failed: %v\n", err)
131+
os.Exit(1)
132+
}
133+
case transportHTTP, transportSSE:
134+
if *port == 0 {
135+
fmt.Fprintln(os.Stderr, "[mcpfixture] --port is required for http/sse transports")
136+
os.Exit(2)
137+
}
138+
listenAddr := fmt.Sprintf("%s:%d", *addr, *port)
139+
if err := serveNetwork(*transportFlag, mcpSrv, listenAddr); err != nil {
140+
fmt.Fprintf(os.Stderr, "[mcpfixture] %s serve failed: %v\n", *transportFlag, err)
141+
os.Exit(1)
142+
}
143+
default:
144+
fmt.Fprintf(os.Stderr, "[mcpfixture] unknown --transport %q (want stdio|http|sse)\n", *transportFlag)
145+
os.Exit(2)
146+
}
147+
148+
fmt.Fprintln(os.Stderr, "[mcpfixture] clean shutdown")
149+
}
150+
151+
// newFixtureServer builds the MCP server with the two deterministic tools.
152+
func newFixtureServer() *server.MCPServer {
153+
s := server.NewMCPServer(fixtureName, fixtureVersion, server.WithToolCapabilities(false))
154+
155+
// Both tools are annotated read-only: without annotations the MCP spec
156+
// default is destructiveHint=true and mcpproxy's intent validation
157+
// (Spec 018) would reject them through call_tool_read.
158+
s.AddTool(mcp.NewTool("echo",
159+
mcp.WithDescription("Returns the provided arguments back as JSON text."),
160+
mcp.WithString("text", mcp.Description("Text to echo back.")),
161+
mcp.WithReadOnlyHintAnnotation(true),
162+
mcp.WithDestructiveHintAnnotation(false),
163+
mcp.WithOpenWorldHintAnnotation(false),
164+
), handleEcho)
165+
166+
s.AddTool(mcp.NewTool("ping",
167+
mcp.WithDescription("Returns pong with a per-process monotonically increasing counter and a per-process instance id."),
168+
mcp.WithReadOnlyHintAnnotation(true),
169+
mcp.WithDestructiveHintAnnotation(false),
170+
mcp.WithOpenWorldHintAnnotation(false),
171+
), handlePing)
172+
173+
return s
174+
}
175+
176+
func handleEcho(_ context.Context, req mcp.CallToolRequest) (*mcp.CallToolResult, error) {
177+
args := req.GetArguments()
178+
if args == nil {
179+
args = map[string]any{}
180+
}
181+
payload, err := json.Marshal(map[string]any{"echo": args})
182+
if err != nil {
183+
return mcp.NewToolResultError(fmt.Sprintf("marshal echo payload: %v", err)), nil
184+
}
185+
return mcp.NewToolResultText(string(payload)), nil
186+
}
187+
188+
func handlePing(_ context.Context, _ mcp.CallToolRequest) (*mcp.CallToolResult, error) {
189+
payload, err := json.Marshal(pingResult{
190+
Message: "pong",
191+
Counter: pingCounter.Add(1),
192+
InstanceID: instanceID,
193+
})
194+
if err != nil {
195+
return mcp.NewToolResultError(fmt.Sprintf("marshal ping payload: %v", err)), nil
196+
}
197+
return mcp.NewToolResultText(string(payload)), nil
198+
}
199+
200+
// networkServer is the common surface of mcp-go's StreamableHTTPServer and
201+
// SSEServer that serveNetwork needs.
202+
type networkServer interface {
203+
Start(addr string) error
204+
Shutdown(ctx context.Context) error
205+
}
206+
207+
// serveNetwork runs the given transport until SIGTERM/SIGINT, then shuts it
208+
// down gracefully (5s deadline). A listen failure surfaces as the returned
209+
// error.
210+
func serveNetwork(transportName string, mcpSrv *server.MCPServer, listenAddr string) error {
211+
var srv networkServer
212+
switch transportName {
213+
case transportHTTP:
214+
srv = server.NewStreamableHTTPServer(mcpSrv)
215+
case transportSSE:
216+
// No WithBaseURL: the endpoint event advertises a relative
217+
// /message?sessionId=… URL, which mcp-go's SSE client (the one
218+
// mcpproxy pins) resolves against its base URL.
219+
srv = server.NewSSEServer(mcpSrv)
220+
default:
221+
return fmt.Errorf("unknown network transport %q", transportName)
222+
}
223+
224+
errCh := make(chan error, 1)
225+
go func() {
226+
err := srv.Start(listenAddr)
227+
if err != nil && !errors.Is(err, http.ErrServerClosed) {
228+
errCh <- err
229+
}
230+
close(errCh)
231+
}()
232+
233+
fmt.Fprintf(os.Stderr, "[mcpfixture] listening on %s\n", listenAddr)
234+
235+
sigCh := make(chan os.Signal, 1)
236+
signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)
237+
defer signal.Stop(sigCh)
238+
239+
select {
240+
case sig := <-sigCh:
241+
fmt.Fprintf(os.Stderr, "[mcpfixture] received %v, shutting down\n", sig)
242+
ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
243+
defer cancel()
244+
if err := srv.Shutdown(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) && !errors.Is(err, context.Canceled) {
245+
return fmt.Errorf("shutdown: %w", err)
246+
}
247+
// Drain the Start goroutine so a late listen error is not lost.
248+
if err, ok := <-errCh; ok && err != nil {
249+
return err
250+
}
251+
return nil
252+
case err, ok := <-errCh:
253+
if ok && err != nil {
254+
return err
255+
}
256+
return nil
257+
}
258+
}

0 commit comments

Comments
 (0)