Skip to content

Commit b24d6c7

Browse files
rdimitrovclaude
rdimitrov
and
claude
authored
fix: harden registry availability and add publish phase timings (#1211)
## Summary Daily ~17:00 UTC `Publish Endpoint Latency` and `Availability < 95%` alerts are caused by both registry pods exit-0'ing within seconds of each other and racing with the still-recovering `registry-pg-rw` endpoint on restart. Separately, the default compute SA had no project IAM roles, so `fluentbit-gke` and the GMP collector were silently dropping every log line and time series — which is why none of this was visible in Cloud Logging. All Kubernetes/IAM changes apply uniformly to staging and prod. ## Changes - **GCP IAM** (`deploy/pkg/providers/gcp/provider.go`) — grant the default compute SA `logging.logWriter`, `monitoring.metricWriter`, `monitoring.viewer`, and `stackdriver.resourceMetadata.writer`. Already applied live; this just encodes it in Pulumi. - **Pod resilience** (`deploy/pkg/k8s/registry.go`) — - `PodDisruptionBudget` (`minAvailable: 1`) prevents a synchronized voluntary disruption. - `TopologySpreadConstraints` (`ScheduleAnyway`, hostname) keeps the PDB from deadlocking node drains when both pods co-locate. - `StartupProbe` (30 × 5s = 150s) protects the DB-retry budget from the liveness probe. - Memory bump 128→256Mi request, 256→512Mi limit. - **Startup retry** (`cmd/registry/main.go`) — replace silent `return` on initial DB-ping failure with bounded retry-with-backoff (8 attempts, 1→8s capped, 10s per-attempt). A brief endpoint flap no longer cascades into a restart loop. - **Diagnostic log** (`internal/service/registry_service.go`) — emit `validate_ms` on each publish (success and failure). Validate fans out to npm/PyPI/OCI with 10s per-host timeouts and is the most likely contributor to publish latency. Total request duration is already covered by the existing `http.request.duration` histogram. ## Calibration The IAM, PDB, topology-spread, startup-probe, and DB-retry pieces are mitigations — they reduce blast radius regardless of root cause. The `validate_ms` log is the diagnostic for identifying the trigger when the alert fires next. Single-PG SPOF (`deploy/pkg/k8s/postgres.go:57`) remains a follow-up; today's evidence shows PG was a victim, not the cause. ## Test plan - [x] `go build ./...` and `go vet ./...` clean for both modules - [x] `make lint` clean (12 pre-existing issues under `deploy/` are not in `make lint`) - [x] `go test -race -count=1 ./internal/... ./cmd/...` green - [x] IAM applied live; `fluentbit-gke` and GMP no longer hitting `IAM_PERMISSION_DENIED` - [ ] After deploy, verify the next 17:00 UTC firing surfaces `validate_ms` in Cloud Logging 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 7722031 commit b24d6c7

4 files changed

Lines changed: 165 additions & 17 deletions

File tree

‎cmd/registry/main.go‎

Lines changed: 25 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,15 +57,31 @@ func main() {
5757
// Initialize configuration
5858
cfg := config.NewConfig()
5959

60-
// Create a context with timeout for PostgreSQL connection
61-
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
62-
defer cancel()
63-
64-
// Connect to PostgreSQL
65-
db, err = database.NewPostgreSQL(ctx, cfg.DatabaseURL)
66-
if err != nil {
67-
log.Printf("Failed to connect to PostgreSQL: %v", err)
68-
return
60+
// Connect to PostgreSQL with bounded retry. The DB endpoint can flap briefly
61+
// during voluntary disruptions (node drains, rollouts) and without retry the
62+
// pod exits cleanly, restarts, then races with the still-recovering service
63+
// — turning a transient blip into a multi-minute outage.
64+
const (
65+
maxStartupAttempts = 8
66+
perAttemptTimeout = 10 * time.Second
67+
initialBackoff = 1 * time.Second
68+
maxBackoff = 8 * time.Second
69+
)
70+
backoff := initialBackoff
71+
for attempt := 1; ; attempt++ {
72+
attemptCtx, attemptCancel := context.WithTimeout(context.Background(), perAttemptTimeout)
73+
db, err = database.NewPostgreSQL(attemptCtx, cfg.DatabaseURL)
74+
attemptCancel()
75+
if err == nil {
76+
break
77+
}
78+
if attempt >= maxStartupAttempts {
79+
log.Printf("Failed to connect to PostgreSQL after %d attempts: %v", attempt, err)
80+
return
81+
}
82+
log.Printf("PostgreSQL not reachable (attempt %d/%d): %v, retrying in %s", attempt, maxStartupAttempts, err, backoff)
83+
time.Sleep(backoff)
84+
backoff = min(backoff*2, maxBackoff)
6985
}
7086

7187
// Store the PostgreSQL instance for later cleanup

‎deploy/pkg/k8s/registry.go‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3"
1111
metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1"
1212
networkingv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/networking/v1"
13+
policyv1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/policy/v1"
1314
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
1415
"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
1516

@@ -95,6 +96,22 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
9596
},
9697
},
9798
Spec: &corev1.PodSpecArgs{
99+
// Soft-spread pods across nodes so the PodDisruptionBudget below can't
100+
// deadlock a node drain when both pods happen to co-locate.
101+
// `ScheduleAnyway` keeps single-node clusters (e.g. constrained dev
102+
// envs) schedulable while still preferring spread under normal load.
103+
TopologySpreadConstraints: corev1.TopologySpreadConstraintArray{
104+
&corev1.TopologySpreadConstraintArgs{
105+
MaxSkew: pulumi.Int(1),
106+
TopologyKey: pulumi.String("kubernetes.io/hostname"),
107+
WhenUnsatisfiable: pulumi.String("ScheduleAnyway"),
108+
LabelSelector: &metav1.LabelSelectorArgs{
109+
MatchLabels: pulumi.StringMap{
110+
"app": pulumi.String("mcp-registry"),
111+
},
112+
},
113+
},
114+
},
98115
Containers: corev1.ContainerArray{
99116
&corev1.ContainerArgs{
100117
Name: pulumi.String("mcp-registry"),
@@ -169,6 +186,18 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
169186
Value: pulumi.String("*"),
170187
},
171188
},
189+
// StartupProbe protects the DB-retry budget in cmd/registry/main.go:
190+
// 30 × 5s = 150s allowed before liveness/readiness take over, which
191+
// covers the worst-case retry budget (8 attempts × 10s + ~40s sleep).
192+
StartupProbe: &corev1.ProbeArgs{
193+
HttpGet: &corev1.HTTPGetActionArgs{
194+
Path: pulumi.String("/v0/health"),
195+
Port: pulumi.Int(8080),
196+
},
197+
PeriodSeconds: pulumi.Int(5),
198+
FailureThreshold: pulumi.Int(30),
199+
TimeoutSeconds: pulumi.Int(3),
200+
},
172201
LivenessProbe: &corev1.ProbeArgs{
173202
HttpGet: &corev1.HTTPGetActionArgs{
174203
Path: pulumi.String("/v0/health"),
@@ -187,11 +216,11 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
187216
},
188217
Resources: &corev1.ResourceRequirementsArgs{
189218
Requests: pulumi.StringMap{
190-
"memory": pulumi.String("128Mi"),
219+
"memory": pulumi.String("256Mi"),
191220
"cpu": pulumi.String("100m"),
192221
},
193222
Limits: pulumi.StringMap{
194-
"memory": pulumi.String("256Mi"),
223+
"memory": pulumi.String("512Mi"),
195224
},
196225
},
197226
},
@@ -204,6 +233,31 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env
204233
return nil, err
205234
}
206235

236+
// Keep at least one registry pod available during voluntary disruptions
237+
// (node drains, evictions, etc.) so a synchronized eviction can't take both
238+
// pods down at once.
239+
_, err = policyv1.NewPodDisruptionBudget(ctx, "mcp-registry", &policyv1.PodDisruptionBudgetArgs{
240+
Metadata: &metav1.ObjectMetaArgs{
241+
Name: pulumi.String("mcp-registry"),
242+
Namespace: pulumi.String("default"),
243+
Labels: pulumi.StringMap{
244+
"app": pulumi.String("mcp-registry"),
245+
"environment": pulumi.String(environment),
246+
},
247+
},
248+
Spec: &policyv1.PodDisruptionBudgetSpecArgs{
249+
MinAvailable: pulumi.Int(1),
250+
Selector: &metav1.LabelSelectorArgs{
251+
MatchLabels: pulumi.StringMap{
252+
"app": pulumi.String("mcp-registry"),
253+
},
254+
},
255+
},
256+
}, pulumi.Provider(cluster.Provider))
257+
if err != nil {
258+
return nil, err
259+
}
260+
207261
// Create Service
208262
service, err := corev1.NewService(ctx, "mcp-registry", &corev1.ServiceArgs{
209263
Metadata: &metav1.ObjectMetaArgs{

‎deploy/pkg/providers/gcp/provider.go‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,12 @@ package gcp
33
import (
44
"encoding/base64"
55
"fmt"
6+
"strings"
67

78
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp"
89
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/container"
10+
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/organizations"
11+
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/projects"
912
"github.com/pulumi/pulumi-gcp/sdk/v8/go/gcp/storage"
1013
"github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes"
1114
corev1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1"
@@ -58,6 +61,49 @@ func createGCPProvider(ctx *pulumi.Context, name string) (*gcp.Provider, error)
5861
return nil, nil
5962
}
6063

64+
// grantNodeServiceAccountRoles grants the default compute service account the standard
65+
// GKE node roles required for log shipping (fluentbit-gke) and metrics scraping
66+
// (managed Prometheus). New GCP projects no longer auto-grant Editor to the default
67+
// compute SA, so these have to be set explicitly or every pod log/metric is dropped.
68+
func grantNodeServiceAccountRoles(ctx *pulumi.Context, projectID string, gcpProvider *gcp.Provider) error {
69+
invokeOpts := []pulumi.InvokeOption{}
70+
resourceOpts := []pulumi.ResourceOption{}
71+
if gcpProvider != nil {
72+
invokeOpts = append(invokeOpts, pulumi.Provider(gcpProvider))
73+
resourceOpts = append(resourceOpts, pulumi.Provider(gcpProvider))
74+
}
75+
76+
project, err := organizations.LookupProject(ctx, &organizations.LookupProjectArgs{
77+
ProjectId: pulumi.StringRef(projectID),
78+
}, invokeOpts...)
79+
if err != nil {
80+
return fmt.Errorf("failed to look up project number for node SA bindings: %w", err)
81+
}
82+
83+
nodeSA := fmt.Sprintf("serviceAccount:%s-compute@developer.gserviceaccount.com", project.Number)
84+
85+
roles := []string{
86+
"roles/logging.logWriter",
87+
"roles/monitoring.metricWriter",
88+
"roles/monitoring.viewer",
89+
"roles/stackdriver.resourceMetadata.writer",
90+
}
91+
92+
for _, role := range roles {
93+
name := fmt.Sprintf("node-sa-%s", strings.ReplaceAll(strings.TrimPrefix(role, "roles/"), ".", "-"))
94+
_, err := projects.NewIAMMember(ctx, name, &projects.IAMMemberArgs{
95+
Project: pulumi.String(projectID),
96+
Role: pulumi.String(role),
97+
Member: pulumi.String(nodeSA),
98+
}, resourceOpts...)
99+
if err != nil {
100+
return fmt.Errorf("failed to grant %s to node SA: %w", role, err)
101+
}
102+
}
103+
104+
return nil
105+
}
106+
61107
// CreateCluster creates a Google Kubernetes Engine cluster
62108
func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*providers.ProviderInfo, error) {
63109
// Get configuration
@@ -152,6 +198,13 @@ func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*prov
152198
return nil, fmt.Errorf("failed to create node pool: %w", err)
153199
}
154200

201+
// Grant the default compute service account the standard GKE node roles so the
202+
// fluentbit-gke and managed Prometheus collectors can ship logs and metrics.
203+
// Without these, every pod log line and time series fails with IAM_PERMISSION_DENIED.
204+
if err := grantNodeServiceAccountRoles(ctx, projectID, gcpProvider); err != nil {
205+
return nil, err
206+
}
207+
155208
// Create Kubernetes provider using the cluster directly
156209
k8sProvider, err := kubernetes.NewProvider(ctx, "k8s-provider", &kubernetes.ProviderArgs{
157210
Kubeconfig: pulumi.All(cluster.Endpoint, cluster.MasterAuth).ApplyT(func(args []any) (string, error) {

‎internal/service/registry_service.go‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55
"errors"
66
"fmt"
7+
"log/slog"
78
"time"
89

910
"github.com/jackc/pgx/v5"
@@ -85,15 +86,29 @@ func (s *registryServiceImpl) CreateServer(ctx context.Context, req *apiv0.Serve
8586
})
8687
}
8788

88-
// createServerInTransaction contains the actual CreateServer logic within a transaction
89+
// createServerInTransaction contains the actual CreateServer logic within a transaction.
8990
func (s *registryServiceImpl) createServerInTransaction(ctx context.Context, tx pgx.Tx, req *apiv0.ServerJSON) (*apiv0.ServerResponse, error) {
90-
// Validate the request
91-
if err := validators.ValidatePublishRequest(ctx, *req, s.cfg); err != nil {
92-
return nil, err
91+
serverJSON := *req
92+
93+
// Validate the request. ValidatePublishRequest fans out to npm/PyPI/OCI for
94+
// registry-ownership checks with 10s per-host timeouts, so it's the most likely
95+
// contributor to publish latency. Log validate_ms on both success and failure
96+
// so the next /v0/publish latency alert is diagnostic — a slow upstream that
97+
// times out into a validation error is exactly the case we need to see.
98+
validateStart := time.Now()
99+
validateErr := validators.ValidatePublishRequest(ctx, serverJSON, s.cfg)
100+
validateMs := time.Since(validateStart).Milliseconds()
101+
if validateErr != nil {
102+
slog.WarnContext(ctx, "publish validate failed",
103+
"server_name", serverJSON.Name,
104+
"version", serverJSON.Version,
105+
"validate_ms", validateMs,
106+
"error", validateErr.Error(),
107+
)
108+
return nil, validateErr
93109
}
94110

95111
publishTime := time.Now()
96-
serverJSON := *req
97112

98113
// Acquire advisory lock to prevent concurrent publishes of the same server
99114
if err := s.db.AcquirePublishLock(ctx, tx, serverJSON.Name); err != nil {
@@ -161,7 +176,17 @@ func (s *registryServiceImpl) createServerInTransaction(ctx context.Context, tx
161176
}
162177

163178
// Insert new server version
164-
return s.db.CreateServer(ctx, tx, &serverJSON, officialMeta)
179+
resp, err := s.db.CreateServer(ctx, tx, &serverJSON, officialMeta)
180+
if err != nil {
181+
return nil, err
182+
}
183+
184+
slog.InfoContext(ctx, "publish complete",
185+
"server_name", serverJSON.Name,
186+
"version", serverJSON.Version,
187+
"validate_ms", validateMs,
188+
)
189+
return resp, nil
165190
}
166191

167192
// recalculateLatest picks the highest non-deleted version of the given server and flags it

0 commit comments

Comments
 (0)