k8s: fix BugBash infra-manifest findings (P0-W5-01, P1-W5-06/07, P2-W5)

mastermanas805 · claude · mastermanas805 · commit 3f33e65e2f84 · 2026-05-18T07:52:16.000+05:30
P0-W5-01: deploy-rbac.yaml ClusterRole was missing batch/jobs — every
Kaniko build Job the api creates (BatchV1().Jobs().Create/Get/Delete)
is RBAC-Forbidden on a manifest rebuild. Added the batch/jobs rule.

P1-W5-06: worker + provisioner livenessProbe pointed at /livez, which
neither binary serves (only /healthz is registered) — 404 → SIGKILL
crashloop every ~3 min. Repointed both liveness probes to /healthz and
corrected the stale manifest comments that wrongly claimed "/livez 404s
and liveness falls back to grace".

P1-W5-07: worker manifest hardcoded ENVIRONMENT=development in prod —
a 60x probe-amplification bug (resource_heartbeat at dev cadence).
Changed to production.

P2-W5: added resources: requests+limits to all three service
Deployments (instant-api 250m/256Mi..1/1Gi, worker + provisioner
100m/128Mi..500m/512Mi). Without resource requests the HPA fires
FailedGetResourceMetric and never scales (P1-W5-15).

P2-W5 (T4): added DEPLOY_DOMAIN + CERT_ISSUER to the ConfigMap — code
otherwise falls back to the hardcoded "instant.dev" default.

P3-W5 (T4): removed dead dashboard-grpc :50052 residue (DASHBOARD_GRPC_PORT
ConfigMap key, app.yaml containerPort + Service port) — dashboard-api was
retired 2026-05-11.

Co-Authored-By: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/k8s/app.yaml b/k8s/app.yaml
@@ -19,7 +19,7 @@ metadata:
     # any delete-old-versions workflow that targets prod tags.
     instanode.dev/image-pin-retention-days: "730"
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: instant-api
@@ -29,6 +29,17 @@ spec:
         app: instant-api
     spec:
       serviceAccountName: instant-api   # grants deploy-manager ClusterRole (see deploy-rbac.yaml)
+      # Spread replicas across nodes when possible — preferred (not required)
+      # so a 1-node dev cluster still schedules both pods.
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: instant-api
+                topologyKey: kubernetes.io/hostname
       initContainers:
         # Wait for platform DB to be ready before starting the API
         - name: wait-for-platform-db
@@ -50,7 +61,6 @@ spec:
           image: instant-api:local   # built with: docker build -t instant-api:local .
           ports:
             - containerPort: 8080
-            - containerPort: 50052
           volumeMounts:
             - name: docker-sock
               mountPath: /var/run/docker.sock
@@ -60,6 +70,16 @@ spec:
           env:
             - name: DOCKER_HOST
               value: unix:///var/run/docker.sock
+            - name: DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: instant-secrets
+                  key: DATABASE_URL
+            - name: CUSTOMER_DATABASE_URL
+              valueFrom:
+                secretKeyRef:
+                  name: instant-secrets
+                  key: CUSTOMER_DATABASE_URL
             - name: JWT_SECRET
               valueFrom:
                 secretKeyRef:
@@ -120,11 +140,6 @@ spec:
                 secretKeyRef:
                   name: instant-secrets
                   key: PROVISIONER_SECRET
-            - name: MIGRATOR_SECRET
-              valueFrom:
-                secretKeyRef:
-                  name: instant-secrets
-                  key: MIGRATOR_SECRET
             - name: R2_API_TOKEN
               valueFrom:
                 secretKeyRef:
@@ -231,22 +246,34 @@ spec:
             initialDelaySeconds: 15
             periodSeconds: 30
             failureThreshold: 6      # 6 × 30s = 3 min before SIGKILL
+          # Resource requests/limits — the api is build-heavy (it shells out
+          # to Kaniko Job orchestration on /deploy/new), so it gets headroom.
+          # The requests are also what the HPA needs to compute utilization;
+          # without them the HPA fires FailedGetResourceMetric and never
+          # scales (P1-W5-15).
+          resources:
+            requests:
+              cpu: 250m
+              memory: 256Mi
+            limits:
+              cpu: "1"
+              memory: 1Gi
 ---
 apiVersion: v1
 kind: Service
 metadata:
   name: instant-api
   namespace: instant
 spec:
+  # Externally reached via the cluster's Ingress (ingress-nginx) — see
+  # infra/k8s/api-ingress.yaml. The Service stays ClusterIP so the only
+  # public entrypoint is the LB → Ingress → Service path.
+  # For local dev / E2E: kubectl port-forward svc/instant-api 8080:8080
+  # (NodePort retired 2026-05-11).
+  type: ClusterIP
   selector:
     app: instant-api
   ports:
     - name: http
       port: 8080
       targetPort: 8080
-      nodePort: 30080   # access locally at http://localhost:30080
-    - name: dashboard-grpc
-      port: 50052
-      targetPort: 50052
-      nodePort: 30052
-  type: NodePort   # Rancher Desktop exposes NodePort via localhost
diff --git a/k8s/configmap.yaml b/k8s/configmap.yaml
@@ -5,25 +5,29 @@ metadata:
   namespace: instant
 data:
   PORT: "8080"
-  DASHBOARD_GRPC_PORT: "50052"
   # OpenTelemetry OTLP gRPC (host:port, no scheme). Empty = tracing disabled (noop).
   OTEL_EXPORTER_OTLP_ENDPOINT: ""
   OTEL_SERVICE_NAME: "instant-api"
   ENVIRONMENT: development
   INSTANT_ENABLED_SERVICES: redis,mongodb,postgres,queue,webhook,storage,deploy
   PLANS_PATH: /app/plans.yaml
   NATS_HOST: nats.instant-data.svc.cluster.local
-  DATABASE_URL: postgres://instant:instant@postgres-platform.instant.svc.cluster.local:5432/instant_platform?sslmode=disable
   REDIS_URL: redis://redis.instant.svc.cluster.local:6379
   GEOLITE2_DB_PATH: /app/GeoLite2-City.mmdb
   GITHUB_CLIENT_ID: ""
   GOOGLE_CLIENT_ID: ""
   # Optional default for GET /auth/google/url (agent API) when redirect_uri query is omitted
   GOOGLE_REDIRECT_URI: ""
   PROVISIONER_ADDR: "instant-provisioner.instant-infra.svc.cluster.local:50051"
-  MIGRATOR_ADDR: "http://instant-migrator.instant-infra.svc.cluster.local:8090"
   COMPUTE_PROVIDER: "k8s"
   STACK_EXPOSE_VIA: "nodeport"
   KUBE_NAMESPACE_APPS: "instant-apps"
   MINIO_ENDPOINT: "minio.instant-data.svc.cluster.local:9000"
   MINIO_BUCKET_NAME: "instant-shared"
+  # Deploy / stack ingress wiring. DEPLOY_DOMAIN is the wildcard zone the
+  # api builds per-deploy hostnames under (<id>.deployment.instanode.dev);
+  # without it the code falls back to the hardcoded default "instant.dev".
+  # CERT_ISSUER names the cert-manager ClusterIssuer used for TLS on those
+  # ingresses.
+  DEPLOY_DOMAIN: "deployment.instanode.dev"
+  CERT_ISSUER: "letsencrypt-http01"
diff --git a/k8s/deploy-rbac.yaml b/k8s/deploy-rbac.yaml
@@ -37,6 +37,14 @@ rules:
     resources: ["deployments"]
     verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
 
+  # Batch Jobs — the api runs each Kaniko build as a Job inside the
+  # per-deploy namespace (client.go BatchV1().Jobs(ns).Create/Get/Delete).
+  # Without this rule every /deploy/new and /stacks/new build is
+  # RBAC-Forbidden at Job creation.
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+
   # Ingresses (Traefik on k3s uses networking.k8s.io/v1).
   - apiGroups: ["networking.k8s.io"]
     resources: ["ingresses", "networkpolicies"]
diff --git a/k8s/provisioner/deployment.yaml b/k8s/provisioner/deployment.yaml
@@ -14,7 +14,7 @@ metadata:
   annotations:
     instanode.dev/image-pin-retention-days: "730"
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: instant-provisioner
@@ -26,6 +26,16 @@ spec:
       # ServiceAccount grants ClusterRole permissions for k8s dedicated backend.
       # When K8S_DEDICATED_BACKEND=false (default), the SA is still present but unused.
       serviceAccountName: instant-provisioner
+      # Spread replicas across nodes so node loss doesn't take all gRPC handlers down.
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: instant-provisioner
+                topologyKey: kubernetes.io/hostname
       initContainers:
         - name: wait-for-provisioner-db
           image: busybox:1.36
@@ -157,14 +167,11 @@ spec:
                 secretKeyRef:
                   name: minio-secrets
                   key: MINIO_ROOT_PASSWORD
-          # Probe split (2026-05-13): readiness hits the /healthz sidecar on
-          # port 8092 (checks DB + pool init). Liveness hits /livez on the
-          # same sidecar — process-only check that returns 200 unconditionally
-          # so a slow pool warm-up can't kill a healthy pod. Startup probe
-          # gives 10 minutes for pool pre-creation before liveness takes over.
-          # See PR body for the api/provisioner follow-up adding GET /livez.
-          # Until that handler ships, /livez 404s and liveness falls back to
-          # failureThreshold=6 grace (≈3 min).
+          # Probes (2026-05-18): the provisioner's /healthz sidecar on port
+          # 8092 registers ONLY /healthz — it never serves /livez. All three
+          # probes hit /healthz (checks DB + pool init). The generous
+          # startupProbe budget (10 min) absorbs hot-pool pre-creation before
+          # the liveness probe takes over.
           startupProbe:
             httpGet:
               path: /healthz
@@ -180,8 +187,15 @@ spec:
             failureThreshold: 3
           livenessProbe:
             httpGet:
-              path: /livez          # process-only; see startupProbe note
+              path: /healthz        # provisioner serves only /healthz (no /livez)
               port: 8092
             initialDelaySeconds: 15
             periodSeconds: 30
             failureThreshold: 6      # 6 × 30s = 3 min before SIGKILL
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
diff --git a/k8s/worker/deployment.yaml b/k8s/worker/deployment.yaml
@@ -14,7 +14,7 @@ metadata:
   annotations:
     instanode.dev/image-pin-retention-days: "730"
 spec:
-  replicas: 1
+  replicas: 2
   selector:
     matchLabels:
       app: instant-worker
@@ -24,20 +24,29 @@ spec:
         app: instant-worker
     spec:
       serviceAccountName: instant-worker
+      # Multi-worker is safe — River queue uses Postgres SELECT FOR UPDATE
+      # SKIP LOCKED so each job lands on exactly one worker.
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: instant-worker
+                topologyKey: kubernetes.io/hostname
       containers:
         - name: worker
           image: instant-worker:local
           imagePullPolicy: Never
           ports:
             - containerPort: 8091
               name: liveness
-          # Probe split (2026-05-13): /healthz checks River queue connectivity
-          # + DB pool (slow on cold start); /livez is process-only. Worker
-          # boots after migrations finish (it owns no migrations of its own),
-          # but heavy job replay on restart can stall /healthz briefly.
-          # See PR body for the worker follow-up adding GET /livez.
-          # Until that handler ships, /livez 404s and liveness falls back to
-          # failureThreshold=6 grace (≈3 min).
+          # Probes (2026-05-18): the worker binary registers ONLY /healthz
+          # (plus /metrics) — it never serves /livez. All three probes hit
+          # /healthz. /healthz checks River queue connectivity + DB pool
+          # (slow on cold start); heavy job replay on restart can stall it
+          # briefly, which the generous startupProbe budget absorbs.
           startupProbe:
             httpGet:
               path: /healthz
@@ -53,11 +62,18 @@ spec:
             failureThreshold: 3
           livenessProbe:
             httpGet:
-              path: /livez          # process-only; see startupProbe note
+              path: /healthz        # worker serves only /healthz (no /livez)
               port: 8091
             initialDelaySeconds: 15
             periodSeconds: 30
             failureThreshold: 6      # 6 × 30s = 3 min before SIGKILL
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
           env:
             # ── Observability (track 6 of 8) ──────────────────────────────────
             # NEW_RELIC_LICENSE_KEY pulled from instant-infra-secrets. When
@@ -97,7 +113,7 @@ spec:
                   name: instant-infra-secrets
                   key: PROVISIONER_SECRET
             - name: ENVIRONMENT
-              value: development
+              value: production
             - name: KUBE_NAMESPACE_APPS
               value: "instant-apps"
             # MinIO storage_bytes scanner — added 2026-05-11 in worker
@@ -123,3 +139,13 @@ spec:
                   name: instant-infra-secrets
                   key: MINIO_ROOT_PASSWORD
                   optional: true
+            # Brevo sender identity for the raw-HTML send path (added
+            # 2026-05-15). The dashboard-template path was inheriting a
+            # personal email from the Brevo sender field — sending these
+            # via code-controlled env vars closes that bug. Defaults in
+            # brevo_provider.go (noreply@instanode.dev / "instanode")
+            # take over if the env var is absent.
+            - name: BREVO_SENDER_EMAIL
+              value: "noreply@instanode.dev"
+            - name: BREVO_SENDER_NAME
+              value: "instanode"
