fix(site-agent): use mTLS for Flow gRPC and protect temporal-certs secret

Three related issues caused the site-agent to fail connecting to Flow gRPC:

1. FLOW_GRPC_SEC_OPT was missing from the site-agent Helm chart values, so
   the Flow gRPC client defaulted to FlowServerTLS (1). Flow requires mTLS
   (RequireAndVerifyClientCert); the server rejected the connection with
   "tls: certificate required". Fix: add FLOW_GRPC_SEC_OPT: "2" as the
   chart default so FlowMutualTLS is used by default.

2. The FlowMutualTLS case in NewFlowGrpcClient used tls.Config{Certificates: ...}
   to present the client cert. Go's TLS stack only selects a cert from that
   field when the cert's issuer matches the server's CertificateRequest acceptable
   CA list; when there is no match it silently sends no cert, and the server
   again rejects with "certificate required". Fix: use GetClientCertificate
   instead, which unconditionally returns the cert — the same approach used in
   rest-api/flow/pkg/certs/certs.go TLSConfig().

3. temporal-certs-secret.yaml created the secret with empty placeholder values
   on every helm upgrade, overwriting certs written by the bootstrap process.
   On the next pod restart the site-agent found the OTP gone, re-attempted
   bootstrap (which fails because the OTP was already consumed), and blocked
   forever — preventing FlowGrpc().Start() from running. Fix: add
   helm.sh/resource-policy: keep so upgrades never touch this secret after
   initial creation.

Author: shayan1995

helm/rest/nico-rest-site-agent/templates/temporal-certs-secret.yaml

@@ -1,14 +1,18 @@
 # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# Temporal client TLS certs for site-agent — placeholder values populated by bootstrap
+# Temporal client TLS certs for site-agent — placeholder values populated by bootstrap.
+# resource-policy: keep prevents helm upgrade from overwriting certs written by the
+# bootstrap process; deleting and re-installing the release creates a fresh placeholder.
 apiVersion: v1
 kind: Secret
 metadata:
   name: {{ .Values.secrets.temporalClientCerts }}
   namespace: {{ include "nico-rest-site-agent.namespace" . }}
   labels:
     {{- include "nico-rest-site-agent.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/resource-policy: keep
 type: Opaque
 stringData:
   otp: ""

helm/rest/nico-rest-site-agent/values.yaml

@@ -82,6 +82,8 @@ envConfig:
   ENABLE_TLS: "true"
   NICO_ADDRESS: ""
   NICO_SEC_OPT: "0"
+  FLOW_GRPC_ENABLED: "false"
+  FLOW_GRPC_SEC_OPT: "2"
   CLUSTER_ID: ""
   TEMPORAL_HOST: "temporal-frontend.temporal"
   TEMPORAL_PORT: "7233"

rest-api/site-workflow/pkg/grpc/client/flow_client.go

@@ -158,9 +158,19 @@ func NewFlowGrpcClient(config *FlowGrpcClientConfig) (client *FlowGrpcClient, er
 		if !capool.AppendCertsFromPEM(cabytes) {
 			return nil, fmt.Errorf("FlowGrpcClient: Failed to append CA cert to CA pool")
 		}
+		// Use GetClientCertificate (not Certificates) to unconditionally present
+		// the client cert. With Certificates, Go's TLS stack only selects a cert
+		// whose issuer matches the acceptable CA list from the server's
+		// CertificateRequest; when no match is found it silently sends no cert,
+		// causing the server to reject with "tls: certificate required".
+		// GetClientCertificate bypasses that matching and always returns the cert,
+		// leaving verification to the server — the same approach used in
+		// rest-api/flow/pkg/certs/certs.go TLSConfig().
 		mutualTLSConfig := &tls.Config{
-			Certificates: []tls.Certificate{clientCert},
-			RootCAs:      capool,
+			GetClientCertificate: func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
+				return &clientCert, nil
+			},
+			RootCAs: capool,
 		}
 		creds := credentials.NewTLS(mutualTLSConfig)