fix: address PR review feedback for OpenShift overlay

- Move SetClusterDefaultsNamespace before buildConfigMapCacheNamespaces
  so the manager cache watches the correct operator namespace instead
  of the hardcoded default
- Add strategic merge patches for OpenShift overlay to replace
  local-dev keycloak.localtest.me URLs with in-cluster
  keycloak-service.keycloak.svc endpoints
- Patch authbridge-runtime-config issuer, keycloak_url, and
  keycloak_realm for both inbound and outbound pipelines
- Patch spiffe-helper-config jwt_audience to use in-cluster URL

Signed-off-by: Ignas Baranauskas <ibaranau@redhat.com>

fix: address PR review feedback for OpenShift overlay

- Protect ClusterDefaultsNamespace with sync.Once to prevent potential
  data races from concurrent reads/writes across reconciler goroutines
- Requeue on ensureNamespaceSCCBinding failure when ClusterRole exists
  (OpenShift) to prevent agent pods from failing with SCC violations
  due to transient API errors
- Fix identity.type nesting in OpenShift authbridge patch to match the
  base template structure (under outbound.plugins[0].config)
- Document root CA dependency chain and recovery procedure in
  certificates.yaml
- Clarify getOperatorNamespace fallback exists for local dev/test runs

Assisted-By: Claude (Anthropic AI) <noreply@anthropic.com>
Signed-off-by: Ignas Baranauskas <ibaranau@redhat.com>

Author: Ygnas

kagenti-operator/cmd/main.go

@@ -76,8 +76,9 @@ func init() {
 }
 
 // getOperatorNamespace returns the namespace the operator is running in.
-// Reads from POD_NAMESPACE environment variable (set via downward API in deployment),
-// falling back to kagenti-system if not set.
+// In production, the manager_webhook_patch.yaml injects POD_NAMESPACE via
+// the downward API, so the fallback is effectively dead code. It exists for
+// local development and test runs where the webhook patch is not applied.
 func getOperatorNamespace() string {
 	if ns := os.Getenv("POD_NAMESPACE"); ns != "" {
 		return ns
@@ -309,6 +310,11 @@ func main() {
 		})
 	}
 
+	// ========================================
+	// Operator namespace resolution
+	// ========================================
+	controller.SetClusterDefaultsNamespace(getOperatorNamespace())
+
 	cmCacheNamespaces := buildConfigMapCacheNamespaces(
 		requireA2ASignature, spireTrustBundleConfigMapName, spireTrustBundleConfigMapNS,
 	)
@@ -365,11 +371,6 @@ func main() {
 		}
 	}
 
-	// ========================================
-	// Operator namespace resolution
-	// ========================================
-	controller.SetClusterDefaultsNamespace(getOperatorNamespace())
-
 	if !requireA2ASignature && !enableVerifiedFetch {
 		setupLog.Info("WARNING: Neither --require-a2a-signature nor --enable-verified-fetch is set. " +
 			"Identity binding requires at least one trust mechanism to function. " +

kagenti-operator/config/istio-trust/certificates.yaml

@@ -1,5 +1,12 @@
 # cert-manager Certificates for multi-mesh Istio shared trust.
 # Root CA in cert-manager namespace, intermediate CAs per Istio control plane.
+#
+# Dependency chain: Root CA (istio-mesh-root-ca, 10-year duration) issues all
+# intermediate CAs via the istio-mesh-ca ClusterIssuer. If the root CA secret
+# (istio-mesh-root-ca-secret) is accidentally deleted, all intermediates become
+# orphaned and workload mTLS will break.
+# Recovery: re-create this Certificate to generate a new root CA, then restart
+# cert-manager to re-issue the intermediate CA certificates.
 ---
 apiVersion: cert-manager.io/v1
 kind: Certificate

kagenti-operator/config/openshift/kustomization.yaml

@@ -9,3 +9,7 @@ resources:
 - ../istio-trust
 - ../authbridge
 - ../security
+
+patches:
+- path: patches/authbridge-keycloak.yaml
+- path: patches/spiffe-helper-keycloak.yaml

kagenti-operator/config/openshift/patches/authbridge-keycloak.yaml

@@ -0,0 +1,27 @@
+# Strategic merge patch: override authbridge-runtime-config with
+# in-cluster Keycloak URLs for OpenShift deployments.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: authbridge-runtime-config
+  namespace: kagenti-operator-system
+data:
+  config.yaml: |
+    spiffe: {}
+    pipeline:
+      inbound:
+        plugins:
+          - name: jwt-validation
+            config:
+              issuer: "http://keycloak-service.keycloak.svc:8080/realms/kagenti"
+              keycloak_url: "http://keycloak-service.keycloak.svc:8080"
+              keycloak_realm: "kagenti"
+      outbound:
+        plugins:
+          - name: token-exchange
+            config:
+              keycloak_url: "http://keycloak-service.keycloak.svc:8080"
+              keycloak_realm: "kagenti"
+              default_policy: "passthrough"
+              identity:
+                type: "client-secret"

kagenti-operator/config/openshift/patches/spiffe-helper-keycloak.yaml

@@ -0,0 +1,20 @@
+# Strategic merge patch: override spiffe-helper-config with
+# in-cluster Keycloak JWT audience for OpenShift deployments.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: spiffe-helper-config
+  namespace: kagenti-operator-system
+data:
+  helper.conf: |
+    agent_address = "/spiffe-workload-api/spire-agent.sock"
+    cmd = ""
+    cmd_args = ""
+    svid_file_name = "/opt/svid.pem"
+    svid_key_file_name = "/opt/svid_key.pem"
+    svid_bundle_file_name = "/opt/svid_bundle.pem"
+    cert_file_mode = 0644
+    key_file_mode = 0640
+    jwt_svids = [{jwt_audience="http://keycloak-service.keycloak.svc:8080/realms/kagenti", jwt_svid_file_name="/opt/jwt_svid.token"}]
+    jwt_svid_file_mode = 0644
+    include_federated_domains = true

kagenti-operator/internal/controller/agentruntime_config.go

@@ -22,6 +22,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
+	"sync"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -50,14 +51,22 @@ const (
 )
 
 // ClusterDefaultsNamespace is the namespace where cluster-level ConfigMaps
-// and template ConfigMaps live. Defaults to "kagenti-system"; set to the
-// operator's own namespace by main() via SetClusterDefaultsNamespace
-var ClusterDefaultsNamespace = clusterDefaultsNamespaceDefault
+// and template ConfigMaps live. Defaults to "kagenti-system"; set once to the
+// operator's own namespace by main() via SetClusterDefaultsNamespace.
+//
+// Write-once semantics: SetClusterDefaultsNamespace must be called exactly once
+// from main() before the manager starts. Subsequent calls are no-ops.
+var (
+	ClusterDefaultsNamespace     = clusterDefaultsNamespaceDefault
+	clusterDefaultsNamespaceOnce sync.Once
+)
 
 func SetClusterDefaultsNamespace(ns string) {
-	if ns != "" {
-		ClusterDefaultsNamespace = ns
-	}
+	clusterDefaultsNamespaceOnce.Do(func() {
+		if ns != "" {
+			ClusterDefaultsNamespace = ns
+		}
+	})
 }
 
 // resolvedConfig is the canonical representation used for hash computation.

kagenti-operator/internal/controller/agentruntime_controller.go

@@ -210,11 +210,19 @@ func (r *AgentRuntimeReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	// 4.7. Ensure SCC RoleBinding exists in the namespace.
 	// Creates a RoleBinding granting all ServiceAccounts in the namespace
 	// access to the kagenti-authbridge SCC. No-op on non-OpenShift clusters.
+	// On OpenShift, a transient failure is retried via requeue to prevent
+	// agent pods from failing with SCC violations at runtime.
 	if err := r.ensureNamespaceSCCBinding(ctx, rt.Namespace); err != nil {
 		logger.Error(err, "Failed to ensure SCC RoleBinding")
 		if r.Recorder != nil {
 			r.Recorder.Event(rt, corev1.EventTypeWarning, "SCCBindingError", err.Error())
 		}
+		r.setPhase(rt, agentv1alpha1.RuntimePhaseError)
+		r.setCondition(rt, ConditionTypeReady, metav1.ConditionFalse, "SCCBindingError", err.Error())
+		if statusErr := r.Status().Update(ctx, rt); statusErr != nil {
+			logger.Error(statusErr, "Failed to update status")
+		}
+		return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
 	}
 
 	// 5. Compute config hash from merged configuration (cluster → namespace)