Merge pull request #744 from aws-samples/fix/eks-cluster-template-issues

fix: EKS cluster template and kro RGD issues

Author: shapirov103

.kiro/steering/aws-env-best-practices.md

@@ -28,6 +28,14 @@ Enforces environment variable usage for all AWS operations to prevent hardcoded
 - When using Terraform specifically, use deployment scripts instead of direct terraform commands (ID: AWS_ENV_TF_SCRIPTS)
 - NEVER commit files containing actual AWS account IDs, access keys, or other sensitive data (ID: AWS_ENV_NO_COMMIT_SENSITIVE)
 
+### Troubleshooting and Reconciliation Best Practices
+
+- AWS controllers (ACK, Kro, ArgoCD via EKS Capabilities) reconcile asynchronously - ALWAYS wait for natural reconciliation before attempting destructive actions (ID: AWS_RECONCILE_PATIENCE)
+- When changes are made (IAM roles, policies, selectors), expect 15-30 minutes for full reconciliation across distributed controllers (ID: AWS_RECONCILE_TIME_EXPECTATION)
+- Troubleshooting should be non-destructive: check status, events, logs, describe resources - NEVER delete resources to "force" reconciliation (ID: AWS_TROUBLESHOOT_NON_DESTRUCTIVE)
+- If a resource appears stuck: (1) Check events with get_k8s_events, (2) Check logs, (3) Verify IAM permissions, (4) Wait longer - do NOT delete (ID: AWS_TROUBLESHOOT_SEQUENCE)
+- Resource deletion breaks status propagation in resource graphs (Kro), disrupts GitOps sync (ArgoCD), and can orphan AWS resources - avoid unless explicitly cleaning up (ID: AWS_DELETE_CONSEQUENCES)
+
 ## Priority
 
 Critical

.kiro/steering/eks-best-practices.md

@@ -15,6 +15,15 @@ Ensures all EKS and Kubernetes operations leverage the EKS MCP server capabiliti
 - For CloudWatch metrics and logs, prefer `get_cloudwatch_logs` and `get_cloudwatch_metrics` MCP tools over AWS CLI commands (ID: EKS_MCP_MONITORING)
 - When applying YAML manifests, use the `apply_yaml` MCP tool to benefit from validation and error handling (ID: EKS_MCP_APPLY)
 
+### Critical: Never Delete Resources During Reconciliation
+
+- **NEVER delete Kubernetes resources (kubectl delete, manage_k8s_resource with delete operation) to "force" reconciliation or troubleshoot** (ID: EKS_NO_DELETE_FORCE_RECONCILE)
+- Controllers (including EKS Capabilities-managed ACK, Kro, ArgoCD) will eventually reconcile - be patient and wait (ID: EKS_WAIT_FOR_RECONCILE)
+- Deleting resources is **destructive** and breaks status propagation chains, especially with Kro resource graphs that depend on child resource status (ID: EKS_DELETE_BREAKS_STATUS)
+- After making changes like adding IAMRoleSelectors, wait 15-30 minutes for reconciliation before investigating further (ID: EKS_RECONCILE_WAIT_TIME)
+- If a resource appears stuck, investigate via events, logs, and status conditions - do NOT delete unless explicitly required for cleanup (ID: EKS_INVESTIGATE_DONT_DELETE)
+- The only acceptable deletions are: (1) Removing finalizers from stuck resources during controlled cleanup, (2) Cleaning up test resources at end of validation, (3) Explicitly requested by user (ID: EKS_DELETE_ONLY_WHEN_REQUIRED)
+
 ### ArgoCD CLI Authentication
 
 - If the `argocd` CLI returns authentication errors or token expired messages, run the `argocd-refresh-token` bash function to obtain a new 12-hour token (ID: EKS_ARGOCD_REFRESH_TOKEN)

gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml

@@ -3,9 +3,8 @@ kind: ResourceGraphDefinition
 metadata:
   name: eksclusterwithvpc.kro.run
   annotations:
-    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true
+    argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true,Replace=false
     argocd.argoproj.io/sync-wave: "-1"
-    argocd.argoproj.io/sync-options: Replace=false
 spec:
   schema:
     apiVersion: v1alpha1
@@ -65,9 +64,9 @@ spec:
         enable_kyverno: string | default="true"
         enable_kyverno_policies: string | default="true"
         enable_kyverno_policy_reporter: string | default="true"
-        enable_cni_metrics_helper: string | default="false"
-        enable_prometheus_node_exporter: string | default="false"
-        enable_kube_state_metrics: string | default="false"
+        enable_cni_metrics_helper: string | default="true"
+        enable_prometheus_node_exporter: string | default="true"
+        enable_kube_state_metrics: string | default="true"
         enable_opentelemetry_operator: string | default="false"
         enable_cert_manager: string | default="true"
         enable_aws_efs_csi_driver: string | default="true"
@@ -131,7 +130,7 @@ spec:
           privateSubnet2Cidr: ${schema.spec.cidr.privateSubnet2Cidr}
   - id: eks
     readyWhen:
-    - ${eks.status.state == "ACTIVE"}
+    - ${eks.status.clusterState == "ACTIVE"}
     template:
       apiVersion: kro.run/v1alpha1
       kind: EksCluster

gitops/addons/charts/multi-acct/templates/eks-capabilities-rbac.yaml

@@ -51,6 +51,10 @@ rules:
 - apiGroups: ["kro.run"]
   resources: ["*"]
   verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+# ACK Secrets Manager
+- apiGroups: ["secretsmanager.services.k8s.aws"]
+  resources: ["*"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

platform/backstage/templates/eks-cluster-template/template.yaml

@@ -85,10 +85,7 @@ spec:
           type: string
           default: platform-on-eks-workshop
           ui:widget: hidden
-        ingressDomainName:
-          type: string
-          default: <ingress_domain_name>
-          ui:widget: hidden
+
     - title: Git Repository URLs
       properties:
         addonsRepoUrl:
@@ -183,6 +180,11 @@ spec:
           type: boolean
           description: Enable Cert Manager addon
           default: true
+        enableAwsEfsCsiDriver:
+          title: Enable AWS EFS CSI Driver
+          type: boolean
+          description: Enable AWS EFS CSI Driver addon
+          default: false
   steps:
     - id: fetchSystem
       name: Fetch System
@@ -223,11 +225,11 @@ spec:
               - url: https://${{ steps['fetchSystem'].output.entity.spec.argocd_hostname }}/applications/argocd/clusters
                 title: ArgoCD App URL
                 icon: externalLink
-              - url: https://${{ steps['fetchSystem'].output.entity.spec.gitlab_hostname }}/${{ steps['fetchSystem'].output.entity.spec.gituser }}/${{ parameters.repoName }}/-/blob/main/fleet/kro-values/tenants/tenant1/kro-clusters/values.yaml
+              - url: https://${{ steps['fetchSystem'].output.entity.spec.gitlab_hostname }}/${{ steps['fetchSystem'].output.entity.spec.gituser }}/${{ parameters.repoName }}/-/blob/main/gitops/fleet/kro-values/tenants/${{ parameters.tenant }}/kro-clusters/values.yaml
                 title: Git Repo URL
                 icon: github
           spec:
-            owner: guests
+            owner: user:guest
             lifecycle: experimental
             type: service
     # - id: mergeConfig
@@ -272,7 +274,7 @@ spec:
           - Added Backstage component definition
         sourcePath: ./repo
         targetPath: .
-        branchName: cluster-${{ parameters.clusterName }}-${{ parameters.environment }}
+        branchName: cluster-${{ parameters.clusterName }}
         removeSourceBranch: true
         commitAction: auto
     - id: register

platform/infra/terraform/scripts/utils.sh

@@ -430,7 +430,11 @@ update_backstage_defaults() {
   CATALOG_INFO_PATH="${GIT_ROOT_PATH}/platform/backstage/templates/catalog-info.yaml"
 
   # Get admin role name from current AWS context
-  ADMIN_ROLE_NAME=$(aws sts get-caller-identity --query 'Arn' --output text | sed 's|.*assumed-role/||' | sed 's|/.*||')
+  # Use WSParticipantRole (Workshop Studio standard role). Fallback to caller identity for self-paced deployments.
+  ADMIN_ROLE_NAME=$(aws iam list-roles --query 'Roles[?contains(RoleName,`WSParticipantRole`)].RoleName' --output text 2>/dev/null)
+  if [[ -z "$ADMIN_ROLE_NAME" ]]; then
+    ADMIN_ROLE_NAME=$(aws sts get-caller-identity --query 'Arn' --output text | sed 's|.*assumed-role/||' | sed 's|/.*||')
+  fi
 
   # Get ArgoCD URL from EKS capability if available
   ARGOCD_SERVER_URL=$(aws eks describe-capability --cluster-name ${CLUSTER_NAME:-${RESOURCE_PREFIX}-hub} --capability-name argocd --query 'capability.configuration.argoCd.serverUrl' --output text 2>/dev/null || echo "")

scripts/validation/backstage-auth.sh

@@ -89,7 +89,7 @@ backstage_scaffolder() {
 }
 
 # When sourced, export BS_TOKEN for direct use
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
+if [[ "${BASH_SOURCE[0]:-}" == "${0}" ]] || [[ "${ZSH_EVAL_CONTEXT:-}" == "toplevel" ]]; then
   backstage_get_token
 else
   export BS_TOKEN