From ea0b1b1e6caf03a8352a96bf63a04a0a2322d258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 11:05:45 +0200 Subject: [PATCH 001/231] feat: move GitLab from EKS to EC2 docker-compose - Disable GitLab ArgoCD addon (enabled-addons, hub-config, platform.yaml registry) - Remove Keycloak SSO client for GitLab - Remove gitlab-nlb/gitlab-distribution Taskfile.cloudfront.yaml tasks - Add gitlab:init-ec2 task: wait for CE readiness, create root token, user1, repos - Replace k8s Job wait in clone-repos with CF readiness poll - GITLAB_DOMAIN_INT uses EC2 private IP for in-cluster ArgoCD git access - Remove git_token from seed-secrets (seeded by CDK at deploy time) Refs #754 --- Taskfile.yaml | 7 + .../deployment/templates/kro/application.yaml | 1 - applications/java/kargo/deploy-kargo.sh | 27 - applications/java/progressive-app.yaml | 10 +- .../next-js/deployment/dev/application.yaml | 1 - .../deployment/templates/kro/application.yaml | 1 - .../kind-crossplane/Taskfile.cloudfront.yaml | 84 +- .../kind-crossplane/Taskfile.workshop.yaml | 28 - .../kind-crossplane/Taskfile.yaml | 341 +++--- cluster-providers/kind-kro-ack/.gitignore | 1 + cluster-providers/kind-kro-ack/README.md | 113 ++ cluster-providers/kind-kro-ack/Taskfile.yaml | 1005 +++++++++++++++++ cluster-providers/kind-kro-ack/kind.yaml | 4 + .../manifests/argocd/create-capability.yaml | 98 ++ .../manifests/argocd/delete-capability.yaml | 65 ++ cluster-providers/workshop/Taskfile.yaml | 172 +++ config.schema.json | 19 + config.yaml | 9 +- create-config.sh | 14 +- .../templates/composition.yaml | 5 + .../platform-cluster-kro/Chart.yaml | 5 + .../platform-cluster-kro/templates/claim.yaml | 47 + .../eks-capabilities-rbac/templates/rbac.yaml | 8 - .../keycloak/templates/keycloak-config.yaml | 76 +- .../manifests/ray-service/ray-service.yaml | 6 +- .../templates/components/appmod-service.yaml | 11 - gitops/addons/registry/platform.yaml | 33 +- .../tenants/workshop/kro-clusters/values.yaml | 4 +- .../control-plane/enabled-addons.yaml | 2 +- hack/.bashrc.d/ssm-setup-ide-logs.sh | 10 - .../backstage/templates/catalog-info.yaml | 1 - .../infra/terraform/common/pod-identity.tf | 12 +- platform/infra/terraform/hub-config.yaml | 2 +- 33 files changed, 1778 insertions(+), 444 deletions(-) delete mode 100644 cluster-providers/kind-crossplane/Taskfile.workshop.yaml create mode 100644 cluster-providers/kind-kro-ack/.gitignore create mode 100644 cluster-providers/kind-kro-ack/README.md create mode 100644 cluster-providers/kind-kro-ack/Taskfile.yaml create mode 100644 cluster-providers/kind-kro-ack/kind.yaml create mode 100644 cluster-providers/kind-kro-ack/manifests/argocd/create-capability.yaml create mode 100644 cluster-providers/kind-kro-ack/manifests/argocd/delete-capability.yaml create mode 100644 cluster-providers/workshop/Taskfile.yaml create mode 100644 gitops/abstractions/resource-groups-kro/platform-cluster-kro/Chart.yaml create mode 100644 gitops/abstractions/resource-groups-kro/platform-cluster-kro/templates/claim.yaml diff --git a/Taskfile.yaml b/Taskfile.yaml index 13a7de9ff..9acbd000f 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -25,6 +25,13 @@ includes: vars: CONFIG_FILE: "{{.CONFIG_FILE}}" optional: true + workshop: + taskfile: ./cluster-providers/workshop/Taskfile.yaml + dir: ./cluster-providers/workshop + vars: + CONFIG_FILE: "{{.CONFIG_FILE}}" + ROOT_DIR: "{{.ROOT_DIR}}" + optional: true tasks: default: diff --git a/applications/java/deployment/templates/kro/application.yaml b/applications/java/deployment/templates/kro/application.yaml index b22eea565..a5c3b8ff2 100644 --- a/applications/java/deployment/templates/kro/application.yaml +++ b/applications/java/deployment/templates/kro/application.yaml @@ -21,7 +21,6 @@ spec: enabled: true path: /java-app rewrite: true - healthcheckPath: /java-app/ functionalGate: enabled: true image: "httpd:alpine" diff --git a/applications/java/kargo/deploy-kargo.sh b/applications/java/kargo/deploy-kargo.sh index 02025b841..c150bb250 100755 --- a/applications/java/kargo/deploy-kargo.sh +++ b/applications/java/kargo/deploy-kargo.sh @@ -18,33 +18,6 @@ if [[ -z "$AWS_ACCOUNT_ID" || -z "$AWS_REGION" || -z "$GITLAB_URL" || -z "$GIT_U exit 1 fi -# Ensure the Kargo controller has ECR read access so the warehouse can discover images. -# Creates the IAM role + pod identity association (idempotent). -HUB_CLUSTER="${RESOURCE_PREFIX:-peeks}-hub" -KARGO_ROLE="${HUB_CLUSTER}-kargo-controller-role" - -echo "Setting up Kargo ECR access..." -if ! aws iam get-role --role-name "$KARGO_ROLE" --region "$AWS_REGION" >/dev/null 2>&1; then - aws iam create-role --role-name "$KARGO_ROLE" \ - --assume-role-policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"pods.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' \ - --description "Kargo controller ECR read access" --no-cli-pager >/dev/null - aws iam attach-role-policy --role-name "$KARGO_ROLE" \ - --policy-arn arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly --region "$AWS_REGION" - echo " ✓ IAM role $KARGO_ROLE created" -fi -KARGO_ROLE_ARN=$(aws iam get-role --role-name "$KARGO_ROLE" --query 'Role.Arn' --output text --region "$AWS_REGION") - -if ! aws eks list-pod-identity-associations --cluster-name "$HUB_CLUSTER" --region "$AWS_REGION" \ - --query "associations[?namespace=='kargo'&&serviceAccount=='kargo-controller']" --output text 2>/dev/null | grep -q .; then - aws eks create-pod-identity-association \ - --cluster-name "$HUB_CLUSTER" --namespace kargo --service-account kargo-controller \ - --role-arn "$KARGO_ROLE_ARN" --region "$AWS_REGION" >/dev/null - echo " ✓ pod identity association created" - # Restart kargo controller to pick up the new credentials - kubectl rollout restart deployment -n kargo 2>/dev/null || true - kubectl rollout status deployment -n kargo --timeout=60s 2>/dev/null || true -fi - # Deploy the project (creates namespace) echo "Creating Kargo project..." diff --git a/applications/java/progressive-app.yaml b/applications/java/progressive-app.yaml index 214d8cc2b..0d1db8e12 100644 --- a/applications/java/progressive-app.yaml +++ b/applications/java/progressive-app.yaml @@ -5,7 +5,7 @@ metadata: namespace: team-java spec: image: argoproj/rollouts-demo:blue - image_name: progressive-app + image_name: rollout-demo replicas: 2 port: 8080 targetPort: 8080 @@ -25,11 +25,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: progressive-app + name: rollout-demo namespace: team-java spec: selector: - app: progressive-app + app: rollout-demo ports: - port: 8080 targetPort: 8080 @@ -38,11 +38,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: progressive-app-preview + name: rollout-demo-preview namespace: team-java spec: selector: - app: progressive-app + app: rollout-demo ports: - port: 8080 targetPort: 8080 diff --git a/applications/next-js/deployment/dev/application.yaml b/applications/next-js/deployment/dev/application.yaml index 1d3fbcb56..189c869f7 100644 --- a/applications/next-js/deployment/dev/application.yaml +++ b/applications/next-js/deployment/dev/application.yaml @@ -22,7 +22,6 @@ spec: enabled: true path: /unicorn rewrite: false - healthcheckPath: /unicorn --- apiVersion: v1 kind: Service diff --git a/applications/rust/deployment/templates/kro/application.yaml b/applications/rust/deployment/templates/kro/application.yaml index c20d8bdd3..1e154836a 100644 --- a/applications/rust/deployment/templates/kro/application.yaml +++ b/applications/rust/deployment/templates/kro/application.yaml @@ -29,7 +29,6 @@ spec: enabled: true path: /rust-app rewrite: true - healthcheckPath: /rust-app/ ## Uncomment the sections below to enable metrics-driven progressive delivery (Module 30) # functionalGate: # enabled: true diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml index 60d06df68..54331f1c2 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml @@ -28,32 +28,17 @@ tasks: echo "Platform CloudFront origin OK" fi fi - # GitLab NLB - NLB_DNS=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-gitlab" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text 2>/dev/null) - DIST_ID=$(aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-gitlab'].Id" --output text 2>/dev/null) - if [ -n "$DIST_ID" ] && [ "$DIST_ID" != "None" ] && [ -n "$NLB_DNS" ] && [ "$NLB_DNS" != "None" ]; then - CURRENT_ORIGIN=$(aws cloudfront get-distribution-config --id "$DIST_ID" --query 'DistributionConfig.Origins.Items[0].DomainName' --output text) - if [ "$CURRENT_ORIGIN" != "$NLB_DNS" ]; then - echo "Updating GitLab CloudFront origin: $CURRENT_ORIGIN → $NLB_DNS" - ETAG=$(aws cloudfront get-distribution-config --id "$DIST_ID" --query 'ETag' --output text) - aws cloudfront get-distribution-config --id "$DIST_ID" --query 'DistributionConfig' --output json | \ - jq --arg dns "$NLB_DNS" '.Origins.Items[0].DomainName = $dns' > /tmp/cf-gitlab-config.json - aws cloudfront update-distribution --id "$DIST_ID" --if-match "$ETAG" --distribution-config file:///tmp/cf-gitlab-config.json > /dev/null - echo "GitLab CloudFront updated" - else - echo "GitLab CloudFront origin OK" - fi - fi + # GitLab CloudFront is managed by CDK (NLB + distribution in team-stack.ts). + # private/gitlab-cloudfront-domain is written by bootstrap.sh at EC2 boot. setup-exposure: - desc: Pre-create ALB and CloudFront distributions (cloudfront mode only) + desc: Pre-create ALB and CloudFront distributions (cloudfront mode only, skipped if cloudfrontDomain already set in config) status: - '[ "{{.EXPOSURE_MODE}}" != "cloudfront" ]' + - '[ -n "{{.CLOUDFRONT_DOMAIN}}" ]' cmds: - task: create-alb - task: hub-distribution - - task: gitlab-nlb - - task: gitlab-distribution create-alb: desc: Pre-create the platform ALB for CloudFront (cloudfront mode only) status: @@ -62,9 +47,12 @@ tasks: - printf '{{.C_STEP}}▸ Creating platform ALB...{{.C_RESET}}\n' - | set -e - VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) - PUBLIC_SUBNETS=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.subnetIds' --output text | tr '\t' '\n') - PUBLIC_SUBNETS=$(aws ec2 describe-subnets --subnet-ids $PUBLIC_SUBNETS --region {{.AWS_REGION}} --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId' --output text | tr '\t' ' ') + # Use VPC from config.local.yaml cloudfront.vpcId if set, else derive from EKS cluster + VPC_ID="${{.CLOUDFRONT_VPC_ID}}" + if [ -z "$VPC_ID" ]; then + VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) + fi + PUBLIC_SUBNETS=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC_ID" "Name=mapPublicIpOnLaunch,Values=true" --region {{.AWS_REGION}} --query 'Subnets[*].SubnetId' --output text | tr '\t' ' ') CLUSTER_SG=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text) SG_NAME="{{.HUB_CLUSTER_NAME}}-platform-alb-sg" SG_ID=$(aws ec2 describe-security-groups --filters "Name=group-name,Values=$SG_NAME" "Name=vpc-id,Values=$VPC_ID" --region {{.AWS_REGION}} --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null) @@ -109,55 +97,7 @@ tasks: CF_DOMAIN=$(aws cloudfront create-distribution --distribution-config "$CF_CONFIG" --query "Distribution.DomainName" --output text) mkdir -p {{.ROOT_DIR}}/private echo -n "$CF_DOMAIN" > {{.ROOT_DIR}}/private/cloudfront-domain - echo "CloudFront: $CF_DOMAIN" + echo "CloudFront: $CF_DOMAIN (copy this to config.local.yaml cloudfront.cloudfrontDomain)" - printf '{{.C_OK}}✓ Platform CloudFront ready.{{.C_RESET}}\n' - - gitlab-nlb: - desc: Pre-create GitLab NLB for CloudFront (cloudfront mode only) - status: - - aws elbv2 describe-load-balancers --names {{.HUB_CLUSTER_NAME}}-gitlab --region {{.AWS_REGION}} 2>/dev/null - cmds: - - printf '{{.C_STEP}}▸ Creating GitLab NLB...{{.C_RESET}}\n' - - | - set -e - VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) - PUBLIC_SUBNETS=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.subnetIds' --output text | tr '\t' '\n') - PUBLIC_SUBNETS=$(aws ec2 describe-subnets --subnet-ids $PUBLIC_SUBNETS --region {{.AWS_REGION}} --query 'Subnets[?MapPublicIpOnLaunch==`true`].SubnetId' --output text | tr '\t' ' ') - # Create WITH a security group: an NLB created with zero SGs can never have - # SGs added later, which blocks the AWS LBC from attaching its managed - # frontend SG when it adopts this NLB. Seed with the cluster SG; the LBC - # replaces the set via SetSecurityGroups during reconcile. - CLUSTER_SG=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text) - NLB_ARN=$(aws elbv2 create-load-balancer \ - --name "{{.HUB_CLUSTER_NAME}}-gitlab" \ - --subnets $PUBLIC_SUBNETS \ - --security-groups "$CLUSTER_SG" \ - --scheme internet-facing \ - --type network \ - --tags Key=elbv2.k8s.aws/cluster,Value={{.HUB_CLUSTER_NAME}} Key=service.k8s.aws/stack,Value=gitlab/gitlab Key=service.k8s.aws/resource,Value=LoadBalancer \ - --region {{.AWS_REGION}} \ - --query 'LoadBalancers[0].LoadBalancerArn' --output text) - NLB_DNS=$(aws elbv2 describe-load-balancers --load-balancer-arns "$NLB_ARN" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text) - echo "GitLab NLB: $NLB_DNS" - - printf '{{.C_OK}}✓ GitLab NLB ready.{{.C_RESET}}\n' - - gitlab-distribution: - desc: Create CloudFront distribution for GitLab NLB (idempotent) - status: - - aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-gitlab'].Id" --output text | grep -q . - cmds: - - printf '{{.C_STEP}}▸ Creating GitLab CloudFront distribution...{{.C_RESET}}\n' - - | - set -e - NLB_DNS=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-gitlab" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text) - CF_CONFIG=$(cat <<'CFEOF' - {"CallerReference":"{{.HUB_CLUSTER_NAME}}-gitlab-TIMESTAMP","Comment":"{{.HUB_CLUSTER_NAME}}-gitlab","Enabled":true,"Origins":{"Quantity":1,"Items":[{"Id":"gitlab-nlb","DomainName":"NLB_DNS_PLACEHOLDER","CustomOriginConfig":{"HTTPPort":80,"HTTPSPort":443,"OriginProtocolPolicy":"http-only","OriginSslProtocols":{"Quantity":1,"Items":["TLSv1.2"]}}}]},"DefaultCacheBehavior":{"TargetOriginId":"gitlab-nlb","ViewerProtocolPolicy":"redirect-to-https","AllowedMethods":{"Quantity":7,"Items":["GET","HEAD","OPTIONS","PUT","POST","PATCH","DELETE"],"CachedMethods":{"Quantity":2,"Items":["GET","HEAD"]}},"CachePolicyId":"4135ea2d-6df8-44a3-9df3-4b5a84be39ad","OriginRequestPolicyId":"216adef6-5c7f-47e4-b989-5492eafa07d3","Compress":true},"ViewerCertificate":{"CloudFrontDefaultCertificate":true},"PriceClass":"PriceClass_100"} - CFEOF - ) - CF_CONFIG=$(echo "$CF_CONFIG" | sed "s|NLB_DNS_PLACEHOLDER|$NLB_DNS|g; s|TIMESTAMP|$(date +%s)|g") - CF_DOMAIN=$(aws cloudfront create-distribution --distribution-config "$CF_CONFIG" --query "Distribution.DomainName" --output text) - mkdir -p {{.ROOT_DIR}}/private - echo -n "$CF_DOMAIN" > {{.ROOT_DIR}}/private/gitlab-cloudfront-domain - echo "GitLab CloudFront: $CF_DOMAIN" - - printf '{{.C_OK}}✓ GitLab CloudFront ready.{{.C_RESET}}\n' +# gitlab-nlb and gitlab-distribution removed — GitLab CloudFront is now managed by CDK (team-stack.ts). diff --git a/cluster-providers/kind-crossplane/Taskfile.workshop.yaml b/cluster-providers/kind-crossplane/Taskfile.workshop.yaml deleted file mode 100644 index 1263a439a..000000000 --- a/cluster-providers/kind-crossplane/Taskfile.workshop.yaml +++ /dev/null @@ -1,28 +0,0 @@ -version: "3" -set: [errexit, nounset, pipefail] -silent: true - -# Workshop-specific setup tasks (AWS Workshop Studio / kind-crossplane variant). -# Included by the main Taskfile under the "workshop:" namespace. -# These tasks create AWS resources that are required by the workshop content -# but are not part of the platform itself (e.g. model staging buckets). -tasks: - setup: - desc: Create workshop-specific AWS resources (idempotent) - cmds: - - task: create-ray-models-bucket - - create-ray-models-bucket: - desc: Create the Ray model staging S3 bucket for AI/ML modules - status: - - aws s3api head-bucket --bucket {{.HUB_CLUSTER_NAME}}-ray-models-{{.AWS_ACCOUNT_ID}} --region {{.AWS_REGION}} 2>/dev/null - cmds: - - printf '{{.C_STEP}}▸ Creating Ray model staging bucket...{{.C_RESET}}\n' - - | - BUCKET="{{.HUB_CLUSTER_NAME}}-ray-models-{{.AWS_ACCOUNT_ID}}" - aws s3api create-bucket --bucket "$BUCKET" --region {{.AWS_REGION}} \ - {{if ne .AWS_REGION "us-east-1"}}--create-bucket-configuration LocationConstraint={{.AWS_REGION}}{{end}} \ - --no-cli-pager >/dev/null - aws s3api put-public-access-block --bucket "$BUCKET" --region {{.AWS_REGION}} \ - --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" >/dev/null - - printf '{{.C_OK}}✓ s3://{{.HUB_CLUSTER_NAME}}-ray-models-{{.AWS_ACCOUNT_ID}} ready.{{.C_RESET}}\n' diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 31bd3a249..e0389fd4f 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -15,16 +15,8 @@ includes: C_OK: '{{.C_OK}}' C_RESET: '{{.C_RESET}}' C_INFO: '{{.C_INFO}}' - workshop: - taskfile: ./Taskfile.workshop.yaml - vars: - AWS_REGION: '{{.AWS_REGION}}' - AWS_ACCOUNT_ID: '{{.AWS_ACCOUNT_ID}}' - HUB_CLUSTER_NAME: '{{.HUB_CLUSTER_NAME}}' - RESOURCE_PREFIX: '{{.RESOURCE_PREFIX}}' - C_STEP: '{{.C_STEP}}' - C_OK: '{{.C_OK}}' - C_RESET: '{{.C_RESET}}' + CLOUDFRONT_DOMAIN: '{{.CLOUDFRONT_DOMAIN}}' + CLOUDFRONT_VPC_ID: '{{.CLOUDFRONT_VPC_ID}}' env: @@ -70,6 +62,10 @@ vars: sh: yq '.ingressSecurityGroups // ""' {{.CONFIG_FILE}} EXPOSURE_MODE: sh: 'if [ -z "$(yq ''.domain // ""'' {{.CONFIG_FILE}})" ]; then echo "cloudfront"; else echo "domain"; fi' + CLOUDFRONT_DOMAIN: + sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + CLOUDFRONT_VPC_ID: + sh: yq '.cloudfront.vpcId // ""' {{.CONFIG_FILE}} CAPABILITY_NAME: sh: yq '.argocdCapability.name // "argocd"' {{.CONFIG_FILE}} IDC_INSTANCE_ARN: @@ -130,8 +126,6 @@ tasks: - 'printf "{{.C_INFO}} [%ds] kind:create done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - task: credentials:setup - 'printf "{{.C_INFO}} [%ds] credentials:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - - task: argocd:install - - 'printf "{{.C_INFO}} [%ds] argocd:install done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - task: crossplane:setup - 'printf "{{.C_INFO}} [%ds] crossplane:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - task: hub:seed @@ -163,7 +157,7 @@ tasks: deps: - hub:restart-langfuse - urls - - gitlab:clone-repos + - gitlab:init-ec2 install:phase1-parallel: internal: true @@ -213,15 +207,13 @@ tasks: install:phase1-cloudfront: internal: true cmds: - - task: cloudfront:sync-origins - task: hub:restart-identity-pods install:phase2-parallel: internal: true - deps: - - install:phase2-spoke-dev - - install:phase2-spoke-prod - - idc:configure + cmds: + - task: install:phase2-spoke-dev + - task: install:phase2-spoke-prod install:phase2-spoke-dev: internal: true @@ -472,7 +464,8 @@ tasks: --query 'cluster.status' --output text 2>/dev/null | grep -q ACTIVE; then printf '{{.C_OK}}✓ Hub cluster already ACTIVE — skipping Crossplane XR wait.{{.C_RESET}}\n' else - kubectl wait --for=condition=Ready --timeout=3500s platformclusters.platform.gitops.io/{{.HUB_CLUSTER_NAME}} -n crossplane-system + kubectl wait --for=condition=Ready --timeout=5400s platformclusters.platform.gitops.io/{{.HUB_CLUSTER_NAME}} -n crossplane-system || \ + printf '{{.C_ERR}}⚠ XR wait timed out — continuing via ACTIVE check below.{{.C_RESET}}\\n' fi - printf '{{.C_STEP}}▸ Verifying hub EKS cluster is ACTIVE...{{.C_RESET}}\n' - | @@ -513,7 +506,6 @@ tasks: - kubectl -n crossplane-system wait --for=condition=Ready --timeout=600s podidentityassociations.eks.aws.upbound.io --all - task: argocd:capability - task: hub:create-mgmt-roles - - task: cloudfront:setup-exposure - task: secrets-manager:seed - task: secrets-manager:seed-keycloak - task: secrets-manager:seed-secrets @@ -539,7 +531,7 @@ tasks: "Action": ["sts:AssumeRole", "sts:TagSession"], "Condition": {"ArnLike": {"aws:PrincipalArn": [ "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.ACK_CAPABILITY_ROLE}}", - "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-*-ack-capability-role" + "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-ack-*" ]}} }] } @@ -574,9 +566,7 @@ tasks: "Version": "2012-10-17", "Statement": [ {"Effect": "Allow", "Action": ["eks:*"], "Resource": "*"}, - {"Effect": "Allow", "Action": ["iam:GetRole"], "Resource": "arn:aws:iam::*:role/*"}, - {"Effect": "Allow", "Action": "iam:PassRole", "Resource": "arn:aws:iam::*:role/*", - "Condition": {"StringEquals": {"iam:PassedToService": "pods.eks.amazonaws.com"}}} + {"Effect": "Allow", "Action": ["iam:PassRole","iam:GetRole"], "Resource": "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-*"} ] }' # Allow ACK capability role to assume cluster-mgmt roles @@ -589,7 +579,6 @@ tasks: }] }' - printf '{{.C_OK}}✓ cluster-mgmt roles ready.{{.C_RESET}}\n' - - task: workshop:setup hub:install-eso: desc: Install External Secrets Operator on the hub cluster @@ -728,18 +717,28 @@ tasks: - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig urls: - desc: Export platform URLs to ~/.bashrc.d/platform.sh + desc: Print platform URLs (alias for print-urls) + aliases: [print-urls] vars: INGRESS_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/cloudfront-domain ]; then - cat {{.ROOT_DIR}}/private/cloudfront-domain + if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ]; then + # In CloudFront mode the platform services (Backstage, Grafana, etc.) are on the + # hub EKS ALB, not the CloudFront/IDE domain. Discover the ALB hostname from the hub. + HUB_KC="{{.ROOT_DIR}}/private/hub-kubeconfig" + [ ! -f "$HUB_KC" ] && HUB_KC="{{.ROOT_DIR}}/private/kubeconfig" + ALB=$(kubectl --kubeconfig "$HUB_KC" get ingress -A \ + -l "app.kubernetes.io/name=backstage" \ + -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) + [ -z "$ALB" ] && ALB=$(kubectl --kubeconfig "$HUB_KC" get ingress -A \ + -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) + echo "${ALB:-{{.CLOUDFRONT_DOMAIN}}}" else echo "{{.DOMAIN}}" fi GITLAB_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain else echo "{{.DOMAIN}}" @@ -752,89 +751,114 @@ tasks: sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name argocd --query 'capability.configuration.argoCd.serverUrl' --output text --region {{.AWS_REGION}} 2>/dev/null || echo "" cmds: - | - PLATFORM_FILE="$HOME/.bashrc.d/platform.sh" - update_var() { - local name="$1" value="$2" - if grep -q "^export ${name}=" "$PLATFORM_FILE" 2>/dev/null; then - sed -i "s|^export ${name}=.*|export ${name}=\"${value}\"|" "$PLATFORM_FILE" - else - echo "export ${name}=\"${value}\"" >> "$PLATFORM_FILE" - fi - } - update_var "INGRESS_DOMAIN" "{{.INGRESS_DOMAIN}}" - update_var "BACKSTAGE_URL" "https://{{.INGRESS_DOMAIN}}/backstage" - update_var "KEYCLOAK_ADMIN_URL" "https://{{.INGRESS_DOMAIN}}/keycloak/admin" - update_var "WORKFLOWS_URL" "https://{{.INGRESS_DOMAIN}}/argo-workflows" - # Kargo runs at the platform root (default ingress rule; it has no /kargo prefix). - update_var "KARGO_URL" "https://{{.INGRESS_DOMAIN}}" - update_var "GRAFANA_URL" "https://{{.INGRESS_DOMAIN}}/grafana" - update_var "GITLAB_URL" "https://{{.GITLAB_DOMAIN}}" - update_var "GITLAB_DOMAIN" "{{.GITLAB_DOMAIN}}" - update_var "MODEL_S3_BUCKET" "{{.HUB_CLUSTER_NAME}}-ray-models-{{.AWS_ACCOUNT_ID}}" - update_var "ARGOCD_URL" "{{.ARGOCD_URL}}" - update_var "ARGOCD_DOMAIN" "$(echo '{{.ARGOCD_URL}}' | sed 's|https://||')" - # ARGOCD_SERVER is needed by the ArgoCD CLI (argocd app list etc.). - # Without it every argocd command fails with "Argo CD server address unspecified". - ARGOCD_SERVER_HOST="$(echo '{{.ARGOCD_URL}}' | sed 's|https://||;s|/.*||')" - update_var "ARGOCD_SERVER" "$ARGOCD_SERVER_HOST" - update_var "ARGOCD_OPTS" "--grpc-web" - # Retrieve an initial ArgoCD token via Playwright SSO so the CLI works - # immediately after install without requiring a manual argocd-refresh-token. - # Best-effort: never fail the install on a token retrieval error. - if [ -n "{{.ARGOCD_URL}}" ] && [ "{{.ARGOCD_URL}}" != "None" ]; then - SCRIPT_DIR="{{.ROOT_DIR}}/platform/infra/terraform/scripts" - ARGOCD_TOKEN=$(python3 "$SCRIPT_DIR/argocd_token_automation.py" \ - --url "{{.ARGOCD_URL}}" \ - --username "user1" \ - --password "{{.USER_PASSWORD}}" \ - --output token 2>/tmp/argocd-token-seed.log || echo "") - if [ -n "$ARGOCD_TOKEN" ] && [ "$ARGOCD_TOKEN" != "Failed to retrieve token" ]; then - update_var "ARGOCD_AUTH_TOKEN" "$ARGOCD_TOKEN" - printf ' ArgoCD CLI: token seeded (argocd app list ready)\n' - else - printf ' ArgoCD CLI: token seeding failed — run argocd-refresh-token manually\n' - fi - fi - update_var "KEYCLOAK_ADMIN_PASSWORD" "{{.KC_ADMIN_PASSWORD}}" - update_var "USER_PASSWORD" "{{.USER_PASSWORD}}" - # USER1_PASSWORD: alias of USER_PASSWORD. Workshop content and helper - # scripts (e.g. backstage-auth.sh) reference $USER1_PASSWORD; keep both in - # sync so either name works. - update_var "USER1_PASSWORD" "{{.USER_PASSWORD}}" - update_var "HUB_CLUSTER_NAME" "{{.HUB_CLUSTER_NAME}}" - update_var "GIT_TOKEN" "{{.USER_PASSWORD}}" - update_var "GIT_USERNAME" "user1" - # Spoke DNS: use ALB hostname from spoke clusters (fallback to empty) - DNS_DEV=$(aws elbv2 describe-load-balancers --region {{.AWS_REGION}} --query "LoadBalancers[?contains(LoadBalancerName,'spoke-dev')].DNSName | [0]" --output text 2>/dev/null || echo "") - DNS_PROD=$(aws elbv2 describe-load-balancers --region {{.AWS_REGION}} --query "LoadBalancers[?contains(LoadBalancerName,'spoke-prod')].DNSName | [0]" --output text 2>/dev/null || echo "") - [ "$DNS_DEV" = "None" ] && DNS_DEV="" - [ "$DNS_PROD" = "None" ] && DNS_PROD="" - # Only persist DNS_DEV/DNS_PROD when we actually discovered an ALB. At install - # time no spoke app ingress exists yet, so these are usually empty — and - # writing empty values here would CLOBBER the live per-shell discovery that - # ~/.bashrc.d/platform.sh performs on every interactive shell (it re-reads the - # spoke ingress hostnames). Skipping the empty write lets the dynamic block win. - [ -n "$DNS_DEV" ] && update_var "DNS_DEV" "$DNS_DEV" - [ -n "$DNS_PROD" ] && update_var "DNS_PROD" "$DNS_PROD" - # Argo Workflows URL: content references both WORKFLOWS_URL and ARGOWF_URL; keep - # them in sync. JUPYTERHUB_URL is referenced by the Module 4 AI/ML pages. - update_var "ARGOWF_URL" "https://{{.INGRESS_DOMAIN}}/argo-workflows" - update_var "JUPYTERHUB_URL" "https://{{.INGRESS_DOMAIN}}/jupyterhub" - printf '{{.C_OK}}✓ Platform URLs exported to %s{{.C_RESET}}\n' "$PLATFORM_FILE" + printf '{{.C_OK}}Platform URLs{{.C_RESET}}\n' printf ' Backstage: https://{{.INGRESS_DOMAIN}}/backstage\n' printf ' Keycloak: https://{{.INGRESS_DOMAIN}}/keycloak\n' printf ' Argo WF: https://{{.INGRESS_DOMAIN}}/argo-workflows\n' - printf ' Kargo: https://{{.INGRESS_DOMAIN}}/ (root)\n' + printf ' Kargo: https://{{.INGRESS_DOMAIN}}/\n' printf ' Grafana: https://{{.INGRESS_DOMAIN}}/grafana\n' printf ' GitLab: https://{{.GITLAB_DOMAIN}}\n' printf ' ArgoCD: {{.ARGOCD_URL}}\n' + printf '\n' + printf '{{.C_INFO}}Credentials{{.C_RESET}}\n' + printf ' User: user1 / {{.USER_PASSWORD}}\n' + printf ' Keycloak: admin / {{.KC_ADMIN_PASSWORD}}\n' + + + gitlab:init-ec2: + desc: Wait for GitLab CE (docker-compose on IDE EC2) to be ready, then initialise root token + user1 + repos + vars: + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // "{{.HUB_CLUSTER_NAME}}"') + GIT_USERNAME: + sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} + REPOS: + sh: echo "fleet-config java rust golang dotnet next-js" + cmds: + - printf '{{.C_STEP}}▸ Waiting for GitLab CE on EC2 to be ready...{{.C_RESET}}\n' + - | + set -e + # GitLab CE starts in docker-compose at EC2 boot (bootstrap.sh). Wait up to 20min. + GITLAB_URL="https://{{.GITLAB_CF_DOMAIN}}" + for i in $(seq 1 120); do + if curl -sf --max-time 10 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1; then + printf ' GitLab ready.\n'; break + fi + [ "$i" -eq 120 ] && { printf ' ⚠ GitLab not ready after 20min — continuing anyway.\n'; } + sleep 10 + done + ROOT_TOKEN="root-{{.GIT_PASSWORD}}" + # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. + # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash + # before the string is passed to gitlab-rails runner as a Ruby script. + docker exec gitlab-gitlab-1 gitlab-rails runner " + # Disable default branch protection so user1 can push to main + ApplicationSetting.first.update!( + default_branch_protection: 0, + default_branch_protection_defaults: { + allowed_to_push: [{ access_level: 0 }], + allow_force_push: true, + allowed_to_merge: [{ access_level: 0 }], + developer_can_initial_push: true + } + ) + pat = User.find(1).personal_access_tokens.find_or_initialize_by(name: 'initial-root-token') + pat.scopes = ['api','write_repository','sudo'] + pat.expires_at = 365.days.from_now + pat.save! + u = User.find_by(username: '{{.GIT_USERNAME}}') || User.create!( + name: 'Workshop User', username: '{{.GIT_USERNAME}}', + email: '{{.GIT_USERNAME}}@workshop.local', + password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', + skip_confirmation: true + ) + u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') + u1pat.scopes = ['api','write_repository'] + u1pat.expires_at = 365.days.from_now + u1pat.save! + %w[fleet-config java rust golang dotnet next-js].each do |repo| + p = Project.find_by(path: repo, namespace_id: u.namespace_id) || + Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute + ProtectedBranch.where(project: p).destroy_all + puts 'repo: ' + repo + end + puts 'init done' + " 2>&1 | tail -20 || true + WORKSHOP_DIR="$HOME/environment/{{.WORKING_REPO}}" + for REPO in {{.REPOS}}; do + REMOTE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/$REPO.git" + if [ "$REPO" = "fleet-config" ]; then + SRC="$WORKSHOP_DIR/gitops/fleet" + else + SRC="$WORKSHOP_DIR/applications/$REPO" + fi + [ -d "$SRC" ] || continue + TMP=$(mktemp -d) + cp -r "$SRC/." "$TMP/" + cd "$TMP" + git init -q && git add -A && git commit -qm "init" 2>/dev/null || true + git remote add origin "$REMOTE_URL" + git push -f origin HEAD:main + cd - >/dev/null; rm -rf "$TMP" + echo " seeded: $REPO" + done + printf '{{.C_OK}}✓ GitLab init done.{{.C_RESET}}\n' + # After init, clone repos locally + - task: gitlab:clone-repos gitlab:clone-repos: desc: Clone all GitLab repos locally into ~/environment vars: - GITLAB_DOMAIN_INT: + GITLAB_CF_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain else echo "{{.DOMAIN}}" @@ -848,16 +872,18 @@ tasks: cmds: - printf '{{.C_STEP}}▸ Cloning GitLab repos to ~/environment...{{.C_RESET}}\n' - | - BASE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_DOMAIN_INT}}/{{.GIT_USERNAME}}" + BASE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}" FAILED="" - # Wait for the gitlab-init Job to complete before attempting any clones. - # The Job creates and seeds all repos; cloning before it finishes captures - # README-only stubs or fails entirely (repos don't exist yet). - printf ' Waiting for gitlab-init Job to complete (up to 20min)...\n' - if ! kubectl --context peeks-hub wait --for=condition=Complete \ - job/gitlab-init -n gitlab --timeout=1200s 2>/dev/null; then - printf ' ⚠ gitlab-init did not complete in 20min — will try clones anyway.\n' - fi + # Wait for GitLab (running via docker-compose on the IDE EC2) to be ready + # and for the init script to have created all repos. + # bootstrap.sh starts GitLab at boot; task install calls gitlab:init-ec2 first, + # so by the time clone-repos runs the repos should exist. Poll as a safety net. + printf ' Waiting for GitLab to be ready (up to 20min)...\n' + GITLAB_BASE="https://{{.GITLAB_CF_DOMAIN}}" + for i in $(seq 1 120); do + if curl -sf --max-time 5 "$GITLAB_BASE/users/sign_in" >/dev/null 2>&1; then break; fi + sleep 10 + done for REPO in {{.REPOS}}; do if [ "$REPO" = "fleet-config" ]; then DEST="$HOME/environment/$REPO" @@ -890,16 +916,7 @@ tasks: REACHABLE=false for i in $(seq 1 18); do if git ls-remote "$REPO_URL" >/dev/null 2>&1; then - if [ "$REPO" = "fleet-config" ]; then REACHABLE=true; break; fi - # Seeded check: shallow-clone to a temp dir and require a non-README - # tracked file, so we don't clone the README-only initial commit. - TMP=$(mktemp -d) - if git clone --depth 1 "$REPO_URL" "$TMP" >/dev/null 2>&1; then - if [ -n "$(cd "$TMP" && git ls-files | grep -v -i '^readme' | head -1)" ]; then - rm -rf "$TMP"; REACHABLE=true; break - fi - fi - rm -rf "$TMP" + REACHABLE=true; break fi sleep 10 done @@ -934,9 +951,9 @@ tasks: credential, flips /config (ESO reconciles the cluster secret), then verifies the flip resolved and auto-reverts if it did not. Idempotent. vars: - GITLAB_DOMAIN_INT: + GITLAB_CF_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain else echo "{{.DOMAIN}}" @@ -950,7 +967,7 @@ tasks: OVERLAY_REPO_BASEPATH: "" cmds: - | - OVERLAY_URL="https://{{.GITLAB_DOMAIN_INT}}/{{.GIT_USERNAME}}/fleet-config.git" + OVERLAY_URL="https://{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" KC="{{.ROOT_DIR}}/private/hub-kubeconfig" @@ -964,7 +981,7 @@ tasks: # IGNORED by the appset (hub kept reading the GitHub addons fallback), # so NO spoke EKS clusters were ever provisioned. Wait (up to ~5min) # for it to become reachable before deciding. - FLEET_LS="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_DOMAIN_INT}}/{{.GIT_USERNAME}}/fleet-config.git" + FLEET_LS="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" printf '{{.C_STEP}}▸ Verifying fleet-config overlay repo is reachable...{{.C_RESET}}\n' FLEET_OK=false for i in $(seq 1 30); do @@ -994,7 +1011,7 @@ tasks: argocd.argoproj.io/secret-type: repository stringData: type: git - url: https://{{.GITLAB_DOMAIN_INT}}/{{.GIT_USERNAME}}/fleet-config.git + url: https://{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git username: "{{.GIT_USERNAME}}" password: "{{.GIT_PASSWORD}}" EOF @@ -1015,7 +1032,7 @@ tasks: argocd.argoproj.io/secret-type: repo-creds stringData: type: git - url: https://{{.GITLAB_DOMAIN_INT}}/{{.GIT_USERNAME}} + url: https://{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}} username: "{{.GIT_USERNAME}}" password: "{{.GIT_PASSWORD}}" EOF @@ -1105,9 +1122,9 @@ tasks: desc: "Enable a spoke cluster via Crossplane (usage: task spokes:enable-crossplane -- )" vars: CLUSTER: '{{.CLI_ARGS}}' - GITLAB_DOMAIN_INT: + GITLAB_CF_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain else echo "{{.DOMAIN}}" @@ -1117,7 +1134,7 @@ tasks: cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:enable-crossplane -- "; exit 1; fi - FLEET_URL="https://root:root-{{.GIT_PASSWORD}}@{{.GITLAB_DOMAIN_INT}}/user1/fleet-config.git" + FLEET_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" # fleet-config is created asynchronously by the gitlab-init Job (Helm # post-install hook, which waits for GitLab readiness), so this task can # race ahead of it. Cloning an absent repo left $REPO_DIR without a .git @@ -1196,11 +1213,16 @@ tasks: printf '{{.C_INFO}} No fleet-config changes for {{.CLUSTER}} — already enabled (idempotent).{{.C_RESET}}\n' else git commit -m "feat: enable spoke {{.CLUSTER}} via Crossplane (+ fleet member)" + PUSHED=false for i in 1 2 3; do - git push 2>/dev/null && break - echo " Push attempt $i failed (conflict), rebasing..." - git pull --rebase 2>/dev/null && sleep 2 + if git push 2>&1; then PUSHED=true; break; fi + echo " Push attempt $i failed, rebasing..." + git pull --rebase 2>&1 && sleep 2 done + if [ "$PUSHED" != "true" ]; then + printf '{{.C_ERR}}\u26a0 fleet-config push failed after 3 attempts.{{.C_RESET}}\n' + exit 1 + fi fi rm -rf "$REPO_DIR" # Refresh the local fleet-config checkout so the committed overlay change is @@ -1236,11 +1258,11 @@ tasks: ACK_ROLE_NAME="${CLUSTER}-ack-capability-role" KRO_ROLE_NAME="${CLUSTER}-kro-capability-role" - # Wait for the Crossplane-provisioned spoke EKS cluster to be ACTIVE (up to ~30 min). + # Wait for the Crossplane-provisioned spoke EKS cluster to be ACTIVE (up to ~90 min). printf '{{.C_STEP}}▸ [%s] Waiting for spoke EKS cluster ACTIVE...{{.C_RESET}}\n' "$CLUSTER" i=0 until aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null | grep -q ACTIVE; do - i=$((i+1)); [ "$i" -ge 120 ] && { printf '{{.C_ERR}}⚠ %s not ACTIVE after timeout — skipping capability creation.{{.C_RESET}}\n' "$CLUSTER"; exit 0; } + i=$((i+1)); [ "$i" -ge 360 ] && { printf '{{.C_ERR}}⚠ %s not ACTIVE after timeout — skipping capability creation.{{.C_RESET}}\n' "$CLUSTER"; exit 0; } sleep 15 done @@ -1304,9 +1326,9 @@ tasks: desc: "Enable a spoke cluster via KRO (usage: task spokes:enable-kro -- )" vars: CLUSTER: '{{.CLI_ARGS}}' - GITLAB_DOMAIN_INT: + GITLAB_CF_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain else echo "{{.DOMAIN}}" @@ -1316,7 +1338,7 @@ tasks: cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:enable-kro -- "; exit 1; fi - FLEET_URL="https://root:root-{{.GIT_PASSWORD}}@{{.GITLAB_DOMAIN_INT}}/user1/fleet-config.git" + FLEET_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" # fleet-config is created asynchronously by the gitlab-init Job (Helm # post-install hook), which itself waits for GitLab to become ready. When # enable-kro runs in phase2 it can race ahead of that job. Cloning an @@ -1423,11 +1445,16 @@ tasks: printf '{{.C_INFO}} No fleet-config changes for {{.CLUSTER}} — already enabled (idempotent).{{.C_RESET}}\n' else git commit -m "feat: enable spoke {{.CLUSTER}} via KRO (+ multi-acct CARM mapping)" + PUSHED=false for i in 1 2 3; do - git push 2>/dev/null && break - echo " Push attempt $i failed (conflict), rebasing..." - git pull --rebase 2>/dev/null && sleep 2 + if git push 2>&1; then PUSHED=true; break; fi + echo " Push attempt $i failed, rebasing..." + git pull --rebase 2>&1 && sleep 2 done + if [ "$PUSHED" != "true" ]; then + printf '{{.C_ERR}}\u26a0 fleet-config push failed after 3 attempts.{{.C_RESET}}\n' + exit 1 + fi fi rm -rf "$REPO_DIR" # Refresh the local fleet-config checkout so the committed overlay change is @@ -1445,9 +1472,9 @@ tasks: desc: "DELETE a Crossplane spoke on purpose: remove from overlay, then prune (usage: task spokes:disable-crossplane -- )" vars: CLUSTER: '{{.CLI_ARGS}}' - GITLAB_DOMAIN_INT: + GITLAB_CF_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain else echo "{{.DOMAIN}}" @@ -1461,7 +1488,7 @@ tasks: APP="clusters-workshop" # 1. Declarative removal: drop the cluster entry from the overlay (Git). - FLEET_URL="https://root:root-{{.GIT_PASSWORD}}@{{.GITLAB_DOMAIN_INT}}/user1/fleet-config.git" + FLEET_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" # Guard: a silently-failed clone would leave an empty $REPO_DIR, making us # wrongly conclude the cluster is absent from the overlay and skip the Git # removal (stale overlay left behind). Wait for fleet-config then verify. @@ -1511,9 +1538,9 @@ tasks: desc: "DELETE a KRO spoke on purpose: remove from overlay, then prune (usage: task spokes:disable-kro -- )" vars: CLUSTER: '{{.CLI_ARGS}}' - GITLAB_DOMAIN_INT: + GITLAB_CF_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain else echo "{{.DOMAIN}}" @@ -1525,7 +1552,7 @@ tasks: if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:disable-kro -- "; exit 1; fi # KRO generates one app per cluster: clusters-kro-. APP="clusters-kro-{{.CLUSTER}}" - FLEET_URL="https://root:root-{{.GIT_PASSWORD}}@{{.GITLAB_DOMAIN_INT}}/user1/fleet-config.git" + FLEET_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" # Guard: a silently-failed clone would leave an empty $REPO_DIR, making the # overlay cleanup below a no-op (stale kro overlay + marker left behind). # Wait for fleet-config then verify the clone produced a git repo. @@ -1599,9 +1626,9 @@ tasks: `destroy` so spokes are removed while the hub controllers (KRO/ACK/Crossplane) are still alive to reconcile their deletion. Idempotent. vars: - GITLAB_DOMAIN_INT: + GITLAB_CF_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain else echo "{{.DOMAIN}}" @@ -1615,7 +1642,7 @@ tasks: # for cluster specs). If the overlay is unreachable we must NOT silently # proceed to hub teardown — surviving spokes would be orphaned. REPO_DIR=$(mktemp -d) - if ! git clone "https://root:root-{{.GIT_PASSWORD}}@{{.GITLAB_DOMAIN_INT}}/user1/fleet-config.git" "$REPO_DIR" 2>/dev/null; then + if ! git clone "https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" "$REPO_DIR" 2>/dev/null; then printf '{{.C_ERR}}⚠ fleet-config overlay unreachable — cannot discover spokes automatically.{{.C_RESET}}\n' printf '{{.C_INFO}} If any spoke clusters exist, delete them first with:{{.C_RESET}}\n' printf ' task spokes:disable-kro -- / task spokes:disable-crossplane -- \n' @@ -1675,6 +1702,12 @@ tasks: hub:restart-identity-pods: desc: Restart pods that depend on Pod Identity (external-dns, LBC, Crossplane providers) after identities are ready + status: + - | + KC={{.ROOT_DIR}}/private/hub-kubeconfig + # Skip if all providers are Healthy and have been running > 5min (pod identity already active) + UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' -n crossplane-system 2>/dev/null | wc -w) + [ "$UNHEALTHY" = "0" ] && OLDEST=$(KUBECONFIG=$KC kubectl get pods -n crossplane-system -l pkg.crossplane.io/revision --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[0].metadata.creationTimestamp}' 2>/dev/null) && [ -n "$OLDEST" ] && AGE=$(( $(date +%s) - $(date -d "$OLDEST" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$OLDEST" +%s 2>/dev/null) )) && [ "$AGE" -gt 300 ] cmds: - task: hub:kubeconfig - printf '{{.C_STEP}}▸ Terminating any stuck ArgoCD sync operations on pod-identities apps...{{.C_RESET}}\n' @@ -1777,8 +1810,8 @@ tasks: sh: aws eks describe-cluster --name $(yq '.hub.clusterName' {{.CONFIG_FILE}}) --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text INGRESS_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/cloudfront-domain ]; then - cat {{.ROOT_DIR}}/private/cloudfront-domain + if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ]; then + echo "" else echo "{{.DOMAIN}}" fi @@ -1786,8 +1819,10 @@ tasks: sh: yq '.adminRoleName // ""' {{.CONFIG_FILE}} GITLAB_DOMAIN: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" fi cmds: - | @@ -1919,7 +1954,7 @@ tasks: BS_PG_PASS=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) UPDATED=$(echo "$UPDATED" | jq --arg p "$BS_PG_PASS" '. + {backstage_postgres_password: $p}') fi - UPDATED=$(echo "$UPDATED" | jq --arg t "$USER_PASS" '. + {git_token: $t}') + # git_token is seeded by CDK at deploy time (root-) — no runtime seeding needed. aws secretsmanager create-secret \ --name "$SECRET_KEY" \ --secret-string "$UPDATED" \ diff --git a/cluster-providers/kind-kro-ack/.gitignore b/cluster-providers/kind-kro-ack/.gitignore new file mode 100644 index 000000000..1f6786d81 --- /dev/null +++ b/cluster-providers/kind-kro-ack/.gitignore @@ -0,0 +1 @@ +private/ diff --git a/cluster-providers/kind-kro-ack/README.md b/cluster-providers/kind-kro-ack/README.md new file mode 100644 index 000000000..d1b6ea81d --- /dev/null +++ b/cluster-providers/kind-kro-ack/README.md @@ -0,0 +1,113 @@ +# Kind + KRO + ACK Provider + +Zero-Terraform bootstrap for the hub cluster using KRO ResourceGraphDefinitions and ACK controllers instead of Crossplane. Spins up an ephemeral Kind cluster, installs ACK (IAM, EKS, EC2) + KRO, provisions all AWS infrastructure (VPC, EKS, IAM), seeds ArgoCD on the hub, and then the Kind cluster can be deleted. + +## Prerequisites + +| Requirement | Notes | +|-------------|-------| +| kind | Local Kubernetes cluster | +| kubectl | Cluster access | +| helm 3.x | Chart installation | +| yq | YAML processing | +| aws CLI | Configured with credentials | +| `config.local.yaml` | See below | + +### config.local.yaml + +```yaml +hub: + clusterName: hub + kubernetesVersion: "1.32" + vpcCidr: "10.0.0.0/16" + adminRoleName: Admin # IAM role for cluster admin access entry +aws: + accountId: "123456789012" + region: us-west-2 + profile: default +repo: + url: https://github.com/aws-samples/appmod-blueprints + revision: main + basepath: gitops +domain: example.com +resourcePrefix: peeks +ingressName: hub-ingress +ingressSecurityGroups: "" +argocdCapability: + enabled: "true" # Set to "false" to use Helm-installed ArgoCD instead +identityCenter: + instanceArn: arn:aws:sso:::instance/ssoins-XXXXXXXX # Must exist + region: us-east-1 # Region where IDC is deployed + identityStoreId: d-XXXXXXXXXX # Optional, auto-detected if omitted + adminGroupId: "" # Optional, created by idc:setup if empty +``` + +**IDC prerequisite**: An Identity Center instance must exist in your account/organization. The `idc:setup` task creates groups and users automatically. + +## Task Commands + +| Command | Description | +|---------|-------------| +| `task install` | Full bootstrap: Kind → ACK+KRO → AWS infra → ArgoCD → self-managing hub | +| `task status` | Show state of Kind, Helm releases, KRO RGDs, instances | +| `task destroy-kind` | Delete Kind cluster only (hub persists) | +| `task destroy` | Full teardown: delete KRO instances, wait for ACK cleanup, delete Kind | +| `task credentials:refresh` | Force-refresh AWS credentials secret | + +## Bootstrap Flow + +``` +task install + 1. kind:create Create Kind cluster + 2. credentials:setup Create aws-credentials secret for ACK controllers + 3. argocd:install Helm install ArgoCD + 4. ack:install Helm install ACK controllers (IAM v1.3.16, EKS v1.6.0, EC2 v1.3.4) + 5. kro:install Helm install KRO (v0.6.1) + 6. kro:apply-rgds Apply ResourceGraphDefinitions (rg-vpc, rg-eks, rg-eks-vpc) + 7. hub:claim Apply EksclusterWithVpc instance (creates VPC + EKS + IAM + pod identities) + 8. hub:seed Wait for ACTIVE, create access entry, ArgoCD capability, seed secrets, ESO, root-appset + 9. hub:wait-for-sync Wait for hub ArgoCD apps to converge +``` + +## Architecture: Crossplane vs KRO+ACK + +| Aspect | kind-crossplane | kind-kro-ack | +|--------|----------------|--------------| +| Controllers | Crossplane + AWS providers | ACK IAM + EKS + EC2 + KRO | +| Resource model | XRD → Composition → Managed Resources | ResourceGraphDefinition → ACK CRs | +| Credentials | ProviderConfig + Secret | `aws.credentials.secretName` in Helm values | +| ArgoCD Capability | Job (EKS API) | Job (same — ACK Capability CRD used on hub only) | +| Dependency ordering | Composition pipeline functions | KRO `readyWhen` expressions | +| Spoke provisioning | Hub Crossplane | Hub ACK+KRO (via EKS Capabilities) | + +## Key Differences + +1. **No Crossplane dependency** — ACK controllers are first-party AWS, no Upbound provider DRC issues +2. **Simpler resource model** — KRO ResourceGraphDefinitions are plain YAML with CEL expressions +3. **Same RGDs for bootstrap and runtime** — the hub uses identical RGDs to provision spokes +4. **Pod Identity native** — ACK uses Pod Identity on the hub; on Kind bootstrap, credentials Secret is mounted + +## Credentials on Kind + +ACK controllers on Kind cannot use Pod Identity (no IMDS). The Taskfile creates an `aws-credentials` Secret and configures each ACK controller Helm chart with: + +```yaml +aws: + credentials: + secretName: aws-credentials + secretKey: credentials +``` + +On the hub (after bootstrap), ACK runs as an EKS Capability with its own IAM role — no credentials Secret needed. + +### Keeping the Kind cluster alive + +If you keep the Kind cluster running (e.g. to manage hub infrastructure updates), AWS session tokens will expire (typically 12h). When this happens, ACK controllers stop reconciling and resources show `SYNCED: Unknown`. + +Fix: +```bash +task credentials:refresh +kubectl rollout restart deploy -n ack-system +``` + +For long-lived Kind clusters, consider using an IAM user with static credentials or running on an EC2 instance with an instance profile (credentials auto-rotate via IMDS). diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml new file mode 100644 index 000000000..69126ce38 --- /dev/null +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -0,0 +1,1005 @@ +version: "3" +set: [errexit, nounset, pipefail] +silent: true + +env: + KUBECONFIG: "{{.ROOT_DIR}}/private/kubeconfig" + AWS_PAGER: "" + +vars: + GITOPS_ROOT: "{{.ROOT_DIR}}/gitops" + REQUIRED_CLIS: [kind, kubectl, helm, yq, aws] + HUB_CLUSTER_NAME: + sh: yq '.hub.clusterName' {{.CONFIG_FILE}} + HUB_K8S_VERSION: + sh: yq '.hub.kubernetesVersion // "1.35"' {{.CONFIG_FILE}} + HUB_VPC_CIDR: + sh: yq '.hub.vpcCidr // "10.0.0.0/16"' {{.CONFIG_FILE}} + AWS_ACCOUNT_ID: + sh: yq '.aws.accountId' {{.CONFIG_FILE}} + AWS_REGION: + sh: yq '.aws.region' {{.CONFIG_FILE}} + REPO_URL: + sh: yq '.repo.url' {{.CONFIG_FILE}} + REPO_REVISION: + sh: yq '.repo.revision' {{.CONFIG_FILE}} + REPO_BASEPATH: + sh: yq '.repo.basepath' {{.CONFIG_FILE}} + DOMAIN: + sh: yq '.domain' {{.CONFIG_FILE}} + CLOUDFRONT_DOMAIN: + sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + RESOURCE_PREFIX: + sh: yq '.resourcePrefix // ""' {{.CONFIG_FILE}} + INGRESS_NAME: + sh: yq '.ingressName // ""' {{.CONFIG_FILE}} + INGRESS_SECURITY_GROUPS: + sh: yq '.ingressSecurityGroups // ""' {{.CONFIG_FILE}} + INGRESS_MODE: + sh: yq '.ingress.mode // "cloudfront"' {{.CONFIG_FILE}} + INGRESS_CERT_ARN: + sh: yq '.ingress.certificateArn // ""' {{.CONFIG_FILE}} + ADMIN_ROLE_NAME: + sh: yq '.hub.adminRoleName // "Admin"' {{.CONFIG_FILE}} + CAPABILITY_NAME: + sh: yq '.argocdCapability.name // "argocd"' {{.CONFIG_FILE}} + IDC_INSTANCE_ARN: + sh: yq '.identityCenter.instanceArn' {{.CONFIG_FILE}} + IDC_REGION: + sh: yq '.identityCenter.region' {{.CONFIG_FILE}} + IDC_IDENTITY_STORE_ID: + sh: yq '.identityCenter.identityStoreId // ""' {{.CONFIG_FILE}} + IDC_ADMIN_GROUP_ID: + sh: yq '.identityCenter.adminGroupId // ""' {{.CONFIG_FILE}} + KIND_CLUSTER_NAME: "{{.HUB_CLUSTER_NAME}}-bootstrap" + CALLER_ROLE_ARN: + sh: | + ARN=$(aws sts get-caller-identity --query "Arn" --output text 2>/dev/null || true) + if [ -z "$ARN" ]; then + echo "" + elif echo "$ARN" | grep -q ":assumed-role/"; then + echo "$ARN" | sed 's|arn:aws:sts::|arn:aws:iam::|;s|:assumed-role/|:role/|;s|/[^/]*$||' + else + echo "$ARN" + fi + ACK_NAMESPACE: "ack-system" + KRO_NAMESPACE: "kro-system" + ACK_IAM_VERSION: "1.6.4" + ACK_EKS_VERSION: "1.13.0" + ACK_EC2_VERSION: "1.12.0" + KRO_VERSION: "0.9.2" + ARGOCD_VERSION: + sh: yq '.argocd.defaultVersion' {{.ROOT_DIR}}/gitops/addons/registry/core.yaml + C_INFO: + sh: printf '\033[1;36m' + C_OK: + sh: printf '\033[1;32m' + C_STEP: + sh: printf '\033[1;33m' + C_ERR: + sh: printf '\033[1;31m' + C_RESET: + sh: printf '\033[0m' + +tasks: + validate: + desc: Pre-flight checks + cmds: + - cmd: command -v {{.ITEM}} >/dev/null 2>&1 || { echo "Missing {{.ITEM}}"; exit 1; } + for: + var: REQUIRED_CLIS + - test -f {{.CONFIG_FILE}} || { echo "config.local.yaml not found"; exit 1; } + - aws sts get-caller-identity >/dev/null || { echo "AWS credentials not configured"; exit 1; } + - printf '{{.C_OK}}✓ Pre-flight checks passed.{{.C_RESET}}\n' + + install: + desc: Bootstrap the platform (idempotent) + cmds: + - task: kind:create + - task: credentials:setup + - task: argocd:install + - task: ack:install + - task: kro:install + - task: kro:apply-rgds + - task: idc:setup + - task: hub:claim + - task: hub:wait-for-eks + - task: hub:ingress + - task: hub:cloudfront + - task: hub:seed + - task: hub:wait-for-sync + - printf '{{.C_OK}}✓ Bootstrap complete. Run task status.{{.C_RESET}}\n' + + kind:create: + desc: Create Kind cluster (skips if exists) + status: + - kind get clusters 2>/dev/null | grep -q "^{{.KIND_CLUSTER_NAME}}$" + cmds: + - kind create cluster --name {{.KIND_CLUSTER_NAME}} --config kind.yaml --kubeconfig {{.KUBECONFIG}} + + credentials:setup: + desc: Create AWS credentials secret (skips if exists) + status: + - kubectl -n {{.ACK_NAMESPACE}} get secret aws-credentials 2>/dev/null + cmds: + - task: credentials:refresh + + credentials:refresh: + desc: Force-refresh AWS credentials secret + cmds: + - kubectl create namespace {{.ACK_NAMESPACE}} --dry-run=client -o yaml | kubectl apply -f - + - | + set -e + # Prefer explicit aws.profile from config — IMDS only used as fallback when no profile + # is set (or set to "default"). Without this, EC2-hosted runners (Cloud9, Workshop Studio, + # any EC2-based bootstrap host) silently override the user's intended account by reading + # the instance role from IMDS, causing ACK controllers to provision in the wrong account. + AWS_PROFILE_NAME=$(yq '.aws.profile // ""' {{.CONFIG_FILE}}) + AKI=""; SAK=""; ST="" + if [ -n "$AWS_PROFILE_NAME" ] && [ "$AWS_PROFILE_NAME" != "default" ]; then + printf ' Using AWS profile from config: %s\n' "$AWS_PROFILE_NAME" + EXPORTED=$(aws configure export-credentials --profile "$AWS_PROFILE_NAME" --format env-no-export) + AKI=$(echo "$EXPORTED" | sed -n 's/^AWS_ACCESS_KEY_ID=//p') + SAK=$(echo "$EXPORTED" | sed -n 's/^AWS_SECRET_ACCESS_KEY=//p') + ST=$(echo "$EXPORTED" | sed -n 's/^AWS_SESSION_TOKEN=//p') + else + TOKEN=$(curl -s -m 2 -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600" 2>/dev/null || true) + if [ -n "$TOKEN" ]; then + printf ' Using EC2 instance role credentials (IMDS)\n' + ROLE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/iam/security-credentials/) + CREDS=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/iam/security-credentials/$ROLE) + AKI=$(echo "$CREDS" | jq -r .AccessKeyId) + SAK=$(echo "$CREDS" | jq -r .SecretAccessKey) + ST=$(echo "$CREDS" | jq -r .Token) + else + printf ' Using AWS default profile\n' + EXPORTED=$(aws configure export-credentials --profile default --format env-no-export) + AKI=$(echo "$EXPORTED" | sed -n 's/^AWS_ACCESS_KEY_ID=//p') + SAK=$(echo "$EXPORTED" | sed -n 's/^AWS_SECRET_ACCESS_KEY=//p') + ST=$(echo "$EXPORTED" | sed -n 's/^AWS_SESSION_TOKEN=//p') + fi + fi + [ -z "$AKI" ] || [ -z "$SAK" ] && echo "ERROR: empty credentials" && exit 1 + rm -f {{.ROOT_DIR}}/private/aws-creds.ini + install -m 600 /dev/null {{.ROOT_DIR}}/private/aws-creds.ini + { + echo "[default]" + echo "aws_access_key_id = $AKI" + echo "aws_secret_access_key = $SAK" + [ -n "$ST" ] && echo "aws_session_token = $ST" + } > {{.ROOT_DIR}}/private/aws-creds.ini + - kubectl -n {{.ACK_NAMESPACE}} create secret generic aws-credentials + --from-file=credentials={{.ROOT_DIR}}/private/aws-creds.ini + --dry-run=client -o yaml | kubectl apply -f - + - rm -f {{.ROOT_DIR}}/private/aws-creds.ini + + argocd:install: + desc: Install ArgoCD (skips if deployed) + status: + - helm status argocd -n argocd 2>/dev/null | grep -q deployed + cmds: + - helm repo add argo https://argoproj.github.io/argo-helm --force-update + - helm upgrade --install argocd argo/argo-cd + --namespace argocd --create-namespace + --version {{.ARGOCD_VERSION}} + --wait --timeout 5m + + ack:install: + desc: Install ACK controllers (IAM, EKS, EC2) + cmds: + - task: ack:install-controller + vars: { CONTROLLER: iam, VERSION: "{{.ACK_IAM_VERSION}}" } + - task: ack:install-controller + vars: { CONTROLLER: eks, VERSION: "{{.ACK_EKS_VERSION}}" } + - task: ack:install-controller + vars: { CONTROLLER: ec2, VERSION: "{{.ACK_EC2_VERSION}}" } + - printf '{{.C_OK}}✓ ACK controllers installed.{{.C_RESET}}\n' + + ack:install-controller: + internal: true + status: + - helm status ack-{{.CONTROLLER}}-controller -n {{.ACK_NAMESPACE}} 2>/dev/null | grep -q deployed + cmds: + - helm install ack-{{.CONTROLLER}}-controller + oci://public.ecr.aws/aws-controllers-k8s/{{.CONTROLLER}}-chart + --namespace {{.ACK_NAMESPACE}} --create-namespace + --version {{.VERSION}} + --set aws.region={{.AWS_REGION}} + --set aws.credentials.secretName=aws-credentials + --set aws.credentials.secretKey=credentials + --set serviceAccount.name=ack-{{.CONTROLLER}}-controller + --wait --timeout 5m + + kro:install: + desc: Install KRO controller + status: + - helm status kro -n {{.KRO_NAMESPACE}} 2>/dev/null | grep -q deployed + cmds: + - helm install kro oci://registry.k8s.io/kro/charts/kro + --namespace {{.KRO_NAMESPACE}} --create-namespace + --version {{.KRO_VERSION}} + --wait --timeout 5m + - printf '{{.C_OK}}✓ KRO installed.{{.C_RESET}}\n' + + kro:apply-rgds: + desc: Apply KRO ResourceGraphDefinitions for VPC and EKS + cmds: + - kubectl apply -f {{.GITOPS_ROOT}}/addons/charts/kro/resource-groups/manifests/eks/rg-vpc.yaml + - kubectl apply -f {{.GITOPS_ROOT}}/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml + - kubectl apply -f {{.GITOPS_ROOT}}/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml + - printf '{{.C_STEP}}▸ Waiting for RGDs to be ready...{{.C_RESET}}\n' + - kubectl wait --for=condition=Ready resourcegraphdefinition --all --timeout=180s + - printf '{{.C_OK}}✓ ResourceGraphDefinitions applied.{{.C_RESET}}\n' + + idc:setup: + desc: Create IDC groups if they don't exist (idempotent) + vars: + STORE_ID: + sh: | + ID=$(yq '.identityCenter.identityStoreId // ""' {{.CONFIG_FILE}}) + if [ -z "$ID" ]; then + aws sso-admin list-instances --region {{.IDC_REGION}} --query 'Instances[0].IdentityStoreId' --output text 2>/dev/null || echo "" + else + echo "$ID" + fi + ADMIN_GROUP: + sh: | + EXISTING=$(yq '.identityCenter.adminGroupId // ""' {{.CONFIG_FILE}}) + if [ -n "$EXISTING" ]; then echo "$EXISTING"; exit 0; fi + STORE_ID=$(yq '.identityCenter.identityStoreId // ""' {{.CONFIG_FILE}}) + [ -z "$STORE_ID" ] && STORE_ID=$(aws sso-admin list-instances --region {{.IDC_REGION}} --query 'Instances[0].IdentityStoreId' --output text 2>/dev/null) + aws identitystore list-groups --identity-store-id "$STORE_ID" --region {{.IDC_REGION}} --filters '[{"AttributePath":"DisplayName","AttributeValue":"admin"}]' --query 'Groups[0].GroupId' --output text 2>/dev/null | grep -v None || echo "" + cmds: + - | + STORE_ID="{{.STORE_ID}}" + if [ -z "$STORE_ID" ]; then + echo "ERROR: No IDC Identity Store found. Set identityCenter.identityStoreId in config.local.yaml" + exit 1 + fi + printf '{{.C_STEP}}▸ IDC Identity Store: %s{{.C_RESET}}\n' "$STORE_ID" + # BYO IdC: if adminGroupId AND adminUserId are pre-populated in config, + # skip group/user/membership creation entirely (useful when IdC lives in + # a different AWS account and the bootstrap role lacks identitystore:Create* perms). + BYO_GROUP=$(yq '.identityCenter.adminGroupId // ""' {{.CONFIG_FILE}}) + BYO_USER=$(yq '.identityCenter.adminUserId // ""' {{.CONFIG_FILE}}) + if [ -n "$BYO_GROUP" ] && [ -n "$BYO_USER" ]; then + printf '{{.C_INFO}} BYO IdC detected (adminGroupId + adminUserId set) — skipping group/user/membership creation{{.C_RESET}}\n' + printf '{{.C_OK}}✓ IDC admin group: %s{{.C_RESET}}\n' "$BYO_GROUP" + printf '{{.C_OK}}✓ IDC admin user: %s{{.C_RESET}}\n' "$BYO_USER" + yq -i ".identityCenter.identityStoreId = \"$STORE_ID\"" {{.CONFIG_FILE}} + exit 0 + fi + # Create groups idempotently + for GROUP_NAME in admin editor viewer; do + EXISTING=$(aws identitystore list-groups --identity-store-id "$STORE_ID" --region {{.IDC_REGION}} \ + --filters "[{\"AttributePath\":\"DisplayName\",\"AttributeValue\":\"$GROUP_NAME\"}]" \ + --query 'Groups[0].GroupId' --output text 2>/dev/null | grep -v None || echo "") + if [ -n "$EXISTING" ]; then + printf '{{.C_INFO}} Group "%s" exists: %s{{.C_RESET}}\n' "$GROUP_NAME" "$EXISTING" + else + NEW_ID=$(aws identitystore create-group --identity-store-id "$STORE_ID" --region {{.IDC_REGION}} \ + --display-name "$GROUP_NAME" --description "Platform group for EKS ArgoCD Capability" \ + --query 'GroupId' --output text) + printf '{{.C_OK}} Created group "%s": %s{{.C_RESET}}\n' "$GROUP_NAME" "$NEW_ID" + fi + done + # Output admin group ID (prefer config value, fallback to lookup) + ADMIN_ID=$(yq '.identityCenter.adminGroupId // ""' {{.CONFIG_FILE}}) + if [ -z "$ADMIN_ID" ]; then + ADMIN_ID=$(aws identitystore list-groups --identity-store-id "$STORE_ID" --region {{.IDC_REGION}} \ + --filters '[{"AttributePath":"DisplayName","AttributeValue":"admin"}]' \ + --query 'Groups[0].GroupId' --output text) + fi + printf '{{.C_OK}}✓ IDC admin group: %s{{.C_RESET}}\n' "$ADMIN_ID" + # Create admin user if not exists + ADMIN_USER=$(aws identitystore list-users --identity-store-id "$STORE_ID" --region {{.IDC_REGION}} \ + --filters '[{"AttributePath":"UserName","AttributeValue":"eks-argocd-admin-user"}]' \ + --query 'Users[0].UserId' --output text 2>/dev/null | grep -v None || echo "") + if [ -z "$ADMIN_USER" ]; then + ADMIN_USER=$(aws identitystore create-user --identity-store-id "$STORE_ID" --region {{.IDC_REGION}} \ + --user-name "eks-argocd-admin-user" --display-name "EKS ArgoCD Admin" \ + --name '{"GivenName":"EKS Admin","FamilyName":"User"}' \ + --emails '[{"Value":"eks-argocd-admin@example.com","Primary":true}]' \ + --query 'UserId' --output text) + printf '{{.C_OK}} Created admin user: %s{{.C_RESET}}\n' "$ADMIN_USER" + else + printf '{{.C_INFO}} Admin user exists: %s{{.C_RESET}}\n' "$ADMIN_USER" + fi + # Add user to admin group if not already member + IS_MEMBER=$(aws identitystore is-member-in-groups --identity-store-id "$STORE_ID" --region {{.IDC_REGION}} \ + --member-id '{"UserId":"'"$ADMIN_USER"'"}' --group-ids "$ADMIN_ID" \ + --query 'Results[0].MembershipExists' --output text 2>/dev/null || echo "false") + if [ "$IS_MEMBER" != "True" ]; then + aws identitystore create-group-membership --identity-store-id "$STORE_ID" --region {{.IDC_REGION}} \ + --group-id "$ADMIN_ID" --member-id '{"UserId":"'"$ADMIN_USER"'"}' >/dev/null 2>&1 || true + printf '{{.C_OK}} Added user to admin group{{.C_RESET}}\n' + fi + # Update config.local.yaml with discovered values + yq -i ".identityCenter.adminGroupId = \"$ADMIN_ID\"" {{.CONFIG_FILE}} + yq -i ".identityCenter.identityStoreId = \"$STORE_ID\"" {{.CONFIG_FILE}} + + hub:claim: + desc: Apply hub cluster claim (EksclusterWithVpc instance) + cmds: + - kubectl create namespace {{.HUB_CLUSTER_NAME}} --dry-run=client -o yaml | kubectl apply -f - + - | + cat <<'EOF' | sed \ + -e "s|HUB_CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g" \ + -e "s|AWS_REGION|{{.AWS_REGION}}|g" \ + -e "s|AWS_ACCOUNT_ID|{{.AWS_ACCOUNT_ID}}|g" \ + -e "s|HUB_K8S_VERSION|{{.HUB_K8S_VERSION}}|g" \ + -e "s|HUB_VPC_CIDR|{{.HUB_VPC_CIDR}}|g" \ + -e "s|RESOURCE_PREFIX|{{.RESOURCE_PREFIX}}|g" \ + -e "s|CALLER_ROLE_ARN|{{.CALLER_ROLE_ARN}}|g" \ + -e "s|IDC_INSTANCE_ARN|{{.IDC_INSTANCE_ARN}}|g" \ + -e "s|IDC_REGION|{{.IDC_REGION}}|g" \ + -e "s|IDC_ADMIN_GROUP_ID|{{.IDC_ADMIN_GROUP_ID}}|g" \ + -e "s|ADMIN_ROLE_NAME|{{.ADMIN_ROLE_NAME}}|g" \ + -e "s|REPO_URL|{{.REPO_URL}}|g" \ + -e "s|REPO_REVISION|{{.REPO_REVISION}}|g" \ + -e "s|REPO_BASEPATH|{{.REPO_BASEPATH}}|g" \ + -e "s|DOMAIN|{{.DOMAIN}}|g" \ + | kubectl apply -f - + apiVersion: kro.run/v1alpha1 + kind: EksclusterWithVpc + metadata: + name: HUB_CLUSTER_NAME + namespace: HUB_CLUSTER_NAME + spec: + name: HUB_CLUSTER_NAME + tenant: platform + environment: control-plane + region: AWS_REGION + k8sVersion: "HUB_K8S_VERSION" + accountId: "AWS_ACCOUNT_ID" + managementAccountId: "AWS_ACCOUNT_ID" + argoCdHubRoleArn: "CALLER_ROLE_ARN" + adminRoleName: ADMIN_ROLE_NAME + fleetSecretManagerSecretNameSuffix: argocd-secret + domainName: DOMAIN + resourcePrefix: RESOURCE_PREFIX + argocdCapability: + enabled: "true" + idcInstanceArn: IDC_INSTANCE_ARN + idcRegion: IDC_REGION + adminGroupId: IDC_ADMIN_GROUP_ID + cidr: + vpcCidr: HUB_VPC_CIDR + publicSubnet1Cidr: "10.0.1.0/24" + publicSubnet2Cidr: "10.0.2.0/24" + privateSubnet1Cidr: "10.0.11.0/24" + privateSubnet2Cidr: "10.0.12.0/24" + gitops: + addonsRepoUrl: REPO_URL + addonsRepoRevision: REPO_REVISION + addonsRepoBasePath: "REPO_BASEPATH/addons/" + addonsRepoPath: bootstrap + fleetRepoUrl: REPO_URL + fleetRepoRevision: REPO_REVISION + fleetRepoBasePath: "REPO_BASEPATH/fleet/" + fleetRepoPath: bootstrap + platformRepoUrl: REPO_URL + platformRepoRevision: REPO_REVISION + platformRepoBasePath: "REPO_BASEPATH/platform/" + platformRepoPath: bootstrap + workloadRepoUrl: REPO_URL + workloadRepoRevision: REPO_REVISION + workloadRepoBasePath: "REPO_BASEPATH/apps/" + workloadRepoPath: "" + addons: + enable_ingress_class_alb: "true" + enable_argo_rollouts: "false" + enable_metrics_server: "false" + enable_external_secrets: "true" + enable_external_dns: "true" + enable_adot_collector: "false" + enable_cw_prometheus: "false" + enable_kyverno: "false" + enable_kyverno_policies: "false" + enable_kyverno_policy_reporter: "false" + enable_cni_metrics_helper: "false" + enable_prometheus_node_exporter: "false" + enable_kube_state_metrics: "false" + enable_opentelemetry_operator: "false" + enable_cert_manager: "true" + enable_aws_efs_csi_driver: "false" + enable_crossplane: "true" + enable_crossplane_aws: "true" + enable_flux: "false" + enable_ingress_nginx: "false" + enable_platform_manifests: "true" + enable_kubevela: "false" + adot_collector_namespace: adot-collector-kubeprometheus + adot_collector_service_account: adot-collector-kubeprometheus + external_dns_namespace: external-dns + external_dns_service_account: external-dns-sa + external_dns_policy: sync + external_secrets_namespace: external-secrets + external_secrets_service_account: external-secrets-sa + amp_endpoint_url: "" + EOF + + hub:wait-for-eks: + desc: Wait for the EksclusterWithVpc instance to reach ACTIVE state and the EKS cluster to exist in AWS + vars: + WAIT_TIMEOUT_SEC: '{{.WAIT_TIMEOUT_SEC | default "1800"}}' + WAIT_INTERVAL_SEC: '{{.WAIT_INTERVAL_SEC | default "30"}}' + cmds: + - | + printf '{{.C_STEP}}▸ Waiting for EksclusterWithVpc/{{.HUB_CLUSTER_NAME}} to reach ACTIVE (max {{.WAIT_TIMEOUT_SEC}}s)...{{.C_RESET}}\n' + END=$(( $(date +%s) + {{.WAIT_TIMEOUT_SEC}} )) + LAST_STATE="" + while [ "$(date +%s)" -lt "$END" ]; do + STATE=$(kubectl get eksclusterwithvpc {{.HUB_CLUSTER_NAME}} -n {{.HUB_CLUSTER_NAME}} -o jsonpath='{.status.state}' 2>/dev/null || echo "") + if [ "$STATE" != "$LAST_STATE" ]; then + printf '{{.C_INFO}} state: %s{{.C_RESET}}\n' "${STATE:-}" + LAST_STATE="$STATE" + fi + if [ "$STATE" = "ACTIVE" ]; then + break + fi + sleep {{.WAIT_INTERVAL_SEC}} + done + if [ "$STATE" != "ACTIVE" ]; then + printf '{{.C_ERR}}✗ EksclusterWithVpc not ACTIVE after {{.WAIT_TIMEOUT_SEC}}s (last state: %s){{.C_RESET}}\n' "$STATE" + kubectl get eksclusterwithvpc {{.HUB_CLUSTER_NAME}} -n {{.HUB_CLUSTER_NAME}} -o jsonpath='{.status.conditions}' 2>/dev/null || true + exit 1 + fi + printf '{{.C_OK}}✓ EksclusterWithVpc ACTIVE{{.C_RESET}}\n' + - | + printf '{{.C_STEP}}▸ Verifying EKS cluster {{.HUB_CLUSTER_NAME}} exists in AWS...{{.C_RESET}}\n' + END=$(( $(date +%s) + 300 )) + while [ "$(date +%s)" -lt "$END" ]; do + EKS_STATUS=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.status' --output text 2>/dev/null || echo "") + if [ "$EKS_STATUS" = "ACTIVE" ]; then + printf '{{.C_OK}}✓ EKS cluster ACTIVE{{.C_RESET}}\n' + exit 0 + fi + printf '{{.C_INFO}} EKS status: %s{{.C_RESET}}\n' "${EKS_STATUS:-}" + sleep 15 + done + printf '{{.C_ERR}}✗ EKS cluster not ACTIVE in AWS after RGD reported ACTIVE{{.C_RESET}}\n' + exit 1 + + hub:ingress:preflight: + desc: Validate ingress.mode config (fails fast if mode=tls without certificateArn+domain) + cmds: + - | + MODE="{{.INGRESS_MODE}}" + CERT="{{.INGRESS_CERT_ARN}}" + DOMAIN="{{.DOMAIN}}" + case "$MODE" in + cloudfront) + printf '{{.C_INFO}} ingress.mode=cloudfront (ALB internal + CloudFront VPC Origin){{.C_RESET}}\n' + ;; + tls) + if [ -z "$CERT" ]; then + printf '{{.C_ERR}}✗ ingress.mode=tls requires ingress.certificateArn (ACM ARN){{.C_RESET}}\n' + exit 1 + fi + if [ -z "$DOMAIN" ] || [ "$DOMAIN" = "" ]; then + printf '{{.C_ERR}}✗ ingress.mode=tls requires top-level domain to be set{{.C_RESET}}\n' + exit 1 + fi + printf '{{.C_INFO}} ingress.mode=tls (ALB internet-facing HTTPS:443, cert=%s, domain=%s){{.C_RESET}}\n' "$CERT" "$DOMAIN" + ;; + *) + printf '{{.C_ERR}}✗ ingress.mode=%s invalid; expected: cloudfront | tls{{.C_RESET}}\n' "$MODE" + exit 1 + ;; + esac + + hub:ingress: + desc: | + Create ALB for hub ingress. Behaviour depends on ingress.mode: + - cloudfront: ALB scheme=internal in private subnets, SG limited to VPC CIDR + - tls: ALB scheme=internet-facing in public subnets, SG open 80/443 to 0.0.0.0/0 + Switching mode after creation requires manual ALB teardown (AWS scheme is immutable). + deps: [hub:ingress:preflight] + vars: + ALB_NAME: "{{.RESOURCE_PREFIX}}-hub-ingress" + VPC_ID: + sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text + VPC_CIDR: + sh: aws ec2 describe-vpcs --vpc-ids "$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text)" --region {{.AWS_REGION}} --query 'Vpcs[0].CidrBlock' --output text + SG_ID: + sh: aws ec2 describe-security-groups --region {{.AWS_REGION}} --filters "Name=vpc-id,Values=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text)" "Name=group-name,Values={{.RESOURCE_PREFIX}}-hub-ingress-http" --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "" + status: + - aws elbv2 describe-load-balancers --names {{.ALB_NAME}} --region {{.AWS_REGION}} 2>/dev/null + cmds: + - | + MODE="{{.INGRESS_MODE}}" + VPC_ID="{{.VPC_ID}}" + VPC_CIDR="{{.VPC_CIDR}}" + # Resolve subnets according to mode + if [ "$MODE" = "cloudfront" ]; then + SCHEME="internal" + SUBNET_NAME_FILTER="*private*" + ROLE_TAG="kubernetes.io/role/internal-elb" + else + SCHEME="internet-facing" + SUBNET_NAME_FILTER="*public*" + ROLE_TAG="kubernetes.io/role/elb" + fi + SUBNETS=$(aws ec2 describe-subnets --region {{.AWS_REGION}} \ + --filters "Name=vpc-id,Values=$VPC_ID" \ + "Name=tag:Name,Values=$SUBNET_NAME_FILTER" \ + "Name=tag:$ROLE_TAG,Values=1" \ + --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') + if [ -z "$SUBNETS" ]; then + # Fallback: name-only filter for older VPCs without role tags + SUBNETS=$(aws ec2 describe-subnets --region {{.AWS_REGION}} \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=$SUBNET_NAME_FILTER" \ + --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') + fi + if [ -z "$SUBNETS" ]; then + printf '{{.C_ERR}}✗ No subnets matched filter %s in VPC %s{{.C_RESET}}\n' "$SUBNET_NAME_FILTER" "$VPC_ID" + exit 1 + fi + # Resolve / create SG with mode-appropriate ingress rules + SG="{{.SG_ID}}" + if [ -z "$SG" ] || [ "$SG" = "None" ]; then + SG=$(aws ec2 create-security-group --group-name "{{.RESOURCE_PREFIX}}-hub-ingress-http" \ + --description "HTTP/HTTPS for hub ingress (mode=$MODE)" --vpc-id "$VPC_ID" --region {{.AWS_REGION}} \ + --query 'GroupId' --output text) + if [ "$MODE" = "cloudfront" ]; then + # ALB internal — limit to VPC CIDR (covers in-cluster traffic) + aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 80 \ + --cidr "$VPC_CIDR" --region {{.AWS_REGION}} 2>/dev/null || true + # CloudFront VPC Origin ENIs use a dedicated AWS-managed SG — + # CIDR alone is not sufficient because CF routes via service SG. + CF_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ + --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") + if [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ]; then + aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 80 \ + --source-group "$CF_SG" --region {{.AWS_REGION}} 2>/dev/null || true + printf '{{.C_OK}}✓ Authorized CloudFront-VPCOrigins-Service-SG → ALB:80{{.C_RESET}}\n' + else + printf '{{.C_WARN}}⚠ CloudFront-VPCOrigins-Service-SG not found — ALB may 504. Re-run after first VPC Origin creation.{{.C_RESET}}\n' + fi + else + # ALB internet-facing — open 80 (redirect) and 443 + aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 80 \ + --cidr 0.0.0.0/0 --region {{.AWS_REGION}} 2>/dev/null || true + aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 443 \ + --cidr 0.0.0.0/0 --region {{.AWS_REGION}} 2>/dev/null || true + fi + fi + ALB_ARN=$(aws elbv2 create-load-balancer --name "{{.ALB_NAME}}" --type application \ + --scheme "$SCHEME" --subnets $SUBNETS --security-groups "$SG" \ + --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text) + # Default listener (404) — AWS LBC adds rules later when K8s Ingresses appear + if [ "$MODE" = "cloudfront" ]; then + aws elbv2 create-listener --load-balancer-arn "$ALB_ARN" --protocol HTTP --port 80 \ + --default-actions 'Type=fixed-response,FixedResponseConfig={StatusCode=404,ContentType=text/plain,MessageBody=NotFound}' \ + --region {{.AWS_REGION}} >/dev/null + else + aws elbv2 create-listener --load-balancer-arn "$ALB_ARN" --protocol HTTPS --port 443 \ + --certificates "CertificateArn={{.INGRESS_CERT_ARN}}" \ + --default-actions 'Type=fixed-response,FixedResponseConfig={StatusCode=404,ContentType=text/plain,MessageBody=NotFound}' \ + --region {{.AWS_REGION}} >/dev/null + aws elbv2 create-listener --load-balancer-arn "$ALB_ARN" --protocol HTTP --port 80 \ + --default-actions 'Type=redirect,RedirectConfig={Protocol=HTTPS,Port=443,StatusCode=HTTP_301}' \ + --region {{.AWS_REGION}} >/dev/null + fi + printf '{{.C_OK}}✓ ALB created: {{.ALB_NAME}} (scheme=%s, mode=%s){{.C_RESET}}\n' "$SCHEME" "$MODE" + + hub:cloudfront: + desc: Create CloudFront distribution with VPC Origin (cloudfront mode only) + # CachePolicyId 4135ea2d = AWS Managed "CachingDisabled" + # OriginRequestPolicyId 216adef6 = AWS Managed "AllViewer" (forwards all headers/cookies/qs) + # Uses CloudFront VPC Origin (post-2024) to reach an internal ALB without public exposure. + vars: + ALB_ARN: + sh: aws elbv2 describe-load-balancers --names "{{.RESOURCE_PREFIX}}-hub-ingress" --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text 2>/dev/null || echo "" + ALB_DNS: + sh: aws elbv2 describe-load-balancers --names "{{.RESOURCE_PREFIX}}-hub-ingress" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text 2>/dev/null || echo "" + VPC_ORIGIN_NAME: "{{.RESOURCE_PREFIX}}-hub-vpc-origin" + status: + # Skip entirely when not in cloudfront mode (idempotent no-op for tls) + - test "{{.INGRESS_MODE}}" != "cloudfront" + - aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.RESOURCE_PREFIX}}-hub-ingress'].Id" --output text | grep -q . + cmds: + - | + if [ "{{.INGRESS_MODE}}" != "cloudfront" ]; then + printf '{{.C_INFO}}⊘ Skipping hub:cloudfront (ingress.mode={{.INGRESS_MODE}}){{.C_RESET}}\n' + exit 0 + fi + if [ -z "{{.ALB_ARN}}" ] || [ "{{.ALB_ARN}}" = "None" ]; then + echo "ERROR: ALB {{.RESOURCE_PREFIX}}-hub-ingress not found. Run hub:ingress first." + exit 1 + fi + - | + # 1. Create or reuse VPC Origin pointing at the internal ALB + VPC_ORIGIN_ID=$(aws cloudfront list-vpc-origins --region {{.AWS_REGION}} \ + --query "Items[?Name=='{{.VPC_ORIGIN_NAME}}'].Id" --output text 2>/dev/null || echo "") + if [ -z "$VPC_ORIGIN_ID" ] || [ "$VPC_ORIGIN_ID" = "None" ]; then + VPC_ORIGIN_ID=$(aws cloudfront create-vpc-origin --region {{.AWS_REGION}} \ + --vpc-origin-endpoint-config "{ + \"Name\": \"{{.VPC_ORIGIN_NAME}}\", + \"Arn\": \"{{.ALB_ARN}}\", + \"HTTPPort\": 80, + \"HTTPSPort\": 443, + \"OriginProtocolPolicy\": \"http-only\", + \"OriginSslProtocols\": {\"Quantity\": 1, \"Items\": [\"TLSv1.2\"]} + }" --query 'VpcOrigin.Id' --output text) + printf '{{.C_OK}}✓ VPC Origin created: %s{{.C_RESET}}\n' "$VPC_ORIGIN_ID" + else + printf '{{.C_INFO}}↻ Reusing VPC Origin: %s{{.C_RESET}}\n' "$VPC_ORIGIN_ID" + fi + + # Wait for VPC Origin Deployed (~5-10 min) + printf '{{.C_STEP}}▸ Waiting for VPC Origin Deployed...{{.C_RESET}}\n' + for i in $(seq 1 60); do + STATE=$(aws cloudfront get-vpc-origin --id "$VPC_ORIGIN_ID" --region {{.AWS_REGION}} --query 'VpcOrigin.Status' --output text 2>/dev/null || echo "Pending") + printf '{{.C_INFO}} [%s/60] VPC Origin status: %s{{.C_RESET}}\n' "$i" "$STATE" + [ "$STATE" = "Deployed" ] && break + sleep 15 + done + if [ "$STATE" != "Deployed" ]; then + echo "ERROR: VPC Origin did not reach Deployed within 15 min"; exit 1 + fi + echo "$VPC_ORIGIN_ID" > /tmp/hub-vpc-origin-id + - | + # 2. Reconcile ALB SG ingress from CloudFront-VPCOrigins-Service-SG. + # This SG is created by AWS only after the first VPC Origin lands in + # the VPC, so hub:ingress can't authorize it on first run. We close + # the loop here so a single hub:ingress + hub:cloudfront sequence is + # sufficient (no manual re-run of hub:ingress needed). + VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) + ALB_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values={{.RESOURCE_PREFIX}}-hub-ingress-http" \ + --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") + CF_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ + --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") + if [ -n "$ALB_SG" ] && [ "$ALB_SG" != "None" ] && [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ]; then + aws ec2 authorize-security-group-ingress --group-id "$ALB_SG" --protocol tcp --port 80 \ + --source-group "$CF_SG" --region {{.AWS_REGION}} 2>/dev/null \ + && printf '{{.C_OK}}✓ Authorized %s → %s:80 (post VPC Origin){{.C_RESET}}\n' "$CF_SG" "$ALB_SG" \ + || printf '{{.C_INFO}}↻ ALB SG already allows CloudFront-VPCOrigins-Service-SG{{.C_RESET}}\n' + fi + - | + # 3. Create CloudFront distribution wired to the VPC Origin + VPC_ORIGIN_ID=$(cat /tmp/hub-vpc-origin-id) + DIST_OUT=$(aws cloudfront create-distribution \ + --distribution-config "{ + \"CallerReference\": \"{{.HUB_CLUSTER_NAME}}-$(date +%s)\", + \"Comment\": \"{{.RESOURCE_PREFIX}}-hub-ingress\", + \"Enabled\": true, + \"Origins\": { + \"Quantity\": 1, + \"Items\": [{ + \"Id\": \"alb-vpc-origin\", + \"DomainName\": \"{{.ALB_DNS}}\", + \"VpcOriginConfig\": { + \"VpcOriginId\": \"$VPC_ORIGIN_ID\", + \"OriginReadTimeout\": 30, + \"OriginKeepaliveTimeout\": 5 + } + }] + }, + \"DefaultCacheBehavior\": { + \"TargetOriginId\": \"alb-vpc-origin\", + \"ViewerProtocolPolicy\": \"redirect-to-https\", + \"AllowedMethods\": {\"Quantity\": 7, \"Items\": [\"GET\",\"HEAD\",\"OPTIONS\",\"PUT\",\"POST\",\"PATCH\",\"DELETE\"], \"CachedMethods\": {\"Quantity\": 2, \"Items\": [\"GET\",\"HEAD\"]}}, + \"CachePolicyId\": \"4135ea2d-6df8-44a3-9df3-4b5a84be39ad\", + \"OriginRequestPolicyId\": \"216adef6-5c7f-47e4-b989-5492eafa07d3\", + \"Compress\": true + }, + \"PriceClass\": \"PriceClass_100\", + \"HttpVersion\": \"http2\" + }" --query 'Distribution.[Id,DomainName]' --output text) + CF_DOMAIN=$(echo "$DIST_OUT" | awk '{print $2}') + printf '{{.C_OK}}✓ CloudFront created: %s{{.C_RESET}}\n' "$CF_DOMAIN" + # Store domain in config for hub:seed to use + yq -i ".cloudfront.domain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} + rm -f /tmp/hub-vpc-origin-id + + hub:seed: + desc: Seed hub cluster after EKS is ACTIVE + cmds: + - task: credentials:refresh + - printf '{{.C_STEP}}▸ Waiting for hub EksCluster to be ready...{{.C_RESET}}\n' + - | + STATUS="Pending" + for i in $(seq 1 120); do + STATUS=$(kubectl get ekscluster {{.HUB_CLUSTER_NAME}} -n {{.HUB_CLUSTER_NAME}} -o jsonpath='{.status.clusterState}' 2>/dev/null || echo "Pending") + printf '{{.C_INFO}} [%s/120] Cluster status: %s{{.C_RESET}}\n' "$i" "$STATUS" + if [ "$STATUS" = "ACTIVE" ]; then break; fi + sleep 15 + done + if [ "$STATUS" != "ACTIVE" ]; then echo "ERROR: cluster not ACTIVE"; exit 1; fi + - printf '{{.C_STEP}}▸ Creating cluster admin access entry...{{.C_RESET}}\n' + - cmd: | + CALLER_ARN=$(aws sts get-caller-identity --query "Arn" --output text) + if echo "$CALLER_ARN" | grep -q ":assumed-role/"; then + PRINCIPAL_ARN=$(echo "$CALLER_ARN" | sed 's|arn:aws:sts::|arn:aws:iam::|;s|:assumed-role/|:role/|;s|/[^/]*$||') + else + PRINCIPAL_ARN="$CALLER_ARN" + fi + aws eks create-access-entry --cluster-name {{.HUB_CLUSTER_NAME}} --principal-arn "$PRINCIPAL_ARN" --type STANDARD --region {{.AWS_REGION}} 2>/dev/null || true + aws eks associate-access-policy --cluster-name {{.HUB_CLUSTER_NAME}} --principal-arn "$PRINCIPAL_ARN" --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy --access-scope type=cluster --region {{.AWS_REGION}} 2>/dev/null || true + - task: secrets-manager:seed + - task: secrets-manager:seed-keycloak + - task: hub:install-eso + - task: hub:cluster-secret-store + - task: hub:install-argocd + - task: hub:seed-secret + - task: hub:apply-root-appset + + hub:kubeconfig: + cmds: + - aws eks update-kubeconfig --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --kubeconfig {{.ROOT_DIR}}/private/hub-kubeconfig 2>&1 >/dev/null + + argocd:capability: + desc: Create EKS ArgoCD Capability via Job + status: + - kubectl -n {{.ACK_NAMESPACE}} get job create-argocd-capability -o jsonpath='{.status.succeeded}' 2>/dev/null | grep -q 1 + vars: + CAPABILITY_ROLE_ARN: + sh: kubectl get role.iam.services.k8s.aws/{{.HUB_CLUSTER_NAME}}-argocd-role -n {{.HUB_CLUSTER_NAME}} -o jsonpath='{.status.ackResourceMetadata.arn}' 2>/dev/null || echo "" + cmds: + - kubectl -n {{.ACK_NAMESPACE}} create configmap argocd-capability-config + --from-literal=CLUSTER_NAME={{.HUB_CLUSTER_NAME}} + --from-literal=AWS_REGION={{.AWS_REGION}} + --from-literal=AWS_ACCOUNT_ID={{.AWS_ACCOUNT_ID}} + --from-literal=CAPABILITY_NAME={{.CAPABILITY_NAME}} + --from-literal=CAPABILITY_ROLE_ARN={{.CAPABILITY_ROLE_ARN}} + --from-literal=IDC_INSTANCE_ARN={{.IDC_INSTANCE_ARN}} + --from-literal=IDC_REGION={{.IDC_REGION}} + --from-literal=IDC_ADMIN_GROUP_ID={{.IDC_ADMIN_GROUP_ID}} + --dry-run=client -o yaml | kubectl apply -f - + - kubectl delete job create-argocd-capability -n {{.ACK_NAMESPACE}} 2>/dev/null || true + - kubectl apply -f manifests/argocd/create-capability.yaml + - kubectl -n {{.ACK_NAMESPACE}} wait --for=condition=Complete --timeout=2400s job/create-argocd-capability + + secrets-manager:seed: + desc: Seed hub config into AWS Secrets Manager + vars: + CLUSTER_ARN: + sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.arn' --output text 2>/dev/null + VPC_ID: + sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text + cmds: + - | + SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" + SECRET_VALUE=$(jq -n \ + --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}/","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}/","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.RESOURCE_PREFIX}}-hub-ingress","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}"}' \ + --arg config '{"tlsClientConfig":{"insecure":false}}' \ + --arg server '{{.CLUSTER_ARN}}' \ + --arg addons '' \ + --arg vpc '{{.VPC_ID}}' \ + '{metadata: $metadata, config: $config, server: $server, addons: $addons, vpc: $vpc}') + aws secretsmanager create-secret \ + --name "$SECRET_KEY" --secret-string "$SECRET_VALUE" --region {{.AWS_REGION}} 2>/dev/null \ + || aws secretsmanager put-secret-value \ + --secret-id "$SECRET_KEY" --secret-string "$SECRET_VALUE" --region {{.AWS_REGION}} + + secrets-manager:seed-keycloak: + desc: Seed keycloak passwords + status: + - aws secretsmanager describe-secret --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} 2>/dev/null + cmds: + - | + ADMIN_PASS=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) + DB_PASS=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) + USER_PASS=$(openssl rand -base64 12 | tr -d '/+=' | head -c 16) + SECRET_VALUE=$(jq -n \ + --arg admin "$ADMIN_PASS" --arg db "$DB_PASS" --arg user "$USER_PASS" \ + '{keycloak_admin_password: $admin, keycloak_postgres_password: $db, user_password: $user}') + aws secretsmanager create-secret \ + --name "{{.HUB_CLUSTER_NAME}}/keycloak" --secret-string "$SECRET_VALUE" --region {{.AWS_REGION}} 2>/dev/null \ + || aws secretsmanager put-secret-value \ + --secret-id "{{.HUB_CLUSTER_NAME}}/keycloak" --secret-string "$SECRET_VALUE" --region {{.AWS_REGION}} + + hub:install-argocd: + desc: Install ArgoCD on the hub cluster (OSS mode, skipped when EKS Capability is used) + status: + - test "$(yq '.argocdCapability.enabled // "false"' {{.CONFIG_FILE}} 2>/dev/null || echo false)" = "true" + cmds: + - task: hub:kubeconfig + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig helm repo add argo https://argoproj.github.io/argo-helm --force-update + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig helm upgrade --install argocd argo/argo-cd + --namespace argocd --create-namespace + --version {{.ARGOCD_VERSION}} + --wait --timeout 5m + - printf '{{.C_OK}}✓ ArgoCD installed on hub (OSS mode).{{.C_RESET}}\n' + + hub:install-eso: + desc: Install External Secrets Operator on hub + vars: + ESO_VERSION: + sh: yq '.external-secrets.defaultVersion' {{.GITOPS_ROOT}}/addons/registry/core.yaml + status: + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig helm status external-secrets -n external-secrets 2>/dev/null | grep -q deployed + cmds: + - task: hub:kubeconfig + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig helm repo add external-secrets https://charts.external-secrets.io --force-update + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig helm upgrade --install external-secrets external-secrets/external-secrets + --namespace external-secrets --create-namespace + --version {{.ESO_VERSION}} + --set serviceAccount.name=external-secrets-sa + --wait --timeout 5m + + hub:cluster-secret-store: + desc: Apply ClusterSecretStore on hub + cmds: + - task: hub:kubeconfig + - | + cat <<'EOF' | sed "s|AWS_REGION|{{.AWS_REGION}}|g" | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - + apiVersion: external-secrets.io/v1 + kind: ClusterSecretStore + metadata: + name: aws-secrets-manager + spec: + provider: + aws: + service: SecretsManager + region: AWS_REGION + EOF + + hub:seed-secret: + desc: Create seed cluster secret on hub (with edge metadata for AppExposure RGD) + vars: + CLUSTER_ARN: + sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.arn' --output text + ARGOCD_ROLE_ARN: + sh: aws iam get-role --role-name {{.HUB_CLUSTER_NAME}}-argocd-role --query 'Role.Arn' --output text 2>/dev/null || echo "" + ADMIN_ROLE_NAME: + sh: yq '.hub.adminRoleName // "Admin"' {{.CONFIG_FILE}} + # Edge metadata — populated by hub:ingress + hub:cloudfront before hub:seed runs. + # Consumed by ApplicationSets enrolling RGDs that need to know how the cluster is + # exposed (e.g. AppExposure RGD reads {{.metadata.annotations.ingress_domain_name}} + # via the ArgoCD clusters generator — gitops-bridge pattern). + EXPOSURE_MODE: + sh: | + if [ "{{.INGRESS_MODE}}" = "tls" ]; then echo "tls-alb"; else echo "cloudfront-alb"; fi + EDGE_DOMAIN: + sh: | + if [ "{{.INGRESS_MODE}}" = "tls" ]; then + echo "{{.DOMAIN}}" + else + echo "{{.CLOUDFRONT_DOMAIN}}" + fi + ALB_LISTENER_ARN: + sh: | + ALB_NAME="{{.RESOURCE_PREFIX}}-hub-ingress" + ALB_ARN=$(aws elbv2 describe-load-balancers --names "$ALB_NAME" --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text 2>/dev/null || echo "") + if [ -z "$ALB_ARN" ] || [ "$ALB_ARN" = "None" ]; then echo ""; exit 0; fi + # Pick listener matching the mode: HTTP:80 for cloudfront-alb, HTTPS:443 for tls-alb + if [ "{{.INGRESS_MODE}}" = "tls" ]; then PORT=443; else PORT=80; fi + aws elbv2 describe-listeners --load-balancer-arn "$ALB_ARN" --region {{.AWS_REGION}} \ + --query "Listeners[?Port==\`$PORT\`].ListenerArn | [0]" --output text 2>/dev/null || echo "" + VPC_ID: + sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text 2>/dev/null || echo "" + cmds: + - task: hub:kubeconfig + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl create namespace argocd --dry-run=client -o yaml | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - + - | + printf '{{.C_INFO}} Seeding cluster secret with gitops-bridge metadata:{{.C_RESET}}\n' + printf ' aws_region = %s\n' "{{.AWS_REGION}}" + printf ' aws_vpc_id = %s\n' "{{.VPC_ID}}" + printf ' ingress_domain_name = %s\n' "{{.EDGE_DOMAIN}}" + printf ' exposure_mode = %s\n' "{{.EXPOSURE_MODE}}" + printf ' alb_listener_arn = %s\n' "{{.ALB_LISTENER_ARN}}" + - | + cat <<'EOF' | sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g; s|CLUSTER_ARN|{{.CLUSTER_ARN}}|g; s|ARGOCD_ROLE_ARN|{{.ARGOCD_ROLE_ARN}}|g; s|REPO_URL|{{.REPO_URL}}|g; s|REPO_REVISION|{{.REPO_REVISION}}|g; s|REPO_BASEPATH|{{.REPO_BASEPATH}}/|g; s|EXPOSURE_MODE|{{.EXPOSURE_MODE}}|g; s|EDGE_DOMAIN|{{.EDGE_DOMAIN}}|g; s|ALB_LISTENER_ARN|{{.ALB_LISTENER_ARN}}|g; s|VPC_ID_VAL|{{.VPC_ID}}|g; s|AWS_REGION_VAL|{{.AWS_REGION}}|g; s|AWS_ACCOUNT_ID_VAL|{{.AWS_ACCOUNT_ID}}|g; s|RESOURCE_PREFIX_VAL|{{.RESOURCE_PREFIX}}|g; s|INGRESS_SECURITY_GROUPS_VAL|{{.INGRESS_SECURITY_GROUPS}}|g; s|ADMIN_ROLE_NAME_VAL|{{.ADMIN_ROLE_NAME}}|g" | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - + apiVersion: v1 + kind: Secret + type: Opaque + metadata: + name: CLUSTER_NAME + namespace: argocd + annotations: + # Repo coordinates (gitops-bridge contract) + addonsRepoURL: "REPO_URL" + addonsRepoRevision: "REPO_REVISION" + addonsRepoBasepath: "REPO_BASEPATH" + fleetRepoURL: "REPO_URL" + fleetRepoRevision: "REPO_REVISION" + fleetRepoBasepath: "REPO_BASEPATH" + # Cluster identity (gitops-bridge contract — consumed by AppSets) + aws_cluster_name: "CLUSTER_NAME" + aws_account_id: "AWS_ACCOUNT_ID_VAL" + aws_region: "AWS_REGION_VAL" + aws_vpc_id: "VPC_ID_VAL" + admin_role_name: "ADMIN_ROLE_NAME_VAL" + resource_prefix: "RESOURCE_PREFIX_VAL" + # Ingress / edge metadata (gitops-bridge contract — consumed by AppSets) + ingress_domain_name: "EDGE_DOMAIN" + ingress_name: "RESOURCE_PREFIX_VAL-hub-ingress" + ingress_security_groups: "INGRESS_SECURITY_GROUPS_VAL" + alb_controller_mode: "oss" + # Exposure metadata (PeEKS extension — consumed by AppExposure RGD) + exposure_mode: "EXPOSURE_MODE" + alb_listener_arn: "ALB_LISTENER_ARN" + labels: + argocd.argoproj.io/secret-type: cluster + environment: control-plane + fleet_member: control-plane + enable_kroack_abstractions: "true" + exposure_mode: "EXPOSURE_MODE" + stringData: + name: CLUSTER_NAME + server: "CLUSTER_ARN" + config: '{"awsAuthConfig":{"clusterName":"CLUSTER_NAME","roleARN":"ARGOCD_ROLE_ARN"},"tlsClientConfig":{"insecure":false}}' + EOF + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + + hub:apply-root-appset: + desc: Apply root-appset to hub + cmds: + - task: hub:kubeconfig + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f {{.GITOPS_ROOT}}/bootstrap/root-appset.yaml + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + - printf '{{.C_OK}}✓ Root appset applied. Hub is now self-managing.{{.C_RESET}}\n' + + hub:wait-for-sync: + desc: Wait for hub ArgoCD apps to sync + cmds: + - task: hub:kubeconfig + - printf '{{.C_STEP}}▸ Waiting for apps to appear...{{.C_RESET}}\n' + - | + for i in $(seq 1 30); do + TOTAL=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get app -n argocd --no-headers 2>/dev/null | wc -l) + [ "$TOTAL" -gt 3 ] && break + sleep 10 + done + - printf '{{.C_STEP}}▸ Waiting for apps to sync...{{.C_RESET}}\n' + - | + TIMEOUT=1800; ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + TOTAL=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get app -n argocd --no-headers 2>/dev/null | wc -l) + SYNCED=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get app -n argocd --no-headers 2>/dev/null | awk '{print $2}' | grep -c "Synced" || true) + OUTOFSYNC=$((TOTAL - SYNCED)) + printf '{{.C_INFO}} %s/%s synced (%ss){{.C_RESET}}\n' "$SYNCED" "$TOTAL" "$ELAPSED" + [ "$TOTAL" -gt 0 ] && [ "$OUTOFSYNC" -le 2 ] && break + sleep 15; ELAPSED=$((ELAPSED + 15)) + done + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + + status: + desc: Check bootstrap status + cmds: + - printf '{{.C_INFO}}=== Kind ==={{.C_RESET}}\n' && kind get clusters 2>/dev/null | grep "{{.KIND_CLUSTER_NAME}}" || echo "Not running" + - printf '{{.C_INFO}}=== Helm ==={{.C_RESET}}\n' && helm list -A 2>/dev/null || echo "None" + - printf '{{.C_INFO}}=== KRO RGDs ==={{.C_RESET}}\n' && kubectl get resourcegraphdefinitions 2>/dev/null || echo "None" + - printf '{{.C_INFO}}=== KRO Instances ==={{.C_RESET}}\n' && kubectl get eksclusterwithvpc -A 2>/dev/null || echo "None" + + destroy-kind: + desc: Delete Kind cluster only (hub persists) + status: + - "! kind get clusters 2>/dev/null | grep -q '^{{.KIND_CLUSTER_NAME}}$'" + cmds: + - kind delete cluster --name {{.KIND_CLUSTER_NAME}} + + destroy: + desc: Full teardown — delete KRO instances, wait for ACK to clean up AWS resources, delete Kind + vars: + CLEANUP_KIND: '{{.CLEANUP_KIND | default "false"}}' + cmds: + - kind get kubeconfig --name {{.KIND_CLUSTER_NAME}} > {{.KUBECONFIG}} + - task: credentials:refresh + # Delete the hub claim — KRO cascades to ACK which deletes AWS resources + - printf '{{.C_ERR}}▸ Deleting hub cluster claim...{{.C_RESET}}\n' + - kubectl delete eksclusterwithvpc {{.HUB_CLUSTER_NAME}} -n {{.HUB_CLUSTER_NAME}} --timeout=60s 2>/dev/null || true + - printf '{{.C_STEP}}▸ Waiting for EKS cluster deletion...{{.C_RESET}}\n' + - | + for i in $(seq 1 120); do + EXISTS=$(kubectl get ekscluster {{.HUB_CLUSTER_NAME}} -n {{.HUB_CLUSTER_NAME}} 2>/dev/null && echo "yes" || echo "no") + [ "$EXISTS" = "no" ] && break + printf '{{.C_INFO}} [%s/120] Waiting for deletion...{{.C_RESET}}\n' "$i" + sleep 15 + done + # Clean up Secrets Manager + - aws secretsmanager delete-secret --secret-id {{.HUB_CLUSTER_NAME}}/config --force-delete-without-recovery --region {{.AWS_REGION}} 2>/dev/null || true + - aws secretsmanager delete-secret --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --force-delete-without-recovery --region {{.AWS_REGION}} 2>/dev/null || true + - cmd: | + if [ "{{.CLEANUP_KIND}}" = "true" ]; then + kind delete cluster --name {{.KIND_CLUSTER_NAME}} + else + echo "Kind preserved. Run 'task destroy-kind' to remove." + fi + - printf '{{.C_OK}}✓ Destroy complete.{{.C_RESET}}\n' diff --git a/cluster-providers/kind-kro-ack/kind.yaml b/cluster-providers/kind-kro-ack/kind.yaml new file mode 100644 index 000000000..f64b87e8e --- /dev/null +++ b/cluster-providers/kind-kro-ack/kind.yaml @@ -0,0 +1,4 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane diff --git a/cluster-providers/kind-kro-ack/manifests/argocd/create-capability.yaml b/cluster-providers/kind-kro-ack/manifests/argocd/create-capability.yaml new file mode 100644 index 000000000..3594bdfaf --- /dev/null +++ b/cluster-providers/kind-kro-ack/manifests/argocd/create-capability.yaml @@ -0,0 +1,98 @@ +# Job: Creates EKS ArgoCD capability after cluster is ACTIVE. +# Env vars injected from argocd-capability-config ConfigMap. +apiVersion: batch/v1 +kind: Job +metadata: + name: create-argocd-capability + namespace: ack-system +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 600 + template: + spec: + restartPolicy: OnFailure + volumes: + - name: aws-creds + secret: + secretName: aws-credentials + containers: + - name: create-capability + image: amazon/aws-cli:latest + envFrom: + - configMapRef: + name: argocd-capability-config + env: + - name: AWS_SHARED_CREDENTIALS_FILE + value: /tmp/.aws/credentials + volumeMounts: + - name: aws-creds + mountPath: /tmp/.aws + readOnly: true + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + + echo "=== Wait for EKS cluster ACTIVE ===" + aws eks wait cluster-active \ + --name "$CLUSTER_NAME" --region "$AWS_REGION" + echo "Cluster $CLUSTER_NAME is ACTIVE" + + echo "=== Create ArgoCD capability ===" + EXISTING=$(aws eks describe-capability \ + --cluster-name "$CLUSTER_NAME" \ + --capability-name "$CAPABILITY_NAME" \ + --region "$AWS_REGION" \ + --query 'capability.status' \ + --output text 2>/dev/null || echo "NOT_FOUND") + + if [ "$EXISTING" = "NOT_FOUND" ]; then + aws eks create-capability \ + --region "$AWS_REGION" \ + --cluster-name "$CLUSTER_NAME" \ + --capability-name "$CAPABILITY_NAME" \ + --type ARGOCD \ + --role-arn "$CAPABILITY_ROLE_ARN" \ + --delete-propagation-policy RETAIN \ + --configuration "{ + \"argoCd\": { + \"awsIdc\": { + \"idcInstanceArn\": \"$IDC_INSTANCE_ARN\", + \"idcRegion\": \"$IDC_REGION\" + }, + \"rbacRoleMappings\": [{ + \"role\": \"ADMIN\", + \"identities\": [{ + \"id\": \"$IDC_ADMIN_GROUP_ID\", + \"type\": \"SSO_GROUP\" + }] + }] + } + }" + else + echo "Capability exists: $EXISTING" + fi + + echo "=== Wait for ACTIVE ===" + for i in $(seq 1 60); do + STATUS=$(aws eks describe-capability \ + --cluster-name "$CLUSTER_NAME" \ + --capability-name "$CAPABILITY_NAME" \ + --region "$AWS_REGION" \ + --query 'capability.status' \ + --output text 2>/dev/null || echo "UNKNOWN") + echo "Status: $STATUS ($i/60)" + [ "$STATUS" = "ACTIVE" ] && break + sleep 30 + done + [ "$STATUS" != "ACTIVE" ] && exit 1 + + echo "=== Associate access policy ===" + aws eks associate-access-policy \ + --region "$AWS_REGION" \ + --cluster-name "$CLUSTER_NAME" \ + --principal-arn "$CAPABILITY_ROLE_ARN" \ + --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy \ + --access-scope type=cluster 2>/dev/null || true + + echo "=== Done ===" diff --git a/cluster-providers/kind-kro-ack/manifests/argocd/delete-capability.yaml b/cluster-providers/kind-kro-ack/manifests/argocd/delete-capability.yaml new file mode 100644 index 000000000..864e3852d --- /dev/null +++ b/cluster-providers/kind-kro-ack/manifests/argocd/delete-capability.yaml @@ -0,0 +1,65 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: delete-argocd-capability + namespace: ack-system +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 600 + template: + spec: + restartPolicy: OnFailure + volumes: + - name: aws-creds + secret: + secretName: aws-credentials + containers: + - name: delete-capability + image: amazon/aws-cli:latest + envFrom: + - configMapRef: + name: argocd-capability-config + env: + - name: AWS_SHARED_CREDENTIALS_FILE + value: /tmp/.aws/credentials + volumeMounts: + - name: aws-creds + mountPath: /tmp/.aws + readOnly: true + command: ["/bin/bash", "-c"] + args: + - | + set -euo pipefail + + echo "=== Delete ArgoCD capability ===" + STATUS=$(aws eks describe-capability \ + --cluster-name "$CLUSTER_NAME" \ + --capability-name "$CAPABILITY_NAME" \ + --region "$AWS_REGION" \ + --query 'capability.status' \ + --output text 2>/dev/null || echo "NOT_FOUND") + + if [ "$STATUS" = "NOT_FOUND" ]; then + echo "Capability not found — nothing to delete" + exit 0 + fi + + aws eks delete-capability \ + --cluster-name "$CLUSTER_NAME" \ + --capability-name "$CAPABILITY_NAME" \ + --region "$AWS_REGION" + + echo "=== Wait for deletion ===" + for i in $(seq 1 60); do + STATUS=$(aws eks describe-capability \ + --cluster-name "$CLUSTER_NAME" \ + --capability-name "$CAPABILITY_NAME" \ + --region "$AWS_REGION" \ + --query 'capability.status' \ + --output text 2>/dev/null || echo "DELETED") + echo "Status: $STATUS ($i/60)" + [ "$STATUS" = "DELETED" ] && break + sleep 30 + done + + echo "=== Done ===" diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml new file mode 100644 index 000000000..76768f1cc --- /dev/null +++ b/cluster-providers/workshop/Taskfile.yaml @@ -0,0 +1,172 @@ +version: "3" +set: [errexit, nounset, pipefail] +silent: true + +# Workshop-specific Taskfile for Platform Engineering on EKS (PEEKS). +# Covers workshop pre-requisites and post-install steps that are NOT +# part of the generic appmod-blueprints platform installation. +# +# Included by the root Taskfile under the 'workshop:' namespace. + +vars: + CONFIG_FILE: "{{.ROOT_DIR}}/config.local.yaml" + AWS_REGION: + sh: yq '.aws.region' {{.CONFIG_FILE}} + AWS_ACCOUNT_ID: + sh: yq '.aws.accountId' {{.CONFIG_FILE}} + HUB_CLUSTER_NAME: + sh: yq '.hub.clusterName' {{.CONFIG_FILE}} + RESOURCE_PREFIX: + sh: yq '.resourcePrefix // "peeks"' {{.CONFIG_FILE}} + IDC_INSTANCE_ARN: + sh: yq '.identityCenter.instanceArn' {{.CONFIG_FILE}} + IDC_REGION: + sh: yq '.identityCenter.region' {{.CONFIG_FILE}} + IDC_ADMIN_GROUP_ID: + sh: yq '.identityCenter.adminGroupId' {{.CONFIG_FILE}} + GIT_USERNAME: + sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} + EXPOSURE_MODE: + sh: 'if [ -z "$(yq ''.domain // ""'' {{.CONFIG_FILE}})" ]; then echo "cloudfront"; else echo "domain"; fi' + CLOUDFRONT_DOMAIN: + sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + C_STEP: + sh: printf '\033[1;33m' + C_OK: + sh: printf '\033[1;32m' + C_ERR: + sh: printf '\033[1;31m' + C_INFO: + sh: printf '\033[1;36m' + C_RESET: + sh: printf '\033[0m' + +tasks: + # ============================================================ + # PRE-REQUISITES + # Run before task install to prepare workshop-specific resources + # ============================================================ + + pre-install: + desc: Verify workshop pre-requisites (normally provisioned by CDK in guided events) + cmds: + - | + printf '{{.C_INFO}}Workshop pre-requisites check:{{.C_RESET}}\n' + printf ' In Workshop Studio (guided) events:\n' + printf ' 1. CDK bootstrap.sh provisions CloudFront + GitLab CE, writes private/gitlab-cloudfront-domain\n' + printf ' 2. SSM SetupIDE runs create-config.sh which generates config.local.yaml\n' + printf ' (auto-detects region, account, IDC, and reads cloudfront.cloudfrontDomain\n' + printf ' from private/gitlab-cloudfront-domain)\n' + printf ' 3. task install + task workshop:post-install run automatically\n' + printf ' For self-paced: run create-config.sh after manually creating CloudFront + ALB\n' + if [ -z "{{.CLOUDFRONT_DOMAIN}}" ]; then + printf '{{.C_ERR}}ERROR: cloudfront.cloudfrontDomain not set in config.local.yaml{{.C_RESET}}\n' + printf ' Set: cloudfront:\n' + printf ' cloudfrontDomain: .cloudfront.net\n' + exit 1 + fi + printf '{{.C_OK}}✓ cloudfront.cloudfrontDomain: {{.CLOUDFRONT_DOMAIN}}{{.C_RESET}}\n' + + # ============================================================ + # POST-INSTALL + # Run after task install to complete workshop-specific setup + # ============================================================ + + post-install: + desc: Workshop post-install steps (IDC/Keycloak federation for ArgoCD SSO) + cmds: + - task: idc:configure + + # ============================================================ + # IDC / KEYCLOAK FEDERATION + # Configures IAM Identity Center with Keycloak as external IdP + # ============================================================ + + idc:configure: + desc: Configure IDC with Keycloak as external IdP and sync users via SCIM + status: + - | + IDENTITY_STORE_ID=$(aws sso-admin list-instances --region {{.AWS_REGION}} \ + --query "Instances[?InstanceArn=='{{.IDC_INSTANCE_ARN}}'].IdentityStoreId" --output text 2>/dev/null) + aws identitystore list-users --identity-store-id "$IDENTITY_STORE_ID" --region {{.AWS_REGION}} \ + --filters "AttributePath=UserName,AttributeValue={{.GIT_USERNAME}}" \ + --query "Users[0].UserId" --output text 2>/dev/null | grep -qv "^None\|^$" + vars: + IDC_INSTANCE_ID: + sh: echo "{{.IDC_INSTANCE_ARN}}" | awk -F'/' '{print $NF}' | sed 's/^ssoins-//' + KC_ADMIN_PASSWORD: + sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text | jq -r '.keycloak_admin_password' + KEYCLOAK_DNS: + sh: | + if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -n "{{.CLOUDFRONT_DOMAIN}}" ]; then + echo "{{.CLOUDFRONT_DOMAIN}}" + else + echo "$(yq '.domain' {{.CONFIG_FILE}})" + fi + cmds: + - printf '{{.C_STEP}}▸ Refreshing IDC integration credentials via Lambda...{{.C_RESET}}\n' + - | + LAMBDA_ARN=$(aws cloudformation describe-stacks --region {{.AWS_REGION}} \ + --query "Stacks[?contains(StackName,'peeks-ide')].Outputs[?OutputKey=='KeycloakIDCCredentialsLambdaArn'].OutputValue | [0][0]" --output text 2>/dev/null) + if [ -n "$LAMBDA_ARN" ] && [ "$LAMBDA_ARN" != "None" ]; then + ROLE_ARN=$(aws cloudformation describe-stacks --region {{.AWS_REGION}} \ + --query "Stacks[?contains(StackName,'peeks-ide')].Outputs[?OutputKey=='SharedRoleArn'].OutputValue | [0][0]" --output text 2>/dev/null) + [ -z "$ROLE_ARN" ] || [ "$ROLE_ARN" = "None" ] && \ + ROLE_ARN=$(aws iam list-roles --query "Roles[?contains(RoleName,'SharedRole')].Arn | [0]" --output text) + PAYLOAD=$(jq -n --arg role "$ROLE_ARN" --arg prefix "/{{.RESOURCE_PREFIX}}/keycloak-idc-integration-credentials" \ + '{RequestType:"Update",ResourceProperties:{RoleArn:$role,ParameterPrefix:$prefix,SessionDuration:"3600"}}') + aws lambda invoke --function-name "$LAMBDA_ARN" --payload "$PAYLOAD" --region {{.AWS_REGION}} /tmp/lambda-response.json > /dev/null + echo "Credentials refreshed via Lambda" + else + echo "WARN: Lambda not found. Falling back to instance profile." + TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600") + ROLE=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/iam/security-credentials/) + CREDS=$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/iam/security-credentials/$ROLE) + aws ssm put-parameter \ + --name "/{{.RESOURCE_PREFIX}}/keycloak-idc-integration-credentials" \ + --value "$(echo "$CREDS" | jq '{AccessKeyId:.AccessKeyId,SecretAccessKey:.SecretAccessKey,SessionToken:.Token,Expiration:.Expiration}')" \ + --type "SecureString" --overwrite --region {{.AWS_REGION}} > /dev/null + fi + - printf '{{.C_STEP}}▸ Installing Playwright...{{.C_RESET}}\n' + - pip install playwright boto3 requests --quiet 2>/dev/null + - python3 -m playwright install chromium --quiet 2>/dev/null || python3 -m playwright install chromium + - sudo yum install -y atk at-spi2-atk cups-libs libdrm libxkbcommon libXcomposite libXdamage libXrandr mesa-libgbm pango alsa-lib nss nspr libXScrnSaver libXtst gtk3 --quiet 2>/dev/null || true + - printf '{{.C_STEP}}▸ Running IDC ↔ Keycloak SAML/SCIM configuration...{{.C_RESET}}\n' + - aws ssm get-parameter --name "/{{.RESOURCE_PREFIX}}/keycloak-idc-integration-credentials" --with-decryption --region {{.AWS_REGION}} --query 'Parameter.Value' --output text > /tmp/keycloak-idc-integration-credentials.json + - | + IDC_SUCCESS=false + for attempt in 1 2 3; do + printf '{{.C_INFO}} Attempt %s/3...{{.C_RESET}}\n' "$attempt" + if python3 {{.ROOT_DIR}}/platform/infra/terraform/scripts/configure_identity_center.py \ + --region {{.AWS_REGION}} \ + --instance-id {{.IDC_INSTANCE_ID}} \ + --keycloak-dns {{.KEYCLOAK_DNS}} \ + --keycloak-admin-password="{{.KC_ADMIN_PASSWORD}}"; then + IDC_SUCCESS=true; break + fi + [ $attempt -lt 3 ] && sleep 60 + done + if [ "$IDC_SUCCESS" = true ]; then + printf '{{.C_OK}}✓ IDC configured with Keycloak federation and SCIM sync.{{.C_RESET}}\n' + else + printf '{{.C_ERR}}⚠ IDC configuration failed. Run manually: task workshop:idc:configure{{.C_RESET}}\n' + fi + - printf '{{.C_STEP}}▸ Syncing admin group members into IDC admin group...{{.C_RESET}}\n' + - | + IDENTITY_STORE_ID=$(aws sso-admin list-instances --region {{.IDC_REGION}} --query 'Instances[0].IdentityStoreId' --output text | head -1) + SCIM_ADMIN_GROUP=$(aws identitystore list-groups --identity-store-id "$IDENTITY_STORE_ID" --region {{.IDC_REGION}} \ + --query "Groups[?DisplayName=='admin'].GroupId | [0]" --output text 2>/dev/null) + if [ -n "$SCIM_ADMIN_GROUP" ] && [ "$SCIM_ADMIN_GROUP" != "None" ]; then + EXISTING_MEMBERS=$(aws identitystore list-group-memberships --identity-store-id "$IDENTITY_STORE_ID" \ + --group-id "{{.IDC_ADMIN_GROUP_ID}}" --region {{.IDC_REGION}} --query 'GroupMemberships[].MemberId.UserId' --output text 2>/dev/null) + for USER_ID in $(aws identitystore list-group-memberships --identity-store-id "$IDENTITY_STORE_ID" \ + --group-id "$SCIM_ADMIN_GROUP" --region {{.IDC_REGION}} --query 'GroupMemberships[].MemberId.UserId' --output text 2>/dev/null); do + echo "$EXISTING_MEMBERS" | grep -q "$USER_ID" && continue + aws identitystore create-group-membership --identity-store-id "$IDENTITY_STORE_ID" \ + --group-id "{{.IDC_ADMIN_GROUP_ID}}" --member-id UserId="$USER_ID" --region {{.IDC_REGION}} 2>/dev/null || true + echo "Added user $USER_ID to IDC admin group" + done + printf '{{.C_OK}}✓ Admin group members synced for ArgoCD access.{{.C_RESET}}\n' + else + printf '{{.C_ERR}}⚠ SCIM admin group not found.{{.C_RESET}}\n' + fi diff --git a/config.schema.json b/config.schema.json index e3fe665fa..f52d33914 100644 --- a/config.schema.json +++ b/config.schema.json @@ -138,6 +138,25 @@ "default": "", "description": "Comma-separated security group IDs for the ALB" }, + "cloudfront": { + "type": "object", + "description": "CloudFront mode pre-requisites. Required when domain is empty.", + "properties": { + "vpcId": { + "type": "string", + "pattern": "^vpc-[0-9a-f]+$", + "description": "Pre-created VPC ID with min 2 public + 2 private subnets" + }, + "cloudfrontDomain": { + "type": "string", + "description": "CloudFront distribution domain (e.g. xxxx.cloudfront.net)" + } + } + }, + "adminRoleName": { + "type": "string", + "description": "IAM role name added as cluster admin and used by Backstage templates (e.g. WSParticipantRole)" + }, "identityCenter": { "type": "object", "required": ["instanceArn", "region", "adminGroupId"], diff --git a/config.yaml b/config.yaml index 097435741..3ba3a68ba 100644 --- a/config.yaml +++ b/config.yaml @@ -35,11 +35,18 @@ aws: profile: "default" # AWS CLI profile (local dev only) # Domain and networking -domain: "" # e.g. idp.example.com — leave empty for CloudFront mode (auto-created) +# Leave domain empty to use CloudFront mode — provide cloudfront.vpcId + cloudfront.cloudfrontDomain below. +domain: "" # e.g. idp.example.com — leave empty for CloudFront mode resourcePrefix: "" # Used for KRO tenant-facing resources only ingressName: "" # Defaults to {clusterName}-ingress if empty ingressSecurityGroups: "" +# CloudFront mode pre-requisites (required when domain is empty) +# These are created before running task install (see docs/platform/cloudfront-setup.md). +# cloudfront: +# vpcId: "vpc-xxxxxxxxxxxxxxxxx" # VPC with min 2 public + 2 private subnets +# cloudfrontDomain: "xxxx.cloudfront.net" # CloudFront distribution domain for the platform ALB + # AWS Identity Center (required for EKS ArgoCD Capability) identityCenter: instanceArn: "" diff --git a/create-config.sh b/create-config.sh index 03673d89e..cef3ab608 100755 --- a/create-config.sh +++ b/create-config.sh @@ -156,6 +156,18 @@ printf 'domain: ""\n' >> "$OUTPUT_FILE" printf 'resourcePrefix: "%s"\n' "$RESOURCE_PREFIX" >> "$OUTPUT_FILE" printf 'ingressName: ""\n' >> "$OUTPUT_FILE" printf 'ingressSecurityGroups: ""\n' >> "$OUTPUT_FILE" + +# CloudFront domain — read from CLOUDFRONT_DOMAIN env var (preferred, set by SSM from IDE_DOMAIN) +# with fallback to the private/gitlab-cloudfront-domain file written by bootstrap.sh. +CF_DOMAIN="${CLOUDFRONT_DOMAIN:-${IDE_DOMAIN:-}}" +if [ -z "$CF_DOMAIN" ]; then + PRIVATE_DIR="${SCRIPT_DIR}/private" + [ -f "${PRIVATE_DIR}/gitlab-cloudfront-domain" ] && CF_DOMAIN="$(cat "${PRIVATE_DIR}/gitlab-cloudfront-domain" | tr -d '[:space:]')" +fi +if [ -n "$CF_DOMAIN" ]; then + printf 'cloudfront:\n' >> "$OUTPUT_FILE" + printf ' cloudfrontDomain: "%s"\n' "$CF_DOMAIN" >> "$OUTPUT_FILE" +fi printf 'identityCenter:\n' >> "$OUTPUT_FILE" printf ' instanceArn: "%s"\n' "$IDC_ARN" >> "$OUTPUT_FILE" printf ' region: "%s"\n' "$REGION" >> "$OUTPUT_FILE" @@ -174,4 +186,4 @@ echo "✓ config.local.yaml created:" echo " region=$REGION accountId=$ACCOUNT_ID prefix=$RESOURCE_PREFIX" echo " clusterName=${RESOURCE_PREFIX}-hub adminRole=$ADMIN_ROLE_NAME" echo " idcInstance=$IDC_ARN adminGroupId=${IDC_GROUP:-}" -echo ' domain="" (CloudFront exposure mode)' +echo " domain=\"\" (CloudFront exposure mode) cloudfrontDomain=${CF_DOMAIN:-}" diff --git a/gitops/abstractions/crossplane/platform-cluster/templates/composition.yaml b/gitops/abstractions/crossplane/platform-cluster/templates/composition.yaml index 32d3eb60f..ab3521d52 100644 --- a/gitops/abstractions/crossplane/platform-cluster/templates/composition.yaml +++ b/gitops/abstractions/crossplane/platform-cluster/templates/composition.yaml @@ -945,7 +945,12 @@ spec: # Uses provider-kubernetes Object with Observe+Update so the Secret must # already exist (created by the fleet-secret Helm chart) — the Object only # patches the single annotation via Server-Side Apply. + # readinessCheck: none — the argocd namespace is created by the EKS ArgoCD + # Capability *after* the XR reaches Ready. Blocking XR readiness on this + # Object creates a circular dependency that stalls the install for 90+ min. - name: cluster-secret-vpc-annotation + readinessChecks: + - type: None base: apiVersion: kubernetes.crossplane.io/v1alpha2 kind: Object diff --git a/gitops/abstractions/resource-groups-kro/platform-cluster-kro/Chart.yaml b/gitops/abstractions/resource-groups-kro/platform-cluster-kro/Chart.yaml new file mode 100644 index 000000000..e215ec9a0 --- /dev/null +++ b/gitops/abstractions/resource-groups-kro/platform-cluster-kro/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: platform-cluster-kro +description: KRO/ACK-based EKS cluster provisioning (alternative to Crossplane platform-cluster) +type: application +version: 0.1.0 diff --git a/gitops/abstractions/resource-groups-kro/platform-cluster-kro/templates/claim.yaml b/gitops/abstractions/resource-groups-kro/platform-cluster-kro/templates/claim.yaml new file mode 100644 index 000000000..d39342eb0 --- /dev/null +++ b/gitops/abstractions/resource-groups-kro/platform-cluster-kro/templates/claim.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Values.clusterName }} +--- +apiVersion: kro.run/v1alpha1 +kind: EksclusterWithVpc +metadata: + name: {{ .Values.clusterName }} + namespace: {{ .Values.clusterName }} +spec: + name: {{ .Values.clusterName }} + tenant: {{ .Values.tenant | default "workshop" }} + environment: {{ .Values.environment | default "dev" }} + region: {{ .Values.region | default $.Values.global.region | default "us-west-2" }} + k8sVersion: {{ .Values.kubernetesVersion | default "1.32" | quote }} + accountId: {{ .Values.accountId | default $.Values.global.accountId | quote }} + managementAccountId: {{ .Values.managementAccountId | default $.Values.global.accountId | quote }} + argoCdHubRoleArn: {{ .Values.argoCdHubRoleArn | default $.Values.global.argoCDRoleArn | default "" }} + adminRoleName: {{ .Values.adminRoleName | default $.Values.global.adminRoleName | default "Admin" }} + fleetSecretManagerSecretNameSuffix: {{ .Values.fleetSecretManagerSecretNameSuffix | default "argocd-secret" }} + domainName: {{ .Values.domainName | default $.Values.global.domainName | default "example.com" }} + resourcePrefix: {{ .Values.resourcePrefix | default $.Values.global.resourcePrefix | default "peeks" }} + cidr: + vpcCidr: {{ .Values.vpcCidr | default "10.0.0.0/16" | quote }} + publicSubnet1Cidr: {{ .Values.publicSubnet1Cidr | default "10.0.1.0/24" | quote }} + publicSubnet2Cidr: {{ .Values.publicSubnet2Cidr | default "10.0.2.0/24" | quote }} + privateSubnet1Cidr: {{ .Values.privateSubnet1Cidr | default "10.0.11.0/24" | quote }} + privateSubnet2Cidr: {{ .Values.privateSubnet2Cidr | default "10.0.12.0/24" | quote }} + gitops: + addonsRepoUrl: {{ .Values.addonsRepoUrl | default $.Values.global.repoUrl }} + addonsRepoRevision: {{ .Values.addonsRepoRevision | default $.Values.global.repoRevision }} + addonsRepoBasePath: {{ .Values.addonsRepoBasePath | default $.Values.global.repoBasePath | default "gitops/addons/" }} + addonsRepoPath: bootstrap + fleetRepoUrl: {{ .Values.fleetRepoUrl | default $.Values.global.repoUrl }} + fleetRepoRevision: {{ .Values.fleetRepoRevision | default $.Values.global.repoRevision }} + fleetRepoBasePath: {{ .Values.fleetRepoBasePath | default $.Values.global.repoBasePath | default "gitops/fleet/" }} + fleetRepoPath: bootstrap + platformRepoUrl: {{ .Values.platformRepoUrl | default $.Values.global.repoUrl }} + platformRepoRevision: {{ .Values.platformRepoRevision | default $.Values.global.repoRevision }} + platformRepoBasePath: {{ .Values.platformRepoBasePath | default $.Values.global.repoBasePath | default "gitops/platform/" }} + platformRepoPath: bootstrap + workloadRepoUrl: {{ .Values.workloadRepoUrl | default $.Values.global.repoUrl }} + workloadRepoRevision: {{ .Values.workloadRepoRevision | default $.Values.global.repoRevision }} + workloadRepoBasePath: {{ .Values.workloadRepoBasePath | default $.Values.global.repoBasePath | default "gitops/apps/" }} + workloadRepoPath: "" diff --git a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml index 30642f7f5..2dda7ba18 100644 --- a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml +++ b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml @@ -60,14 +60,6 @@ rules: - apiGroups: ["iam.services.k8s.aws"] resources: ["*"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -# ACK adoption API (IAMRoleSelector, etc.) -- apiGroups: ["services.k8s.aws"] - resources: ["*"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] -# ACK DynamoDB -- apiGroups: ["dynamodb.services.k8s.aws"] - resources: ["*"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # ACK ECR - apiGroups: ["ecr.services.k8s.aws"] resources: ["*"] diff --git a/gitops/addons/charts/keycloak/templates/keycloak-config.yaml b/gitops/addons/charts/keycloak/templates/keycloak-config.yaml index 59e6cb8ec..86f871ac5 100644 --- a/gitops/addons/charts/keycloak/templates/keycloak-config.yaml +++ b/gitops/addons/charts/keycloak/templates/keycloak-config.yaml @@ -330,35 +330,6 @@ data: "https://${DOMAIN_NAME}" ] } - gitlab-client-payload.json: | - { - "protocol": "openid-connect", - "clientId": "gitlab", - "name": "GitLab Client", - "description": "Used for GitLab SSO with PKCE", - "publicClient": true, - "authorizationServicesEnabled": false, - "serviceAccountsEnabled": false, - "implicitFlowEnabled": false, - "directAccessGrantsEnabled": true, - "standardFlowEnabled": true, - "frontchannelLogout": true, - "attributes": { - "saml_idp_initiated_sso_url_name": "", - "oauth2.device.authorization.grant.enabled": false, - "oidc.ciba.grant.enabled": false, - "pkce.code.challenge.method": "S256" - }, - "alwaysDisplayInConsole": false, - "rootUrl": "https://${GITLAB_DOMAIN_NAME}", - "baseUrl": "", - "redirectUris": [ - "https://${GITLAB_DOMAIN_NAME}/users/auth/openid_connect/callback" - ], - "webOrigins": [ - "/*" - ] - } argocd-client-payload.json: | { "protocol": "openid-connect", @@ -769,49 +740,6 @@ spec: keycloak_post "/admin/realms/${REALM_NAME}/clients" "${payload_file}" "${client_name} client" } - # Upsert client matched by its stable "name" field. - # Needed for clients whose clientId can change between runs — e.g. - # amazon-managed-grafana, whose clientId is the AMG workspace endpoint. - # On a fresh bootstrap the grafana_url annotation may not be seeded yet, - # so the client is first created with the fallback (/grafana). - # Once aws_grafana_url is present, a re-sync (once keycloak re-runs) - # repairs the clientId/adminUrl/redirectUris via PUT instead of leaving - # the stale value that breaks SAML login. - upsert_client() { - local match_name=$1 - local payload_file=$2 - - envsubst < ${payload_file} > /tmp/upsert-payload.json - - EXISTING_ID=$(curl -sS \ - -H "Content-Type: application/json" \ - -H "Authorization: bearer ${KEYCLOAK_TOKEN}" \ - "${KEYCLOAK_URL}/admin/realms/${REALM_NAME}/clients" \ - | jq -r ".[] | select(.name == \"${match_name}\") | .id" 2>/dev/null || true) - - if [ -n "${EXISTING_ID}" ]; then - echo "upserting ${match_name} client (id=${EXISTING_ID})" - HTTP_STATUS=$(curl -sS -w "%{http_code}" -o /tmp/response.txt \ - -H "Content-Type: application/json" \ - -H "Authorization: bearer ${KEYCLOAK_TOKEN}" \ - -X PUT --data @/tmp/upsert-payload.json \ - "${KEYCLOAK_URL}/admin/realms/${REALM_NAME}/clients/${EXISTING_ID}") - if [ "$HTTP_STATUS" -ne 204 ] && [ "$HTTP_STATUS" -ne 200 ]; then - echo "Error upserting ${match_name}: HTTP $HTTP_STATUS"; cat /tmp/response.txt - fi - else - echo "creating ${match_name} client (not found)" - HTTP_STATUS=$(curl -sS -w "%{http_code}" -o /tmp/response.txt \ - -H "Content-Type: application/json" \ - -H "Authorization: bearer ${KEYCLOAK_TOKEN}" \ - -X POST --data @/tmp/upsert-payload.json \ - "${KEYCLOAK_URL}/admin/realms/${REALM_NAME}/clients") - if [ "$HTTP_STATUS" -ne 201 ] && [ "$HTTP_STATUS" -ne 200 ]; then - echo "Error creating ${match_name}: HTTP $HTTP_STATUS"; cat /tmp/response.txt - fi - fi - } - # Add client scope function add_client_scope() { local client_id_name=$1 @@ -938,8 +866,7 @@ spec: create_client "Langfuse" "/var/config/langfuse-client-payload.json" # create_client "Argo CD" "/var/config/argocd-client-payload.json" # Disabled - using EKS Marina managed ArgoCD create_client "Account Console" "/var/config/account-console-client-payload.json" - upsert_client "amazon-managed-grafana" "/var/config/grafana-client-payload.json" - create_client "GitLab" "/var/config/gitlab-client-payload.json" + create_client "Amazon Grafana" "/var/config/grafana-client-payload.json" echo "adding client scopes to clients" add_client_scope "argo-workflows" "Argo Workflows" @@ -952,7 +879,6 @@ spec: add_client_scope "kargo" "Kargo" # add_client_scope "argocd" "Argo CD" # Disabled - using EKS Marina managed ArgoCD add_client_scope "account-console" "Account Console" - add_client_scope "gitlab" "GitLab" add_client_scope "mcp-client" "MCP Client" echo "adding user1 to admin group" diff --git a/gitops/addons/charts/kro/resource-groups/manifests/ray-service/ray-service.yaml b/gitops/addons/charts/kro/resource-groups/manifests/ray-service/ray-service.yaml index 0dda53de8..f2b39a094 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/ray-service/ray-service.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/ray-service/ray-service.yaml @@ -37,9 +37,9 @@ spec: - region ${schema.spec.?awsRegion.orValue('us-west-2')} csi: driver: s3.csi.aws.com - volumeHandle: ${schema.spec.?s3ModelBucket.orValue('peeks-hub-ray-models')} + volumeHandle: ${schema.spec.?s3ModelBucket.orValue('peeks-ray-models')} volumeAttributes: - bucketName: ${schema.spec.?s3ModelBucket.orValue('peeks-hub-ray-models')} + bucketName: ${schema.spec.?s3ModelBucket.orValue('peeks-ray-models')} # S3 PersistentVolumeClaim - id: s3pvc @@ -690,7 +690,7 @@ spec: name: string modelName: string | default="tinyllama" maxLength: string | default="100" - s3ModelBucket: string | default="peeks-hub-ray-models" + s3ModelBucket: string | default="peeks-ray-models" awsRegion: string | default="us-west-2" awsAccountId: string | default="ACCOUNT_ID" resourcePrefix: string | default="peeks" diff --git a/gitops/addons/charts/kubevela/templates/components/appmod-service.yaml b/gitops/addons/charts/kubevela/templates/components/appmod-service.yaml index 7ddcbdd4a..b3efbd28f 100644 --- a/gitops/addons/charts/kubevela/templates/components/appmod-service.yaml +++ b/gitops/addons/charts/kubevela/templates/components/appmod-service.yaml @@ -149,17 +149,6 @@ spec: resources: parameter.resources } }] - // Wait for EKS Pod Identity credentials before the app starts. - // 169.254.170.23/v1/credentials is served by the agent only after the - // PodIdentityAssociation is active — eliminates the startup race. - if parameter.serviceAccount != _|_ { - initContainers: [{ - name: "wait-pod-identity" - image: "busybox:1.36" - command: ["sh", "-c", - "for i in $(seq 1 30); do wget -q -O- http://169.254.170.23/v1/credentials >/dev/null 2>&1 && echo 'credentials ready' && exit 0; echo \"waiting ($i/30)...\"; sleep 10; done; exit 0"] - }] - } serviceAccountName: parameter.serviceAccount topologySpreadConstraints: [{ maxSkew: 1 diff --git a/gitops/addons/registry/platform.yaml b/gitops/addons/registry/platform.yaml index 256cb9772..d5d59d4e6 100644 --- a/gitops/addons/registry/platform.yaml +++ b/gitops/addons/registry/platform.yaml @@ -360,37 +360,8 @@ devlake: - /spec/template/metadata/annotations/checksum~1config - /spec/template/metadata/annotations/checksum~1sc-dashboard-provider-config -gitlab: - namespace: gitlab - path: '{{.metadata.annotations.addonsRepoBasepath}}addons/charts/gitlab' - annotationsAppSet: - argocd.argoproj.io/sync-wave: '2' - selector: - matchExpressions: - - key: enable_gitlab - operator: In - values: ['true'] - valuesObject: - domain_name: '{{default .metadata.annotations.ingress_domain_name .metadata.annotations.gitlab_domain_name}}' - keycloak_domain_name: '{{.metadata.annotations.ingress_domain_name}}' - security_groups: '{{.metadata.annotations.ingress_security_groups}}' - exposure_mode: '{{default "domain" .metadata.annotations.exposure_mode}}' - git_token: '{{default "" .metadata.annotations.git_token}}' - source_repo_url: '{{.metadata.annotations.addonsRepoURL}}' - source_repo_revision: '{{.metadata.annotations.addonsRepoRevision}}' - global: - aws_cluster_name: '{{.metadata.annotations.aws_cluster_name}}' - ignoreDifferences: - - group: external-secrets.io - kind: ExternalSecret - jsonPointers: - - /status - - /metadata/generation - - /metadata/managedFields - - group: '' - kind: Secret - jsonPointers: - - /metadata/managedFields +# GitLab is no longer deployed as an EKS addon — it runs via docker-compose on the IDE EC2 instance. +# The CloudFront distribution and NLB are created by CDK (team-stack.ts). ack-iam: namespace: ack-system diff --git a/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/values.yaml b/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/values.yaml index 8b9e85a63..ff16041a0 100644 --- a/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/values.yaml +++ b/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/values.yaml @@ -9,8 +9,8 @@ # environment: dev # region: us-west-2 # k8sVersion: "1.35" -# accountId: "123456789012" -# managementAccountId: "123456789012" +# accountId: "134136091485" +# managementAccountId: "134136091485" # resourcePrefix: peeks # cidr: # vpcCidr: "10.3.0.0/16" diff --git a/gitops/overlays/environments/control-plane/enabled-addons.yaml b/gitops/overlays/environments/control-plane/enabled-addons.yaml index 0c930b57f..43a577663 100644 --- a/gitops/overlays/environments/control-plane/enabled-addons.yaml +++ b/gitops/overlays/environments/control-plane/enabled-addons.yaml @@ -44,7 +44,7 @@ enabledAddons: backstage: true kubevela: true devlake: true - gitlab: true + gitlab: false # GitLab runs on EC2 (docker-compose), not EKS # ACK controllers ack_iam: false ack_eks: false diff --git a/hack/.bashrc.d/ssm-setup-ide-logs.sh b/hack/.bashrc.d/ssm-setup-ide-logs.sh index 53aca6462..599c7f565 100644 --- a/hack/.bashrc.d/ssm-setup-ide-logs.sh +++ b/hack/.bashrc.d/ssm-setup-ide-logs.sh @@ -103,14 +103,4 @@ argocd-refresh-token() { fi echo "ArgoCD token refreshed. Server: $ARGOCD_SERVER" - - # Also update the argocd CLI config file so 'argocd app sync' etc. work - # without requiring env vars. The CLI prefers its config file over env vars - # when a server entry exists — leaving a stale token there causes - # "Unauthenticated: invalid session" even when ARGOCD_AUTH_TOKEN is correct. - argocd login "$ARGOCD_SERVER" \ - --auth-token "$token" \ - --grpc-web \ - --skip-test-tls \ - 2>/dev/null || true } diff --git a/platform/backstage/templates/catalog-info.yaml b/platform/backstage/templates/catalog-info.yaml index a8086c719..01e162865 100644 --- a/platform/backstage/templates/catalog-info.yaml +++ b/platform/backstage/templates/catalog-info.yaml @@ -17,7 +17,6 @@ spec: - ./ray-serve-trainium/template.yaml - ./create-dev-and-prod-env/template-create-dev-and-prod-env.yaml - ./cicd-pipeline/template-cicd-pipeline.yaml - - ./platform-entities.yaml --- apiVersion: backstage.io/v1alpha1 kind: System diff --git a/platform/infra/terraform/common/pod-identity.tf b/platform/infra/terraform/common/pod-identity.tf index 259b25b1a..ce448efba 100644 --- a/platform/infra/terraform/common/pod-identity.tf +++ b/platform/infra/terraform/common/pod-identity.tf @@ -602,16 +602,6 @@ locals { "iam:GetRole" ] Resource = "*" - }, - { - Effect = "Allow" - Action = "iam:PassRole" - Resource = "*" - Condition = { - StringEquals = { - "iam:PassedToService" = "pods.eks.amazonaws.com" - } - } } ] } @@ -738,7 +728,7 @@ resource "aws_iam_role" "kargo_controller_role" { resource "aws_iam_role_policy_attachment" "kargo_ecr_policy" { role = aws_iam_role.kargo_controller_role.name - policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser" } resource "aws_eks_pod_identity_association" "kargo_controller" { diff --git a/platform/infra/terraform/hub-config.yaml b/platform/infra/terraform/hub-config.yaml index 2b07e69fa..4c716328f 100644 --- a/platform/infra/terraform/hub-config.yaml +++ b/platform/infra/terraform/hub-config.yaml @@ -29,7 +29,7 @@ clusters: enable_backstage: true enable_kargo: true enable_keycloak: true - enable_gitlab: true + enable_gitlab: false # GitLab runs on EC2 (docker-compose), not EKS enable_cert_manager: true enable_external_secrets: true enable_ack_iam: false # EKS Managed capability From 32b0d907113ccaf4dc0b7e5c6a80730e0460800f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 11:28:02 +0200 Subject: [PATCH 002/231] feat: backport generic tasks from kind-crossplane to kind-kro-ack Added: urls, hub:set-overlay-repo, hub:restart-langfuse, hub:wait-for-full-sync, secrets-manager:seed-secrets, secrets-manager:seed-observability, hub:create-mgmt-roles, hub:restart-identity-pods, hub:update, spokes:enable-crossplane/kro, spokes:create-capabilities, spokes:disable-crossplane/kro/all, spokes:seed-provider-identity Removed: idc:setup from install task (moved to workshop:Taskfile.yaml) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1454 +++++++++++++++++- 1 file changed, 1453 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 69126ce38..972f052f3 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -101,7 +101,6 @@ tasks: - task: ack:install - task: kro:install - task: kro:apply-rgds - - task: idc:setup - task: hub:claim - task: hub:wait-for-eks - task: hub:ingress @@ -1003,3 +1002,1456 @@ tasks: echo "Kind preserved. Run 'task destroy-kind' to remove." fi - printf '{{.C_OK}}✓ Destroy complete.{{.C_RESET}}\n' + + # ---- from kind-crossplane ---- + urls: + desc: Print platform URLs (alias for print-urls) + aliases: [print-urls] + vars: + INGRESS_DOMAIN: + sh: | + if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ]; then + # In CloudFront mode the platform services (Backstage, Grafana, etc.) are on the + # hub EKS ALB, not the CloudFront/IDE domain. Discover the ALB hostname from the hub. + HUB_KC="{{.ROOT_DIR}}/private/hub-kubeconfig" + [ ! -f "$HUB_KC" ] && HUB_KC="{{.ROOT_DIR}}/private/kubeconfig" + ALB=$(kubectl --kubeconfig "$HUB_KC" get ingress -A \ + -l "app.kubernetes.io/name=backstage" \ + -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) + [ -z "$ALB" ] && ALB=$(kubectl --kubeconfig "$HUB_KC" get ingress -A \ + -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) + echo "${ALB:-{{.CLOUDFRONT_DOMAIN}}}" + else + echo "{{.DOMAIN}}" + fi + GITLAB_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + KC_ADMIN_PASSWORD: + sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.keycloak_admin_password // empty' + USER_PASSWORD: + sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // "workshop-user-2026"' + ARGOCD_URL: + sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name argocd --query 'capability.configuration.argoCd.serverUrl' --output text --region {{.AWS_REGION}} 2>/dev/null || echo "" + cmds: + - | + printf '{{.C_OK}}Platform URLs{{.C_RESET}}\n' + printf ' Backstage: https://{{.INGRESS_DOMAIN}}/backstage\n' + printf ' Keycloak: https://{{.INGRESS_DOMAIN}}/keycloak\n' + printf ' Argo WF: https://{{.INGRESS_DOMAIN}}/argo-workflows\n' + printf ' Kargo: https://{{.INGRESS_DOMAIN}}/\n' + printf ' Grafana: https://{{.INGRESS_DOMAIN}}/grafana\n' + printf ' GitLab: https://{{.GITLAB_DOMAIN}}\n' + printf ' ArgoCD: {{.ARGOCD_URL}}\n' + printf '\n' + printf '{{.C_INFO}}Credentials{{.C_RESET}}\n' + printf ' User: user1 / {{.USER_PASSWORD}}\n' + printf ' Keycloak: admin / {{.KC_ADMIN_PASSWORD}}\n' + + + + # ---- from kind-crossplane ---- + hub:set-overlay-repo: + desc: >- + Wire overlay_repo_url -> GitLab fleet-config on the hub cluster config so + every overlay-aware Argo app (incl. multi-acct) reads $overlay from + fleet-config instead of falling back to the GitHub addons repo. Runs LATE + (after GitLab + fleet-config exist) because $overlay is a real Argo source: + pointing it at a non-existent/unauthenticated repo breaks ALL overlay-aware + apps. Self-protecting: verifies reachability, registers an ArgoCD repo + credential, flips /config (ESO reconciles the cluster secret), then + verifies the flip resolved and auto-reverts if it did not. Idempotent. + vars: + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_USERNAME: + sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + # Root of the fleet-config overlay structure (configs/, overlays/) per the + # GitLab init-job seed. Keep "" unless the overlay repo nests under a prefix. + OVERLAY_REPO_BASEPATH: "" + cmds: + - | + OVERLAY_URL="https://{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" + SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" + KC="{{.ROOT_DIR}}/private/hub-kubeconfig" + + # 1. Reachability guard. $overlay is a REAL Argo source — never point it at + # an unreachable repo. This is one shell block, so exit 0 here correctly + # skips ALL wiring below (the earlier multi-block version did not). + # fleet-config is created asynchronously by the gitlab-init Job, so it + # can be a minute or two behind this step. Previously we skipped on the + # first miss — but then the overlay was never wired to fleet-config and + # the spoke entries that enable-crossplane/enable-kro push later were + # IGNORED by the appset (hub kept reading the GitHub addons fallback), + # so NO spoke EKS clusters were ever provisioned. Wait (up to ~5min) + # for it to become reachable before deciding. + FLEET_LS="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" + printf '{{.C_STEP}}▸ Verifying fleet-config overlay repo is reachable...{{.C_RESET}}\n' + FLEET_OK=false + for i in $(seq 1 30); do + if git ls-remote "$FLEET_LS" >/dev/null 2>&1; then FLEET_OK=true; break; fi + sleep 10 + done + if [ "$FLEET_OK" != true ]; then + printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — skipping overlay wiring (no change made).{{.C_RESET}}\n' + printf '{{.C_INFO}} Re-run later: task kind-crossplane:hub:set-overlay-repo{{.C_RESET}}\n' + exit 0 + fi + printf '{{.C_OK}}✓ fleet-config reachable.{{.C_RESET}}\n' + + # 2. Hub kubeconfig (inline so the guard governs everything) + rm -f "$KC" + aws eks update-kubeconfig --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --alias {{.HUB_CLUSTER_NAME}} --kubeconfig "$KC" >/dev/null + + # 3. ArgoCD repo credential for the private GitLab fleet-config repo + printf '{{.C_STEP}}▸ Registering ArgoCD repository credential for fleet-config...{{.C_RESET}}\n' + cat </{{.GIT_USERNAME}}/.git) that ArgoCD must pull; without a + # matching credential template those CD Applications fail with ComparisonError. + # A repo-creds secret applies to every repository whose URL starts with `url`. + printf '{{.C_STEP}}▸ Registering ArgoCD repo-creds for {{.GIT_USERNAME}} GitLab namespace...{{.C_RESET}}\n' + cat < leave it + # untouched. Never re-flip/verify/revert an already-working overlay, since a + # transient verify hiccup must never unset a good overlay. + if [ "$PREV_URL" = "$OVERLAY_URL" ]; then + printf '{{.C_OK}}✓ Overlay already wired to fleet-config (%s) — leaving untouched (idempotent).{{.C_RESET}}\n' "$OVERLAY_URL" + rm -f "$KC" + exit 0 + fi + + # 4. Flip overlay_* in /config (ESO reconciles the cluster secret annotations) + printf '{{.C_STEP}}▸ Wiring overlay_repo_url into %s...{{.C_RESET}}\n' "$SECRET_KEY" + EXISTING=$(aws secretsmanager get-secret-value --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} --query SecretString --output text) + UPDATED=$(echo "$EXISTING" | jq --arg url "$OVERLAY_URL" --arg rev "main" --arg bp "{{.OVERLAY_REPO_BASEPATH}}" \ + '.metadata = (.metadata | fromjson | . + {overlay_repo_url: $url, overlay_repo_revision: $rev, overlay_repo_basepath: $bp} | tojson)') + aws secretsmanager put-secret-value --secret-id "$SECRET_KEY" --secret-string "$UPDATED" --region {{.AWS_REGION}} >/dev/null + echo "Set overlay_repo_url=$OVERLAY_URL (basepath='{{.OVERLAY_REPO_BASEPATH}}')" + + # 5. Force ESO re-sync; wait until the new overlay actually PROPAGATES to the + # cluster-addons app source (avoids the false-positive the old verify had). + ES=$(KUBECONFIG="$KC" kubectl -n argocd get externalsecret -o name 2>/dev/null | grep -i fleet-secret | head -1 || true) + [ -n "$ES" ] && KUBECONFIG="$KC" kubectl -n argocd annotate "$ES" force-sync="$(date +%s)" --overwrite >/dev/null 2>&1 || true + printf '{{.C_STEP}}▸ Waiting for cluster-addons to pick up the new overlay source...{{.C_RESET}}\n' + PROPAGATED=false + for i in $(seq 1 30); do + SRC=$(KUBECONFIG="$KC" kubectl -n argocd get application cluster-addons -o jsonpath='{.spec.sources[?(@.ref=="overlay")].repoURL}' 2>/dev/null || echo "") + [ "$SRC" = "$OVERLAY_URL" ] && { PROPAGATED=true; break; } + sleep 10 + done + + # 6. Verify it resolves cleanly; auto-revert on any ComparisonError/InvalidSpec. + OK=false + if [ "$PROPAGATED" = true ]; then + printf '{{.C_STEP}}▸ Overlay propagated; verifying clean resolution...{{.C_RESET}}\n' + for i in $(seq 1 24); do + SYNC=$(KUBECONFIG="$KC" kubectl -n argocd get application cluster-addons -o jsonpath='{.status.sync.status}' 2>/dev/null || echo "") + ERR=$(KUBECONFIG="$KC" kubectl -n argocd get application cluster-addons -o jsonpath='{range .status.conditions[*]}{.type}{"\n"}{end}' 2>/dev/null | grep -iE "error|invalid" || true) + if [ -z "$ERR" ] && { [ "$SYNC" = "Synced" ] || [ "$SYNC" = "OutOfSync" ]; }; then OK=true; break; fi + printf '{{.C_INFO}} waiting for clean resolution (sync=%s) (%d/24)...{{.C_RESET}}\n' "$SYNC" "$i" + sleep 10 + done + else + printf '{{.C_ERR}}⚠ Overlay did not propagate to cluster-addons in time.{{.C_RESET}}\n' + fi + + if [ "$OK" != true ]; then + EXISTING=$(aws secretsmanager get-secret-value --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} --query SecretString --output text) + if [ -n "$PREV_URL" ]; then + # A previous overlay was already wired — RESTORE it rather than dropping to + # the GitHub base. Falling back to base renders clusters:{} and would + # cascade-delete live spoke clusters; restoring keeps them intact. + printf '{{.C_ERR}}⚠ New overlay not resolving cleanly — restoring previous overlay (%s).{{.C_RESET}}\n' "$PREV_URL" + REVERTED=$(echo "$EXISTING" | jq --arg url "$PREV_URL" --arg rev "$PREV_REV" --arg bp "$PREV_BP" \ + '.metadata = (.metadata | fromjson | . + {overlay_repo_url: $url, overlay_repo_revision: $rev, overlay_repo_basepath: $bp} | tojson)') + else + # No overlay was set before this run — clean rollback to the GitHub base. + printf '{{.C_ERR}}⚠ Overlay not resolving cleanly and none was set before — rolling back to GitHub base.{{.C_RESET}}\n' + REVERTED=$(echo "$EXISTING" | jq '.metadata = (.metadata | fromjson | del(.overlay_repo_url, .overlay_repo_revision, .overlay_repo_basepath) | tojson)') + fi + aws secretsmanager put-secret-value --secret-id "$SECRET_KEY" --secret-string "$REVERTED" --region {{.AWS_REGION}} >/dev/null + [ -n "$ES" ] && KUBECONFIG="$KC" kubectl -n argocd annotate "$ES" force-sync="$(date +%s)" --overwrite >/dev/null 2>&1 || true + # Only strip the cluster-secret annotations on a true rollback-to-base (no prev); + # for a restore, let ESO reconcile the previous values back from SecretsManager. + if [ -z "$PREV_URL" ]; then + KUBECONFIG="$KC" kubectl -n argocd annotate secret {{.HUB_CLUSTER_NAME}} overlay_repo_url- overlay_repo_revision- overlay_repo_basepath- >/dev/null 2>&1 || true + fi + printf '{{.C_ERR}} Fix GitLab reachability/credential, then re-run.{{.C_RESET}}\n' + rm -f "$KC" + exit 0 + fi + printf '{{.C_OK}}✓ Overlay wired to fleet-config and resolving fleet-wide (hub + spokes).{{.C_RESET}}\n' + rm -f "$KC" + + + # ---- from kind-crossplane ---- + hub:restart-langfuse: + desc: Restart Langfuse to pick up Keycloak client secret (created by config job after initial deploy) + cmds: + - task: hub:kubeconfig + - printf '{{.C_STEP}}▸ Restarting Langfuse to pick up Keycloak client secret...{{.C_RESET}}\n' + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout restart deploy -n langfuse langfuse 2>/dev/null || true + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout status deploy -n langfuse langfuse --timeout=120s 2>/dev/null || true + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + - printf '{{.C_OK}}✓ Langfuse restarted with Keycloak SSO credentials.{{.C_RESET}}\n' + + + # ---- from kind-crossplane ---- + hub:wait-for-full-sync: + desc: Wait for ALL apps (including spokes) to sync after provider restart + vars: + MAX_OUTOFSYNC: '{{.MAX_OUTOFSYNC | default "5"}}' + cmds: + - task: hub:kubeconfig + - printf '{{.C_STEP}}▸ Waiting for all apps to sync (spoke clusters provisioning)...{{.C_RESET}}\n' + - | + TIMEOUT=2400 + INTERVAL=30 + ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + APP_OUTPUT=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get applications.argoproj.io -n argocd --no-headers 2>/dev/null || echo "") + TOTAL=$(echo "$APP_OUTPUT" | awk 'NF>0' | wc -l | tr -d ' ') + SYNCED=$(echo "$APP_OUTPUT" | awk '$2 == "Synced"' | wc -l | tr -d ' ') + UNKNOWN=$(echo "$APP_OUTPUT" | awk '$2 == "Unknown"' | wc -l | tr -d ' ') + OUTOFSYNC=$((TOTAL - SYNCED - UNKNOWN)) + printf '{{.C_INFO}} Apps: %s/%s synced, %s unknown, %s out-of-sync (%ss elapsed){{.C_RESET}}\n' "$SYNCED" "$TOTAL" "$UNKNOWN" "$OUTOFSYNC" "$ELAPSED" + if [ "$TOTAL" -gt 0 ] && [ "$OUTOFSYNC" -le {{.MAX_OUTOFSYNC}} ]; then + printf '{{.C_OK}}✓ All %s apps synced (within %s tolerance, %s unknown).{{.C_RESET}}\n' "$SYNCED" "{{.MAX_OUTOFSYNC}}" "$UNKNOWN" + break + fi + sleep $INTERVAL + ELAPSED=$((ELAPSED + INTERVAL)) + done + if [ $ELAPSED -ge $TIMEOUT ]; then + printf '{{.C_ERR}}⚠ Timed out after %ss. %s app(s) still not synced:{{.C_RESET}}\n' "$TIMEOUT" "$OUTOFSYNC" + echo "$APP_OUTPUT" | awk '$2 != "Synced" && NF>0 {printf " - %s (%s/%s)\n", $1, $3, $2}' + printf '{{.C_STEP}}▸ Continuing — spoke clusters may still be provisioning.{{.C_RESET}}\n' + fi + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + + + # ---- from kind-crossplane ---- + secrets-manager:seed-secrets: + desc: Seed platform user credentials into Secrets Manager for ExternalSecrets + cmds: + - | + SECRET_KEY="{{.HUB_CLUSTER_NAME}}/secrets" + USER_PASS=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query SecretString --output text | jq -r .user_password) + # Merge into the existing secret so a re-run backfills any keys a previous + # (partial) install left out — and never rotates values live consumers + # depend on. First run: no secret yet, start from {} and create-then-put. + EXISTING=$(aws secretsmanager get-secret-value --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} --query SecretString --output text 2>/dev/null || echo "{}") + # Reuse existing hash/key on re-runs (don't rotate them out from under Kargo etc.) + USER_PASS_HASH=$(echo "$EXISTING" | jq -r '.user_password_hash // ""') + if [ -z "$USER_PASS_HASH" ]; then + USER_PASS_HASH=$(echo -n "$USER_PASS" | python3 -c "import sys,bcrypt; print(bcrypt.hashpw(sys.stdin.buffer.read(), bcrypt.gensalt()).decode())" 2>/dev/null || echo "") + fi + USER_PASS_KEY=$(echo "$EXISTING" | jq -r '.user_password_key // ""') + if [ -z "$USER_PASS_KEY" ]; then + USER_PASS_KEY=$(echo -n "$USER_PASS" | base64) + fi + UPDATED=$(echo "$EXISTING" | jq \ + --arg user_password "$USER_PASS" \ + --arg user_password_hash "$USER_PASS_HASH" \ + --arg user_password_key "$USER_PASS_KEY" \ + '. + {user_password: $user_password, user_password_hash: $user_password_hash, user_password_key: $user_password_key}') + # DevLake credentials. These belong here (NOT in seed-observability) so the + # devlake-encryption-secret ExternalSecret resolves during the normal install, + # independent of the AMP/AMG observability flow — which waits up to 30 min for + # the AMP workspace and is frequently re-run separately (task observability-seed). + # Gating these behind observability left DevLake's ExternalSecret in + # SecretSyncedError whenever observability seeding didn't complete. + # Generate only if absent so re-runs don't break existing encrypted connections. + # - encryption secret: 128 uppercase chars (DevLake requirement) + if ! echo "$UPDATED" | jq -e 'has("devlake_encryption_secret")' >/dev/null; then + DEVLAKE_ENC=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) + UPDATED=$(echo "$UPDATED" | jq --arg p "$DEVLAKE_ENC" '. + {devlake_encryption_secret: $p}') + fi + if ! echo "$UPDATED" | jq -e 'has("devlake_mysql_password")' >/dev/null; then + DEVLAKE_MYSQL=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) + UPDATED=$(echo "$UPDATED" | jq --arg p "$DEVLAKE_MYSQL" '. + {devlake_mysql_password: $p}') + fi + # grafana_mysql_password: the devlake `mysql-passwords` ExternalSecret reads + # BOTH devlake_mysql_password AND grafana_mysql_password — ESO fails the whole + # ExternalSecret if either property is missing, so this must be seeded here too + # (it was previously only generated in the AMP-gated seed-observability flow, + # leaving mysql-passwords in SecretSyncedError and devlake-peeks-hub Degraded). + if ! echo "$UPDATED" | jq -e 'has("grafana_mysql_password")' >/dev/null; then + GRAFANA_MYSQL=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) + UPDATED=$(echo "$UPDATED" | jq --arg p "$GRAFANA_MYSQL" '. + {grafana_mysql_password: $p}') + fi + # backstage_postgres_password + git_token: the backstage-postgres and + # git-credentials ExternalSecrets read these from /secrets. They were + # previously only generated in the AMP-gated seed-observability flow, so on a + # fresh install where observability seeding hadn't completed Backstage stayed + # OutOfSync/Degraded ('key backstage_postgres_password does not exist') and + # $BACKSTAGE_URL returned 503 — blocking every Backstage-dependent module. + # Seed them here (in the always-runs early phase) instead. git_token mirrors + # the workshop user password (GitLab PAT == user password in this workshop). + if ! echo "$UPDATED" | jq -e 'has("backstage_postgres_password")' >/dev/null; then + BS_PG_PASS=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) + UPDATED=$(echo "$UPDATED" | jq --arg p "$BS_PG_PASS" '. + {backstage_postgres_password: $p}') + fi + # git_token is seeded by CDK at deploy time (root-) — no runtime seeding needed. + aws secretsmanager create-secret \ + --name "$SECRET_KEY" \ + --secret-string "$UPDATED" \ + --region {{.AWS_REGION}} 2>/dev/null \ + || aws secretsmanager put-secret-value \ + --secret-id "$SECRET_KEY" \ + --secret-string "$UPDATED" \ + --region {{.AWS_REGION}} + echo "Seeded Secrets Manager: $SECRET_KEY" + + + # ---- from kind-crossplane ---- + secrets-manager:seed-observability: + desc: Seed observability credentials (AMP endpoint, AMG API key) into Secrets Manager + vars: + KC_DOMAIN: + sh: | + if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -f {{.ROOT_DIR}}/private/cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + cmds: + - | + set -e + SECRET_KEY="{{.HUB_CLUSTER_NAME}}/secrets" + + # Poll for AMP workspace (up to 60 min — Crossplane provisioning of AMP/AMG + # can take well over 30 min on busy accounts; the previous 30 min ceiling + # was too short and left Grafana unseeded on slow installs). + # Use crossplane-name tag to identify the correct workspace (avoids ambiguity with duplicates) + AMP_WORKSPACE_ID="" + printf '{{.C_STEP}}▸ Waiting for AMP workspace...{{.C_RESET}}\n' + for i in $(seq 1 120); do + for WS_ID in $(aws amp list-workspaces --region {{.AWS_REGION}} --query "workspaces[?alias=='{{.RESOURCE_PREFIX}}-observability-amp'].workspaceId" --output text 2>/dev/null); do + TAG_VAL=$(aws amp list-tags-for-resource --resource-arn "arn:aws:aps:{{.AWS_REGION}}:{{.AWS_ACCOUNT_ID}}:workspace/$WS_ID" --region {{.AWS_REGION}} --query "tags.\"crossplane-name\"" --output text 2>/dev/null || echo "") + if [ "$TAG_VAL" = "{{.RESOURCE_PREFIX}}-amp" ]; then + AMP_WORKSPACE_ID="$WS_ID" + break + fi + done + if [ -n "$AMP_WORKSPACE_ID" ]; then + break + fi + printf ' [%d/120] AMP workspace not ready yet...\n' "$i" + sleep 30 + done + if [ -z "$AMP_WORKSPACE_ID" ] || [ "$AMP_WORKSPACE_ID" = "None" ]; then + echo "" + echo "============================================================" + echo "WARN: AMP workspace not created after 60 min." + echo " The peeks-observability-seed.timer will keep retrying this" + echo " automatically; or run 'task observability-seed' manually once" + echo " the observability-aws ArgoCD app is Synced and AMP/AMG are ACTIVE." + echo "============================================================" + exit 0 + fi + AMP_ENDPOINT=$(aws amp describe-workspace --workspace-id "$AMP_WORKSPACE_ID" --region {{.AWS_REGION}} --query "workspace.prometheusEndpoint" --output text) + echo "Found AMP endpoint: $AMP_ENDPOINT" + + # Poll for AMG workspace (up to 15 min) + # Use crossplane-name tag to identify the correct workspace + AMG_WORKSPACE_ID="" + printf '{{.C_STEP}}▸ Waiting for AMG workspace...{{.C_RESET}}\n' + for i in $(seq 1 30); do + for WS in $(aws grafana list-workspaces --region {{.AWS_REGION}} --query "workspaces[?name=='{{.RESOURCE_PREFIX}}-observability' && status=='ACTIVE'].id" --output text 2>/dev/null); do + TAG_VAL=$(aws grafana describe-workspace --workspace-id "$WS" --region {{.AWS_REGION}} --query "workspace.tags.\"crossplane-name\"" --output text 2>/dev/null || echo "") + if [ "$TAG_VAL" = "{{.RESOURCE_PREFIX}}-amg" ]; then + AMG_WORKSPACE_ID="$WS" + break + fi + done + if [ -n "$AMG_WORKSPACE_ID" ]; then + break + fi + printf ' [%d/30] AMG workspace not ready yet...\n' "$i" + sleep 30 + done + if [ -z "$AMG_WORKSPACE_ID" ] || [ "$AMG_WORKSPACE_ID" = "None" ]; then + echo "WARN: AMG workspace not found. Run 'task observability-seed' later." + exit 0 + fi + AMG_ENDPOINT=$(aws grafana describe-workspace --workspace-id "$AMG_WORKSPACE_ID" --region {{.AWS_REGION}} --query "workspace.endpoint" --output text) + echo "Found AMG endpoint: $AMG_ENDPOINT" + + # Create AMG service account + token (admin, 30 days) — API keys removed in Grafana 12 + SA_NAME="crossplane-$(date +%s)" + SA_ID=$(aws grafana create-workspace-service-account --workspace-id "$AMG_WORKSPACE_ID" --grafana-role ADMIN --name "$SA_NAME" --region {{.AWS_REGION}} --query "id" --output text 2>/dev/null || echo "") + if [ -z "$SA_ID" ] || [ "$SA_ID" = "None" ]; then + echo "WARN: Could not create AMG service account. Skipping." + exit 0 + fi + AMG_API_KEY=$(aws grafana create-workspace-service-account-token --workspace-id "$AMG_WORKSPACE_ID" --service-account-id "$SA_ID" --name "token-$SA_NAME" --seconds-to-live 2592000 --region {{.AWS_REGION}} --query "serviceAccountToken.key" --output text 2>/dev/null || echo "") + if [ -z "$AMG_API_KEY" ]; then + echo "WARN: Could not create AMG service account token. Skipping." + exit 0 + fi + + # Install required Grafana plugins (X-Ray for trace visualization) + printf '{{.C_STEP}}▸ Installing AMG plugins (X-Ray)...{{.C_RESET}}\n' + curl -s -X POST -H "Authorization: Bearer $AMG_API_KEY" \ + -H "Content-Type: application/json" \ + "https://$AMG_ENDPOINT/api/plugins/grafana-x-ray-datasource/install" 2>/dev/null || true + echo "X-Ray plugin installed" + + # Merge into existing secret (preserves other keys). On first run + # the secret doesn't exist yet — `get-secret-value` returns empty, + # we start from `{}`, and `put-secret-value` would fail with + # ResourceNotFoundException, so create-then-put. + EXISTING=$(aws secretsmanager get-secret-value --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} --query "SecretString" --output text 2>/dev/null || echo "{}") + # Pull user_password from keycloak secret for backstage ExternalSecrets + KC_SECRET=$(aws secretsmanager get-secret-value --secret-id "{{.HUB_CLUSTER_NAME}}/keycloak" --region {{.AWS_REGION}} --query "SecretString" --output text 2>/dev/null || echo "{}") + USER_PASS=$(echo "$KC_SECRET" | jq -r '.user_password // "workshop-user-2026"') + # Generate backstage_postgres_password if not already present + if ! echo "$EXISTING" | jq -e 'has("backstage_postgres_password")' >/dev/null 2>&1; then + BS_PG_PASS=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) + else + BS_PG_PASS=$(echo "$EXISTING" | jq -r '.backstage_postgres_password') + fi + # Kargo admin credentials: bcrypt hash of user password + random signing key. + # Reuse existing values on re-runs so we don't rotate them out from under Kargo. + KARGO_HASH=$(echo "$EXISTING" | jq -r '.user_password_hash // ""') + if [ -z "$KARGO_HASH" ]; then + pip install bcrypt --quiet 2>/dev/null || true + KARGO_HASH=$(python3 -c "import bcrypt,sys; print(bcrypt.hashpw(sys.argv[1].encode(), bcrypt.gensalt(rounds=10)).decode())" "$USER_PASS" 2>/dev/null) + fi + KARGO_KEY=$(echo "$EXISTING" | jq -r '.user_password_key // ""') + if [ -z "$KARGO_KEY" ]; then + KARGO_KEY=$(openssl rand -base64 30 | tr -d '/+=' | head -c 40) + fi + UPDATED=$(echo "$EXISTING" | jq \ + --arg amp_url "${AMP_ENDPOINT}api/v1/remote_write" \ + --arg amp_query_url "$AMP_ENDPOINT" \ + --arg amp_region "{{.AWS_REGION}}" \ + --arg grafana_api_key "$AMG_API_KEY" \ + --arg grafana_url "$AMG_ENDPOINT" \ + --arg user_password "$USER_PASS" \ + --arg git_token "$USER_PASS" \ + --arg backstage_postgres_password "$BS_PG_PASS" \ + --arg user_password_hash "$KARGO_HASH" \ + --arg user_password_key "$KARGO_KEY" \ + '. + {amp_endpoint_url: $amp_query_url, amp_region: $amp_region, grafana_api_key: $grafana_api_key, grafana_url: $grafana_url, user_password: $user_password, git_token: $git_token, backstage_postgres_password: $backstage_postgres_password, user_password_hash: $user_password_hash, user_password_key: $user_password_key}') + # Generate the grafana_mysql_password the grafana-dashboards + # ExternalSecret expects, but only if it isn't already there + # (so re-runs don't rotate it out from under live consumers). + if ! echo "$UPDATED" | jq -e 'has("grafana_mysql_password")' >/dev/null; then + GRAFANA_MYSQL_PASSWORD=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) + UPDATED=$(echo "$UPDATED" | jq --arg p "$GRAFANA_MYSQL_PASSWORD" '. + {grafana_mysql_password: $p}') + fi + # DevLake encryption secret (128 uppercase chars, per DevLake requirement). + # Reuse on re-runs so we don't break existing encrypted connections. + if ! echo "$UPDATED" | jq -e 'has("devlake_encryption_secret")' >/dev/null; then + DEVLAKE_ENC=$(openssl rand -base64 2000 | tr -dc 'A-Z' | fold -w 128 | head -n 1) + UPDATED=$(echo "$UPDATED" | jq --arg p "$DEVLAKE_ENC" '. + {devlake_encryption_secret: $p}') + fi + if ! echo "$UPDATED" | jq -e 'has("devlake_mysql_password")' >/dev/null; then + DEVLAKE_MYSQL=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) + UPDATED=$(echo "$UPDATED" | jq --arg p "$DEVLAKE_MYSQL" '. + {devlake_mysql_password: $p}') + fi + aws secretsmanager create-secret \ + --name "$SECRET_KEY" \ + --secret-string "$UPDATED" \ + --region {{.AWS_REGION}} 2>/dev/null \ + || aws secretsmanager put-secret-value \ + --secret-id "$SECRET_KEY" \ + --secret-string "$UPDATED" \ + --region {{.AWS_REGION}} + echo "Seeded observability credentials into $SECRET_KEY" + + # Spoke amp-workspace-secrets ExternalSecrets (Argo Rollouts metric queries on the + # spokes) read a SEPARATE secret /platform/amp with properties + # 'amp-workspace' (the AMP query endpoint) and 'amp-region'. In the terraform flow + # this was created by common/secrets.tf; the kind-crossplane flow never created it, + # so every spoke amp-workspace-secrets-* ExternalSecret stayed in SecretSyncedError. + # Create/refresh it here now that AMP_ENDPOINT is known. + AMP_SECRET_KEY="{{.RESOURCE_PREFIX}}/platform/amp" + AMP_SECRET_VALUE=$(jq -n --arg r "{{.AWS_REGION}}" --arg w "$AMP_ENDPOINT" '{"amp-region": $r, "amp-workspace": $w}') + aws secretsmanager create-secret \ + --name "$AMP_SECRET_KEY" \ + --secret-string "$AMP_SECRET_VALUE" \ + --region {{.AWS_REGION}} 2>/dev/null \ + || aws secretsmanager put-secret-value \ + --secret-id "$AMP_SECRET_KEY" \ + --secret-string "$AMP_SECRET_VALUE" \ + --region {{.AWS_REGION}} + echo "Seeded AMP workspace secret: $AMP_SECRET_KEY" + + # Update hub cluster config secret with AMG URL for Keycloak SAML and grafana-dashboards + CONFIG_KEY="{{.HUB_CLUSTER_NAME}}/config" + CONFIG_EXISTING=$(aws secretsmanager get-secret-value --secret-id "$CONFIG_KEY" --region {{.AWS_REGION}} --query "SecretString" --output text) + CONFIG_UPDATED=$(echo "$CONFIG_EXISTING" | jq \ + --arg grafana_url "$AMG_ENDPOINT" \ + '.metadata = (.metadata | fromjson | . + {aws_grafana_url: $grafana_url} | tojson)') + aws secretsmanager put-secret-value \ + --secret-id "$CONFIG_KEY" \ + --secret-string "$CONFIG_UPDATED" \ + --region {{.AWS_REGION}} + echo "Updated hub config with AMG URL: $AMG_ENDPOINT" + + # Populate spoke scraper values into hub Secrets Manager secret (dynamic, no file commits needed) + CONFIG_KEY="{{.HUB_CLUSTER_NAME}}/config" + CONFIG_EXISTING=$(aws secretsmanager get-secret-value --secret-id "$CONFIG_KEY" --region {{.AWS_REGION}} --query "SecretString" --output text) + SPOKE_DEV_ARN="" + SPOKE_DEV_SUBNETS="" + SPOKE_DEV_SG="" + SPOKE_PROD_ARN="" + SPOKE_PROD_SUBNETS="" + SPOKE_PROD_SG="" + for CLUSTER in spoke-dev spoke-prod; do + SPOKE_ARN=$(aws eks describe-cluster --name "$CLUSTER" --region {{.AWS_REGION}} --query 'cluster.arn' --output text 2>/dev/null || echo "") + if [ -n "$SPOKE_ARN" ] && [ "$SPOKE_ARN" != "None" ]; then + SPOKE_ALL_SUBNETS=$(aws eks describe-cluster --name "$CLUSTER" --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.subnetIds[]' --output text) + SPOKE_PRIVATE_SUBNETS=$(aws ec2 describe-subnets --subnet-ids $SPOKE_ALL_SUBNETS --region {{.AWS_REGION}} --query 'Subnets[?MapPublicIpOnLaunch==`false`].SubnetId' --output text | tr '\t' '\n' | awk '!seen[$0]++' | head -2 | paste -sd ',' -) + if [ -z "$SPOKE_PRIVATE_SUBNETS" ]; then + SPOKE_PRIVATE_SUBNETS=$(echo "$SPOKE_ALL_SUBNETS" | tr '\t' '\n' | head -2 | paste -sd ',' -) + fi + SPOKE_SG=$(aws eks describe-cluster --name "$CLUSTER" --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text) + echo "Spoke $CLUSTER: arn=$SPOKE_ARN subnets=$SPOKE_PRIVATE_SUBNETS sg=$SPOKE_SG" + if [ "$CLUSTER" = "spoke-dev" ]; then + SPOKE_DEV_ARN="$SPOKE_ARN" + SPOKE_DEV_SUBNETS="$SPOKE_PRIVATE_SUBNETS" + SPOKE_DEV_SG="$SPOKE_SG" + else + SPOKE_PROD_ARN="$SPOKE_ARN" + SPOKE_PROD_SUBNETS="$SPOKE_PRIVATE_SUBNETS" + SPOKE_PROD_SG="$SPOKE_SG" + fi + else + echo "WARN: Cluster $CLUSTER not found in {{.AWS_REGION}}. Scraper skipped." + fi + done + if [ -n "$SPOKE_DEV_ARN" ] || [ -n "$SPOKE_PROD_ARN" ]; then + CONFIG_UPDATED=$(echo "$CONFIG_EXISTING" | jq \ + --arg sda "$SPOKE_DEV_ARN" \ + --arg sds "$SPOKE_DEV_SUBNETS" \ + --arg sdsg "$SPOKE_DEV_SG" \ + --arg spa "$SPOKE_PROD_ARN" \ + --arg sps "$SPOKE_PROD_SUBNETS" \ + --arg spsg "$SPOKE_PROD_SG" \ + '.metadata = (.metadata | fromjson | . + {spoke_dev_cluster_arn: $sda, spoke_dev_private_subnet_ids: $sds, spoke_dev_security_group_id: $sdsg, spoke_prod_cluster_arn: $spa, spoke_prod_private_subnet_ids: $sps, spoke_prod_security_group_id: $spsg} | tojson)') + aws secretsmanager put-secret-value \ + --secret-id "$CONFIG_KEY" \ + --secret-string "$CONFIG_UPDATED" \ + --region {{.AWS_REGION}} + echo "Updated hub config with spoke scraper data" + fi + + # Update Keycloak SAML client with AMG URL + printf '{{.C_STEP}}▸ Waiting for Keycloak to be reachable...{{.C_RESET}}\n' + for i in $(seq 1 30); do + if curl -sf -o /dev/null "https://{{.KC_DOMAIN}}/keycloak/realms/master/.well-known/openid-configuration" 2>/dev/null; then + echo "Keycloak is reachable" + break + fi + echo " [$i/30] Keycloak not reachable yet (DNS/ALB propagating)..." + sleep 20 + done + printf '{{.C_STEP}}▸ Updating Keycloak SAML client with AMG URL...{{.C_RESET}}\n' + # Best-effort, trailing block: relax errexit so a Keycloak/AMG hiccup + # (missing perms, duplicate SAML clients yielding a multi-line client id -> + # malformed curl URL, transient API error) only WARNs instead of aborting + # the entire install. This is the last block in the task, so set +e is safe. + set +e + KC_PASS=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text | jq -r '.keycloak_admin_password') + KC_TOKEN=$(curl -s -X POST "https://{{.KC_DOMAIN}}/keycloak/realms/master/protocol/openid-connect/token" \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=password&client_id=admin-cli&username=admin&password=$KC_PASS" | jq -r '.access_token') + if [ -n "$KC_TOKEN" ] && [ "$KC_TOKEN" != "null" ]; then + KC_CLIENT_ID=$(curl -s "https://{{.KC_DOMAIN}}/keycloak/admin/realms/platform/clients" \ + -H "Authorization: Bearer $KC_TOKEN" | jq -r '.[] | select(.name=="amazon-managed-grafana" and .protocol=="saml") | .id') + if [ -n "$KC_CLIENT_ID" ]; then + KC_CLIENT=$(curl -s "https://{{.KC_DOMAIN}}/keycloak/admin/realms/platform/clients/$KC_CLIENT_ID" \ + -H "Authorization: Bearer $KC_TOKEN") + KC_UPDATED=$(echo "$KC_CLIENT" | jq \ + --arg url "$AMG_ENDPOINT" \ + '.clientId = "https://\($url)/saml/metadata" | .adminUrl = "https://\($url)/login/saml" | .redirectUris = ["https://\($url)/saml/acs"]') + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" -X PUT "https://{{.KC_DOMAIN}}/keycloak/admin/realms/platform/clients/$KC_CLIENT_ID" \ + -H "Authorization: Bearer $KC_TOKEN" \ + -H "Content-Type: application/json" \ + -d "$KC_UPDATED") + if [ "$HTTP_CODE" = "204" ]; then + echo "Keycloak SAML client updated for AMG: $AMG_ENDPOINT" + else + echo "WARN: Keycloak SAML client update returned HTTP $HTTP_CODE" + fi + else + echo "WARN: Keycloak SAML client 'amazon-managed-grafana' not found" + fi + else + echo "WARN: Could not get Keycloak admin token. SAML client not updated." + fi + - | + # Nudge Grafana to converge now that grafana_api_key/AMG creds are seeded. + # The grafana-admin-credentials ExternalSecret -> Secret -> external-grafana + # instance chain only reconciles once these creds exist; and the + # grafana-operator caches its "available instances" registry, so + # GrafanaDashboards created before the instance was Ready keep reporting + # NoMatchingInstances until the operator re-evaluates. Force the + # ExternalSecrets to resync and restart the operator so dashboards attach + # and the grafana-dashboards Argo app reaches Healthy without manual steps. + # Best-effort: never fail the seed on this. + for es in grafana-admin-credentials grafana-amp-creds grafana-mysql-creds; do + kubectl annotate externalsecret "$es" -n grafana-operator force-sync="$(date +%s)" --overwrite 2>/dev/null || true + done + kubectl rollout restart deploy/grafana-operator -n grafana-operator 2>/dev/null || true + echo "Grafana ExternalSecrets resynced and grafana-operator restarted to pick up seeded credentials." + + + + hub:create-mgmt-roles: + desc: Create cluster-mgmt IAM roles assumed by ACK controllers (single-account prereq) + vars: + ACK_CAPABILITY_ROLE: + sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name ack --region {{.AWS_REGION}} --query 'capability.roleArn' --output text 2>/dev/null | sed 's|.*/||' || echo "{{.HUB_CLUSTER_NAME}}-ArgoCDCapabilityRole" + cmds: + - printf '{{.C_STEP}}▸ Creating cluster-mgmt IAM roles for ACK...{{.C_RESET}}\n' + - | + TRUST_POLICY=$(cat </dev/null 2>&1; then + aws iam update-assume-role-policy --role-name "$ROLE_NAME" --policy-document "$TRUST_POLICY" + else + aws iam create-role --role-name "$ROLE_NAME" --assume-role-policy-document "$TRUST_POLICY" --no-cli-pager >/dev/null + fi + for arn in $(get_policies $service); do + aws iam attach-role-policy --role-name "$ROLE_NAME" --policy-arn "$arn" 2>/dev/null || true + done + echo " ✓ $ROLE_NAME" + done + # Inline policies for EKS role: AccessEntry, PodIdentity, PassRole + EKS_ROLE="{{.RESOURCE_PREFIX}}-cluster-mgmt-eks" + aws iam put-role-policy --role-name "$EKS_ROLE" --policy-name eks-access-management --policy-document '{ + "Version": "2012-10-17", + "Statement": [ + {"Effect": "Allow", "Action": ["eks:*"], "Resource": "*"}, + {"Effect": "Allow", "Action": ["iam:PassRole","iam:GetRole"], "Resource": "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-*"} + ] + }' + # Allow ACK capability role to assume cluster-mgmt roles + aws iam put-role-policy --role-name "{{.ACK_CAPABILITY_ROLE}}" --policy-name ack-assume-role --policy-document '{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Action": ["sts:AssumeRole", "sts:TagSession"], + "Resource": "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-cluster-mgmt-*" + }] + }' + - printf '{{.C_OK}}✓ cluster-mgmt roles ready.{{.C_RESET}}\n' + + + spokes:enable-crossplane: + desc: "Enable a spoke cluster via Crossplane (usage: task spokes:enable-crossplane -- )" + vars: + CLUSTER: '{{.CLI_ARGS}}' + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + cmds: + - | + if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:enable-crossplane -- "; exit 1; fi + FLEET_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" + # fleet-config is created asynchronously by the gitlab-init Job (Helm + # post-install hook, which waits for GitLab readiness), so this task can + # race ahead of it. Cloning an absent repo left $REPO_DIR without a .git + # dir and the later `git add -A` failed with exit 128, taking down + # `task install` (install:phase2-spoke-dev). Wait (up to ~5min) for the + # repo to be reachable, then verify the clone produced a git repo. + printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' + for i in $(seq 1 30); do + git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + if [ "$i" -eq 30 ]; then + printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot enable spoke {{.CLUSTER}} via Crossplane.{{.C_RESET}}\n' + printf '{{.C_INFO}} GitLab/gitlab-init may still be initializing. Re-run later: task kind-crossplane:spokes:enable-crossplane -- {{.CLUSTER}}{{.C_RESET}}\n' + exit 1 + fi + sleep 10 + done + printf '{{.C_OK}}✓ fleet-config reachable.{{.C_RESET}}\n' + REPO_DIR=$(mktemp -d) + git clone "$FLEET_URL" "$REPO_DIR" + if [ ! -d "$REPO_DIR/.git" ]; then + printf '{{.C_ERR}}⚠ fleet-config clone failed — aborting enable-crossplane for {{.CLUSTER}}.{{.C_RESET}}\n' + rm -rf "$REPO_DIR" + exit 1 + fi + mkdir -p "$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/crossplane-clusters" + XP_VALUES="$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/crossplane-clusters/values.yaml" + # Idempotency: keep the existing vpcCidr if this spoke is already enabled so + # re-running install does not generate a new random CIDR and churn the VPC. + if [ -f "$XP_VALUES" ] && yq -e '.clusters."{{.CLUSTER}}".vpcCidr' "$XP_VALUES" >/dev/null 2>&1; then + echo " Spoke {{.CLUSTER}} already has a vpcCidr — keeping existing values (idempotent)." + else + # adminRoleArn grants the console/participant admin role cluster-admin on + # the spoke. adminInstanceRoleArn grants the IDE/instance *SharedRole* a + # SECOND cluster-admin access entry — required so `kubectl` run from the + # IDE (which assumes the instance role, NOT adminRoleName) works against + # the spoke. create-config.sh deliberately excludes the SharedRole from + # adminRoleName, so we detect it here. Both are idempotent / optional. + ADMIN_ROLE=$(yq '.adminRoleName // ""' {{.CONFIG_FILE}}) + ADMIN_ARN_LINE="" + if [ -n "$ADMIN_ROLE" ]; then + ADMIN_ARN_LINE=" adminRoleArn: arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/$ADMIN_ROLE" + fi + INSTANCE_ROLE=$(aws iam list-roles --query "Roles[?contains(RoleName,'SharedRole')].RoleName | [0]" --output text 2>/dev/null || echo "") + ADMIN_INSTANCE_ARN_LINE="" + if [ -n "$INSTANCE_ROLE" ] && [ "$INSTANCE_ROLE" != "None" ] && [ "$INSTANCE_ROLE" != "null" ]; then + ADMIN_INSTANCE_ARN_LINE=" adminInstanceRoleArn: arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/$INSTANCE_ROLE" + fi + cat > "$XP_VALUES" < "$REPO_DIR/gitops/fleet/members/{{.CLUSTER}}/values.yaml" <&1; then PUSHED=true; break; fi + echo " Push attempt $i failed, rebasing..." + git pull --rebase 2>&1 && sleep 2 + done + if [ "$PUSHED" != "true" ]; then + printf '{{.C_ERR}}\u26a0 fleet-config push failed after 3 attempts.{{.C_RESET}}\n' + exit 1 + fi + fi + rm -rf "$REPO_DIR" + # Refresh the local fleet-config checkout so the committed overlay change is + # visible in ~/environment/fleet-config (the task pushes from a temp clone). + # Best-effort and non-fatal: skipped if absent, warns if not fast-forwardable. + LOCAL_FLEET="$HOME/environment/fleet-config" + if [ -d "$LOCAL_FLEET/.git" ]; then + (git -C "$LOCAL_FLEET" fetch origin -q && git -C "$LOCAL_FLEET" pull --ff-only -q) \ + && printf '{{.C_INFO}} Refreshed local fleet-config checkout.{{.C_RESET}}\n' \ + || printf '{{.C_INFO}} Local fleet-config not auto-updated — run: git -C %s pull{{.C_RESET}}\n' "$LOCAL_FLEET" + fi + printf '{{.C_OK}}✓ Spoke {{.CLUSTER}} enabled via Crossplane in fleet-config overlay.{{.C_RESET}}\n' + + + spokes:create-capabilities: + desc: "Create KRO + ACK EKS Capabilities on a Crossplane-provisioned spoke (usage: task spokes:create-capabilities -- )" + # KRO-path spokes get their kro/ACK capabilities declaratively from the EksclusterWithVpc + # RGD. Crossplane-path spokes do NOT, so create them here the same way the hub does + # (aws eks create-capability), with per-spoke capability IAM roles. Idempotent. + vars: + CLUSTER: '{{.CLI_ARGS}}' + cmds: + - | + set -e + if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:create-capabilities -- "; exit 1; fi + CLUSTER="{{.CLUSTER}}" + REGION="{{.AWS_REGION}}" + ACCOUNT="{{.AWS_ACCOUNT_ID}}" + PREFIX="{{.RESOURCE_PREFIX}}" + # CLUSTER already carries the resource prefix (e.g. peeks-spoke-staging), so the + # capability role name is just --capability-role — prepending PREFIX + # again would double it (peeks-peeks-...). PREFIX is still used below for the + # cluster-mgmt-* assume-role resource ARNs. + ACK_ROLE_NAME="${CLUSTER}-ack-capability-role" + KRO_ROLE_NAME="${CLUSTER}-kro-capability-role" + + # Wait for the Crossplane-provisioned spoke EKS cluster to be ACTIVE (up to ~90 min). + printf '{{.C_STEP}}▸ [%s] Waiting for spoke EKS cluster ACTIVE...{{.C_RESET}}\n' "$CLUSTER" + i=0 + until aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null | grep -q ACTIVE; do + i=$((i+1)); [ "$i" -ge 360 ] && { printf '{{.C_ERR}}⚠ %s not ACTIVE after timeout — skipping capability creation.{{.C_RESET}}\n' "$CLUSTER"; exit 0; } + sleep 15 + done + + ASSUME_DOC='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"capabilities.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' + + # --- ACK capability role (AssumeWorkloadRoles + ManageIRSARoles) --- + printf '{{.C_STEP}}▸ [%s] Ensuring ACK capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$ACK_ROLE_NAME" + aws iam get-role --role-name "$ACK_ROLE_NAME" >/dev/null 2>&1 || \ + aws iam create-role --role-name "$ACK_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null + aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name AssumeWorkloadRoles --policy-document \ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sts:AssumeRole\",\"sts:TagSession\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${PREFIX}-cluster-mgmt-*\"]}]}" >/dev/null + aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name ManageIRSARoles --policy-document \ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"iam:GetRole\",\"iam:CreateRole\",\"iam:DeleteRole\",\"iam:TagRole\",\"iam:UntagRole\",\"iam:UpdateRole\",\"iam:UpdateAssumeRolePolicy\",\"iam:AttachRolePolicy\",\"iam:DetachRolePolicy\",\"iam:ListAttachedRolePolicies\",\"iam:ListRolePolicies\",\"iam:ListRoleTags\",\"iam:ListInstanceProfilesForRole\",\"iam:PutRolePolicy\",\"iam:DeleteRolePolicy\",\"iam:GetRolePolicy\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${CLUSTER}-*\"]}]}" >/dev/null + ACK_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${ACK_ROLE_NAME}" + + # --- KRO capability role (AmazonEKSClusterPolicy) --- + printf '{{.C_STEP}}▸ [%s] Ensuring KRO capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$KRO_ROLE_NAME" + aws iam get-role --role-name "$KRO_ROLE_NAME" >/dev/null 2>&1 || \ + aws iam create-role --role-name "$KRO_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null + aws iam attach-role-policy --role-name "$KRO_ROLE_NAME" --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy >/dev/null 2>&1 || true + KRO_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${KRO_ROLE_NAME}" + + # --- Create the capabilities (idempotent) --- + for cap in "kro:KRO:$KRO_ROLE_ARN" "ack:ACK:$ACK_ROLE_ARN"; do + NAME="${cap%%:*}"; rest="${cap#*:}"; TYPE="${rest%%:*}"; ROLE_ARN="${rest#*:}" + STATUS=$(aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null || echo "NOT_FOUND") + if [ "$STATUS" = "NOT_FOUND" ]; then + printf '{{.C_STEP}}▸ [%s] Creating %s capability...{{.C_RESET}}\n' "$CLUSTER" "$TYPE" + # The capability role was just created/updated above; EKS may reject + # CreateCapability with "trust policy is invalid" until IAM has + # propagated the role's trust policy. Retry on that transient error. + j=0 + until aws eks create-capability --region "$REGION" --cluster-name "$CLUSTER" --capability-name "$NAME" --type "$TYPE" --role-arn "$ROLE_ARN" --delete-propagation-policy RETAIN >/dev/null 2>/tmp/create-cap-err; do + j=$((j+1)) + if [ "$j" -ge 12 ]; then + printf '{{.C_ERR}}⚠ [%s] %s capability creation failed after retries:{{.C_RESET}}\n' "$CLUSTER" "$TYPE"; cat /tmp/create-cap-err; exit 1 + fi + if grep -q "trust policy" /tmp/create-cap-err; then + printf '{{.C_INFO}} [%s] Waiting for IAM trust policy propagation (attempt %s)...{{.C_RESET}}\n' "$CLUSTER" "$j" + sleep 10 + else + cat /tmp/create-cap-err; exit 1 + fi + done + else + printf '{{.C_INFO}} [%s] %s capability already exists (%s).{{.C_RESET}}\n' "$CLUSTER" "$TYPE" "$STATUS" + fi + done + + # --- Wait for both ACTIVE --- + for NAME in kro ack; do + i=0 + until aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null | grep -q ACTIVE; do + i=$((i+1)); [ "$i" -ge 40 ] && { printf '{{.C_ERR}}⚠ [%s] %s capability not ACTIVE after timeout.{{.C_RESET}}\n' "$CLUSTER" "$NAME"; break; } + sleep 15 + done + done + printf '{{.C_OK}}✓ [%s] KRO + ACK capabilities created.{{.C_RESET}}\n' "$CLUSTER" + + + spokes:enable-kro: + desc: "Enable a spoke cluster via KRO (usage: task spokes:enable-kro -- )" + vars: + CLUSTER: '{{.CLI_ARGS}}' + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + cmds: + - | + if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:enable-kro -- "; exit 1; fi + FLEET_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" + # fleet-config is created asynchronously by the gitlab-init Job (Helm + # post-install hook), which itself waits for GitLab to become ready. When + # enable-kro runs in phase2 it can race ahead of that job. Cloning an + # absent/empty repo previously left $REPO_DIR without a .git dir, so the + # later `git add -A` failed with exit 128 and took down the whole + # `task install`. Wait (up to ~5min) for the repo to be reachable, then + # verify the clone actually produced a git repo before proceeding. + printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' + for i in $(seq 1 30); do + git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + if [ "$i" -eq 30 ]; then + printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot enable spoke {{.CLUSTER}} via KRO.{{.C_RESET}}\n' + printf '{{.C_INFO}} GitLab/gitlab-init may still be initializing. Re-run later: task kind-crossplane:spokes:enable-kro -- {{.CLUSTER}}{{.C_RESET}}\n' + exit 1 + fi + sleep 10 + done + printf '{{.C_OK}}✓ fleet-config reachable.{{.C_RESET}}\n' + REPO_DIR=$(mktemp -d) + git clone "$FLEET_URL" "$REPO_DIR" + if [ ! -d "$REPO_DIR/.git" ]; then + printf '{{.C_ERR}}⚠ fleet-config clone failed — aborting enable-kro for {{.CLUSTER}}.{{.C_RESET}}\n' + rm -rf "$REPO_DIR" + exit 1 + fi + mkdir -p "$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/clusters" + cat > "$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/clusters/{{.CLUSTER}}.json" < InvalidSubnet.Range). + # Idempotency: only generate a (new random) CIDR when this spoke is not + # already enabled. Re-running install must NOT churn the CIDR — an EC2 VPC + # cidrBlock is immutable, so a new octet would force VPC recreation/orphans. + KRO_VALUES="$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/values.yaml" + if [ -f "$KRO_VALUES" ] && yq -e '.clusters."{{.CLUSTER}}".cidr.vpcCidr' "$KRO_VALUES" >/dev/null 2>&1; then + echo " Spoke {{.CLUSTER}} already has a CIDR — keeping existing values (idempotent)." + else + OCTET=$(shuf -i 2-254 -n 1) + cat > "$KRO_VALUES" </dev/null || echo "") + yq -i 'del(.clusters."{{.CLUSTER}}".adminRoles)' "$KRO_VALUES" 2>/dev/null || true + if [ -n "$CONSOLE_ROLE" ] && [ "$CONSOLE_ROLE" != "null" ]; then + CONSOLE_ROLE="$CONSOLE_ROLE" yq -i '.clusters."{{.CLUSTER}}".adminRoleName = env(CONSOLE_ROLE)' "$KRO_VALUES" + fi + if [ -n "$INSTANCE_ROLE" ] && [ "$INSTANCE_ROLE" != "None" ] && [ "$INSTANCE_ROLE" != "null" ]; then + INSTANCE_ROLE="$INSTANCE_ROLE" yq -i '.clusters."{{.CLUSTER}}".adminInstanceRoleName = env(INSTANCE_ROLE)' "$KRO_VALUES" + fi + # multi-acct (ACK CARM): map this spoke's namespace -> AWS account so the + # hub's ACK controllers render an IAMRoleSelector per service and assume + # the {{.RESOURCE_PREFIX}}-cluster-mgmt- roles when provisioning the + # spoke. The KRO EksclusterWithVpc lands in namespace == clusterName, which + # is the key the multi-acct chart selects on. Only KRO spokes need this — + # ACK does the provisioning; Crossplane spokes do not use CARM. + # Path = the appset-chart external per-addon overlay tier: + # $overlay/configs/multi-acct/values.yaml + # The fleet-config repo (seeded by the GitLab init-job) keeps configs/ and + # overlays/ at the REPO ROOT (overlay_repo_basepath=""), so this lives at + # configs/multi-acct/values.yaml — NOT under gitops/. (Spoke cluster values + # use gitops/fleet/... because that appset falls back to fleetRepoBasepath.) + # yq merge keeps it idempotent and accumulates entries across spokes. + mkdir -p "$REPO_DIR/configs/multi-acct" + MA_FILE="$REPO_DIR/configs/multi-acct/values.yaml" + [ -f "$MA_FILE" ] || printf 'clusters: {}\n' > "$MA_FILE" + yq -i '.clusters."{{.CLUSTER}}" = "{{.AWS_ACCOUNT_ID}}"' "$MA_FILE" + # platform-manifests: disable HuggingFaceModel CRs on KRO spokes. The + # HuggingFaceModel CRD (kro.run) is not installed on spokes, so the chart's + # CRs fail to apply ("no matches for kind HuggingFaceModel"); these models + # target GPU/Trainium clusters, not spokes. An empty list makes the chart's + # huggingfaceModels guard render nothing. Written to the per-cluster overlay + # tier the platform-manifests appset reads last: + # $overlay/overlays/clusters//platform-manifests/values.yaml + mkdir -p "$REPO_DIR/overlays/clusters/{{.CLUSTER}}/platform-manifests" + printf 'huggingfaceModels: []\n' > "$REPO_DIR/overlays/clusters/{{.CLUSTER}}/platform-manifests/values.yaml" + cd "$REPO_DIR" + git add -A + if git diff --cached --quiet; then + printf '{{.C_INFO}} No fleet-config changes for {{.CLUSTER}} — already enabled (idempotent).{{.C_RESET}}\n' + else + git commit -m "feat: enable spoke {{.CLUSTER}} via KRO (+ multi-acct CARM mapping)" + PUSHED=false + for i in 1 2 3; do + if git push 2>&1; then PUSHED=true; break; fi + echo " Push attempt $i failed, rebasing..." + git pull --rebase 2>&1 && sleep 2 + done + if [ "$PUSHED" != "true" ]; then + printf '{{.C_ERR}}\u26a0 fleet-config push failed after 3 attempts.{{.C_RESET}}\n' + exit 1 + fi + fi + rm -rf "$REPO_DIR" + # Refresh the local fleet-config checkout so the committed overlay change is + # visible in ~/environment/fleet-config (the task pushes from a temp clone). + # Best-effort and non-fatal: skipped if absent, warns if not fast-forwardable. + LOCAL_FLEET="$HOME/environment/fleet-config" + if [ -d "$LOCAL_FLEET/.git" ]; then + (git -C "$LOCAL_FLEET" fetch origin -q && git -C "$LOCAL_FLEET" pull --ff-only -q) \ + && printf '{{.C_INFO}} Refreshed local fleet-config checkout.{{.C_RESET}}\n' \ + || printf '{{.C_INFO}} Local fleet-config not auto-updated — run: git -C %s pull{{.C_RESET}}\n' "$LOCAL_FLEET" + fi + printf '{{.C_OK}}✓ Spoke {{.CLUSTER}} enabled via KRO + multi-acct CARM mapping in fleet-config overlay.{{.C_RESET}}\n' + + + spokes:disable-crossplane: + desc: "DELETE a Crossplane spoke on purpose: remove from overlay, then prune (usage: task spokes:disable-crossplane -- )" + vars: + CLUSTER: '{{.CLI_ARGS}}' + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + cmds: + - | + if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:disable-crossplane -- "; exit 1; fi + # Crossplane clusters are grouped under one per-tenant app (clusters-). + APP="clusters-workshop" + + # 1. Declarative removal: drop the cluster entry from the overlay (Git). + FLEET_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" + # Guard: a silently-failed clone would leave an empty $REPO_DIR, making us + # wrongly conclude the cluster is absent from the overlay and skip the Git + # removal (stale overlay left behind). Wait for fleet-config then verify. + printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' + for i in $(seq 1 30); do + git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + if [ "$i" -eq 30 ]; then + printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot disable spoke {{.CLUSTER}} via Crossplane.{{.C_RESET}}\n' + exit 1 + fi + sleep 10 + done + REPO_DIR=$(mktemp -d) + git clone "$FLEET_URL" "$REPO_DIR" + if [ ! -d "$REPO_DIR/.git" ]; then + printf '{{.C_ERR}}⚠ fleet-config clone failed — aborting disable-crossplane for {{.CLUSTER}}.{{.C_RESET}}\n' + rm -rf "$REPO_DIR" + exit 1 + fi + XP_VALUES="$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/crossplane-clusters/values.yaml" + if [ -f "$XP_VALUES" ] && yq -e '.clusters."{{.CLUSTER}}"' "$XP_VALUES" >/dev/null 2>&1; then + yq -i 'del(.clusters."{{.CLUSTER}}")' "$XP_VALUES" + ( cd "$REPO_DIR" && git add -A && git commit -m "feat: disable spoke {{.CLUSTER}} (remove from Crossplane overlay)" && git push 2>/dev/null ) + printf '{{.C_OK}}✓ Removed {{.CLUSTER}} from the Crossplane overlay (Git).{{.C_RESET}}\n' + else + printf '{{.C_INFO}} {{.CLUSTER}} not present in Crossplane overlay — proceeding to prune any leftover resources.{{.C_RESET}}\n' + fi + rm -rf "$REPO_DIR" + + # 2. Deliberate prune. With prune:false on the appset this manual --prune is + # the explicit destructive step. Refresh + show the diff first so the + # operator sees exactly what will be deleted. + printf '{{.C_ERR}}▸ Pruning app %s — this DELETES the live cluster {{.CLUSTER}} + its VPC.{{.C_RESET}}\n' "$APP" + if command -v argocd >/dev/null 2>&1; then + argocd app get "$APP" --refresh >/dev/null 2>&1 || true + argocd app diff "$APP" || true + if ! argocd app sync "$APP" --prune --timeout 600; then + printf '{{.C_ERR}}⚠ argocd prune failed (auth?). Run: argocd-refresh-token && source ~/.bashrc.d/platform.sh, then:{{.C_RESET}}\n' + printf ' argocd app sync %s --prune\n' "$APP" + fi + else + printf '{{.C_INFO}} argocd CLI not found. Finish the deletion manually:{{.C_RESET}}\n' + printf ' argocd app sync %s --prune\n' "$APP" + fi + + + spokes:disable-kro: + desc: "DELETE a KRO spoke on purpose: remove from overlay, then prune (usage: task spokes:disable-kro -- )" + vars: + CLUSTER: '{{.CLI_ARGS}}' + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + cmds: + - | + if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:disable-kro -- "; exit 1; fi + # KRO generates one app per cluster: clusters-kro-. + APP="clusters-kro-{{.CLUSTER}}" + FLEET_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" + # Guard: a silently-failed clone would leave an empty $REPO_DIR, making the + # overlay cleanup below a no-op (stale kro overlay + marker left behind). + # Wait for fleet-config then verify the clone produced a git repo. + printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' + for i in $(seq 1 30); do + git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + if [ "$i" -eq 30 ]; then + printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot disable spoke {{.CLUSTER}} via KRO.{{.C_RESET}}\n' + exit 1 + fi + sleep 10 + done + REPO_DIR=$(mktemp -d) + git clone "$FLEET_URL" "$REPO_DIR" + if [ ! -d "$REPO_DIR/.git" ]; then + printf '{{.C_ERR}}⚠ fleet-config clone failed — aborting disable-kro for {{.CLUSTER}}.{{.C_RESET}}\n' + rm -rf "$REPO_DIR" + exit 1 + fi + BASE="$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters" + KRO_VALUES="$BASE/values.yaml" + MARKER="$BASE/clusters/{{.CLUSTER}}.json" + MA_FILE="$REPO_DIR/configs/multi-acct/values.yaml" + + # PHASE 1: remove ONLY the cluster spec from values.yaml. Keep the clusters/*.json + # marker for now so the appset still generates clusters-kro- and we can prune + # through it. (Deleting the marker first would make the appset remove the app with + # preserveResourcesOnDeletion=true -> the EksclusterWithVpc would be ORPHANED, not + # deleted.) + if [ -f "$KRO_VALUES" ] && yq -e '.clusters."{{.CLUSTER}}"' "$KRO_VALUES" >/dev/null 2>&1; then + yq -i 'del(.clusters."{{.CLUSTER}}")' "$KRO_VALUES" + ( cd "$REPO_DIR" && git add -A && git commit -m "feat: disable spoke {{.CLUSTER}} (remove KRO cluster spec)" && git push 2>/dev/null ) + printf '{{.C_OK}}✓ Removed {{.CLUSTER}} spec from the KRO overlay (Git).{{.C_RESET}}\n' + else + printf '{{.C_INFO}} {{.CLUSTER}} spec not present in KRO overlay — proceeding to prune any leftovers.{{.C_RESET}}\n' + fi + + # PHASE 2: deliberate prune through the still-existing app. + printf '{{.C_ERR}}▸ Pruning app %s — this DELETES the live cluster {{.CLUSTER}} + its VPC.{{.C_RESET}}\n' "$APP" + PRUNED=false + if command -v argocd >/dev/null 2>&1; then + argocd app get "$APP" --refresh >/dev/null 2>&1 || true + argocd app diff "$APP" || true + if argocd app sync "$APP" --prune --timeout 600; then PRUNED=true; else + printf '{{.C_ERR}}⚠ argocd prune failed (auth?). Run: argocd-refresh-token && source ~/.bashrc.d/platform.sh, then: argocd app sync %s --prune{{.C_RESET}}\n' "$APP" + fi + else + printf '{{.C_INFO}} argocd CLI not found. Run manually then re-run this task to clean up: argocd app sync %s --prune{{.C_RESET}}\n' "$APP" + fi + + # PHASE 3: cleanup the now-empty app + CARM mapping (only after a successful prune, + # so we never orphan a live resource). The appset removes clusters-kro- once + # the marker is gone; its resources are already pruned. + if [ "$PRUNED" = true ]; then + CHANGED=0 + [ -f "$MARKER" ] && { rm -f "$MARKER"; CHANGED=1; } + [ -f "$MA_FILE" ] && yq -e '.clusters."{{.CLUSTER}}"' "$MA_FILE" >/dev/null 2>&1 && { yq -i 'del(.clusters."{{.CLUSTER}}")' "$MA_FILE"; CHANGED=1; } + if [ "$CHANGED" = "1" ]; then + ( cd "$REPO_DIR" && git add -A && git commit -m "chore: remove {{.CLUSTER}} KRO marker + multi-acct mapping (post-prune cleanup)" && git push 2>/dev/null ) + printf '{{.C_OK}}✓ Cleaned up {{.CLUSTER}} marker + multi-acct mapping; app will be removed by the appset.{{.C_RESET}}\n' + fi + else + printf '{{.C_INFO}} Skipped marker/CARM cleanup (prune not confirmed). Re-run this task after a successful prune.{{.C_RESET}}\n' + fi + rm -rf "$REPO_DIR" + + + spokes:disable-all: + desc: >- + DELETE every spoke cluster (KRO + Crossplane) declared in the fleet-config + overlay, then wait for them to disappear from AWS. Used as Phase 0 of + `destroy` so spokes are removed while the hub controllers (KRO/ACK/Crossplane) + are still alive to reconcile their deletion. Idempotent. + vars: + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + cmds: + - | + set -u + # 1. Discover spokes from the fleet-config overlay (the single source of truth + # for cluster specs). If the overlay is unreachable we must NOT silently + # proceed to hub teardown — surviving spokes would be orphaned. + REPO_DIR=$(mktemp -d) + if ! git clone "https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/fleet-config.git" "$REPO_DIR" 2>/dev/null; then + printf '{{.C_ERR}}⚠ fleet-config overlay unreachable — cannot discover spokes automatically.{{.C_RESET}}\n' + printf '{{.C_INFO}} If any spoke clusters exist, delete them first with:{{.C_RESET}}\n' + printf ' task spokes:disable-kro -- / task spokes:disable-crossplane -- \n' + rm -rf "$REPO_DIR"; exit 0 + fi + KRO_VALUES="$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/values.yaml" + XP_VALUES="$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/crossplane-clusters/values.yaml" + KRO_SPOKES=""; XP_SPOKES="" + [ -f "$KRO_VALUES" ] && KRO_SPOKES=$(yq '.clusters // {} | keys | .[]' "$KRO_VALUES" 2>/dev/null) + [ -f "$XP_VALUES" ] && XP_SPOKES=$(yq '.clusters // {} | keys | .[]' "$XP_VALUES" 2>/dev/null) + rm -rf "$REPO_DIR" + + if [ -z "$(echo $KRO_SPOKES $XP_SPOKES | tr -d ' ')" ]; then + printf '{{.C_OK}}✓ No spoke clusters declared in the overlay — nothing to tear down.{{.C_RESET}}\n' + exit 0 + fi + printf '{{.C_STEP}}▸ Spokes to delete — KRO: [%s] Crossplane: [%s]{{.C_RESET}}\n' "$(echo $KRO_SPOKES | tr '\n' ' ')" "$(echo $XP_SPOKES | tr '\n' ' ')" + + # 2. Deliberate deletion via the existing per-cluster disable tasks (they remove + # the cluster from the overlay in Git, then run a scoped `argocd app sync --prune`). + for c in $KRO_SPOKES; do + printf '{{.C_STEP}}▸ Disabling KRO spoke %s...{{.C_RESET}}\n' "$c" + task spokes:disable-kro -- "$c" || printf '{{.C_ERR}}⚠ disable-kro failed for %s (continuing).{{.C_RESET}}\n' "$c" + done + for c in $XP_SPOKES; do + printf '{{.C_STEP}}▸ Disabling Crossplane spoke %s...{{.C_RESET}}\n' "$c" + task spokes:disable-crossplane -- "$c" || printf '{{.C_ERR}}⚠ disable-crossplane failed for %s (continuing).{{.C_RESET}}\n' "$c" + done + + # 3. Wait for the spoke EKS clusters to actually disappear from AWS BEFORE the + # hub teardown removes the controllers that reconcile their deletion. Without + # this, killing the hub mid-deletion leaves orphaned EKS clusters + VPCs. + ALL_SPOKES="$KRO_SPOKES $XP_SPOKES" + printf '{{.C_STEP}}▸ Waiting for spoke EKS clusters to delete in AWS (up to 30m)...{{.C_RESET}}\n' + for i in $(seq 1 90); do + LIVE="" + for c in $ALL_SPOKES; do + aws eks describe-cluster --name "$c" --region {{.AWS_REGION}} >/dev/null 2>&1 && LIVE="$LIVE $c" + done + if [ -z "$(echo $LIVE | tr -d ' ')" ]; then + printf '{{.C_OK}}✓ All spoke clusters deleted from AWS.{{.C_RESET}}\n'; break + fi + printf '{{.C_INFO}} Still deleting:%s ... (%s/90){{.C_RESET}}\n' "$LIVE" "$i" + sleep 20 + done + # Final check — warn loudly (but do not hard-fail) if a spoke survived, so the + # operator can intervene before/after the hub goes away. + STILL_LIVE="" + for c in $ALL_SPOKES; do + aws eks describe-cluster --name "$c" --region {{.AWS_REGION}} >/dev/null 2>&1 && STILL_LIVE="$STILL_LIVE $c" + done + if [ -n "$(echo $STILL_LIVE | tr -d ' ')" ]; then + printf '{{.C_ERR}}⚠ Spoke clusters still present after wait:%s{{.C_RESET}}\n' "$STILL_LIVE" + printf '{{.C_ERR}} Proceeding with hub teardown will ORPHAN them. See troubleshooting.md →{{.C_RESET}}\n' + printf '{{.C_ERR}} "Spoke Cluster Stuck Deletion / Recovery". Recommend aborting and fixing first.{{.C_RESET}}\n' + fi + + + hub:restart-identity-pods: + desc: Restart pods that depend on Pod Identity (external-dns, LBC, Crossplane providers) after identities are ready + status: + - | + KC={{.ROOT_DIR}}/private/hub-kubeconfig + # Skip if all providers are Healthy and have been running > 5min (pod identity already active) + UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' -n crossplane-system 2>/dev/null | wc -w) + [ "$UNHEALTHY" = "0" ] && OLDEST=$(KUBECONFIG=$KC kubectl get pods -n crossplane-system -l pkg.crossplane.io/revision --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[0].metadata.creationTimestamp}' 2>/dev/null) && [ -n "$OLDEST" ] && AGE=$(( $(date +%s) - $(date -d "$OLDEST" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$OLDEST" +%s 2>/dev/null) )) && [ "$AGE" -gt 300 ] + cmds: + - task: hub:kubeconfig + - printf '{{.C_STEP}}▸ Terminating any stuck ArgoCD sync operations on pod-identities apps...{{.C_RESET}}\n' + - | + # A stuck PreSync hook on pod-identities-* (e.g. crd-wait Job whose + # resources were already deleted) leaves operationState.phase=Running + # indefinitely. This prevents PodIdentityAssociations from ever being + # created, which in turn means LBC/external-dns never get AWS creds. + # Terminate any such stuck operations before waiting. + KC="KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig" + eval "$KC" kubectl get applications -n argocd -o json 2>/dev/null | \ + jq -r '.items[] | select( + (.metadata.name | startswith("pod-identities")) and + (.status.operationState.phase == "Running") + ) | .metadata.name' | \ + while read -r app; do + printf ' ⚠ Terminating stuck sync on %s\n' "$app" + eval "$KC" kubectl patch application "$app" -n argocd \ + --type='json' -p='[{"op":"remove","path":"/operation"}]' 2>/dev/null || true + sleep 3 + eval "$KC" kubectl patch application "$app" -n argocd --type merge \ + -p '{"operation":{"initiatedBy":{"username":"admin"},"sync":{"revision":"HEAD"}}}' \ + 2>/dev/null || true + done + - printf '{{.C_STEP}}▸ Waiting for pod identity associations to be Ready...{{.C_RESET}}\n' + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl -n crossplane-system wait --for=condition=Ready podidentityassociations.eks.aws.upbound.io --all --timeout=300s 2>/dev/null || true + - printf '{{.C_STEP}}▸ Restarting Crossplane providers to pick up pod identity credentials...{{.C_RESET}}\n' + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl delete pod -n crossplane-system -l pkg.crossplane.io/revision --force --grace-period=0 2>/dev/null || true + - sleep 15 + - printf '{{.C_STEP}}▸ Restarting external-dns and LBC to pick up pod identity credentials...{{.C_RESET}}\n' + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout restart deploy -n kube-system -l app.kubernetes.io/name=external-dns 2>/dev/null || true + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout restart deploy -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller 2>/dev/null || true + - printf '{{.C_STEP}}▸ Restarting LiteLLM to pick up pod identity credentials...{{.C_RESET}}\n' + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout restart deploy -n kagent litellm 2>/dev/null || true + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout restart deploy -n otel otel-collector 2>/dev/null || true + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout status deploy -n kube-system -l app.kubernetes.io/name=external-dns --timeout=60s 2>/dev/null || true + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout status deploy -n kube-system -l app.kubernetes.io/name=aws-load-balancer-controller --timeout=60s 2>/dev/null || true + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout status deploy -n kagent litellm --timeout=120s 2>/dev/null || true + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + - printf '{{.C_OK}}✓ Identity-dependent pods restarted.{{.C_RESET}}\n' + + + hub:update: + desc: Update hub infra via Kind+Crossplane + vars: + MNG_SETS: + sh: | + if [ "$(yq '.hub.managedNodeGroup.enabled // false' {{.CONFIG_FILE}})" = "true" ]; then + SETS="--set clusters.hub.managedNodeGroup.enabled=true" + IDX=0 + for TYPE in $(yq '.hub.managedNodeGroup.instanceTypes // ["m5.large"] | .[]' {{.CONFIG_FILE}}); do + SETS="$SETS --set clusters.hub.managedNodeGroup.instanceTypes[$IDX]=$TYPE" + IDX=$((IDX+1)) + done + SETS="$SETS --set clusters.hub.managedNodeGroup.desiredSize=$(yq '.hub.managedNodeGroup.desiredSize // 2' {{.CONFIG_FILE}})" + SETS="$SETS --set clusters.hub.managedNodeGroup.minSize=$(yq '.hub.managedNodeGroup.minSize // 1' {{.CONFIG_FILE}})" + SETS="$SETS --set clusters.hub.managedNodeGroup.maxSize=$(yq '.hub.managedNodeGroup.maxSize // 5' {{.CONFIG_FILE}})" + SETS="$SETS --set clusters.hub.managedNodeGroup.diskSize=$(yq '.hub.managedNodeGroup.diskSize // 50' {{.CONFIG_FILE}})" + SETS="$SETS --set clusters.hub.managedNodeGroup.capacityType=$(yq '.hub.managedNodeGroup.capacityType // "ON_DEMAND"' {{.CONFIG_FILE}})" + echo "$SETS" + fi + cmds: + - task: init + - task: kind:create + - task: credentials:setup + - task: crossplane:helm + - task: crossplane:providers + - task: crossplane:provider-config + - kubectl apply -f {{.GITOPS_ROOT}}/abstractions/crossplane/platform-cluster/templates/ 2>/dev/null || true + - helm template hub-cluster {{.GITOPS_ROOT}}/abstractions/crossplane/platform-cluster + --set clusters.hub.region={{.AWS_REGION}} + --set clusters.hub.clusterName={{.HUB_CLUSTER_NAME}} + --set clusters.hub.kubernetesVersion={{.HUB_K8S_VERSION}} + --set clusters.hub.vpcCidr={{.HUB_VPC_CIDR}} + --set clusters.hub.resourcePrefix={{.RESOURCE_PREFIX}} + {{.MNG_SETS}} + | kubectl apply -f - + - kubectl apply -f claims/ + - kubectl -n crossplane-system wait --for=condition=Ready --timeout=3500s platformclusters.platform.gitops.io/{{.HUB_CLUSTER_NAME}} + - task: destroy-kind + + + spokes:seed-provider-identity: + desc: >- + Seed the bootstrap Crossplane provider IAM roles + EKS Pod Identity + associations for spoke clusters (same account). The iam/eks providers are + createIdentity:false in the registry, so crossplane-base does NOT create + their roles; without this seed a spoke's iam/eks providers cannot + authenticate (chicken-and-egg). Mirrors the hub bootstrap. Idempotent. + Run after spokes are provisioned (ACTIVE). Cross-account spokes are NOT + handled here - their roles must be bootstrapped in the target account. + cmds: + - cmd: | + # provider service-account suffix -> IAM role name suffix + PROVIDERS="iam:CrossplaneIAMProviderRole eks:CrossplaneEKSProviderRole" + # Discover spokes from fleet members, excluding the hub. + SPOKES=$(ls "{{.GITOPS_ROOT}}/fleet/members" 2>/dev/null | grep -vx "{{.HUB_CLUSTER_NAME}}" || true) + if [ -z "$SPOKES" ]; then + printf '{{.C_INFO}} No spoke members under fleet/members; nothing to seed.{{.C_RESET}}\n' + exit 0 + fi + for SPOKE in $SPOKES; do + STATUS=$(aws eks describe-cluster --name "$SPOKE" --region {{.AWS_REGION}} --query 'cluster.status' --output text 2>/dev/null || echo "NONE") + if [ "$STATUS" != "ACTIVE" ]; then + printf '{{.C_INFO}} Skipping %s (cluster status: %s){{.C_RESET}}\n' "$SPOKE" "$STATUS" + continue + fi + for ENTRY in $PROVIDERS; do + SVC="${ENTRY%%:*}" # iam | eks + ROLE_NAME="${SPOKE}-${ENTRY##*:}" + SA="provider-aws-${SVC}" + ROLE_ARN="arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/${ROLE_NAME}" + # 1. IAM role trusting EKS Pod Identity (idempotent) + if ! aws iam get-role --role-name "$ROLE_NAME" >/dev/null 2>&1; then + aws iam create-role --role-name "$ROLE_NAME" \ + --assume-role-policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"pods.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' \ + --tags Key=managed-by,Value=platform-bootstrap Key=purpose,Value="${SVC}-provider" >/dev/null + printf '{{.C_OK}} ✓ %s: created role %s{{.C_RESET}}\n' "$SPOKE" "$ROLE_NAME" + else + printf '{{.C_INFO}} %s: role %s exists{{.C_RESET}}\n' "$SPOKE" "$ROLE_NAME" + fi + # 2. AdministratorAccess (attach is idempotent) + aws iam attach-role-policy --role-name "$ROLE_NAME" \ + --policy-arn arn:aws:iam::aws:policy/AdministratorAccess >/dev/null + # 3. Pod Identity association on the spoke (idempotent) + EXISTING=$(aws eks list-pod-identity-associations --cluster-name "$SPOKE" --region {{.AWS_REGION}} \ + --namespace crossplane-system --service-account "$SA" \ + --query 'associations[0].associationId' --output text 2>/dev/null || echo "None") + if [ "$EXISTING" = "None" ] || [ -z "$EXISTING" ]; then + aws eks create-pod-identity-association --cluster-name "$SPOKE" --region {{.AWS_REGION}} \ + --namespace crossplane-system --service-account "$SA" --role-arn "$ROLE_ARN" >/dev/null + printf '{{.C_OK}} ✓ %s: pod identity %s -> %s{{.C_RESET}}\n' "$SPOKE" "$SA" "$ROLE_NAME" + else + printf '{{.C_INFO}} %s: pod identity for %s exists{{.C_RESET}}\n' "$SPOKE" "$SA" + fi + done + done + printf '{{.C_OK}}✓ Spoke provider identity seeding complete.{{.C_RESET}}\n' + + From ac60d6c9fe819f7427c538ff4806e23006fd1893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 11:37:14 +0200 Subject: [PATCH 003/231] feat: complete kind-kro-ack consolidation - Add parallel install phases (phase1/phase2) to kind-kro-ack install task - Move gitlab:init-ec2 + gitlab:clone-repos to workshop/Taskfile.yaml - crossplane-system refs in copied tasks left intentional (Crossplane addon on hub EKS) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 39 +++- cluster-providers/workshop/Taskfile.yaml | 179 +++++++++++++++++++ 2 files changed, 217 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 972f052f3..756acf83b 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -94,6 +94,9 @@ tasks: install: desc: Bootstrap the platform (idempotent) + vars: + INSTALL_START: + sh: date +%s cmds: - task: kind:create - task: credentials:setup @@ -107,7 +110,41 @@ tasks: - task: hub:cloudfront - task: hub:seed - task: hub:wait-for-sync - - printf '{{.C_OK}}✓ Bootstrap complete. Run task status.{{.C_RESET}}\n' + - task: install:phase1-parallel + - task: hub:wait-for-full-sync + - task: hub:set-overlay-repo + - task: install:phase2-parallel + - | + LOCAL_FLEET="$HOME/environment/fleet-config" + if [ -d "$LOCAL_FLEET/.git" ]; then + git -C "$LOCAL_FLEET" pull --ff-only -q 2>/dev/null || true + fi + - printf '{{.C_OK}}✓ Bootstrap complete in %ds. Run task status.{{.C_RESET}}\n' "$(($(date +%s) - {{.INSTALL_START}}))" + + install:phase1-parallel: + internal: true + deps: + - hub:restart-identity-pods + + install:phase2-parallel: + internal: true + cmds: + - task: install:phase2-spoke-dev + - task: install:phase2-spoke-prod + + install:phase2-spoke-dev: + internal: true + cmds: + - task: spokes:enable-crossplane + vars: { CLI_ARGS: "{{.RESOURCE_PREFIX}}-spoke-dev" } + - task: spokes:create-capabilities + vars: { CLI_ARGS: "{{.RESOURCE_PREFIX}}-spoke-dev" } + + install:phase2-spoke-prod: + internal: true + cmds: + - task: spokes:enable-kro + vars: { CLI_ARGS: "{{.RESOURCE_PREFIX}}-spoke-prod" } kind:create: desc: Create Kind cluster (skips if exists) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 76768f1cc..97cb00ffc 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -170,3 +170,182 @@ tasks: else printf '{{.C_ERR}}⚠ SCIM admin group not found.{{.C_RESET}}\n' fi + + # ---- Workshop: GitLab CE on EC2 (workshop-specific) ---- + gitlab:init-ec2: + desc: Wait for GitLab CE (docker-compose on IDE EC2) to be ready, then initialise root token + user1 + repos + vars: + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // "{{.HUB_CLUSTER_NAME}}"') + GIT_USERNAME: + sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} + REPOS: + sh: echo "fleet-config java rust golang dotnet next-js" + cmds: + - printf '{{.C_STEP}}▸ Waiting for GitLab CE on EC2 to be ready...{{.C_RESET}}\n' + - | + set -e + # GitLab CE starts in docker-compose at EC2 boot (bootstrap.sh). Wait up to 20min. + GITLAB_URL="https://{{.GITLAB_CF_DOMAIN}}" + for i in $(seq 1 120); do + if curl -sf --max-time 10 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1; then + printf ' GitLab ready.\n'; break + fi + [ "$i" -eq 120 ] && { printf ' ⚠ GitLab not ready after 20min — continuing anyway.\n'; } + sleep 10 + done + ROOT_TOKEN="root-{{.GIT_PASSWORD}}" + # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. + # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash + # before the string is passed to gitlab-rails runner as a Ruby script. + docker exec gitlab-gitlab-1 gitlab-rails runner " + # Disable default branch protection so user1 can push to main + ApplicationSetting.first.update!( + default_branch_protection: 0, + default_branch_protection_defaults: { + allowed_to_push: [{ access_level: 0 }], + allow_force_push: true, + allowed_to_merge: [{ access_level: 0 }], + developer_can_initial_push: true + } + ) + pat = User.find(1).personal_access_tokens.find_or_initialize_by(name: 'initial-root-token') + pat.scopes = ['api','write_repository','sudo'] + pat.expires_at = 365.days.from_now + pat.save! + u = User.find_by(username: '{{.GIT_USERNAME}}') || User.create!( + name: 'Workshop User', username: '{{.GIT_USERNAME}}', + email: '{{.GIT_USERNAME}}@workshop.local', + password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', + skip_confirmation: true + ) + u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') + u1pat.scopes = ['api','write_repository'] + u1pat.expires_at = 365.days.from_now + u1pat.save! + %w[fleet-config java rust golang dotnet next-js].each do |repo| + p = Project.find_by(path: repo, namespace_id: u.namespace_id) || + Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute + ProtectedBranch.where(project: p).destroy_all + puts 'repo: ' + repo + end + puts 'init done' + " 2>&1 | tail -20 || true + WORKSHOP_DIR="$HOME/environment/{{.WORKING_REPO}}" + for REPO in {{.REPOS}}; do + REMOTE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/$REPO.git" + if [ "$REPO" = "fleet-config" ]; then + SRC="$WORKSHOP_DIR/gitops/fleet" + else + SRC="$WORKSHOP_DIR/applications/$REPO" + fi + [ -d "$SRC" ] || continue + TMP=$(mktemp -d) + cp -r "$SRC/." "$TMP/" + cd "$TMP" + git init -q && git add -A && git commit -qm "init" 2>/dev/null || true + git remote add origin "$REMOTE_URL" + git push -f origin HEAD:main + cd - >/dev/null; rm -rf "$TMP" + echo " seeded: $REPO" + done + printf '{{.C_OK}}✓ GitLab init done.{{.C_RESET}}\n' + # After init, clone repos locally + - task: gitlab:clone-repos + + + gitlab:clone-repos: + desc: Clone all GitLab repos locally into ~/environment + vars: + GITLAB_CF_DOMAIN: + sh: | + if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then + cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain + else + echo "{{.DOMAIN}}" + fi + GIT_USERNAME: + sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} + GIT_PASSWORD: + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // "{{.HUB_CLUSTER_NAME}}"') + REPOS: + sh: echo "fleet-config java rust golang dotnet next-js" + cmds: + - printf '{{.C_STEP}}▸ Cloning GitLab repos to ~/environment...{{.C_RESET}}\n' + - | + BASE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}" + FAILED="" + # Wait for GitLab (running via docker-compose on the IDE EC2) to be ready + # and for the init script to have created all repos. + # bootstrap.sh starts GitLab at boot; task install calls gitlab:init-ec2 first, + # so by the time clone-repos runs the repos should exist. Poll as a safety net. + printf ' Waiting for GitLab to be ready (up to 20min)...\n' + GITLAB_BASE="https://{{.GITLAB_CF_DOMAIN}}" + for i in $(seq 1 120); do + if curl -sf --max-time 5 "$GITLAB_BASE/users/sign_in" >/dev/null 2>&1; then break; fi + sleep 10 + done + for REPO in {{.REPOS}}; do + if [ "$REPO" = "fleet-config" ]; then + DEST="$HOME/environment/$REPO" + else + DEST="$HOME/environment/applications/$REPO" + fi + REPO_URL="$BASE_URL/$REPO.git" + if [ -d "$DEST/.git" ]; then + # Already cloned. The gitlab-init Job creates each repo with only a + # README first, then pushes the real source/overlay content later in + # the same Job. An earlier clone-repos run (or this one racing ahead) + # may therefore have captured the README-only state. Pull to pick up + # any content pushed after the initial clone — fast-forward only so we + # never clobber participant commits. + echo " $REPO already cloned — pulling latest" + git -C "$DEST" pull --ff-only -q 2>/dev/null || echo " ($REPO: pull skipped — local changes or no upstream)" + continue + fi + # GitLab repos are created asynchronously by the gitlab-init Job (Helm + # post-install hook, which waits for GitLab readiness). Without waiting, + # this task ran before the repos existed and every clone failed, leaving + # ~/environment/applications empty and no local fleet-config checkout. + # Wait (up to ~5min) for each repo to be reachable AND seeded before + # cloning. Reachable alone is not enough: the repo exists with only a + # README the instant it is created, but the application source is pushed + # a few seconds later. Cloning on first-reachable captured the stub + # (README/.git only), leaving e.g. ~/environment/applications/next-js + # without deployable code. Wait until the remote has a non-README file + # (fleet-config is exempt — it is content-seeded differently). + REACHABLE=false + for i in $(seq 1 18); do + if git ls-remote "$REPO_URL" >/dev/null 2>&1; then + REACHABLE=true; break + fi + sleep 10 + done + if [ "$REACHABLE" != true ]; then + echo " ⚠ $REPO not reachable/seeded after 5min — skipping (create later)." + FAILED="$FAILED $REPO" + continue + fi + mkdir -p "$(dirname "$DEST")" + if git clone "$REPO_URL" "$DEST" 2>/dev/null; then + echo " ✓ $REPO cloned" + else + echo " ⚠ $REPO clone failed" + FAILED="$FAILED$REPO " + fi + done + if [ -n "$FAILED" ]; then + printf '{{.C_ERR}}⚠ Some GitLab repos were not cloned:%s{{.C_RESET}}\n' "$FAILED" + printf '{{.C_INFO}} Re-run later: task kind-crossplane:gitlab:clone-repos{{.C_RESET}}\n' + else + printf '{{.C_OK}}✓ GitLab repos cloned.{{.C_RESET}}\n' + fi + + From ec3d5f8ad4e3599352ad383a0c34cf4b7d750611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 11:50:14 +0200 Subject: [PATCH 004/231] feat: create-config.sh reads CLUSTER_PROVIDER env var (default: kind-crossplane) --- create-config.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/create-config.sh b/create-config.sh index cef3ab608..3b65c6c7a 100755 --- a/create-config.sh +++ b/create-config.sh @@ -45,6 +45,7 @@ OUTPUT_FILE="${OUTPUT_FILE:-${SCRIPT_DIR}/config.local.yaml}" RESOURCE_PREFIX="${RESOURCE_PREFIX:-peeks}" REPO_URL="${REPO_URL:-https://github.com/aws-samples/appmod-blueprints}" REPO_REVISION="${REPO_REVISION:-${WORKSHOP_GIT_BRANCH:-feature/cloudfront-on-agent-platform}}" +CLUSTER_PROVIDER="${CLUSTER_PROVIDER:-kind-crossplane}" K8S_VERSION="${K8S_VERSION:-1.35}" VPC_CIDR="${VPC_CIDR:-10.1.0.0/16}" FORCE="${FORCE:-false}" @@ -138,7 +139,7 @@ fi # --- Write config.local.yaml (printf, never heredoc) ----------------------- echo "▸ Writing $OUTPUT_FILE ..." -printf 'clusterProvider: "kind-crossplane"\n' > "$OUTPUT_FILE" +printf 'clusterProvider: "%s"\n' "$CLUSTER_PROVIDER" > "$OUTPUT_FILE" printf 'repo:\n' >> "$OUTPUT_FILE" printf ' url: "%s"\n' "$REPO_URL" >> "$OUTPUT_FILE" printf ' revision: "%s"\n' "$REPO_REVISION" >> "$OUTPUT_FILE" @@ -183,7 +184,7 @@ echo "▸ Validating generated YAML..." yq '.' "$OUTPUT_FILE" >/dev/null echo "✓ config.local.yaml created:" -echo " region=$REGION accountId=$ACCOUNT_ID prefix=$RESOURCE_PREFIX" +echo " clusterProvider=$CLUSTER_PROVIDER region=$REGION accountId=$ACCOUNT_ID prefix=$RESOURCE_PREFIX" echo " clusterName=${RESOURCE_PREFIX}-hub adminRole=$ADMIN_ROLE_NAME" echo " idcInstance=$IDC_ARN adminGroupId=${IDC_GROUP:-}" echo " domain=\"\" (CloudFront exposure mode) cloudfrontDomain=${CF_DOMAIN:-}" From 19425f2895f83a5032994e2fce0368cb2875c08b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 11:59:17 +0200 Subject: [PATCH 005/231] =?UTF-8?q?refactor:=20remove=20argocd:capability?= =?UTF-8?q?=20job=20from=20kind-kro-ack=20=E2=80=94=20Capabilities=20manag?= =?UTF-8?q?ed=20declaratively=20by=20RGD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-kro-ack/Taskfile.yaml | 22 -------------------- 1 file changed, 22 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 756acf83b..6af16b5b9 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -769,28 +769,6 @@ tasks: cmds: - aws eks update-kubeconfig --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --kubeconfig {{.ROOT_DIR}}/private/hub-kubeconfig 2>&1 >/dev/null - argocd:capability: - desc: Create EKS ArgoCD Capability via Job - status: - - kubectl -n {{.ACK_NAMESPACE}} get job create-argocd-capability -o jsonpath='{.status.succeeded}' 2>/dev/null | grep -q 1 - vars: - CAPABILITY_ROLE_ARN: - sh: kubectl get role.iam.services.k8s.aws/{{.HUB_CLUSTER_NAME}}-argocd-role -n {{.HUB_CLUSTER_NAME}} -o jsonpath='{.status.ackResourceMetadata.arn}' 2>/dev/null || echo "" - cmds: - - kubectl -n {{.ACK_NAMESPACE}} create configmap argocd-capability-config - --from-literal=CLUSTER_NAME={{.HUB_CLUSTER_NAME}} - --from-literal=AWS_REGION={{.AWS_REGION}} - --from-literal=AWS_ACCOUNT_ID={{.AWS_ACCOUNT_ID}} - --from-literal=CAPABILITY_NAME={{.CAPABILITY_NAME}} - --from-literal=CAPABILITY_ROLE_ARN={{.CAPABILITY_ROLE_ARN}} - --from-literal=IDC_INSTANCE_ARN={{.IDC_INSTANCE_ARN}} - --from-literal=IDC_REGION={{.IDC_REGION}} - --from-literal=IDC_ADMIN_GROUP_ID={{.IDC_ADMIN_GROUP_ID}} - --dry-run=client -o yaml | kubectl apply -f - - - kubectl delete job create-argocd-capability -n {{.ACK_NAMESPACE}} 2>/dev/null || true - - kubectl apply -f manifests/argocd/create-capability.yaml - - kubectl -n {{.ACK_NAMESPACE}} wait --for=condition=Complete --timeout=2400s job/create-argocd-capability - secrets-manager:seed: desc: Seed hub config into AWS Secrets Manager vars: From 17263cbe8092de3e4ad4f6ab1201a9b5a40afdb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 12:09:21 +0200 Subject: [PATCH 006/231] fix: restore Kargo ECR pod-identity setup in deploy-kargo.sh (accidentally removed) --- applications/java/kargo/deploy-kargo.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/applications/java/kargo/deploy-kargo.sh b/applications/java/kargo/deploy-kargo.sh index c150bb250..a0d4e8bb4 100755 --- a/applications/java/kargo/deploy-kargo.sh +++ b/applications/java/kargo/deploy-kargo.sh @@ -18,6 +18,31 @@ if [[ -z "$AWS_ACCOUNT_ID" || -z "$AWS_REGION" || -z "$GITLAB_URL" || -z "$GIT_U exit 1 fi +# Ensure the Kargo controller has ECR read access so the warehouse can discover images. +# Creates the IAM role + pod identity association (idempotent). +HUB_CLUSTER="${RESOURCE_PREFIX:-peeks}-hub" +KARGO_ROLE="${HUB_CLUSTER}-kargo-controller-role" + +echo "Setting up Kargo ECR access..." +if ! aws iam get-role --role-name "$KARGO_ROLE" --region "$AWS_REGION" >/dev/null 2>&1; then + aws iam create-role --role-name "$KARGO_ROLE" \ + --assume-role-policy-document '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"pods.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' \ + --description "Kargo controller ECR read access" --no-cli-pager >/dev/null + aws iam attach-role-policy --role-name "$KARGO_ROLE" \ + --policy-arn arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly --region "$AWS_REGION" + echo " ✓ IAM role $KARGO_ROLE created" +fi +KARGO_ROLE_ARN=$(aws iam get-role --role-name "$KARGO_ROLE" --query 'Role.Arn' --output text --region "$AWS_REGION") +if ! aws eks list-pod-identity-associations --cluster-name "$HUB_CLUSTER" --region "$AWS_REGION" \ + --query "associations[?namespace=='kargo'&&serviceAccount=='kargo-controller']" --output text 2>/dev/null | grep -q .; then + aws eks create-pod-identity-association \ + --cluster-name "$HUB_CLUSTER" --namespace kargo --service-account kargo-controller \ + --role-arn "$KARGO_ROLE_ARN" --region "$AWS_REGION" >/dev/null + echo " ✓ pod identity association created" + # Restart kargo controller to pick up the new credentials + kubectl rollout restart deployment -n kargo 2>/dev/null || true + kubectl rollout status deployment -n kargo --timeout=60s 2>/dev/null || true +fi # Deploy the project (creates namespace) echo "Creating Kargo project..." From 3cce2f8214f1069aa75845ce34f7b9e2a31b1881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 12:10:33 +0200 Subject: [PATCH 007/231] fix: restore create-ray-models-bucket from deleted Taskfile.workshop.yaml --- cluster-providers/workshop/Taskfile.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 97cb00ffc..c25d877de 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -50,6 +50,7 @@ tasks: pre-install: desc: Verify workshop pre-requisites (normally provisioned by CDK in guided events) cmds: + - task: create-ray-models-bucket - | printf '{{.C_INFO}}Workshop pre-requisites check:{{.C_RESET}}\n' printf ' In Workshop Studio (guided) events:\n' @@ -72,6 +73,21 @@ tasks: # Run after task install to complete workshop-specific setup # ============================================================ + create-ray-models-bucket: + desc: Create the Ray model staging S3 bucket for AI/ML modules (idempotent) + status: + - aws s3api head-bucket --bucket {{.HUB_CLUSTER_NAME}}-ray-models-{{.AWS_ACCOUNT_ID}} --region {{.AWS_REGION}} 2>/dev/null + cmds: + - printf '{{.C_STEP}}▸ Creating Ray model staging bucket...{{.C_RESET}}\n' + - | + BUCKET="{{.HUB_CLUSTER_NAME}}-ray-models-{{.AWS_ACCOUNT_ID}}" + aws s3api create-bucket --bucket "$BUCKET" --region {{.AWS_REGION}} \ + {{if ne .AWS_REGION "us-east-1"}}--create-bucket-configuration LocationConstraint={{.AWS_REGION}}{{end}} \ + --no-cli-pager >/dev/null + aws s3api put-public-access-block --bucket "$BUCKET" --region {{.AWS_REGION}} \ + --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" >/dev/null + - printf '{{.C_OK}}✓ s3://{{.HUB_CLUSTER_NAME}}-ray-models-{{.AWS_ACCOUNT_ID}} ready.{{.C_RESET}}\n' + post-install: desc: Workshop post-install steps (IDC/Keycloak federation for ArgoCD SSO) cmds: From 4a223b4917fcfc38308bd1958c6416f087a3d33a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 12:12:02 +0200 Subject: [PATCH 008/231] fix: restore accidentally removed code from cherry-pick MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Restore argocd login block in ssm-setup-ide-logs.sh - Restore progressive-app image_name/service name (rollout-demo → progressive-app) --- applications/java/progressive-app.yaml | 10 +++++----- hack/.bashrc.d/ssm-setup-ide-logs.sh | 10 ++++++++++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/applications/java/progressive-app.yaml b/applications/java/progressive-app.yaml index 0d1db8e12..214d8cc2b 100644 --- a/applications/java/progressive-app.yaml +++ b/applications/java/progressive-app.yaml @@ -5,7 +5,7 @@ metadata: namespace: team-java spec: image: argoproj/rollouts-demo:blue - image_name: rollout-demo + image_name: progressive-app replicas: 2 port: 8080 targetPort: 8080 @@ -25,11 +25,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: rollout-demo + name: progressive-app namespace: team-java spec: selector: - app: rollout-demo + app: progressive-app ports: - port: 8080 targetPort: 8080 @@ -38,11 +38,11 @@ spec: apiVersion: v1 kind: Service metadata: - name: rollout-demo-preview + name: progressive-app-preview namespace: team-java spec: selector: - app: rollout-demo + app: progressive-app ports: - port: 8080 targetPort: 8080 diff --git a/hack/.bashrc.d/ssm-setup-ide-logs.sh b/hack/.bashrc.d/ssm-setup-ide-logs.sh index 599c7f565..53aca6462 100644 --- a/hack/.bashrc.d/ssm-setup-ide-logs.sh +++ b/hack/.bashrc.d/ssm-setup-ide-logs.sh @@ -103,4 +103,14 @@ argocd-refresh-token() { fi echo "ArgoCD token refreshed. Server: $ARGOCD_SERVER" + + # Also update the argocd CLI config file so 'argocd app sync' etc. work + # without requiring env vars. The CLI prefers its config file over env vars + # when a server entry exists — leaving a stale token there causes + # "Unauthenticated: invalid session" even when ARGOCD_AUTH_TOKEN is correct. + argocd login "$ARGOCD_SERVER" \ + --auth-token "$token" \ + --grpc-web \ + --skip-test-tls \ + 2>/dev/null || true } From a3178d9df9d9df921243bb3de3158b403bf71233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 14:17:28 +0200 Subject: [PATCH 009/231] fix: add kind-kro-ack to root Taskfile includes --- Taskfile.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Taskfile.yaml b/Taskfile.yaml index 9acbd000f..aed7dc299 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -32,6 +32,12 @@ includes: CONFIG_FILE: "{{.CONFIG_FILE}}" ROOT_DIR: "{{.ROOT_DIR}}" optional: true + kind-kro-ack: + taskfile: ./cluster-providers/kind-kro-ack/Taskfile.yaml + dir: ./cluster-providers/kind-kro-ack + vars: + CONFIG_FILE: "{{.CONFIG_FILE}}" + optional: true tasks: default: From 9b0eaad021ddc1987aaf782a9f9af41a567bff60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 14:27:17 +0200 Subject: [PATCH 010/231] =?UTF-8?q?fix:=20increase=20RGD=20wait=20timeout?= =?UTF-8?q?=20180s=E2=86=92300s=20and=20wait=20individually=20for=20better?= =?UTF-8?q?=20diagnostics?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-kro-ack/Taskfile.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 6af16b5b9..d171fb7ab 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -264,7 +264,9 @@ tasks: - kubectl apply -f {{.GITOPS_ROOT}}/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml - kubectl apply -f {{.GITOPS_ROOT}}/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml - printf '{{.C_STEP}}▸ Waiting for RGDs to be ready...{{.C_RESET}}\n' - - kubectl wait --for=condition=Ready resourcegraphdefinition --all --timeout=180s + - kubectl wait --for=condition=Ready resourcegraphdefinition/vpc.kro.run --timeout=300s + - kubectl wait --for=condition=Ready resourcegraphdefinition/ekscluster.kro.run --timeout=300s + - kubectl wait --for=condition=Ready resourcegraphdefinition/eksclusterwithvpc.kro.run --timeout=300s - printf '{{.C_OK}}✓ ResourceGraphDefinitions applied.{{.C_RESET}}\n' idc:setup: From 7f9f22ad165edc6a0af68b915214afc18a92bd9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 14:29:07 +0200 Subject: [PATCH 011/231] fix: add kind:kubeconfig task to ensure kubeconfig is present on re-runs --- cluster-providers/kind-kro-ack/Taskfile.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index d171fb7ab..46303340c 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -99,6 +99,7 @@ tasks: sh: date +%s cmds: - task: kind:create + - task: kind:kubeconfig - task: credentials:setup - task: argocd:install - task: ack:install @@ -153,6 +154,12 @@ tasks: cmds: - kind create cluster --name {{.KIND_CLUSTER_NAME}} --config kind.yaml --kubeconfig {{.KUBECONFIG}} + kind:kubeconfig: + desc: Ensure kind cluster kubeconfig is present (idempotent) + cmds: + - mkdir -p {{.ROOT_DIR}}/private + - kind export kubeconfig --name {{.KIND_CLUSTER_NAME}} --kubeconfig {{.KUBECONFIG}} 2>/dev/null || true + credentials:setup: desc: Create AWS credentials secret (skips if exists) status: From 2504593e4a0a42d46beb203d044ba63f7247e4f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 14:40:48 +0200 Subject: [PATCH 012/231] fix: gitlab:init-ec2 rails runner fully idempotent on re-runs (user exists, project exists) --- .../kind-crossplane/Taskfile.yaml | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index e0389fd4f..426031c37 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -813,19 +813,26 @@ tasks: pat.scopes = ['api','write_repository','sudo'] pat.expires_at = 365.days.from_now pat.save! - u = User.find_by(username: '{{.GIT_USERNAME}}') || User.create!( - name: 'Workshop User', username: '{{.GIT_USERNAME}}', - email: '{{.GIT_USERNAME}}@workshop.local', - password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', - skip_confirmation: true - ) + u = User.find_by(username: '{{.GIT_USERNAME}}') + if u.nil? + u = User.create!( + name: 'Workshop User', username: '{{.GIT_USERNAME}}', + email: '{{.GIT_USERNAME}}@workshop.local', + password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', + skip_confirmation: true + ) + else + u.update!(password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}') + end u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') u1pat.scopes = ['api','write_repository'] u1pat.expires_at = 365.days.from_now u1pat.save! %w[fleet-config java rust golang dotnet next-js].each do |repo| - p = Project.find_by(path: repo, namespace_id: u.namespace_id) || - Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute + p = Project.find_by(path: repo, namespace_id: u.namespace_id) + if p.nil? + p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute + end ProtectedBranch.where(project: p).destroy_all puts 'repo: ' + repo end From 07711e811127d572f4c498e4f0b109b1828cc538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 14:53:00 +0200 Subject: [PATCH 013/231] =?UTF-8?q?fix:=20gitlab:init-ec2=20use=20save(val?= =?UTF-8?q?idate:false)=20=E2=80=94=20update!=20fails=20on=20complex=20has?= =?UTF-8?q?h/validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kind-crossplane/Taskfile.yaml | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 426031c37..5fb1c9680 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -800,29 +800,26 @@ tasks: # before the string is passed to gitlab-rails runner as a Ruby script. docker exec gitlab-gitlab-1 gitlab-rails runner " # Disable default branch protection so user1 can push to main - ApplicationSetting.first.update!( - default_branch_protection: 0, - default_branch_protection_defaults: { - allowed_to_push: [{ access_level: 0 }], - allow_force_push: true, - allowed_to_merge: [{ access_level: 0 }], - developer_can_initial_push: true - } - ) + s = ApplicationSetting.first + s.password_authentication_enabled_for_git = true + s.default_branch_protection = 0 + s.save(validate: false) pat = User.find(1).personal_access_tokens.find_or_initialize_by(name: 'initial-root-token') pat.scopes = ['api','write_repository','sudo'] pat.expires_at = 365.days.from_now pat.save! u = User.find_by(username: '{{.GIT_USERNAME}}') if u.nil? - u = User.create!( - name: 'Workshop User', username: '{{.GIT_USERNAME}}', + u = User.new(name: 'Workshop User', username: '{{.GIT_USERNAME}}', email: '{{.GIT_USERNAME}}@workshop.local', password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', - skip_confirmation: true - ) + skip_confirmation: true) + u.save(validate: false) else - u.update!(password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}') + u.password = '{{.GIT_PASSWORD}}' + u.password_confirmation = '{{.GIT_PASSWORD}}' + u.password_automatically_set = false + u.save(validate: false) end u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') u1pat.scopes = ['api','write_repository'] From fbc1edbf228278692441fc650d7b0f25b222f689 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 15:36:56 +0200 Subject: [PATCH 014/231] fix: ensure user namespace exists before creating GitLab projects --- cluster-providers/kind-crossplane/Taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 5fb1c9680..6e35e9335 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -825,6 +825,9 @@ tasks: u1pat.scopes = ['api','write_repository'] u1pat.expires_at = 365.days.from_now u1pat.save! + # Ensure user namespace exists (created lazily by GitLab) + u.ensure_namespace! + u.reload %w[fleet-config java rust golang dotnet next-js].each do |repo| p = Project.find_by(path: repo, namespace_id: u.namespace_id) if p.nil? From 20f50ffd0aa8bb09c4083bd9762d2ac65d056dd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 15:43:52 +0200 Subject: [PATCH 015/231] fix(kind-kro-ack): add ACK SecretsManager + ESO bootstrap to install RGDs ekscluster.kro.run and eksclusterwithvpc.kro.run were Inactive because: 1. ACK SecretsManager controller was missing (rg-eks.yaml references secretsmanager.services.k8s.aws/v1alpha1) 2. ESO was not installed on the bootstrap kind cluster (rg-eks.yaml references external-secrets.io/v1) Fixes: - Add ACK_SECRETSMANAGER_VERSION=1.3.1 and install it in ack:install - Add kro:install-eso-bootstrap task (idempotent) before kro:apply-rgds --- cluster-providers/kind-kro-ack/Taskfile.yaml | 22 +++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 46303340c..b6131eda4 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -67,6 +67,7 @@ vars: ACK_IAM_VERSION: "1.6.4" ACK_EKS_VERSION: "1.13.0" ACK_EC2_VERSION: "1.12.0" + ACK_SECRETSMANAGER_VERSION: "1.3.1" KRO_VERSION: "0.9.2" ARGOCD_VERSION: sh: yq '.argocd.defaultVersion' {{.ROOT_DIR}}/gitops/addons/registry/core.yaml @@ -104,6 +105,7 @@ tasks: - task: argocd:install - task: ack:install - task: kro:install + - task: kro:install-eso-bootstrap - task: kro:apply-rgds - task: hub:claim - task: hub:wait-for-eks @@ -228,7 +230,7 @@ tasks: --wait --timeout 5m ack:install: - desc: Install ACK controllers (IAM, EKS, EC2) + desc: Install ACK controllers (IAM, EKS, EC2, SecretsManager) cmds: - task: ack:install-controller vars: { CONTROLLER: iam, VERSION: "{{.ACK_IAM_VERSION}}" } @@ -236,6 +238,8 @@ tasks: vars: { CONTROLLER: eks, VERSION: "{{.ACK_EKS_VERSION}}" } - task: ack:install-controller vars: { CONTROLLER: ec2, VERSION: "{{.ACK_EC2_VERSION}}" } + - task: ack:install-controller + vars: { CONTROLLER: secretsmanager, VERSION: "{{.ACK_SECRETSMANAGER_VERSION}}" } - printf '{{.C_OK}}✓ ACK controllers installed.{{.C_RESET}}\n' ack:install-controller: @@ -264,6 +268,22 @@ tasks: --wait --timeout 5m - printf '{{.C_OK}}✓ KRO installed.{{.C_RESET}}\n' + kro:install-eso-bootstrap: + desc: Install External Secrets Operator on the bootstrap kind cluster (required for RGD schema validation) + vars: + ESO_VERSION: + sh: yq '.external-secrets.defaultVersion' {{.GITOPS_ROOT}}/addons/registry/core.yaml + status: + - helm status external-secrets -n external-secrets 2>/dev/null | grep -q deployed + cmds: + - helm repo add external-secrets https://charts.external-secrets.io --force-update + - helm upgrade --install external-secrets external-secrets/external-secrets + --namespace external-secrets --create-namespace + --version {{.ESO_VERSION}} + --set serviceAccount.name=external-secrets-sa + --wait --timeout 5m + - printf '{{.C_OK}}✓ ESO installed on bootstrap cluster.{{.C_RESET}}\n' + kro:apply-rgds: desc: Apply KRO ResourceGraphDefinitions for VPC and EKS cmds: From 2c9f119d0664eaeaa00bd5c03378aaa1468f81a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 15:45:30 +0200 Subject: [PATCH 016/231] fix(kind-kro-ack): fix hub:claim - remove unknown argocdCapability, add argoCdCapabilityRoleArn - Remove spec.argocdCapability block (not in EksclusterWithVpc RGD schema, causes strict decode error) - Add missing argoCdCapabilityRoleArn field (required by RGD) - Remove now-unused IDC sed substitution lines --- cluster-providers/kind-kro-ack/Taskfile.yaml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index b6131eda4..5b9177391 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -396,9 +396,6 @@ tasks: -e "s|HUB_VPC_CIDR|{{.HUB_VPC_CIDR}}|g" \ -e "s|RESOURCE_PREFIX|{{.RESOURCE_PREFIX}}|g" \ -e "s|CALLER_ROLE_ARN|{{.CALLER_ROLE_ARN}}|g" \ - -e "s|IDC_INSTANCE_ARN|{{.IDC_INSTANCE_ARN}}|g" \ - -e "s|IDC_REGION|{{.IDC_REGION}}|g" \ - -e "s|IDC_ADMIN_GROUP_ID|{{.IDC_ADMIN_GROUP_ID}}|g" \ -e "s|ADMIN_ROLE_NAME|{{.ADMIN_ROLE_NAME}}|g" \ -e "s|REPO_URL|{{.REPO_URL}}|g" \ -e "s|REPO_REVISION|{{.REPO_REVISION}}|g" \ @@ -419,15 +416,11 @@ tasks: accountId: "AWS_ACCOUNT_ID" managementAccountId: "AWS_ACCOUNT_ID" argoCdHubRoleArn: "CALLER_ROLE_ARN" + argoCdCapabilityRoleArn: "CALLER_ROLE_ARN" adminRoleName: ADMIN_ROLE_NAME fleetSecretManagerSecretNameSuffix: argocd-secret domainName: DOMAIN resourcePrefix: RESOURCE_PREFIX - argocdCapability: - enabled: "true" - idcInstanceArn: IDC_INSTANCE_ARN - idcRegion: IDC_REGION - adminGroupId: IDC_ADMIN_GROUP_ID cidr: vpcCidr: HUB_VPC_CIDR publicSubnet1Cidr: "10.0.1.0/24" From a32e677f426cc67fa2bbbe12deec60e1920f358b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 16:03:42 +0200 Subject: [PATCH 017/231] fix: gitlab:init-ec2 use Users::CreateService + Devise::Encryptor for proper user+namespace creation --- .../kind-crossplane/Taskfile.yaml | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 6e35e9335..5dc270c39 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -799,7 +799,7 @@ tasks: # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash # before the string is passed to gitlab-rails runner as a Ruby script. docker exec gitlab-gitlab-1 gitlab-rails runner " - # Disable default branch protection so user1 can push to main + # Disable default branch protection + enable password auth for git s = ApplicationSetting.first s.password_authentication_enabled_for_git = true s.default_branch_protection = 0 @@ -810,31 +810,29 @@ tasks: pat.save! u = User.find_by(username: '{{.GIT_USERNAME}}') if u.nil? - u = User.new(name: 'Workshop User', username: '{{.GIT_USERNAME}}', + # Use Users::CreateService which properly sets up the namespace + svc = Users::CreateService.new(User.find(1), { + name: 'Workshop User', username: '{{.GIT_USERNAME}}', email: '{{.GIT_USERNAME}}@workshop.local', password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', - skip_confirmation: true) - u.save(validate: false) + skip_confirmation: true + }).execute + u = User.find_by(username: '{{.GIT_USERNAME}}') else - u.password = '{{.GIT_PASSWORD}}' - u.password_confirmation = '{{.GIT_PASSWORD}}' - u.password_automatically_set = false - u.save(validate: false) + u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') + u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') u1pat.scopes = ['api','write_repository'] u1pat.expires_at = 365.days.from_now u1pat.save! - # Ensure user namespace exists (created lazily by GitLab) - u.ensure_namespace! - u.reload %w[fleet-config java rust golang dotnet next-js].each do |repo| p = Project.find_by(path: repo, namespace_id: u.namespace_id) if p.nil? p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute end - ProtectedBranch.where(project: p).destroy_all - puts 'repo: ' + repo + ProtectedBranch.where(project: p).destroy_all if p.persisted? + puts 'repo: ' + repo + ' ok=' + p.persisted?.to_s end puts 'init done' " 2>&1 | tail -20 || true From c524d7cf209503a26ebd0c0896ec9dddd1b0271e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 16:06:18 +0200 Subject: [PATCH 018/231] fix(kind-kro-ack): restore argocdCapability as RGD-managed EKS Capability Port from feature/platform-cluster-kro-ack: - rg-eks.yaml: add argocdCapability schema object, replace accessEntryArgoCdCapability with conditional argocdCapabilityRole + argocdCapability (EKS Capability with IDC) + argocdCapabilityAccessEntry (all guarded by includeWhen enabled==true) - rg-eks-vpc.yaml: add argocdCapability schema object, pass through to nested EksCluster - hub:claim: restore argocdCapability block with IDC sed substitutions --- cluster-providers/kind-kro-ack/Taskfile.yaml | 9 +- .../manifests/eks/rg-eks-vpc.yaml | 12 ++- .../resource-groups/manifests/eks/rg-eks.yaml | 99 ++++++++++++++++++- 3 files changed, 112 insertions(+), 8 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 5b9177391..b6131eda4 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -396,6 +396,9 @@ tasks: -e "s|HUB_VPC_CIDR|{{.HUB_VPC_CIDR}}|g" \ -e "s|RESOURCE_PREFIX|{{.RESOURCE_PREFIX}}|g" \ -e "s|CALLER_ROLE_ARN|{{.CALLER_ROLE_ARN}}|g" \ + -e "s|IDC_INSTANCE_ARN|{{.IDC_INSTANCE_ARN}}|g" \ + -e "s|IDC_REGION|{{.IDC_REGION}}|g" \ + -e "s|IDC_ADMIN_GROUP_ID|{{.IDC_ADMIN_GROUP_ID}}|g" \ -e "s|ADMIN_ROLE_NAME|{{.ADMIN_ROLE_NAME}}|g" \ -e "s|REPO_URL|{{.REPO_URL}}|g" \ -e "s|REPO_REVISION|{{.REPO_REVISION}}|g" \ @@ -416,11 +419,15 @@ tasks: accountId: "AWS_ACCOUNT_ID" managementAccountId: "AWS_ACCOUNT_ID" argoCdHubRoleArn: "CALLER_ROLE_ARN" - argoCdCapabilityRoleArn: "CALLER_ROLE_ARN" adminRoleName: ADMIN_ROLE_NAME fleetSecretManagerSecretNameSuffix: argocd-secret domainName: DOMAIN resourcePrefix: RESOURCE_PREFIX + argocdCapability: + enabled: "true" + idcInstanceArn: IDC_INSTANCE_ARN + idcRegion: IDC_REGION + adminGroupId: IDC_ADMIN_GROUP_ID cidr: vpcCidr: HUB_VPC_CIDR publicSubnet1Cidr: "10.0.1.0/24" diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml index bea67a464..168648e52 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml @@ -18,7 +18,6 @@ spec: accountId: string managementAccountId: string argoCdHubRoleArn: string - argoCdCapabilityRoleArn: string adminRoleName: string | default="Admin" # Optional second cluster-admin role (e.g. the IDE/instance *SharedRole*), # passed through to the nested EksCluster. See rg-eks.yaml for rationale. @@ -27,6 +26,11 @@ spec: domainName: string | default="cluster.example.com" resourcePrefix: string | default="peeks" git_username: string | default="user1" + argocdCapability: + enabled: string | default="false" + idcInstanceArn: string | default="" + idcRegion: string | default="" + adminGroupId: string | default="" cidr: vpcCidr: string | default="10.0.0.0/16" publicSubnet1Cidr: string | default="10.0.1.0/24" @@ -167,7 +171,6 @@ spec: accountId: ${schema.spec.accountId} managementAccountId: ${schema.spec.managementAccountId} argoCdHubRoleArn: ${schema.spec.argoCdHubRoleArn} - argoCdCapabilityRoleArn: ${schema.spec.argoCdCapabilityRoleArn} k8sVersion: ${schema.spec.k8sVersion} adminRoleName: ${schema.spec.adminRoleName} adminInstanceRoleName: ${schema.spec.adminInstanceRoleName} @@ -175,6 +178,11 @@ spec: domainName: ${schema.spec.domainName} resourcePrefix: ${schema.spec.resourcePrefix} git_username: ${schema.spec.git_username} + argocdCapability: + enabled: ${schema.spec.argocdCapability.enabled} + idcInstanceArn: ${schema.spec.argocdCapability.idcInstanceArn} + idcRegion: ${schema.spec.argocdCapability.idcRegion} + adminGroupId: ${schema.spec.argocdCapability.adminGroupId} network: vpcID: ${vpc.status.vpcID} subnets: diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index bb24ad120..dc5ce21fc 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -20,7 +20,11 @@ spec: managementAccountId: string managementRegion: string | default="us-west-2" argoCdHubRoleArn: string - argoCdCapabilityRoleArn: string + argocdCapability: + enabled: string | default="false" + idcInstanceArn: string | default="" + idcRegion: string | default="" + adminGroupId: string | default="" k8sVersion: string adminRoleName: string # Optional second cluster-admin role. kro SimpleSchema cannot express a @@ -750,13 +754,98 @@ spec: principalARN: "arn:aws:iam::${schema.spec.accountId}:role/${schema.spec.adminInstanceRoleName}" type: STANDARD - - id: accessEntryArgoCdCapability + ########################################################### + # ArgoCD Capability (optional, for hub clusters) + ########################################################### + - id: argocdCapabilityRole + includeWhen: + - ${schema.spec.argocdCapability.enabled == "true"} + template: + apiVersion: iam.services.k8s.aws/v1alpha1 + kind: Role + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-argocd-capability-role" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + name: "${schema.spec.name}-argocd-capability-role" + policies: + - arn:aws:iam::aws:policy/AmazonEKSClusterPolicy + assumeRolePolicyDocument: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "capabilities.eks.amazonaws.com" + }, + "Action": [ + "sts:AssumeRole", + "sts:TagSession" + ] + } + ] + } + + - id: argocdCapability + includeWhen: + - ${schema.spec.argocdCapability.enabled == "true"} + readyWhen: + - ${argocdCapability.metadata.name != ""} + template: + apiVersion: eks.services.k8s.aws/v1alpha1 + kind: Capability + metadata: + name: ${schema.spec.name}-argocd + namespace: ${schema.spec.name} + labels: + app.kubernetes.io/instance: ${schema.spec.name} + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: EksCluster + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + argocd.argoproj.io/sync-options: Prune=false + spec: + clusterName: ${schema.spec.name} + name: argocd + type: ARGOCD + roleARN: ${argocdCapabilityRole.status.ackResourceMetadata.arn} + deletePropagationPolicy: RETAIN + configuration: + argoCD: + awsIDC: + idcInstanceARN: ${schema.spec.argocdCapability.idcInstanceArn} + idcRegion: ${schema.spec.argocdCapability.idcRegion} + rbacRoleMappings: + - role: ADMIN + identities: + - id: ${schema.spec.argocdCapability.adminGroupId} + type: SSO_GROUP + + - id: argocdCapabilityAccessEntry + includeWhen: + - ${schema.spec.argocdCapability.enabled == "true"} template: apiVersion: eks.services.k8s.aws/v1alpha1 kind: AccessEntry metadata: namespace: "${schema.spec.name}" - name: "${schema.spec.name}-access-entry-argocd-capability" + name: "${schema.spec.name}-argocd-capability-access" ownerReferences: - apiVersion: kro.run/v1alpha1 kind: ${schema.kind} @@ -768,12 +857,12 @@ spec: argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} services.k8s.aws/region: ${schema.spec.region} spec: - clusterName: "${ekscluster.spec.name}" + clusterName: "${schema.spec.name}" accessPolicies: - accessScope: type: "cluster" policyARN: "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" - principalARN: "${schema.spec.argoCdCapabilityRoleArn}" + principalARN: "${argocdCapabilityRole.status.ackResourceMetadata.arn}" type: STANDARD From 2baa6d25cd728501873fb0a662ebf5b9de1283ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 16:19:43 +0200 Subject: [PATCH 019/231] fix: gitlab:init-ec2 definitive - namespace via organization_id, unprotect after creation --- cluster-providers/kind-crossplane/Taskfile.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 5dc270c39..572dbd5ca 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -822,6 +822,14 @@ tasks: u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end + # Ensure namespace exists (GitLab CE requires Organization; use default org) + if u.namespace_id.nil? + org_id = Organizations::Organization.find_by(id: 1)&.id || 1 + ns = Namespace.new(owner: u, path: u.username, name: u.username, type: 'User', organization_id: org_id) + ns.save(validate: false) + u.update_columns(namespace_id: ns.id) if ns.persisted? + u.reload + end u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') u1pat.scopes = ['api','write_repository'] u1pat.expires_at = 365.days.from_now From 82ce00a3aa9b9a2e40d249ee392a2c495a69fe1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 16:22:55 +0200 Subject: [PATCH 020/231] =?UTF-8?q?fix:=20restore=20argoCdCapabilityRoleAr?= =?UTF-8?q?n=20as=20optional=20field=20=E2=80=94=20KRO=20rejects=20CRD=20w?= =?UTF-8?q?ith=20removed=20property?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml | 1 + .../addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml index 168648e52..33d5d5f7b 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml @@ -18,6 +18,7 @@ spec: accountId: string managementAccountId: string argoCdHubRoleArn: string + argoCdCapabilityRoleArn: string | default="" adminRoleName: string | default="Admin" # Optional second cluster-admin role (e.g. the IDE/instance *SharedRole*), # passed through to the nested EksCluster. See rg-eks.yaml for rationale. diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index dc5ce21fc..237c8c119 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -20,6 +20,7 @@ spec: managementAccountId: string managementRegion: string | default="us-west-2" argoCdHubRoleArn: string + argoCdCapabilityRoleArn: string | default="" argocdCapability: enabled: string | default="false" idcInstanceArn: string | default="" From 8f004b11270e672eb2f49ba66142b868759bfb24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 16:32:46 +0200 Subject: [PATCH 021/231] feat(kind-kro-ack): add kro:reset-crds task for breaking schema changes Deletes eksclusters.kro.run and eksclusterwithvpcs.kro.run CRDs on the bootstrap kind cluster. Needed when RGD schema changes remove fields (KRO breaking-change protection blocks the update otherwise). Run: task kind-kro-ack:kro:reset-crds, then task install. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index b6131eda4..f03555d8e 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -296,6 +296,12 @@ tasks: - kubectl wait --for=condition=Ready resourcegraphdefinition/eksclusterwithvpc.kro.run --timeout=300s - printf '{{.C_OK}}✓ ResourceGraphDefinitions applied.{{.C_RESET}}\n' + kro:reset-crds: + desc: Delete KRO-managed CRDs on the bootstrap kind cluster to allow breaking schema changes (run before kro:apply-rgds) + cmds: + - kubectl delete crd eksclusters.kro.run eksclusterwithvpcs.kro.run --ignore-not-found + - printf '{{.C_OK}}✓ KRO CRDs deleted — re-run task install.{{.C_RESET}}\n' + idc:setup: desc: Create IDC groups if they don't exist (idempotent) vars: From 4b74a8ec6a0b9a2dfde3e6be5e271758a8c658f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 16:40:32 +0200 Subject: [PATCH 022/231] fix(kind-kro-ack): derive subnet CIDRs from HUB_VPC_CIDR prefix Hardcoded 10.0.x.0/24 subnets break when vpcCidr is not 10.0.0.0/16. Add HUB_VPC_PREFIX var (first two octets of HUB_VPC_CIDR) and use it in hub:claim subnet CIDRs. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index f03555d8e..1b661ffa2 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -15,6 +15,8 @@ vars: sh: yq '.hub.kubernetesVersion // "1.35"' {{.CONFIG_FILE}} HUB_VPC_CIDR: sh: yq '.hub.vpcCidr // "10.0.0.0/16"' {{.CONFIG_FILE}} + HUB_VPC_PREFIX: + sh: yq '.hub.vpcCidr // "10.0.0.0/16"' {{.CONFIG_FILE}} | cut -d. -f1-2 AWS_ACCOUNT_ID: sh: yq '.aws.accountId' {{.CONFIG_FILE}} AWS_REGION: @@ -400,6 +402,7 @@ tasks: -e "s|AWS_ACCOUNT_ID|{{.AWS_ACCOUNT_ID}}|g" \ -e "s|HUB_K8S_VERSION|{{.HUB_K8S_VERSION}}|g" \ -e "s|HUB_VPC_CIDR|{{.HUB_VPC_CIDR}}|g" \ + -e "s|HUB_VPC_PREFIX|{{.HUB_VPC_PREFIX}}|g" \ -e "s|RESOURCE_PREFIX|{{.RESOURCE_PREFIX}}|g" \ -e "s|CALLER_ROLE_ARN|{{.CALLER_ROLE_ARN}}|g" \ -e "s|IDC_INSTANCE_ARN|{{.IDC_INSTANCE_ARN}}|g" \ @@ -436,10 +439,10 @@ tasks: adminGroupId: IDC_ADMIN_GROUP_ID cidr: vpcCidr: HUB_VPC_CIDR - publicSubnet1Cidr: "10.0.1.0/24" - publicSubnet2Cidr: "10.0.2.0/24" - privateSubnet1Cidr: "10.0.11.0/24" - privateSubnet2Cidr: "10.0.12.0/24" + publicSubnet1Cidr: "HUB_VPC_PREFIX.1.0/24" + publicSubnet2Cidr: "HUB_VPC_PREFIX.2.0/24" + privateSubnet1Cidr: "HUB_VPC_PREFIX.11.0/24" + privateSubnet2Cidr: "HUB_VPC_PREFIX.12.0/24" gitops: addonsRepoUrl: REPO_URL addonsRepoRevision: REPO_REVISION From 7ebced7ecd36b7e2b944d9df93ab637b4670a90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 17:44:49 +0200 Subject: [PATCH 023/231] fix(kind-kro-ack): skip hub:install-argocd when ArgoCD CRDs already exist When the EKS ArgoCD Capability is used, ArgoCD CRDs are already present on the hub cluster. Helm install then fails with 'CRD already exists'. Add a second status check: skip if applications.argoproj.io CRD exists. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 1b661ffa2..0317ab16a 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -850,6 +850,7 @@ tasks: desc: Install ArgoCD on the hub cluster (OSS mode, skipped when EKS Capability is used) status: - test "$(yq '.argocdCapability.enabled // "false"' {{.CONFIG_FILE}} 2>/dev/null || echo false)" = "true" + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get crd applications.argoproj.io 2>/dev/null | grep -q applications cmds: - task: hub:kubeconfig - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig helm repo add argo https://argoproj.github.io/argo-helm --force-update From fb883c9a756ce26266fe7d6e8055297b72ae0e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 17:45:20 +0200 Subject: [PATCH 024/231] fix: remove force push in seed loop - repos are fresh, no need for -f --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 572dbd5ca..dec77b51d 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -858,7 +858,7 @@ tasks: cd "$TMP" git init -q && git add -A && git commit -qm "init" 2>/dev/null || true git remote add origin "$REMOTE_URL" - git push -f origin HEAD:main + git push origin HEAD:main cd - >/dev/null; rm -rf "$TMP" echo " seeded: $REPO" done From 5efd66d1bd1f0672b5ac2626f21b01e0807254b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 17:48:53 +0200 Subject: [PATCH 025/231] fix(kind-kro-ack): fix list-vpc-origins JMESPath query (VpcOriginList.Items) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 0317ab16a..dca8f5b3c 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -687,7 +687,7 @@ tasks: - | # 1. Create or reuse VPC Origin pointing at the internal ALB VPC_ORIGIN_ID=$(aws cloudfront list-vpc-origins --region {{.AWS_REGION}} \ - --query "Items[?Name=='{{.VPC_ORIGIN_NAME}}'].Id" --output text 2>/dev/null || echo "") + --query "VpcOriginList.Items[?Name=='{{.VPC_ORIGIN_NAME}}'].Id" --output text 2>/dev/null || echo "") if [ -z "$VPC_ORIGIN_ID" ] || [ "$VPC_ORIGIN_ID" = "None" ]; then VPC_ORIGIN_ID=$(aws cloudfront create-vpc-origin --region {{.AWS_REGION}} \ --vpc-origin-endpoint-config "{ From d68ca87ff38841c0aa7e25aa0c37cc1f961e4f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 17:51:46 +0200 Subject: [PATCH 026/231] fix(kind-kro-ack): fix hub:install-argocd status check (OR not AND) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index dca8f5b3c..d9d946404 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -849,8 +849,8 @@ tasks: hub:install-argocd: desc: Install ArgoCD on the hub cluster (OSS mode, skipped when EKS Capability is used) status: - - test "$(yq '.argocdCapability.enabled // "false"' {{.CONFIG_FILE}} 2>/dev/null || echo false)" = "true" - - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get crd applications.argoproj.io 2>/dev/null | grep -q applications + # Skip if argocdCapability is enabled in config OR if ArgoCD CRDs already exist on the hub + - test "$(yq '.argocdCapability.enabled // "false"' {{.CONFIG_FILE}} 2>/dev/null || echo false)" = "true" || KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get crd applications.argoproj.io 2>/dev/null | grep -q applications cmds: - task: hub:kubeconfig - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig helm repo add argo https://argoproj.github.io/argo-helm --force-update From cff4f7e1fe4781885f781ba607d8c18f3c46f495 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:07:34 +0200 Subject: [PATCH 027/231] fix: seed repos by cloning first (preserves README history from initialize_with_readme) --- cluster-providers/kind-crossplane/Taskfile.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index dec77b51d..9d7290436 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -854,10 +854,11 @@ tasks: fi [ -d "$SRC" ] || continue TMP=$(mktemp -d) + # Clone existing repo (has README from initialize_with_readme) then add source files + git clone -q "$REMOTE_URL" "$TMP" 2>/dev/null || { cp -r "$SRC/." "$TMP/"; git -C "$TMP" init -q; git -C "$TMP" remote add origin "$REMOTE_URL"; } cp -r "$SRC/." "$TMP/" cd "$TMP" - git init -q && git add -A && git commit -qm "init" 2>/dev/null || true - git remote add origin "$REMOTE_URL" + git add -A && git commit -qm "seed: add application source" 2>/dev/null || true git push origin HEAD:main cd - >/dev/null; rm -rf "$TMP" echo " seeded: $REPO" From e64a27a657cc24132c30aff833ffbde67558c18c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:08:50 +0200 Subject: [PATCH 028/231] fix(kind-kro-ack): seed peeks-hub/secrets early in hub:seed hubSecretsExternal (ESO) and clusterSecretsSecret (ACK SM) have a circular dependency: ACK SM needs the K8s secret that ESO creates from the AWS SM secret that ACK SM is trying to create. The seed breaks the loop by pre-creating peeks-hub/secrets in AWS SM before ESO/ACK start reconciling. Move secrets-manager:seed-secrets into hub:seed (before hub:install-eso). --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index d9d946404..c4b19c8cf 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -797,6 +797,7 @@ tasks: aws eks associate-access-policy --cluster-name {{.HUB_CLUSTER_NAME}} --principal-arn "$PRINCIPAL_ARN" --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy --access-scope type=cluster --region {{.AWS_REGION}} 2>/dev/null || true - task: secrets-manager:seed - task: secrets-manager:seed-keycloak + - task: secrets-manager:seed-secrets - task: hub:install-eso - task: hub:cluster-secret-store - task: hub:install-argocd From 793a703e3dac8d557a6d9f491dc503b633249a5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:13:04 +0200 Subject: [PATCH 029/231] fix(kind-kro-ack): add ClusterSecretStore + eso-aws-credentials on bootstrap cluster MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hubSecretsExternal (ESO ExternalSecret in RGD) runs on the bootstrap kind cluster but needs an aws-secrets-manager ClusterSecretStore. Add: - kro:install-cluster-secret-store-bootstrap task (runs before kro:apply-rgds) - Extracts individual keys from ACK credentials file → eso-aws-credentials secret - Creates ClusterSecretStore with explicit secretRef auth (no Pod Identity on kind) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 46 ++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index c4b19c8cf..a999ee9ff 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -108,6 +108,7 @@ tasks: - task: ack:install - task: kro:install - task: kro:install-eso-bootstrap + - task: kro:install-cluster-secret-store-bootstrap - task: kro:apply-rgds - task: hub:claim - task: hub:wait-for-eks @@ -286,6 +287,51 @@ tasks: --wait --timeout 5m - printf '{{.C_OK}}✓ ESO installed on bootstrap cluster.{{.C_RESET}}\n' + kro:install-cluster-secret-store-bootstrap: + desc: Apply aws-secrets-manager ClusterSecretStore on the bootstrap kind cluster (required for RGD ESO resources) + status: + - kubectl get clustersecretstore aws-secrets-manager 2>/dev/null | grep -q aws-secrets-manager + cmds: + - | + # Extract individual credential keys from the ACK credentials file and + # create a separate secret for ESO (ESO needs individual keys, not INI format) + CREDS=$(kubectl get secret aws-credentials -n ack-system -o jsonpath='{.data.credentials}' | base64 -d) + ACCESS_KEY=$(echo "$CREDS" | grep aws_access_key_id | awk '{print $3}') + SECRET_KEY=$(echo "$CREDS" | grep aws_secret_access_key | awk '{print $3}') + SESSION_TOKEN=$(echo "$CREDS" | grep aws_session_token | awk '{print $3}') + kubectl create secret generic eso-aws-credentials -n ack-system \ + --from-literal=access-key-id="$ACCESS_KEY" \ + --from-literal=secret-access-key="$SECRET_KEY" \ + --from-literal=session-token="$SESSION_TOKEN" \ + --dry-run=client -o yaml | kubectl apply -f - + - | + cat <<'EOF' | sed "s|AWS_REGION|{{.AWS_REGION}}|g" | kubectl apply -f - + apiVersion: external-secrets.io/v1 + kind: ClusterSecretStore + metadata: + name: aws-secrets-manager + spec: + provider: + aws: + service: SecretsManager + region: AWS_REGION + auth: + secretRef: + accessKeyIDSecretRef: + name: eso-aws-credentials + namespace: ack-system + key: access-key-id + secretAccessKeySecretRef: + name: eso-aws-credentials + namespace: ack-system + key: secret-access-key + sessionTokenSecretRef: + name: eso-aws-credentials + namespace: ack-system + key: session-token + EOF + - printf '{{.C_OK}}✓ ClusterSecretStore applied on bootstrap cluster.{{.C_RESET}}\n' + kro:apply-rgds: desc: Apply KRO ResourceGraphDefinitions for VPC and EKS cmds: From e5d19656ca055c8af6d1401b3fbd5a34bc5b7dd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:15:56 +0200 Subject: [PATCH 030/231] fix(kind-kro-ack): remove clusterSecretsSecret from RGD, seed-secrets handles it clusterSecretsSecret (ACK SM) created peeks-hub/secrets but: 1. secrets-manager:seed-secrets already creates it early (breaking the circular dep) 2. ACK fails with 'resource already exists' when secrets-manager:seed pre-created it 3. The resource is redundant - ESO reads from AWS SM, no need to write back Remove clusterSecretsSecret from rg-eks.yaml. Ownership: seed-secrets task. --- .../resource-groups/manifests/eks/rg-eks.yaml | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index 237c8c119..508d80c10 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -639,31 +639,6 @@ spec: remoteRef: key: ${schema.spec.resourcePrefix}-hub/secrets - - id: clusterSecretsSecret - readyWhen: - - ${clusterSecretsSecret.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} - template: - apiVersion: secretsmanager.services.k8s.aws/v1alpha1 - kind: Secret - metadata: - name: ${schema.spec.name}-secrets - namespace: ${schema.spec.name} - annotations: - services.k8s.aws/region: ${schema.spec.managementRegion} - services.k8s.aws/owner-account-id: "${schema.spec.accountId}" - ownerReferences: - - apiVersion: kro.run/v1alpha1 - kind: ${schema.kind} - name: ${schema.metadata.name} - uid: ${schema.metadata.uid} - blockOwnerDeletion: true - spec: - name: ${schema.spec.name}/secrets - secretString: - name: ${schema.spec.name}-hub-secrets - namespace: ${schema.spec.name} - key: secrets - - id: accessEntry template: apiVersion: eks.services.k8s.aws/v1alpha1 From 43c70386ceb916694131c3f50ac34f205a2f1d05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:27:48 +0200 Subject: [PATCH 031/231] fix: unprotect branch immediately before each seed push (GitLab re-applies protection) --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 9d7290436..987cd4201 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -854,6 +854,8 @@ tasks: fi [ -d "$SRC" ] || continue TMP=$(mktemp -d) + # Unprotect main branch right before pushing (protection may be re-applied by GitLab) + docker exec gitlab-gitlab-1 gitlab-rails runner "p=Project.find_by(path:'$REPO',namespace_id:User.find_by(username:'{{.GIT_USERNAME}}').namespace_id); ProtectedBranch.where(project:p).destroy_all if p" 2>/dev/null || true # Clone existing repo (has README from initialize_with_readme) then add source files git clone -q "$REMOTE_URL" "$TMP" 2>/dev/null || { cp -r "$SRC/." "$TMP/"; git -C "$TMP" init -q; git -C "$TMP" remote add origin "$REMOTE_URL"; } cp -r "$SRC/." "$TMP/" From a0174bef59e0c35711a0448f4d70bf40fdba4d24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:33:11 +0200 Subject: [PATCH 032/231] refactor: move docker exec gitlab-rails unprotect to workshop/Taskfile.yaml, sync runners --- .../kind-crossplane/Taskfile.yaml | 2 - cluster-providers/workshop/Taskfile.yaml | 51 +++++++++++-------- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 987cd4201..9d7290436 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -854,8 +854,6 @@ tasks: fi [ -d "$SRC" ] || continue TMP=$(mktemp -d) - # Unprotect main branch right before pushing (protection may be re-applied by GitLab) - docker exec gitlab-gitlab-1 gitlab-rails runner "p=Project.find_by(path:'$REPO',namespace_id:User.find_by(username:'{{.GIT_USERNAME}}').namespace_id); ProtectedBranch.where(project:p).destroy_all if p" 2>/dev/null || true # Clone existing repo (has README from initialize_with_readme) then add source files git clone -q "$REMOTE_URL" "$TMP" 2>/dev/null || { cp -r "$SRC/." "$TMP/"; git -C "$TMP" init -q; git -C "$TMP" remote add origin "$REMOTE_URL"; } cp -r "$SRC/." "$TMP/" diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index c25d877de..90560e7d4 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -222,35 +222,46 @@ tasks: # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash # before the string is passed to gitlab-rails runner as a Ruby script. docker exec gitlab-gitlab-1 gitlab-rails runner " - # Disable default branch protection so user1 can push to main - ApplicationSetting.first.update!( - default_branch_protection: 0, - default_branch_protection_defaults: { - allowed_to_push: [{ access_level: 0 }], - allow_force_push: true, - allowed_to_merge: [{ access_level: 0 }], - developer_can_initial_push: true - } - ) + # Disable default branch protection + enable password auth for git + s = ApplicationSetting.first + s.password_authentication_enabled_for_git = true + s.default_branch_protection = 0 + s.save(validate: false) pat = User.find(1).personal_access_tokens.find_or_initialize_by(name: 'initial-root-token') pat.scopes = ['api','write_repository','sudo'] pat.expires_at = 365.days.from_now pat.save! - u = User.find_by(username: '{{.GIT_USERNAME}}') || User.create!( - name: 'Workshop User', username: '{{.GIT_USERNAME}}', - email: '{{.GIT_USERNAME}}@workshop.local', - password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', - skip_confirmation: true - ) + u = User.find_by(username: '{{.GIT_USERNAME}}') + if u.nil? + svc = Users::CreateService.new(User.find(1), { + name: 'Workshop User', username: '{{.GIT_USERNAME}}', + email: '{{.GIT_USERNAME}}@workshop.local', + password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', + skip_confirmation: true + }).execute + u = User.find_by(username: '{{.GIT_USERNAME}}') + else + u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') + u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) + end + if u.namespace_id.nil? + org_id = Organizations::Organization.find_by(id: 1)&.id || 1 + ns = Namespace.new(owner: u, path: u.username, name: u.username, type: 'User', organization_id: org_id) + ns.save(validate: false) + u.update_columns(namespace_id: ns.id) if ns.persisted? + u.reload + end u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') u1pat.scopes = ['api','write_repository'] u1pat.expires_at = 365.days.from_now u1pat.save! %w[fleet-config java rust golang dotnet next-js].each do |repo| - p = Project.find_by(path: repo, namespace_id: u.namespace_id) || - Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute - ProtectedBranch.where(project: p).destroy_all - puts 'repo: ' + repo + p = Project.find_by(path: repo, namespace_id: u.namespace_id) + if p.nil? + p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute + end + ProtectedBranch.where(project: p).destroy_all if p.persisted? + puts 'repo: ' + repo + ' ok=' + p.persisted?.to_s end puts 'init done' " 2>&1 | tail -20 || true From 2fbc37a17409e5aec684a089e9f3b60de6e13fd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:46:29 +0200 Subject: [PATCH 033/231] chore: reference #757 in secrets-manager:seed-secrets (GitLab creds separation) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index a999ee9ff..7dbe5140d 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1384,6 +1384,9 @@ tasks: # ---- from kind-crossplane ---- secrets-manager:seed-secrets: desc: Seed platform user credentials into Secrets Manager for ExternalSecrets + # TODO(#757): gitlab_root_password and git_token should move to peeks-hub/gitlab + # (separate secret, gated on enable_gitlab) so the platform can run without GitLab. + # https://github.com/aws-samples/appmod-blueprints/issues/757 cmds: - | SECRET_KEY="{{.HUB_CLUSTER_NAME}}/secrets" From c8cafc0e1a16d8c093cda36678eec314ce341d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:49:54 +0200 Subject: [PATCH 034/231] =?UTF-8?q?fix:=20use=20initialize=5Fwith=5Freadme?= =?UTF-8?q?:false=20=E2=80=94=20empty=20repos=20have=20no=20protected=20ma?= =?UTF-8?q?in=20branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 7 +++---- cluster-providers/workshop/Taskfile.yaml | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 9d7290436..7825942f8 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -837,7 +837,7 @@ tasks: %w[fleet-config java rust golang dotnet next-js].each do |repo| p = Project.find_by(path: repo, namespace_id: u.namespace_id) if p.nil? - p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute + p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: false).execute end ProtectedBranch.where(project: p).destroy_all if p.persisted? puts 'repo: ' + repo + ' ok=' + p.persisted?.to_s @@ -854,11 +854,10 @@ tasks: fi [ -d "$SRC" ] || continue TMP=$(mktemp -d) - # Clone existing repo (has README from initialize_with_readme) then add source files - git clone -q "$REMOTE_URL" "$TMP" 2>/dev/null || { cp -r "$SRC/." "$TMP/"; git -C "$TMP" init -q; git -C "$TMP" remote add origin "$REMOTE_URL"; } cp -r "$SRC/." "$TMP/" cd "$TMP" - git add -A && git commit -qm "seed: add application source" 2>/dev/null || true + git init -q && git add -A && git commit -qm "init" 2>/dev/null || true + git remote add origin "$REMOTE_URL" git push origin HEAD:main cd - >/dev/null; rm -rf "$TMP" echo " seeded: $REPO" diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 90560e7d4..e2f2963e4 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -258,7 +258,7 @@ tasks: %w[fleet-config java rust golang dotnet next-js].each do |repo| p = Project.find_by(path: repo, namespace_id: u.namespace_id) if p.nil? - p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: true).execute + p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: false).execute end ProtectedBranch.where(project: p).destroy_all if p.persisted? puts 'repo: ' + repo + ' ok=' + p.persisted?.to_s From f62a43ea9aca072222d5641d55907257d414042b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 18:54:48 +0200 Subject: [PATCH 035/231] fix(kind-kro-ack): guard kro:reset-crds against live instances --- cluster-providers/kind-kro-ack/Taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 7dbe5140d..1d3ec85d5 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -346,6 +346,9 @@ tasks: kro:reset-crds: desc: Delete KRO-managed CRDs on the bootstrap kind cluster to allow breaking schema changes (run before kro:apply-rgds) + preconditions: + - sh: "! kubectl get ekscluster,eksclusterwithvpc -A --no-headers 2>/dev/null | grep -qv '^$'" + msg: "Live EksCluster/EksclusterWithVpc instances exist — delete them first (task kind-kro-ack:destroy) before resetting CRDs." cmds: - kubectl delete crd eksclusters.kro.run eksclusterwithvpcs.kro.run --ignore-not-found - printf '{{.C_OK}}✓ KRO CRDs deleted — re-run task install.{{.C_RESET}}\n' From 4cf3704c0d5e7261b7eea2780250c622018a42b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 19:03:30 +0200 Subject: [PATCH 036/231] fix: set default_branch_protection_defaults to allow all pushes (stops re-protection on branch creation) --- cluster-providers/kind-crossplane/Taskfile.yaml | 1 + cluster-providers/workshop/Taskfile.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 7825942f8..83eebb775 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -803,6 +803,7 @@ tasks: s = ApplicationSetting.first s.password_authentication_enabled_for_git = true s.default_branch_protection = 0 + s.default_branch_protection_defaults = {allowed_to_push: [{access_level: 0}], allow_force_push: true, allowed_to_merge: [{access_level: 0}], developer_can_initial_push: true} s.save(validate: false) pat = User.find(1).personal_access_tokens.find_or_initialize_by(name: 'initial-root-token') pat.scopes = ['api','write_repository','sudo'] diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index e2f2963e4..4b36ad039 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -226,6 +226,7 @@ tasks: s = ApplicationSetting.first s.password_authentication_enabled_for_git = true s.default_branch_protection = 0 + s.default_branch_protection_defaults = {allowed_to_push: [{access_level: 0}], allow_force_push: true, allowed_to_merge: [{access_level: 0}], developer_can_initial_push: true} s.save(validate: false) pat = User.find(1).personal_access_tokens.find_or_initialize_by(name: 'initial-root-token') pat.scopes = ['api','write_repository','sudo'] From 522df809536a30c9f8323445dbebcec541c0dc76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 22:06:52 +0200 Subject: [PATCH 037/231] fix(kro): remove readyWhen from clusterConfigSecret and hubSecretsExternal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit These two resources block EKS cluster creation due to ACK/ESO backoff on re-installs. Nothing downstream gates on them being Ready — removing readyWhen makes them fire-and-forget: they sync eventually without blocking the graph. --- .../charts/kro/resource-groups/manifests/eks/rg-eks.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index 508d80c10..a05edf709 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -581,8 +581,6 @@ spec: {"vpc":{"id":"${schema.spec.network.vpcID}","cluster_security_group_id":"","subnet_ids":["${schema.spec.network.subnets.workers.subnet1ID}","${schema.spec.network.subnets.workers.subnet2ID}"]},"server":"${ekscluster.status.ackResourceMetadata.arn}","metadata":{"cluster_name":"${schema.spec.name}","region":"${schema.spec.region}","account_id":"${schema.spec.accountId}"}} - id: clusterConfigSecret - readyWhen: - - ${clusterConfigSecret.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} template: apiVersion: secretsmanager.services.k8s.aws/v1alpha1 kind: Secret @@ -606,8 +604,6 @@ spec: key: config - id: hubSecretsExternal - readyWhen: - - ${hubSecretsExternal.status.conditions.exists(x, x.type == 'Ready' && x.status == "True")} template: apiVersion: external-secrets.io/v1 kind: ExternalSecret From c3a3e0394193edd46b0e0b25a25723443d48b60a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 22:17:08 +0200 Subject: [PATCH 038/231] fix(kind-kro-ack): refresh eso-aws-credentials in credentials:refresh --- cluster-providers/kind-kro-ack/Taskfile.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 1d3ec85d5..ff5e180f4 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -219,6 +219,14 @@ tasks: - kubectl -n {{.ACK_NAMESPACE}} create secret generic aws-credentials --from-file=credentials={{.ROOT_DIR}}/private/aws-creds.ini --dry-run=client -o yaml | kubectl apply -f - + - | + # Also refresh eso-aws-credentials (ESO needs individual keys, not INI format) + CREDS=$(kubectl -n {{.ACK_NAMESPACE}} get secret aws-credentials -o jsonpath='{.data.credentials}' | base64 -d) + kubectl -n {{.ACK_NAMESPACE}} create secret generic eso-aws-credentials \ + --from-literal=access-key-id="$(echo "$CREDS" | grep aws_access_key_id | awk '{print $3}')" \ + --from-literal=secret-access-key="$(echo "$CREDS" | grep aws_secret_access_key | awk '{print $3}')" \ + --from-literal=session-token="$(echo "$CREDS" | grep aws_session_token | awk '{print $3}')" \ + --dry-run=client -o yaml | kubectl apply -f - - rm -f {{.ROOT_DIR}}/private/aws-creds.ini argocd:install: From 6db7de905f527ec16f1acef411bd7ff0abee3891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 22:31:30 +0200 Subject: [PATCH 039/231] =?UTF-8?q?fix:=20restore=20git=20push=20-f=20for?= =?UTF-8?q?=20seed=20=E2=80=94=20user1=20owns=20repos,=20force=20push=20is?= =?UTF-8?q?=20correct?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 83eebb775..910227f5a 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -859,7 +859,7 @@ tasks: cd "$TMP" git init -q && git add -A && git commit -qm "init" 2>/dev/null || true git remote add origin "$REMOTE_URL" - git push origin HEAD:main + git push -f origin HEAD:main cd - >/dev/null; rm -rf "$TMP" echo " seeded: $REPO" done From 2aad9a38f07aa4e3cd7810c6b4461d6f27f812ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 26 Jun 2026 22:34:04 +0200 Subject: [PATCH 040/231] fix(kind-kro-ack): restore-secret before create/put in all seed tasks Prevents InvalidRequestException when secrets are marked for deletion (30-day retention period) after a failed install/teardown cycle. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index ff5e180f4..91f7d3777 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -882,6 +882,7 @@ tasks: --arg addons '' \ --arg vpc '{{.VPC_ID}}' \ '{metadata: $metadata, config: $config, server: $server, addons: $addons, vpc: $vpc}') + aws secretsmanager restore-secret --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} 2>/dev/null || true aws secretsmanager create-secret \ --name "$SECRET_KEY" --secret-string "$SECRET_VALUE" --region {{.AWS_REGION}} 2>/dev/null \ || aws secretsmanager put-secret-value \ @@ -899,6 +900,7 @@ tasks: SECRET_VALUE=$(jq -n \ --arg admin "$ADMIN_PASS" --arg db "$DB_PASS" --arg user "$USER_PASS" \ '{keycloak_admin_password: $admin, keycloak_postgres_password: $db, user_password: $user}') + aws secretsmanager restore-secret --secret-id "{{.HUB_CLUSTER_NAME}}/keycloak" --region {{.AWS_REGION}} 2>/dev/null || true aws secretsmanager create-secret \ --name "{{.HUB_CLUSTER_NAME}}/keycloak" --secret-string "$SECRET_VALUE" --region {{.AWS_REGION}} 2>/dev/null \ || aws secretsmanager put-secret-value \ @@ -1458,6 +1460,7 @@ tasks: UPDATED=$(echo "$UPDATED" | jq --arg p "$BS_PG_PASS" '. + {backstage_postgres_password: $p}') fi # git_token is seeded by CDK at deploy time (root-) — no runtime seeding needed. + aws secretsmanager restore-secret --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} 2>/dev/null || true aws secretsmanager create-secret \ --name "$SECRET_KEY" \ --secret-string "$UPDATED" \ @@ -1617,6 +1620,7 @@ tasks: DEVLAKE_MYSQL=$(openssl rand -base64 18 | tr -d '/+=' | head -c 24) UPDATED=$(echo "$UPDATED" | jq --arg p "$DEVLAKE_MYSQL" '. + {devlake_mysql_password: $p}') fi + aws secretsmanager restore-secret --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} 2>/dev/null || true aws secretsmanager create-secret \ --name "$SECRET_KEY" \ --secret-string "$UPDATED" \ From 6c6b30e2c230f6c9ad3fa4032360811c1739361d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 15:44:06 +0200 Subject: [PATCH 041/231] fix(kind-kro-ack): remove double slash in addonsRepoBasepath (REPO_BASEPATH already has trailing /) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 91f7d3777..a0717bf31 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -876,7 +876,7 @@ tasks: - | SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" SECRET_VALUE=$(jq -n \ - --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}/","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}/","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.RESOURCE_PREFIX}}-hub-ingress","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}"}' \ + --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.RESOURCE_PREFIX}}-hub-ingress","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}"}' \ --arg config '{"tlsClientConfig":{"insecure":false}}' \ --arg server '{{.CLUSTER_ARN}}' \ --arg addons '' \ @@ -998,7 +998,7 @@ tasks: printf ' exposure_mode = %s\n' "{{.EXPOSURE_MODE}}" printf ' alb_listener_arn = %s\n' "{{.ALB_LISTENER_ARN}}" - | - cat <<'EOF' | sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g; s|CLUSTER_ARN|{{.CLUSTER_ARN}}|g; s|ARGOCD_ROLE_ARN|{{.ARGOCD_ROLE_ARN}}|g; s|REPO_URL|{{.REPO_URL}}|g; s|REPO_REVISION|{{.REPO_REVISION}}|g; s|REPO_BASEPATH|{{.REPO_BASEPATH}}/|g; s|EXPOSURE_MODE|{{.EXPOSURE_MODE}}|g; s|EDGE_DOMAIN|{{.EDGE_DOMAIN}}|g; s|ALB_LISTENER_ARN|{{.ALB_LISTENER_ARN}}|g; s|VPC_ID_VAL|{{.VPC_ID}}|g; s|AWS_REGION_VAL|{{.AWS_REGION}}|g; s|AWS_ACCOUNT_ID_VAL|{{.AWS_ACCOUNT_ID}}|g; s|RESOURCE_PREFIX_VAL|{{.RESOURCE_PREFIX}}|g; s|INGRESS_SECURITY_GROUPS_VAL|{{.INGRESS_SECURITY_GROUPS}}|g; s|ADMIN_ROLE_NAME_VAL|{{.ADMIN_ROLE_NAME}}|g" | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - + cat <<'EOF' | sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g; s|CLUSTER_ARN|{{.CLUSTER_ARN}}|g; s|ARGOCD_ROLE_ARN|{{.ARGOCD_ROLE_ARN}}|g; s|REPO_URL|{{.REPO_URL}}|g; s|REPO_REVISION|{{.REPO_REVISION}}|g; s|REPO_BASEPATH|{{.REPO_BASEPATH}}|g; s|EXPOSURE_MODE|{{.EXPOSURE_MODE}}|g; s|EDGE_DOMAIN|{{.EDGE_DOMAIN}}|g; s|ALB_LISTENER_ARN|{{.ALB_LISTENER_ARN}}|g; s|VPC_ID_VAL|{{.VPC_ID}}|g; s|AWS_REGION_VAL|{{.AWS_REGION}}|g; s|AWS_ACCOUNT_ID_VAL|{{.AWS_ACCOUNT_ID}}|g; s|RESOURCE_PREFIX_VAL|{{.RESOURCE_PREFIX}}|g; s|INGRESS_SECURITY_GROUPS_VAL|{{.INGRESS_SECURITY_GROUPS}}|g; s|ADMIN_ROLE_NAME_VAL|{{.ADMIN_ROLE_NAME}}|g" | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - apiVersion: v1 kind: Secret type: Opaque From d29320d4fa2f7554b69c6a393ed966baf8b3b13e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 15:49:54 +0200 Subject: [PATCH 042/231] fix(kind-kro-ack): use HTTPS endpoint not ARN as ArgoCD cluster server ArgoCD OSS requires the HTTPS endpoint as server URL. The EKS ARN (arn:aws:eks:...) causes parse error in ArgoCD. Add CLUSTER_ENDPOINT var and use it in hub:seed-secret. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index a0717bf31..36f50392c 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -958,6 +958,8 @@ tasks: vars: CLUSTER_ARN: sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.arn' --output text + CLUSTER_ENDPOINT: + sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.endpoint' --output text ARGOCD_ROLE_ARN: sh: aws iam get-role --role-name {{.HUB_CLUSTER_NAME}}-argocd-role --query 'Role.Arn' --output text 2>/dev/null || echo "" ADMIN_ROLE_NAME: @@ -998,7 +1000,7 @@ tasks: printf ' exposure_mode = %s\n' "{{.EXPOSURE_MODE}}" printf ' alb_listener_arn = %s\n' "{{.ALB_LISTENER_ARN}}" - | - cat <<'EOF' | sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g; s|CLUSTER_ARN|{{.CLUSTER_ARN}}|g; s|ARGOCD_ROLE_ARN|{{.ARGOCD_ROLE_ARN}}|g; s|REPO_URL|{{.REPO_URL}}|g; s|REPO_REVISION|{{.REPO_REVISION}}|g; s|REPO_BASEPATH|{{.REPO_BASEPATH}}|g; s|EXPOSURE_MODE|{{.EXPOSURE_MODE}}|g; s|EDGE_DOMAIN|{{.EDGE_DOMAIN}}|g; s|ALB_LISTENER_ARN|{{.ALB_LISTENER_ARN}}|g; s|VPC_ID_VAL|{{.VPC_ID}}|g; s|AWS_REGION_VAL|{{.AWS_REGION}}|g; s|AWS_ACCOUNT_ID_VAL|{{.AWS_ACCOUNT_ID}}|g; s|RESOURCE_PREFIX_VAL|{{.RESOURCE_PREFIX}}|g; s|INGRESS_SECURITY_GROUPS_VAL|{{.INGRESS_SECURITY_GROUPS}}|g; s|ADMIN_ROLE_NAME_VAL|{{.ADMIN_ROLE_NAME}}|g" | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - + cat <<'EOF' | sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g; s|CLUSTER_ARN|{{.CLUSTER_ARN}}|g; s|CLUSTER_ENDPOINT|{{.CLUSTER_ENDPOINT}}|g; s|ARGOCD_ROLE_ARN|{{.ARGOCD_ROLE_ARN}}|g; s|REPO_URL|{{.REPO_URL}}|g; s|REPO_REVISION|{{.REPO_REVISION}}|g; s|REPO_BASEPATH|{{.REPO_BASEPATH}}|g; s|EXPOSURE_MODE|{{.EXPOSURE_MODE}}|g; s|EDGE_DOMAIN|{{.EDGE_DOMAIN}}|g; s|ALB_LISTENER_ARN|{{.ALB_LISTENER_ARN}}|g; s|VPC_ID_VAL|{{.VPC_ID}}|g; s|AWS_REGION_VAL|{{.AWS_REGION}}|g; s|AWS_ACCOUNT_ID_VAL|{{.AWS_ACCOUNT_ID}}|g; s|RESOURCE_PREFIX_VAL|{{.RESOURCE_PREFIX}}|g; s|INGRESS_SECURITY_GROUPS_VAL|{{.INGRESS_SECURITY_GROUPS}}|g; s|ADMIN_ROLE_NAME_VAL|{{.ADMIN_ROLE_NAME}}|g" | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - apiVersion: v1 kind: Secret type: Opaque @@ -1036,7 +1038,7 @@ tasks: exposure_mode: "EXPOSURE_MODE" stringData: name: CLUSTER_NAME - server: "CLUSTER_ARN" + server: "CLUSTER_ENDPOINT" config: '{"awsAuthConfig":{"clusterName":"CLUSTER_NAME","roleARN":"ARGOCD_ROLE_ARN"},"tlsClientConfig":{"insecure":false}}' EOF - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig From 37554e1a6a0df9781f9712d399ce4987011d4085 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 20:51:34 +0200 Subject: [PATCH 043/231] fix: add tolerations to eks-pod-identity-agent addon for EKS Auto Mode nodes EKS Auto Mode nodes require Exists toleration on the pod-identity-agent DaemonSet. Without it, 0 pods are scheduled and Pod Identity doesn't work, blocking apps from accessing AWS resources (DynamoDB etc.). --- .../addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index a05edf709..86e1dcd1b 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -386,6 +386,7 @@ spec: name: eks-pod-identity-agent addonVersion: v1.3.4-eksbuild.1 clusterName: "${schema.spec.name}" + configurationValues: '{"tolerations":[{"operator":"Exists"}]}' ########################################################### # ArgoCD Integration From c761d1d0f095287032267719823ebb6199528094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 20:57:09 +0200 Subject: [PATCH 044/231] fix(kind-kro-ack): remove hub:install-argocd from hub:seed EKS ArgoCD Capability manages ArgoCD externally (no pods in cluster). Installing OSS ArgoCD via Helm conflicts with the Capability and causes argocd-k8s-auth failures. kro-ack always uses the Capability. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 36f50392c..143e17493 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -857,7 +857,6 @@ tasks: - task: secrets-manager:seed-secrets - task: hub:install-eso - task: hub:cluster-secret-store - - task: hub:install-argocd - task: hub:seed-secret - task: hub:apply-root-appset From 027ad1c6bae2220449a86a8fae6a467ea1c841d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 21:01:14 +0200 Subject: [PATCH 045/231] fix(kind-kro-ack): align hub:seed-secret with crossplane - use ARN + no awsAuthConfig EKS Capability ArgoCD resolves the EKS ARN natively without argocd-k8s-auth. Matches crossplane provider which uses server=ARN + tlsClientConfig only. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 143e17493..b52d43e31 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1037,8 +1037,8 @@ tasks: exposure_mode: "EXPOSURE_MODE" stringData: name: CLUSTER_NAME - server: "CLUSTER_ENDPOINT" - config: '{"awsAuthConfig":{"clusterName":"CLUSTER_NAME","roleARN":"ARGOCD_ROLE_ARN"},"tlsClientConfig":{"insecure":false}}' + server: "CLUSTER_ARN" + config: '{"tlsClientConfig":{"insecure":false}}' EOF - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig From 9335c0a9275c34111ff6872f165c9dae9011dacd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 21:06:53 +0200 Subject: [PATCH 046/231] fix(kind-kro-ack): remove argocd:install from bootstrap - not needed kind-crossplane doesn't install ArgoCD on the bootstrap cluster. ArgoCD on hub is managed by EKS Capability. The root appset is applied via hub:apply-root-appset using hub-kubeconfig directly. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index b52d43e31..7db53e69b 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -104,7 +104,6 @@ tasks: - task: kind:create - task: kind:kubeconfig - task: credentials:setup - - task: argocd:install - task: ack:install - task: kro:install - task: kro:install-eso-bootstrap From 3eb77a335a6e62ccc5be84993181fa7801cf5fdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 21:12:26 +0200 Subject: [PATCH 047/231] fix(kind-kro-ack): enforce bootstrap context after kind:kubeconfig to prevent hub context contamination --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 7db53e69b..6e8379c48 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -163,6 +163,7 @@ tasks: cmds: - mkdir -p {{.ROOT_DIR}}/private - kind export kubeconfig --name {{.KIND_CLUSTER_NAME}} --kubeconfig {{.KUBECONFIG}} 2>/dev/null || true + - kubectl config use-context kind-{{.KIND_CLUSTER_NAME}} --kubeconfig {{.KUBECONFIG}} 2>/dev/null || true credentials:setup: desc: Create AWS credentials secret (skips if exists) From 80a21c784e3bd9e806ed1fbaaba8346190f41d87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 23:57:13 +0200 Subject: [PATCH 048/231] revert: remove unnecessary pod-identity-agent tolerations (EKS Auto Mode has built-in support at 127.0.0.1:23443) --- .../addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index 86e1dcd1b..a05edf709 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -386,7 +386,6 @@ spec: name: eks-pod-identity-agent addonVersion: v1.3.4-eksbuild.1 clusterName: "${schema.spec.name}" - configurationValues: '{"tolerations":[{"operator":"Exists"}]}' ########################################################### # ArgoCD Integration From a7b5bd0467a3f3d4861622b2db90a24842ea3c60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 27 Jun 2026 23:57:40 +0200 Subject: [PATCH 049/231] fix(kro): add argocd-capability-role as trusted principal in argocdRole EKS ArgoCD Capability uses argocd-capability-role to assume argocd-role when calling argocd-k8s-auth to generate cluster tokens. Without this trust the Capability ArgoCD cannot connect to the cluster. --- .../charts/kro/resource-groups/manifests/eks/rg-eks.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index a05edf709..7c676a0cf 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -423,7 +423,10 @@ spec: { "Effect": "Allow", "Principal": { - "AWS": "${schema.spec.argoCdHubRoleArn}" + "AWS": [ + "${schema.spec.argoCdHubRoleArn}", + "arn:${schema.spec.aws_partition}:iam::${schema.spec.accountId}:role/${schema.spec.name}-argocd-capability-role" + ] }, "Action": [ "sts:TagSession", From f3204f592535c218374878475f26490c9d5abeca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 11:26:23 +0200 Subject: [PATCH 050/231] fix: seed platform/backstage/templates into fleet-config so Backstage catalog works in cloudfront mode --- cluster-providers/kind-crossplane/Taskfile.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 910227f5a..7857e94c7 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -856,6 +856,11 @@ tasks: [ -d "$SRC" ] || continue TMP=$(mktemp -d) cp -r "$SRC/." "$TMP/" + # Also seed Backstage templates into fleet-config so Backstage catalog works in cloudfront mode + if [ "$REPO" = "fleet-config" ] && [ -d "$WORKSHOP_DIR/platform/backstage/templates" ]; then + mkdir -p "$TMP/platform/backstage" + cp -r "$WORKSHOP_DIR/platform/backstage/templates" "$TMP/platform/backstage/" + fi cd "$TMP" git init -q && git add -A && git commit -qm "init" 2>/dev/null || true git remote add origin "$REMOTE_URL" From 9920cf0af597ea81e88e6aec1f2a7d3e5a4fe29b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 11:27:44 +0200 Subject: [PATCH 051/231] fix: capture user1 PAT token after GitLab init and update git_token in peeks-hub/secrets Ensures the CICDPipeline setup-workflow has a valid GitLab token for webhook creation (issue #2). Also removes the broken GIT_TOKEN='' output from runner. --- cluster-providers/kind-crossplane/Taskfile.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 7857e94c7..3a59c354a 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -835,6 +835,8 @@ tasks: u1pat.scopes = ['api','write_repository'] u1pat.expires_at = 365.days.from_now u1pat.save! + # Output the plaintext token on creation (only available once) + puts 'GIT_TOKEN=' + (u1pat.token || '') %w[fleet-config java rust golang dotnet next-js].each do |repo| p = Project.find_by(path: repo, namespace_id: u.namespace_id) if p.nil? @@ -845,6 +847,18 @@ tasks: end puts 'init done' " 2>&1 | tail -20 || true + # Capture user1 PAT and update git_token in peeks-hub/secrets for webhook creation + GIT_TOKEN=$(docker exec gitlab-gitlab-1 gitlab-rails runner ' + u = User.find_by(username: "user1") + pat = u&.personal_access_tokens&.find_by(name: "workshop-token") + puts pat ? pat.token.to_s : "" + ' 2>/dev/null | tail -1) + if [ -n "$GIT_TOKEN" ]; then + SECRET_KEY="{{.HUB_CLUSTER_NAME}}/secrets" + EXISTING=$(aws secretsmanager get-secret-value --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} --query SecretString --output text 2>/dev/null || echo "{}") + UPDATED=$(echo "$EXISTING" | python3 -c "import json,sys; d=json.load(sys.stdin); d['git_token']='$GIT_TOKEN'; print(json.dumps(d))" 2>/dev/null) + [ -n "$UPDATED" ] && aws secretsmanager put-secret-value --secret-id "$SECRET_KEY" --secret-string "$UPDATED" --region {{.AWS_REGION}} >/dev/null 2>&1 && printf ' git_token updated in peeks-hub/secrets\n' || true + fi WORKSHOP_DIR="$HOME/environment/{{.WORKING_REPO}}" for REPO in {{.REPOS}}; do REMOTE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/$REPO.git" From 1fead6e047c193c474f81d6bb9c3ed63b0b49ddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 12:53:21 +0200 Subject: [PATCH 052/231] fix(kind-kro-ack): add phase1b-parallel with gitlab:init-ec2 before phase2 Missing from install flow: GitLab init must run before spokes:enable-crossplane which needs fleet-config repo and GIT_TOKEN. Mirrors kind-crossplane structure. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 6e8379c48..1e19083ea 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -117,6 +117,7 @@ tasks: - task: hub:wait-for-sync - task: install:phase1-parallel - task: hub:wait-for-full-sync + - task: install:phase1b-parallel - task: hub:set-overlay-repo - task: install:phase2-parallel - | @@ -131,6 +132,11 @@ tasks: deps: - hub:restart-identity-pods + install:phase1b-parallel: + internal: true + deps: + - workshop:gitlab:init-ec2 + install:phase2-parallel: internal: true cmds: From 5d50a309ff76436bcfa49fc853679458cd4cffae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 14:44:49 +0200 Subject: [PATCH 053/231] fix(kind-kro-ack): fix gitlab:init-ec2 cross-namespace delegation --- cluster-providers/kind-kro-ack/Taskfile.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 1e19083ea..e9df10cf6 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -134,8 +134,13 @@ tasks: install:phase1b-parallel: internal: true - deps: - - workshop:gitlab:init-ec2 + cmds: + - task: gitlab:init-ec2 + + gitlab:init-ec2: + desc: Wait for GitLab CE on IDE EC2 to be ready and initialize (delegates to workshop provider) + cmds: + - cd {{.ROOT_DIR}} && task workshop:gitlab:init-ec2 install:phase2-parallel: internal: true From 585e7a53756599be144bb757312b36f06dbfa06f Mon Sep 17 00:00:00 2001 From: Workshop User Date: Sun, 28 Jun 2026 10:43:04 +0000 Subject: [PATCH 054/231] fix(kro): grant ArgoCD Capability Role direct access to KRO-provisioned spokes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The KRO EKS cluster RGD creates a separate argocd-role for each spoke and configures the ArgoCD cluster secret with roleARN pointing to it. However, the ArgoCD Capability Role's IAM policy only allows assuming 'peeks-cluster-mgmt-*' roles — the spoke-specific roles don't match. This caused ArgoCD to fail connecting to KRO-provisioned spokes with: 'error getting cluster RESTConfig: failed to verify the access entry' Fix: 1. Remove roleARN from the cluster secret config — ArgoCD Capability uses its own role directly (matching the Crossplane-provisioned spoke pattern) 2. Add a new accessEntryArgocdCapability resource that grants the Capability Role direct cluster-admin access to the spoke (using the existing but previously unused argoCdCapabilityRoleArn schema field) The includeWhen guard ensures backward compatibility — the access entry is only created when argoCdCapabilityRoleArn is provided. --- .../resource-groups/manifests/eks/rg-eks.yaml | 31 +++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index 7c676a0cf..b14d4e5ab 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -555,8 +555,7 @@ spec: config: | { "awsAuthConfig": { - "clusterName": "${schema.spec.name}", - "roleARN": "${argocdRole.status.ackResourceMetadata.arn}" + "clusterName": "${schema.spec.name}" }, "tlsClientConfig": { "caData": "${ekscluster.status.certificateAuthority.data}", @@ -668,6 +667,34 @@ spec: principalARN: "${argocdRole.status.ackResourceMetadata.arn}" type: STANDARD + - id: accessEntryArgocdCapability + includeWhen: + - ${schema.spec.argoCdCapabilityRoleArn != ""} + template: + apiVersion: eks.services.k8s.aws/v1alpha1 + kind: AccessEntry + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-access-entry-argocd-capability" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + clusterName: "${schema.spec.name}" + accessPolicies: + - accessScope: + type: "cluster" + policyARN: "arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy" + principalARN: "${schema.spec.argoCdCapabilityRoleArn}" + type: STANDARD + - id: accessEntryAdmin template: apiVersion: eks.services.k8s.aws/v1alpha1 From 38e9f3a40c3338c422b68c96eed8f835729e6415 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 16:01:54 +0200 Subject: [PATCH 055/231] fix(kind-kro-ack): skip hub:seed-secret when ARN matches, fix hub:wait-for-sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - hub:seed-secret: add status check to skip when cluster secret ARN already correct — prevents 20+ min ArgoCD cache invalidation on re-runs - hub:wait-for-sync: filter Unknown apps (spoke apps on non-existent clusters), add MAX_OUTOFSYNC tolerance, non-fatal timeout --- cluster-providers/kind-kro-ack/Taskfile.yaml | 36 ++++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index e9df10cf6..4622c254e 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -972,6 +972,12 @@ tasks: sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.endpoint' --output text ARGOCD_ROLE_ARN: sh: aws iam get-role --role-name {{.HUB_CLUSTER_NAME}}-argocd-role --query 'Role.Arn' --output text 2>/dev/null || echo "" + status: + # Skip if the cluster secret already has the correct server ARN — avoids triggering + # a 20+ minute ArgoCD cache invalidation on every re-run of task install. + - | + CURRENT=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get secret {{.HUB_CLUSTER_NAME}} -n argocd -o jsonpath='{.data.server}' 2>/dev/null | base64 -d 2>/dev/null) + [ "$CURRENT" = "{{.CLUSTER_ARN}}" ] ADMIN_ROLE_NAME: sh: yq '.hub.adminRoleName // "Admin"' {{.CONFIG_FILE}} # Edge metadata — populated by hub:ingress + hub:cloudfront before hub:seed runs. @@ -1063,26 +1069,36 @@ tasks: hub:wait-for-sync: desc: Wait for hub ArgoCD apps to sync + vars: + MAX_OUTOFSYNC: '{{.MAX_OUTOFSYNC | default "5"}}' cmds: - task: hub:kubeconfig - printf '{{.C_STEP}}▸ Waiting for apps to appear...{{.C_RESET}}\n' - | - for i in $(seq 1 30); do - TOTAL=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get app -n argocd --no-headers 2>/dev/null | wc -l) - [ "$TOTAL" -gt 3 ] && break - sleep 10 + TIMEOUT=300; ELAPSED=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + TOTAL=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get applications.argoproj.io -n argocd --no-headers 2>/dev/null | wc -l | tr -d ' ') + [ "$TOTAL" -gt 3 ] && printf '{{.C_INFO}} %s apps detected{{.C_RESET}}\n' "$TOTAL" && break + printf '{{.C_INFO}} %s apps so far, waiting...{{.C_RESET}}\n' "$TOTAL" + sleep 10; ELAPSED=$((ELAPSED + 10)) done - printf '{{.C_STEP}}▸ Waiting for apps to sync...{{.C_RESET}}\n' - | - TIMEOUT=1800; ELAPSED=0 + TIMEOUT=1800; INTERVAL=15; ELAPSED=0 while [ $ELAPSED -lt $TIMEOUT ]; do - TOTAL=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get app -n argocd --no-headers 2>/dev/null | wc -l) - SYNCED=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get app -n argocd --no-headers 2>/dev/null | awk '{print $2}' | grep -c "Synced" || true) + APP_OUTPUT=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get applications.argoproj.io -n argocd --no-headers 2>/dev/null || echo "") + TOTAL=$(echo "$APP_OUTPUT" | awk '$2 != "Unknown" && NF>0' | wc -l | tr -d ' ') + SYNCED=$(echo "$APP_OUTPUT" | awk '$2 == "Synced"' | wc -l | tr -d ' ') + UNKNOWN=$(echo "$APP_OUTPUT" | awk '$2 == "Unknown"' | wc -l | tr -d ' ') OUTOFSYNC=$((TOTAL - SYNCED)) - printf '{{.C_INFO}} %s/%s synced (%ss){{.C_RESET}}\n' "$SYNCED" "$TOTAL" "$ELAPSED" - [ "$TOTAL" -gt 0 ] && [ "$OUTOFSYNC" -le 2 ] && break - sleep 15; ELAPSED=$((ELAPSED + 15)) + printf '{{.C_INFO}} Apps: %s/%s synced, %s unknown, %s out-of-sync (%ss elapsed){{.C_RESET}}\n' "$SYNCED" "$TOTAL" "$UNKNOWN" "$OUTOFSYNC" "$ELAPSED" + if [ "$TOTAL" -gt 0 ] && [ "$OUTOFSYNC" -le {{.MAX_OUTOFSYNC}} ]; then + printf '{{.C_OK}}✓ All %s apps synced (within %s tolerance, %s unknown).{{.C_RESET}}\n' "$SYNCED" "{{.MAX_OUTOFSYNC}}" "$UNKNOWN" + break + fi + sleep $INTERVAL; ELAPSED=$((ELAPSED + INTERVAL)) done + [ $ELAPSED -ge $TIMEOUT ] && printf '{{.C_WARN}}⚠ Sync wait timed out after %ss — continuing.{{.C_RESET}}\n' "$TIMEOUT" || true - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig status: From ff037ce5994a862c47e8394427b0d0f0f9fe96f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 16:03:23 +0200 Subject: [PATCH 056/231] feat(kind-kro-ack): port phase1b improvements from kind-crossplane Add hub:restart-langfuse and urls to phase1b-parallel deps (mirrors kind-crossplane). Langfuse needs restart to pick up Keycloak OIDC client secret generated post-install. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 4622c254e..ff042a270 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -134,8 +134,10 @@ tasks: install:phase1b-parallel: internal: true - cmds: - - task: gitlab:init-ec2 + deps: + - hub:restart-langfuse + - urls + - gitlab:init-ec2 gitlab:init-ec2: desc: Wait for GitLab CE on IDE EC2 to be ready and initialize (delegates to workshop provider) From de919070e507f37cc5f36b6afa958074f78fcf31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 16:20:42 +0200 Subject: [PATCH 057/231] feat(kind-kro-ack): add observability seeding in background during phase1 Port install:phase1-observability-bg from kind-crossplane. Runs secrets-manager:seed-observability in background (non-blocking) so AMP/AMG workspace provisioning doesn't delay install completion. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index ff042a270..d06773af5 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -131,6 +131,16 @@ tasks: internal: true deps: - hub:restart-identity-pods + - install:phase1-observability-bg + + install:phase1-observability-bg: + internal: true + desc: Seed observability in background — non-blocking so slow AMP provisioning doesn't delay install + cmds: + - | + REPO_ROOT="${WORKSPACE_PATH:-/home/ec2-user/environment}/${WORKING_REPO:-platform-on-eks-workshop}" + ( cd "$REPO_ROOT" && task kind-kro-ack:secrets-manager:seed-observability >> /tmp/observability-seed.log 2>&1 ) & + printf '{{.C_INFO}} Observability seeding running in background (log: /tmp/observability-seed.log){{.C_RESET}}\n' install:phase1b-parallel: internal: true From 44ae67b774f73debeeb7a5e31fac01faa0c10bba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 16:22:50 +0200 Subject: [PATCH 058/231] fix(kind-kro-ack): move status check to correct position in hub:seed-secret --- cluster-providers/kind-kro-ack/Taskfile.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index d06773af5..c3e3f0ed5 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -984,12 +984,6 @@ tasks: sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.endpoint' --output text ARGOCD_ROLE_ARN: sh: aws iam get-role --role-name {{.HUB_CLUSTER_NAME}}-argocd-role --query 'Role.Arn' --output text 2>/dev/null || echo "" - status: - # Skip if the cluster secret already has the correct server ARN — avoids triggering - # a 20+ minute ArgoCD cache invalidation on every re-run of task install. - - | - CURRENT=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get secret {{.HUB_CLUSTER_NAME}} -n argocd -o jsonpath='{.data.server}' 2>/dev/null | base64 -d 2>/dev/null) - [ "$CURRENT" = "{{.CLUSTER_ARN}}" ] ADMIN_ROLE_NAME: sh: yq '.hub.adminRoleName // "Admin"' {{.CONFIG_FILE}} # Edge metadata — populated by hub:ingress + hub:cloudfront before hub:seed runs. @@ -1017,6 +1011,12 @@ tasks: --query "Listeners[?Port==\`$PORT\`].ListenerArn | [0]" --output text 2>/dev/null || echo "" VPC_ID: sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text 2>/dev/null || echo "" + status: + # Skip if the cluster secret already has the correct server ARN — avoids triggering + # a 20+ minute ArgoCD cache invalidation on every re-run of task install. + - | + CURRENT=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get secret {{.HUB_CLUSTER_NAME}} -n argocd -o jsonpath='{.data.server}' 2>/dev/null | base64 -d 2>/dev/null) + [ "$CURRENT" = "{{.CLUSTER_ARN}}" ] cmds: - task: hub:kubeconfig - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl create namespace argocd --dry-run=client -o yaml | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - From 04dc7ade6a1ee039a535e1bbd40b7d8ed84cf1c8 Mon Sep 17 00:00:00 2001 From: Workshop User Date: Sun, 28 Jun 2026 14:35:34 +0000 Subject: [PATCH 059/231] fix(argo-workflows): use gitlab_domain_name as fallback for SSO issuer in cloudfront mode In cloudfront exposure mode, ingress_domain_name is empty. The SSO config rendered as https:///keycloak/... (no host), causing argo-server to crash with 'no Host in request URL'. Use gitlab_domain_name (which holds the CloudFront domain) as fallback. --- gitops/addons/charts/argo-workflows/templates/install.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitops/addons/charts/argo-workflows/templates/install.yaml b/gitops/addons/charts/argo-workflows/templates/install.yaml index 10e4119ee..449f27b0c 100644 --- a/gitops/addons/charts/argo-workflows/templates/install.yaml +++ b/gitops/addons/charts/argo-workflows/templates/install.yaml @@ -3051,14 +3051,14 @@ data: config: |- sso: # insecureSkipVerify: true - issuer: https://{{ .Values.ingress_domain_name }}/keycloak/realms/platform + issuer: https://{{ .Values.ingress_domain_name | default .Values.gitlab_domain_name }}/keycloak/realms/platform clientId: name: keycloak-oidc key: client-id clientSecret: name: keycloak-oidc key: secret-key - redirectUrl: https://{{ .Values.ingress_domain_name }}/argo-workflows/oauth2/callback + redirectUrl: https://{{ .Values.ingress_domain_name | default .Values.gitlab_domain_name }}/argo-workflows/oauth2/callback rbac: enabled: true scopes: From d73dd60442963c576d2601cfa9ca4d3c0eec4f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 18:04:38 +0200 Subject: [PATCH 060/231] fix(gitlab): add branch protection + auth config to OMNIBUS_CONFIG, guard nil user - gitlab.yaml OMNIBUS_CONFIG: add initial_default_branch_protection=0 and password_authentication_enabled_for_git=true (prevents git push 401 on pre-receive hook due to mismatched internal secrets) - workshop:gitlab:init-ec2: add nil guard before namespace_id check to prevent 'undefined method namespace_id for nil' crash when user creation fails --- cluster-providers/workshop/Taskfile.yaml | 1 + gitops/addons/charts/gitlab/templates/gitlab.yaml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 4b36ad039..6c6b3363f 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -245,6 +245,7 @@ tasks: u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end + raise "Failed to create/find user {{.GIT_USERNAME}}" if u.nil? if u.namespace_id.nil? org_id = Organizations::Organization.find_by(id: 1)&.id || 1 ns = Namespace.new(owner: u, path: u.username, name: u.username, type: 'User', organization_id: org_id) diff --git a/gitops/addons/charts/gitlab/templates/gitlab.yaml b/gitops/addons/charts/gitlab/templates/gitlab.yaml index 4a6f66b12..7b7120e80 100644 --- a/gitops/addons/charts/gitlab/templates/gitlab.yaml +++ b/gitops/addons/charts/gitlab/templates/gitlab.yaml @@ -67,6 +67,8 @@ spec: external_url 'https://{{ .Values.domain_name }}' gitlab_rails['gitlab_shell_ssh_port'] = 22 gitlab_rails['gitlab_shell_git_timeout'] = 800 + gitlab_rails['initial_default_branch_protection'] = 0 + gitlab_rails['password_authentication_enabled_for_git'] = true {{- if eq (default "domain" .Values.exposure_mode) "cloudfront" }} gitlab_rails['enabled_git_access_protocol'] = 'http' {{- else }} From a8907d5df9a988b886c068a6b9c11c561eb19a5a Mon Sep 17 00:00:00 2001 From: Workshop User Date: Sun, 28 Jun 2026 16:05:57 +0000 Subject: [PATCH 061/231] fix(argo-workflows): pass gitlab_domain_name to chart for cloudfront SSO The argo-workflows chart template now uses gitlab_domain_name as fallback when ingress_domain_name is empty. But the value was not passed from the ApplicationSet valuesObject. Add it from the cluster secret annotation. --- gitops/addons/charts/argo-workflows/templates/install.yaml | 6 +++++- gitops/addons/registry/gitops.yaml | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/gitops/addons/charts/argo-workflows/templates/install.yaml b/gitops/addons/charts/argo-workflows/templates/install.yaml index 449f27b0c..d579cc0fd 100644 --- a/gitops/addons/charts/argo-workflows/templates/install.yaml +++ b/gitops/addons/charts/argo-workflows/templates/install.yaml @@ -3050,8 +3050,12 @@ metadata: data: config: |- sso: - # insecureSkipVerify: true + {{- if eq (default "domain" .Values.exposure_mode) "cloudfront" }} + insecureSkipVerify: true + issuer: http://keycloak.keycloak.svc.cluster.local:8080/keycloak/realms/platform + {{- else }} issuer: https://{{ .Values.ingress_domain_name | default .Values.gitlab_domain_name }}/keycloak/realms/platform + {{- end }} clientId: name: keycloak-oidc key: client-id diff --git a/gitops/addons/registry/gitops.yaml b/gitops/addons/registry/gitops.yaml index a154f575c..7b2f946d3 100644 --- a/gitops/addons/registry/gitops.yaml +++ b/gitops/addons/registry/gitops.yaml @@ -51,6 +51,7 @@ argo-workflows: values: ['true'] valuesObject: ingress_domain_name: '{{.metadata.annotations.ingress_domain_name}}' + gitlab_domain_name: '{{.metadata.annotations.gitlab_domain_name}}' exposure_mode: '{{default "domain" .metadata.annotations.exposure_mode}}' aws_cluster_name: '{{.metadata.annotations.aws_cluster_name}}' ignoreDifferences: From d4780fe6b76b815c2e6a21076d868f4c73918d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 18:16:54 +0200 Subject: [PATCH 062/231] fix(workshop): fix raise syntax in gitlab:init-ec2 (RuntimeError not bare string) --- cluster-providers/workshop/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 6c6b3363f..276dfcb04 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -245,7 +245,7 @@ tasks: u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end - raise "Failed to create/find user {{.GIT_USERNAME}}" if u.nil? + raise RuntimeError, "Failed to create/find user {{.GIT_USERNAME}}" if u.nil? if u.namespace_id.nil? org_id = Organizations::Organization.find_by(id: 1)&.id || 1 ns = Namespace.new(owner: u, path: u.username, name: u.username, type: 'User', organization_id: org_id) From 20c42715b448d2b0941c3db6b43f2b04b4e42020 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 18:20:20 +0200 Subject: [PATCH 063/231] fix(workshop): use abort instead of raise for nil user check --- cluster-providers/workshop/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 276dfcb04..e9cee10bf 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -245,7 +245,7 @@ tasks: u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end - raise RuntimeError, "Failed to create/find user {{.GIT_USERNAME}}" if u.nil? + abort "user {{.GIT_USERNAME}} not found" if u.nil? if u.namespace_id.nil? org_id = Organizations::Organization.find_by(id: 1)&.id || 1 ns = Namespace.new(owner: u, path: u.username, name: u.username, type: 'User', organization_id: org_id) From d9ea18be57ee1bda820469099ec43085a7cfa157 Mon Sep 17 00:00:00 2001 From: Workshop User Date: Sun, 28 Jun 2026 16:21:20 +0000 Subject: [PATCH 064/231] fix(keycloak,argo-workflows): use gitlab_domain_name fallback for hostname in cloudfront mode In cloudfront exposure mode, ingress_domain_name is empty. This caused: - Keycloak hostname=https:///keycloak (broken issuer in OIDC discovery) - argo-server crash: 'no Host in request URL' then issuer mismatch Fix: use '| default .Values.global.gitlab_domain_name' which holds the CloudFront domain (d1k9v3wf33443g.cloudfront.net). --- gitops/addons/charts/argo-workflows/templates/install.yaml | 4 ---- gitops/addons/charts/keycloak/templates/install.yaml | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/gitops/addons/charts/argo-workflows/templates/install.yaml b/gitops/addons/charts/argo-workflows/templates/install.yaml index d579cc0fd..3ae89c0f0 100644 --- a/gitops/addons/charts/argo-workflows/templates/install.yaml +++ b/gitops/addons/charts/argo-workflows/templates/install.yaml @@ -3050,12 +3050,8 @@ metadata: data: config: |- sso: - {{- if eq (default "domain" .Values.exposure_mode) "cloudfront" }} insecureSkipVerify: true - issuer: http://keycloak.keycloak.svc.cluster.local:8080/keycloak/realms/platform - {{- else }} issuer: https://{{ .Values.ingress_domain_name | default .Values.gitlab_domain_name }}/keycloak/realms/platform - {{- end }} clientId: name: keycloak-oidc key: client-id diff --git a/gitops/addons/charts/keycloak/templates/install.yaml b/gitops/addons/charts/keycloak/templates/install.yaml index a293ad7a8..f89029533 100644 --- a/gitops/addons/charts/keycloak/templates/install.yaml +++ b/gitops/addons/charts/keycloak/templates/install.yaml @@ -151,8 +151,8 @@ data: db-url=jdbc:postgresql://postgresql.keycloak.svc.cluster.local:5432/postgres # hostname settings - hostname=https://{{ .Values.global.ingress_domain_name }}/keycloak - hostname-admin=https://{{ .Values.global.ingress_domain_name }}/keycloak + hostname=https://{{ .Values.global.ingress_domain_name | default .Values.global.gitlab_domain_name }}/keycloak + hostname-admin=https://{{ .Values.global.ingress_domain_name | default .Values.global.gitlab_domain_name }}/keycloak hostname-strict=false http-relative-path=keycloak http-enabled=true From e0c31c8d97ec8ff678b871a8cea14dfa8877fa3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 18:23:14 +0200 Subject: [PATCH 065/231] fix(workshop): remove nil guard (caused Ruby parse errors in heredoc) --- cluster-providers/workshop/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index e9cee10bf..22b4929bc 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -245,7 +245,7 @@ tasks: u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end - abort "user {{.GIT_USERNAME}} not found" if u.nil? + if u.namespace_id.nil? if u.namespace_id.nil? org_id = Organizations::Organization.find_by(id: 1)&.id || 1 ns = Namespace.new(owner: u, path: u.username, name: u.username, type: 'User', organization_id: org_id) From 30fdec068ee34385c66072378bbfbb321528037c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 18:26:11 +0200 Subject: [PATCH 066/231] fix(workshop): remove duplicate if block in gitlab:init-ec2 ruby script --- cluster-providers/workshop/Taskfile.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 22b4929bc..4b36ad039 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -245,7 +245,6 @@ tasks: u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end - if u.namespace_id.nil? if u.namespace_id.nil? org_id = Organizations::Organization.find_by(id: 1)&.id || 1 ns = Namespace.new(owner: u, path: u.username, name: u.username, type: 'User', organization_id: org_id) From beaaeeac7684a5179e6c548485b70160f27e3473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 18:34:02 +0200 Subject: [PATCH 067/231] fix(workshop): handle new GitLab CE ServiceResponse API in CreateService In newer GitLab CE, Users::CreateService.execute returns ServiceResponse with payload[:user] instead of the user directly. Handle both old and new API. --- cluster-providers/workshop/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 4b36ad039..8bf22a2bd 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -240,7 +240,7 @@ tasks: password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', skip_confirmation: true }).execute - u = User.find_by(username: '{{.GIT_USERNAME}}') + u = svc.respond_to?(:payload) ? svc.payload[:user] : User.find_by(username: '{{.GIT_USERNAME}}') else u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) From bb6c142d825af54662452e2dc7f73aa9f42f85b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 18:44:21 +0200 Subject: [PATCH 068/231] revert: remove gitlab.yaml chart change (docker-compose not k8s chart) --- gitops/addons/charts/gitlab/templates/gitlab.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/gitops/addons/charts/gitlab/templates/gitlab.yaml b/gitops/addons/charts/gitlab/templates/gitlab.yaml index 7b7120e80..4a6f66b12 100644 --- a/gitops/addons/charts/gitlab/templates/gitlab.yaml +++ b/gitops/addons/charts/gitlab/templates/gitlab.yaml @@ -67,8 +67,6 @@ spec: external_url 'https://{{ .Values.domain_name }}' gitlab_rails['gitlab_shell_ssh_port'] = 22 gitlab_rails['gitlab_shell_git_timeout'] = 800 - gitlab_rails['initial_default_branch_protection'] = 0 - gitlab_rails['password_authentication_enabled_for_git'] = true {{- if eq (default "domain" .Values.exposure_mode) "cloudfront" }} gitlab_rails['enabled_git_access_protocol'] = 'http' {{- else }} From 8e6e8ea7add3e33b89c1fb4be03e1e72d0ff4cf1 Mon Sep 17 00:00:00 2001 From: Workshop User Date: Sun, 28 Jun 2026 16:49:29 +0000 Subject: [PATCH 069/231] feat(ray): add ray:setup and ray:prestage-models tasks Ports Ray ML infrastructure from Terraform common module to Taskfile: - ray:setup: S3 bucket, ECR repo, vLLM image build, S3 CSI driver, IAM + Pod Identity - ray:prestage-models: downloads TinyLlama to S3 for serving Usage: task ray:setup && task ray:prestage-models --- .../kind-crossplane/Taskfile.yaml | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 3a59c354a..89d6926cc 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -2794,6 +2794,102 @@ tasks: done printf '{{.C_OK}}✓ Spoke provider identity seeding complete.{{.C_RESET}}\n' + + ray:setup: + desc: "Set up Ray ML infrastructure: S3 bucket, ECR repo, vLLM image, S3 CSI driver, IAM + Pod Identity" + vars: + RAY_BUCKET: "{{.RESOURCE_PREFIX}}-ray-models-{{.AWS_ACCOUNT_ID}}" + RAY_ECR_REPO: "{{.RESOURCE_PREFIX}}-ray-vllm-custom" + RAY_ROLE_PRESTAGE: "{{.RESOURCE_PREFIX}}-model-prestage-role" + RAY_ROLE_WORKER: "{{.RESOURCE_PREFIX}}-ray-worker-role" + DOCKERFILE_PATH: "{{.ROOT_DIR}}/platform/infra/terraform/common/Dockerfile.ray-vllm" + cmds: + - echo "▸ Setting up Ray ML infrastructure..." + # 1. S3 bucket + - cmd: | + if aws s3api head-bucket --bucket "{{.RAY_BUCKET}}" --region {{.AWS_REGION}} 2>/dev/null; then + echo " ✓ S3 bucket {{.RAY_BUCKET}} exists" + else + aws s3api create-bucket --bucket "{{.RAY_BUCKET}}" --region {{.AWS_REGION}} \ + --create-bucket-configuration LocationConstraint={{.AWS_REGION}} --output text >/dev/null + aws s3api put-public-access-block --bucket "{{.RAY_BUCKET}}" \ + --public-access-block-configuration BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true + echo " ✓ S3 bucket {{.RAY_BUCKET}} created" + fi + # 2. ECR repo + - cmd: | + if aws ecr describe-repositories --repository-names "{{.RAY_ECR_REPO}}" --region {{.AWS_REGION}} >/dev/null 2>&1; then + echo " ✓ ECR repo {{.RAY_ECR_REPO}} exists" + else + aws ecr create-repository --repository-name "{{.RAY_ECR_REPO}}" --region {{.AWS_REGION}} \ + --image-tag-mutability MUTABLE --output text >/dev/null + echo " ✓ ECR repo {{.RAY_ECR_REPO}} created" + fi + # 3. Build and push vLLM image + - cmd: | + IMAGE_URI="{{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com/{{.RAY_ECR_REPO}}:latest" + if aws ecr describe-images --repository-name "{{.RAY_ECR_REPO}}" --image-ids imageTag=latest --region {{.AWS_REGION}} >/dev/null 2>&1; then + echo " ✓ vLLM image already exists" + else + echo " ▸ Building vLLM image (5-10 min)..." + aws ecr get-login-password --region {{.AWS_REGION}} | docker login --username AWS --password-stdin {{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com + docker build -t "$IMAGE_URI" -f "{{.DOCKERFILE_PATH}}" "$(dirname {{.DOCKERFILE_PATH}})" + docker push "$IMAGE_URI" + echo " ✓ vLLM image pushed" + fi + # 4. S3 CSI driver addon + - cmd: | + if aws eks describe-addon --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} >/dev/null 2>&1; then + echo " ✓ S3 CSI driver addon installed" + else + aws eks create-addon --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} --output text >/dev/null + aws eks wait addon-active --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} 2>/dev/null || true + echo " ✓ S3 CSI driver addon installed" + fi + # 5. IAM roles + - cmd: | + TRUST='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"pods.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' + S3_RW='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket"],"Resource":["arn:aws:s3:::{{.RAY_BUCKET}}","arn:aws:s3:::{{.RAY_BUCKET}}/*"]}]}' + S3_RO='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:ListBucket"],"Resource":["arn:aws:s3:::{{.RAY_BUCKET}}","arn:aws:s3:::{{.RAY_BUCKET}}/*"]}]}' + for ROLE_INFO in "{{.RAY_ROLE_PRESTAGE}}|$S3_RW" "{{.RAY_ROLE_WORKER}}|$S3_RO"; do + ROLE="${ROLE_INFO%%|*}"; POLICY="${ROLE_INFO#*|}" + if aws iam get-role --role-name "$ROLE" >/dev/null 2>&1; then + echo " ✓ IAM role $ROLE exists" + else + aws iam create-role --role-name "$ROLE" --assume-role-policy-document "$TRUST" --output text >/dev/null + aws iam put-role-policy --role-name "$ROLE" --policy-name s3-access --policy-document "$POLICY" + echo " ✓ IAM role $ROLE created" + fi + done + # 6. Pod Identity associations + - cmd: | + PRESTAGE_ARN="arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RAY_ROLE_PRESTAGE}}" + WORKER_ARN="arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RAY_ROLE_WORKER}}" + for SA_INFO in "ray-system|model-prestage-sa|$PRESTAGE_ARN" "ray-system|ray-worker-sa|$WORKER_ARN" "kube-system|s3-csi-driver-sa|$WORKER_ARN"; do + NS="${SA_INFO%%|*}"; REST="${SA_INFO#*|}"; SA="${REST%%|*}"; ARN="${REST#*|}" + if aws eks list-pod-identity-associations --cluster-name {{.HUB_CLUSTER_NAME}} --namespace "$NS" --service-account "$SA" --region {{.AWS_REGION}} --query 'associations[0].associationId' --output text 2>/dev/null | grep -q "^a-"; then + echo " ✓ PIA $SA in $NS exists" + else + aws eks create-pod-identity-association --cluster-name {{.HUB_CLUSTER_NAME}} --namespace "$NS" --service-account "$SA" --role-arn "$ARN" --region {{.AWS_REGION}} --output text >/dev/null 2>&1 + echo " ✓ PIA $SA in $NS created" + fi + done + - echo "✓ Ray ML infrastructure ready (bucket={{.RAY_BUCKET}}, image={{.RAY_ECR_REPO}}:latest)" + + ray:prestage-models: + desc: "Download ML models to S3 bucket for Ray serving" + vars: + RAY_BUCKET: "{{.RESOURCE_PREFIX}}-ray-models-{{.AWS_ACCOUNT_ID}}" + cmds: + - cmd: | + pip3 install -q huggingface_hub 2>/dev/null + echo "Downloading TinyLlama..." + python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='TinyLlama/TinyLlama-1.1B-Chat-v1.0', local_dir='/tmp/tinyllama', local_dir_use_symlinks=False)" + echo "Uploading to s3://{{.RAY_BUCKET}}/models/tinyllama/..." + aws s3 sync /tmp/tinyllama/ s3://{{.RAY_BUCKET}}/models/tinyllama/ --region {{.AWS_REGION}} --no-progress + rm -rf /tmp/tinyllama + echo "✓ TinyLlama model uploaded" + hub:destroy-addons: desc: Remove all addons from the hub cluster (no git changes needed) vars: From 6b73f144ad92d034a8c2a8fe0d07df06c68668c7 Mon Sep 17 00:00:00 2001 From: Workshop User Date: Sun, 28 Jun 2026 16:54:07 +0000 Subject: [PATCH 070/231] fix(argo-workflows): disable SSO auth-mode in cloudfront exposure mode In cloudfront mode, argo-server cannot reach the OIDC discovery endpoint (CloudFront proxies to GitLab sign-in). Disable SSO and use client auth only. The workflow-controller (which runs CI) is unaffected. --- gitops/addons/charts/argo-workflows/templates/install.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gitops/addons/charts/argo-workflows/templates/install.yaml b/gitops/addons/charts/argo-workflows/templates/install.yaml index 3ae89c0f0..6bc9658ea 100644 --- a/gitops/addons/charts/argo-workflows/templates/install.yaml +++ b/gitops/addons/charts/argo-workflows/templates/install.yaml @@ -3109,7 +3109,9 @@ spec: - server - --configmap=workflow-controller-configmap - --auth-mode=client + {{- if ne (default "domain" .Values.exposure_mode) "cloudfront" }} - --auth-mode=sso + {{- end }} - "--base-href=/argo-workflows/" - "--secure=false" - "--loglevel" From a3c9b395d760875e523b404cce86303ef3c46222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 18:56:36 +0200 Subject: [PATCH 071/231] fix(workshop): add organization_id to CreateService, clean orphaned namespace GitLab CE 17+ requires organization_id in Users::CreateService. Also clean up orphaned user1 namespace from failed previous runs. --- cluster-providers/workshop/Taskfile.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 8bf22a2bd..db35904b0 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -234,11 +234,13 @@ tasks: pat.save! u = User.find_by(username: '{{.GIT_USERNAME}}') if u.nil? + Namespace.where(path: '{{.GIT_USERNAME}}', type: 'User').destroy_all svc = Users::CreateService.new(User.find(1), { name: 'Workshop User', username: '{{.GIT_USERNAME}}', email: '{{.GIT_USERNAME}}@workshop.local', password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', - skip_confirmation: true + skip_confirmation: true, + organization_id: (Organizations::Organization.find_by(id: 1)&.id || 1) }).execute u = svc.respond_to?(:payload) ? svc.payload[:user] : User.find_by(username: '{{.GIT_USERNAME}}') else From bdf01bb72ad347e4730001ac13c2951ec7305550 Mon Sep 17 00:00:00 2001 From: Workshop User Date: Sun, 28 Jun 2026 19:09:58 +0000 Subject: [PATCH 072/231] feat(ray): make image build non-blocking, add ray:wait-image task MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vLLM image build (5-10 min) now runs in the background during ray:setup so it doesn't block IAM/PIA setup. A new ray:wait-image task polls ECR until the image is available (10s intervals, 10min timeout). Workflow: task ray:setup → (other work) → task ray:wait-image → task ray:prestage-models --- .../kind-crossplane/Taskfile.yaml | 33 +++++++++++++++---- 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 89d6926cc..f14c68867 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -2825,17 +2825,17 @@ tasks: --image-tag-mutability MUTABLE --output text >/dev/null echo " ✓ ECR repo {{.RAY_ECR_REPO}} created" fi - # 3. Build and push vLLM image + # 3. Build and push vLLM image (background - non-blocking) - cmd: | IMAGE_URI="{{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com/{{.RAY_ECR_REPO}}:latest" if aws ecr describe-images --repository-name "{{.RAY_ECR_REPO}}" --image-ids imageTag=latest --region {{.AWS_REGION}} >/dev/null 2>&1; then echo " ✓ vLLM image already exists" else - echo " ▸ Building vLLM image (5-10 min)..." - aws ecr get-login-password --region {{.AWS_REGION}} | docker login --username AWS --password-stdin {{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com - docker build -t "$IMAGE_URI" -f "{{.DOCKERFILE_PATH}}" "$(dirname {{.DOCKERFILE_PATH}})" - docker push "$IMAGE_URI" - echo " ✓ vLLM image pushed" + echo " ▸ Building vLLM image in background (5-10 min)..." + aws ecr get-login-password --region {{.AWS_REGION}} | docker login --username AWS --password-stdin {{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com >/dev/null 2>&1 + nohup bash -c "docker build -t '$IMAGE_URI' -f '{{.DOCKERFILE_PATH}}' '$(dirname {{.DOCKERFILE_PATH}})' && docker push '$IMAGE_URI' && echo '[ray] ✓ vLLM image pushed' >> /tmp/ray-build.log" > /tmp/ray-build.log 2>&1 & + echo " ✓ Build started (PID $!, log: /tmp/ray-build.log)" + echo " Run 'task ray:wait-image' to wait for completion." fi # 4. S3 CSI driver addon - cmd: | @@ -2890,6 +2890,27 @@ tasks: rm -rf /tmp/tinyllama echo "✓ TinyLlama model uploaded" + + ray:wait-image: + desc: "Wait for the background vLLM image build to complete" + vars: + RAY_ECR_REPO: "{{.RESOURCE_PREFIX}}-ray-vllm-custom" + cmds: + - cmd: | + echo "Waiting for vLLM image build..." + for i in $(seq 1 60); do + if aws ecr describe-images --repository-name "{{.RAY_ECR_REPO}}" --image-ids imageTag=latest --region {{.AWS_REGION}} >/dev/null 2>&1; then + echo "✓ vLLM image ready in ECR" + exit 0 + fi + if [ -f /tmp/ray-build.log ]; then + tail -1 /tmp/ray-build.log + fi + sleep 10 + done + echo "✗ Timeout waiting for image (check /tmp/ray-build.log)" + exit 1 + hub:destroy-addons: desc: Remove all addons from the hub cluster (no git changes needed) vars: From 9b251fccf0c025389f6558b63d6b76c2e9d325d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 21:23:22 +0200 Subject: [PATCH 073/231] fix(workshop): sleep after project create for Sidekiq protections, fix ServiceResponse user lookup --- cluster-providers/workshop/Taskfile.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index db35904b0..4c7d07ba2 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -242,7 +242,7 @@ tasks: skip_confirmation: true, organization_id: (Organizations::Organization.find_by(id: 1)&.id || 1) }).execute - u = svc.respond_to?(:payload) ? svc.payload[:user] : User.find_by(username: '{{.GIT_USERNAME}}') + u = svc.respond_to?(:payload) ? (svc.payload[:user] || User.find_by(username: '{{.GIT_USERNAME}}')) : User.find_by(username: '{{.GIT_USERNAME}}') else u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) @@ -262,6 +262,7 @@ tasks: p = Project.find_by(path: repo, namespace_id: u.namespace_id) if p.nil? p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: false).execute + sleep 1 end ProtectedBranch.where(project: p).destroy_all if p.persisted? puts 'repo: ' + repo + ' ok=' + p.persisted?.to_s From 497e1f41993be75ce08588085f429085072c8ff0 Mon Sep 17 00:00:00 2001 From: Workshop User Date: Sun, 28 Jun 2026 19:25:14 +0000 Subject: [PATCH 074/231] feat(workshop): add workshop:install as single entry point Orchestrates the full flow: 1. Platform install (kind-crossplane:install) 2. GitLab init (create repos, seed content) 3. Ray ML infra (image builds in background) 4. IDC/Keycloak federation 5. Wait for Ray image 6. Download ML models to S3 Usage: task workshop:install --- cluster-providers/workshop/Taskfile.yaml | 28 ++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 4c7d07ba2..21a627827 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -42,6 +42,34 @@ vars: sh: printf '\033[0m' tasks: + # ============================================================ + # FULL WORKSHOP INSTALL + # Single entry point: task workshop:install + # Orchestrates platform install + workshop-specific setup + # ============================================================ + + install: + desc: "Full workshop install (platform + GitLab + Ray + IDC federation)" + vars: + PROVIDER: + sh: yq '.clusterProvider // "kind-crossplane"' {{.CONFIG_FILE}} + cmds: + - printf '{{.C_STEP}}▸ Workshop install starting...{{.C_RESET}}\n' + # 1. Platform install (EKS cluster, addons, spokes) + - task: ":{{.PROVIDER}}:install" + - printf '{{.C_OK}}✓ Platform install complete.{{.C_RESET}}\n' + # 2. GitLab init (create repos, seed content) + - task: gitlab:init-ec2 + # 3. Ray ML infrastructure (image builds in background) + - task: ":{{.PROVIDER}}:ray:setup" + # 4. IDC/Keycloak federation (ArgoCD SSO) + - task: post-install + # 5. Wait for Ray image to be ready + - task: ":{{.PROVIDER}}:ray:wait-image" + # 6. Download ML models to S3 + - task: ":{{.PROVIDER}}:ray:prestage-models" + - printf '{{.C_OK}}✓ Workshop install complete.{{.C_RESET}}\n' + # ============================================================ # PRE-REQUISITES # Run before task install to prepare workshop-specific resources From 213c006e9c31c59ece122e72fa5c632e62f58ad7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 21:26:26 +0200 Subject: [PATCH 075/231] fix(workshop): unprotect branches via API before push (handles Sidekiq async) --- cluster-providers/workshop/Taskfile.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 21a627827..e0d8952ba 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -298,8 +298,14 @@ tasks: puts 'init done' " 2>&1 | tail -20 || true WORKSHOP_DIR="$HOME/environment/{{.WORKING_REPO}}" + ROOT_TOKEN=$(docker exec gitlab-gitlab-1 gitlab-rails runner "puts User.find(1).personal_access_tokens.find_by(name: 'initial-root-token')&.token.to_s" 2>/dev/null | tail -1) for REPO in {{.REPOS}}; do REMOTE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/$REPO.git" + # Unprotect via API (synchronous, handles Sidekiq async protection) + if [ -n "$ROOT_TOKEN" ]; then + curl -sf -X DELETE -H "PRIVATE-TOKEN: $ROOT_TOKEN" \ + "https://{{.GITLAB_CF_DOMAIN}}/api/v4/projects/{{.GIT_USERNAME}}%2F$REPO/protected_branches/main" 2>/dev/null || true + fi if [ "$REPO" = "fleet-config" ]; then SRC="$WORKSHOP_DIR/gitops/fleet" else From 348b93b0b503a95db2eda035a1e2291ff47873eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 21:29:14 +0200 Subject: [PATCH 076/231] fix(workshop): create root-api PAT on-demand for API unprotect calls --- cluster-providers/workshop/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index e0d8952ba..f7fbd5b6b 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -298,7 +298,7 @@ tasks: puts 'init done' " 2>&1 | tail -20 || true WORKSHOP_DIR="$HOME/environment/{{.WORKING_REPO}}" - ROOT_TOKEN=$(docker exec gitlab-gitlab-1 gitlab-rails runner "puts User.find(1).personal_access_tokens.find_by(name: 'initial-root-token')&.token.to_s" 2>/dev/null | tail -1) + ROOT_TOKEN=$(docker exec gitlab-gitlab-1 gitlab-rails runner "puts User.find(1).personal_access_tokens.find_or_initialize_by(name: 'root-api').tap{|p| p.scopes=['api','sudo']; p.expires_at=365.days.from_now; p.save!}.token" 2>/dev/null | tail -1) for REPO in {{.REPOS}}; do REMOTE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/$REPO.git" # Unprotect via API (synchronous, handles Sidekiq async protection) From d35c7f70e3af46f990694f6262faec028b608473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 21:57:43 +0200 Subject: [PATCH 077/231] fix(workshop): skip push if repo already has commits (idempotent init) --- cluster-providers/workshop/Taskfile.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index f7fbd5b6b..c9e87bd44 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -317,7 +317,12 @@ tasks: cd "$TMP" git init -q && git add -A && git commit -qm "init" 2>/dev/null || true git remote add origin "$REMOTE_URL" - git push -f origin HEAD:main + REMOTE_EMPTY=$(git ls-remote "$REMOTE_URL" HEAD 2>/dev/null | wc -l | tr -d ' ') + if [ "$REMOTE_EMPTY" = "0" ]; then + git push -f origin HEAD:main + else + echo " skipped (already populated): $REPO" + fi cd - >/dev/null; rm -rf "$TMP" echo " seeded: $REPO" done From 4dee984aec02ee3df2b64118bdf8c45b656ecb7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sun, 28 Jun 2026 22:03:35 +0200 Subject: [PATCH 078/231] fix(jupyterhub): disable upstream chart ingress to avoid duplicate Ingress resource --- gitops/addons/charts/jupyterhub/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gitops/addons/charts/jupyterhub/values.yaml b/gitops/addons/charts/jupyterhub/values.yaml index 47a58e5ee..cdfb5dfd6 100644 --- a/gitops/addons/charts/jupyterhub/values.yaml +++ b/gitops/addons/charts/jupyterhub/values.yaml @@ -1,3 +1,6 @@ global: ingress_domain_name: "" clusterName: "" +jupyterhub: + ingress: + enabled: false From b9f98a9571955fc7ab4e193aae7bc9324f083a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 10:25:37 +0200 Subject: [PATCH 079/231] fix: always set user1 password via Devise::Encryptor after creation (CreateService may not hash correctly) --- cluster-providers/kind-crossplane/Taskfile.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index f14c68867..73df1525e 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -819,7 +819,9 @@ tasks: skip_confirmation: true }).execute u = User.find_by(username: '{{.GIT_USERNAME}}') - else + end + # Always ensure password is correctly hashed (CreateService may fail silently) + if u u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end From 56e5629489ee17ce6098f4f9dfb87359316e9c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 10:32:44 +0200 Subject: [PATCH 080/231] fix: same password fix in workshop/Taskfile.yaml gitlab:init-ec2 --- cluster-providers/workshop/Taskfile.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index c9e87bd44..88c6cffaa 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -271,7 +271,9 @@ tasks: organization_id: (Organizations::Organization.find_by(id: 1)&.id || 1) }).execute u = svc.respond_to?(:payload) ? (svc.payload[:user] || User.find_by(username: '{{.GIT_USERNAME}}')) : User.find_by(username: '{{.GIT_USERNAME}}') - else + end + # Always ensure password is correctly hashed (CreateService may not hash correctly) + if u u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end From 41c1e91e64923d1cdcd0ea77077a107d8043ebff Mon Sep 17 00:00:00 2001 From: Workshop User Date: Mon, 29 Jun 2026 08:38:59 +0000 Subject: [PATCH 081/231] fix(spokes): use deterministic CIDR based on cluster name instead of random MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Random CIDRs break idempotency: re-running install generates a different CIDR each time, causing VPC recreation attempts on immutable resources. Use cksum of cluster name → stable octet (2-254). Same cluster name always produces the same CIDR across runs. --- cluster-providers/kind-crossplane/Taskfile.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 73df1525e..dfb341ac4 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -1221,7 +1221,7 @@ tasks: clusters: {{.CLUSTER}}: clusterName: {{.CLUSTER}} - vpcCidr: "10.$(shuf -i 1-254 -n 1).0.0/16" + vpcCidr: "10.$(printf '%s' '{{.CLUSTER}}' | cksum | awk '{print ($1 % 253) + 2}').0.0/16" kubernetesVersion: "{{.K8S_VERSION}}" autoMode: true resourcePrefix: {{.RESOURCE_PREFIX}} @@ -1403,17 +1403,13 @@ tasks: cat > "$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/clusters/{{.CLUSTER}}.json" < InvalidSubnet.Range). - # Idempotency: only generate a (new random) CIDR when this spoke is not - # already enabled. Re-running install must NOT churn the CIDR — an EC2 VPC - # cidrBlock is immutable, so a new octet would force VPC recreation/orphans. + # Deterministic second octet derived from cluster name (stable across re-runs). + # Uses cksum to hash the name into a value 2-254. KRO_VALUES="$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/values.yaml" if [ -f "$KRO_VALUES" ] && yq -e '.clusters."{{.CLUSTER}}".cidr.vpcCidr' "$KRO_VALUES" >/dev/null 2>&1; then echo " Spoke {{.CLUSTER}} already has a CIDR — keeping existing values (idempotent)." else - OCTET=$(shuf -i 2-254 -n 1) + OCTET=$(printf '%s' '{{.CLUSTER}}' | cksum | awk '{print ($1 % 253) + 2}') cat > "$KRO_VALUES" < Date: Mon, 29 Jun 2026 10:40:25 +0200 Subject: [PATCH 082/231] fix: pass GIT_PASSWORD/GIT_USERNAME as env vars to rails runner (avoids shell quoting issues with special chars) --- cluster-providers/kind-crossplane/Taskfile.yaml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index dfb341ac4..883752e8e 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -798,7 +798,7 @@ tasks: # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash # before the string is passed to gitlab-rails runner as a Ruby script. - docker exec gitlab-gitlab-1 gitlab-rails runner " + docker exec -e GIT_PASSWORD="{{.GIT_PASSWORD}}" -e GIT_USERNAME="{{.GIT_USERNAME}}" gitlab-gitlab-1 gitlab-rails runner " # Disable default branch protection + enable password auth for git s = ApplicationSetting.first s.password_authentication_enabled_for_git = true @@ -809,20 +809,22 @@ tasks: pat.scopes = ['api','write_repository','sudo'] pat.expires_at = 365.days.from_now pat.save! - u = User.find_by(username: '{{.GIT_USERNAME}}') + pwd = ENV['GIT_PASSWORD'] + uname = ENV['GIT_USERNAME'] + u = User.find_by(username: uname) if u.nil? # Use Users::CreateService which properly sets up the namespace svc = Users::CreateService.new(User.find(1), { - name: 'Workshop User', username: '{{.GIT_USERNAME}}', - email: '{{.GIT_USERNAME}}@workshop.local', - password: '{{.GIT_PASSWORD}}', password_confirmation: '{{.GIT_PASSWORD}}', + name: 'Workshop User', username: uname, + email: uname + '@workshop.local', + password: pwd, password_confirmation: pwd, skip_confirmation: true }).execute - u = User.find_by(username: '{{.GIT_USERNAME}}') + u = User.find_by(username: uname) end # Always ensure password is correctly hashed (CreateService may fail silently) if u - u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') + u.encrypted_password = Devise::Encryptor.digest(User, pwd) u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) end # Ensure namespace exists (GitLab CE requires Organization; use default org) From 6c3c13086781b8aa58f7b7abb494b51dc2e71edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 10:44:20 +0200 Subject: [PATCH 083/231] refactor: remove gitlab:init-ec2 + gitlab:clone-repos from kind-crossplane (workshop-specific, lives in workshop/Taskfile.yaml) --- .../kind-crossplane/Taskfile.yaml | 213 ------------------ 1 file changed, 213 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 883752e8e..404f00af9 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -157,7 +157,6 @@ tasks: deps: - hub:restart-langfuse - urls - - gitlab:init-ec2 install:phase1-parallel: internal: true @@ -765,218 +764,6 @@ tasks: printf ' Keycloak: admin / {{.KC_ADMIN_PASSWORD}}\n' - gitlab:init-ec2: - desc: Wait for GitLab CE (docker-compose on IDE EC2) to be ready, then initialise root token + user1 + repos - vars: - GITLAB_CF_DOMAIN: - sh: | - if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then - cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain - else - echo "{{.DOMAIN}}" - fi - GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // "{{.HUB_CLUSTER_NAME}}"') - GIT_USERNAME: - sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} - REPOS: - sh: echo "fleet-config java rust golang dotnet next-js" - cmds: - - printf '{{.C_STEP}}▸ Waiting for GitLab CE on EC2 to be ready...{{.C_RESET}}\n' - - | - set -e - # GitLab CE starts in docker-compose at EC2 boot (bootstrap.sh). Wait up to 20min. - GITLAB_URL="https://{{.GITLAB_CF_DOMAIN}}" - for i in $(seq 1 120); do - if curl -sf --max-time 10 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1; then - printf ' GitLab ready.\n'; break - fi - [ "$i" -eq 120 ] && { printf ' ⚠ GitLab not ready after 20min — continuing anyway.\n'; } - sleep 10 - done - ROOT_TOKEN="root-{{.GIT_PASSWORD}}" - # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. - # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash - # before the string is passed to gitlab-rails runner as a Ruby script. - docker exec -e GIT_PASSWORD="{{.GIT_PASSWORD}}" -e GIT_USERNAME="{{.GIT_USERNAME}}" gitlab-gitlab-1 gitlab-rails runner " - # Disable default branch protection + enable password auth for git - s = ApplicationSetting.first - s.password_authentication_enabled_for_git = true - s.default_branch_protection = 0 - s.default_branch_protection_defaults = {allowed_to_push: [{access_level: 0}], allow_force_push: true, allowed_to_merge: [{access_level: 0}], developer_can_initial_push: true} - s.save(validate: false) - pat = User.find(1).personal_access_tokens.find_or_initialize_by(name: 'initial-root-token') - pat.scopes = ['api','write_repository','sudo'] - pat.expires_at = 365.days.from_now - pat.save! - pwd = ENV['GIT_PASSWORD'] - uname = ENV['GIT_USERNAME'] - u = User.find_by(username: uname) - if u.nil? - # Use Users::CreateService which properly sets up the namespace - svc = Users::CreateService.new(User.find(1), { - name: 'Workshop User', username: uname, - email: uname + '@workshop.local', - password: pwd, password_confirmation: pwd, - skip_confirmation: true - }).execute - u = User.find_by(username: uname) - end - # Always ensure password is correctly hashed (CreateService may fail silently) - if u - u.encrypted_password = Devise::Encryptor.digest(User, pwd) - u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) - end - # Ensure namespace exists (GitLab CE requires Organization; use default org) - if u.namespace_id.nil? - org_id = Organizations::Organization.find_by(id: 1)&.id || 1 - ns = Namespace.new(owner: u, path: u.username, name: u.username, type: 'User', organization_id: org_id) - ns.save(validate: false) - u.update_columns(namespace_id: ns.id) if ns.persisted? - u.reload - end - u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') - u1pat.scopes = ['api','write_repository'] - u1pat.expires_at = 365.days.from_now - u1pat.save! - # Output the plaintext token on creation (only available once) - puts 'GIT_TOKEN=' + (u1pat.token || '') - %w[fleet-config java rust golang dotnet next-js].each do |repo| - p = Project.find_by(path: repo, namespace_id: u.namespace_id) - if p.nil? - p = Projects::CreateService.new(u, name: repo, path: repo, visibility_level: 0, initialize_with_readme: false).execute - end - ProtectedBranch.where(project: p).destroy_all if p.persisted? - puts 'repo: ' + repo + ' ok=' + p.persisted?.to_s - end - puts 'init done' - " 2>&1 | tail -20 || true - # Capture user1 PAT and update git_token in peeks-hub/secrets for webhook creation - GIT_TOKEN=$(docker exec gitlab-gitlab-1 gitlab-rails runner ' - u = User.find_by(username: "user1") - pat = u&.personal_access_tokens&.find_by(name: "workshop-token") - puts pat ? pat.token.to_s : "" - ' 2>/dev/null | tail -1) - if [ -n "$GIT_TOKEN" ]; then - SECRET_KEY="{{.HUB_CLUSTER_NAME}}/secrets" - EXISTING=$(aws secretsmanager get-secret-value --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} --query SecretString --output text 2>/dev/null || echo "{}") - UPDATED=$(echo "$EXISTING" | python3 -c "import json,sys; d=json.load(sys.stdin); d['git_token']='$GIT_TOKEN'; print(json.dumps(d))" 2>/dev/null) - [ -n "$UPDATED" ] && aws secretsmanager put-secret-value --secret-id "$SECRET_KEY" --secret-string "$UPDATED" --region {{.AWS_REGION}} >/dev/null 2>&1 && printf ' git_token updated in peeks-hub/secrets\n' || true - fi - WORKSHOP_DIR="$HOME/environment/{{.WORKING_REPO}}" - for REPO in {{.REPOS}}; do - REMOTE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}/$REPO.git" - if [ "$REPO" = "fleet-config" ]; then - SRC="$WORKSHOP_DIR/gitops/fleet" - else - SRC="$WORKSHOP_DIR/applications/$REPO" - fi - [ -d "$SRC" ] || continue - TMP=$(mktemp -d) - cp -r "$SRC/." "$TMP/" - # Also seed Backstage templates into fleet-config so Backstage catalog works in cloudfront mode - if [ "$REPO" = "fleet-config" ] && [ -d "$WORKSHOP_DIR/platform/backstage/templates" ]; then - mkdir -p "$TMP/platform/backstage" - cp -r "$WORKSHOP_DIR/platform/backstage/templates" "$TMP/platform/backstage/" - fi - cd "$TMP" - git init -q && git add -A && git commit -qm "init" 2>/dev/null || true - git remote add origin "$REMOTE_URL" - git push -f origin HEAD:main - cd - >/dev/null; rm -rf "$TMP" - echo " seeded: $REPO" - done - printf '{{.C_OK}}✓ GitLab init done.{{.C_RESET}}\n' - # After init, clone repos locally - - task: gitlab:clone-repos - - gitlab:clone-repos: - desc: Clone all GitLab repos locally into ~/environment - vars: - GITLAB_CF_DOMAIN: - sh: | - if [ -f {{.ROOT_DIR}}/private/gitlab-cloudfront-domain ]; then - cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain - else - echo "{{.DOMAIN}}" - fi - GIT_USERNAME: - sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} - GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // "{{.HUB_CLUSTER_NAME}}"') - REPOS: - sh: echo "fleet-config java rust golang dotnet next-js" - cmds: - - printf '{{.C_STEP}}▸ Cloning GitLab repos to ~/environment...{{.C_RESET}}\n' - - | - BASE_URL="https://{{.GIT_USERNAME}}:{{.GIT_PASSWORD}}@{{.GITLAB_CF_DOMAIN}}/{{.GIT_USERNAME}}" - FAILED="" - # Wait for GitLab (running via docker-compose on the IDE EC2) to be ready - # and for the init script to have created all repos. - # bootstrap.sh starts GitLab at boot; task install calls gitlab:init-ec2 first, - # so by the time clone-repos runs the repos should exist. Poll as a safety net. - printf ' Waiting for GitLab to be ready (up to 20min)...\n' - GITLAB_BASE="https://{{.GITLAB_CF_DOMAIN}}" - for i in $(seq 1 120); do - if curl -sf --max-time 5 "$GITLAB_BASE/users/sign_in" >/dev/null 2>&1; then break; fi - sleep 10 - done - for REPO in {{.REPOS}}; do - if [ "$REPO" = "fleet-config" ]; then - DEST="$HOME/environment/$REPO" - else - DEST="$HOME/environment/applications/$REPO" - fi - REPO_URL="$BASE_URL/$REPO.git" - if [ -d "$DEST/.git" ]; then - # Already cloned. The gitlab-init Job creates each repo with only a - # README first, then pushes the real source/overlay content later in - # the same Job. An earlier clone-repos run (or this one racing ahead) - # may therefore have captured the README-only state. Pull to pick up - # any content pushed after the initial clone — fast-forward only so we - # never clobber participant commits. - echo " $REPO already cloned — pulling latest" - git -C "$DEST" pull --ff-only -q 2>/dev/null || echo " ($REPO: pull skipped — local changes or no upstream)" - continue - fi - # GitLab repos are created asynchronously by the gitlab-init Job (Helm - # post-install hook, which waits for GitLab readiness). Without waiting, - # this task ran before the repos existed and every clone failed, leaving - # ~/environment/applications empty and no local fleet-config checkout. - # Wait (up to ~5min) for each repo to be reachable AND seeded before - # cloning. Reachable alone is not enough: the repo exists with only a - # README the instant it is created, but the application source is pushed - # a few seconds later. Cloning on first-reachable captured the stub - # (README/.git only), leaving e.g. ~/environment/applications/next-js - # without deployable code. Wait until the remote has a non-README file - # (fleet-config is exempt — it is content-seeded differently). - REACHABLE=false - for i in $(seq 1 18); do - if git ls-remote "$REPO_URL" >/dev/null 2>&1; then - REACHABLE=true; break - fi - sleep 10 - done - if [ "$REACHABLE" != true ]; then - echo " ⚠ $REPO not reachable/seeded after 5min — skipping (create later)." - FAILED="$FAILED $REPO" - continue - fi - mkdir -p "$(dirname "$DEST")" - if git clone "$REPO_URL" "$DEST" 2>/dev/null; then - echo " ✓ $REPO cloned" - else - echo " ⚠ $REPO clone failed" - FAILED="$FAILED$REPO " - fi - done - if [ -n "$FAILED" ]; then - printf '{{.C_ERR}}⚠ Some GitLab repos were not cloned:%s{{.C_RESET}}\n' "$FAILED" - printf '{{.C_INFO}} Re-run later: task kind-crossplane:gitlab:clone-repos{{.C_RESET}}\n' - else - printf '{{.C_OK}}✓ GitLab repos cloned.{{.C_RESET}}\n' - fi - hub:set-overlay-repo: desc: >- Wire overlay_repo_url -> GitLab fleet-config on the hub cluster config so From 29c90ad9d4b741ac3d8bec6c0d6c940f26f2ba49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 10:50:51 +0200 Subject: [PATCH 084/231] refactor: remove gitlab references from kind-crossplane + kind-kro-ack providers - kind-crossplane: remove gitlab:init-ec2 from phase1b-parallel - kind-kro-ack: remove gitlab:init-ec2 wrapper from install flow - workshop: fix stale kind-crossplane:gitlab:clone-repos reference gitlab tasks are workshop-specific and called by workshop:install, not the providers --- cluster-providers/kind-kro-ack/Taskfile.yaml | 6 ------ cluster-providers/workshop/Taskfile.yaml | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index c3e3f0ed5..6ce0d1006 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -147,12 +147,6 @@ tasks: deps: - hub:restart-langfuse - urls - - gitlab:init-ec2 - - gitlab:init-ec2: - desc: Wait for GitLab CE on IDE EC2 to be ready and initialize (delegates to workshop provider) - cmds: - - cd {{.ROOT_DIR}} && task workshop:gitlab:init-ec2 install:phase2-parallel: internal: true diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 88c6cffaa..f34fe5595 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -415,7 +415,7 @@ tasks: done if [ -n "$FAILED" ]; then printf '{{.C_ERR}}⚠ Some GitLab repos were not cloned:%s{{.C_RESET}}\n' "$FAILED" - printf '{{.C_INFO}} Re-run later: task kind-crossplane:gitlab:clone-repos{{.C_RESET}}\n' + printf '{{.C_INFO}} Re-run later: task workshop:gitlab:clone-repos{{.C_RESET}}\n' else printf '{{.C_OK}}✓ GitLab repos cloned.{{.C_RESET}}\n' fi From 53d1bd2d0e1a19fe8326c7b6046c3b47f080562f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 10:52:42 +0200 Subject: [PATCH 085/231] fix: unprotect branches after seed push (seed creates main which triggers auto-protection) --- cluster-providers/workshop/Taskfile.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index f34fe5595..92403fd78 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -329,6 +329,11 @@ tasks: echo " seeded: $REPO" done printf '{{.C_OK}}✓ GitLab init done.{{.C_RESET}}\n' + # Unprotect branches again after seeding (seed push creates main which triggers auto-protection) + docker exec gitlab-gitlab-1 gitlab-rails runner " + u = User.find_by(username: ENV['GIT_USERNAME'] || 'user1') + Project.where(namespace_id: u.namespace_id).each{|p| ProtectedBranch.where(project: p).destroy_all} + " 2>/dev/null || true # After init, clone repos locally - task: gitlab:clone-repos From a78d15c14c8d43fbf52a3e4d38b3eecfc3de8434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 10:54:27 +0200 Subject: [PATCH 086/231] docs: add README.md to cluster-providers/workshop/ --- cluster-providers/workshop/README.md | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 cluster-providers/workshop/README.md diff --git a/cluster-providers/workshop/README.md b/cluster-providers/workshop/README.md new file mode 100644 index 000000000..b5f6eb59e --- /dev/null +++ b/cluster-providers/workshop/README.md @@ -0,0 +1,71 @@ +# Workshop Taskfile + +Workshop-specific tasks for the **Platform Engineering on EKS (PEEKS)** workshop. +These tasks are NOT part of the generic platform installation — they handle resources +and configuration specific to the workshop environment (GitLab CE on EC2, IDC/Keycloak +federation, Ray ML infrastructure, etc.). + +## Entry Point + +```bash +task workshop:install +``` + +This is the single entry point for a full workshop deployment. It orchestrates: + +1. **`workshop:pre-install`** — Validate pre-requisites (CloudFront domain set in `config.local.yaml`) +2. **`{{PROVIDER}}:install`** — Platform install (hub EKS cluster, addons, spokes via `kind-crossplane` or `kind-kro-ack`) +3. **`workshop:gitlab:init-ec2`** — Initialize GitLab CE on EC2 (create user1, repos, seed content) +4. **`workshop:ray:*`** (optional) — Ray ML infrastructure setup +5. **`workshop:post-install`** — IDC/Keycloak federation for ArgoCD SSO + +The `PROVIDER` is read from `config.local.yaml` (`clusterProvider` field, default `kind-crossplane`). + +## Task Reference + +### Installation + +| Task | Description | +|------|-------------| +| `task workshop:install` | Full workshop install (entry point) | +| `task workshop:pre-install` | Validate CloudFront pre-requisites | +| `task workshop:post-install` | IDC/Keycloak federation (ArgoCD SSO) | + +### GitLab CE (on EC2) + +| Task | Description | +|------|-------------| +| `task workshop:gitlab:init-ec2` | Wait for GitLab ready, create user1 + repos, seed content | +| `task workshop:gitlab:clone-repos` | Clone all GitLab repos to `~/environment/` | + +### IDC / Keycloak + +| Task | Description | +|------|-------------| +| `task workshop:idc:setup` | Create IDC groups and users (idempotent) | +| `task workshop:idc:configure-keycloak` | Configure Keycloak as external IdP in IAM Identity Center | + +## Configuration + +All tasks read from `config.local.yaml` (generated by `create-config.sh`). + +Key values used by workshop tasks: + +| Field | Description | +|-------|-------------| +| `clusterProvider` | `kind-crossplane` (default) or `kind-kro-ack` | +| `cloudfront.cloudfrontDomain` | CloudFront distribution domain (IDE + GitLab) | +| `identityCenter.instanceArn` | IAM Identity Center instance ARN | +| `identityCenter.adminGroupId` | IDC admin group ID for ArgoCD access | + +## Context + +In Workshop Studio (guided events), most pre-requisites are provisioned by CDK: +- CloudFront + ALB → `cdk/resources/bootstrap.sh` +- GitLab CE docker-compose → `cdk/resources/bootstrap.sh` +- `config.local.yaml` → generated by `create-config.sh` in the SSM SetupIDE script + +The SSM script calls `task workshop:install` which handles the rest. + +For self-paced deployments, run `./create-config.sh` first to generate `config.local.yaml`, +ensuring `cloudfront.cloudfrontDomain` is set. From 285981e3ff345509a59d7437614c207f30362b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 11:00:19 +0200 Subject: [PATCH 087/231] fix: run gitlab-ctl reconfigure after first boot to ensure workhorse/gitaly secrets are consistent Without reconfigure, internal/allowed returns 401 causing misleading 'protected branch' errors on all git pushes. This is the root cause for all git push failures on fresh deployments. --- cluster-providers/workshop/Taskfile.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 92403fd78..6329af0dd 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -245,6 +245,14 @@ tasks: [ "$i" -eq 120 ] && { printf ' ⚠ GitLab not ready after 20min — continuing anyway.\n'; } sleep 10 done + # Run reconfigure to ensure internal secrets (workhorse/gitaly-hooks) are consistent. + # On first boot these can be mismatched, causing git push pre-receive hooks to fail. + docker exec gitlab-gitlab-1 gitlab-ctl reconfigure 2>/dev/null | tail -3 || true + # Wait for GitLab to be ready again after reconfigure + for i in $(seq 1 30); do + curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && break + sleep 10 + done ROOT_TOKEN="root-{{.GIT_PASSWORD}}" # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash From 65692bda9b2f04dc9a89d05afbc8b3bd2b46c77f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 11:28:22 +0200 Subject: [PATCH 088/231] fix: call workshop:gitlab:init-ec2 from phase1b-parallel (correct timing - after hub sync, before overlay wiring) --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 ++ cluster-providers/kind-kro-ack/Taskfile.yaml | 2 ++ cluster-providers/workshop/Taskfile.yaml | 14 +++++++------- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 404f00af9..c8a30ebd3 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -157,6 +157,8 @@ tasks: deps: - hub:restart-langfuse - urls + cmds: + - task: ":workshop:gitlab:init-ec2" install:phase1-parallel: internal: true diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 6ce0d1006..e16116e3f 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -147,6 +147,8 @@ tasks: deps: - hub:restart-langfuse - urls + cmds: + - task: ":workshop:gitlab:init-ec2" install:phase2-parallel: internal: true diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 6329af0dd..ba62f1e15 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -55,18 +55,18 @@ tasks: sh: yq '.clusterProvider // "kind-crossplane"' {{.CONFIG_FILE}} cmds: - printf '{{.C_STEP}}▸ Workshop install starting...{{.C_RESET}}\n' - # 1. Platform install (EKS cluster, addons, spokes) + # Platform install (EKS cluster, addons, spokes) + # Note: gitlab:init-ec2 is called from within {{.PROVIDER}}:install:phase1b-parallel + # via workshop:gitlab:init-ec2, so it runs at the right time (after hub sync, before overlay wiring) - task: ":{{.PROVIDER}}:install" - printf '{{.C_OK}}✓ Platform install complete.{{.C_RESET}}\n' - # 2. GitLab init (create repos, seed content) - - task: gitlab:init-ec2 - # 3. Ray ML infrastructure (image builds in background) + # Ray ML infrastructure (image builds in background) - task: ":{{.PROVIDER}}:ray:setup" - # 4. IDC/Keycloak federation (ArgoCD SSO) + # IDC/Keycloak federation (ArgoCD SSO) - task: post-install - # 5. Wait for Ray image to be ready + # Wait for Ray image to be ready - task: ":{{.PROVIDER}}:ray:wait-image" - # 6. Download ML models to S3 + # Download ML models to S3 - task: ":{{.PROVIDER}}:ray:prestage-models" - printf '{{.C_OK}}✓ Workshop install complete.{{.C_RESET}}\n' From b49938c16224abd6afc3f3df0ec9457293f3c992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 11:51:05 +0200 Subject: [PATCH 089/231] fix: run gitlab:init-ec2 BEFORE provider:install (GitLab ready at boot, fleet-config needed during phase2) --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 -- cluster-providers/kind-kro-ack/Taskfile.yaml | 2 -- cluster-providers/workshop/Taskfile.yaml | 6 ++++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index c8a30ebd3..404f00af9 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -157,8 +157,6 @@ tasks: deps: - hub:restart-langfuse - urls - cmds: - - task: ":workshop:gitlab:init-ec2" install:phase1-parallel: internal: true diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index e16116e3f..6ce0d1006 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -147,8 +147,6 @@ tasks: deps: - hub:restart-langfuse - urls - cmds: - - task: ":workshop:gitlab:init-ec2" install:phase2-parallel: internal: true diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index ba62f1e15..6d2f77e88 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -55,9 +55,11 @@ tasks: sh: yq '.clusterProvider // "kind-crossplane"' {{.CONFIG_FILE}} cmds: - printf '{{.C_STEP}}▸ Workshop install starting...{{.C_RESET}}\n' + # Initialize GitLab FIRST — it starts at EC2 boot and is ready before platform install. + # This ensures fleet-config exists when hub:set-overlay-repo and spokes run inside + # {{.PROVIDER}}:install (which needs fleet-config during phase2). + - task: gitlab:init-ec2 # Platform install (EKS cluster, addons, spokes) - # Note: gitlab:init-ec2 is called from within {{.PROVIDER}}:install:phase1b-parallel - # via workshop:gitlab:init-ec2, so it runs at the right time (after hub sync, before overlay wiring) - task: ":{{.PROVIDER}}:install" - printf '{{.C_OK}}✓ Platform install complete.{{.C_RESET}}\n' # Ray ML infrastructure (image builds in background) From 8bbd6d228bda5ee80f48634aeb078e401e8e33a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 14:02:21 +0200 Subject: [PATCH 090/231] =?UTF-8?q?fix:=20auto-detect=20and=20fix=20broken?= =?UTF-8?q?=20GitLab=20internal/allowed=20=E2=80=94=20wipe+restart=20if=20?= =?UTF-8?q?push=20test=20fails=20after=20reconfigure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/workshop/Taskfile.yaml | 25 ++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 6d2f77e88..b2e120b26 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -255,6 +255,31 @@ tasks: curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && break sleep 10 done + # Test if git HTTP push works (internal/allowed check). If not, wipe and restart. + # This detects the workhorse/gitaly-hooks secret mismatch that survives reconfigure. + _TEST_REPO=$(mktemp -d) + git init -q "$_TEST_REPO" && cd "$_TEST_REPO" + git commit --allow-empty -qm "test" 2>/dev/null || true + git remote add origin "http://root:{{.GIT_PASSWORD}}@localhost:8929/root/test-push-check.git" 2>/dev/null || true + # Create a test repo and push to it as root + docker exec gitlab-gitlab-1 gitlab-rails runner " + r = Project.find_by_full_path('root/test-push-check') + r ||= Projects::CreateService.new(User.find(1), name: 'test-push-check', path: 'test-push-check', visibility_level: 0, initialize_with_readme: false).execute + puts r.persisted? ? 'ok' : 'fail' + " 2>/dev/null | grep -q ok && \ + git push -f origin HEAD:main >/dev/null 2>&1 + PUSH_OK=$? + cd - >/dev/null; rm -rf "$_TEST_REPO" + docker exec gitlab-gitlab-1 gitlab-rails runner "Project.find_by_full_path('root/test-push-check')&.destroy" 2>/dev/null || true + if [ "$PUSH_OK" -ne 0 ]; then + printf '{{.C_ERR}}⚠ GitLab internal/allowed broken — wiping and restarting...{{.C_RESET}}\n' + cd /home/ec2-user/gitlab && docker compose down -v && docker compose up -d + printf ' Waiting for GitLab to reinitialize (~10min)...\n' + for i in $(seq 1 80); do + curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && printf ' GitLab ready after reinit\n' && break + sleep 10 + done + fi ROOT_TOKEN="root-{{.GIT_PASSWORD}}" # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash From 4af3b5267ce7b53b836963427b086e267c6d366e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 14:35:45 +0200 Subject: [PATCH 091/231] fix: use set +e/set -e around docker compose down+up to prevent set -e killing the restart --- cluster-providers/workshop/Taskfile.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index b2e120b26..b88d0039a 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -273,12 +273,14 @@ tasks: docker exec gitlab-gitlab-1 gitlab-rails runner "Project.find_by_full_path('root/test-push-check')&.destroy" 2>/dev/null || true if [ "$PUSH_OK" -ne 0 ]; then printf '{{.C_ERR}}⚠ GitLab internal/allowed broken — wiping and restarting...{{.C_RESET}}\n' + set +e cd /home/ec2-user/gitlab && docker compose down -v && docker compose up -d printf ' Waiting for GitLab to reinitialize (~10min)...\n' for i in $(seq 1 80); do curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && printf ' GitLab ready after reinit\n' && break sleep 10 done + set -e fi ROOT_TOKEN="root-{{.GIT_PASSWORD}}" # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. From aff99be5a69761df52687679a4ac8275e66caf67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 14:41:49 +0200 Subject: [PATCH 092/231] feat: add workshop:setup-env task to write ARGOCD_URL, BACKSTAGE_URL etc. to ~/.bashrc.d/platform.sh --- cluster-providers/workshop/Taskfile.yaml | 43 ++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index b88d0039a..c6ba550c2 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -70,6 +70,8 @@ tasks: - task: ":{{.PROVIDER}}:ray:wait-image" # Download ML models to S3 - task: ":{{.PROVIDER}}:ray:prestage-models" + # Write platform URLs + credentials to ~/.bashrc.d/platform.sh for IDE use + - task: setup-env - printf '{{.C_OK}}✓ Workshop install complete.{{.C_RESET}}\n' # ============================================================ @@ -123,6 +125,47 @@ tasks: cmds: - task: idc:configure + + setup-env: + desc: Write platform URLs and credentials to ~/.bashrc.d/platform.sh (workshop IDE env vars) + vars: + ARGOCD_URL: + sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name argocd --query 'capability.configuration.argoCd.serverUrl' --output text --region {{.AWS_REGION}} 2>/dev/null || echo "" + CF_DOMAIN: + sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + USER_PASSWORD: + sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // ""' + KC_ADMIN_PASSWORD: + sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.keycloak_admin_password // ""' + cmds: + - | + PLATFORM_FILE="$HOME/.bashrc.d/platform.sh" + set_var() { local n="$1" v="$2" + grep -q "^export ${n}=" "$PLATFORM_FILE" 2>/dev/null && sed -i "s|^export ${n}=.*|export ${n}=\"${v}\"|" "$PLATFORM_FILE" || echo "export ${n}=\"${v}\"" >> "$PLATFORM_FILE" + } + CF="{{.CF_DOMAIN}}" + ARGOCD="{{.ARGOCD_URL}}" + ARGOCD_SERVER=$(echo "$ARGOCD" | sed 's|https://||;s|/.*||') + UP="{{.USER_PASSWORD}}" + KC="{{.KC_ADMIN_PASSWORD}}" + set_var BACKSTAGE_URL "https://${CF}/backstage" + set_var KEYCLOAK_ADMIN_URL "https://${CF}/keycloak/admin" + set_var WORKFLOWS_URL "https://${CF}/argo-workflows" + set_var KARGO_URL "https://${CF}" + set_var GRAFANA_URL "https://${CF}/grafana" + set_var GITLAB_URL "https://${CF}" + set_var GITLAB_DOMAIN "${CF}" + set_var ARGOCD_URL "${ARGOCD}" + set_var ARGOCD_SERVER "${ARGOCD_SERVER}" + set_var ARGOCD_OPTS "--grpc-web" + set_var USER_PASSWORD "${UP}" + set_var USER1_PASSWORD "${UP}" + set_var GIT_TOKEN "${UP}" + set_var GIT_USERNAME "user1" + set_var KEYCLOAK_ADMIN_PASSWORD "${KC}" + printf '{{.C_OK}}✓ Platform env vars written to %s{{.C_RESET}}\n' "$PLATFORM_FILE" + source "$PLATFORM_FILE" 2>/dev/null || true + # ============================================================ # IDC / KEYCLOAK FEDERATION # Configures IAM Identity Center with Keycloak as external IdP From dc3c05fd1274acc54047e64b0a997287eaeaffee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 14:48:35 +0200 Subject: [PATCH 093/231] fix: add set +e to workshop:setup-env to prevent exit status 1 --- cluster-providers/workshop/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index c6ba550c2..eb9f92754 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -139,6 +139,7 @@ tasks: sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.keycloak_admin_password // ""' cmds: - | + set +e PLATFORM_FILE="$HOME/.bashrc.d/platform.sh" set_var() { local n="$1" v="$2" grep -q "^export ${n}=" "$PLATFORM_FILE" 2>/dev/null && sed -i "s|^export ${n}=.*|export ${n}=\"${v}\"|" "$PLATFORM_FILE" || echo "export ${n}=\"${v}\"" >> "$PLATFORM_FILE" From 69af5cdb56e9143dc739a0b25d6ff39d11d37b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 14:52:51 +0200 Subject: [PATCH 094/231] fix: wrap push test in set +e to prevent exit 128 on rails runner failure --- cluster-providers/workshop/Taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index eb9f92754..04261c781 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -301,6 +301,7 @@ tasks: done # Test if git HTTP push works (internal/allowed check). If not, wipe and restart. # This detects the workhorse/gitaly-hooks secret mismatch that survives reconfigure. + set +e _TEST_REPO=$(mktemp -d) git init -q "$_TEST_REPO" && cd "$_TEST_REPO" git commit --allow-empty -qm "test" 2>/dev/null || true @@ -315,16 +316,15 @@ tasks: PUSH_OK=$? cd - >/dev/null; rm -rf "$_TEST_REPO" docker exec gitlab-gitlab-1 gitlab-rails runner "Project.find_by_full_path('root/test-push-check')&.destroy" 2>/dev/null || true + set -e if [ "$PUSH_OK" -ne 0 ]; then printf '{{.C_ERR}}⚠ GitLab internal/allowed broken — wiping and restarting...{{.C_RESET}}\n' - set +e cd /home/ec2-user/gitlab && docker compose down -v && docker compose up -d printf ' Waiting for GitLab to reinitialize (~10min)...\n' for i in $(seq 1 80); do curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && printf ' GitLab ready after reinit\n' && break sleep 10 done - set -e fi ROOT_TOKEN="root-{{.GIT_PASSWORD}}" # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. From 6900ef6323fdbd8d5e101e0d3cff5246d65a86b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 14:58:19 +0200 Subject: [PATCH 095/231] perf: run spoke-dev and spoke-prod enables in parallel (deps vs sequential cmds) --- cluster-providers/kind-crossplane/Taskfile.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 404f00af9..76b5a8609 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -210,9 +210,9 @@ tasks: install:phase2-parallel: internal: true - cmds: - - task: install:phase2-spoke-dev - - task: install:phase2-spoke-prod + deps: + - install:phase2-spoke-dev + - install:phase2-spoke-prod install:phase2-spoke-dev: internal: true From 5cfe37ce6747426ec23154d41bb6bf4a4b2fa050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 15:00:56 +0200 Subject: [PATCH 096/231] fix: always do docker compose down -v + up (unconditional, eliminates brittle push test) --- cluster-providers/workshop/Taskfile.yaml | 40 +++++------------------- 1 file changed, 8 insertions(+), 32 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 04261c781..9a76bc2a1 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -291,41 +291,17 @@ tasks: [ "$i" -eq 120 ] && { printf ' ⚠ GitLab not ready after 20min — continuing anyway.\n'; } sleep 10 done - # Run reconfigure to ensure internal secrets (workhorse/gitaly-hooks) are consistent. - # On first boot these can be mismatched, causing git push pre-receive hooks to fail. - docker exec gitlab-gitlab-1 gitlab-ctl reconfigure 2>/dev/null | tail -3 || true - # Wait for GitLab to be ready again after reconfigure - for i in $(seq 1 30); do - curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && break + # Wipe and restart GitLab to ensure internal secrets are consistent. + # docker compose down -v reinitializes gitaly/workhorse/gitlab-shell secrets + # which prevents the misleading "protected branch" error on git pushes. + set +e + cd /home/ec2-user/gitlab && docker compose down -v && docker compose up -d + printf ' Waiting for GitLab to reinitialize (~10min)...\n' + for i in $(seq 1 80); do + curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && printf ' GitLab ready.\n' && break sleep 10 done - # Test if git HTTP push works (internal/allowed check). If not, wipe and restart. - # This detects the workhorse/gitaly-hooks secret mismatch that survives reconfigure. - set +e - _TEST_REPO=$(mktemp -d) - git init -q "$_TEST_REPO" && cd "$_TEST_REPO" - git commit --allow-empty -qm "test" 2>/dev/null || true - git remote add origin "http://root:{{.GIT_PASSWORD}}@localhost:8929/root/test-push-check.git" 2>/dev/null || true - # Create a test repo and push to it as root - docker exec gitlab-gitlab-1 gitlab-rails runner " - r = Project.find_by_full_path('root/test-push-check') - r ||= Projects::CreateService.new(User.find(1), name: 'test-push-check', path: 'test-push-check', visibility_level: 0, initialize_with_readme: false).execute - puts r.persisted? ? 'ok' : 'fail' - " 2>/dev/null | grep -q ok && \ - git push -f origin HEAD:main >/dev/null 2>&1 - PUSH_OK=$? - cd - >/dev/null; rm -rf "$_TEST_REPO" - docker exec gitlab-gitlab-1 gitlab-rails runner "Project.find_by_full_path('root/test-push-check')&.destroy" 2>/dev/null || true set -e - if [ "$PUSH_OK" -ne 0 ]; then - printf '{{.C_ERR}}⚠ GitLab internal/allowed broken — wiping and restarting...{{.C_RESET}}\n' - cd /home/ec2-user/gitlab && docker compose down -v && docker compose up -d - printf ' Waiting for GitLab to reinitialize (~10min)...\n' - for i in $(seq 1 80); do - curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && printf ' GitLab ready after reinit\n' && break - sleep 10 - done - fi ROOT_TOKEN="root-{{.GIT_PASSWORD}}" # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash From fc0ccc762107e8d23007b0081a889d7f60e49e74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 16:32:58 +0200 Subject: [PATCH 097/231] fix: separate gitlab domain (CDK) from platform domain (ALB CloudFront) in config and setup-env --- .../kind-crossplane/Taskfile.cloudfront.yaml | 2 + cluster-providers/workshop/Taskfile.yaml | 9 +++- config.schema.json | 52 +++++++++++++++---- create-config.sh | 14 ++--- 4 files changed, 56 insertions(+), 21 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml index 54331f1c2..8ec117ba8 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml @@ -97,6 +97,8 @@ tasks: CF_DOMAIN=$(aws cloudfront create-distribution --distribution-config "$CF_CONFIG" --query "Distribution.DomainName" --output text) mkdir -p {{.ROOT_DIR}}/private echo -n "$CF_DOMAIN" > {{.ROOT_DIR}}/private/cloudfront-domain + # Write cloudfrontDomain into config.local.yaml so all tasks use the ALB domain + yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} echo "CloudFront: $CF_DOMAIN (copy this to config.local.yaml cloudfront.cloudfrontDomain)" - printf '{{.C_OK}}✓ Platform CloudFront ready.{{.C_RESET}}\n' # gitlab-nlb and gitlab-distribution removed — GitLab CloudFront is now managed by CDK (team-stack.ts). diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 9a76bc2a1..36d40af14 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -133,6 +133,8 @@ tasks: sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name argocd --query 'capability.configuration.argoCd.serverUrl' --output text --region {{.AWS_REGION}} 2>/dev/null || echo "" CF_DOMAIN: sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + GITLAB_CF_DOMAIN_VAR: + sh: yq '.cloudfront.gitlabDomain // .cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} USER_PASSWORD: sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // ""' KC_ADMIN_PASSWORD: @@ -145,6 +147,9 @@ tasks: grep -q "^export ${n}=" "$PLATFORM_FILE" 2>/dev/null && sed -i "s|^export ${n}=.*|export ${n}=\"${v}\"|" "$PLATFORM_FILE" || echo "export ${n}=\"${v}\"" >> "$PLATFORM_FILE" } CF="{{.CF_DOMAIN}}" + GITLAB_CF="{{.GITLAB_CF_DOMAIN_VAR}}" + # Fall back to platform domain if gitlabDomain not set + [ -z "$GITLAB_CF" ] && GITLAB_CF="$CF" ARGOCD="{{.ARGOCD_URL}}" ARGOCD_SERVER=$(echo "$ARGOCD" | sed 's|https://||;s|/.*||') UP="{{.USER_PASSWORD}}" @@ -154,8 +159,8 @@ tasks: set_var WORKFLOWS_URL "https://${CF}/argo-workflows" set_var KARGO_URL "https://${CF}" set_var GRAFANA_URL "https://${CF}/grafana" - set_var GITLAB_URL "https://${CF}" - set_var GITLAB_DOMAIN "${CF}" + set_var GITLAB_URL "https://${GITLAB_CF}" + set_var GITLAB_DOMAIN "${GITLAB_CF}" set_var ARGOCD_URL "${ARGOCD}" set_var ARGOCD_SERVER "${ARGOCD_SERVER}" set_var ARGOCD_OPTS "--grpc-web" diff --git a/config.schema.json b/config.schema.json index f52d33914..3359d112c 100644 --- a/config.schema.json +++ b/config.schema.json @@ -3,17 +3,29 @@ "title": "Platform Bootstrap Configuration", "description": "Configuration for the GitOps addon management platform. Copy to config.local.yaml and customize.", "type": "object", - "required": ["hub", "aws", "repo", "domain"], + "required": [ + "hub", + "aws", + "repo", + "domain" + ], "properties": { "clusterProvider": { "type": "string", - "enum": ["kind-crossplane", "byoc"], + "enum": [ + "kind-crossplane", + "byoc" + ], "default": "kind-crossplane", "description": "Which cluster provider to use for hub provisioning" }, "repo": { "type": "object", - "required": ["url", "revision", "basepath"], + "required": [ + "url", + "revision", + "basepath" + ], "properties": { "url": { "type": "string", @@ -34,7 +46,9 @@ }, "hub": { "type": "object", - "required": ["clusterName"], + "required": [ + "clusterName" + ], "properties": { "clusterName": { "type": "string", @@ -63,8 +77,12 @@ }, "instanceTypes": { "type": "array", - "items": { "type": "string" }, - "default": ["m5.large"], + "items": { + "type": "string" + }, + "default": [ + "m5.large" + ], "description": "EC2 instance types for the node group" }, "desiredSize": { @@ -90,7 +108,10 @@ }, "capacityType": { "type": "string", - "enum": ["ON_DEMAND", "SPOT"], + "enum": [ + "ON_DEMAND", + "SPOT" + ], "default": "ON_DEMAND" } } @@ -99,7 +120,10 @@ }, "aws": { "type": "object", - "required": ["region", "accountId"], + "required": [ + "region", + "accountId" + ], "properties": { "region": { "type": "string", @@ -121,7 +145,7 @@ "domain": { "type": "string", "pattern": "^[a-z0-9][a-z0-9.-]+[a-z0-9]$", - "description": "Base domain for ingress hostnames. Must be unique per stack — two stacks sharing a domain will have DNS and TLS conflicts." + "description": "Base domain for ingress hostnames. Must be unique per stack \u2014 two stacks sharing a domain will have DNS and TLS conflicts." }, "resourcePrefix": { "type": "string", @@ -150,6 +174,10 @@ "cloudfrontDomain": { "type": "string", "description": "CloudFront distribution domain (e.g. xxxx.cloudfront.net)" + }, + "gitlabDomain": { + "type": "string", + "description": "GitLab/IDE CloudFront domain (CDK-managed, EC2 origin)" } } }, @@ -159,7 +187,11 @@ }, "identityCenter": { "type": "object", - "required": ["instanceArn", "region", "adminGroupId"], + "required": [ + "instanceArn", + "region", + "adminGroupId" + ], "description": "AWS Identity Center config for EKS ArgoCD Capability", "properties": { "instanceArn": { diff --git a/create-config.sh b/create-config.sh index 3b65c6c7a..55b515f51 100755 --- a/create-config.sh +++ b/create-config.sh @@ -158,16 +158,12 @@ printf 'resourcePrefix: "%s"\n' "$RESOURCE_PREFIX" >> "$OUTPUT_FILE" printf 'ingressName: ""\n' >> "$OUTPUT_FILE" printf 'ingressSecurityGroups: ""\n' >> "$OUTPUT_FILE" -# CloudFront domain — read from CLOUDFRONT_DOMAIN env var (preferred, set by SSM from IDE_DOMAIN) -# with fallback to the private/gitlab-cloudfront-domain file written by bootstrap.sh. -CF_DOMAIN="${CLOUDFRONT_DOMAIN:-${IDE_DOMAIN:-}}" -if [ -z "$CF_DOMAIN" ]; then - PRIVATE_DIR="${SCRIPT_DIR}/private" - [ -f "${PRIVATE_DIR}/gitlab-cloudfront-domain" ] && CF_DOMAIN="$(cat "${PRIVATE_DIR}/gitlab-cloudfront-domain" | tr -d '[:space:]')" -fi -if [ -n "$CF_DOMAIN" ]; then +# CloudFront domain is NOT written here — it is set by the hub-distribution task +# after the platform ALB CloudFront is created (pointing to the EKS ALB, not EC2/GitLab). +# The GitLab/IDE CloudFront domain is available as $CLOUDFRONT_DOMAIN / $IDE_DOMAIN env vars. +if [ -n "${CLOUDFRONT_DOMAIN:-}" ] || [ -n "${IDE_DOMAIN:-}" ]; then printf 'cloudfront:\n' >> "$OUTPUT_FILE" - printf ' cloudfrontDomain: "%s"\n' "$CF_DOMAIN" >> "$OUTPUT_FILE" + printf ' gitlabDomain: "%s"\n' "${CLOUDFRONT_DOMAIN:-${IDE_DOMAIN:-}}" >> "$OUTPUT_FILE" fi printf 'identityCenter:\n' >> "$OUTPUT_FILE" printf ' instanceArn: "%s"\n' "$IDC_ARN" >> "$OUTPUT_FILE" From 7cb392f21eb58ce880c515c475e7babb226705c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 17:24:55 +0200 Subject: [PATCH 098/231] fix(kind-kro-ack): increase MAX_OUTOFSYNC=20 for initial hub:wait-for-sync Spoke cluster apps show OutOfSync (not Unknown) before spoke EKS clusters exist. With default tolerance of 5, the wait never passes. 20 gives room for the typical spoke-targeting apps that are legitimately not yet synced. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 6ce0d1006..52ba6a8a1 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -115,6 +115,7 @@ tasks: - task: hub:cloudfront - task: hub:seed - task: hub:wait-for-sync + vars: { MAX_OUTOFSYNC: "20" } - task: install:phase1-parallel - task: hub:wait-for-full-sync - task: install:phase1b-parallel From 782d98c7b1e5806170a6191b675e28581a4407d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 17:49:32 +0200 Subject: [PATCH 099/231] =?UTF-8?q?fix:=20hub:cloudfront=20and=20hub-distr?= =?UTF-8?q?ibution=20status=20checks=20=E2=80=94=20use=20OR=20logic=20to?= =?UTF-8?q?=20prevent=20duplicate=20distributions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml | 3 +-- cluster-providers/kind-kro-ack/Taskfile.yaml | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml index 8ec117ba8..e28f9ced7 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml @@ -34,8 +34,7 @@ tasks: setup-exposure: desc: Pre-create ALB and CloudFront distributions (cloudfront mode only, skipped if cloudfrontDomain already set in config) status: - - '[ "{{.EXPOSURE_MODE}}" != "cloudfront" ]' - - '[ -n "{{.CLOUDFRONT_DOMAIN}}" ]' + - '[ "{{.EXPOSURE_MODE}}" != "cloudfront" ] || aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='"'"'{{.HUB_CLUSTER_NAME}}-platform'"'"'].Id" --output text | grep -q .' cmds: - task: create-alb - task: hub-distribution diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 52ba6a8a1..e61848aaa 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -746,9 +746,10 @@ tasks: sh: aws elbv2 describe-load-balancers --names "{{.RESOURCE_PREFIX}}-hub-ingress" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text 2>/dev/null || echo "" VPC_ORIGIN_NAME: "{{.RESOURCE_PREFIX}}-hub-vpc-origin" status: - # Skip entirely when not in cloudfront mode (idempotent no-op for tls) - - test "{{.INGRESS_MODE}}" != "cloudfront" - - aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.RESOURCE_PREFIX}}-hub-ingress'].Id" --output text | grep -q . + # Skip when not in cloudfront mode OR when distribution already exists + - | + test "{{.INGRESS_MODE}}" != "cloudfront" || \ + aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.RESOURCE_PREFIX}}-hub-ingress'].Id" --output text 2>/dev/null | grep -q . cmds: - | if [ "{{.INGRESS_MODE}}" != "cloudfront" ]; then From e4ba51ccff0a59d4826b6c41d7d3de336daf96ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 17:55:06 +0200 Subject: [PATCH 100/231] fix: use YAML block scalar for status check to avoid single-quote escaping issue --- cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml index e28f9ced7..d96ded63c 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml @@ -34,7 +34,9 @@ tasks: setup-exposure: desc: Pre-create ALB and CloudFront distributions (cloudfront mode only, skipped if cloudfrontDomain already set in config) status: - - '[ "{{.EXPOSURE_MODE}}" != "cloudfront" ] || aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='"'"'{{.HUB_CLUSTER_NAME}}-platform'"'"'].Id" --output text | grep -q .' + - | + [ "{{.EXPOSURE_MODE}}" != "cloudfront" ] || \ + aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" --output text | grep -q . cmds: - task: create-alb - task: hub-distribution From 9f799fce6d9134056392140b23f1c17fd956c74f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 18:01:16 +0200 Subject: [PATCH 101/231] =?UTF-8?q?fix:=20only=20wipe=20GitLab=20on=20firs?= =?UTF-8?q?t=20run=20=E2=80=94=20skip=20down=20-v=20if=20fleet-config=20re?= =?UTF-8?q?po=20already=20seeded?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/workshop/Taskfile.yaml | 27 ++++++++++++++---------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 36d40af14..a1efd1b18 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -296,17 +296,22 @@ tasks: [ "$i" -eq 120 ] && { printf ' ⚠ GitLab not ready after 20min — continuing anyway.\n'; } sleep 10 done - # Wipe and restart GitLab to ensure internal secrets are consistent. - # docker compose down -v reinitializes gitaly/workhorse/gitlab-shell secrets - # which prevents the misleading "protected branch" error on git pushes. - set +e - cd /home/ec2-user/gitlab && docker compose down -v && docker compose up -d - printf ' Waiting for GitLab to reinitialize (~10min)...\n' - for i in $(seq 1 80); do - curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && printf ' GitLab ready.\n' && break - sleep 10 - done - set -e + # Wipe and restart GitLab ONLY on first run (no existing repos/data). + # On retries, the data is already there — skip the wipe to preserve repos. + # Detection: check if fleet-config repo exists (seeded on first successful init). + ALREADY_SEEDED=$(docker exec gitlab-gitlab-1 gitlab-rails runner \ + "puts Project.find_by_full_path('user1/fleet-config') ? 'yes' : 'no'" 2>/dev/null | grep -E '^yes|^no' | tail -1) + if [ "$ALREADY_SEEDED" = "yes" ]; then + printf ' GitLab already initialized — skipping wipe+restart.\n' + else + printf '{{.C_STEP}} First run — reinitializing GitLab secrets (down -v + up)...{{.C_RESET}}\n' + cd /home/ec2-user/gitlab && docker compose down -v && docker compose up -d + printf ' Waiting for GitLab to reinitialize (~10min)...\n' + for i in $(seq 1 80); do + curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && printf ' GitLab ready.\n' && break + sleep 10 + done + fi ROOT_TOKEN="root-{{.GIT_PASSWORD}}" # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash From 991bebe1e10cfa75f142ab223a34625f3239fe81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 18:02:09 +0200 Subject: [PATCH 102/231] =?UTF-8?q?fix:=20remove=20down=20-v=20wipe=20logi?= =?UTF-8?q?c=20=E2=80=94=20CDK=20starts=20GitLab=20fresh=20on=20new=20even?= =?UTF-8?q?ts,=20no=20secret=20mismatch=20possible?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/workshop/Taskfile.yaml | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index a1efd1b18..6cd246637 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -296,22 +296,8 @@ tasks: [ "$i" -eq 120 ] && { printf ' ⚠ GitLab not ready after 20min — continuing anyway.\n'; } sleep 10 done - # Wipe and restart GitLab ONLY on first run (no existing repos/data). - # On retries, the data is already there — skip the wipe to preserve repos. - # Detection: check if fleet-config repo exists (seeded on first successful init). - ALREADY_SEEDED=$(docker exec gitlab-gitlab-1 gitlab-rails runner \ - "puts Project.find_by_full_path('user1/fleet-config') ? 'yes' : 'no'" 2>/dev/null | grep -E '^yes|^no' | tail -1) - if [ "$ALREADY_SEEDED" = "yes" ]; then - printf ' GitLab already initialized — skipping wipe+restart.\n' - else - printf '{{.C_STEP}} First run — reinitializing GitLab secrets (down -v + up)...{{.C_RESET}}\n' - cd /home/ec2-user/gitlab && docker compose down -v && docker compose up -d - printf ' Waiting for GitLab to reinitialize (~10min)...\n' - for i in $(seq 1 80); do - curl -sf --max-time 5 "$GITLAB_URL/users/sign_in" >/dev/null 2>&1 && printf ' GitLab ready.\n' && break - sleep 10 - done - fi + # GitLab was started fresh by CDK bootstrap.sh — secrets are consistent on new events. + # No need to wipe and restart. ROOT_TOKEN="root-{{.GIT_PASSWORD}}" # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash From 98c345759e0b714852dfc34be08276f97f513cf5 Mon Sep 17 00:00:00 2001 From: Workshop User Date: Mon, 29 Jun 2026 16:04:12 +0000 Subject: [PATCH 103/231] fix(kro): propagate argoCdCapabilityRoleArn to child EksCluster in rg-eks-vpc The EksclusterWithVpc wrapper passed argoCdHubRoleArn and argocdCapability.* to the child EksCluster but omitted argoCdCapabilityRoleArn, so the child fell back to "" and the accessEntryArgocdCapability gate (argoCdCapabilityRoleArn != "") was skipped. This left KRO-provisioned spokes without an EKS access entry for the hub ArgoCD capability role, so ArgoCD could not connect (cluster status Unknown). --- .../charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml index 33d5d5f7b..50744e62f 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml @@ -172,6 +172,7 @@ spec: accountId: ${schema.spec.accountId} managementAccountId: ${schema.spec.managementAccountId} argoCdHubRoleArn: ${schema.spec.argoCdHubRoleArn} + argoCdCapabilityRoleArn: ${schema.spec.argoCdCapabilityRoleArn} k8sVersion: ${schema.spec.k8sVersion} adminRoleName: ${schema.spec.adminRoleName} adminInstanceRoleName: ${schema.spec.adminInstanceRoleName} From e6ad282502b6624ea60b09990e41d5153f75a304 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 18:19:37 +0200 Subject: [PATCH 104/231] fix: guard update_columns with persisted? to surface CreateService errors --- cluster-providers/workshop/Taskfile.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 6cd246637..1f646038a 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -326,9 +326,11 @@ tasks: u = svc.respond_to?(:payload) ? (svc.payload[:user] || User.find_by(username: '{{.GIT_USERNAME}}')) : User.find_by(username: '{{.GIT_USERNAME}}') end # Always ensure password is correctly hashed (CreateService may not hash correctly) - if u + if u && u.persisted? u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) + elsif u && !u.persisted? + raise "user1 not persisted: #{u.errors.full_messages.join(', ')}" end if u.namespace_id.nil? org_id = Organizations::Organization.find_by(id: 1)&.id || 1 From c9a69b00a640589ee3ced757c00c7df0efc02bd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 18:21:39 +0200 Subject: [PATCH 105/231] =?UTF-8?q?fix:=20avoid=20shell=20interpolation=20?= =?UTF-8?q?in=20ruby=20raise=20=E2=80=94=20use=20stderr.puts=20+=20exit=20?= =?UTF-8?q?1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/workshop/Taskfile.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 1f646038a..aad33a73d 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -330,7 +330,8 @@ tasks: u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) elsif u && !u.persisted? - raise "user1 not persisted: #{u.errors.full_messages.join(', ')}" + $stderr.puts 'ERROR: user1 not persisted: ' + u.errors.full_messages.inspect + exit 1 end if u.namespace_id.nil? org_id = Organizations::Organization.find_by(id: 1)&.id || 1 From 391ad6db60172e0f785fa1ffb00ddbafe97d02fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 18:22:26 +0200 Subject: [PATCH 106/231] fix: use STDERR constant instead of $stderr to avoid bash variable expansion --- cluster-providers/workshop/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index aad33a73d..0d9478b33 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -330,7 +330,7 @@ tasks: u.encrypted_password = Devise::Encryptor.digest(User, '{{.GIT_PASSWORD}}') u.update_columns(encrypted_password: u.encrypted_password, password_automatically_set: false) elsif u && !u.persisted? - $stderr.puts 'ERROR: user1 not persisted: ' + u.errors.full_messages.inspect + STDERR.puts 'ERROR: user1 not persisted: ' + u.errors.full_messages.inspect exit 1 end if u.namespace_id.nil? From 9eddcb95f13d03730b50ef61c48df58765de77bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 20:04:11 +0200 Subject: [PATCH 107/231] fix: GIT_PASSWORD reads from peeks-ide-password secret (available at boot) before peeks-hub/keycloak --- cluster-providers/workshop/Taskfile.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 0d9478b33..d9c38878a 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -278,7 +278,11 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // "{{.HUB_CLUSTER_NAME}}"') + sh: | + P=$(yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$') + [ -z "$P" ] && P=$(aws secretsmanager get-secret-value --secret-id peeks-ide-password --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.password // empty') + [ -z "$P" ] && P=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty') + echo "$P" GIT_USERNAME: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} REPOS: @@ -407,7 +411,11 @@ tasks: GIT_USERNAME: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // "{{.HUB_CLUSTER_NAME}}"') + sh: | + P=$(yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$') + [ -z "$P" ] && P=$(aws secretsmanager get-secret-value --secret-id peeks-ide-password --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.password // empty') + [ -z "$P" ] && P=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty') + echo "$P" REPOS: sh: echo "fleet-config java rust golang dotnet next-js" cmds: From 66706c8f66f1679dc6041c7ad583414b743696a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 20:11:41 +0200 Subject: [PATCH 108/231] fix: GIT_PASSWORD as single-line chained command (sh: block doesn't support multiline) --- cluster-providers/workshop/Taskfile.yaml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index d9c38878a..aed501f3e 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -278,11 +278,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: | - P=$(yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$') - [ -z "$P" ] && P=$(aws secretsmanager get-secret-value --secret-id peeks-ide-password --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.password // empty') - [ -z "$P" ] && P=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty') - echo "$P" + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id peeks-ide-password --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.password // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty' GIT_USERNAME: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} REPOS: @@ -411,11 +407,7 @@ tasks: GIT_USERNAME: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} GIT_PASSWORD: - sh: | - P=$(yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$') - [ -z "$P" ] && P=$(aws secretsmanager get-secret-value --secret-id peeks-ide-password --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.password // empty') - [ -z "$P" ] && P=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty') - echo "$P" + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id peeks-ide-password --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.password // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty' REPOS: sh: echo "fleet-config java rust golang dotnet next-js" cmds: From 3c1d9bc7f3c32755157aeb56281731f7b749c9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 20:20:42 +0200 Subject: [PATCH 109/231] fix: GIT_PASSWORD reads from peeks-hub/secrets.git_token (seeded by CDK at boot) --- cluster-providers/workshop/Taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index aed501f3e..f2a6ee501 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -278,7 +278,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id peeks-ide-password --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.password // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty' + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty' GIT_USERNAME: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} REPOS: @@ -407,7 +407,7 @@ tasks: GIT_USERNAME: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id peeks-ide-password --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.password // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty' + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // empty' REPOS: sh: echo "fleet-config java rust golang dotnet next-js" cmds: From 704c474c3c0bb64e79e3be79bd0eed05f82517f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 21:21:37 +0200 Subject: [PATCH 110/231] fix: write cloudfront.cloudfrontDomain (not cloudfront.domain) in kind-kro-ack hub:cloudfront --- cluster-providers/kind-kro-ack/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index e61848aaa..069393bd8 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -844,7 +844,7 @@ tasks: CF_DOMAIN=$(echo "$DIST_OUT" | awk '{print $2}') printf '{{.C_OK}}✓ CloudFront created: %s{{.C_RESET}}\n' "$CF_DOMAIN" # Store domain in config for hub:seed to use - yq -i ".cloudfront.domain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} + yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} rm -f /tmp/hub-vpc-origin-id hub:seed: From 8ced210d4404bb76953534204eff08a56c451bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 21:24:18 +0200 Subject: [PATCH 111/231] fix: unprotect fleet-config main before each spoke push attempt (re-protection after seed) --- cluster-providers/kind-crossplane/Taskfile.yaml | 4 ++++ cluster-providers/kind-kro-ack/Taskfile.yaml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 76b5a8609..cfff60896 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -1039,6 +1039,8 @@ tasks: git commit -m "feat: enable spoke {{.CLUSTER}} via Crossplane (+ fleet member)" PUSHED=false for i in 1 2 3; do + # Ensure fleet-config main branch is not protected before pushing + docker exec gitlab-gitlab-1 gitlab-rails runner "u=User.find_by(username:'user1'); p=Project.find_by(path:'fleet-config',namespace_id:u&.namespace_id); ProtectedBranch.where(project:p).destroy_all" 2>/dev/null || true if git push 2>&1; then PUSHED=true; break; fi echo " Push attempt $i failed, rebasing..." git pull --rebase 2>&1 && sleep 2 @@ -1267,6 +1269,8 @@ tasks: git commit -m "feat: enable spoke {{.CLUSTER}} via KRO (+ multi-acct CARM mapping)" PUSHED=false for i in 1 2 3; do + # Ensure fleet-config main branch is not protected before pushing + docker exec gitlab-gitlab-1 gitlab-rails runner "u=User.find_by(username:'user1'); p=Project.find_by(path:'fleet-config',namespace_id:u&.namespace_id); ProtectedBranch.where(project:p).destroy_all" 2>/dev/null || true if git push 2>&1; then PUSHED=true; break; fi echo " Push attempt $i failed, rebasing..." git pull --rebase 2>&1 && sleep 2 diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 069393bd8..f3de67ee6 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1971,6 +1971,8 @@ tasks: git commit -m "feat: enable spoke {{.CLUSTER}} via Crossplane (+ fleet member)" PUSHED=false for i in 1 2 3; do + # Ensure fleet-config main branch is not protected before pushing + docker exec gitlab-gitlab-1 gitlab-rails runner "u=User.find_by(username:'user1'); p=Project.find_by(path:'fleet-config',namespace_id:u&.namespace_id); ProtectedBranch.where(project:p).destroy_all" 2>/dev/null || true if git push 2>&1; then PUSHED=true; break; fi echo " Push attempt $i failed, rebasing..." git pull --rebase 2>&1 && sleep 2 @@ -2205,6 +2207,8 @@ tasks: git commit -m "feat: enable spoke {{.CLUSTER}} via KRO (+ multi-acct CARM mapping)" PUSHED=false for i in 1 2 3; do + # Ensure fleet-config main branch is not protected before pushing + docker exec gitlab-gitlab-1 gitlab-rails runner "u=User.find_by(username:'user1'); p=Project.find_by(path:'fleet-config',namespace_id:u&.namespace_id); ProtectedBranch.where(project:p).destroy_all" 2>/dev/null || true if git push 2>&1; then PUSHED=true; break; fi echo " Push attempt $i failed, rebasing..." git pull --rebase 2>&1 && sleep 2 From 5a7007708d272d0fcf51481ac79edd41a63044c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 29 Jun 2026 21:25:27 +0200 Subject: [PATCH 112/231] fix: flush Rails cache after ApplicationSetting save so all workers pick up default_branch_protection=0 --- cluster-providers/workshop/Taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index f2a6ee501..3b02afd06 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -309,6 +309,9 @@ tasks: s.default_branch_protection = 0 s.default_branch_protection_defaults = {allowed_to_push: [{access_level: 0}], allow_force_push: true, allowed_to_merge: [{access_level: 0}], developer_can_initial_push: true} s.save(validate: false) + # Flush Rails cache so all workers (Sidekiq, web) pick up the new settings immediately + Rails.cache.clear + ApplicationSetting.expire pat = User.find(1).personal_access_tokens.find_or_initialize_by(name: 'initial-root-token') pat.scopes = ['api','write_repository','sudo'] pat.expires_at = 365.days.from_now From ca8bfdb3b2716778d3bf3ca45ceff92ea78eae33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 09:04:32 +0200 Subject: [PATCH 113/231] fix: use access_level 30 (Developer+) not 0 (No one) in default_branch_protection_defaults --- cluster-providers/workshop/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 3b02afd06..378c6a911 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -307,7 +307,7 @@ tasks: s = ApplicationSetting.first s.password_authentication_enabled_for_git = true s.default_branch_protection = 0 - s.default_branch_protection_defaults = {allowed_to_push: [{access_level: 0}], allow_force_push: true, allowed_to_merge: [{access_level: 0}], developer_can_initial_push: true} + s.default_branch_protection_defaults = {allowed_to_push: [{access_level: 30}], allow_force_push: true, allowed_to_merge: [{access_level: 30}], developer_can_initial_push: true} s.save(validate: false) # Flush Rails cache so all workers (Sidekiq, web) pick up the new settings immediately Rails.cache.clear From db3a0ccf2b623d5ea35262beeb70408f2c1c3c09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 10:15:47 +0200 Subject: [PATCH 114/231] fix: call cloudfront:hub-distribution in install:phase1-cloudfront (was missing) --- cluster-providers/kind-crossplane/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index cfff60896..516f2339f 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -206,6 +206,7 @@ tasks: install:phase1-cloudfront: internal: true cmds: + - task: cloudfront:hub-distribution - task: hub:restart-identity-pods install:phase2-parallel: From c1d0986937e5fcc3d42340ae7c30bedfb05cf789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 10:42:21 +0200 Subject: [PATCH 115/231] perf: skip gitlab:init-ec2 rails runner if user1 PAT + fleet-config already exist (idempotent fast-path) --- cluster-providers/workshop/Taskfile.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 378c6a911..574bde490 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -269,6 +269,12 @@ tasks: # ---- Workshop: GitLab CE on EC2 (workshop-specific) ---- gitlab:init-ec2: desc: Wait for GitLab CE (docker-compose on IDE EC2) to be ready, then initialise root token + user1 + repos + status: + # Skip if user1 PAT already exists and fleet-config repo is seeded — already initialized + - | + GIT_PASS=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty') + GITLAB_URL=$(cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain 2>/dev/null || echo "{{.DOMAIN}}") + [ -n "$GIT_PASS" ] && curl -sf --max-time 5 "http://localhost:8929/api/v4/projects/user1%2Ffleet-config" -H "PRIVATE-TOKEN: $GIT_PASS" | jq -e '.id' >/dev/null 2>&1 vars: GITLAB_CF_DOMAIN: sh: | From 7aa6314009bfb91d054b3f7b17a071748cc3703f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 10:44:37 +0200 Subject: [PATCH 116/231] ux: print progress message before gitlab-rails runner (takes ~60s with no output) --- cluster-providers/workshop/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 574bde490..67caf0998 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -308,6 +308,7 @@ tasks: # Bootstrap via gitlab-rails runner: create root PAT, user1, repos, unprotect branches. # Shell variables (ROOT_TOKEN, GIT_USERNAME, GIT_PASSWORD) are expanded by bash # before the string is passed to gitlab-rails runner as a Ruby script. + printf ' Running GitLab Rails bootstrap (~60s)...\n' docker exec gitlab-gitlab-1 gitlab-rails runner " # Disable default branch protection + enable password auth for git s = ApplicationSetting.first From 6d5b05fed2b98bf93c6a8a20809cc7c4618522fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 10:47:31 +0200 Subject: [PATCH 117/231] =?UTF-8?q?perf:=20fast=20file-based=20status=20ch?= =?UTF-8?q?eck=20for=20gitlab:init-ec2=20=E2=80=94=20skip=20if=20repos=20a?= =?UTF-8?q?lready=20on=20disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/workshop/Taskfile.yaml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 67caf0998..b29b1b00d 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -270,11 +270,8 @@ tasks: gitlab:init-ec2: desc: Wait for GitLab CE (docker-compose on IDE EC2) to be ready, then initialise root token + user1 + repos status: - # Skip if user1 PAT already exists and fleet-config repo is seeded — already initialized - - | - GIT_PASS=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty') - GITLAB_URL=$(cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain 2>/dev/null || echo "{{.DOMAIN}}") - [ -n "$GIT_PASS" ] && curl -sf --max-time 5 "http://localhost:8929/api/v4/projects/user1%2Ffleet-config" -H "PRIVATE-TOKEN: $GIT_PASS" | jq -e '.id' >/dev/null 2>&1 + # Fast check: if fleet-config git repo exists on disk, init already ran + - docker exec gitlab-gitlab-1 test -d /var/opt/gitlab/git-data/repositories/@hashed 2>/dev/null && docker exec gitlab-gitlab-1 find /var/opt/gitlab/git-data/repositories/@hashed -name "*.git" -maxdepth 3 | grep -q . vars: GITLAB_CF_DOMAIN: sh: | From af5e42c31636a693f6f80791f9ba3aaa279b961d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 11:14:31 +0200 Subject: [PATCH 118/231] fix(install): add hub:create-mgmt-roles to hub:seed flow hub:create-mgmt-roles was defined but never called in kind-kro-ack, causing pod-identities-peeks-hub to stay Degraded and hub:wait-for-full-sync to time out (6 apps stuck). This cascaded to fleet-config not reachable and spokes:enable-crossplane failing. The task was already called in kind-crossplane but missing from kind-kro-ack. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index f3de67ee6..43988f1b5 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -119,6 +119,7 @@ tasks: - task: install:phase1-parallel - task: hub:wait-for-full-sync - task: install:phase1b-parallel + - task: workshop:gitlab:clone-repos - task: hub:set-overlay-repo - task: install:phase2-parallel - | @@ -148,6 +149,7 @@ tasks: deps: - hub:restart-langfuse - urls + - workshop:gitlab:init-ec2 install:phase2-parallel: internal: true @@ -878,6 +880,7 @@ tasks: - task: hub:cluster-secret-store - task: hub:seed-secret - task: hub:apply-root-appset + - task: hub:create-mgmt-roles hub:kubeconfig: cmds: From f2d58d7d5e7bccf4208a54ff75799cc9aa56e8ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 11:31:00 +0200 Subject: [PATCH 119/231] refactor(kro-ack): align with crossplane Taskfile; remove AppExposure RGD pattern MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Align kind-kro-ack to the tested kind-crossplane reference; only hub creation differs: - hub:seed-secret: revert to minimal seed (repo coords + fleet labels). Removed AppExposure RGD edge metadata (alb_listener_arn, exposure_mode, enable_kroack_abstractions) — that was a previous-test artifact. Edge metadata (ingress_domain_name etc.) still flows via secrets-manager:seed + gitops-bridge ExternalSecret, same as crossplane. - spokes CIDR: deterministic cksum (10..0.0/16) instead of random shuf, for both enable-crossplane and enable-kro — stable across re-runs, no parallel collision. - install flow: remove workshop:gitlab:* references (GitLab is workshop-specific, handled by workshop:install which calls gitlab:init-ec2 before provider:install). - hub:create-mgmt-roles wired into hub:seed (was missing). Note: phase2 stays sequential (parallel fleet-config pushes collide) and IDC stays in workshop:post-install — both intentional workshop-architecture differences. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 97 ++++---------------- 1 file changed, 19 insertions(+), 78 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 43988f1b5..c191bf831 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -119,7 +119,6 @@ tasks: - task: install:phase1-parallel - task: hub:wait-for-full-sync - task: install:phase1b-parallel - - task: workshop:gitlab:clone-repos - task: hub:set-overlay-repo - task: install:phase2-parallel - | @@ -149,7 +148,6 @@ tasks: deps: - hub:restart-langfuse - urls - - workshop:gitlab:init-ec2 install:phase2-parallel: internal: true @@ -975,59 +973,17 @@ tasks: EOF hub:seed-secret: - desc: Create seed cluster secret on hub (with edge metadata for AppExposure RGD) + desc: Create minimal seed cluster secret on the hub (repo coords + fleet labels only) vars: CLUSTER_ARN: - sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.arn' --output text - CLUSTER_ENDPOINT: - sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.endpoint' --output text - ARGOCD_ROLE_ARN: - sh: aws iam get-role --role-name {{.HUB_CLUSTER_NAME}}-argocd-role --query 'Role.Arn' --output text 2>/dev/null || echo "" - ADMIN_ROLE_NAME: - sh: yq '.hub.adminRoleName // "Admin"' {{.CONFIG_FILE}} - # Edge metadata — populated by hub:ingress + hub:cloudfront before hub:seed runs. - # Consumed by ApplicationSets enrolling RGDs that need to know how the cluster is - # exposed (e.g. AppExposure RGD reads {{.metadata.annotations.ingress_domain_name}} - # via the ArgoCD clusters generator — gitops-bridge pattern). - EXPOSURE_MODE: - sh: | - if [ "{{.INGRESS_MODE}}" = "tls" ]; then echo "tls-alb"; else echo "cloudfront-alb"; fi - EDGE_DOMAIN: - sh: | - if [ "{{.INGRESS_MODE}}" = "tls" ]; then - echo "{{.DOMAIN}}" - else - echo "{{.CLOUDFRONT_DOMAIN}}" - fi - ALB_LISTENER_ARN: - sh: | - ALB_NAME="{{.RESOURCE_PREFIX}}-hub-ingress" - ALB_ARN=$(aws elbv2 describe-load-balancers --names "$ALB_NAME" --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text 2>/dev/null || echo "") - if [ -z "$ALB_ARN" ] || [ "$ALB_ARN" = "None" ]; then echo ""; exit 0; fi - # Pick listener matching the mode: HTTP:80 for cloudfront-alb, HTTPS:443 for tls-alb - if [ "{{.INGRESS_MODE}}" = "tls" ]; then PORT=443; else PORT=80; fi - aws elbv2 describe-listeners --load-balancer-arn "$ALB_ARN" --region {{.AWS_REGION}} \ - --query "Listeners[?Port==\`$PORT\`].ListenerArn | [0]" --output text 2>/dev/null || echo "" - VPC_ID: - sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text 2>/dev/null || echo "" - status: - # Skip if the cluster secret already has the correct server ARN — avoids triggering - # a 20+ minute ArgoCD cache invalidation on every re-run of task install. - - | - CURRENT=$(KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl get secret {{.HUB_CLUSTER_NAME}} -n argocd -o jsonpath='{.data.server}' 2>/dev/null | base64 -d 2>/dev/null) - [ "$CURRENT" = "{{.CLUSTER_ARN}}" ] + sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.arn' --output text 2>/dev/null || echo "arn:aws:eks:{{.AWS_REGION}}:{{.AWS_ACCOUNT_ID}}:cluster/{{.HUB_CLUSTER_NAME}}" cmds: + - printf '{{.C_STEP}}▸ Getting hub kubeconfig...{{.C_RESET}}\n' - task: hub:kubeconfig + - printf '{{.C_STEP}}▸ Creating seed cluster secret on hub...{{.C_RESET}}\n' - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl create namespace argocd --dry-run=client -o yaml | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - - | - printf '{{.C_INFO}} Seeding cluster secret with gitops-bridge metadata:{{.C_RESET}}\n' - printf ' aws_region = %s\n' "{{.AWS_REGION}}" - printf ' aws_vpc_id = %s\n' "{{.VPC_ID}}" - printf ' ingress_domain_name = %s\n' "{{.EDGE_DOMAIN}}" - printf ' exposure_mode = %s\n' "{{.EXPOSURE_MODE}}" - printf ' alb_listener_arn = %s\n' "{{.ALB_LISTENER_ARN}}" - - | - cat <<'EOF' | sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g; s|CLUSTER_ARN|{{.CLUSTER_ARN}}|g; s|CLUSTER_ENDPOINT|{{.CLUSTER_ENDPOINT}}|g; s|ARGOCD_ROLE_ARN|{{.ARGOCD_ROLE_ARN}}|g; s|REPO_URL|{{.REPO_URL}}|g; s|REPO_REVISION|{{.REPO_REVISION}}|g; s|REPO_BASEPATH|{{.REPO_BASEPATH}}|g; s|EXPOSURE_MODE|{{.EXPOSURE_MODE}}|g; s|EDGE_DOMAIN|{{.EDGE_DOMAIN}}|g; s|ALB_LISTENER_ARN|{{.ALB_LISTENER_ARN}}|g; s|VPC_ID_VAL|{{.VPC_ID}}|g; s|AWS_REGION_VAL|{{.AWS_REGION}}|g; s|AWS_ACCOUNT_ID_VAL|{{.AWS_ACCOUNT_ID}}|g; s|RESOURCE_PREFIX_VAL|{{.RESOURCE_PREFIX}}|g; s|INGRESS_SECURITY_GROUPS_VAL|{{.INGRESS_SECURITY_GROUPS}}|g; s|ADMIN_ROLE_NAME_VAL|{{.ADMIN_ROLE_NAME}}|g" | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - + cat <<'EOF' | sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g; s|CLUSTER_ARN|{{.CLUSTER_ARN}}|g; s|FLEET_REPO_URL|{{.FLEET_REPO_URL}}|g; s|FLEET_REPO_REVISION|{{.FLEET_REPO_REVISION}}|g; s|FLEET_REPO_BASEPATH|{{.FLEET_REPO_BASEPATH}}|g; s|REPO_URL|{{.REPO_URL}}|g; s|REPO_REVISION|{{.REPO_REVISION}}|g; s|REPO_BASEPATH|{{.REPO_BASEPATH}}|g" | KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f - apiVersion: v1 kind: Secret type: Opaque @@ -1035,39 +991,24 @@ tasks: name: CLUSTER_NAME namespace: argocd annotations: - # Repo coordinates (gitops-bridge contract) addonsRepoURL: "REPO_URL" addonsRepoRevision: "REPO_REVISION" addonsRepoBasepath: "REPO_BASEPATH" - fleetRepoURL: "REPO_URL" - fleetRepoRevision: "REPO_REVISION" - fleetRepoBasepath: "REPO_BASEPATH" - # Cluster identity (gitops-bridge contract — consumed by AppSets) - aws_cluster_name: "CLUSTER_NAME" - aws_account_id: "AWS_ACCOUNT_ID_VAL" - aws_region: "AWS_REGION_VAL" - aws_vpc_id: "VPC_ID_VAL" - admin_role_name: "ADMIN_ROLE_NAME_VAL" - resource_prefix: "RESOURCE_PREFIX_VAL" - # Ingress / edge metadata (gitops-bridge contract — consumed by AppSets) - ingress_domain_name: "EDGE_DOMAIN" - ingress_name: "RESOURCE_PREFIX_VAL-hub-ingress" - ingress_security_groups: "INGRESS_SECURITY_GROUPS_VAL" - alb_controller_mode: "oss" - # Exposure metadata (PeEKS extension — consumed by AppExposure RGD) - exposure_mode: "EXPOSURE_MODE" - alb_listener_arn: "ALB_LISTENER_ARN" + fleetRepoURL: "FLEET_REPO_URL" + fleetRepoRevision: "FLEET_REPO_REVISION" + fleetRepoBasepath: "FLEET_REPO_BASEPATH" + aws_cluster_name: {{.HUB_CLUSTER_NAME}} labels: argocd.argoproj.io/secret-type: cluster environment: control-plane fleet_member: control-plane - enable_kroack_abstractions: "true" - exposure_mode: "EXPOSURE_MODE" stringData: name: CLUSTER_NAME server: "CLUSTER_ARN" config: '{"tlsClientConfig":{"insecure":false}}' + project: default EOF + - printf '{{.C_OK}}✓ Seed secret created on hub{{.C_RESET}}\n' - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig hub:apply-root-appset: @@ -1945,7 +1886,7 @@ tasks: clusters: {{.CLUSTER}}: clusterName: {{.CLUSTER}} - vpcCidr: "10.$(shuf -i 1-254 -n 1).0.0/16" + vpcCidr: "10.$(printf '%s' '{{.CLUSTER}}' | cksum | awk '{print ($1 % 253) + 2}').0.0/16" kubernetesVersion: "{{.K8S_VERSION}}" autoMode: true resourcePrefix: {{.RESOURCE_PREFIX}} @@ -2131,17 +2072,17 @@ tasks: cat > "$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/clusters/{{.CLUSTER}}.json" < InvalidSubnet.Range). - # Idempotency: only generate a (new random) CIDR when this spoke is not - # already enabled. Re-running install must NOT churn the CIDR — an EC2 VPC - # cidrBlock is immutable, so a new octet would force VPC recreation/orphans. + # Deterministic second octet derived from the cluster name via cksum so the + # VPC and all subnets share one octet and stay within the VPC CIDR. Deterministic + # (not random) so re-runs are stable and parallel spoke enables never collide. + # Idempotency: only generate when this spoke is not already enabled. Re-running + # install must NOT churn the CIDR — an EC2 VPC cidrBlock is immutable, so a new + # octet would force VPC recreation/orphans. KRO_VALUES="$REPO_DIR/gitops/fleet/spoke-values/tenants/workshop/kro-clusters/values.yaml" if [ -f "$KRO_VALUES" ] && yq -e '.clusters."{{.CLUSTER}}".cidr.vpcCidr' "$KRO_VALUES" >/dev/null 2>&1; then echo " Spoke {{.CLUSTER}} already has a CIDR — keeping existing values (idempotent)." else - OCTET=$(shuf -i 2-254 -n 1) + OCTET=$(printf '%s' '{{.CLUSTER}}' | cksum | awk '{print ($1 % 253) + 2}') cat > "$KRO_VALUES" < Date: Tue, 30 Jun 2026 15:13:39 +0200 Subject: [PATCH 120/231] fix: create-config.sh reads gitlabDomain from gitlab-cloudfront-domain file as fallback --- create-config.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/create-config.sh b/create-config.sh index 55b515f51..93a6ba37b 100755 --- a/create-config.sh +++ b/create-config.sh @@ -160,10 +160,14 @@ printf 'ingressSecurityGroups: ""\n' >> "$OUTPUT_FILE" # CloudFront domain is NOT written here — it is set by the hub-distribution task # after the platform ALB CloudFront is created (pointing to the EKS ALB, not EC2/GitLab). -# The GitLab/IDE CloudFront domain is available as $CLOUDFRONT_DOMAIN / $IDE_DOMAIN env vars. -if [ -n "${CLOUDFRONT_DOMAIN:-}" ] || [ -n "${IDE_DOMAIN:-}" ]; then +# The GitLab/IDE CloudFront domain is available as $CLOUDFRONT_DOMAIN / $IDE_DOMAIN env vars, +# or from the private/gitlab-cloudfront-domain file written by bootstrap.sh. +GITLAB_CF_DOMAIN="${CLOUDFRONT_DOMAIN:-${IDE_DOMAIN:-}}" +[ -z "$GITLAB_CF_DOMAIN" ] && [ -f "${SCRIPT_DIR}/private/gitlab-cloudfront-domain" ] && \ + GITLAB_CF_DOMAIN="$(cat "${SCRIPT_DIR}/private/gitlab-cloudfront-domain" | tr -d '[:space:]')" +if [ -n "$GITLAB_CF_DOMAIN" ]; then printf 'cloudfront:\n' >> "$OUTPUT_FILE" - printf ' gitlabDomain: "%s"\n' "${CLOUDFRONT_DOMAIN:-${IDE_DOMAIN:-}}" >> "$OUTPUT_FILE" + printf ' gitlabDomain: "%s"\n' "$GITLAB_CF_DOMAIN" >> "$OUTPUT_FILE" fi printf 'identityCenter:\n' >> "$OUTPUT_FILE" printf ' instanceArn: "%s"\n' "$IDC_ARN" >> "$OUTPUT_FILE" From ae9286b9cc3e729c47dbb4ae0d45873664dbbf96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 16:33:25 +0200 Subject: [PATCH 121/231] fix: add ignore_error to setup-env so && chaining doesn't break on source exit code --- cluster-providers/workshop/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index b29b1b00d..83ad6ba47 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -128,6 +128,7 @@ tasks: setup-env: desc: Write platform URLs and credentials to ~/.bashrc.d/platform.sh (workshop IDE env vars) + ignore_error: true vars: ARGOCD_URL: sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name argocd --query 'capability.configuration.argoCd.serverUrl' --output text --region {{.AWS_REGION}} 2>/dev/null || echo "" From 774fe4dae0a110072a65d6ae82f06929031a3b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 17:02:04 +0200 Subject: [PATCH 122/231] =?UTF-8?q?fix:=20more=20robust=20pod=20age=20chec?= =?UTF-8?q?k=20in=20restart-identity-pods=20status=20=E2=80=94=20clearer?= =?UTF-8?q?=20GNU=20date=20usage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kind-crossplane/Taskfile.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 516f2339f..c7c8e633f 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -1530,9 +1530,22 @@ tasks: status: - | KC={{.ROOT_DIR}}/private/hub-kubeconfig - # Skip if all providers are Healthy and have been running > 5min (pod identity already active) - UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' -n crossplane-system 2>/dev/null | wc -w) - [ "$UNHEALTHY" = "0" ] && OLDEST=$(KUBECONFIG=$KC kubectl get pods -n crossplane-system -l pkg.crossplane.io/revision --sort-by=.metadata.creationTimestamp -o jsonpath='{.items[0].metadata.creationTimestamp}' 2>/dev/null) && [ -n "$OLDEST" ] && AGE=$(( $(date +%s) - $(date -d "$OLDEST" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%SZ" "$OLDEST" +%s 2>/dev/null) )) && [ "$AGE" -gt 300 ] + # Skip if all providers are Healthy and oldest provider pod is >5min old + UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io \ + -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' \ + 2>/dev/null | wc -w) + [ "$UNHEALTHY" != "0" ] && exit 1 + # Use kubectl to get pod start time as epoch via jsonpath, then compare + OLDEST_EPOCH=$(KUBECONFIG=$KC kubectl get pods -n crossplane-system \ + -l pkg.crossplane.io/revision \ + --sort-by=.metadata.creationTimestamp \ + -o jsonpath='{.items[0].metadata.creationTimestamp}' 2>/dev/null) + [ -z "$OLDEST_EPOCH" ] && exit 1 + # Convert ISO8601 to epoch (GNU date) + POD_START=$(date -d "$OLDEST_EPOCH" +%s 2>/dev/null || echo 0) + NOW=$(date +%s) + AGE=$(( NOW - POD_START )) + [ "$AGE" -gt 300 ] cmds: - task: hub:kubeconfig - printf '{{.C_STEP}}▸ Terminating any stuck ArgoCD sync operations on pod-identities apps...{{.C_RESET}}\n' From ba9cbf1d8992a1c8e107efa7c9ce32fb88c993d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 18:14:38 +0200 Subject: [PATCH 123/231] fix: add hub:update-ingress-domain to patch cluster secrets with cloudfrontDomain after hub-distribution runs --- .../kind-crossplane/Taskfile.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index c7c8e633f..367502cd8 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -207,6 +207,8 @@ tasks: internal: true cmds: - task: cloudfront:hub-distribution + # Update ingress_domain_name in the ArgoCD cluster secret now that CloudFront domain is known + - task: hub:update-ingress-domain - task: hub:restart-identity-pods install:phase2-parallel: @@ -663,6 +665,31 @@ tasks: - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig - printf '{{.C_OK}}✓ Root appset applied. Hub is now self-managing.{{.C_RESET}}\n' + hub:update-ingress-domain: + desc: Patch ingress_domain_name in ArgoCD cluster secrets with the CloudFront platform domain + internal: true + vars: + CF_DOMAIN: + sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + cmds: + - | + CF="{{.CF_DOMAIN}}" + [ -z "$CF" ] && CF=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true) + [ -z "$CF" ] && printf '{{.C_INFO}} Skipping ingress_domain_name update — cloudfrontDomain not set yet{{.C_RESET}}\n' && exit 0 + KC={{.ROOT_DIR}}/private/hub-kubeconfig + [ ! -f "$KC" ] && KC={{.ROOT_DIR}}/private/kubeconfig + for CLUSTER in peeks-hub peeks-spoke-dev peeks-spoke-prod; do + CURRENT=$(kubectl --kubeconfig "$KC" get secret "$CLUSTER" -n argocd \ + -o jsonpath='{.data.ingress_domain_name}' 2>/dev/null | base64 -d 2>/dev/null || true) + if [ "$CURRENT" != "$CF" ]; then + kubectl --kubeconfig "$KC" patch secret "$CLUSTER" -n argocd \ + --type='json' -p="[{\"op\":\"replace\",\"path\":\"/data/ingress_domain_name\",\"value\":\"$(echo -n "$CF" | base64)\"}]" \ + 2>/dev/null || true + printf ' Updated ingress_domain_name=%s on %s\n' "$CF" "$CLUSTER" + fi + done + printf '{{.C_OK}}✓ ingress_domain_name patched on cluster secrets{{.C_RESET}}\n' + hub:wait-for-sync: desc: Wait for hub ArgoCD apps to reach Synced state vars: From 8b299e3f5bfdc723b8fca7a0631d2129f9cb5ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 18:53:37 +0200 Subject: [PATCH 124/231] fix: hub:update-ingress-domain updates Secrets Manager (not k8s annotation) so ESO propagates correctly --- .../kind-crossplane/Taskfile.yaml | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 367502cd8..7873618ff 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -666,7 +666,7 @@ tasks: - printf '{{.C_OK}}✓ Root appset applied. Hub is now self-managing.{{.C_RESET}}\n' hub:update-ingress-domain: - desc: Patch ingress_domain_name in ArgoCD cluster secrets with the CloudFront platform domain + desc: Update ingress_domain_name in Secrets Manager metadata so ESO propagates the CloudFront domain to cluster secrets internal: true vars: CF_DOMAIN: @@ -676,19 +676,22 @@ tasks: CF="{{.CF_DOMAIN}}" [ -z "$CF" ] && CF=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true) [ -z "$CF" ] && printf '{{.C_INFO}} Skipping ingress_domain_name update — cloudfrontDomain not set yet{{.C_RESET}}\n' && exit 0 - KC={{.ROOT_DIR}}/private/hub-kubeconfig - [ ! -f "$KC" ] && KC={{.ROOT_DIR}}/private/kubeconfig - for CLUSTER in peeks-hub peeks-spoke-dev peeks-spoke-prod; do - CURRENT=$(kubectl --kubeconfig "$KC" get secret "$CLUSTER" -n argocd \ - -o jsonpath='{.data.ingress_domain_name}' 2>/dev/null | base64 -d 2>/dev/null || true) - if [ "$CURRENT" != "$CF" ]; then - kubectl --kubeconfig "$KC" patch secret "$CLUSTER" -n argocd \ - --type='json' -p="[{\"op\":\"replace\",\"path\":\"/data/ingress_domain_name\",\"value\":\"$(echo -n "$CF" | base64)\"}]" \ - 2>/dev/null || true - printf ' Updated ingress_domain_name=%s on %s\n' "$CF" "$CLUSTER" + for SECRET_ID in {{.HUB_CLUSTER_NAME}}/config; do + CUR=$(aws secretsmanager get-secret-value --secret-id "$SECRET_ID" --region {{.AWS_REGION}} --query SecretString --output text 2>/dev/null) + [ -z "$CUR" ] && printf ' Skipping %s — not found\n' "$SECRET_ID" && continue + CURRENT_VAL=$(echo "$CUR" | jq -r '.metadata | fromjson | .ingress_domain_name // empty' 2>/dev/null) + if [ "$CURRENT_VAL" = "$CF" ]; then + printf ' ingress_domain_name already correct in %s\n' "$SECRET_ID" + continue fi + NEW=$(echo "$CUR" | jq -c --arg cf "$CF" '.metadata = ((.metadata | fromjson | .ingress_domain_name = $cf) | tojson)') + printf '%s' "$NEW" > /tmp/secret_ingress_update.json + aws secretsmanager put-secret-value --secret-id "$SECRET_ID" --region {{.AWS_REGION}} \ + --secret-string file:///tmp/secret_ingress_update.json --query 'Name' --output text + rm -f /tmp/secret_ingress_update.json + printf ' Updated ingress_domain_name=%s in %s\n' "$CF" "$SECRET_ID" done - printf '{{.C_OK}}✓ ingress_domain_name patched on cluster secrets{{.C_RESET}}\n' + printf '{{.C_OK}}✓ ingress_domain_name updated in Secrets Manager (ESO will propagate within 1min){{.C_RESET}}\n' hub:wait-for-sync: desc: Wait for hub ArgoCD apps to reach Synced state From ae43d18939554e471aaf1c4bd4bb2ec1df85c388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 18:57:59 +0200 Subject: [PATCH 125/231] =?UTF-8?q?refactor:=20move=20cloudfront:setup-exp?= =?UTF-8?q?osure=20before=20hub:seed=20in=20crossplane=20=E2=80=94=20ALB+C?= =?UTF-8?q?F=20created=20upfront=20like=20kro-ack?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 7873618ff..3a51d027a 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -128,6 +128,8 @@ tasks: - 'printf "{{.C_INFO}} [%ds] credentials:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - task: crossplane:setup - 'printf "{{.C_INFO}} [%ds] crossplane:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' + # Create ALB + CloudFront BEFORE hub:seed so ingress_domain_name is correct from the start + - task: cloudfront:setup-exposure - task: hub:seed - 'printf "{{.C_INFO}} [%ds] hub:seed done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - task: hub:wait-for-sync @@ -206,8 +208,8 @@ tasks: install:phase1-cloudfront: internal: true cmds: - - task: cloudfront:hub-distribution - # Update ingress_domain_name in the ArgoCD cluster secret now that CloudFront domain is known + # setup-exposure (create-alb + hub-distribution) now runs before hub:seed + # This task only handles pod identity restart - task: hub:update-ingress-domain - task: hub:restart-identity-pods From d21e65e87de88bd4959f6488c910aef7fd2f666d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 19:11:42 +0200 Subject: [PATCH 126/231] =?UTF-8?q?refactor:=20align=20crossplane=20CloudF?= =?UTF-8?q?ront=20with=20kro-ack=20=E2=80=94=20internal=20ALB=20+=20VPC=20?= =?UTF-8?q?Origin=20instead=20of=20internet-facing=20ALB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kind-crossplane/Taskfile.cloudfront.yaml | 178 +++++++++++++----- 1 file changed, 130 insertions(+), 48 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml index d96ded63c..ef21846cc 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml @@ -5,34 +5,23 @@ silent: true # CloudFront exposure-mode tasks (extracted from the main Taskfile). # Only run when EXPOSURE_MODE=cloudfront (guarded per-task via status:). # Included by the main Taskfile under the "cloudfront:" namespace. +# Uses internal ALB + CloudFront VPC Origin (same pattern as kind-kro-ack). tasks: sync-origins: - desc: Ensure CloudFront distributions point to the current ALB/NLB DNS (idempotent safety net) + desc: Ensure CloudFront distributions point to the current ALB DNS (idempotent safety net) cmds: - cmd: | if [ "{{.EXPOSURE_MODE}}" != "cloudfront" ]; then exit 0; fi printf '{{.C_STEP}}▸ Syncing CloudFront origins with current LB DNS...{{.C_RESET}}\n' - # Platform ALB - ALB_DNS=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text 2>/dev/null) DIST_ID=$(aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" --output text 2>/dev/null) - if [ -n "$DIST_ID" ] && [ "$DIST_ID" != "None" ] && [ -n "$ALB_DNS" ] && [ "$ALB_DNS" != "None" ]; then - CURRENT_ORIGIN=$(aws cloudfront get-distribution-config --id "$DIST_ID" --query 'DistributionConfig.Origins.Items[0].DomainName' --output text) - if [ "$CURRENT_ORIGIN" != "$ALB_DNS" ]; then - echo "Updating platform CloudFront origin: $CURRENT_ORIGIN → $ALB_DNS" - ETAG=$(aws cloudfront get-distribution-config --id "$DIST_ID" --query 'ETag' --output text) - aws cloudfront get-distribution-config --id "$DIST_ID" --query 'DistributionConfig' --output json | \ - jq --arg dns "$ALB_DNS" '.Origins.Items[0].DomainName = $dns' > /tmp/cf-platform-config.json - aws cloudfront update-distribution --id "$DIST_ID" --if-match "$ETAG" --distribution-config file:///tmp/cf-platform-config.json > /dev/null - echo "Platform CloudFront updated" - else - echo "Platform CloudFront origin OK" - fi + ALB_ARN=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text 2>/dev/null) + if [ -n "$DIST_ID" ] && [ "$DIST_ID" != "None" ] && [ -n "$ALB_ARN" ] && [ "$ALB_ARN" != "None" ]; then + echo "Platform CloudFront origin OK (VPC Origin — no DNS update needed)" fi - # GitLab CloudFront is managed by CDK (NLB + distribution in team-stack.ts). - # private/gitlab-cloudfront-domain is written by bootstrap.sh at EC2 boot. + # GitLab CloudFront is managed by CDK (team-stack.ts). setup-exposure: - desc: Pre-create ALB and CloudFront distributions (cloudfront mode only, skipped if cloudfrontDomain already set in config) + desc: Pre-create internal ALB and CloudFront VPC Origin distribution (cloudfront mode, idempotent) status: - | [ "{{.EXPOSURE_MODE}}" != "cloudfront" ] || \ @@ -40,67 +29,160 @@ tasks: cmds: - task: create-alb - task: hub-distribution + create-alb: - desc: Pre-create the platform ALB for CloudFront (cloudfront mode only) + desc: Pre-create the internal platform ALB for CloudFront VPC Origin (cloudfront mode only) status: - aws elbv2 describe-load-balancers --names {{.HUB_CLUSTER_NAME}}-platform --region {{.AWS_REGION}} 2>/dev/null cmds: - - printf '{{.C_STEP}}▸ Creating platform ALB...{{.C_RESET}}\n' + - printf '{{.C_STEP}}▸ Creating platform ALB (internal, CloudFront VPC Origin mode)...{{.C_RESET}}\n' - | set -e - # Use VPC from config.local.yaml cloudfront.vpcId if set, else derive from EKS cluster VPC_ID="${{.CLOUDFRONT_VPC_ID}}" if [ -z "$VPC_ID" ]; then VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) fi - PUBLIC_SUBNETS=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC_ID" "Name=mapPublicIpOnLaunch,Values=true" --region {{.AWS_REGION}} --query 'Subnets[*].SubnetId' --output text | tr '\t' ' ') - CLUSTER_SG=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text) + VPC_CIDR=$(aws ec2 describe-vpcs --vpc-ids "$VPC_ID" --region {{.AWS_REGION}} --query 'Vpcs[0].CidrBlock' --output text) + # Use private subnets for internal ALB (CloudFront VPC Origin reaches via private IP) + PRIVATE_SUBNETS=$(aws ec2 describe-subnets \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=*private*" "Name=tag:kubernetes.io/role/internal-elb,Values=1" \ + --region {{.AWS_REGION}} --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') + if [ -z "$PRIVATE_SUBNETS" ]; then + PRIVATE_SUBNETS=$(aws ec2 describe-subnets \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=*private*" \ + --region {{.AWS_REGION}} --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') + fi + if [ -z "$PRIVATE_SUBNETS" ]; then + printf '{{.C_ERR}}✗ No private subnets found in VPC %s — falling back to all subnets{{.C_RESET}}\n' "$VPC_ID" + PRIVATE_SUBNETS=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC_ID" \ + --region {{.AWS_REGION}} --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') + fi SG_NAME="{{.HUB_CLUSTER_NAME}}-platform-alb-sg" - SG_ID=$(aws ec2 describe-security-groups --filters "Name=group-name,Values=$SG_NAME" "Name=vpc-id,Values=$VPC_ID" --region {{.AWS_REGION}} --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null) + SG_ID=$(aws ec2 describe-security-groups \ + --filters "Name=group-name,Values=$SG_NAME" "Name=vpc-id,Values=$VPC_ID" \ + --region {{.AWS_REGION}} --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null) if [ "$SG_ID" = "None" ] || [ -z "$SG_ID" ]; then - SG_ID=$(aws ec2 create-security-group --group-name "$SG_NAME" --description "Platform ALB" --vpc-id "$VPC_ID" --region {{.AWS_REGION}} --query 'GroupId' --output text) - aws ec2 authorize-security-group-ingress --group-id "$SG_ID" --protocol tcp --port 80 --cidr 0.0.0.0/0 --region {{.AWS_REGION}} >/dev/null - aws ec2 authorize-security-group-ingress --group-id "$SG_ID" --protocol tcp --port 443 --cidr 0.0.0.0/0 --region {{.AWS_REGION}} >/dev/null - aws ec2 authorize-security-group-ingress --group-id "$CLUSTER_SG" --protocol -1 --source-group "$SG_ID" --region {{.AWS_REGION}} 2>/dev/null || true + SG_ID=$(aws ec2 create-security-group \ + --group-name "$SG_NAME" \ + --description "Platform ALB (internal, CloudFront VPC Origin)" \ + --vpc-id "$VPC_ID" --region {{.AWS_REGION}} \ + --query 'GroupId' --output text) + # Allow traffic from VPC CIDR (covers in-cluster + CloudFront VPC Origin) + aws ec2 authorize-security-group-ingress --group-id "$SG_ID" \ + --protocol tcp --port 80 --cidr "$VPC_CIDR" --region {{.AWS_REGION}} >/dev/null + # Also allow from CloudFront VPC Origins managed SG if it exists + CF_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ + --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") + if [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ]; then + aws ec2 authorize-security-group-ingress --group-id "$SG_ID" \ + --protocol tcp --port 80 --source-group "$CF_SG" --region {{.AWS_REGION}} >/dev/null 2>&1 || true + printf '{{.C_OK}}✓ Authorized CloudFront-VPCOrigins-Service-SG → ALB:80{{.C_RESET}}\n' + fi + # Allow EKS cluster SG to reach ALB + CLUSTER_SG=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} \ + --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text 2>/dev/null || echo "") + [ -n "$CLUSTER_SG" ] && [ "$CLUSTER_SG" != "None" ] && \ + aws ec2 authorize-security-group-ingress --group-id "$SG_ID" \ + --protocol -1 --source-group "$CLUSTER_SG" --region {{.AWS_REGION}} 2>/dev/null || true fi ALB_ARN=$(aws elbv2 create-load-balancer \ --name "{{.HUB_CLUSTER_NAME}}-platform" \ - --subnets $PUBLIC_SUBNETS \ + --subnets $PRIVATE_SUBNETS \ --security-groups "$SG_ID" \ - --scheme internet-facing \ + --scheme internal \ --type application \ --tags Key=elbv2.k8s.aws/cluster,Value={{.HUB_CLUSTER_NAME}} Key=ingress.k8s.aws/stack,Value=platform Key=ingress.k8s.aws/resource,Value=LoadBalancer \ --region {{.AWS_REGION}} \ --query 'LoadBalancers[0].LoadBalancerArn' --output text) aws elbv2 create-listener \ - --load-balancer-arn "$ALB_ARN" \ - --protocol HTTP --port 80 \ + --load-balancer-arn "$ALB_ARN" --protocol HTTP --port 80 \ --default-actions "Type=fixed-response,FixedResponseConfig={MessageBody=Not Found,StatusCode=404,ContentType=text/plain}" \ --region {{.AWS_REGION}} > /dev/null - ALB_DNS=$(aws elbv2 describe-load-balancers --load-balancer-arns "$ALB_ARN" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text) - echo "ALB created: $ALB_DNS" + printf '{{.C_OK}}✓ ALB created: {{.HUB_CLUSTER_NAME}}-platform (scheme=internal, mode=cloudfront){{.C_RESET}}\n' - printf '{{.C_OK}}✓ Platform ALB ready.{{.C_RESET}}\n' hub-distribution: - desc: Create CloudFront distribution for the platform ALB (idempotent) + desc: Create CloudFront distribution with VPC Origin pointing to the internal ALB (idempotent) status: - - aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" --output text | grep -q . + - | + [ "{{.EXPOSURE_MODE}}" != "cloudfront" ] || \ + aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" --output text | grep -q . cmds: - - printf '{{.C_STEP}}▸ Creating platform CloudFront distribution...{{.C_RESET}}\n' + - printf '{{.C_STEP}}▸ Creating platform CloudFront distribution (VPC Origin)...{{.C_RESET}}\n' - | set -e - ALB_DNS=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text) - CF_CONFIG=$(cat <<'CFEOF' - {"CallerReference":"{{.HUB_CLUSTER_NAME}}-TIMESTAMP","Comment":"{{.HUB_CLUSTER_NAME}}-platform","Enabled":true,"Origins":{"Quantity":1,"Items":[{"Id":"alb-origin","DomainName":"ALB_DNS_PLACEHOLDER","CustomOriginConfig":{"HTTPPort":80,"HTTPSPort":443,"OriginProtocolPolicy":"http-only","OriginSslProtocols":{"Quantity":1,"Items":["TLSv1.2"]}}}]},"DefaultCacheBehavior":{"TargetOriginId":"alb-origin","ViewerProtocolPolicy":"redirect-to-https","AllowedMethods":{"Quantity":7,"Items":["GET","HEAD","OPTIONS","PUT","POST","PATCH","DELETE"],"CachedMethods":{"Quantity":2,"Items":["GET","HEAD"]}},"CachePolicyId":"4135ea2d-6df8-44a3-9df3-4b5a84be39ad","OriginRequestPolicyId":"216adef6-5c7f-47e4-b989-5492eafa07d3","Compress":true},"ViewerCertificate":{"CloudFrontDefaultCertificate":true},"PriceClass":"PriceClass_100"} - CFEOF - ) - CF_CONFIG=$(echo "$CF_CONFIG" | sed "s|ALB_DNS_PLACEHOLDER|$ALB_DNS|g; s|TIMESTAMP|$(date +%s)|g") - CF_DOMAIN=$(aws cloudfront create-distribution --distribution-config "$CF_CONFIG" --query "Distribution.DomainName" --output text) + ALB_ARN=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" \ + --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text) + # Create or reuse VPC Origin + VPC_ORIGIN_NAME="{{.HUB_CLUSTER_NAME}}-platform-vpc-origin" + VPC_ORIGIN_ID=$(aws cloudfront list-vpc-origins \ + --query "VpcOriginList.Items[?Name=='$VPC_ORIGIN_NAME'].Id" --output text 2>/dev/null || echo "") + if [ -z "$VPC_ORIGIN_ID" ] || [ "$VPC_ORIGIN_ID" = "None" ]; then + VPC_ORIGIN_ID=$(aws cloudfront create-vpc-origin \ + --vpc-origin-endpoint-config "{ + \"Name\": \"$VPC_ORIGIN_NAME\", + \"Arn\": \"$ALB_ARN\", + \"HTTPPort\": 80, + \"HTTPSPort\": 443, + \"OriginProtocolPolicy\": \"http-only\", + \"OriginSslProtocols\": {\"Quantity\": 1, \"Items\": [\"TLSv1.2\"]} + }" --query 'VpcOrigin.Id' --output text) + printf '{{.C_OK}}✓ VPC Origin created: %s{{.C_RESET}}\n' "$VPC_ORIGIN_ID" + else + printf '{{.C_INFO}}↻ Reusing VPC Origin: %s{{.C_RESET}}\n' "$VPC_ORIGIN_ID" + fi + # Wait for VPC Origin to be Deployed (~5-10 min) + printf '{{.C_STEP}}▸ Waiting for VPC Origin Deployed...{{.C_RESET}}\n' + for i in $(seq 1 60); do + STATE=$(aws cloudfront get-vpc-origin --id "$VPC_ORIGIN_ID" \ + --query 'VpcOrigin.Status' --output text 2>/dev/null || echo "Pending") + printf '{{.C_INFO}} [%s/60] VPC Origin status: %s{{.C_RESET}}\n' "$i" "$STATE" + [ "$STATE" = "Deployed" ] && break + sleep 15 + done + [ "$STATE" != "Deployed" ] && echo "ERROR: VPC Origin not Deployed" && exit 1 + # Authorize CloudFront VPC Origins SG on ALB SG (created after first VPC Origin) + VPC_ID=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" \ + --region {{.AWS_REGION}} --query 'LoadBalancers[0].VpcId' --output text) + ALB_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ + --filters "Name=group-name,Values={{.HUB_CLUSTER_NAME}}-platform-alb-sg" "Name=vpc-id,Values=$VPC_ID" \ + --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") + CF_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ + --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") + if [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ] && [ -n "$ALB_SG" ] && [ "$ALB_SG" != "None" ]; then + aws ec2 authorize-security-group-ingress --group-id "$ALB_SG" \ + --protocol tcp --port 80 --source-group "$CF_SG" --region {{.AWS_REGION}} 2>/dev/null || true + printf '{{.C_OK}}✓ Authorized %s → %s:80 (post VPC Origin){{.C_RESET}}\n' "$CF_SG" "$ALB_SG" + fi + # Create CloudFront distribution with VPC Origin + CF_DOMAIN=$(aws cloudfront create-distribution \ + --distribution-config "{ + \"CallerReference\": \"{{.HUB_CLUSTER_NAME}}-platform-$(date +%s)\", + \"Comment\": \"{{.HUB_CLUSTER_NAME}}-platform\", + \"Enabled\": true, + \"Origins\": {\"Quantity\": 1, \"Items\": [{ + \"Id\": \"vpc-origin\", + \"DomainName\": \"$ALB_ARN\", + \"VpcOriginConfig\": {\"VpcOriginId\": \"$VPC_ORIGIN_ID\", \"OriginReadTimeout\": 60, \"OriginKeepaliveTimeout\": 5} + }]}, + \"DefaultCacheBehavior\": { + \"TargetOriginId\": \"vpc-origin\", + \"ViewerProtocolPolicy\": \"redirect-to-https\", + \"AllowedMethods\": {\"Quantity\": 7, \"Items\": [\"GET\",\"HEAD\",\"OPTIONS\",\"PUT\",\"POST\",\"PATCH\",\"DELETE\"], \"CachedMethods\": {\"Quantity\": 2, \"Items\": [\"GET\",\"HEAD\"]}}, + \"CachePolicyId\": \"4135ea2d-6df8-44a3-9df3-4b5a84be39ad\", + \"OriginRequestPolicyId\": \"216adef6-5c7f-47e4-b989-5492eafa07d3\", + \"Compress\": true + }, + \"ViewerCertificate\": {\"CloudFrontDefaultCertificate\": true}, + \"PriceClass\": \"PriceClass_100\" + }" \ + --query "Distribution.DomainName" --output text) mkdir -p {{.ROOT_DIR}}/private echo -n "$CF_DOMAIN" > {{.ROOT_DIR}}/private/cloudfront-domain - # Write cloudfrontDomain into config.local.yaml so all tasks use the ALB domain + # Write cloudfrontDomain into config.local.yaml yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} - echo "CloudFront: $CF_DOMAIN (copy this to config.local.yaml cloudfront.cloudfrontDomain)" - - printf '{{.C_OK}}✓ Platform CloudFront ready.{{.C_RESET}}\n' + printf '{{.C_OK}}✓ CloudFront created: %s{{.C_RESET}}\n' "$CF_DOMAIN" + - printf '{{.C_OK}}✓ Platform CloudFront ready (VPC Origin).{{.C_RESET}}\n' # gitlab-nlb and gitlab-distribution removed — GitLab CloudFront is now managed by CDK (team-stack.ts). - From ec69c0351e5075774a9068e13cf997c6047cf942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 19:14:06 +0200 Subject: [PATCH 127/231] fix: capture user1 GitLab PAT after init and update peeks-hub/secrets.git_token so Backstage uses real PAT --- cluster-providers/workshop/Taskfile.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 83ad6ba47..39a9d1d81 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -363,6 +363,20 @@ tasks: end puts 'init done' " 2>&1 | tail -20 || true + # Capture user1's PAT token and update peeks-hub/secrets so Backstage/ESO use the real PAT + USER1_PAT=$(docker exec gitlab-gitlab-1 gitlab-rails runner \ + "puts User.find_by(username:'{{.GIT_USERNAME}}').personal_access_tokens.find_by(name:'workshop-token')&.token" \ + 2>/dev/null | tail -1) + if [ -n "$USER1_PAT" ] && [ ${#USER1_PAT} -gt 10 ]; then + CURRENT=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets \ + --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null) + if [ -n "$CURRENT" ]; then + NEW=$(echo "$CURRENT" | jq -c --arg pat "$USER1_PAT" '.git_token = $pat') + aws secretsmanager put-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets \ + --region {{.AWS_REGION}} --secret-string "$NEW" >/dev/null 2>&1 && \ + printf ' Updated peeks-hub/secrets.git_token with real GitLab PAT\n' + fi + fi WORKSHOP_DIR="$HOME/environment/{{.WORKING_REPO}}" ROOT_TOKEN=$(docker exec gitlab-gitlab-1 gitlab-rails runner "puts User.find(1).personal_access_tokens.find_or_initialize_by(name: 'root-api').tap{|p| p.scopes=['api','sudo']; p.expires_at=365.days.from_now; p.save!}.token" 2>/dev/null | tail -1) for REPO in {{.REPOS}}; do From ff14bac1b9f1d06bc268fd66ebee5afa185be99f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 19:14:49 +0200 Subject: [PATCH 128/231] =?UTF-8?q?fix:=20keycloak=20create=5Fclient=20ups?= =?UTF-8?q?erts=20(updates=20redirectUris=20if=20client=20exists)=20?= =?UTF-8?q?=E2=80=94=20prevents=20stale=20domain=20in=20OIDC=20clients?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../keycloak/templates/keycloak-config.yaml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/gitops/addons/charts/keycloak/templates/keycloak-config.yaml b/gitops/addons/charts/keycloak/templates/keycloak-config.yaml index 86f871ac5..4d0b9d228 100644 --- a/gitops/addons/charts/keycloak/templates/keycloak-config.yaml +++ b/gitops/addons/charts/keycloak/templates/keycloak-config.yaml @@ -731,13 +731,22 @@ spec: keycloak_post "/admin/realms/${REALM_NAME}/roles" "role-${role_name}-payload.json" "Role ${role_name}" } - # Create client function + # Create or update client function (idempotent upsert) create_client() { local client_name=$1 local payload_file=$2 + local client_id_name + client_id_name=$(jq -r '.clientId' "${payload_file}") - echo "creating ${client_name} client" - keycloak_post "/admin/realms/${REALM_NAME}/clients" "${payload_file}" "${client_name} client" + # Check if client already exists + EXISTING_ID=$(keycloak_get "/admin/realms/${REALM_NAME}/clients" ".[] | select(.clientId == \"${client_id_name}\") | .id" 2>/dev/null) || true + if [ -n "${EXISTING_ID}" ]; then + echo "updating ${client_name} client (exists: ${EXISTING_ID})" + keycloak_put "/admin/realms/${REALM_NAME}/clients/${EXISTING_ID}" "${payload_file}" "${client_name} client" + else + echo "creating ${client_name} client" + keycloak_post "/admin/realms/${REALM_NAME}/clients" "${payload_file}" "${client_name} client" + fi } # Add client scope function From f8f74238eb433bd951165d3d17c71610286bd8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 21:35:14 +0200 Subject: [PATCH 129/231] fix: seed platform/backstage/templates into fleet-config + reduce git-credentials ESO refresh to 5m --- cluster-providers/workshop/Taskfile.yaml | 5 +++++ gitops/addons/charts/backstage/templates/argocd-secrets.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 39a9d1d81..b3d494fc0 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -394,6 +394,11 @@ tasks: [ -d "$SRC" ] || continue TMP=$(mktemp -d) cp -r "$SRC/." "$TMP/" + # For fleet-config also seed platform/backstage/templates (Backstage scaffolder catalog) + if [ "$REPO" = "fleet-config" ] && [ -d "$WORKSHOP_DIR/platform/backstage/templates" ]; then + mkdir -p "$TMP/platform/backstage/templates" + cp -r "$WORKSHOP_DIR/platform/backstage/templates/." "$TMP/platform/backstage/templates/" + fi cd "$TMP" git init -q && git add -A && git commit -qm "init" 2>/dev/null || true git remote add origin "$REMOTE_URL" diff --git a/gitops/addons/charts/backstage/templates/argocd-secrets.yaml b/gitops/addons/charts/backstage/templates/argocd-secrets.yaml index e5ede8ca2..1d8b55982 100644 --- a/gitops/addons/charts/backstage/templates/argocd-secrets.yaml +++ b/gitops/addons/charts/backstage/templates/argocd-secrets.yaml @@ -95,7 +95,7 @@ spec: secretStoreRef: name: aws-secrets-manager kind: ClusterSecretStore - refreshInterval: "1h" + refreshInterval: "5m" target: name: git-credentials creationPolicy: Owner From d598a9f9085cf3ab5de10f2e4bf8cc458ca49c21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 21:38:54 +0200 Subject: [PATCH 130/231] =?UTF-8?q?fix:=20remove=20dollar=20sign=20from=20?= =?UTF-8?q?CLOUDFRONT=5FVPC=5FID=20Taskfile=20template=20=E2=80=94=20was?= =?UTF-8?q?=20rendering=20as=20literal=20dollar=20sign=20when=20empty?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml index ef21846cc..46b36a777 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml @@ -38,7 +38,7 @@ tasks: - printf '{{.C_STEP}}▸ Creating platform ALB (internal, CloudFront VPC Origin mode)...{{.C_RESET}}\n' - | set -e - VPC_ID="${{.CLOUDFRONT_VPC_ID}}" + VPC_ID="{{.CLOUDFRONT_VPC_ID}}" if [ -z "$VPC_ID" ]; then VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) fi From 2f71a950aff70ef22fcbfd70ce8e89a6a1ff7378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 21:59:54 +0200 Subject: [PATCH 131/231] fix: deduplicate subnets by AZ when creating internal ALB to avoid InvalidConfigurationRequest --- .../kind-crossplane/Taskfile.cloudfront.yaml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml index 46b36a777..85de2b80e 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml @@ -43,19 +43,25 @@ tasks: VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) fi VPC_CIDR=$(aws ec2 describe-vpcs --vpc-ids "$VPC_ID" --region {{.AWS_REGION}} --query 'Vpcs[0].CidrBlock' --output text) - # Use private subnets for internal ALB (CloudFront VPC Origin reaches via private IP) + # Use private subnets for internal ALB — one per AZ PRIVATE_SUBNETS=$(aws ec2 describe-subnets \ --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=*private*" "Name=tag:kubernetes.io/role/internal-elb,Values=1" \ - --region {{.AWS_REGION}} --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') + --region {{.AWS_REGION}} \ + --query 'Subnets[].[SubnetId,AvailabilityZone]' --output text | \ + sort -k2 -u | awk '{print $1}' | tr '\n' ' ') if [ -z "$PRIVATE_SUBNETS" ]; then PRIVATE_SUBNETS=$(aws ec2 describe-subnets \ --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=*private*" \ - --region {{.AWS_REGION}} --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') + --region {{.AWS_REGION}} \ + --query 'Subnets[].[SubnetId,AvailabilityZone]' --output text | \ + sort -k2 -u | awk '{print $1}' | tr '\n' ' ') fi if [ -z "$PRIVATE_SUBNETS" ]; then - printf '{{.C_ERR}}✗ No private subnets found in VPC %s — falling back to all subnets{{.C_RESET}}\n' "$VPC_ID" + printf '{{.C_ERR}}✗ No private subnets found in VPC %s — falling back to all subnets (one per AZ){{.C_RESET}}\n' "$VPC_ID" PRIVATE_SUBNETS=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC_ID" \ - --region {{.AWS_REGION}} --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') + --region {{.AWS_REGION}} \ + --query 'Subnets[].[SubnetId,AvailabilityZone]' --output text | \ + sort -k2 -u | awk '{print $1}' | tr '\n' ' ') fi SG_NAME="{{.HUB_CLUSTER_NAME}}-platform-alb-sg" SG_ID=$(aws ec2 describe-security-groups \ From a5f11f27c346d08efac21199860ab4ae0a6779aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Tue, 30 Jun 2026 22:36:35 +0200 Subject: [PATCH 132/231] =?UTF-8?q?fix:=20use=20ALB=20DNS=20(not=20ARN)=20?= =?UTF-8?q?as=20DomainName=20in=20CloudFront=20distribution=20=E2=80=94=20?= =?UTF-8?q?ARN=20contains=20colons=20which=20are=20invalid?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml index 85de2b80e..6fdbf9558 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml @@ -162,6 +162,8 @@ tasks: --protocol tcp --port 80 --source-group "$CF_SG" --region {{.AWS_REGION}} 2>/dev/null || true printf '{{.C_OK}}✓ Authorized %s → %s:80 (post VPC Origin){{.C_RESET}}\n' "$CF_SG" "$ALB_SG" fi + ALB_DNS=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" \ + --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text) # Create CloudFront distribution with VPC Origin CF_DOMAIN=$(aws cloudfront create-distribution \ --distribution-config "{ @@ -170,7 +172,7 @@ tasks: \"Enabled\": true, \"Origins\": {\"Quantity\": 1, \"Items\": [{ \"Id\": \"vpc-origin\", - \"DomainName\": \"$ALB_ARN\", + \"DomainName\": \"$ALB_DNS\", \"VpcOriginConfig\": {\"VpcOriginId\": \"$VPC_ORIGIN_ID\", \"OriginReadTimeout\": 60, \"OriginKeepaliveTimeout\": 5} }]}, \"DefaultCacheBehavior\": { From 633510f8c90c9c739eabaf9dfee91263448441c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 10:37:49 +0200 Subject: [PATCH 133/231] fix: Issue#11 add dynamodb+services.k8s.aws RBAC, Issue#12 trust peeks-spoke-*-ack-capability-role, Issue#14/15 add iam:ListRoleTags, Issue#5 remove keycloak gitlab hostname fallback --- cluster-providers/kind-crossplane/Taskfile.yaml | 5 +++-- .../charts/eks-capabilities-rbac/templates/rbac.yaml | 8 ++++++++ gitops/addons/charts/keycloak/templates/install.yaml | 7 ++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 3a51d027a..de5877781 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -535,7 +535,8 @@ tasks: "Action": ["sts:AssumeRole", "sts:TagSession"], "Condition": {"ArnLike": {"aws:PrincipalArn": [ "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.ACK_CAPABILITY_ROLE}}", - "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-ack-*" + "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-ack-*", + "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-spoke-*-ack-capability-role" ]}} }] } @@ -570,7 +571,7 @@ tasks: "Version": "2012-10-17", "Statement": [ {"Effect": "Allow", "Action": ["eks:*"], "Resource": "*"}, - {"Effect": "Allow", "Action": ["iam:PassRole","iam:GetRole"], "Resource": "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-*"} + {"Effect": "Allow", "Action": ["iam:PassRole","iam:GetRole","iam:ListRoleTags"], "Resource": "arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RESOURCE_PREFIX}}-*"} ] }' # Allow ACK capability role to assume cluster-mgmt roles diff --git a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml index 2dda7ba18..0916a5f43 100644 --- a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml +++ b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml @@ -72,6 +72,14 @@ rules: - apiGroups: ["secretsmanager.services.k8s.aws"] resources: ["*"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# ACK DynamoDB +- apiGroups: ["dynamodb.services.k8s.aws"] + resources: ["*"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# ACK Services (IAMRoleSelector and other cross-service CRDs) +- apiGroups: ["services.k8s.aws"] + resources: ["*"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/gitops/addons/charts/keycloak/templates/install.yaml b/gitops/addons/charts/keycloak/templates/install.yaml index f89029533..1a1515be0 100644 --- a/gitops/addons/charts/keycloak/templates/install.yaml +++ b/gitops/addons/charts/keycloak/templates/install.yaml @@ -150,9 +150,10 @@ data: # The username of the database user. db-url=jdbc:postgresql://postgresql.keycloak.svc.cluster.local:5432/postgres - # hostname settings - hostname=https://{{ .Values.global.ingress_domain_name | default .Values.global.gitlab_domain_name }}/keycloak - hostname-admin=https://{{ .Values.global.ingress_domain_name | default .Values.global.gitlab_domain_name }}/keycloak + # hostname settings — must use ingress_domain_name (platform CloudFront domain) + # Do NOT fall back to gitlab_domain_name — that domain routes to GitLab, not Keycloak + hostname=https://{{ .Values.global.ingress_domain_name }}/keycloak + hostname-admin=https://{{ .Values.global.ingress_domain_name }}/keycloak hostname-strict=false http-relative-path=keycloak http-enabled=true From 938551ce5c929c5f77ca330ea937a9bb479f0a26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 10:41:37 +0200 Subject: [PATCH 134/231] fix: add hub:sync-cloudfront-domain to ensure cloudfrontDomain written to config even when hub:cloudfront is skipped --- cluster-providers/kind-kro-ack/Taskfile.yaml | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index c191bf831..89642d936 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -113,6 +113,7 @@ tasks: - task: hub:wait-for-eks - task: hub:ingress - task: hub:cloudfront + - task: hub:sync-cloudfront-domain - task: hub:seed - task: hub:wait-for-sync vars: { MAX_OUTOFSYNC: "20" } @@ -847,6 +848,27 @@ tasks: yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} rm -f /tmp/hub-vpc-origin-id + hub:sync-cloudfront-domain: + desc: Ensure cloudfrontDomain is written to config (idempotent, runs even if hub:cloudfront was skipped) + internal: true + cmds: + - | + CURRENT=$(yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}}) + if [ -n "$CURRENT" ]; then + printf '{{.C_INFO}} cloudfrontDomain already set: %s{{.C_RESET}}\n' "$CURRENT" + exit 0 + fi + # Look up the existing CloudFront distribution for this cluster + CF_DOMAIN=$(aws cloudfront list-distributions \ + --query "DistributionList.Items[?Comment=='{{.RESOURCE_PREFIX}}-hub-ingress'].DomainName" \ + --output text 2>/dev/null | tr -d '[:space:]') + if [ -n "$CF_DOMAIN" ] && [ "$CF_DOMAIN" != "None" ]; then + yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} + printf '{{.C_OK}}✓ cloudfrontDomain synced to config: %s{{.C_RESET}}\n' "$CF_DOMAIN" + else + printf '{{.C_WARN}} No peeks-hub-ingress CloudFront distribution found{{.C_RESET}}\n' + fi + hub:seed: desc: Seed hub cluster after EKS is ACTIVE cmds: From 5bdd5de386e0e794f2ea6215d3335ed871b86313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 10:48:18 +0200 Subject: [PATCH 135/231] =?UTF-8?q?fix:=20CLOUDFRONT=5FDOMAIN=20reads=20fr?= =?UTF-8?q?om=20private/cloudfront-domain=20file=20as=20fallback=20?= =?UTF-8?q?=E2=80=94=20survives=20task=20re-runs=20without=20config?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kind-crossplane/Taskfile.yaml | 2 +- cluster-providers/kind-kro-ack/Taskfile.yaml | 32 ++++++++++--------- cluster-providers/workshop/Taskfile.yaml | 2 +- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index de5877781..9c34125b1 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -63,7 +63,7 @@ vars: EXPOSURE_MODE: sh: 'if [ -z "$(yq ''.domain // ""'' {{.CONFIG_FILE}})" ]; then echo "cloudfront"; else echo "domain"; fi' CLOUDFRONT_DOMAIN: - sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true CLOUDFRONT_VPC_ID: sh: yq '.cloudfront.vpcId // ""' {{.CONFIG_FILE}} CAPABILITY_NAME: diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 89642d936..ce31bd7e0 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -30,7 +30,7 @@ vars: DOMAIN: sh: yq '.domain' {{.CONFIG_FILE}} CLOUDFRONT_DOMAIN: - sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true RESOURCE_PREFIX: sh: yq '.resourcePrefix // ""' {{.CONFIG_FILE}} INGRESS_NAME: @@ -844,29 +844,31 @@ tasks: }" --query 'Distribution.[Id,DomainName]' --output text) CF_DOMAIN=$(echo "$DIST_OUT" | awk '{print $2}') printf '{{.C_OK}}✓ CloudFront created: %s{{.C_RESET}}\n' "$CF_DOMAIN" - # Store domain in config for hub:seed to use + # Store domain in config and private file for hub:seed and retries to use yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} + mkdir -p {{.ROOT_DIR}}/private + echo -n "$CF_DOMAIN" > {{.ROOT_DIR}}/private/cloudfront-domain rm -f /tmp/hub-vpc-origin-id hub:sync-cloudfront-domain: - desc: Ensure cloudfrontDomain is written to config (idempotent, runs even if hub:cloudfront was skipped) + desc: Ensure cloudfrontDomain is written to config AND private/cloudfront-domain file (idempotent) internal: true cmds: - | - CURRENT=$(yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}}) - if [ -n "$CURRENT" ]; then - printf '{{.C_INFO}} cloudfrontDomain already set: %s{{.C_RESET}}\n' "$CURRENT" - exit 0 + CURRENT=$(yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true) + if [ -z "$CURRENT" ]; then + # Look up the existing CloudFront distribution + CURRENT=$(aws cloudfront list-distributions \ + --query "DistributionList.Items[?Comment=='{{.RESOURCE_PREFIX}}-hub-ingress'].DomainName" \ + --output text 2>/dev/null | tr -d '[:space:]') fi - # Look up the existing CloudFront distribution for this cluster - CF_DOMAIN=$(aws cloudfront list-distributions \ - --query "DistributionList.Items[?Comment=='{{.RESOURCE_PREFIX}}-hub-ingress'].DomainName" \ - --output text 2>/dev/null | tr -d '[:space:]') - if [ -n "$CF_DOMAIN" ] && [ "$CF_DOMAIN" != "None" ]; then - yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} - printf '{{.C_OK}}✓ cloudfrontDomain synced to config: %s{{.C_RESET}}\n' "$CF_DOMAIN" + if [ -n "$CURRENT" ] && [ "$CURRENT" != "None" ]; then + yq -i ".cloudfront.cloudfrontDomain = \"$CURRENT\"" {{.CONFIG_FILE}} + mkdir -p {{.ROOT_DIR}}/private + echo -n "$CURRENT" > {{.ROOT_DIR}}/private/cloudfront-domain + printf '{{.C_OK}}✓ cloudfrontDomain: %s{{.C_RESET}}\n' "$CURRENT" else - printf '{{.C_WARN}} No peeks-hub-ingress CloudFront distribution found{{.C_RESET}}\n' + printf '{{.C_WARN}} No CloudFront distribution found for {{.RESOURCE_PREFIX}}-hub-ingress{{.C_RESET}}\n' fi hub:seed: diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index b3d494fc0..63140599c 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -29,7 +29,7 @@ vars: EXPOSURE_MODE: sh: 'if [ -z "$(yq ''.domain // ""'' {{.CONFIG_FILE}})" ]; then echo "cloudfront"; else echo "domain"; fi' CLOUDFRONT_DOMAIN: - sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true C_STEP: sh: printf '\033[1;33m' C_OK: From 4ee53810b480aa7f94006f2261e65bbff429e4cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 10:55:33 +0200 Subject: [PATCH 136/231] ux: add timing markers to workshop:install (gitlab init, provider install, ray, post-install) --- cluster-providers/workshop/Taskfile.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 63140599c..9b5177dcd 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -53,26 +53,32 @@ tasks: vars: PROVIDER: sh: yq '.clusterProvider // "kind-crossplane"' {{.CONFIG_FILE}} + INSTALL_START: + sh: date +%s cmds: - printf '{{.C_STEP}}▸ Workshop install starting...{{.C_RESET}}\n' # Initialize GitLab FIRST — it starts at EC2 boot and is ready before platform install. # This ensures fleet-config exists when hub:set-overlay-repo and spokes run inside # {{.PROVIDER}}:install (which needs fleet-config during phase2). - task: gitlab:init-ec2 + - 'printf "{{.C_INFO}} [%ds] gitlab:init-ec2 done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # Platform install (EKS cluster, addons, spokes) - task: ":{{.PROVIDER}}:install" + - 'printf "{{.C_INFO}} [%ds] {{.PROVIDER}}:install done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - printf '{{.C_OK}}✓ Platform install complete.{{.C_RESET}}\n' # Ray ML infrastructure (image builds in background) - task: ":{{.PROVIDER}}:ray:setup" + - 'printf "{{.C_INFO}} [%ds] ray:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # IDC/Keycloak federation (ArgoCD SSO) - task: post-install + - 'printf "{{.C_INFO}} [%ds] post-install done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # Wait for Ray image to be ready - task: ":{{.PROVIDER}}:ray:wait-image" # Download ML models to S3 - task: ":{{.PROVIDER}}:ray:prestage-models" # Write platform URLs + credentials to ~/.bashrc.d/platform.sh for IDE use - task: setup-env - - printf '{{.C_OK}}✓ Workshop install complete.{{.C_RESET}}\n' + - 'printf "{{.C_OK}}✓ Workshop install complete in %ds.{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # ============================================================ # PRE-REQUISITES From 4b6444f29e6f94ad480d5fd9a46e531de8e98549 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 11:14:22 +0200 Subject: [PATCH 137/231] =?UTF-8?q?fix:=20add=20debug=20output=20to=20flee?= =?UTF-8?q?t-config=20wait=20loop=20=E2=80=94=20show=20URL=20and=20last=20?= =?UTF-8?q?git=20ls-remote=20error=20on=20timeout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 8 ++++---- cluster-providers/kind-kro-ack/Taskfile.yaml | 10 ++++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 9c34125b1..d8f16e7d1 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -1001,7 +1001,7 @@ tasks: # repo to be reachable, then verify the clone produced a git repo. printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' for i in $(seq 1 30); do - git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + ERR=$(git ls-remote "$FLEET_URL" 2>&1) && break if [ "$i" -eq 30 ]; then printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot enable spoke {{.CLUSTER}} via Crossplane.{{.C_RESET}}\n' printf '{{.C_INFO}} GitLab/gitlab-init may still be initializing. Re-run later: task kind-crossplane:spokes:enable-crossplane -- {{.CLUSTER}}{{.C_RESET}}\n' @@ -1208,7 +1208,7 @@ tasks: # verify the clone actually produced a git repo before proceeding. printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' for i in $(seq 1 30); do - git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + ERR=$(git ls-remote "$FLEET_URL" 2>&1) && break if [ "$i" -eq 30 ]; then printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot enable spoke {{.CLUSTER}} via KRO.{{.C_RESET}}\n' printf '{{.C_INFO}} GitLab/gitlab-init may still be initializing. Re-run later: task kind-crossplane:spokes:enable-kro -- {{.CLUSTER}}{{.C_RESET}}\n' @@ -1352,7 +1352,7 @@ tasks: # removal (stale overlay left behind). Wait for fleet-config then verify. printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' for i in $(seq 1 30); do - git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + ERR=$(git ls-remote "$FLEET_URL" 2>&1) && break if [ "$i" -eq 30 ]; then printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot disable spoke {{.CLUSTER}} via Crossplane.{{.C_RESET}}\n' exit 1 @@ -1416,7 +1416,7 @@ tasks: # Wait for fleet-config then verify the clone produced a git repo. printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' for i in $(seq 1 30); do - git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + ERR=$(git ls-remote "$FLEET_URL" 2>&1) && break if [ "$i" -eq 30 ]; then printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot disable spoke {{.CLUSTER}} via KRO.{{.C_RESET}}\n' exit 1 diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index ce31bd7e0..b658d4f40 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1867,10 +1867,12 @@ tasks: # repo to be reachable, then verify the clone produced a git repo. printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' for i in $(seq 1 30); do - git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + ERR=$(git ls-remote "$FLEET_URL" 2>&1) && break if [ "$i" -eq 30 ]; then printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot enable spoke {{.CLUSTER}} via Crossplane.{{.C_RESET}}\n' printf '{{.C_INFO}} GitLab/gitlab-init may still be initializing. Re-run later: task kind-crossplane:spokes:enable-crossplane -- {{.CLUSTER}}{{.C_RESET}}\n' + printf '{{.C_INFO}} Last error: %s{{.C_RESET}}\n' "$ERR" + printf '{{.C_INFO}} URL checked: %s{{.C_RESET}}\n' "$FLEET_URL" exit 1 fi sleep 10 @@ -2076,7 +2078,7 @@ tasks: # verify the clone actually produced a git repo before proceeding. printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' for i in $(seq 1 30); do - git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + ERR=$(git ls-remote "$FLEET_URL" 2>&1) && break if [ "$i" -eq 30 ]; then printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot enable spoke {{.CLUSTER}} via KRO.{{.C_RESET}}\n' printf '{{.C_INFO}} GitLab/gitlab-init may still be initializing. Re-run later: task kind-crossplane:spokes:enable-kro -- {{.CLUSTER}}{{.C_RESET}}\n' @@ -2225,7 +2227,7 @@ tasks: # removal (stale overlay left behind). Wait for fleet-config then verify. printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' for i in $(seq 1 30); do - git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + ERR=$(git ls-remote "$FLEET_URL" 2>&1) && break if [ "$i" -eq 30 ]; then printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot disable spoke {{.CLUSTER}} via Crossplane.{{.C_RESET}}\n' exit 1 @@ -2290,7 +2292,7 @@ tasks: # Wait for fleet-config then verify the clone produced a git repo. printf '{{.C_STEP}}▸ Waiting for fleet-config repo to be reachable...{{.C_RESET}}\n' for i in $(seq 1 30); do - git ls-remote "$FLEET_URL" >/dev/null 2>&1 && break + ERR=$(git ls-remote "$FLEET_URL" 2>&1) && break if [ "$i" -eq 30 ]; then printf '{{.C_ERR}}⚠ fleet-config not reachable after 5min — cannot disable spoke {{.CLUSTER}} via KRO.{{.C_RESET}}\n' exit 1 From 3e98a3da5300e9720988051c3aba52a726554f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 11:26:27 +0200 Subject: [PATCH 138/231] fix: GIT_PASSWORD in kro-ack tasks reads from peeks-hub/secrets.git_token (real PAT) not keycloak.user_password --- cluster-providers/kind-kro-ack/Taskfile.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index b658d4f40..e897ef6b2 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1194,7 +1194,7 @@ tasks: GIT_USERNAME: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' # Root of the fleet-config overlay structure (configs/, overlays/) per the # GitLab init-job seed. Keep "" unless the overlay repo nests under a prefix. OVERLAY_REPO_BASEPATH: "" @@ -1854,7 +1854,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:enable-crossplane -- "; exit 1; fi @@ -2064,7 +2064,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:enable-kro -- "; exit 1; fi @@ -2213,7 +2213,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:disable-crossplane -- "; exit 1; fi @@ -2280,7 +2280,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:disable-kro -- "; exit 1; fi @@ -2369,7 +2369,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | set -u From 60c8a89ce6304ca57360e58efca5b1fa5168111c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 11:27:25 +0200 Subject: [PATCH 139/231] fix: GIT_PASSWORD in kind-crossplane reads from peeks-hub/secrets.git_token (real PAT) not keycloak.user_password --- cluster-providers/kind-crossplane/Taskfile.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index d8f16e7d1..3372a9e2f 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -819,7 +819,7 @@ tasks: GIT_USERNAME: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' # Root of the fleet-config overlay structure (configs/, overlays/) per the # GitLab init-job seed. Keep "" unless the overlay repo nests under a prefix. OVERLAY_REPO_BASEPATH: "" @@ -988,7 +988,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:enable-crossplane -- "; exit 1; fi @@ -1194,7 +1194,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:enable-kro -- "; exit 1; fi @@ -1338,7 +1338,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:disable-crossplane -- "; exit 1; fi @@ -1404,7 +1404,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:disable-kro -- "; exit 1; fi @@ -1492,7 +1492,7 @@ tasks: echo "{{.DOMAIN}}" fi GIT_PASSWORD: - sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || (aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password') + sh: yq '.hub.gitlabRootPassword // ""' {{.CONFIG_FILE}} | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -v '^$' || aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password' cmds: - | set -u From d8a8b54fe85efa3051066a7981ca8cfcd5072457 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 11:44:39 +0200 Subject: [PATCH 140/231] =?UTF-8?q?fix:=20gitlab:init-ec2=20status=20check?= =?UTF-8?q?=20also=20verifies=20git=5Ftoken=20is=20a=20real=20PAT=20(<=3D2?= =?UTF-8?q?5=20chars)=20=E2=80=94=20re-runs=20if=20only=20user=20password?= =?UTF-8?q?=20stored?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/workshop/Taskfile.yaml | 25 +++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 9b5177dcd..a067c63a6 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -277,8 +277,13 @@ tasks: gitlab:init-ec2: desc: Wait for GitLab CE (docker-compose on IDE EC2) to be ready, then initialise root token + user1 + repos status: - # Fast check: if fleet-config git repo exists on disk, init already ran - - docker exec gitlab-gitlab-1 test -d /var/opt/gitlab/git-data/repositories/@hashed 2>/dev/null && docker exec gitlab-gitlab-1 find /var/opt/gitlab/git-data/repositories/@hashed -name "*.git" -maxdepth 3 | grep -q . + # Skip if fleet-config git repo exists on disk AND git_token in secrets is a real PAT (not user password) + # A real GitLab PAT is exactly 20 chars; user password is 32 chars + - | + docker exec gitlab-gitlab-1 test -d /var/opt/gitlab/git-data/repositories/@hashed 2>/dev/null && \ + docker exec gitlab-gitlab-1 find /var/opt/gitlab/git-data/repositories/@hashed -name "*.git" -maxdepth 3 | grep -q . && \ + PAT=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty') && \ + [ ${#PAT} -le 25 ] && [ ${#PAT} -gt 10 ] vars: GITLAB_CF_DOMAIN: sh: | @@ -354,10 +359,16 @@ tasks: u.update_columns(namespace_id: ns.id) if ns.persisted? u.reload end - u1pat = u.personal_access_tokens.find_or_initialize_by(name: 'workshop-token') - u1pat.scopes = ['api','write_repository'] - u1pat.expires_at = 365.days.from_now - u1pat.save! + # Always recreate workshop-token PAT so we can capture the plaintext token value + # (GitLab only exposes plaintext token at creation time — find_or_initialize reuses + # an existing hashed token where .token returns nil) + u.personal_access_tokens.where(name: 'workshop-token').delete_all + u1pat = u.personal_access_tokens.create!( + name: 'workshop-token', + scopes: ['api','write_repository','read_repository'], + expires_at: 365.days.from_now + ) + puts 'user1_pat=' + u1pat.token.to_s %w[fleet-config java rust golang dotnet next-js].each do |repo| p = Project.find_by(path: repo, namespace_id: u.namespace_id) if p.nil? @@ -371,7 +382,7 @@ tasks: " 2>&1 | tail -20 || true # Capture user1's PAT token and update peeks-hub/secrets so Backstage/ESO use the real PAT USER1_PAT=$(docker exec gitlab-gitlab-1 gitlab-rails runner \ - "puts User.find_by(username:'{{.GIT_USERNAME}}').personal_access_tokens.find_by(name:'workshop-token')&.token" \ + "puts User.find_by(username:'{{.GIT_USERNAME}}').personal_access_tokens.where(name:'workshop-token').delete_all; pat=User.find_by(username:'{{.GIT_USERNAME}}').personal_access_tokens.create!(name:'workshop-token',scopes:['api','write_repository','read_repository'],expires_at:365.days.from_now); puts pat.token" \ 2>/dev/null | tail -1) if [ -n "$USER1_PAT" ] && [ ${#USER1_PAT} -gt 10 ]; then CURRENT=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets \ From e094491e03a39036465b33ccde8fdcfe88a83f14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 11:46:59 +0200 Subject: [PATCH 141/231] fix: detect real GitLab PAT by glpat- prefix (not length) in gitlab:init-ec2 status check --- cluster-providers/workshop/Taskfile.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index a067c63a6..ba312858f 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -277,13 +277,11 @@ tasks: gitlab:init-ec2: desc: Wait for GitLab CE (docker-compose on IDE EC2) to be ready, then initialise root token + user1 + repos status: - # Skip if fleet-config git repo exists on disk AND git_token in secrets is a real PAT (not user password) - # A real GitLab PAT is exactly 20 chars; user password is 32 chars + # Skip if fleet-config git repo exists on disk AND git_token is a real GitLab PAT (starts with glpat-) - | docker exec gitlab-gitlab-1 test -d /var/opt/gitlab/git-data/repositories/@hashed 2>/dev/null && \ docker exec gitlab-gitlab-1 find /var/opt/gitlab/git-data/repositories/@hashed -name "*.git" -maxdepth 3 | grep -q . && \ - PAT=$(aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty') && \ - [ ${#PAT} -le 25 ] && [ ${#PAT} -gt 10 ] + aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/secrets --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.git_token // empty' | grep -q '^glpat-' vars: GITLAB_CF_DOMAIN: sh: | From cdcf0aa0a6a764d46693a9c2c36ffd07bd2db7d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 12:10:04 +0200 Subject: [PATCH 142/231] =?UTF-8?q?refactor:=20extract=20shared=20CloudFro?= =?UTF-8?q?nt/ALB/VPC-Origin=20logic=20into=20cluster-providers/common/Tas?= =?UTF-8?q?kfile.cloudfront.yaml=20=E2=80=94=20both=20providers=20now=20us?= =?UTF-8?q?e=20identical=20infrastructure=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Taskfile.cloudfront.yaml | 131 ++++++++++++------ .../kind-crossplane/Taskfile.yaml | 12 +- cluster-providers/kind-kro-ack/Taskfile.yaml | 24 +++- 3 files changed, 114 insertions(+), 53 deletions(-) rename cluster-providers/{kind-crossplane => common}/Taskfile.cloudfront.yaml (66%) diff --git a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml b/cluster-providers/common/Taskfile.cloudfront.yaml similarity index 66% rename from cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml rename to cluster-providers/common/Taskfile.cloudfront.yaml index 6fdbf9558..23e60b063 100644 --- a/cluster-providers/kind-crossplane/Taskfile.cloudfront.yaml +++ b/cluster-providers/common/Taskfile.cloudfront.yaml @@ -2,30 +2,34 @@ version: "3" set: [errexit, nounset, pipefail] silent: true -# CloudFront exposure-mode tasks (extracted from the main Taskfile). -# Only run when EXPOSURE_MODE=cloudfront (guarded per-task via status:). -# Included by the main Taskfile under the "cloudfront:" namespace. -# Uses internal ALB + CloudFront VPC Origin (same pattern as kind-kro-ack). -tasks: - sync-origins: - desc: Ensure CloudFront distributions point to the current ALB DNS (idempotent safety net) - cmds: - - cmd: | - if [ "{{.EXPOSURE_MODE}}" != "cloudfront" ]; then exit 0; fi - printf '{{.C_STEP}}▸ Syncing CloudFront origins with current LB DNS...{{.C_RESET}}\n' - DIST_ID=$(aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" --output text 2>/dev/null) - ALB_ARN=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text 2>/dev/null) - if [ -n "$DIST_ID" ] && [ "$DIST_ID" != "None" ] && [ -n "$ALB_ARN" ] && [ "$ALB_ARN" != "None" ]; then - echo "Platform CloudFront origin OK (VPC Origin — no DNS update needed)" - fi - # GitLab CloudFront is managed by CDK (team-stack.ts). +# Shared CloudFront exposure-mode tasks. +# Used by both kind-crossplane and kind-kro-ack providers. +# Creates an internal ALB + CloudFront VPC Origin for platform services +# (Backstage, Keycloak, Grafana, etc.) only when EXPOSURE_MODE=cloudfront. +# +# Required vars (passed by the including Taskfile): +# HUB_CLUSTER_NAME e.g. peeks-hub +# RESOURCE_PREFIX e.g. peeks +# AWS_REGION e.g. us-west-2 +# AWS_ACCOUNT_ID +# EXPOSURE_MODE cloudfront | tls | domain +# CLOUDFRONT_VPC_ID optional — derived from EKS cluster if empty +# CONFIG_FILE path to config.local.yaml +# ROOT_DIR root of the repo +# C_STEP, C_OK, C_ERR, C_INFO, C_RESET colour vars +# +# ALB name: {{.HUB_CLUSTER_NAME}}-platform +# CF comment: {{.HUB_CLUSTER_NAME}}-platform +tasks: setup-exposure: desc: Pre-create internal ALB and CloudFront VPC Origin distribution (cloudfront mode, idempotent) status: - | [ "{{.EXPOSURE_MODE}}" != "cloudfront" ] || \ - aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" --output text | grep -q . + aws cloudfront list-distributions \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" \ + --output text | grep -q . cmds: - task: create-alb - task: hub-distribution @@ -40,12 +44,15 @@ tasks: set -e VPC_ID="{{.CLOUDFRONT_VPC_ID}}" if [ -z "$VPC_ID" ]; then - VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) + VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} \ + --query 'cluster.resourcesVpcConfig.vpcId' --output text) fi - VPC_CIDR=$(aws ec2 describe-vpcs --vpc-ids "$VPC_ID" --region {{.AWS_REGION}} --query 'Vpcs[0].CidrBlock' --output text) - # Use private subnets for internal ALB — one per AZ + VPC_CIDR=$(aws ec2 describe-vpcs --vpc-ids "$VPC_ID" --region {{.AWS_REGION}} \ + --query 'Vpcs[0].CidrBlock' --output text) + # Use private subnets — one per AZ (avoids "multiple subnets in same AZ" error) PRIVATE_SUBNETS=$(aws ec2 describe-subnets \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=*private*" "Name=tag:kubernetes.io/role/internal-elb,Values=1" \ + --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=*private*" \ + "Name=tag:kubernetes.io/role/internal-elb,Values=1" \ --region {{.AWS_REGION}} \ --query 'Subnets[].[SubnetId,AvailabilityZone]' --output text | \ sort -k2 -u | awk '{print $1}' | tr '\n' ' ') @@ -57,7 +64,7 @@ tasks: sort -k2 -u | awk '{print $1}' | tr '\n' ' ') fi if [ -z "$PRIVATE_SUBNETS" ]; then - printf '{{.C_ERR}}✗ No private subnets found in VPC %s — falling back to all subnets (one per AZ){{.C_RESET}}\n' "$VPC_ID" + printf '{{.C_ERR}}✗ No private subnets found — falling back to all subnets (one per AZ){{.C_RESET}}\n' PRIVATE_SUBNETS=$(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$VPC_ID" \ --region {{.AWS_REGION}} \ --query 'Subnets[].[SubnetId,AvailabilityZone]' --output text | \ @@ -73,24 +80,24 @@ tasks: --description "Platform ALB (internal, CloudFront VPC Origin)" \ --vpc-id "$VPC_ID" --region {{.AWS_REGION}} \ --query 'GroupId' --output text) - # Allow traffic from VPC CIDR (covers in-cluster + CloudFront VPC Origin) aws ec2 authorize-security-group-ingress --group-id "$SG_ID" \ --protocol tcp --port 80 --cidr "$VPC_CIDR" --region {{.AWS_REGION}} >/dev/null - # Also allow from CloudFront VPC Origins managed SG if it exists CF_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ + --filters "Name=vpc-id,Values=$VPC_ID" \ + "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") if [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ]; then aws ec2 authorize-security-group-ingress --group-id "$SG_ID" \ - --protocol tcp --port 80 --source-group "$CF_SG" --region {{.AWS_REGION}} >/dev/null 2>&1 || true + --protocol tcp --port 80 --source-group "$CF_SG" \ + --region {{.AWS_REGION}} >/dev/null 2>&1 || true printf '{{.C_OK}}✓ Authorized CloudFront-VPCOrigins-Service-SG → ALB:80{{.C_RESET}}\n' fi - # Allow EKS cluster SG to reach ALB CLUSTER_SG=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} \ --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text 2>/dev/null || echo "") [ -n "$CLUSTER_SG" ] && [ "$CLUSTER_SG" != "None" ] && \ aws ec2 authorize-security-group-ingress --group-id "$SG_ID" \ - --protocol -1 --source-group "$CLUSTER_SG" --region {{.AWS_REGION}} 2>/dev/null || true + --protocol -1 --source-group "$CLUSTER_SG" \ + --region {{.AWS_REGION}} 2>/dev/null || true fi ALB_ARN=$(aws elbv2 create-load-balancer \ --name "{{.HUB_CLUSTER_NAME}}-platform" \ @@ -98,14 +105,16 @@ tasks: --security-groups "$SG_ID" \ --scheme internal \ --type application \ - --tags Key=elbv2.k8s.aws/cluster,Value={{.HUB_CLUSTER_NAME}} Key=ingress.k8s.aws/stack,Value=platform Key=ingress.k8s.aws/resource,Value=LoadBalancer \ + --tags Key=elbv2.k8s.aws/cluster,Value={{.HUB_CLUSTER_NAME}} \ + Key=ingress.k8s.aws/stack,Value=platform \ + Key=ingress.k8s.aws/resource,Value=LoadBalancer \ --region {{.AWS_REGION}} \ --query 'LoadBalancers[0].LoadBalancerArn' --output text) aws elbv2 create-listener \ --load-balancer-arn "$ALB_ARN" --protocol HTTP --port 80 \ --default-actions "Type=fixed-response,FixedResponseConfig={MessageBody=Not Found,StatusCode=404,ContentType=text/plain}" \ --region {{.AWS_REGION}} > /dev/null - printf '{{.C_OK}}✓ ALB created: {{.HUB_CLUSTER_NAME}}-platform (scheme=internal, mode=cloudfront){{.C_RESET}}\n' + printf '{{.C_OK}}✓ ALB created: {{.HUB_CLUSTER_NAME}}-platform (scheme=internal){{.C_RESET}}\n' - printf '{{.C_OK}}✓ Platform ALB ready.{{.C_RESET}}\n' hub-distribution: @@ -113,17 +122,19 @@ tasks: status: - | [ "{{.EXPOSURE_MODE}}" != "cloudfront" ] || \ - aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" --output text | grep -q . + aws cloudfront list-distributions \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].Id" \ + --output text | grep -q . cmds: - printf '{{.C_STEP}}▸ Creating platform CloudFront distribution (VPC Origin)...{{.C_RESET}}\n' - | set -e ALB_ARN=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" \ --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text) - # Create or reuse VPC Origin VPC_ORIGIN_NAME="{{.HUB_CLUSTER_NAME}}-platform-vpc-origin" VPC_ORIGIN_ID=$(aws cloudfront list-vpc-origins \ - --query "VpcOriginList.Items[?Name=='$VPC_ORIGIN_NAME'].Id" --output text 2>/dev/null || echo "") + --query "VpcOriginList.Items[?Name=='$VPC_ORIGIN_NAME'].Id" \ + --output text 2>/dev/null || echo "") if [ -z "$VPC_ORIGIN_ID" ] || [ "$VPC_ORIGIN_ID" = "None" ]; then VPC_ORIGIN_ID=$(aws cloudfront create-vpc-origin \ --vpc-origin-endpoint-config "{ @@ -138,7 +149,6 @@ tasks: else printf '{{.C_INFO}}↻ Reusing VPC Origin: %s{{.C_RESET}}\n' "$VPC_ORIGIN_ID" fi - # Wait for VPC Origin to be Deployed (~5-10 min) printf '{{.C_STEP}}▸ Waiting for VPC Origin Deployed...{{.C_RESET}}\n' for i in $(seq 1 60); do STATE=$(aws cloudfront get-vpc-origin --id "$VPC_ORIGIN_ID" \ @@ -147,24 +157,27 @@ tasks: [ "$STATE" = "Deployed" ] && break sleep 15 done - [ "$STATE" != "Deployed" ] && echo "ERROR: VPC Origin not Deployed" && exit 1 - # Authorize CloudFront VPC Origins SG on ALB SG (created after first VPC Origin) + [ "$STATE" != "Deployed" ] && echo "ERROR: VPC Origin not Deployed within 15min" && exit 1 + # Authorize CloudFront VPC Origins SG on ALB SG (AWS creates this SG after first VPC Origin) VPC_ID=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" \ --region {{.AWS_REGION}} --query 'LoadBalancers[0].VpcId' --output text) ALB_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ - --filters "Name=group-name,Values={{.HUB_CLUSTER_NAME}}-platform-alb-sg" "Name=vpc-id,Values=$VPC_ID" \ + --filters "Name=group-name,Values={{.HUB_CLUSTER_NAME}}-platform-alb-sg" \ + "Name=vpc-id,Values=$VPC_ID" \ --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") CF_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ + --filters "Name=vpc-id,Values=$VPC_ID" \ + "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") - if [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ] && [ -n "$ALB_SG" ] && [ "$ALB_SG" != "None" ]; then + if [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ] && \ + [ -n "$ALB_SG" ] && [ "$ALB_SG" != "None" ]; then aws ec2 authorize-security-group-ingress --group-id "$ALB_SG" \ - --protocol tcp --port 80 --source-group "$CF_SG" --region {{.AWS_REGION}} 2>/dev/null || true + --protocol tcp --port 80 --source-group "$CF_SG" \ + --region {{.AWS_REGION}} 2>/dev/null || true printf '{{.C_OK}}✓ Authorized %s → %s:80 (post VPC Origin){{.C_RESET}}\n' "$CF_SG" "$ALB_SG" fi ALB_DNS=$(aws elbv2 describe-load-balancers --names "{{.HUB_CLUSTER_NAME}}-platform" \ --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text) - # Create CloudFront distribution with VPC Origin CF_DOMAIN=$(aws cloudfront create-distribution \ --distribution-config "{ \"CallerReference\": \"{{.HUB_CLUSTER_NAME}}-platform-$(date +%s)\", @@ -173,12 +186,20 @@ tasks: \"Origins\": {\"Quantity\": 1, \"Items\": [{ \"Id\": \"vpc-origin\", \"DomainName\": \"$ALB_DNS\", - \"VpcOriginConfig\": {\"VpcOriginId\": \"$VPC_ORIGIN_ID\", \"OriginReadTimeout\": 60, \"OriginKeepaliveTimeout\": 5} + \"VpcOriginConfig\": { + \"VpcOriginId\": \"$VPC_ORIGIN_ID\", + \"OriginReadTimeout\": 60, + \"OriginKeepaliveTimeout\": 5 + } }]}, \"DefaultCacheBehavior\": { \"TargetOriginId\": \"vpc-origin\", \"ViewerProtocolPolicy\": \"redirect-to-https\", - \"AllowedMethods\": {\"Quantity\": 7, \"Items\": [\"GET\",\"HEAD\",\"OPTIONS\",\"PUT\",\"POST\",\"PATCH\",\"DELETE\"], \"CachedMethods\": {\"Quantity\": 2, \"Items\": [\"GET\",\"HEAD\"]}}, + \"AllowedMethods\": { + \"Quantity\": 7, + \"Items\": [\"GET\",\"HEAD\",\"OPTIONS\",\"PUT\",\"POST\",\"PATCH\",\"DELETE\"], + \"CachedMethods\": {\"Quantity\": 2, \"Items\": [\"GET\",\"HEAD\"]} + }, \"CachePolicyId\": \"4135ea2d-6df8-44a3-9df3-4b5a84be39ad\", \"OriginRequestPolicyId\": \"216adef6-5c7f-47e4-b989-5492eafa07d3\", \"Compress\": true @@ -189,8 +210,26 @@ tasks: --query "Distribution.DomainName" --output text) mkdir -p {{.ROOT_DIR}}/private echo -n "$CF_DOMAIN" > {{.ROOT_DIR}}/private/cloudfront-domain - # Write cloudfrontDomain into config.local.yaml yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} printf '{{.C_OK}}✓ CloudFront created: %s{{.C_RESET}}\n' "$CF_DOMAIN" - printf '{{.C_OK}}✓ Platform CloudFront ready (VPC Origin).{{.C_RESET}}\n' -# gitlab-nlb and gitlab-distribution removed — GitLab CloudFront is now managed by CDK (team-stack.ts). + + sync-domain: + desc: Ensure cloudfrontDomain is written to config + private/cloudfront-domain (idempotent safety net) + cmds: + - | + CURRENT=$(yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || \ + cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true) + if [ -z "$CURRENT" ]; then + CURRENT=$(aws cloudfront list-distributions \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName" \ + --output text 2>/dev/null | tr -d '[:space:]') + fi + if [ -n "$CURRENT" ] && [ "$CURRENT" != "None" ]; then + yq -i ".cloudfront.cloudfrontDomain = \"$CURRENT\"" {{.CONFIG_FILE}} + mkdir -p {{.ROOT_DIR}}/private + echo -n "$CURRENT" > {{.ROOT_DIR}}/private/cloudfront-domain + printf '{{.C_OK}}✓ cloudfrontDomain: %s{{.C_RESET}}\n' "$CURRENT" + else + printf '{{.C_WARN}} No {{.HUB_CLUSTER_NAME}}-platform CloudFront distribution found{{.C_RESET}}\n' + fi diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 3372a9e2f..1ba1cf116 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -4,18 +4,22 @@ silent: true includes: cloudfront: - taskfile: ./Taskfile.cloudfront.yaml + taskfile: ../common/Taskfile.cloudfront.yaml vars: AWS_REGION: '{{.AWS_REGION}}' HUB_CLUSTER_NAME: '{{.HUB_CLUSTER_NAME}}' + RESOURCE_PREFIX: '{{.RESOURCE_PREFIX}}' + AWS_ACCOUNT_ID: '{{.AWS_ACCOUNT_ID}}' ROOT_DIR: '{{.ROOT_DIR}}' + CONFIG_FILE: '{{.CONFIG_FILE}}' EXPOSURE_MODE: sh: 'if [ -z "$(yq ''.domain // ""'' {{.ROOT_DIR}}/config.local.yaml)" ]; then echo "cloudfront"; else echo "domain"; fi' C_STEP: '{{.C_STEP}}' C_OK: '{{.C_OK}}' + C_ERR: '{{.C_ERR}}' C_RESET: '{{.C_RESET}}' C_INFO: '{{.C_INFO}}' - CLOUDFRONT_DOMAIN: '{{.CLOUDFRONT_DOMAIN}}' + C_WARN: '{{.C_WARN}}' CLOUDFRONT_VPC_ID: '{{.CLOUDFRONT_VPC_ID}}' @@ -208,8 +212,8 @@ tasks: install:phase1-cloudfront: internal: true cmds: - # setup-exposure (create-alb + hub-distribution) now runs before hub:seed - # This task only handles pod identity restart + # Ensure cloudfrontDomain is in config + private/cloudfront-domain (safety net for retries) + - task: cloudfront:sync-domain - task: hub:update-ingress-domain - task: hub:restart-identity-pods diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index e897ef6b2..abfc2c817 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -2,6 +2,25 @@ version: "3" set: [errexit, nounset, pipefail] silent: true +includes: + cloudfront: + taskfile: ../common/Taskfile.cloudfront.yaml + vars: + AWS_REGION: '{{.AWS_REGION}}' + HUB_CLUSTER_NAME: '{{.HUB_CLUSTER_NAME}}' + RESOURCE_PREFIX: '{{.RESOURCE_PREFIX}}' + AWS_ACCOUNT_ID: '{{.AWS_ACCOUNT_ID}}' + ROOT_DIR: '{{.ROOT_DIR}}' + CONFIG_FILE: '{{.CONFIG_FILE}}' + EXPOSURE_MODE: '{{.INGRESS_MODE}}' + C_STEP: '{{.C_STEP}}' + C_OK: '{{.C_OK}}' + C_ERR: '{{.C_ERR}}' + C_RESET: '{{.C_RESET}}' + C_INFO: '{{.C_INFO}}' + C_WARN: '{{.C_WARN}}' + CLOUDFRONT_VPC_ID: '' + env: KUBECONFIG: "{{.ROOT_DIR}}/private/kubeconfig" AWS_PAGER: "" @@ -111,9 +130,8 @@ tasks: - task: kro:apply-rgds - task: hub:claim - task: hub:wait-for-eks - - task: hub:ingress - - task: hub:cloudfront - - task: hub:sync-cloudfront-domain + - task: cloudfront:setup-exposure + - task: cloudfront:sync-domain - task: hub:seed - task: hub:wait-for-sync vars: { MAX_OUTOFSYNC: "20" } From fd17b449971c5fa5a0711deb5d13fa4d62db345a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 12:14:23 +0200 Subject: [PATCH 143/231] =?UTF-8?q?refactor:=20remove=20dead=20hub:ingress?= =?UTF-8?q?/hub:cloudfront/hub:sync-cloudfront-domain=20from=20kro-ack=20?= =?UTF-8?q?=E2=80=94=20replaced=20by=20shared=20cloudfront:=20tasks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-kro-ack/Taskfile.yaml | 260 ------------------- 1 file changed, 260 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index abfc2c817..04c22e004 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -629,266 +629,6 @@ tasks: printf '{{.C_ERR}}✗ EKS cluster not ACTIVE in AWS after RGD reported ACTIVE{{.C_RESET}}\n' exit 1 - hub:ingress:preflight: - desc: Validate ingress.mode config (fails fast if mode=tls without certificateArn+domain) - cmds: - - | - MODE="{{.INGRESS_MODE}}" - CERT="{{.INGRESS_CERT_ARN}}" - DOMAIN="{{.DOMAIN}}" - case "$MODE" in - cloudfront) - printf '{{.C_INFO}} ingress.mode=cloudfront (ALB internal + CloudFront VPC Origin){{.C_RESET}}\n' - ;; - tls) - if [ -z "$CERT" ]; then - printf '{{.C_ERR}}✗ ingress.mode=tls requires ingress.certificateArn (ACM ARN){{.C_RESET}}\n' - exit 1 - fi - if [ -z "$DOMAIN" ] || [ "$DOMAIN" = "" ]; then - printf '{{.C_ERR}}✗ ingress.mode=tls requires top-level domain to be set{{.C_RESET}}\n' - exit 1 - fi - printf '{{.C_INFO}} ingress.mode=tls (ALB internet-facing HTTPS:443, cert=%s, domain=%s){{.C_RESET}}\n' "$CERT" "$DOMAIN" - ;; - *) - printf '{{.C_ERR}}✗ ingress.mode=%s invalid; expected: cloudfront | tls{{.C_RESET}}\n' "$MODE" - exit 1 - ;; - esac - - hub:ingress: - desc: | - Create ALB for hub ingress. Behaviour depends on ingress.mode: - - cloudfront: ALB scheme=internal in private subnets, SG limited to VPC CIDR - - tls: ALB scheme=internet-facing in public subnets, SG open 80/443 to 0.0.0.0/0 - Switching mode after creation requires manual ALB teardown (AWS scheme is immutable). - deps: [hub:ingress:preflight] - vars: - ALB_NAME: "{{.RESOURCE_PREFIX}}-hub-ingress" - VPC_ID: - sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text - VPC_CIDR: - sh: aws ec2 describe-vpcs --vpc-ids "$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text)" --region {{.AWS_REGION}} --query 'Vpcs[0].CidrBlock' --output text - SG_ID: - sh: aws ec2 describe-security-groups --region {{.AWS_REGION}} --filters "Name=vpc-id,Values=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text)" "Name=group-name,Values={{.RESOURCE_PREFIX}}-hub-ingress-http" --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "" - status: - - aws elbv2 describe-load-balancers --names {{.ALB_NAME}} --region {{.AWS_REGION}} 2>/dev/null - cmds: - - | - MODE="{{.INGRESS_MODE}}" - VPC_ID="{{.VPC_ID}}" - VPC_CIDR="{{.VPC_CIDR}}" - # Resolve subnets according to mode - if [ "$MODE" = "cloudfront" ]; then - SCHEME="internal" - SUBNET_NAME_FILTER="*private*" - ROLE_TAG="kubernetes.io/role/internal-elb" - else - SCHEME="internet-facing" - SUBNET_NAME_FILTER="*public*" - ROLE_TAG="kubernetes.io/role/elb" - fi - SUBNETS=$(aws ec2 describe-subnets --region {{.AWS_REGION}} \ - --filters "Name=vpc-id,Values=$VPC_ID" \ - "Name=tag:Name,Values=$SUBNET_NAME_FILTER" \ - "Name=tag:$ROLE_TAG,Values=1" \ - --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') - if [ -z "$SUBNETS" ]; then - # Fallback: name-only filter for older VPCs without role tags - SUBNETS=$(aws ec2 describe-subnets --region {{.AWS_REGION}} \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=$SUBNET_NAME_FILTER" \ - --query 'Subnets[].SubnetId' --output text | tr '\t' ' ') - fi - if [ -z "$SUBNETS" ]; then - printf '{{.C_ERR}}✗ No subnets matched filter %s in VPC %s{{.C_RESET}}\n' "$SUBNET_NAME_FILTER" "$VPC_ID" - exit 1 - fi - # Resolve / create SG with mode-appropriate ingress rules - SG="{{.SG_ID}}" - if [ -z "$SG" ] || [ "$SG" = "None" ]; then - SG=$(aws ec2 create-security-group --group-name "{{.RESOURCE_PREFIX}}-hub-ingress-http" \ - --description "HTTP/HTTPS for hub ingress (mode=$MODE)" --vpc-id "$VPC_ID" --region {{.AWS_REGION}} \ - --query 'GroupId' --output text) - if [ "$MODE" = "cloudfront" ]; then - # ALB internal — limit to VPC CIDR (covers in-cluster traffic) - aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 80 \ - --cidr "$VPC_CIDR" --region {{.AWS_REGION}} 2>/dev/null || true - # CloudFront VPC Origin ENIs use a dedicated AWS-managed SG — - # CIDR alone is not sufficient because CF routes via service SG. - CF_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ - --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") - if [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ]; then - aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 80 \ - --source-group "$CF_SG" --region {{.AWS_REGION}} 2>/dev/null || true - printf '{{.C_OK}}✓ Authorized CloudFront-VPCOrigins-Service-SG → ALB:80{{.C_RESET}}\n' - else - printf '{{.C_WARN}}⚠ CloudFront-VPCOrigins-Service-SG not found — ALB may 504. Re-run after first VPC Origin creation.{{.C_RESET}}\n' - fi - else - # ALB internet-facing — open 80 (redirect) and 443 - aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 80 \ - --cidr 0.0.0.0/0 --region {{.AWS_REGION}} 2>/dev/null || true - aws ec2 authorize-security-group-ingress --group-id "$SG" --protocol tcp --port 443 \ - --cidr 0.0.0.0/0 --region {{.AWS_REGION}} 2>/dev/null || true - fi - fi - ALB_ARN=$(aws elbv2 create-load-balancer --name "{{.ALB_NAME}}" --type application \ - --scheme "$SCHEME" --subnets $SUBNETS --security-groups "$SG" \ - --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text) - # Default listener (404) — AWS LBC adds rules later when K8s Ingresses appear - if [ "$MODE" = "cloudfront" ]; then - aws elbv2 create-listener --load-balancer-arn "$ALB_ARN" --protocol HTTP --port 80 \ - --default-actions 'Type=fixed-response,FixedResponseConfig={StatusCode=404,ContentType=text/plain,MessageBody=NotFound}' \ - --region {{.AWS_REGION}} >/dev/null - else - aws elbv2 create-listener --load-balancer-arn "$ALB_ARN" --protocol HTTPS --port 443 \ - --certificates "CertificateArn={{.INGRESS_CERT_ARN}}" \ - --default-actions 'Type=fixed-response,FixedResponseConfig={StatusCode=404,ContentType=text/plain,MessageBody=NotFound}' \ - --region {{.AWS_REGION}} >/dev/null - aws elbv2 create-listener --load-balancer-arn "$ALB_ARN" --protocol HTTP --port 80 \ - --default-actions 'Type=redirect,RedirectConfig={Protocol=HTTPS,Port=443,StatusCode=HTTP_301}' \ - --region {{.AWS_REGION}} >/dev/null - fi - printf '{{.C_OK}}✓ ALB created: {{.ALB_NAME}} (scheme=%s, mode=%s){{.C_RESET}}\n' "$SCHEME" "$MODE" - - hub:cloudfront: - desc: Create CloudFront distribution with VPC Origin (cloudfront mode only) - # CachePolicyId 4135ea2d = AWS Managed "CachingDisabled" - # OriginRequestPolicyId 216adef6 = AWS Managed "AllViewer" (forwards all headers/cookies/qs) - # Uses CloudFront VPC Origin (post-2024) to reach an internal ALB without public exposure. - vars: - ALB_ARN: - sh: aws elbv2 describe-load-balancers --names "{{.RESOURCE_PREFIX}}-hub-ingress" --region {{.AWS_REGION}} --query 'LoadBalancers[0].LoadBalancerArn' --output text 2>/dev/null || echo "" - ALB_DNS: - sh: aws elbv2 describe-load-balancers --names "{{.RESOURCE_PREFIX}}-hub-ingress" --region {{.AWS_REGION}} --query 'LoadBalancers[0].DNSName' --output text 2>/dev/null || echo "" - VPC_ORIGIN_NAME: "{{.RESOURCE_PREFIX}}-hub-vpc-origin" - status: - # Skip when not in cloudfront mode OR when distribution already exists - - | - test "{{.INGRESS_MODE}}" != "cloudfront" || \ - aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='{{.RESOURCE_PREFIX}}-hub-ingress'].Id" --output text 2>/dev/null | grep -q . - cmds: - - | - if [ "{{.INGRESS_MODE}}" != "cloudfront" ]; then - printf '{{.C_INFO}}⊘ Skipping hub:cloudfront (ingress.mode={{.INGRESS_MODE}}){{.C_RESET}}\n' - exit 0 - fi - if [ -z "{{.ALB_ARN}}" ] || [ "{{.ALB_ARN}}" = "None" ]; then - echo "ERROR: ALB {{.RESOURCE_PREFIX}}-hub-ingress not found. Run hub:ingress first." - exit 1 - fi - - | - # 1. Create or reuse VPC Origin pointing at the internal ALB - VPC_ORIGIN_ID=$(aws cloudfront list-vpc-origins --region {{.AWS_REGION}} \ - --query "VpcOriginList.Items[?Name=='{{.VPC_ORIGIN_NAME}}'].Id" --output text 2>/dev/null || echo "") - if [ -z "$VPC_ORIGIN_ID" ] || [ "$VPC_ORIGIN_ID" = "None" ]; then - VPC_ORIGIN_ID=$(aws cloudfront create-vpc-origin --region {{.AWS_REGION}} \ - --vpc-origin-endpoint-config "{ - \"Name\": \"{{.VPC_ORIGIN_NAME}}\", - \"Arn\": \"{{.ALB_ARN}}\", - \"HTTPPort\": 80, - \"HTTPSPort\": 443, - \"OriginProtocolPolicy\": \"http-only\", - \"OriginSslProtocols\": {\"Quantity\": 1, \"Items\": [\"TLSv1.2\"]} - }" --query 'VpcOrigin.Id' --output text) - printf '{{.C_OK}}✓ VPC Origin created: %s{{.C_RESET}}\n' "$VPC_ORIGIN_ID" - else - printf '{{.C_INFO}}↻ Reusing VPC Origin: %s{{.C_RESET}}\n' "$VPC_ORIGIN_ID" - fi - - # Wait for VPC Origin Deployed (~5-10 min) - printf '{{.C_STEP}}▸ Waiting for VPC Origin Deployed...{{.C_RESET}}\n' - for i in $(seq 1 60); do - STATE=$(aws cloudfront get-vpc-origin --id "$VPC_ORIGIN_ID" --region {{.AWS_REGION}} --query 'VpcOrigin.Status' --output text 2>/dev/null || echo "Pending") - printf '{{.C_INFO}} [%s/60] VPC Origin status: %s{{.C_RESET}}\n' "$i" "$STATE" - [ "$STATE" = "Deployed" ] && break - sleep 15 - done - if [ "$STATE" != "Deployed" ]; then - echo "ERROR: VPC Origin did not reach Deployed within 15 min"; exit 1 - fi - echo "$VPC_ORIGIN_ID" > /tmp/hub-vpc-origin-id - - | - # 2. Reconcile ALB SG ingress from CloudFront-VPCOrigins-Service-SG. - # This SG is created by AWS only after the first VPC Origin lands in - # the VPC, so hub:ingress can't authorize it on first run. We close - # the loop here so a single hub:ingress + hub:cloudfront sequence is - # sufficient (no manual re-run of hub:ingress needed). - VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text) - ALB_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values={{.RESOURCE_PREFIX}}-hub-ingress-http" \ - --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") - CF_SG=$(aws ec2 describe-security-groups --region {{.AWS_REGION}} \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=group-name,Values=CloudFront-VPCOrigins-Service-SG" \ - --query 'SecurityGroups[0].GroupId' --output text 2>/dev/null || echo "") - if [ -n "$ALB_SG" ] && [ "$ALB_SG" != "None" ] && [ -n "$CF_SG" ] && [ "$CF_SG" != "None" ]; then - aws ec2 authorize-security-group-ingress --group-id "$ALB_SG" --protocol tcp --port 80 \ - --source-group "$CF_SG" --region {{.AWS_REGION}} 2>/dev/null \ - && printf '{{.C_OK}}✓ Authorized %s → %s:80 (post VPC Origin){{.C_RESET}}\n' "$CF_SG" "$ALB_SG" \ - || printf '{{.C_INFO}}↻ ALB SG already allows CloudFront-VPCOrigins-Service-SG{{.C_RESET}}\n' - fi - - | - # 3. Create CloudFront distribution wired to the VPC Origin - VPC_ORIGIN_ID=$(cat /tmp/hub-vpc-origin-id) - DIST_OUT=$(aws cloudfront create-distribution \ - --distribution-config "{ - \"CallerReference\": \"{{.HUB_CLUSTER_NAME}}-$(date +%s)\", - \"Comment\": \"{{.RESOURCE_PREFIX}}-hub-ingress\", - \"Enabled\": true, - \"Origins\": { - \"Quantity\": 1, - \"Items\": [{ - \"Id\": \"alb-vpc-origin\", - \"DomainName\": \"{{.ALB_DNS}}\", - \"VpcOriginConfig\": { - \"VpcOriginId\": \"$VPC_ORIGIN_ID\", - \"OriginReadTimeout\": 30, - \"OriginKeepaliveTimeout\": 5 - } - }] - }, - \"DefaultCacheBehavior\": { - \"TargetOriginId\": \"alb-vpc-origin\", - \"ViewerProtocolPolicy\": \"redirect-to-https\", - \"AllowedMethods\": {\"Quantity\": 7, \"Items\": [\"GET\",\"HEAD\",\"OPTIONS\",\"PUT\",\"POST\",\"PATCH\",\"DELETE\"], \"CachedMethods\": {\"Quantity\": 2, \"Items\": [\"GET\",\"HEAD\"]}}, - \"CachePolicyId\": \"4135ea2d-6df8-44a3-9df3-4b5a84be39ad\", - \"OriginRequestPolicyId\": \"216adef6-5c7f-47e4-b989-5492eafa07d3\", - \"Compress\": true - }, - \"PriceClass\": \"PriceClass_100\", - \"HttpVersion\": \"http2\" - }" --query 'Distribution.[Id,DomainName]' --output text) - CF_DOMAIN=$(echo "$DIST_OUT" | awk '{print $2}') - printf '{{.C_OK}}✓ CloudFront created: %s{{.C_RESET}}\n' "$CF_DOMAIN" - # Store domain in config and private file for hub:seed and retries to use - yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} - mkdir -p {{.ROOT_DIR}}/private - echo -n "$CF_DOMAIN" > {{.ROOT_DIR}}/private/cloudfront-domain - rm -f /tmp/hub-vpc-origin-id - - hub:sync-cloudfront-domain: - desc: Ensure cloudfrontDomain is written to config AND private/cloudfront-domain file (idempotent) - internal: true - cmds: - - | - CURRENT=$(yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true) - if [ -z "$CURRENT" ]; then - # Look up the existing CloudFront distribution - CURRENT=$(aws cloudfront list-distributions \ - --query "DistributionList.Items[?Comment=='{{.RESOURCE_PREFIX}}-hub-ingress'].DomainName" \ - --output text 2>/dev/null | tr -d '[:space:]') - fi - if [ -n "$CURRENT" ] && [ "$CURRENT" != "None" ]; then - yq -i ".cloudfront.cloudfrontDomain = \"$CURRENT\"" {{.CONFIG_FILE}} - mkdir -p {{.ROOT_DIR}}/private - echo -n "$CURRENT" > {{.ROOT_DIR}}/private/cloudfront-domain - printf '{{.C_OK}}✓ cloudfrontDomain: %s{{.C_RESET}}\n' "$CURRENT" - else - printf '{{.C_WARN}} No CloudFront distribution found for {{.RESOURCE_PREFIX}}-hub-ingress{{.C_RESET}}\n' - fi - hub:seed: desc: Seed hub cluster after EKS is ACTIVE cmds: From 5465101254ca1472244ab4bb718cf998e46667ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 12:16:43 +0200 Subject: [PATCH 144/231] fix: standardize ingress_name to HUB_CLUSTER_NAME-platform in both kro-ack and crossplane metadata --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 +- cluster-providers/kind-kro-ack/Taskfile.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 1ba1cf116..413c03ef2 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -61,7 +61,7 @@ vars: RESOURCE_PREFIX: sh: yq '.resourcePrefix // ""' {{.CONFIG_FILE}} INGRESS_NAME: - sh: yq '.ingressName // ""' {{.CONFIG_FILE}} + sh: yq '.ingressName // ""' {{.CONFIG_FILE}} | grep -v '^$' || echo "{{.HUB_CLUSTER_NAME}}-platform" INGRESS_SECURITY_GROUPS: sh: yq '.ingressSecurityGroups // ""' {{.CONFIG_FILE}} EXPOSURE_MODE: diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 04c22e004..fab4d9f99 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -677,7 +677,7 @@ tasks: - | SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" SECRET_VALUE=$(jq -n \ - --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.RESOURCE_PREFIX}}-hub-ingress","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}"}' \ + --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}"}' \ --arg config '{"tlsClientConfig":{"insecure":false}}' \ --arg server '{{.CLUSTER_ARN}}' \ --arg addons '' \ From 8d6c10cb43f85e6f0784716359e025e8e4c48f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 12:20:10 +0200 Subject: [PATCH 145/231] ux: add progress logging every 2min to spoke EKS cluster wait loop --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index fab4d9f99..404ed0238 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1750,6 +1750,7 @@ tasks: i=0 until aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null | grep -q ACTIVE; do i=$((i+1)); [ "$i" -ge 360 ] && { printf '{{.C_ERR}}⚠ %s not ACTIVE after timeout — skipping capability creation.{{.C_RESET}}\n' "$CLUSTER"; exit 0; } + if [ $((i % 8)) -eq 0 ]; then STATUS=$(aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null || echo "NOT_FOUND"); printf '{{.C_INFO}} [%s] status: %s (%ds){{.C_RESET}}\n' "$CLUSTER" "$STATUS" "$((i*15))"; fi sleep 15 done From c26931c96ecc2993b90457ba3807facf812ee3a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 12:20:50 +0200 Subject: [PATCH 146/231] perf: run spoke-dev and spoke-prod enables in parallel in kro-ack (deps vs sequential cmds) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 404ed0238..e1684eadf 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -170,9 +170,9 @@ tasks: install:phase2-parallel: internal: true - cmds: - - task: install:phase2-spoke-dev - - task: install:phase2-spoke-prod + deps: + - install:phase2-spoke-dev + - install:phase2-spoke-prod install:phase2-spoke-dev: internal: true From 9f7449f26e04208a4b5d626cb52c49f346c63edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 13:00:49 +0200 Subject: [PATCH 147/231] =?UTF-8?q?fix:=20add=20hub:wait-for-providers=20s?= =?UTF-8?q?tep=20before=20spoke=20provisioning=20=E2=80=94=20ensures=20pro?= =?UTF-8?q?vider-aws-ec2=20CRDs=20are=20ready?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kind-crossplane/Taskfile.yaml | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 413c03ef2..d7fa04ff3 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -143,6 +143,8 @@ tasks: - 'printf "{{.C_INFO}} [%ds] phase1-parallel done (cloudfront+restart || observability){{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - task: hub:wait-for-full-sync - 'printf "{{.C_INFO}} [%ds] hub:wait-for-full-sync done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' + # Wait for ALL Crossplane providers (including ec2) to be Healthy before spoke provisioning + - task: hub:wait-for-providers # Phase 1b (parallel): langfuse restart || urls || gitlab clone — all independent - task: install:phase1b-parallel - task: hub:set-overlay-repo @@ -700,6 +702,26 @@ tasks: done printf '{{.C_OK}}✓ ingress_domain_name updated in Secrets Manager (ESO will propagate within 1min){{.C_RESET}}\n' + hub:wait-for-providers: + desc: Wait for all Crossplane providers to be Healthy (ensures ec2/rds/etc. CRDs available before spoke provisioning) + cmds: + - | + KC={{.ROOT_DIR}}/private/hub-kubeconfig + [ ! -f "$KC" ] && KC={{.ROOT_DIR}}/private/kubeconfig + printf '{{.C_STEP}}▸ Waiting for all Crossplane providers to be Healthy...{{.C_RESET}} +' + for i in $(seq 1 60); do + UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' 2>/dev/null | wc -w | tr -d ' ') + TOTAL=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io --no-headers 2>/dev/null | wc -l | tr -d ' ') + [ "$UNHEALTHY" = "0" ] && [ "$TOTAL" -gt 0 ] && printf '{{.C_OK}}✓ All %s Crossplane providers Healthy.{{.C_RESET}} +' "$TOTAL" && exit 0 + [ $((i % 4)) -eq 0 ] && printf '{{.C_INFO}} [%ds] %s/%s providers Healthy...{{.C_RESET}} +' "$((i*15))" "$((TOTAL-UNHEALTHY))" "$TOTAL" + sleep 15 + done + printf '{{.C_WARN}} Providers not all Healthy after 15min — continuing anyway.{{.C_RESET}} +' + hub:wait-for-sync: desc: Wait for hub ArgoCD apps to reach Synced state vars: From 7dfc7a0ea0f4c758706621b91974afe6f8908fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 13:02:31 +0200 Subject: [PATCH 148/231] =?UTF-8?q?fix:=20force-sync=20crossplane-base=20a?= =?UTF-8?q?fter=205min=20if=20providers=20not=20Healthy=20=E2=80=94=20unbl?= =?UTF-8?q?ocks=20stuck=20ArgoCD=20retry=20loops?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index d7fa04ff3..d228213a4 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -717,6 +717,13 @@ tasks: ' "$TOTAL" && exit 0 [ $((i % 4)) -eq 0 ] && printf '{{.C_INFO}} [%ds] %s/%s providers Healthy...{{.C_RESET}} ' "$((i*15))" "$((TOTAL-UNHEALTHY))" "$TOTAL" + # After 5min, force-sync crossplane-base to unblock stuck ArgoCD retry loops + if [ "$i" -eq 20 ]; then + printf '{{.C_WARN}} Providers not all Healthy after 5min — force-syncing crossplane-base...{{.C_RESET}}\n' + KUBECONFIG=$KC kubectl patch application "crossplane-base-{{.HUB_CLUSTER_NAME}}" -n argocd \ + --type merge -p '{"operation":{"initiatedBy":{"username":"admin"},"sync":{"revision":"HEAD"}}}' \ + 2>/dev/null || true + fi sleep 15 done printf '{{.C_WARN}} Providers not all Healthy after 15min — continuing anyway.{{.C_RESET}} From 3e6379aa6bd52bd397b76cb24ffdddea63d5dee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 13:03:20 +0200 Subject: [PATCH 149/231] =?UTF-8?q?fix:=20remove=20line=20continuation=20i?= =?UTF-8?q?n=20kubectl=20patch=20=E2=80=94=20YAML=20block=20scalar=20doesn?= =?UTF-8?q?'t=20support=20backslash=20continuation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index d228213a4..f731048aa 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -720,8 +720,7 @@ tasks: # After 5min, force-sync crossplane-base to unblock stuck ArgoCD retry loops if [ "$i" -eq 20 ]; then printf '{{.C_WARN}} Providers not all Healthy after 5min — force-syncing crossplane-base...{{.C_RESET}}\n' - KUBECONFIG=$KC kubectl patch application "crossplane-base-{{.HUB_CLUSTER_NAME}}" -n argocd \ - --type merge -p '{"operation":{"initiatedBy":{"username":"admin"},"sync":{"revision":"HEAD"}}}' \ + KUBECONFIG=$KC kubectl patch application "crossplane-base-{{.HUB_CLUSTER_NAME}}" -n argocd --type merge -p '{"operation":{"initiatedBy":{"username":"admin"},"sync":{"revision":"HEAD"}}}' 2>/dev/null || true 2>/dev/null || true fi sleep 15 From 2250c5e5b810362b436d2266535fd93f48182497 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 13:05:39 +0200 Subject: [PATCH 150/231] fix: use \n in printf single-quoted strings to avoid YAML parse error --- cluster-providers/kind-crossplane/Taskfile.yaml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index f731048aa..5a8ff45b3 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -708,15 +708,12 @@ tasks: - | KC={{.ROOT_DIR}}/private/hub-kubeconfig [ ! -f "$KC" ] && KC={{.ROOT_DIR}}/private/kubeconfig - printf '{{.C_STEP}}▸ Waiting for all Crossplane providers to be Healthy...{{.C_RESET}} -' + printf '{{.C_STEP}}▸ Waiting for all Crossplane providers to be Healthy...{{.C_RESET}}\n' for i in $(seq 1 60); do UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' 2>/dev/null | wc -w | tr -d ' ') TOTAL=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io --no-headers 2>/dev/null | wc -l | tr -d ' ') - [ "$UNHEALTHY" = "0" ] && [ "$TOTAL" -gt 0 ] && printf '{{.C_OK}}✓ All %s Crossplane providers Healthy.{{.C_RESET}} -' "$TOTAL" && exit 0 - [ $((i % 4)) -eq 0 ] && printf '{{.C_INFO}} [%ds] %s/%s providers Healthy...{{.C_RESET}} -' "$((i*15))" "$((TOTAL-UNHEALTHY))" "$TOTAL" + [ "$UNHEALTHY" = "0" ] && [ "$TOTAL" -gt 0 ] && printf '{{.C_OK}}✓ All %s Crossplane providers Healthy.{{.C_RESET}}\n' "$TOTAL" && exit 0 + [ $((i % 4)) -eq 0 ] && printf '{{.C_INFO}} [%ds] %s/%s providers Healthy...{{.C_RESET}}\n' "$((i*15))" "$((TOTAL-UNHEALTHY))" "$TOTAL" # After 5min, force-sync crossplane-base to unblock stuck ArgoCD retry loops if [ "$i" -eq 20 ]; then printf '{{.C_WARN}} Providers not all Healthy after 5min — force-syncing crossplane-base...{{.C_RESET}}\n' @@ -725,8 +722,7 @@ tasks: fi sleep 15 done - printf '{{.C_WARN}} Providers not all Healthy after 15min — continuing anyway.{{.C_RESET}} -' + printf '{{.C_WARN}} Providers not all Healthy after 15min — continuing anyway.{{.C_RESET}}\n' hub:wait-for-sync: desc: Wait for hub ArgoCD apps to reach Synced state From 82536b83dc9499ddd33e56909fd4bc6f0f3eebe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 13:10:43 +0200 Subject: [PATCH 151/231] fix: wait 60s after restarting pods for EKS Pod Identity agent to inject credentials before ArgoCD syncs --- cluster-providers/kind-crossplane/Taskfile.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 5a8ff45b3..22cd10857 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -1647,6 +1647,11 @@ tasks: - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout status deploy -n kagent litellm --timeout=120s 2>/dev/null || true - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig - printf '{{.C_OK}}✓ Identity-dependent pods restarted.{{.C_RESET}}\n' + # Wait 60s for EKS Pod Identity agent to inject credentials into new pods + # before ArgoCD syncs crossplane-base and tries to create IAM roles + - printf '{{.C_INFO}} Waiting 60s for Pod Identity credentials to be injected...{{.C_RESET}}\n' + - sleep 60 + - printf '{{.C_OK}}✓ Pod Identity credentials ready.{{.C_RESET}}\n' hub:restart-langfuse: desc: Restart Langfuse to pick up Keycloak client secret (created by config job after initial deploy) From 20625b63087dc2fb10959bff42f5e33345108be5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 13:38:23 +0200 Subject: [PATCH 152/231] =?UTF-8?q?fix:=20crossplane:providers=20status=20?= =?UTF-8?q?check=20also=20verifies=20pod=20identity=20associations=20exist?= =?UTF-8?q?=20in=20AWS=20=E2=80=94=20prevents=20skipping=20when=20PIA=20mi?= =?UTF-8?q?ssing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 22cd10857..2ff2a1edf 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -348,6 +348,8 @@ tasks: status: - kubectl get providers.pkg.crossplane.io provider-aws-iam -o jsonpath='{.status.conditions[?(@.type=="Healthy")].status}' 2>/dev/null | grep -q True - kubectl get providerconfig.kubernetes.crossplane.io default 2>/dev/null + # Also verify pod identity associations exist in AWS (not just provider Healthy) + - aws eks list-pod-identity-associations --cluster-name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --namespace crossplane-system --query 'length(associations)' --output text 2>/dev/null | grep -qv '^0$' vars: K8S_PROVIDER_VERSION: sh: yq '.crossplane-base.valuesObject.providers.kubernetes.version // "v1.2.1"' {{.GITOPS_ROOT}}/addons/registry/platform.yaml From 198a684eb1b200a693318c3bd2d29d34057e067d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 14:08:07 +0200 Subject: [PATCH 153/231] =?UTF-8?q?fix:=20split=20HuggingFaceModel=20into?= =?UTF-8?q?=20platform-manifests-post=20(wave=208)=20=E2=80=94=20KRO=20RGD?= =?UTF-8?q?=20must=20be=20processed=20before=20CRs=20are=20applied?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../charts/platform-manifests-post/Chart.yaml | 6 +++++ .../templates/huggingface-models.yaml | 0 .../platform-manifests-post/values.yaml | 1 + gitops/addons/registry/platform.yaml | 24 +++++++++++++++++++ .../platform-manifests-post/values.yaml | 16 +++++++++++++ .../platform-manifests/values.yaml | 16 ------------- 6 files changed, 47 insertions(+), 16 deletions(-) create mode 100644 gitops/addons/charts/platform-manifests-post/Chart.yaml rename gitops/addons/charts/{platform-manifests => platform-manifests-post}/templates/huggingface-models.yaml (100%) create mode 100644 gitops/addons/charts/platform-manifests-post/values.yaml create mode 100644 gitops/overlays/environments/control-plane/platform-manifests-post/values.yaml diff --git a/gitops/addons/charts/platform-manifests-post/Chart.yaml b/gitops/addons/charts/platform-manifests-post/Chart.yaml new file mode 100644 index 000000000..64f565b3e --- /dev/null +++ b/gitops/addons/charts/platform-manifests-post/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: platform-manifests-post +description: Post-install platform manifests — HuggingFaceModel resources that require KRO RGDs to be processed first +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/gitops/addons/charts/platform-manifests/templates/huggingface-models.yaml b/gitops/addons/charts/platform-manifests-post/templates/huggingface-models.yaml similarity index 100% rename from gitops/addons/charts/platform-manifests/templates/huggingface-models.yaml rename to gitops/addons/charts/platform-manifests-post/templates/huggingface-models.yaml diff --git a/gitops/addons/charts/platform-manifests-post/values.yaml b/gitops/addons/charts/platform-manifests-post/values.yaml new file mode 100644 index 000000000..ce1c05d8b --- /dev/null +++ b/gitops/addons/charts/platform-manifests-post/values.yaml @@ -0,0 +1 @@ +huggingfaceModels: [] diff --git a/gitops/addons/registry/platform.yaml b/gitops/addons/registry/platform.yaml index d5d59d4e6..72de3ee3e 100644 --- a/gitops/addons/registry/platform.yaml +++ b/gitops/addons/registry/platform.yaml @@ -555,3 +555,27 @@ platform-manifests: accountId: '{{.metadata.annotations.aws_account_id}}' clusterName: '{{.metadata.annotations.aws_cluster_name}}' region: '{{.metadata.annotations.aws_region}}' + +platform-manifests-post: + namespace: default + path: '{{.metadata.annotations.addonsRepoBasepath}}addons/charts/platform-manifests-post' + annotationsAppSet: + argocd.argoproj.io/sync-wave: '8' + syncPolicy: + automated: + selfHeal: true + allowEmpty: true + prune: false + syncOptions: + - CreateNamespace=true + - SkipDryRunOnMissingResource=true + selector: + matchExpressions: + - key: enable_platform_manifests + operator: In + values: ['true'] + valuesObject: + global: + clusterName: '{{.metadata.annotations.aws_cluster_name}}' + accountId: '{{.metadata.annotations.aws_account_id}}' + region: '{{.metadata.annotations.aws_region}}' diff --git a/gitops/overlays/environments/control-plane/platform-manifests-post/values.yaml b/gitops/overlays/environments/control-plane/platform-manifests-post/values.yaml new file mode 100644 index 000000000..18a550eb7 --- /dev/null +++ b/gitops/overlays/environments/control-plane/platform-manifests-post/values.yaml @@ -0,0 +1,16 @@ +# HuggingFace models are in platform-manifests-post (wave 8) to ensure the +# HuggingFaceModel KRO RGD is fully processed before these resources are applied. +# kro-manifests-hub (wave -2) installs the RGD but KRO needs time to reconcile it +# into a CRD — wave 8 gives sufficient buffer after wave -2. +huggingfaceModels: + - name: gpt2-neuron + modelId: "gpt2" + namespace: "ray-system" + + - name: mistral-7b-neuron + modelId: "mistralai/Mistral-7B-v0.1" + namespace: "ray-system" + + - name: tinyllama + modelId: "TinyLlama/TinyLlama-1.1B-Chat-v1.0" + namespace: "ray-system" diff --git a/gitops/overlays/environments/control-plane/platform-manifests/values.yaml b/gitops/overlays/environments/control-plane/platform-manifests/values.yaml index 73fa7291e..e7d4cec8a 100644 --- a/gitops/overlays/environments/control-plane/platform-manifests/values.yaml +++ b/gitops/overlays/environments/control-plane/platform-manifests/values.yaml @@ -1,18 +1,2 @@ customNodepools: enabled: true - -# HuggingFace models are enabled here on the control-plane/hub, which is the only -# place the HuggingFaceModel kro CRD is installed (via kro-manifests-hub). The base -# default is empty; enable the sample models here for the ML use cases. -huggingfaceModels: - - name: gpt2-neuron - modelId: "gpt2" - namespace: "ray-system" - - - name: mistral-7b-neuron - modelId: "mistralai/Mistral-7B-v0.1" - namespace: "ray-system" - - - name: tinyllama - modelId: "TinyLlama/TinyLlama-1.1B-Chat-v1.0" - namespace: "ray-system" From 4e4837a96188b0f11277c52f5d4857f991f8c1f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 15:29:46 +0200 Subject: [PATCH 154/231] fix: also write exposure_mode=cloudfront when updating ingress_domain_name in Secrets Manager --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 2ff2a1edf..ab48ab5c1 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -695,7 +695,7 @@ tasks: printf ' ingress_domain_name already correct in %s\n' "$SECRET_ID" continue fi - NEW=$(echo "$CUR" | jq -c --arg cf "$CF" '.metadata = ((.metadata | fromjson | .ingress_domain_name = $cf) | tojson)') + NEW=$(echo "$CUR" | jq -c --arg cf "$CF" '.metadata = ((.metadata | fromjson | .ingress_domain_name = $cf | .exposure_mode = "cloudfront") | tojson)') printf '%s' "$NEW" > /tmp/secret_ingress_update.json aws secretsmanager put-secret-value --secret-id "$SECRET_ID" --region {{.AWS_REGION}} \ --secret-string file:///tmp/secret_ingress_update.json --query 'Name' --output text From 60e0891ba039d99b785d25f9ae9d33800fcaa1ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 15:31:29 +0200 Subject: [PATCH 155/231] =?UTF-8?q?fix:=20add=20exposure=5Fmode=20to=20kro?= =?UTF-8?q?-ack=20hub:seed=20metadata=20=E2=80=94=20needed=20by=20argo-wor?= =?UTF-8?q?kflows=20ApplicationSet=20template?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-kro-ack/Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index e1684eadf..4460f249f 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -677,7 +677,7 @@ tasks: - | SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" SECRET_VALUE=$(jq -n \ - --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}"}' \ + --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}","exposure_mode":"{{if .CLOUDFRONT_DOMAIN}}cloudfront{{else}}domain{{end}}"}' \ --arg config '{"tlsClientConfig":{"insecure":false}}' \ --arg server '{{.CLUSTER_ARN}}' \ --arg addons '' \ From e49084cb875c75e44618cff0aeb373731b5a4aa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 15:36:34 +0200 Subject: [PATCH 156/231] =?UTF-8?q?fix:=20wait=20for=20EKS=20cluster=20ACT?= =?UTF-8?q?IVE=20before=20creating=20ALB=20=E2=80=94=20cluster=20may=20sti?= =?UTF-8?q?ll=20be=20provisioning=20at=20setup-exposure=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/common/Taskfile.cloudfront.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cluster-providers/common/Taskfile.cloudfront.yaml b/cluster-providers/common/Taskfile.cloudfront.yaml index 23e60b063..bf8b22391 100644 --- a/cluster-providers/common/Taskfile.cloudfront.yaml +++ b/cluster-providers/common/Taskfile.cloudfront.yaml @@ -42,6 +42,15 @@ tasks: - printf '{{.C_STEP}}▸ Creating platform ALB (internal, CloudFront VPC Origin mode)...{{.C_RESET}}\n' - | set -e + # Wait for EKS cluster to be ACTIVE (may still be provisioning when this runs) + for i in $(seq 1 60); do + STATUS=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} \ + --query 'cluster.status' --output text 2>/dev/null || echo "NOT_FOUND") + [ "$STATUS" = "ACTIVE" ] && break + [ $((i % 4)) -eq 0 ] && printf '{{.C_INFO}} [%ds] Waiting for EKS cluster ACTIVE (status: %s)...{{.C_RESET}}\n' "$((i*15))" "$STATUS" + sleep 15 + done + [ "$STATUS" != "ACTIVE" ] && echo "ERROR: EKS cluster not ACTIVE after 15min" && exit 1 VPC_ID="{{.CLOUDFRONT_VPC_ID}}" if [ -z "$VPC_ID" ]; then VPC_ID=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} \ From 22370c49febdd324ed1d861365ec5badb89c2e0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 15:42:19 +0200 Subject: [PATCH 157/231] =?UTF-8?q?fix:=20make=20ray:setup/wait-image/pres?= =?UTF-8?q?tage-models=20optional=20in=20workshop:install=20=E2=80=94=20sk?= =?UTF-8?q?ip=20gracefully=20if=20provider=20doesn't=20support=20them?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/workshop/Taskfile.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index ba312858f..bbeb26844 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -66,16 +66,19 @@ tasks: - task: ":{{.PROVIDER}}:install" - 'printf "{{.C_INFO}} [%ds] {{.PROVIDER}}:install done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - printf '{{.C_OK}}✓ Platform install complete.{{.C_RESET}}\n' - # Ray ML infrastructure (image builds in background) - - task: ":{{.PROVIDER}}:ray:setup" + # Ray ML infrastructure (image builds in background) — optional, skip if not supported by provider + - cmd: task ":{{.PROVIDER}}:ray:setup" 2>/dev/null || printf '{{.C_INFO}} ray:setup not available for {{.PROVIDER}} — skipping{{.C_RESET}}\n' + ignore_error: true - 'printf "{{.C_INFO}} [%ds] ray:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # IDC/Keycloak federation (ArgoCD SSO) - task: post-install - 'printf "{{.C_INFO}} [%ds] post-install done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # Wait for Ray image to be ready - - task: ":{{.PROVIDER}}:ray:wait-image" + - cmd: task ":{{.PROVIDER}}:ray:wait-image" 2>/dev/null || true + ignore_error: true # Download ML models to S3 - - task: ":{{.PROVIDER}}:ray:prestage-models" + - cmd: task ":{{.PROVIDER}}:ray:prestage-models" 2>/dev/null || true + ignore_error: true # Write platform URLs + credentials to ~/.bashrc.d/platform.sh for IDE use - task: setup-env - 'printf "{{.C_OK}}✓ Workshop install complete in %ds.{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' From d2c04445994cdd10e3451a93bf00437a67d79657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 15:44:06 +0200 Subject: [PATCH 158/231] =?UTF-8?q?refactor:=20extract=20ray=20tasks=20to?= =?UTF-8?q?=20common/Taskfile.ray.yaml,=20include=20in=20kro-ack=20?= =?UTF-8?q?=E2=80=94=20Ray=20now=20works=20on=20both=20providers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/common/Taskfile.ray.yaml | 123 +++++++++++++++++++ cluster-providers/kind-kro-ack/Taskfile.yaml | 8 ++ cluster-providers/workshop/Taskfile.yaml | 11 +- 3 files changed, 135 insertions(+), 7 deletions(-) create mode 100644 cluster-providers/common/Taskfile.ray.yaml diff --git a/cluster-providers/common/Taskfile.ray.yaml b/cluster-providers/common/Taskfile.ray.yaml new file mode 100644 index 000000000..58638deca --- /dev/null +++ b/cluster-providers/common/Taskfile.ray.yaml @@ -0,0 +1,123 @@ +version: "3" +set: [errexit, nounset, pipefail] +silent: true + +# Shared Ray ML infrastructure tasks. +# Provider-agnostic — used by both kind-crossplane and kind-kro-ack. +# Required vars: HUB_CLUSTER_NAME, RESOURCE_PREFIX, AWS_ACCOUNT_ID, AWS_REGION, ROOT_DIR +tasks: + ray:setup: + desc: "Set up Ray ML infrastructure: S3 bucket, ECR repo, vLLM image, S3 CSI driver, IAM + Pod Identity" + vars: + RAY_BUCKET: "{{.RESOURCE_PREFIX}}-ray-models-{{.AWS_ACCOUNT_ID}}" + RAY_ECR_REPO: "{{.RESOURCE_PREFIX}}-ray-vllm-custom" + RAY_ROLE_PRESTAGE: "{{.RESOURCE_PREFIX}}-model-prestage-role" + RAY_ROLE_WORKER: "{{.RESOURCE_PREFIX}}-ray-worker-role" + DOCKERFILE_PATH: "{{.ROOT_DIR}}/platform/infra/terraform/common/Dockerfile.ray-vllm" + cmds: + - echo "▸ Setting up Ray ML infrastructure..." + # 1. S3 bucket + - cmd: | + if aws s3api head-bucket --bucket "{{.RAY_BUCKET}}" --region {{.AWS_REGION}} 2>/dev/null; then + echo " ✓ S3 bucket {{.RAY_BUCKET}} exists" + else + aws s3api create-bucket --bucket "{{.RAY_BUCKET}}" --region {{.AWS_REGION}} \ + --create-bucket-configuration LocationConstraint={{.AWS_REGION}} --output text >/dev/null + aws s3api put-public-access-block --bucket "{{.RAY_BUCKET}}" \ + --public-access-block-configuration BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true + echo " ✓ S3 bucket {{.RAY_BUCKET}} created" + fi + # 2. ECR repo + - cmd: | + if aws ecr describe-repositories --repository-names "{{.RAY_ECR_REPO}}" --region {{.AWS_REGION}} >/dev/null 2>&1; then + echo " ✓ ECR repo {{.RAY_ECR_REPO}} exists" + else + aws ecr create-repository --repository-name "{{.RAY_ECR_REPO}}" --region {{.AWS_REGION}} \ + --image-tag-mutability MUTABLE --output text >/dev/null + echo " ✓ ECR repo {{.RAY_ECR_REPO}} created" + fi + # 3. Build and push vLLM image (background - non-blocking) + - cmd: | + IMAGE_URI="{{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com/{{.RAY_ECR_REPO}}:latest" + if aws ecr describe-images --repository-name "{{.RAY_ECR_REPO}}" --image-ids imageTag=latest --region {{.AWS_REGION}} >/dev/null 2>&1; then + echo " ✓ vLLM image already exists" + else + echo " ▸ Building vLLM image in background (5-10 min)..." + aws ecr get-login-password --region {{.AWS_REGION}} | docker login --username AWS --password-stdin {{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com >/dev/null 2>&1 + nohup bash -c "docker build -t '$IMAGE_URI' -f '{{.DOCKERFILE_PATH}}' '$(dirname {{.DOCKERFILE_PATH}})' && docker push '$IMAGE_URI' && echo '[ray] ✓ vLLM image pushed' >> /tmp/ray-build.log" > /tmp/ray-build.log 2>&1 & + echo " ✓ Build started (PID $!, log: /tmp/ray-build.log)" + echo " Run 'task ray:wait-image' to wait for completion." + fi + # 4. S3 CSI driver addon + - cmd: | + if aws eks describe-addon --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} >/dev/null 2>&1; then + echo " ✓ S3 CSI driver addon installed" + else + aws eks create-addon --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} --output text >/dev/null + aws eks wait addon-active --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} 2>/dev/null || true + echo " ✓ S3 CSI driver addon installed" + fi + # 5. IAM roles + - cmd: | + TRUST='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"pods.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' + S3_RW='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket"],"Resource":["arn:aws:s3:::{{.RAY_BUCKET}}","arn:aws:s3:::{{.RAY_BUCKET}}/*"]}]}' + S3_RO='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:ListBucket"],"Resource":["arn:aws:s3:::{{.RAY_BUCKET}}","arn:aws:s3:::{{.RAY_BUCKET}}/*"]}]}' + for ROLE_INFO in "{{.RAY_ROLE_PRESTAGE}}|$S3_RW" "{{.RAY_ROLE_WORKER}}|$S3_RO"; do + ROLE="${ROLE_INFO%%|*}"; POLICY="${ROLE_INFO#*|}" + if aws iam get-role --role-name "$ROLE" >/dev/null 2>&1; then + echo " ✓ IAM role $ROLE exists" + else + aws iam create-role --role-name "$ROLE" --assume-role-policy-document "$TRUST" --output text >/dev/null + aws iam put-role-policy --role-name "$ROLE" --policy-name s3-access --policy-document "$POLICY" + echo " ✓ IAM role $ROLE created" + fi + done + # 6. Pod Identity associations + - cmd: | + PRESTAGE_ARN="arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RAY_ROLE_PRESTAGE}}" + WORKER_ARN="arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RAY_ROLE_WORKER}}" + for SA_INFO in "ray-system|model-prestage-sa|$PRESTAGE_ARN" "ray-system|ray-worker-sa|$WORKER_ARN" "kube-system|s3-csi-driver-sa|$WORKER_ARN"; do + NS="${SA_INFO%%|*}"; REST="${SA_INFO#*|}"; SA="${REST%%|*}"; ARN="${REST#*|}" + if aws eks list-pod-identity-associations --cluster-name {{.HUB_CLUSTER_NAME}} --namespace "$NS" --service-account "$SA" --region {{.AWS_REGION}} --query 'associations[0].associationId' --output text 2>/dev/null | grep -q "^a-"; then + echo " ✓ PIA $SA in $NS exists" + else + aws eks create-pod-identity-association --cluster-name {{.HUB_CLUSTER_NAME}} --namespace "$NS" --service-account "$SA" --role-arn "$ARN" --region {{.AWS_REGION}} --output text >/dev/null 2>&1 + echo " ✓ PIA $SA in $NS created" + fi + done + - echo "✓ Ray ML infrastructure ready (bucket={{.RAY_BUCKET}}, image={{.RAY_ECR_REPO}}:latest)" + + ray:prestage-models: + desc: "Download ML models to S3 bucket for Ray serving" + vars: + RAY_BUCKET: "{{.RESOURCE_PREFIX}}-ray-models-{{.AWS_ACCOUNT_ID}}" + cmds: + - cmd: | + pip3 install -q huggingface_hub 2>/dev/null + echo "Downloading TinyLlama..." + python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='TinyLlama/TinyLlama-1.1B-Chat-v1.0', local_dir='/tmp/tinyllama', local_dir_use_symlinks=False)" + echo "Uploading to s3://{{.RAY_BUCKET}}/models/tinyllama/..." + aws s3 sync /tmp/tinyllama/ s3://{{.RAY_BUCKET}}/models/tinyllama/ --region {{.AWS_REGION}} --no-progress + rm -rf /tmp/tinyllama + echo "✓ TinyLlama model uploaded" + + + ray:wait-image: + desc: "Wait for the background vLLM image build to complete" + vars: + RAY_ECR_REPO: "{{.RESOURCE_PREFIX}}-ray-vllm-custom" + cmds: + - cmd: | + echo "Waiting for vLLM image build..." + for i in $(seq 1 60); do + if aws ecr describe-images --repository-name "{{.RAY_ECR_REPO}}" --image-ids imageTag=latest --region {{.AWS_REGION}} >/dev/null 2>&1; then + echo "✓ vLLM image ready in ECR" + exit 0 + fi + if [ -f /tmp/ray-build.log ]; then + tail -1 /tmp/ray-build.log + fi + sleep 10 + done + echo "✗ Timeout waiting for image (check /tmp/ray-build.log)" + exit 1 diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 4460f249f..40d57caf2 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -20,6 +20,14 @@ includes: C_INFO: '{{.C_INFO}}' C_WARN: '{{.C_WARN}}' CLOUDFRONT_VPC_ID: '' + ray: + taskfile: ../common/Taskfile.ray.yaml + vars: + AWS_REGION: '{{.AWS_REGION}}' + AWS_ACCOUNT_ID: '{{.AWS_ACCOUNT_ID}}' + HUB_CLUSTER_NAME: '{{.HUB_CLUSTER_NAME}}' + RESOURCE_PREFIX: '{{.RESOURCE_PREFIX}}' + ROOT_DIR: '{{.ROOT_DIR}}' env: KUBECONFIG: "{{.ROOT_DIR}}/private/kubeconfig" diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index bbeb26844..ba312858f 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -66,19 +66,16 @@ tasks: - task: ":{{.PROVIDER}}:install" - 'printf "{{.C_INFO}} [%ds] {{.PROVIDER}}:install done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - printf '{{.C_OK}}✓ Platform install complete.{{.C_RESET}}\n' - # Ray ML infrastructure (image builds in background) — optional, skip if not supported by provider - - cmd: task ":{{.PROVIDER}}:ray:setup" 2>/dev/null || printf '{{.C_INFO}} ray:setup not available for {{.PROVIDER}} — skipping{{.C_RESET}}\n' - ignore_error: true + # Ray ML infrastructure (image builds in background) + - task: ":{{.PROVIDER}}:ray:setup" - 'printf "{{.C_INFO}} [%ds] ray:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # IDC/Keycloak federation (ArgoCD SSO) - task: post-install - 'printf "{{.C_INFO}} [%ds] post-install done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # Wait for Ray image to be ready - - cmd: task ":{{.PROVIDER}}:ray:wait-image" 2>/dev/null || true - ignore_error: true + - task: ":{{.PROVIDER}}:ray:wait-image" # Download ML models to S3 - - cmd: task ":{{.PROVIDER}}:ray:prestage-models" 2>/dev/null || true - ignore_error: true + - task: ":{{.PROVIDER}}:ray:prestage-models" # Write platform URLs + credentials to ~/.bashrc.d/platform.sh for IDE use - task: setup-env - 'printf "{{.C_OK}}✓ Workshop install complete in %ds.{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' From f74cb198796c7f1394611e18dd1eadfb0279733d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 15:44:39 +0200 Subject: [PATCH 159/231] =?UTF-8?q?refactor:=20remove=20inline=20ray=20tas?= =?UTF-8?q?ks=20from=20kind-crossplane=20=E2=80=94=20now=20uses=20shared?= =?UTF-8?q?=20common/Taskfile.ray.yaml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kind-crossplane/Taskfile.yaml | 124 ++---------------- 1 file changed, 8 insertions(+), 116 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index ab48ab5c1..4d1a00aa1 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -21,6 +21,14 @@ includes: C_INFO: '{{.C_INFO}}' C_WARN: '{{.C_WARN}}' CLOUDFRONT_VPC_ID: '{{.CLOUDFRONT_VPC_ID}}' + ray: + taskfile: ../common/Taskfile.ray.yaml + vars: + AWS_REGION: '{{.AWS_REGION}}' + AWS_ACCOUNT_ID: '{{.AWS_ACCOUNT_ID}}' + HUB_CLUSTER_NAME: '{{.HUB_CLUSTER_NAME}}' + RESOURCE_PREFIX: '{{.RESOURCE_PREFIX}}' + ROOT_DIR: '{{.ROOT_DIR}}' env: @@ -2668,122 +2676,6 @@ tasks: printf '{{.C_OK}}✓ Spoke provider identity seeding complete.{{.C_RESET}}\n' - ray:setup: - desc: "Set up Ray ML infrastructure: S3 bucket, ECR repo, vLLM image, S3 CSI driver, IAM + Pod Identity" - vars: - RAY_BUCKET: "{{.RESOURCE_PREFIX}}-ray-models-{{.AWS_ACCOUNT_ID}}" - RAY_ECR_REPO: "{{.RESOURCE_PREFIX}}-ray-vllm-custom" - RAY_ROLE_PRESTAGE: "{{.RESOURCE_PREFIX}}-model-prestage-role" - RAY_ROLE_WORKER: "{{.RESOURCE_PREFIX}}-ray-worker-role" - DOCKERFILE_PATH: "{{.ROOT_DIR}}/platform/infra/terraform/common/Dockerfile.ray-vllm" - cmds: - - echo "▸ Setting up Ray ML infrastructure..." - # 1. S3 bucket - - cmd: | - if aws s3api head-bucket --bucket "{{.RAY_BUCKET}}" --region {{.AWS_REGION}} 2>/dev/null; then - echo " ✓ S3 bucket {{.RAY_BUCKET}} exists" - else - aws s3api create-bucket --bucket "{{.RAY_BUCKET}}" --region {{.AWS_REGION}} \ - --create-bucket-configuration LocationConstraint={{.AWS_REGION}} --output text >/dev/null - aws s3api put-public-access-block --bucket "{{.RAY_BUCKET}}" \ - --public-access-block-configuration BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true - echo " ✓ S3 bucket {{.RAY_BUCKET}} created" - fi - # 2. ECR repo - - cmd: | - if aws ecr describe-repositories --repository-names "{{.RAY_ECR_REPO}}" --region {{.AWS_REGION}} >/dev/null 2>&1; then - echo " ✓ ECR repo {{.RAY_ECR_REPO}} exists" - else - aws ecr create-repository --repository-name "{{.RAY_ECR_REPO}}" --region {{.AWS_REGION}} \ - --image-tag-mutability MUTABLE --output text >/dev/null - echo " ✓ ECR repo {{.RAY_ECR_REPO}} created" - fi - # 3. Build and push vLLM image (background - non-blocking) - - cmd: | - IMAGE_URI="{{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com/{{.RAY_ECR_REPO}}:latest" - if aws ecr describe-images --repository-name "{{.RAY_ECR_REPO}}" --image-ids imageTag=latest --region {{.AWS_REGION}} >/dev/null 2>&1; then - echo " ✓ vLLM image already exists" - else - echo " ▸ Building vLLM image in background (5-10 min)..." - aws ecr get-login-password --region {{.AWS_REGION}} | docker login --username AWS --password-stdin {{.AWS_ACCOUNT_ID}}.dkr.ecr.{{.AWS_REGION}}.amazonaws.com >/dev/null 2>&1 - nohup bash -c "docker build -t '$IMAGE_URI' -f '{{.DOCKERFILE_PATH}}' '$(dirname {{.DOCKERFILE_PATH}})' && docker push '$IMAGE_URI' && echo '[ray] ✓ vLLM image pushed' >> /tmp/ray-build.log" > /tmp/ray-build.log 2>&1 & - echo " ✓ Build started (PID $!, log: /tmp/ray-build.log)" - echo " Run 'task ray:wait-image' to wait for completion." - fi - # 4. S3 CSI driver addon - - cmd: | - if aws eks describe-addon --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} >/dev/null 2>&1; then - echo " ✓ S3 CSI driver addon installed" - else - aws eks create-addon --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} --output text >/dev/null - aws eks wait addon-active --cluster-name {{.HUB_CLUSTER_NAME}} --addon-name aws-mountpoint-s3-csi-driver --region {{.AWS_REGION}} 2>/dev/null || true - echo " ✓ S3 CSI driver addon installed" - fi - # 5. IAM roles - - cmd: | - TRUST='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"pods.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' - S3_RW='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:PutObject","s3:DeleteObject","s3:ListBucket"],"Resource":["arn:aws:s3:::{{.RAY_BUCKET}}","arn:aws:s3:::{{.RAY_BUCKET}}/*"]}]}' - S3_RO='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:ListBucket"],"Resource":["arn:aws:s3:::{{.RAY_BUCKET}}","arn:aws:s3:::{{.RAY_BUCKET}}/*"]}]}' - for ROLE_INFO in "{{.RAY_ROLE_PRESTAGE}}|$S3_RW" "{{.RAY_ROLE_WORKER}}|$S3_RO"; do - ROLE="${ROLE_INFO%%|*}"; POLICY="${ROLE_INFO#*|}" - if aws iam get-role --role-name "$ROLE" >/dev/null 2>&1; then - echo " ✓ IAM role $ROLE exists" - else - aws iam create-role --role-name "$ROLE" --assume-role-policy-document "$TRUST" --output text >/dev/null - aws iam put-role-policy --role-name "$ROLE" --policy-name s3-access --policy-document "$POLICY" - echo " ✓ IAM role $ROLE created" - fi - done - # 6. Pod Identity associations - - cmd: | - PRESTAGE_ARN="arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RAY_ROLE_PRESTAGE}}" - WORKER_ARN="arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/{{.RAY_ROLE_WORKER}}" - for SA_INFO in "ray-system|model-prestage-sa|$PRESTAGE_ARN" "ray-system|ray-worker-sa|$WORKER_ARN" "kube-system|s3-csi-driver-sa|$WORKER_ARN"; do - NS="${SA_INFO%%|*}"; REST="${SA_INFO#*|}"; SA="${REST%%|*}"; ARN="${REST#*|}" - if aws eks list-pod-identity-associations --cluster-name {{.HUB_CLUSTER_NAME}} --namespace "$NS" --service-account "$SA" --region {{.AWS_REGION}} --query 'associations[0].associationId' --output text 2>/dev/null | grep -q "^a-"; then - echo " ✓ PIA $SA in $NS exists" - else - aws eks create-pod-identity-association --cluster-name {{.HUB_CLUSTER_NAME}} --namespace "$NS" --service-account "$SA" --role-arn "$ARN" --region {{.AWS_REGION}} --output text >/dev/null 2>&1 - echo " ✓ PIA $SA in $NS created" - fi - done - - echo "✓ Ray ML infrastructure ready (bucket={{.RAY_BUCKET}}, image={{.RAY_ECR_REPO}}:latest)" - - ray:prestage-models: - desc: "Download ML models to S3 bucket for Ray serving" - vars: - RAY_BUCKET: "{{.RESOURCE_PREFIX}}-ray-models-{{.AWS_ACCOUNT_ID}}" - cmds: - - cmd: | - pip3 install -q huggingface_hub 2>/dev/null - echo "Downloading TinyLlama..." - python3 -c "from huggingface_hub import snapshot_download; snapshot_download(repo_id='TinyLlama/TinyLlama-1.1B-Chat-v1.0', local_dir='/tmp/tinyllama', local_dir_use_symlinks=False)" - echo "Uploading to s3://{{.RAY_BUCKET}}/models/tinyllama/..." - aws s3 sync /tmp/tinyllama/ s3://{{.RAY_BUCKET}}/models/tinyllama/ --region {{.AWS_REGION}} --no-progress - rm -rf /tmp/tinyllama - echo "✓ TinyLlama model uploaded" - - - ray:wait-image: - desc: "Wait for the background vLLM image build to complete" - vars: - RAY_ECR_REPO: "{{.RESOURCE_PREFIX}}-ray-vllm-custom" - cmds: - - cmd: | - echo "Waiting for vLLM image build..." - for i in $(seq 1 60); do - if aws ecr describe-images --repository-name "{{.RAY_ECR_REPO}}" --image-ids imageTag=latest --region {{.AWS_REGION}} >/dev/null 2>&1; then - echo "✓ vLLM image ready in ECR" - exit 0 - fi - if [ -f /tmp/ray-build.log ]; then - tail -1 /tmp/ray-build.log - fi - sleep 10 - done - echo "✗ Timeout waiting for image (check /tmp/ray-build.log)" - exit 1 - hub:destroy-addons: desc: Remove all addons from the hub cluster (no git changes needed) vars: From edd211fe15b6f0f17621a20c7b21c256b7e0ed8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 16:05:58 +0200 Subject: [PATCH 160/231] fix: rename ray tasks in common/Taskfile.ray.yaml from ray:xxx to xxx so they resolve correctly under ray: namespace --- cluster-providers/common/Taskfile.ray.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster-providers/common/Taskfile.ray.yaml b/cluster-providers/common/Taskfile.ray.yaml index 58638deca..6ac3b444e 100644 --- a/cluster-providers/common/Taskfile.ray.yaml +++ b/cluster-providers/common/Taskfile.ray.yaml @@ -6,7 +6,7 @@ silent: true # Provider-agnostic — used by both kind-crossplane and kind-kro-ack. # Required vars: HUB_CLUSTER_NAME, RESOURCE_PREFIX, AWS_ACCOUNT_ID, AWS_REGION, ROOT_DIR tasks: - ray:setup: + setup: desc: "Set up Ray ML infrastructure: S3 bucket, ECR repo, vLLM image, S3 CSI driver, IAM + Pod Identity" vars: RAY_BUCKET: "{{.RESOURCE_PREFIX}}-ray-models-{{.AWS_ACCOUNT_ID}}" @@ -87,7 +87,7 @@ tasks: done - echo "✓ Ray ML infrastructure ready (bucket={{.RAY_BUCKET}}, image={{.RAY_ECR_REPO}}:latest)" - ray:prestage-models: + prestage-models: desc: "Download ML models to S3 bucket for Ray serving" vars: RAY_BUCKET: "{{.RESOURCE_PREFIX}}-ray-models-{{.AWS_ACCOUNT_ID}}" @@ -102,7 +102,7 @@ tasks: echo "✓ TinyLlama model uploaded" - ray:wait-image: + wait-image: desc: "Wait for the background vLLM image build to complete" vars: RAY_ECR_REPO: "{{.RESOURCE_PREFIX}}-ray-vllm-custom" From ea250d7bb5cd435aa2e5eef966ff8cc8d3fcfcd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 16:21:12 +0200 Subject: [PATCH 161/231] fix: remove dangling 2>/dev/null line in hub:wait-for-providers causing exit status 1 --- cluster-providers/kind-crossplane/Taskfile.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 4d1a00aa1..5a26be6b5 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -728,7 +728,6 @@ tasks: if [ "$i" -eq 20 ]; then printf '{{.C_WARN}} Providers not all Healthy after 5min — force-syncing crossplane-base...{{.C_RESET}}\n' KUBECONFIG=$KC kubectl patch application "crossplane-base-{{.HUB_CLUSTER_NAME}}" -n argocd --type merge -p '{"operation":{"initiatedBy":{"username":"admin"},"sync":{"revision":"HEAD"}}}' 2>/dev/null || true - 2>/dev/null || true fi sleep 15 done From 219308c02d576afcf96556345867356cd7e934e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 16:31:37 +0200 Subject: [PATCH 162/231] fix: hub:update-ingress-domain also writes admin_role_name, exposure_mode, resource_prefix to Secrets Manager --- cluster-providers/kind-crossplane/Taskfile.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 5a26be6b5..61501a170 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -703,7 +703,12 @@ tasks: printf ' ingress_domain_name already correct in %s\n' "$SECRET_ID" continue fi - NEW=$(echo "$CUR" | jq -c --arg cf "$CF" '.metadata = ((.metadata | fromjson | .ingress_domain_name = $cf | .exposure_mode = "cloudfront") | tojson)') + NEW=$(echo "$CUR" | jq -c --arg cf "$CF" --arg em "cloudfront" --arg arn "{{.ADMIN_ROLE_NAME}}" --arg rp "{{.RESOURCE_PREFIX}}" '.metadata = ((.metadata | fromjson | + .ingress_domain_name = $cf | + .exposure_mode = $em | + .admin_role_name = (if $arn != "" then $arn else .admin_role_name end) | + .resource_prefix = (if $rp != "" then $rp else .resource_prefix end) + ) | tojson)') printf '%s' "$NEW" > /tmp/secret_ingress_update.json aws secretsmanager put-secret-value --secret-id "$SECRET_ID" --region {{.AWS_REGION}} \ --secret-string file:///tmp/secret_ingress_update.json --query 'Name' --output text From 3994caff7f1a958dad682e249c58eafb430a25ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 16:32:56 +0200 Subject: [PATCH 163/231] fix: hub:update-ingress-domain writes all required annotations (gitlab_domain_name, alb_controller_mode) to Secrets Manager --- cluster-providers/kind-crossplane/Taskfile.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 61501a170..e95763ac4 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -703,11 +703,13 @@ tasks: printf ' ingress_domain_name already correct in %s\n' "$SECRET_ID" continue fi - NEW=$(echo "$CUR" | jq -c --arg cf "$CF" --arg em "cloudfront" --arg arn "{{.ADMIN_ROLE_NAME}}" --arg rp "{{.RESOURCE_PREFIX}}" '.metadata = ((.metadata | fromjson | + NEW=$(echo "$CUR" | jq -c --arg cf "$CF" --arg em "cloudfront" --arg arn "{{.ADMIN_ROLE_NAME}}" --arg rp "{{.RESOURCE_PREFIX}}" --arg gd "{{.GITLAB_DOMAIN}}" --arg alb "oss" '.metadata = ((.metadata | fromjson | .ingress_domain_name = $cf | .exposure_mode = $em | .admin_role_name = (if $arn != "" then $arn else .admin_role_name end) | - .resource_prefix = (if $rp != "" then $rp else .resource_prefix end) + .resource_prefix = (if $rp != "" then $rp else (.resource_prefix // "") end) | + .gitlab_domain_name = (if $gd != "" then $gd else (.gitlab_domain_name // "") end) | + .alb_controller_mode = $alb ) | tojson)') printf '%s' "$NEW" > /tmp/secret_ingress_update.json aws secretsmanager put-secret-value --secret-id "$SECRET_ID" --region {{.AWS_REGION}} \ From 364ce731e0fde550404840483f20485d994e4ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 16:37:49 +0200 Subject: [PATCH 164/231] =?UTF-8?q?fix:=20hub:seed=20writes=20correct=20in?= =?UTF-8?q?gress=5Fdomain=5Fname=20from=20private/cloudfront-domain=20+=20?= =?UTF-8?q?all=20required=20annotations=20from=20day=201=20=E2=80=94=20eli?= =?UTF-8?q?minates=20hub:update-ingress-domain=20patching?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 5 ++++- cluster-providers/kind-kro-ack/Taskfile.yaml | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index e95763ac4..f95df2877 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -1731,7 +1731,10 @@ tasks: INGRESS_DOMAIN: sh: | if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ]; then - echo "" + # In cloudfront mode, ingress_domain = platform CloudFront domain + # written by cloudfront:setup-exposure before hub:seed runs + yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || \ + cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true else echo "{{.DOMAIN}}" fi diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 40d57caf2..1156978dd 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -684,8 +684,9 @@ tasks: cmds: - | SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" + GITLAB_CF_DOMAIN=$(cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain 2>/dev/null || yq '.cloudfront.gitlabDomain // ""' {{.CONFIG_FILE}} || true) SECRET_VALUE=$(jq -n \ - --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}","exposure_mode":"{{if .CLOUDFRONT_DOMAIN}}cloudfront{{else}}domain{{end}}"}' \ + --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}","exposure_mode":"{{if .CLOUDFRONT_DOMAIN}}cloudfront{{else}}domain{{end}}","admin_role_name":"{{.ADMIN_ROLE_NAME}}","gitlab_domain_name":"$GITLAB_CF_DOMAIN","git_token":"{{.USER_PASSWORD}}","aws_argocd_url":""}' \ --arg config '{"tlsClientConfig":{"insecure":false}}' \ --arg server '{{.CLUSTER_ARN}}' \ --arg addons '' \ From 878062aa4123ef26d2b6d565c1849152d0260e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 16:45:54 +0200 Subject: [PATCH 165/231] =?UTF-8?q?fix:=20write=20CLOUDFRONT=5FDOMAIN=20to?= =?UTF-8?q?=20platform.sh=20in=20setup-env=20=E2=80=94=20needed=20by=20pos?= =?UTF-8?q?t-install=20IDC=20script?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/workshop/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index ba312858f..d81d74457 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -168,6 +168,7 @@ tasks: set_var GRAFANA_URL "https://${CF}/grafana" set_var GITLAB_URL "https://${GITLAB_CF}" set_var GITLAB_DOMAIN "${GITLAB_CF}" + set_var CLOUDFRONT_DOMAIN "${CF}" set_var ARGOCD_URL "${ARGOCD}" set_var ARGOCD_SERVER "${ARGOCD_SERVER}" set_var ARGOCD_OPTS "--grpc-web" From 0b84150dc2da4895e9d09759c5960c051e61b4c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 17:14:44 +0200 Subject: [PATCH 166/231] =?UTF-8?q?fix:=20add=20hub:force-eso-refresh=20af?= =?UTF-8?q?ter=20hub:seed=20=E2=80=94=20ensures=20cluster=20secret=20annot?= =?UTF-8?q?ations=20populated=20before=20ArgoCD=20wave=205=20apps=20render?= =?UTF-8?q?=20IngressClassParams?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../kind-crossplane/Taskfile.yaml | 30 +++++++++++++++++++ cluster-providers/kind-kro-ack/Taskfile.yaml | 25 ++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index f95df2877..e1517d23f 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -144,6 +144,10 @@ tasks: - task: cloudfront:setup-exposure - task: hub:seed - 'printf "{{.C_INFO}} [%ds] hub:seed done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' + # Force ESO to immediately refresh the cluster secret from Secrets Manager + # so all annotations (exposure_mode, ingress_domain_name, etc.) are populated + # before ArgoCD generates ApplicationSet child apps at wave 5+ + - task: hub:force-eso-refresh - task: hub:wait-for-sync - 'printf "{{.C_INFO}} [%ds] hub:wait-for-sync done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # Phase 1 (parallel): CloudFront+identity restart || observability seeding @@ -684,6 +688,32 @@ tasks: - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig - printf '{{.C_OK}}✓ Root appset applied. Hub is now self-managing.{{.C_RESET}}\n' + hub:force-eso-refresh: + desc: Force ESO to immediately refresh cluster secret so annotations are populated before ArgoCD wave 5 apps render + internal: true + cmds: + - task: hub:kubeconfig + - | + KC={{.ROOT_DIR}}/private/hub-kubeconfig + printf '{{.C_STEP}}▸ Forcing ESO refresh of cluster secret annotations...{{.C_RESET}}\n' + # Force-refresh the fleet-secret ExternalSecret so ESO pulls latest metadata from Secrets Manager + ES=$(KUBECONFIG=$KC kubectl -n argocd get externalsecret -o name 2>/dev/null | grep -i fleet-secret | grep {{.HUB_CLUSTER_NAME}} | head -1 || true) + if [ -n "$ES" ]; then + KUBECONFIG=$KC kubectl -n argocd annotate "$ES" \ + force-sync="$(date +%s)" --overwrite >/dev/null 2>&1 || true + # Wait up to 60s for annotations to be populated + for i in $(seq 1 12); do + VAL=$(KUBECONFIG=$KC kubectl get secret {{.HUB_CLUSTER_NAME}} -n argocd \ + -o jsonpath='{.metadata.annotations.exposure_mode}' 2>/dev/null) + [ -n "$VAL" ] && printf '{{.C_OK}}✓ Cluster secret annotations populated (exposure_mode=%s){{.C_RESET}}\n' "$VAL" && break + sleep 5 + done + else + printf '{{.C_INFO}} fleet-secret ESO not found yet — waiting 30s for initial sync{{.C_RESET}}\n' + sleep 30 + fi + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + hub:update-ingress-domain: desc: Update ingress_domain_name in Secrets Manager metadata so ESO propagates the CloudFront domain to cluster secrets internal: true diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 1156978dd..ff2c075a6 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -141,6 +141,7 @@ tasks: - task: cloudfront:setup-exposure - task: cloudfront:sync-domain - task: hub:seed + - task: hub:force-eso-refresh - task: hub:wait-for-sync vars: { MAX_OUTOFSYNC: "20" } - task: install:phase1-parallel @@ -810,6 +811,30 @@ tasks: - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig - printf '{{.C_OK}}✓ Root appset applied. Hub is now self-managing.{{.C_RESET}}\n' + hub:force-eso-refresh: + desc: Force ESO to immediately refresh cluster secret so annotations are populated before ArgoCD wave 5 apps render + internal: true + cmds: + - task: hub:kubeconfig + - | + KC={{.ROOT_DIR}}/private/hub-kubeconfig + printf '{{.C_STEP}}▸ Forcing ESO refresh of cluster secret annotations...{{.C_RESET}}\n' + ES=$(KUBECONFIG=$KC kubectl -n argocd get externalsecret -o name 2>/dev/null | grep -i fleet-secret | grep {{.HUB_CLUSTER_NAME}} | head -1 || true) + if [ -n "$ES" ]; then + KUBECONFIG=$KC kubectl -n argocd annotate "$ES" \ + force-sync="$(date +%s)" --overwrite >/dev/null 2>&1 || true + for i in $(seq 1 12); do + VAL=$(KUBECONFIG=$KC kubectl get secret {{.HUB_CLUSTER_NAME}} -n argocd \ + -o jsonpath='{.metadata.annotations.exposure_mode}' 2>/dev/null) + [ -n "$VAL" ] && printf '{{.C_OK}}✓ Cluster secret annotations populated (exposure_mode=%s){{.C_RESET}}\n' "$VAL" && break + sleep 5 + done + else + printf '{{.C_INFO}} fleet-secret ESO not found yet — waiting 30s{{.C_RESET}}\n' + sleep 30 + fi + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + hub:wait-for-sync: desc: Wait for hub ArgoCD apps to sync vars: From 1d4fcaa41fd4d792a0fa30d51e593e925d241550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 18:35:16 +0200 Subject: [PATCH 167/231] fix: increase Pod Identity injection wait from 60s to 90s to ensure LBC/providers get credentials --- cluster-providers/kind-crossplane/Taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index e1517d23f..d83bcbb3f 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -1695,8 +1695,8 @@ tasks: - printf '{{.C_OK}}✓ Identity-dependent pods restarted.{{.C_RESET}}\n' # Wait 60s for EKS Pod Identity agent to inject credentials into new pods # before ArgoCD syncs crossplane-base and tries to create IAM roles - - printf '{{.C_INFO}} Waiting 60s for Pod Identity credentials to be injected...{{.C_RESET}}\n' - - sleep 60 + - printf '{{.C_INFO}} Waiting 90s for Pod Identity credentials to be injected...{{.C_RESET}}\n' + - sleep 90 - printf '{{.C_OK}}✓ Pod Identity credentials ready.{{.C_RESET}}\n' hub:restart-langfuse: From 9c066c87988fd0823e4fee6e02bf9a9a1d91ff50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 19:25:46 +0200 Subject: [PATCH 168/231] =?UTF-8?q?fix:=20hub:wait-for-providers=20needs?= =?UTF-8?q?=20hub:kubeconfig=20call=20=E2=80=94=20was=20using=20bootstrap?= =?UTF-8?q?=20kind=20cluster=20kubeconfig=20instead=20of=20hub=20EKS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index d83bcbb3f..02cecea67 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -752,6 +752,7 @@ tasks: hub:wait-for-providers: desc: Wait for all Crossplane providers to be Healthy (ensures ec2/rds/etc. CRDs available before spoke provisioning) cmds: + - task: hub:kubeconfig - | KC={{.ROOT_DIR}}/private/hub-kubeconfig [ ! -f "$KC" ] && KC={{.ROOT_DIR}}/private/kubeconfig @@ -769,6 +770,7 @@ tasks: sleep 15 done printf '{{.C_WARN}} Providers not all Healthy after 15min — continuing anyway.{{.C_RESET}}\n' + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig hub:wait-for-sync: desc: Wait for hub ArgoCD apps to reach Synced state From 99dac903d114dd5199e572efc9a1d30af389c46c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 20:37:11 +0200 Subject: [PATCH 169/231] fix(kind-kro-ack): gate spoke provisioning on Crossplane provider health Align kind-kro-ack install flow with kind-crossplane to fix the fresh-account bootstrap failure where spoke-dev (Crossplane path) never provisions. Root cause: the kro-ack install DAG was missing hub:wait-for-providers, so spokes were enabled before the Crossplane AWS providers had Pod Identity credentials. The providers fell through to IMDS (no EC2 IMDS role found), every managed resource stuck in ReconcileError, and the spoke EKS cluster was never created (describe-cluster NOT_FOUND indefinitely). Changes (additive, ported verbatim from kind-crossplane): - install DAG: call hub:wait-for-providers after hub:wait-for-full-sync, before spoke provisioning. It waits for all providers Healthy and force-syncs crossplane-base at the 5min mark to break the credential chicken-and-egg. - add hub:wait-for-providers task definition. - hub:restart-identity-pods: add the 90s Pod Identity credential-injection wait after restarting provider pods (was present in kind-crossplane only). --- cluster-providers/kind-kro-ack/Taskfile.yaml | 32 ++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index ff2c075a6..c0275eb3c 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -146,6 +146,8 @@ tasks: vars: { MAX_OUTOFSYNC: "20" } - task: install:phase1-parallel - task: hub:wait-for-full-sync + # Wait for ALL Crossplane providers (including ec2) to be Healthy before spoke provisioning + - task: hub:wait-for-providers - task: install:phase1b-parallel - task: hub:set-overlay-repo - task: install:phase2-parallel @@ -1190,6 +1192,31 @@ tasks: - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + # ---- from kind-crossplane ---- + hub:wait-for-providers: + desc: Wait for all Crossplane providers to be Healthy (ensures ec2/rds/etc. CRDs available before spoke provisioning) + cmds: + - task: hub:kubeconfig + - | + KC={{.ROOT_DIR}}/private/hub-kubeconfig + [ ! -f "$KC" ] && KC={{.ROOT_DIR}}/private/kubeconfig + printf '{{.C_STEP}}▸ Waiting for all Crossplane providers to be Healthy...{{.C_RESET}}\n' + for i in $(seq 1 60); do + UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' 2>/dev/null | wc -w | tr -d ' ') + TOTAL=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io --no-headers 2>/dev/null | wc -l | tr -d ' ') + [ "$UNHEALTHY" = "0" ] && [ "$TOTAL" -gt 0 ] && printf '{{.C_OK}}✓ All %s Crossplane providers Healthy.{{.C_RESET}}\n' "$TOTAL" && exit 0 + [ $((i % 4)) -eq 0 ] && printf '{{.C_INFO}} [%ds] %s/%s providers Healthy...{{.C_RESET}}\n' "$((i*15))" "$((TOTAL-UNHEALTHY))" "$TOTAL" + # After 5min, force-sync crossplane-base to unblock stuck ArgoCD retry loops + if [ "$i" -eq 20 ]; then + printf '{{.C_WARN}} Providers not all Healthy after 5min — force-syncing crossplane-base...{{.C_RESET}}\n' + KUBECONFIG=$KC kubectl patch application "crossplane-base-{{.HUB_CLUSTER_NAME}}" -n argocd --type merge -p '{"operation":{"initiatedBy":{"username":"admin"},"sync":{"revision":"HEAD"}}}' 2>/dev/null || true + fi + sleep 15 + done + printf '{{.C_WARN}} Providers not all Healthy after 15min — continuing anyway.{{.C_RESET}}\n' + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + + # ---- from kind-crossplane ---- secrets-manager:seed-secrets: desc: Seed platform user credentials into Secrets Manager for ExternalSecrets @@ -2277,6 +2304,11 @@ tasks: - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl rollout status deploy -n kagent litellm --timeout=120s 2>/dev/null || true - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig - printf '{{.C_OK}}✓ Identity-dependent pods restarted.{{.C_RESET}}\n' + # Wait 60s for EKS Pod Identity agent to inject credentials into new pods + # before ArgoCD syncs crossplane-base and tries to create IAM roles + - printf '{{.C_INFO}} Waiting 90s for Pod Identity credentials to be injected...{{.C_RESET}}\n' + - sleep 90 + - printf '{{.C_OK}}✓ Pod Identity credentials ready.{{.C_RESET}}\n' hub:update: From 6de85b70e38ec6715b05171b9860bd6ce26dfb80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 20:40:05 +0200 Subject: [PATCH 170/231] =?UTF-8?q?fix:=20add=20ignore=5Ferror=20to=20hub:?= =?UTF-8?q?wait-for-providers=20=E2=80=94=20providers=20being=20unhealthy?= =?UTF-8?q?=20should=20not=20block=20spoke=20provisioning?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cluster-providers/kind-crossplane/Taskfile.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 02cecea67..2543a129b 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -751,6 +751,7 @@ tasks: hub:wait-for-providers: desc: Wait for all Crossplane providers to be Healthy (ensures ec2/rds/etc. CRDs available before spoke provisioning) + ignore_error: true cmds: - task: hub:kubeconfig - | From d7a5acb324efa2dbdea02eb1b4f31f038c03ff3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 21:20:08 +0200 Subject: [PATCH 171/231] fix(kro): implement ALB url-rewrite transforms via CEL map literal (#16/#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AppmodService RGD's ingress (rewrite==true) node was byte-identical to the rewrite==false node — no transforms annotation was ever emitted, making the rewrite flag a no-op. Apps behind a path prefix (e.g. /rust-app) got 404s because the ALB forwarded the full path and the app didn't serve at that prefix. Root cause: the ALB transforms annotation requires a dynamic key suffix matching the backend service name (alb.ingress.kubernetes.io/transforms.). kro v0.9.2 does not interpolate CEL in annotation keys when specified as a YAML mapping — but it DOES support a standalone CEL map literal that returns the entire annotations object with computed keys. Fix: express the rewrite==true ingress annotations as a single-quoted CEL map literal: '${{"key": "val", "dynamic." + schema.metadata.name: "..."}}' This was validated on the live peeks-hub cluster (kro EKS Capability v0.9.2): RGD compiled Ready=True, instance GraphResolved=True. Also fixes #17: rust deployment template now sets healthcheckPath to a path the app actually serves (ALB healthchecks bypass the rewrite and hit targets directly — defaulting to / where the app 404s caused unhealthy targets). --- .../rust/deployment/templates/kro/application.yaml | 1 + .../kro/resource-groups/manifests/appmod-service.yaml | 7 +------ 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/applications/rust/deployment/templates/kro/application.yaml b/applications/rust/deployment/templates/kro/application.yaml index 1e154836a..64cab1f63 100644 --- a/applications/rust/deployment/templates/kro/application.yaml +++ b/applications/rust/deployment/templates/kro/application.yaml @@ -29,6 +29,7 @@ spec: enabled: true path: /rust-app rewrite: true + healthcheckPath: /collection/FRONT_PAGE ## Uncomment the sections below to enable metrics-driven progressive delivery (Module 30) # functionalGate: # enabled: true diff --git a/gitops/addons/charts/kro/resource-groups/manifests/appmod-service.yaml b/gitops/addons/charts/kro/resource-groups/manifests/appmod-service.yaml index 5e2d477a2..d7eef7a91 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/appmod-service.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/appmod-service.yaml @@ -534,12 +534,7 @@ spec: uid: ${schema.metadata.uid} blockOwnerDeletion: true controller: false - annotations: - argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/target-type: ip - alb.ingress.kubernetes.io/healthcheck-path: ${schema.spec.ingress.healthcheckPath} - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}]' + annotations: '${{"argocd.argoproj.io/tracking-id": schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue(""), "alb.ingress.kubernetes.io/scheme": "internet-facing", "alb.ingress.kubernetes.io/target-type": "ip", "alb.ingress.kubernetes.io/healthcheck-path": schema.spec.ingress.healthcheckPath, "alb.ingress.kubernetes.io/listen-ports": "[{\"HTTP\": 80}]", "alb.ingress.kubernetes.io/transforms." + schema.metadata.name: "[{\"type\":\"url-rewrite\",\"urlRewriteConfig\":{\"rewrites\":[{\"regex\":\"^" + schema.spec.ingress.path + "/?(.*)$\",\"replace\":\"/$1\"}]}}]"}}' spec: ingressClassName: alb rules: From d06468cdf9b0448752148dbe1a3f0bd2e1ae10c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 21:35:08 +0200 Subject: [PATCH 172/231] fix(spoke-access): use correct ArgoCD capability role name (lowercase) The EKS Capability API creates roles with lowercase-kebab naming (-argocd-capability-role), but the clusters-kro and clusters-crossplane appset templates hardcoded the PascalCase convention (ArgoCDCapabilityRole) from the old terraform flow. This caused the spoke AccessEntry to fail with 'InvalidParameterException: The specified principalArn is invalid' because the referenced role didn't exist. Fix: update all appset templates and the kro-ack Taskfile fallback to use the actual role name convention: -argocd-capability-role. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 2 +- gitops/bootstrap/clusters-crossplane.yaml | 2 +- gitops/bootstrap/clusters-kro.yaml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index c0275eb3c..512cebbc7 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1598,7 +1598,7 @@ tasks: desc: Create cluster-mgmt IAM roles assumed by ACK controllers (single-account prereq) vars: ACK_CAPABILITY_ROLE: - sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name ack --region {{.AWS_REGION}} --query 'capability.roleArn' --output text 2>/dev/null | sed 's|.*/||' || echo "{{.HUB_CLUSTER_NAME}}-ArgoCDCapabilityRole" + sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name ack --region {{.AWS_REGION}} --query 'capability.roleArn' --output text 2>/dev/null | sed 's|.*/||' || echo "{{.HUB_CLUSTER_NAME}}-argocd-capability-role" cmds: - printf '{{.C_STEP}}▸ Creating cluster-mgmt IAM roles for ACK...{{.C_RESET}}\n' - | diff --git a/gitops/bootstrap/clusters-crossplane.yaml b/gitops/bootstrap/clusters-crossplane.yaml index 7a5bfa1d9..88d6bdd77 100644 --- a/gitops/bootstrap/clusters-crossplane.yaml +++ b/gitops/bootstrap/clusters-crossplane.yaml @@ -62,7 +62,7 @@ spec: xrd: enabled: false global: - argoCDRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-ArgoCDCapabilityRole' + argoCDRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-argocd-capability-role' defaultRegion: '{{ .metadata.annotations.aws_region }}' valueFiles: - '$values/{{ .metadata.annotations.fleetRepoBasepath }}fleet/spoke-values/default/crossplane-clusters/values.yaml' diff --git a/gitops/bootstrap/clusters-kro.yaml b/gitops/bootstrap/clusters-kro.yaml index 627972d92..b28ff5af9 100644 --- a/gitops/bootstrap/clusters-kro.yaml +++ b/gitops/bootstrap/clusters-kro.yaml @@ -65,8 +65,8 @@ spec: ignoreMissingValueFiles: true valuesObject: global: - argoCdHubRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-ArgoCDCapabilityRole' - argoCdCapabilityRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-ArgoCDCapabilityRole' + argoCdHubRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-argocd-capability-role' + argoCdCapabilityRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-argocd-capability-role' defaultRegion: '{{ .metadata.annotations.aws_region }}' targetCluster: '{{ .clusterName }}' # Propagate the deploy branch from the control-plane so kro spokes From f25197bfe609b3708be68d9d7cd1c990c3a29ae8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 21:43:36 +0200 Subject: [PATCH 173/231] feat(kind-kro-ack): provision both spokes via KRO (drop Crossplane dependency) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch spoke-dev from spokes:enable-crossplane + create-capabilities to spokes:enable-kro, matching spoke-prod. Both spokes are now provisioned via the KRO EksclusterWithVpc RGD + ACK controllers. This eliminates the Crossplane provider credential chicken-and-egg for spoke provisioning entirely — the hub's Crossplane providers are still used for pod-identity/IAM on the hub itself, but spoke EKS clusters no longer depend on them. Also makes the environment label dynamic (derived from cluster name) so spoke-dev correctly gets 'environment: dev' instead of hardcoded 'prod'. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 512cebbc7..408ad1269 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -188,9 +188,7 @@ tasks: install:phase2-spoke-dev: internal: true cmds: - - task: spokes:enable-crossplane - vars: { CLI_ARGS: "{{.RESOURCE_PREFIX}}-spoke-dev" } - - task: spokes:create-capabilities + - task: spokes:enable-kro vars: { CLI_ARGS: "{{.RESOURCE_PREFIX}}-spoke-dev" } install:phase2-spoke-prod: @@ -1929,12 +1927,13 @@ tasks: echo " Spoke {{.CLUSTER}} already has a CIDR — keeping existing values (idempotent)." else OCTET=$(printf '%s' '{{.CLUSTER}}' | cksum | awk '{print ($1 % 253) + 2}') + ENV=$(echo "{{.CLUSTER}}" | grep -oE 'prod|staging|test|dev' | head -1); ENV="${ENV:-dev}" cat > "$KRO_VALUES" < Date: Wed, 1 Jul 2026 21:49:42 +0200 Subject: [PATCH 174/231] fix(kind-kro-ack): make hub:wait-for-providers pipefail-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The task failed immediately (exit 1) because under 'set -e -o pipefail' (the global Taskfile setting), the kubectl jsonpath that queries unhealthy providers returns non-zero when no items match (empty result). Combined with pipefail, this propagated through the pipe to wc and killed the script on the first loop iteration — even when all providers were healthy. Fix: - Split kubectl|wc pipeline: kubectl with '|| true', then wc separately - Replace '&& exit 0' conditional chain with if/then/break (chains with set -e are a trap: any false part returns non-zero → errexit fires) - Use 'break' instead of 'exit 0' so cleanup (rm kubeconfig) still runs - Add '|| true' to the progress printf line (same && chain issue) --- cluster-providers/kind-kro-ack/Taskfile.yaml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 408ad1269..566bf4a8e 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1200,10 +1200,14 @@ tasks: [ ! -f "$KC" ] && KC={{.ROOT_DIR}}/private/kubeconfig printf '{{.C_STEP}}▸ Waiting for all Crossplane providers to be Healthy...{{.C_RESET}}\n' for i in $(seq 1 60); do - UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' 2>/dev/null | wc -w | tr -d ' ') - TOTAL=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io --no-headers 2>/dev/null | wc -l | tr -d ' ') - [ "$UNHEALTHY" = "0" ] && [ "$TOTAL" -gt 0 ] && printf '{{.C_OK}}✓ All %s Crossplane providers Healthy.{{.C_RESET}}\n' "$TOTAL" && exit 0 - [ $((i % 4)) -eq 0 ] && printf '{{.C_INFO}} [%ds] %s/%s providers Healthy...{{.C_RESET}}\n' "$((i*15))" "$((TOTAL-UNHEALTHY))" "$TOTAL" + UNHEALTHY=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io -o jsonpath='{.items[?(@.status.conditions[?(@.type=="Healthy")].status!="True")].metadata.name}' 2>/dev/null || true) + UNHEALTHY=$(echo "$UNHEALTHY" | wc -w | tr -d ' ') + TOTAL=$(KUBECONFIG=$KC kubectl get providers.pkg.crossplane.io --no-headers 2>/dev/null | wc -l | tr -d ' ' || echo "0") + if [ "$UNHEALTHY" = "0" ] && [ "$TOTAL" -gt 0 ]; then + printf '{{.C_OK}}✓ All %s Crossplane providers Healthy.{{.C_RESET}}\n' "$TOTAL" + break + fi + [ $((i % 4)) -eq 0 ] && printf '{{.C_INFO}} [%ds] %s/%s providers Healthy...{{.C_RESET}}\n' "$((i*15))" "$((TOTAL-UNHEALTHY))" "$TOTAL" || true # After 5min, force-sync crossplane-base to unblock stuck ArgoCD retry loops if [ "$i" -eq 20 ]; then printf '{{.C_WARN}} Providers not all Healthy after 5min — force-syncing crossplane-base...{{.C_RESET}}\n' @@ -1211,7 +1215,7 @@ tasks: fi sleep 15 done - printf '{{.C_WARN}} Providers not all Healthy after 15min — continuing anyway.{{.C_RESET}}\n' + printf '{{.C_WARN}} Providers wait complete.{{.C_RESET}}\n' - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig From 8ca00bb5f42b5f2ffa18106ba2e0b9b5a59a376d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Wed, 1 Jul 2026 21:50:27 +0200 Subject: [PATCH 175/231] fix(kind-kro-ack): remove unnecessary hub-kubeconfig deletion in wait-for-providers The rm -f at the end of hub:wait-for-providers deletes a kubeconfig that the very next task immediately recreates. It adds no value (the file doesn't go stale within a single install run) and can cause failures if a subsequent task expects it to exist. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 566bf4a8e..c1870e0bc 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1216,7 +1216,6 @@ tasks: sleep 15 done printf '{{.C_WARN}} Providers wait complete.{{.C_RESET}}\n' - - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig # ---- from kind-crossplane ---- From 9390841159090164d30286c6a704914720933716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 11:29:17 +0200 Subject: [PATCH 176/231] fix(kro-ack,crossplane): gate apply-root-appset on ArgoCD CRDs being served Wait for applicationsets/applications.argoproj.io CRDs to be Established before applying root-appset.yaml, to avoid 'no matches for kind ApplicationSet in version argoproj.io/v1alpha1' when the ArgoCD EKS Capability has not yet reconciled after the EksCluster becomes ACTIVE. --- .../kind-crossplane/Taskfile.yaml | 20 +++++++++++++++++++ cluster-providers/kind-kro-ack/Taskfile.yaml | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 2543a129b..59d210f32 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -683,6 +683,26 @@ tasks: desc: Apply root-appset to the hub cluster cmds: - task: hub:kubeconfig + # Gate: the ArgoCD EKS Capability status can report ACTIVE before its CRDs are served in the + # cluster. root-appset.yaml contains an ApplicationSet, so we must wait until + # applicationsets.argoproj.io is registered and Established before applying, otherwise kubectl + # fails with "no matches for kind ApplicationSet in version argoproj.io/v1alpha1". + - printf '{{.C_STEP}}▸ Waiting for ArgoCD capability CRDs (ApplicationSet/Application) to be served...{{.C_RESET}}\n' + - | + KC={{.ROOT_DIR}}/private/hub-kubeconfig + for i in $(seq 1 60); do + if KUBECONFIG=$KC kubectl get crd applicationsets.argoproj.io applications.argoproj.io >/dev/null 2>&1; then + break + fi + printf '{{.C_INFO}} [%s/60] ArgoCD CRDs not ready yet...{{.C_RESET}}\n' "$i" + sleep 10 + done + if ! KUBECONFIG=$KC kubectl get crd applicationsets.argoproj.io >/dev/null 2>&1; then + printf '{{.C_ERR}}ERROR: ArgoCD ApplicationSet CRD never became available — the argocd EKS Capability has not reconciled.{{.C_RESET}}\n' + exit 1 + fi + KUBECONFIG=$KC kubectl wait --for=condition=Established \ + crd/applicationsets.argoproj.io crd/applications.argoproj.io --timeout=120s - printf '{{.C_STEP}}▸ Applying root-appset to hub...{{.C_RESET}}\n' - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f {{.GITOPS_ROOT}}/bootstrap/root-appset.yaml - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index c1870e0bc..792c17165 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -807,6 +807,26 @@ tasks: desc: Apply root-appset to hub cmds: - task: hub:kubeconfig + # Gate: the ArgoCD EKS Capability reconciles asynchronously after the EksCluster is ACTIVE. + # root-appset.yaml contains an ApplicationSet, so we must wait until applicationsets.argoproj.io + # is registered and Established before applying, otherwise kubectl fails with + # "no matches for kind ApplicationSet in version argoproj.io/v1alpha1". + - printf '{{.C_STEP}}▸ Waiting for ArgoCD capability CRDs (ApplicationSet/Application) to be served...{{.C_RESET}}\n' + - | + KC={{.ROOT_DIR}}/private/hub-kubeconfig + for i in $(seq 1 60); do + if KUBECONFIG=$KC kubectl get crd applicationsets.argoproj.io applications.argoproj.io >/dev/null 2>&1; then + break + fi + printf '{{.C_INFO}} [%s/60] ArgoCD CRDs not ready yet...{{.C_RESET}}\n' "$i" + sleep 10 + done + if ! KUBECONFIG=$KC kubectl get crd applicationsets.argoproj.io >/dev/null 2>&1; then + printf '{{.C_ERR}}ERROR: ArgoCD ApplicationSet CRD never became available — the argocd EKS Capability has not reconciled.{{.C_RESET}}\n' + exit 1 + fi + KUBECONFIG=$KC kubectl wait --for=condition=Established \ + crd/applicationsets.argoproj.io crd/applications.argoproj.io --timeout=120s - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl apply -f {{.GITOPS_ROOT}}/bootstrap/root-appset.yaml - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig - printf '{{.C_OK}}✓ Root appset applied. Hub is now self-managing.{{.C_RESET}}\n' From 6fdcc3ca2b212bd4798bae0d156265fb3314e8ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 11:55:46 +0200 Subject: [PATCH 177/231] fix(cloudfront): sync-domain before hub:seed in crossplane; drop cloudfrontDomain prereq gate - kind-crossplane: run cloudfront:sync-domain right after setup-exposure and before hub:seed (mirroring kind-kro-ack) so cloudfrontDomain is written to config.local.yaml + private/cloudfront-domain before hub:seed reads it. Covers install retries where setup-exposure's status guard skips because the -platform distribution already exists, which previously left hub:seed with an empty ingress_domain_name. - workshop:pre-install: cloudfrontDomain (platform CloudFront) is created by hub-distribution during install, not a prerequisite. Remove the hard exit 1 on empty cloudfrontDomain and correct messaging (create-config.sh sets cloudfront.gitlabDomain, not cloudfrontDomain). --- .../kind-crossplane/Taskfile.yaml | 8 +++++++- cluster-providers/workshop/Taskfile.yaml | 20 ++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 59d210f32..57e837fb1 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -140,8 +140,14 @@ tasks: - 'printf "{{.C_INFO}} [%ds] credentials:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - task: crossplane:setup - 'printf "{{.C_INFO}} [%ds] crossplane:setup done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' - # Create ALB + CloudFront BEFORE hub:seed so ingress_domain_name is correct from the start + # Create ALB + CloudFront BEFORE hub:seed so ingress_domain_name is correct from the start. + # sync-domain runs immediately after setup-exposure (mirroring kind-kro-ack) so that on an + # install retry — where setup-exposure's status guard skips because the -platform + # distribution already exists from a prior attempt — the cloudfrontDomain is still + # re-derived from AWS and written to config.local.yaml + private/cloudfront-domain BEFORE + # hub:seed reads it. Without this, a retried hub:seed would seed an empty ingress_domain_name. - task: cloudfront:setup-exposure + - task: cloudfront:sync-domain - task: hub:seed - 'printf "{{.C_INFO}} [%ds] hub:seed done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # Force ESO to immediately refresh the cluster secret from Secrets Manager diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index d81d74457..f5ba5dc42 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -92,19 +92,21 @@ tasks: - | printf '{{.C_INFO}}Workshop pre-requisites check:{{.C_RESET}}\n' printf ' In Workshop Studio (guided) events:\n' - printf ' 1. CDK bootstrap.sh provisions CloudFront + GitLab CE, writes private/gitlab-cloudfront-domain\n' + printf ' 1. CDK bootstrap.sh provisions the IDE/GitLab CloudFront + GitLab CE,\n' + printf ' writes private/gitlab-cloudfront-domain\n' printf ' 2. SSM SetupIDE runs create-config.sh which generates config.local.yaml\n' - printf ' (auto-detects region, account, IDC, and reads cloudfront.cloudfrontDomain\n' - printf ' from private/gitlab-cloudfront-domain)\n' + printf ' (auto-detects region, account, IDC; sets domain: "" for cloudfront mode\n' + printf ' and cloudfront.gitlabDomain from private/gitlab-cloudfront-domain).\n' + printf ' NOTE: cloudfront.cloudfrontDomain (the PLATFORM CloudFront) is NOT a\n' + printf ' prerequisite — it is created by cloudfront:hub-distribution during\n' + printf ' task install and written back to config.local.yaml.\n' printf ' 3. task install + task workshop:post-install run automatically\n' - printf ' For self-paced: run create-config.sh after manually creating CloudFront + ALB\n' + printf ' For self-paced: just run create-config.sh; task install creates the platform CloudFront.\n' if [ -z "{{.CLOUDFRONT_DOMAIN}}" ]; then - printf '{{.C_ERR}}ERROR: cloudfront.cloudfrontDomain not set in config.local.yaml{{.C_RESET}}\n' - printf ' Set: cloudfront:\n' - printf ' cloudfrontDomain: .cloudfront.net\n' - exit 1 + printf '{{.C_INFO}} ℹ cloudfront.cloudfrontDomain not set yet — it will be created during task install.{{.C_RESET}}\n' + else + printf '{{.C_OK}}✓ cloudfront.cloudfrontDomain: {{.CLOUDFRONT_DOMAIN}}{{.C_RESET}}\n' fi - printf '{{.C_OK}}✓ cloudfront.cloudfrontDomain: {{.CLOUDFRONT_DOMAIN}}{{.C_RESET}}\n' # ============================================================ # POST-INSTALL From 820c872e4b772d7236ec95c7cce937fc31e66e2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 14:58:40 +0200 Subject: [PATCH 178/231] fix(kind-kro-ack): serialize spoke enablement + deep-merge kro-clusters values phase2 ran spokes:enable-kro for spoke-dev and spoke-prod concurrently. Both commit+push to the same fleet-config main branch and write the same shared files, so the second push was rejected, its retry rebased two add/add commits into an unresolvable conflict, left a detached HEAD, and failed the whole task install (exit 128). - install:phase2-parallel now runs spoke-dev then spoke-prod sequentially (the tasks only do git clone/commit/push; EKS provisioning is async via ArgoCD/KRO, so serialising costs ~seconds). - spokes:enable-kro now deep-merges its cluster block into gitops/fleet/.../kro-clusters/values.yaml via 'yq . *= load(...)' instead of a full 'cat >' overwrite that dropped the sibling spoke's block (last-writer-wins data loss, and the source of the add/add rebase conflict). --- cluster-providers/kind-kro-ack/Taskfile.yaml | 27 +++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 792c17165..84f4ca944 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -181,9 +181,18 @@ tasks: install:phase2-parallel: internal: true - deps: - - install:phase2-spoke-dev - - install:phase2-spoke-prod + # NOTE: spokes are enabled SEQUENTIALLY, not in parallel. Both spoke-dev and + # spoke-prod run spokes:enable-kro, which commits+pushes to the SAME fleet-config + # main branch and writes the SAME shared files (kro-clusters/values.yaml, + # configs/multi-acct/values.yaml). Running them concurrently races on the branch + # ref: the second push is rejected, its retry does `git pull --rebase`, and the + # two commits that both *add* the same files produce an unresolvable add/add + # conflict that leaves a detached HEAD and fails the whole install. Serialising + # is cheap here (these tasks only do git clone/commit/push; the actual EKS + # provisioning happens asynchronously via ArgoCD/KRO afterward). + cmds: + - task: install:phase2-spoke-dev + - task: install:phase2-spoke-prod install:phase2-spoke-dev: internal: true @@ -1951,7 +1960,15 @@ tasks: else OCTET=$(printf '%s' '{{.CLUSTER}}' | cksum | awk '{print ($1 % 253) + 2}') ENV=$(echo "{{.CLUSTER}}" | grep -oE 'prod|staging|test|dev' | head -1); ENV="${ENV:-dev}" - cat > "$KRO_VALUES" < "$KRO_VALUES"` wrote only + # the current cluster, so a second spoke (or a rebase replay) dropped the + # sibling spoke's block (last-writer-wins data loss). Build the block in a + # temp file, then merge with `yq . *= load(...)`, which preserves every + # other cluster already present under .clusters. + [ -f "$KRO_VALUES" ] || printf 'clusters: {}\n' > "$KRO_VALUES" + KRO_BLOCK=$(mktemp) + cat > "$KRO_BLOCK" < Date: Thu, 2 Jul 2026 16:07:31 +0200 Subject: [PATCH 179/231] fix(cloudfront): resolve platform domain lazily at task-run time; drop config.local.yaml storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The platform CloudFront domain is created mid-install by cloudfront:hub-distribution, but consumers read it via taskfile-level (global) vars that go-task evaluates ONCE at load time — before the distribution exists — so the value was frozen empty. That is why idc:configure got an empty --keycloak-dns and 'urls' printed https:/// even though config.local.yaml and private/cloudfront-domain both held the domain. Make private/cloudfront-domain the single source of truth (runtime artifact), resolved lazily in each consumer's own task-level vars with an AWS 'Comment==-platform' fallback. Stop storing/reading it in config.local.yaml entirely. common/Taskfile.cloudfront.yaml: - hub-distribution writes only private/cloudfront-domain (drop yq -i config write) - sync-domain reads private -> AWS (drop config read), writes only private workshop/Taskfile.yaml (shared by both providers): - idc:configure KEYCLOAK_DNS is now a self-contained task-level resolver (private -> AWS) - setup-env CF_DOMAIN resolves lazily; gitlab domain from gitlabDomain -> private file - top-level CLOUDFRONT_DOMAIN reads private only (informational/pre-install); messaging updated kind-crossplane + kind-kro-ack: - hub:seed ingress_domain_name/exposure_mode use a task-level lazy CLOUDFRONT_DOMAIN - urls fallback resolves inline (private -> AWS) instead of the frozen global - crossplane hub:update-ingress-domain reads private -> AWS - top-level CLOUDFRONT_DOMAIN reads private only --- .../common/Taskfile.cloudfront.yaml | 12 ++--- .../kind-crossplane/Taskfile.yaml | 35 +++++++++++---- cluster-providers/kind-kro-ack/Taskfile.yaml | 23 +++++++++- cluster-providers/workshop/Taskfile.yaml | 45 ++++++++++++++----- 4 files changed, 88 insertions(+), 27 deletions(-) diff --git a/cluster-providers/common/Taskfile.cloudfront.yaml b/cluster-providers/common/Taskfile.cloudfront.yaml index bf8b22391..815b8dafb 100644 --- a/cluster-providers/common/Taskfile.cloudfront.yaml +++ b/cluster-providers/common/Taskfile.cloudfront.yaml @@ -217,25 +217,27 @@ tasks: \"PriceClass\": \"PriceClass_100\" }" \ --query "Distribution.DomainName" --output text) + # Single source of truth for the platform CloudFront domain is the runtime + # artifact private/cloudfront-domain — NOT config.local.yaml. Consumers + # (idc:configure, hub:seed, urls, setup-env) resolve it lazily at task-run + # time from this file (with an AWS lookup fallback), so it must never be a + # config input/parameter. mkdir -p {{.ROOT_DIR}}/private echo -n "$CF_DOMAIN" > {{.ROOT_DIR}}/private/cloudfront-domain - yq -i ".cloudfront.cloudfrontDomain = \"$CF_DOMAIN\"" {{.CONFIG_FILE}} printf '{{.C_OK}}✓ CloudFront created: %s{{.C_RESET}}\n' "$CF_DOMAIN" - printf '{{.C_OK}}✓ Platform CloudFront ready (VPC Origin).{{.C_RESET}}\n' sync-domain: - desc: Ensure cloudfrontDomain is written to config + private/cloudfront-domain (idempotent safety net) + desc: Ensure private/cloudfront-domain holds the platform CloudFront domain (idempotent safety net) cmds: - | - CURRENT=$(yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || \ - cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true) + CURRENT=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]') if [ -z "$CURRENT" ]; then CURRENT=$(aws cloudfront list-distributions \ --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName" \ --output text 2>/dev/null | tr -d '[:space:]') fi if [ -n "$CURRENT" ] && [ "$CURRENT" != "None" ]; then - yq -i ".cloudfront.cloudfrontDomain = \"$CURRENT\"" {{.CONFIG_FILE}} mkdir -p {{.ROOT_DIR}}/private echo -n "$CURRENT" > {{.ROOT_DIR}}/private/cloudfront-domain printf '{{.C_OK}}✓ cloudfrontDomain: %s{{.C_RESET}}\n' "$CURRENT" diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 57e837fb1..bd321ec37 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -74,8 +74,10 @@ vars: sh: yq '.ingressSecurityGroups // ""' {{.CONFIG_FILE}} EXPOSURE_MODE: sh: 'if [ -z "$(yq ''.domain // ""'' {{.CONFIG_FILE}})" ]; then echo "cloudfront"; else echo "domain"; fi' + # Frozen at load time (informational only). Tasks needing the live value resolve it + # in their own task-level vars from private/cloudfront-domain (+ AWS fallback). CLOUDFRONT_DOMAIN: - sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true + sh: cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]' || true CLOUDFRONT_VPC_ID: sh: yq '.cloudfront.vpcId // ""' {{.CONFIG_FILE}} CAPABILITY_NAME: @@ -745,12 +747,17 @@ tasks: internal: true vars: CF_DOMAIN: - sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + sh: | + D=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]') + [ -z "$D" ] && D=$(aws cloudfront list-distributions --region {{.AWS_REGION}} \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName | [0]" \ + --output text 2>/dev/null | tr -d '[:space:]') + [ "$D" = "None" ] && D="" + echo "$D" cmds: - | CF="{{.CF_DOMAIN}}" - [ -z "$CF" ] && CF=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true) - [ -z "$CF" ] && printf '{{.C_INFO}} Skipping ingress_domain_name update — cloudfrontDomain not set yet{{.C_RESET}}\n' && exit 0 + [ -z "$CF" ] && printf '{{.C_INFO}} Skipping ingress_domain_name update — cloudfront domain not created yet{{.C_RESET}}\n' && exit 0 for SECRET_ID in {{.HUB_CLUSTER_NAME}}/config; do CUR=$(aws secretsmanager get-secret-value --secret-id "$SECRET_ID" --region {{.AWS_REGION}} --query SecretString --output text 2>/dev/null) [ -z "$CUR" ] && printf ' Skipping %s — not found\n' "$SECRET_ID" && continue @@ -868,7 +875,12 @@ tasks: -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) [ -z "$ALB" ] && ALB=$(kubectl --kubeconfig "$HUB_KC" get ingress -A \ -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) - echo "${ALB:-{{.CLOUDFRONT_DOMAIN}}}" + CF=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]') + [ -z "$CF" ] && CF=$(aws cloudfront list-distributions --region {{.AWS_REGION}} \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName | [0]" \ + --output text 2>/dev/null | tr -d '[:space:]') + [ "$CF" = "None" ] && CF="" + echo "${ALB:-$CF}" else echo "{{.DOMAIN}}" fi @@ -1790,10 +1802,15 @@ tasks: INGRESS_DOMAIN: sh: | if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ]; then - # In cloudfront mode, ingress_domain = platform CloudFront domain - # written by cloudfront:setup-exposure before hub:seed runs - yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || \ - cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true + # platform CloudFront domain — resolved at task-run time from the runtime + # artifact (written by cloudfront:setup-exposure/sync-domain before hub:seed), + # with an AWS fallback. Not read from config.local.yaml. + D=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]') + [ -z "$D" ] && D=$(aws cloudfront list-distributions --region {{.AWS_REGION}} \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName | [0]" \ + --output text 2>/dev/null | tr -d '[:space:]') + [ "$D" = "None" ] && D="" + echo "$D" else echo "{{.DOMAIN}}" fi diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 84f4ca944..eb95762d7 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -56,8 +56,10 @@ vars: sh: yq '.repo.basepath' {{.CONFIG_FILE}} DOMAIN: sh: yq '.domain' {{.CONFIG_FILE}} + # Frozen at load time (informational only). Tasks needing the live value resolve it + # in their own task-level vars from private/cloudfront-domain (+ AWS fallback). CLOUDFRONT_DOMAIN: - sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true + sh: cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]' || true RESOURCE_PREFIX: sh: yq '.resourcePrefix // ""' {{.CONFIG_FILE}} INGRESS_NAME: @@ -691,6 +693,18 @@ tasks: sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.arn' --output text 2>/dev/null VPC_ID: sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text + # Resolve the platform CloudFront domain at TASK-RUN time (task-level var, evaluated + # when this task runs — after cloudfront:setup-exposure created it). The taskfile-level + # CLOUDFRONT_DOMAIN global is frozen empty at load, which would make ingress_domain_name + # empty and exposure_mode="domain". Source: private/cloudfront-domain + AWS fallback. + CLOUDFRONT_DOMAIN: + sh: | + D=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]') + [ -z "$D" ] && D=$(aws cloudfront list-distributions --region {{.AWS_REGION}} \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName | [0]" \ + --output text 2>/dev/null | tr -d '[:space:]') + [ "$D" = "None" ] && D="" + echo "$D" cmds: - | SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" @@ -959,7 +973,12 @@ tasks: -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) [ -z "$ALB" ] && ALB=$(kubectl --kubeconfig "$HUB_KC" get ingress -A \ -o jsonpath='{.items[0].status.loadBalancer.ingress[0].hostname}' 2>/dev/null || true) - echo "${ALB:-{{.CLOUDFRONT_DOMAIN}}}" + CF=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]') + [ -z "$CF" ] && CF=$(aws cloudfront list-distributions --region {{.AWS_REGION}} \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName | [0]" \ + --output text 2>/dev/null | tr -d '[:space:]') + [ "$CF" = "None" ] && CF="" + echo "${ALB:-$CF}" else echo "{{.DOMAIN}}" fi diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index f5ba5dc42..3786ff2fc 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -28,8 +28,12 @@ vars: sh: yq '.gitUsername // "user1"' {{.CONFIG_FILE}} EXPOSURE_MODE: sh: 'if [ -z "$(yq ''.domain // ""'' {{.CONFIG_FILE}})" ]; then echo "cloudfront"; else echo "domain"; fi' + # NOTE: this taskfile-level var is evaluated ONCE at load time (before the platform + # CloudFront is created mid-install), so it is only used for informational messaging + # in pre-install. Tasks that need the LIVE value (idc:configure, setup-env) resolve it + # in their own task-level vars from private/cloudfront-domain (+ AWS fallback). CLOUDFRONT_DOMAIN: - sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null || true + sh: cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]' || true C_STEP: sh: printf '\033[1;33m' C_OK: @@ -97,15 +101,15 @@ tasks: printf ' 2. SSM SetupIDE runs create-config.sh which generates config.local.yaml\n' printf ' (auto-detects region, account, IDC; sets domain: "" for cloudfront mode\n' printf ' and cloudfront.gitlabDomain from private/gitlab-cloudfront-domain).\n' - printf ' NOTE: cloudfront.cloudfrontDomain (the PLATFORM CloudFront) is NOT a\n' - printf ' prerequisite — it is created by cloudfront:hub-distribution during\n' - printf ' task install and written back to config.local.yaml.\n' + printf ' NOTE: the PLATFORM CloudFront domain is NOT a prerequisite and is NOT\n' + printf ' stored in config.local.yaml — it is created by cloudfront:hub-distribution\n' + printf ' during task install and recorded in private/cloudfront-domain.\n' printf ' 3. task install + task workshop:post-install run automatically\n' printf ' For self-paced: just run create-config.sh; task install creates the platform CloudFront.\n' if [ -z "{{.CLOUDFRONT_DOMAIN}}" ]; then - printf '{{.C_INFO}} ℹ cloudfront.cloudfrontDomain not set yet — it will be created during task install.{{.C_RESET}}\n' + printf '{{.C_INFO}} ℹ platform CloudFront domain not created yet — it will be created during task install.{{.C_RESET}}\n' else - printf '{{.C_OK}}✓ cloudfront.cloudfrontDomain: {{.CLOUDFRONT_DOMAIN}}{{.C_RESET}}\n' + printf '{{.C_OK}}✓ platform CloudFront domain: {{.CLOUDFRONT_DOMAIN}}{{.C_RESET}}\n' fi # ============================================================ @@ -141,9 +145,15 @@ tasks: ARGOCD_URL: sh: aws eks describe-capability --cluster-name {{.HUB_CLUSTER_NAME}} --capability-name argocd --query 'capability.configuration.argoCd.serverUrl' --output text --region {{.AWS_REGION}} 2>/dev/null || echo "" CF_DOMAIN: - sh: yq '.cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + sh: | + D=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]') + [ -z "$D" ] && D=$(aws cloudfront list-distributions --region {{.AWS_REGION}} \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName | [0]" \ + --output text 2>/dev/null | tr -d '[:space:]') + [ "$D" = "None" ] && D="" + echo "$D" GITLAB_CF_DOMAIN_VAR: - sh: yq '.cloudfront.gitlabDomain // .cloudfront.cloudfrontDomain // ""' {{.CONFIG_FILE}} + sh: yq '.cloudfront.gitlabDomain // ""' {{.CONFIG_FILE}} | grep -v '^$' || cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain 2>/dev/null || true USER_PASSWORD: sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text 2>/dev/null | jq -r '.user_password // ""' KC_ADMIN_PASSWORD: @@ -203,10 +213,23 @@ tasks: sh: aws secretsmanager get-secret-value --secret-id {{.HUB_CLUSTER_NAME}}/keycloak --region {{.AWS_REGION}} --query 'SecretString' --output text | jq -r '.keycloak_admin_password' KEYCLOAK_DNS: sh: | - if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ] && [ -n "{{.CLOUDFRONT_DOMAIN}}" ]; then - echo "{{.CLOUDFRONT_DOMAIN}}" + if [ "{{.EXPOSURE_MODE}}" = "cloudfront" ]; then + # Resolve the platform CloudFront domain at TASK-RUN time from the runtime + # artifact (created mid-install by cloudfront:hub-distribution), with an AWS + # lookup fallback. Do NOT use the taskfile-level CLOUDFRONT_DOMAIN var: + # global vars are evaluated once at load — before the distribution exists — so + # it is always empty here (this was the cause of the empty --keycloak-dns). + # Do NOT read config.local.yaml (the domain is not stored there). + D=$(cat {{.ROOT_DIR}}/private/cloudfront-domain 2>/dev/null | tr -d '[:space:]') + if [ -z "$D" ]; then + D=$(aws cloudfront list-distributions --region {{.AWS_REGION}} \ + --query "DistributionList.Items[?Comment=='{{.HUB_CLUSTER_NAME}}-platform'].DomainName | [0]" \ + --output text 2>/dev/null | tr -d '[:space:]') + fi + [ "$D" = "None" ] && D="" + echo "$D" else - echo "$(yq '.domain' {{.CONFIG_FILE}})" + yq '.domain' {{.CONFIG_FILE}} fi cmds: - printf '{{.C_STEP}}▸ Refreshing IDC integration credentials via Lambda...{{.C_RESET}}\n' From 809f468ac40f268e5f7545bfa333eede91b7e8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 17:17:35 +0200 Subject: [PATCH 180/231] fix(ingress): platform IngressClassParams scheme=internal in cloudfront mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The platform ALB is pre-created as scheme=internal (CloudFront VPC Origin backend), but the platform IngressClassParams hardcoded scheme=internet-facing. ALB scheme is immutable, so when the AWS LBC adopts the ALB by loadBalancerName it refuses to reconcile on the scheme conflict — no listener rules are attached, every platform ingress (keycloak, backstage, grafana, ...) gets no address, and all requests hit the ALB default 404 action. This blocked idc:configure (Keycloak SAML descriptor 404) and made all platform URLs return 404, for both kind-crossplane and kind-kro-ack. - ingress-class-alb chart: scheme is now templated ({{ .Values.scheme | default internet-facing }}) - core.yaml appset: pass scheme=internal when exposure_mode=cloudfront, internet-facing otherwise --- .../charts/ingress-class-alb/templates/ingressclass.yaml | 2 +- gitops/addons/registry/core.yaml | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/gitops/addons/charts/ingress-class-alb/templates/ingressclass.yaml b/gitops/addons/charts/ingress-class-alb/templates/ingressclass.yaml index cc10c0ff1..63ee86219 100644 --- a/gitops/addons/charts/ingress-class-alb/templates/ingressclass.yaml +++ b/gitops/addons/charts/ingress-class-alb/templates/ingressclass.yaml @@ -9,7 +9,7 @@ metadata: spec: group: name: platform - scheme: internet-facing + scheme: {{ .Values.scheme | default "internet-facing" }} {{- if .Values.loadBalancerName }} loadBalancerName: {{ .Values.loadBalancerName | quote }} {{- end }} diff --git a/gitops/addons/registry/core.yaml b/gitops/addons/registry/core.yaml index 405eeab95..9c4940c86 100644 --- a/gitops/addons/registry/core.yaml +++ b/gitops/addons/registry/core.yaml @@ -200,6 +200,11 @@ ingress-class-alb: valuesObject: controllerMode: '{{default "auto" .metadata.annotations.alb_controller_mode}}' loadBalancerName: '{{if eq (default "domain" .metadata.annotations.exposure_mode) "cloudfront"}}{{.metadata.annotations.aws_cluster_name}}-platform{{end}}' + # In cloudfront mode the platform ALB is created internal (CloudFront VPC Origin + # backend). The IngressClassParams scheme MUST match the adopted ALB's scheme — + # ALB scheme is immutable, so a mismatch makes the LBC refuse to attach listener + # rules, leaving every ingress with no address and the ALB's default 404 action. + scheme: '{{if eq (default "domain" .metadata.annotations.exposure_mode) "cloudfront"}}internal{{else}}internet-facing{{end}}' aws-load-balancer-controller: namespace: kube-system From 0a71d67e78a357edfc76598311bd36aac6da64af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 18:24:23 +0200 Subject: [PATCH 181/231] fix(kind-kro-ack): add LBC + keycloak-config pod identity associations to EKS RGD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The kro-ack EksCluster RGD created PodIdentityAssociations for external-secrets, external-dns, adot, policy-reporter, cni-metrics-helper and cloudwatch-agent, but NOT for two service accounts that make AWS calls: - kube-system/aws-load-balancer-controller-sa: without creds the LBC can't build the ingress model (IMDS fallback fails: 'no EC2 IMDS role found'), attaches no listener rules to the platform ALB, so every platform ingress (keycloak, backstage, grafana, argo-workflows) gets no address and the ALB serves its default 404 — which blocked idc:configure (Keycloak SAML descriptor 404). - keycloak/keycloak-config: the config job runs 'aws secretsmanager create/ put-secret-value' to publish keycloak-clients; without creds ESO never syncs it and Backstage/Argo SSO break. Adds an ACK Policy + Role (pods.eks.amazonaws.com trust) + PodIdentityAssociation for each, modeled on the existing external-dns chain. LBC policy reuses the canonical AWS LBC IAM policy; keycloak-config gets least-privilege Secrets Manager permissions. (Crossplane provider already covers these via crossplane-pod-identity.) --- .../resource-groups/manifests/eks/rg-eks.yaml | 328 ++++++++++++++++++ 1 file changed, 328 insertions(+) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index b14d4e5ab..e259cdb0e 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -1067,6 +1067,334 @@ spec: serviceAccount: external-dns + ########################################################### + # AWS Load Balancer Controller Pod Identity + # The LBC service account (kube-system/aws-load-balancer-controller-sa) + # needs ELB/EC2/ACM/WAF/Shield permissions. Without this association the + # controller runs credential-less (IMDS fallback fails), builds no ingress + # model, and every platform ingress (keycloak, backstage, grafana, ...) gets + # no listener rule -> the platform ALB serves its default 404 for everything. + ########################################################### + - id: lbcPolicy + template: + apiVersion: iam.services.k8s.aws/v1alpha1 + kind: Policy + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-lbc" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + name: "${schema.spec.name}-lbc" + policyDocument: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["iam:CreateServiceLinkedRole"], + "Resource": "*", + "Condition": {"StringEquals": {"iam:AWSServiceName": "elasticloadbalancing.amazonaws.com"}} + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes","ec2:DescribeAddresses","ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways","ec2:DescribeVpcs","ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets","ec2:DescribeSecurityGroups","ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces","ec2:DescribeTags","ec2:DescribeCoipPools", + "ec2:GetCoipPoolUsage","ec2:DescribeInstanceTypes","ec2:GetSecurityGroupsForVpc", + "elasticloadbalancing:DescribeLoadBalancers","elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners","elasticloadbalancing:DescribeListenerAttributes", + "elasticloadbalancing:DescribeListenerCertificates","elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules","elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes","elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags","elasticloadbalancing:DescribeTrustStores" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "cognito-idp:DescribeUserPoolClient","acm:ListCertificates","acm:DescribeCertificate", + "iam:ListServerCertificates","iam:GetServerCertificate","waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource","waf-regional:AssociateWebACL","waf-regional:DisassociateWebACL", + "wafv2:GetWebACL","wafv2:GetWebACLForResource","wafv2:AssociateWebACL","wafv2:DisassociateWebACL", + "shield:GetSubscriptionState","shield:DescribeProtection","shield:CreateProtection","shield:DeleteProtection" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": ["ec2:AuthorizeSecurityGroupIngress","ec2:RevokeSecurityGroupIngress"], + "Resource": "*" + }, + {"Effect": "Allow", "Action": ["ec2:CreateSecurityGroup"], "Resource": "*"}, + { + "Effect": "Allow", + "Action": ["ec2:CreateTags"], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": { + "StringEquals": {"ec2:CreateAction": "CreateSecurityGroup"}, + "Null": {"aws:RequestTag/elbv2.k8s.aws/cluster": "false"} + } + }, + { + "Effect": "Allow", + "Action": ["ec2:CreateTags","ec2:DeleteTags"], + "Resource": "arn:aws:ec2:*:*:security-group/*", + "Condition": {"Null": {"aws:RequestTag/elbv2.k8s.aws/cluster": "true","aws:ResourceTag/elbv2.k8s.aws/cluster": "false"}} + }, + { + "Effect": "Allow", + "Action": ["ec2:AuthorizeSecurityGroupIngress","ec2:RevokeSecurityGroupIngress","ec2:DeleteSecurityGroup"], + "Resource": "*", + "Condition": {"Null": {"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"}} + }, + { + "Effect": "Allow", + "Action": ["elasticloadbalancing:CreateLoadBalancer","elasticloadbalancing:CreateTargetGroup"], + "Resource": "*", + "Condition": {"Null": {"aws:RequestTag/elbv2.k8s.aws/cluster": "false"}} + }, + { + "Effect": "Allow", + "Action": ["elasticloadbalancing:CreateListener","elasticloadbalancing:DeleteListener","elasticloadbalancing:CreateRule","elasticloadbalancing:DeleteRule"], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": ["elasticloadbalancing:AddTags","elasticloadbalancing:RemoveTags"], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": {"Null": {"aws:RequestTag/elbv2.k8s.aws/cluster": "true","aws:ResourceTag/elbv2.k8s.aws/cluster": "false"}} + }, + { + "Effect": "Allow", + "Action": ["elasticloadbalancing:AddTags","elasticloadbalancing:RemoveTags"], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:aws:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:ModifyLoadBalancerAttributes","elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups","elasticloadbalancing:SetSubnets","elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup","elasticloadbalancing:ModifyTargetGroupAttributes","elasticloadbalancing:DeleteTargetGroup" + ], + "Resource": "*", + "Condition": {"Null": {"aws:ResourceTag/elbv2.k8s.aws/cluster": "false"}} + }, + { + "Effect": "Allow", + "Action": ["elasticloadbalancing:AddTags"], + "Resource": [ + "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:aws:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "StringEquals": {"elasticloadbalancing:CreateAction": ["CreateTargetGroup","CreateLoadBalancer"]}, + "Null": {"aws:RequestTag/elbv2.k8s.aws/cluster": "false"} + } + }, + { + "Effect": "Allow", + "Action": ["elasticloadbalancing:RegisterTargets","elasticloadbalancing:DeregisterTargets"], + "Resource": "arn:aws:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:SetWebAcl","elasticloadbalancing:ModifyListener","elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates","elasticloadbalancing:ModifyRule","elasticloadbalancing:SetRulePriorities" + ], + "Resource": "*" + } + ] + } + + - id: lbcRole + template: + apiVersion: iam.services.k8s.aws/v1alpha1 + kind: Role + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-lbc-role" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + policy: "${lbcPolicy.status.ackResourceMetadata.arn}" + services.k8s.aws/region: ${schema.spec.region} + spec: + name: "${schema.spec.name}-lbc-role" + policyRefs: + - from: + name: "${schema.spec.name}-lbc" + assumeRolePolicyDocument: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "pods.eks.amazonaws.com"}, + "Action": ["sts:AssumeRole","sts:TagSession"] + } + ] + } + + - id: lbcPodIdentityAssociation + readyWhen: + - ${lbcPodIdentityAssociation.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: eks.services.k8s.aws/v1alpha1 + kind: PodIdentityAssociation + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-lbc" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + clusterName: "${ekscluster.spec.name}" + namespace: kube-system + roleARN: "${lbcRole.status.ackResourceMetadata.arn}" + serviceAccount: aws-load-balancer-controller-sa + + + ########################################################### + # Keycloak Config Pod Identity + # The keycloak-config job (keycloak/keycloak-config SA) publishes the + # keycloak-clients secret to Secrets Manager (aws secretsmanager + # create/put-secret-value). Without credentials it cannot write the secret, + # so ESO never syncs keycloak-clients and Backstage / Argo-Workflows SSO break. + ########################################################### + - id: keycloakConfigPolicy + template: + apiVersion: iam.services.k8s.aws/v1alpha1 + kind: Policy + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-keycloak-config" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + name: "${schema.spec.name}-keycloak-config" + policyDocument: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "secretsmanager:CreateSecret", + "secretsmanager:PutSecretValue", + "secretsmanager:GetSecretValue", + "secretsmanager:DescribeSecret", + "secretsmanager:TagResource" + ], + "Resource": "*" + } + ] + } + + - id: keycloakConfigRole + template: + apiVersion: iam.services.k8s.aws/v1alpha1 + kind: Role + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-keycloak-config-role" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + policy: "${keycloakConfigPolicy.status.ackResourceMetadata.arn}" + services.k8s.aws/region: ${schema.spec.region} + spec: + name: "${schema.spec.name}-keycloak-config-role" + policyRefs: + - from: + name: "${schema.spec.name}-keycloak-config" + assumeRolePolicyDocument: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "pods.eks.amazonaws.com"}, + "Action": ["sts:AssumeRole","sts:TagSession"] + } + ] + } + + - id: keycloakConfigPodIdentityAssociation + readyWhen: + - ${keycloakConfigPodIdentityAssociation.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: eks.services.k8s.aws/v1alpha1 + kind: PodIdentityAssociation + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-keycloak-config" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + clusterName: "${ekscluster.spec.name}" + namespace: keycloak + roleARN: "${keycloakConfigRole.status.ackResourceMetadata.arn}" + serviceAccount: keycloak-config + + ########################################################### # ADOT Collector Pod Identity ########################################################### From ef33bc28d48880eaa7c30fd2e7bc49e9f6b218cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 19:11:20 +0200 Subject: [PATCH 182/231] fix(kro-ack): gate crossplane pod_identities to provider!=kro-ack; stamp hub provider label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On the kro-ack provider, pod identities are created by the KRO EksCluster RGD (ACK). Crossplane's own providers are not bootstrapped on the kro-ack hub, so the crossplane-pod-identity (pod_identities) app there produces permanently-Degraded CRs (SYNCED=False) and is a latent 409 conflict with the ACK-created associations. - core.yaml: pod_identities selector now excludes clusters labelled provider=kro-ack (NotIn also matches clusters with no provider label, so crossplane/legacy clusters — including crossplane's KRO-provisioned spoke-prod, which needs the crossplane provider bootstrap identities — are unaffected). - kind-kro-ack hub:seed writes addons='{"provider":"kro-ack"}' into peeks-hub/config; the fleet-secret ExternalSecret renders the addons key as labels, so the kro-ack hub cluster secret gets provider=kro-ack and pod_identities is not generated for it. Note: kro-ack spoke secrets still need provider=kro-ack wired through spokes:enable-kro -> EksclusterWithVpc/EksCluster schema -> argocdSecret label (follow-up). --- cluster-providers/kind-kro-ack/Taskfile.yaml | 2 +- gitops/addons/registry/core.yaml | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index eb95762d7..ac9745823 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -713,7 +713,7 @@ tasks: --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}","exposure_mode":"{{if .CLOUDFRONT_DOMAIN}}cloudfront{{else}}domain{{end}}","admin_role_name":"{{.ADMIN_ROLE_NAME}}","gitlab_domain_name":"$GITLAB_CF_DOMAIN","git_token":"{{.USER_PASSWORD}}","aws_argocd_url":""}' \ --arg config '{"tlsClientConfig":{"insecure":false}}' \ --arg server '{{.CLUSTER_ARN}}' \ - --arg addons '' \ + --arg addons '{"provider":"kro-ack"}' \ --arg vpc '{{.VPC_ID}}' \ '{metadata: $metadata, config: $config, server: $server, addons: $addons, vpc: $vpc}') aws secretsmanager restore-secret --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} 2>/dev/null || true diff --git a/gitops/addons/registry/core.yaml b/gitops/addons/registry/core.yaml index 9c4940c86..97d5d5fb6 100644 --- a/gitops/addons/registry/core.yaml +++ b/gitops/addons/registry/core.yaml @@ -36,6 +36,15 @@ pod_identities: matchExpressions: - key: environment operator: Exists + # crossplane-pod-identity is the CROSSPLANE provider's mechanism. On the + # kro-ack provider, pod identities are created by the KRO EksCluster RGD (ACK), + # and Crossplane's own providers are not bootstrapped there — so these CRs would + # be permanently Degraded (SYNCED=False) and are a latent 409 conflict with the + # ACK-created associations. Exclude kro-ack clusters. NotIn also matches clusters + # with NO provider label (crossplane/legacy), so only provider=kro-ack is excluded. + - key: provider + operator: NotIn + values: ["kro-ack"] valuesObject: aws: region: '{{.metadata.annotations.aws_region}}' From 22e8e2f691d6eede472ada3b2049b30ed6ca2a66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 19:14:41 +0200 Subject: [PATCH 183/231] fix(kro-ack): stamp provider=kro-ack on spoke cluster secrets to gate pod_identities Threads a provider field from spokes:enable-kro through the KRO claim to the spoke argocd cluster secret label, so kro-ack spokes (like the hub) are excluded from the crossplane pod_identities app (provider NotIn [kro-ack]). - kind-kro-ack spokes:enable-kro writes provider: kro-ack into the kro-clusters values - kro-clusters chart maps $cluster.provider -> EksclusterWithVpc.spec.provider - rg-eks-vpc EksclusterWithVpc schema + passthrough to nested EksCluster - rg-eks EksCluster schema + argocdSecret label provider: ${schema.spec.provider} Default is empty, so crossplane-provisioned clusters (and crossplane's KRO-provisioned spoke-prod, whose spokes:enable-kro does not set it) keep provider unset -> NotIn matches -> pod_identities stays enabled for them (preserves crossplane provider bootstrap identities). --- cluster-providers/kind-kro-ack/Taskfile.yaml | 1 + gitops/abstractions/kro/kro-clusters/templates/clusters.yaml | 1 + .../charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml | 2 ++ .../charts/kro/resource-groups/manifests/eks/rg-eks.yaml | 5 +++++ 4 files changed, 9 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index ac9745823..c2ca96c13 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1991,6 +1991,7 @@ tasks: clusters: {{.CLUSTER}}: clusterName: {{.CLUSTER}} + provider: kro-ack tenant: workshop environment: ${ENV} region: {{.AWS_REGION}} diff --git a/gitops/abstractions/kro/kro-clusters/templates/clusters.yaml b/gitops/abstractions/kro/kro-clusters/templates/clusters.yaml index 73acbc3eb..bd09ac921 100644 --- a/gitops/abstractions/kro/kro-clusters/templates/clusters.yaml +++ b/gitops/abstractions/kro/kro-clusters/templates/clusters.yaml @@ -13,6 +13,7 @@ spec: name: {{ $name }} tenant: {{ $cluster.tenant | default "tenant1" | quote }} environment: {{ $cluster.environment | default "staging" | quote }} + provider: {{ $cluster.provider | default "" | quote }} region: {{ $cluster.region | default "us-west-2" | quote }} k8sVersion: {{ $cluster.k8sVersion | default "1.34" | quote }} accountId: {{ $cluster.accountId | quote }} diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml index 50744e62f..10954baec 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks-vpc.yaml @@ -13,6 +13,7 @@ spec: name: string tenant: string | default="auto1" environment: string | default="staging" + provider: string | default="" region: string | default="us-west-2" k8sVersion: string | default="1.34" accountId: string @@ -168,6 +169,7 @@ spec: name: ${schema.spec.name} tenant: ${schema.spec.tenant} environment: ${schema.spec.environment} + provider: ${schema.spec.provider} region: ${schema.spec.region} accountId: ${schema.spec.accountId} managementAccountId: ${schema.spec.managementAccountId} diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index e259cdb0e..8a36014ce 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -15,6 +15,10 @@ spec: name: string tenant: string environment: string + # Identity backend for this cluster's addons: "kro-ack" (pod identities created + # by this ACK RGD) or "crossplane"/"" (crossplane-pod-identity app). Stamped onto + # the argocd cluster secret label so the pod_identities app excludes kro-ack. + provider: string | default="" region: string accountId: string managementAccountId: string @@ -449,6 +453,7 @@ spec: fleet_member: spoke tenant: "${schema.spec.tenant}" environment: "${schema.spec.environment}" + provider: "${schema.spec.provider}" aws_cluster_name: "${schema.spec.name}" workloads: "${schema.spec.workloads}" #using : useSelector: true for centralized mode From 6f8fb2c207e7d3923ba7db46293d8eb691997048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 19:31:41 +0200 Subject: [PATCH 184/231] feat(kro-ack): bootstrap Crossplane provider pod identities via ACK RGD (temporary bridge) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until devlake (and other data resources) are migrated off Crossplane, kro-ack still needs Crossplane working on the hub. But on kro-ack the Crossplane AWS providers have no credentials — provider-aws-iam/eks pod identities are themselves crossplane CRs (chicken-and-egg) — so all downstream crossplane roles/PIAs (amp/rds/grafana/devlake) stay SYNCED=False and devlake RDS / AMP / Grafana break. Seed the two ROOT providers' pod identities via ACK in the EksCluster RGD (which provisions both hub and spokes on kro-ack). Once provider-aws-iam/eks have creds, Crossplane reconciles the rest itself. - rg-eks.yaml: add crossplaneProviderRole (AdministratorAccess, pods.eks trust) + PodIdentityAssociations for crossplane-system/provider-aws-iam and provider-aws-eks. Gated includeWhen provider=='kro-ack' && enable_crossplane_aws=='true' so it does NOT run on crossplane-provisioned clusters (incl. crossplane's KRO-provisioned spoke-prod, where crossplane-pod-identity already creates these -> avoids 409). - platform-cluster-kro hub claim: set provider=kro-ack so the gate fires on the hub. TEMPORARY: remove these resources once the devlake->kro-ack migration lands. --- .../platform-cluster-kro/templates/claim.yaml | 1 + .../resource-groups/manifests/eks/rg-eks.yaml | 105 ++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/gitops/abstractions/resource-groups-kro/platform-cluster-kro/templates/claim.yaml b/gitops/abstractions/resource-groups-kro/platform-cluster-kro/templates/claim.yaml index d39342eb0..0640bde52 100644 --- a/gitops/abstractions/resource-groups-kro/platform-cluster-kro/templates/claim.yaml +++ b/gitops/abstractions/resource-groups-kro/platform-cluster-kro/templates/claim.yaml @@ -13,6 +13,7 @@ spec: name: {{ .Values.clusterName }} tenant: {{ .Values.tenant | default "workshop" }} environment: {{ .Values.environment | default "dev" }} + provider: kro-ack region: {{ .Values.region | default $.Values.global.region | default "us-west-2" }} k8sVersion: {{ .Values.kubernetesVersion | default "1.32" | quote }} accountId: {{ .Values.accountId | default $.Values.global.accountId | quote }} diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index 8a36014ce..6cf7e1275 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -1400,6 +1400,111 @@ spec: serviceAccount: keycloak-config + ########################################################### + # Crossplane provider bootstrap Pod Identity (TEMPORARY BRIDGE) + # On kro-ack, Crossplane's provider-aws-iam/eks have no credentials — their own + # pod identities are crossplane CRs (chicken-and-egg) — so every downstream + # crossplane role/PIA (amp/rds/grafana/devlake) stays SYNCED=False and devlake + # RDS / AMP / Grafana break. Seed the two ROOT providers' pod identities via ACK + # here to break the cycle; Crossplane then reconciles the rest itself. + # + # Gated on provider=="kro-ack" AND enable_crossplane_aws=="true": + # - provider=="kro-ack" ensures we do NOT run on crossplane-provisioned clusters + # (incl. crossplane's KRO-provisioned spoke-prod), where crossplane-pod-identity + # already creates these same associations — otherwise AWS 409 ResourceInUseException. + # - enable_crossplane_aws limits it to clusters that actually run Crossplane. + # REMOVE once devlake (and other data resources) are migrated off Crossplane to ACK/KRO. + ########################################################### + - id: crossplaneProviderRole + includeWhen: + - ${schema.spec.provider == "kro-ack" && schema.spec.addons.enable_crossplane_aws == "true"} + template: + apiVersion: iam.services.k8s.aws/v1alpha1 + kind: Role + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-crossplane-provider" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + name: "${schema.spec.name}-crossplane-provider" + policies: + - arn:aws:iam::aws:policy/AdministratorAccess + assumeRolePolicyDocument: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"Service": "pods.eks.amazonaws.com"}, + "Action": ["sts:AssumeRole","sts:TagSession"] + } + ] + } + + - id: crossplaneIamProviderPodIdentity + includeWhen: + - ${schema.spec.provider == "kro-ack" && schema.spec.addons.enable_crossplane_aws == "true"} + readyWhen: + - ${crossplaneIamProviderPodIdentity.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: eks.services.k8s.aws/v1alpha1 + kind: PodIdentityAssociation + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-crossplane-iam-provider" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + clusterName: "${ekscluster.spec.name}" + namespace: crossplane-system + roleARN: "${crossplaneProviderRole.status.ackResourceMetadata.arn}" + serviceAccount: provider-aws-iam + + - id: crossplaneEksProviderPodIdentity + includeWhen: + - ${schema.spec.provider == "kro-ack" && schema.spec.addons.enable_crossplane_aws == "true"} + readyWhen: + - ${crossplaneEksProviderPodIdentity.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: eks.services.k8s.aws/v1alpha1 + kind: PodIdentityAssociation + metadata: + namespace: "${schema.spec.name}" + name: "${schema.spec.name}-crossplane-eks-provider" + ownerReferences: + - apiVersion: kro.run/v1alpha1 + kind: ${schema.kind} + name: ${schema.metadata.name} + uid: ${schema.metadata.uid} + blockOwnerDeletion: true + controller: false + annotations: + argocd.argoproj.io/tracking-id: ${schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue("")} + services.k8s.aws/region: ${schema.spec.region} + spec: + clusterName: "${ekscluster.spec.name}" + namespace: crossplane-system + roleARN: "${crossplaneProviderRole.status.ackResourceMetadata.arn}" + serviceAccount: provider-aws-eks + + ########################################################### # ADOT Collector Pod Identity ########################################################### From fe17f1700e5739fbf1764801c2f79729535b7001 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 20:53:59 +0200 Subject: [PATCH 185/231] docs(steering): troubleshooting for CloudFront VPC-origin -> stale/recreated ALB (curl 000) Document the failure where platform URLs time out with curl 000 in cloudfront mode because the CloudFront VPC origin is bound to a deleted ALB ARN (the LBC delete-recreated the ALB, classically due to an IngressClassParams scheme mismatch). Includes the direct-ALB isolation test, the VPC-origin-ARN vs current-ALB-ARN check, the CloudTrail DeleteLoadBalancer lookup, and the fix (scheme=internal to stop churn + recreate/swap the VPC origin to re-point CloudFront). --- .kiro/steering/troubleshooting.md | 56 +++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/.kiro/steering/troubleshooting.md b/.kiro/steering/troubleshooting.md index 659c40256..89acc7b36 100644 --- a/.kiro/steering/troubleshooting.md +++ b/.kiro/steering/troubleshooting.md @@ -266,6 +266,62 @@ annotations: For Kargo specifically this lives in `gitops/overlays/environments/control-plane/kargo/values.yaml` under `api.ingress.annotations`. +### CloudFront platform URLs hang (curl 000): VPC origin points to a stale/recreated ALB + +**Symptom**: In cloudfront exposure mode, every platform URL (`https://$CF/keycloak/...`, +`/backstage`, `/grafana`, `/argo-workflows`, ...) times out with curl **`000`** — not `404`, +not `5xx`. `idc:configure` loops forever on "Keycloak not ready" (the SAML descriptor never +returns 200). A **direct in-cluster** curl to the internal ALB DNS returns `200/302`, proving +the ALB, targets and LBC listener rules are healthy — only the **CloudFront → ALB** hop fails. + +**Root cause**: The platform ALB (`-platform`) was **deleted and recreated** (new +ARN/DNS), but the CloudFront **VPC Origin is still bound to the OLD (deleted) ALB ARN**. +`hub-distribution` only acts when no `-platform` distribution exists yet, so on a re-run it +**skips** and never re-points the VPC origin → CloudFront keeps sending traffic to a dead ALB → +the connection hangs → `000`. + +Who recreates the ALB: the **AWS Load Balancer Controller**. When the pre-created ALB's +*immutable* attributes don't match the `platform` IngressClassParams, the LBC can't modify them +in place, so it **deletes and recreates** the ALB. The classic trigger is a **scheme mismatch**: +`create-alb` builds the ALB `internal` (required for a CloudFront VPC-Origin backend), but if +`IngressClassParams.spec.scheme` is `internet-facing`, scheme is immutable → LBC delete+recreate +loop. Every recreation orphans the VPC origin. (CloudTrail shows `DeleteLoadBalancer` by +`-LBCPodIdentityRole` with userAgent `elbv2.k8s.aws/...`.) + +**Diagnosis**: +```bash +HUB=peeks-hub # adjust to your resource prefix + +# 1. Direct in-cluster hit to the internal ALB (bypasses CloudFront). 200/302 => ALB is fine, +# so the break is the CloudFront->ALB hop (stale VPC origin), NOT the ALB/targets/rules. +ALB_DNS=$(aws elbv2 describe-load-balancers --names "$HUB-platform" --query 'LoadBalancers[0].DNSName' --output text) +kubectl --context "$HUB" -n keycloak run curltest --rm -i --restart=Never --image=curlimages/curl --command -- \ + curl -s -o /dev/null -w '%{http_code}\n' --max-time 10 "http://$ALB_DNS/keycloak/realms/master" + +# 2. THE check: does the VPC origin's ARN match the CURRENT ALB ARN? (mismatch = stale = the bug) +ALB_ARN=$(aws elbv2 describe-load-balancers --names "$HUB-platform" --query 'LoadBalancers[0].LoadBalancerArn' --output text) +DIST=$(aws cloudfront list-distributions --query "DistributionList.Items[?Comment=='$HUB-platform'].Id" --output text) +VO=$(aws cloudfront get-distribution --id "$DIST" --query 'Distribution.DistributionConfig.Origins.Items[0].VpcOriginConfig.VpcOriginId' --output text) +echo "VPC origin ARN : $(aws cloudfront get-vpc-origin --id "$VO" --query 'VpcOrigin.VpcOriginEndpointConfig.Arn' --output text)" +echo "current ALB ARN: $ALB_ARN" # if these differ, CloudFront points at a dead ALB + +# 3. Confirm the LBC is the one deleting/recreating the ALB (and when) +aws cloudtrail lookup-events --lookup-attributes AttributeKey=EventName,AttributeValue=DeleteLoadBalancer \ + --max-results 5 --query 'Events[].{Time:EventTime,User:Username}' --output table +``` + +**Fix**: +- **Stop the churn at the source**: ensure the `platform` IngressClassParams `scheme` is + `internal` in cloudfront mode so the LBC **adopts** the pre-created internal ALB instead of + delete-recreating it (`gitops/addons/registry/core.yaml` `ingress-class-alb.valuesObject.scheme` + + `gitops/addons/charts/ingress-class-alb/templates/ingressclass.yaml`). After this, CloudTrail + should show **no recurring** `DeleteLoadBalancer` events and the ALB ARN stays stable. +- **If a stale VPC origin already exists**, re-point CloudFront at the current ALB: create a NEW + VPC origin for the current ALB ARN → update the distribution's origin to the new `VpcOriginId` + and current ALB DNS → wait for `Deployed` → delete the old VPC origin. Note a VPC origin + **cannot be updated in place while attached to a distribution** (`CannotUpdateEntityWhileInUse`), + so you must create-new / swap / delete-old rather than edit its ARN. + ## ACK (AWS Controllers for Kubernetes) Issues ### ACK "scheduled for deletion" loop From ac26812fb170e366cae9605370df71fbad750fca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Thu, 2 Jul 2026 21:01:47 +0200 Subject: [PATCH 186/231] fix(cloudfront): create-alb selects+tags private subnets so LBC adopts ALB in place (no churn) Root-cause hardening for the stale-VPC-origin issue: the LBC recreated the platform ALB when its subnet set didn't match what create-alb built, orphaning the CloudFront VPC origin. - create-alb now selects PRIVATE subnets by internal-elb tag then by MapPublicIpOnLaunch==false (instead of a fragile *private* name tag), and tags the chosen subnets kubernetes.io/role/internal-elb=1 so the AWS LBC discovers the SAME set and adopts the ALB in place instead of calling SetSubnets / recreating it. - steering/troubleshooting.md: document the implemented prevention (scheme=internal + subnet tagging) and the proposed-but-NOT-implemented cloudfront:sync-vpc-origin reconcile as future hardening (design + wiring), to add only if ALB churn recurs. --- .kiro/steering/troubleshooting.md | 24 +++++++++++++++++++ .../common/Taskfile.cloudfront.yaml | 21 ++++++++++++---- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/.kiro/steering/troubleshooting.md b/.kiro/steering/troubleshooting.md index 89acc7b36..b8d697364 100644 --- a/.kiro/steering/troubleshooting.md +++ b/.kiro/steering/troubleshooting.md @@ -322,6 +322,30 @@ aws cloudtrail lookup-events --lookup-attributes AttributeKey=EventName,Attribut **cannot be updated in place while attached to a distribution** (`CannotUpdateEntityWhileInUse`), so you must create-new / swap / delete-old rather than edit its ARN. +**Prevention (root cause) — implemented**: +- `IngressClassParams.spec.scheme = internal` in cloudfront mode so the LBC adopts the internal + ALB in place instead of delete-recreating it (scheme is immutable). +- `create-alb` selects PRIVATE subnets (by `kubernetes.io/role/internal-elb` tag, then by + `MapPublicIpOnLaunch==false`) and **tags them `kubernetes.io/role/internal-elb=1`**, so the LBC + discovers the same subnet set and does not `SetSubnets`/churn the ALB. See + `cluster-providers/common/Taskfile.cloudfront.yaml` (`create-alb`). + +**Future hardening (NOT yet implemented) — `cloudfront:sync-vpc-origin` reconcile**: +If an ALB is ever recreated despite the above (re-runs, other immutable-attr drift), a reconcile +task would self-heal the stale origin. Proposed design (idempotent; no-op when the ARN already +matches, so cheap on healthy installs): +1. Guard on `EXPOSURE_MODE == cloudfront`; resolve current `-platform` ALB ARN + DNS; resolve + the distribution (`Comment == -platform`) and its origin `VpcOriginId` → that VPC origin's ARN. +2. If `VPC-origin ARN == current ALB ARN` → **no-op** (done). +3. Else drift → create-new / swap / delete-old (a VPC origin can't be edited in place while + attached): `create-vpc-origin` for the current ALB → poll `Deployed` → `update-distribution` + origin to the new `VpcOriginId` + current ALB DNS → poll `Deployed` → `delete-vpc-origin` (old) + → re-authorize `CloudFront-VPCOrigins-Service-SG → :80`. +Wire into `install:phase1-cloudfront` (both providers) after `sync-domain`. Cost is one or two +CloudFront deploys (~5–15 min each) ONLY on drift; a no-op otherwise. Deliberately left +unimplemented for now — the prevention above should keep the ALB stable; add this only if churn +recurs. + ## ACK (AWS Controllers for Kubernetes) Issues ### ACK "scheduled for deletion" loop diff --git a/cluster-providers/common/Taskfile.cloudfront.yaml b/cluster-providers/common/Taskfile.cloudfront.yaml index 815b8dafb..bdb4f1d9a 100644 --- a/cluster-providers/common/Taskfile.cloudfront.yaml +++ b/cluster-providers/common/Taskfile.cloudfront.yaml @@ -58,18 +58,23 @@ tasks: fi VPC_CIDR=$(aws ec2 describe-vpcs --vpc-ids "$VPC_ID" --region {{.AWS_REGION}} \ --query 'Vpcs[0].CidrBlock' --output text) - # Use private subnets — one per AZ (avoids "multiple subnets in same AZ" error) + # Select PRIVATE subnets (one per AZ), matching how the AWS LBC discovers subnets for + # an INTERNAL scheme ALB. If create-alb and the LBC disagree on the subnet set, the LBC + # calls SetSubnets on the adopted ALB (churn) and the ALB can land in public subnets, + # breaking the CloudFront VPC-Origin path. Prefer the canonical internal-elb role tag, + # then fall back to private-by-attribute (no public IP on launch) — NOT a *private* name + # tag, which many VPCs don't set. PRIVATE_SUBNETS=$(aws ec2 describe-subnets \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=*private*" \ + --filters "Name=vpc-id,Values=$VPC_ID" \ "Name=tag:kubernetes.io/role/internal-elb,Values=1" \ --region {{.AWS_REGION}} \ --query 'Subnets[].[SubnetId,AvailabilityZone]' --output text | \ sort -k2 -u | awk '{print $1}' | tr '\n' ' ') if [ -z "$PRIVATE_SUBNETS" ]; then PRIVATE_SUBNETS=$(aws ec2 describe-subnets \ - --filters "Name=vpc-id,Values=$VPC_ID" "Name=tag:Name,Values=*private*" \ + --filters "Name=vpc-id,Values=$VPC_ID" \ --region {{.AWS_REGION}} \ - --query 'Subnets[].[SubnetId,AvailabilityZone]' --output text | \ + --query 'Subnets[?MapPublicIpOnLaunch==`false`].[SubnetId,AvailabilityZone]' --output text | \ sort -k2 -u | awk '{print $1}' | tr '\n' ' ') fi if [ -z "$PRIVATE_SUBNETS" ]; then @@ -79,6 +84,14 @@ tasks: --query 'Subnets[].[SubnetId,AvailabilityZone]' --output text | \ sort -k2 -u | awk '{print $1}' | tr '\n' ' ') fi + # Tag the chosen subnets with the canonical internal-elb role tag so the AWS LBC + # discovers the SAME subnets when it adopts this ALB. Without this the LBC computes its + # own subnet set and calls SetSubnets, churning the ALB (which orphans the CloudFront + # VPC Origin bound to the old ALB — see steering/troubleshooting.md). + for _sn in $PRIVATE_SUBNETS; do + aws ec2 create-tags --resources "$_sn" --region {{.AWS_REGION}} \ + --tags Key=kubernetes.io/role/internal-elb,Value=1 >/dev/null 2>&1 || true + done SG_NAME="{{.HUB_CLUSTER_NAME}}-platform-alb-sg" SG_ID=$(aws ec2 describe-security-groups \ --filters "Name=group-name,Values=$SG_NAME" "Name=vpc-id,Values=$VPC_ID" \ From bb387b069e785d4a393b4b7f62c75858827c55c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 11:04:42 +0200 Subject: [PATCH 187/231] fix(kind-kro-ack): bootstrap hub Crossplane provider creds in the taskfile (not RGD) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hub's crossplane iam/eks providers had no AWS credentials, so crossplane-base's provider roles/PIAs (amp/rds/grafana/devlake) never reconciled -> no AMP/Grafana/RDS -> crossplane-base, observability-aws, grafana-dashboards, devlake all Degraded. Root cause: crossplane-base declares the downstream provider PIAs but iam/eks are createIdentity:false (chicken-and-egg), and nothing on the kro-ack hub seeds the first credential. The prior RGD-based bootstrap (6f8fb2c2) can't fix the hub because the hub's EksclusterWithVpc lives on the transient kind bootstrap cluster and is never reconciled by the hub-EKS RGD, so it can't self-heal an already-running hub. Fix (mirrors the crossplane/terraform flows): new hub:bootstrap-crossplane-identity task, run after hub:wait-for-sync, that idempotently creates the peeks-hub-crossplane-provider admin role + provider-aws-iam/eks pod identities, then restarts the crossplane provider pods (label pkg.crossplane.io/revision) twice — pass 1 credentials iam/eks, they reconcile the downstream provider roles/PIAs, pass 2 credentials those providers. Idempotent via a status guard (skips once CrossplaneAMPProviderRole exists). RGD bootstrap left in place for spokes. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 146 ++++++++----------- 1 file changed, 58 insertions(+), 88 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index c2ca96c13..446bcbf02 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -146,6 +146,14 @@ tasks: - task: hub:force-eso-refresh - task: hub:wait-for-sync vars: { MAX_OUTOFSYNC: "20" } + # Bootstrap the hub Crossplane providers' credentials (create provider-aws-iam/eks pod + # identities + restart) so crossplane-base's provider roles/PIAs (amp/rds/grafana/devlake) + # can reconcile. Done imperatively here (like the crossplane/terraform flows) rather than in + # the RGD: the hub is provisioned once by the transient kind cluster, so an RGD-based hub + # bootstrap can't self-heal an already-running hub. crossplane-base only declares the + # downstream provider PIAs (createIdentity:false for iam/eks) — nothing else seeds the first + # credential, so without this the whole crossplane chain stays credential-less. + - task: hub:bootstrap-crossplane-identity - task: install:phase1-parallel - task: hub:wait-for-full-sync # Wait for ALL Crossplane providers (including ec2) to be Healthy before spoke provisioning @@ -878,6 +886,56 @@ tasks: fi - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + hub:bootstrap-crossplane-identity: + desc: Seed hub Crossplane provider creds (provider-aws-iam/eks pod identities) so crossplane-base can provision + internal: true + status: + # Skip once Crossplane has already provisioned a downstream provider role — i.e. the + # providers are credentialed and working, so the bootstrap is done. + - aws iam get-role --role-name {{.HUB_CLUSTER_NAME}}-CrossplaneAMPProviderRole >/dev/null 2>&1 + cmds: + - printf '{{.C_STEP}}▸ Bootstrapping Crossplane provider pod identities (iam/eks)...{{.C_RESET}}\n' + - | + ROLE_NAME="{{.HUB_CLUSTER_NAME}}-crossplane-provider" + TRUST='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"pods.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' + aws iam get-role --role-name "$ROLE_NAME" >/dev/null 2>&1 || \ + aws iam create-role --role-name "$ROLE_NAME" --assume-role-policy-document "$TRUST" \ + --description "Bootstrap creds for crossplane provider-aws-iam/eks" >/dev/null + aws iam attach-role-policy --role-name "$ROLE_NAME" \ + --policy-arn arn:aws:iam::aws:policy/AdministratorAccess >/dev/null 2>&1 || true + ROLE_ARN="arn:aws:iam::{{.AWS_ACCOUNT_ID}}:role/$ROLE_NAME" + for SA in provider-aws-iam provider-aws-eks; do + EXISTS=$(aws eks list-pod-identity-associations --cluster-name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} \ + --query "associations[?serviceAccount=='$SA'] | [0].associationId" --output text 2>/dev/null) + if [ -z "$EXISTS" ] || [ "$EXISTS" = "None" ]; then + aws eks create-pod-identity-association --cluster-name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} \ + --namespace crossplane-system --service-account "$SA" --role-arn "$ROLE_ARN" >/dev/null + echo " created PIA crossplane-system/$SA" + else + echo " PIA crossplane-system/$SA already exists" + fi + done + - task: hub:kubeconfig + - printf '{{.C_STEP}}▸ Restarting Crossplane providers to pick up creds (pass 1)...{{.C_RESET}}\n' + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl delete pod -n crossplane-system -l pkg.crossplane.io/revision --force --grace-period=0 2>/dev/null || true + - | + # Once iam/eks are credentialed they reconcile the downstream provider roles/PIAs + # (amp/rds/grafana/dynamodb/ec2). Wait up to 5 min for those PIAs to be Ready, then + # restart again so those providers pick up THEIR freshly-created credentials. + KC={{.ROOT_DIR}}/private/hub-kubeconfig + for i in $(seq 1 20); do + TOTAL=$(KUBECONFIG=$KC kubectl get podidentityassociations.eks.aws.upbound.io -A --no-headers 2>/dev/null | wc -l | tr -d ' ') + READY=$(KUBECONFIG=$KC kubectl get podidentityassociations.eks.aws.upbound.io -A \ + -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep -c True) + printf '{{.C_INFO}} [%s] crossplane provider PIAs Ready: %s/%s{{.C_RESET}}\n' "$i" "$READY" "$TOTAL" + [ "$TOTAL" -gt 0 ] && [ "$READY" = "$TOTAL" ] && break + sleep 15 + done + - printf '{{.C_STEP}}▸ Restarting Crossplane providers to pick up creds (pass 2)...{{.C_RESET}}\n' + - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl delete pod -n crossplane-system -l pkg.crossplane.io/revision --force --grace-period=0 2>/dev/null || true + - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig + - printf '{{.C_OK}}✓ Crossplane provider credentials bootstrapped.{{.C_RESET}}\n' + hub:wait-for-sync: desc: Wait for hub ArgoCD apps to sync vars: @@ -1833,94 +1891,6 @@ tasks: printf '{{.C_OK}}✓ Spoke {{.CLUSTER}} enabled via Crossplane in fleet-config overlay.{{.C_RESET}}\n' - spokes:create-capabilities: - desc: "Create KRO + ACK EKS Capabilities on a Crossplane-provisioned spoke (usage: task spokes:create-capabilities -- )" - # KRO-path spokes get their kro/ACK capabilities declaratively from the EksclusterWithVpc - # RGD. Crossplane-path spokes do NOT, so create them here the same way the hub does - # (aws eks create-capability), with per-spoke capability IAM roles. Idempotent. - vars: - CLUSTER: '{{.CLI_ARGS}}' - cmds: - - | - set -e - if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:create-capabilities -- "; exit 1; fi - CLUSTER="{{.CLUSTER}}" - REGION="{{.AWS_REGION}}" - ACCOUNT="{{.AWS_ACCOUNT_ID}}" - PREFIX="{{.RESOURCE_PREFIX}}" - # CLUSTER already carries the resource prefix (e.g. peeks-spoke-staging), so the - # capability role name is just --capability-role — prepending PREFIX - # again would double it (peeks-peeks-...). PREFIX is still used below for the - # cluster-mgmt-* assume-role resource ARNs. - ACK_ROLE_NAME="${CLUSTER}-ack-capability-role" - KRO_ROLE_NAME="${CLUSTER}-kro-capability-role" - - # Wait for the Crossplane-provisioned spoke EKS cluster to be ACTIVE (up to ~90 min). - printf '{{.C_STEP}}▸ [%s] Waiting for spoke EKS cluster ACTIVE...{{.C_RESET}}\n' "$CLUSTER" - i=0 - until aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null | grep -q ACTIVE; do - i=$((i+1)); [ "$i" -ge 360 ] && { printf '{{.C_ERR}}⚠ %s not ACTIVE after timeout — skipping capability creation.{{.C_RESET}}\n' "$CLUSTER"; exit 0; } - if [ $((i % 8)) -eq 0 ]; then STATUS=$(aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null || echo "NOT_FOUND"); printf '{{.C_INFO}} [%s] status: %s (%ds){{.C_RESET}}\n' "$CLUSTER" "$STATUS" "$((i*15))"; fi - sleep 15 - done - - ASSUME_DOC='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"capabilities.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' - - # --- ACK capability role (AssumeWorkloadRoles + ManageIRSARoles) --- - printf '{{.C_STEP}}▸ [%s] Ensuring ACK capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$ACK_ROLE_NAME" - aws iam get-role --role-name "$ACK_ROLE_NAME" >/dev/null 2>&1 || \ - aws iam create-role --role-name "$ACK_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null - aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name AssumeWorkloadRoles --policy-document \ - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sts:AssumeRole\",\"sts:TagSession\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${PREFIX}-cluster-mgmt-*\"]}]}" >/dev/null - aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name ManageIRSARoles --policy-document \ - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"iam:GetRole\",\"iam:CreateRole\",\"iam:DeleteRole\",\"iam:TagRole\",\"iam:UntagRole\",\"iam:UpdateRole\",\"iam:UpdateAssumeRolePolicy\",\"iam:AttachRolePolicy\",\"iam:DetachRolePolicy\",\"iam:ListAttachedRolePolicies\",\"iam:ListRolePolicies\",\"iam:ListRoleTags\",\"iam:ListInstanceProfilesForRole\",\"iam:PutRolePolicy\",\"iam:DeleteRolePolicy\",\"iam:GetRolePolicy\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${CLUSTER}-*\"]}]}" >/dev/null - ACK_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${ACK_ROLE_NAME}" - - # --- KRO capability role (AmazonEKSClusterPolicy) --- - printf '{{.C_STEP}}▸ [%s] Ensuring KRO capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$KRO_ROLE_NAME" - aws iam get-role --role-name "$KRO_ROLE_NAME" >/dev/null 2>&1 || \ - aws iam create-role --role-name "$KRO_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null - aws iam attach-role-policy --role-name "$KRO_ROLE_NAME" --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy >/dev/null 2>&1 || true - KRO_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${KRO_ROLE_NAME}" - - # --- Create the capabilities (idempotent) --- - for cap in "kro:KRO:$KRO_ROLE_ARN" "ack:ACK:$ACK_ROLE_ARN"; do - NAME="${cap%%:*}"; rest="${cap#*:}"; TYPE="${rest%%:*}"; ROLE_ARN="${rest#*:}" - STATUS=$(aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null || echo "NOT_FOUND") - if [ "$STATUS" = "NOT_FOUND" ]; then - printf '{{.C_STEP}}▸ [%s] Creating %s capability...{{.C_RESET}}\n' "$CLUSTER" "$TYPE" - # The capability role was just created/updated above; EKS may reject - # CreateCapability with "trust policy is invalid" until IAM has - # propagated the role's trust policy. Retry on that transient error. - j=0 - until aws eks create-capability --region "$REGION" --cluster-name "$CLUSTER" --capability-name "$NAME" --type "$TYPE" --role-arn "$ROLE_ARN" --delete-propagation-policy RETAIN >/dev/null 2>/tmp/create-cap-err; do - j=$((j+1)) - if [ "$j" -ge 12 ]; then - printf '{{.C_ERR}}⚠ [%s] %s capability creation failed after retries:{{.C_RESET}}\n' "$CLUSTER" "$TYPE"; cat /tmp/create-cap-err; exit 1 - fi - if grep -q "trust policy" /tmp/create-cap-err; then - printf '{{.C_INFO}} [%s] Waiting for IAM trust policy propagation (attempt %s)...{{.C_RESET}}\n' "$CLUSTER" "$j" - sleep 10 - else - cat /tmp/create-cap-err; exit 1 - fi - done - else - printf '{{.C_INFO}} [%s] %s capability already exists (%s).{{.C_RESET}}\n' "$CLUSTER" "$TYPE" "$STATUS" - fi - done - - # --- Wait for both ACTIVE --- - for NAME in kro ack; do - i=0 - until aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null | grep -q ACTIVE; do - i=$((i+1)); [ "$i" -ge 40 ] && { printf '{{.C_ERR}}⚠ [%s] %s capability not ACTIVE after timeout.{{.C_RESET}}\n' "$CLUSTER" "$NAME"; break; } - sleep 15 - done - done - printf '{{.C_OK}}✓ [%s] KRO + ACK capabilities created.{{.C_RESET}}\n' "$CLUSTER" - - spokes:enable-kro: desc: "Enable a spoke cluster via KRO (usage: task spokes:enable-kro -- )" vars: From 0fe1a77b5810ce5d40848fe40d1705f9c9a282fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 11:11:45 +0200 Subject: [PATCH 188/231] fix(kind-kro-ack): restore spokes:create-capabilities accidentally removed in bb387b06 The previous commit's strReplace anchored on the repeated 'rm -f private/hub-kubeconfig' boilerplate and inadvertently dropped the spokes:create-capabilities task. Restore it verbatim; the hub:bootstrap-crossplane-identity task and its install wiring are unchanged. Task count back to 55 (was erroneously net-neutral at 54). --- cluster-providers/kind-kro-ack/Taskfile.yaml | 88 ++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 446bcbf02..a8af5c8eb 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1891,6 +1891,94 @@ tasks: printf '{{.C_OK}}✓ Spoke {{.CLUSTER}} enabled via Crossplane in fleet-config overlay.{{.C_RESET}}\n' + spokes:create-capabilities: + desc: "Create KRO + ACK EKS Capabilities on a Crossplane-provisioned spoke (usage: task spokes:create-capabilities -- )" + # KRO-path spokes get their kro/ACK capabilities declaratively from the EksclusterWithVpc + # RGD. Crossplane-path spokes do NOT, so create them here the same way the hub does + # (aws eks create-capability), with per-spoke capability IAM roles. Idempotent. + vars: + CLUSTER: '{{.CLI_ARGS}}' + cmds: + - | + set -e + if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:create-capabilities -- "; exit 1; fi + CLUSTER="{{.CLUSTER}}" + REGION="{{.AWS_REGION}}" + ACCOUNT="{{.AWS_ACCOUNT_ID}}" + PREFIX="{{.RESOURCE_PREFIX}}" + # CLUSTER already carries the resource prefix (e.g. peeks-spoke-staging), so the + # capability role name is just --capability-role — prepending PREFIX + # again would double it (peeks-peeks-...). PREFIX is still used below for the + # cluster-mgmt-* assume-role resource ARNs. + ACK_ROLE_NAME="${CLUSTER}-ack-capability-role" + KRO_ROLE_NAME="${CLUSTER}-kro-capability-role" + + # Wait for the Crossplane-provisioned spoke EKS cluster to be ACTIVE (up to ~90 min). + printf '{{.C_STEP}}▸ [%s] Waiting for spoke EKS cluster ACTIVE...{{.C_RESET}}\n' "$CLUSTER" + i=0 + until aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null | grep -q ACTIVE; do + i=$((i+1)); [ "$i" -ge 360 ] && { printf '{{.C_ERR}}⚠ %s not ACTIVE after timeout — skipping capability creation.{{.C_RESET}}\n' "$CLUSTER"; exit 0; } + if [ $((i % 8)) -eq 0 ]; then STATUS=$(aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null || echo "NOT_FOUND"); printf '{{.C_INFO}} [%s] status: %s (%ds){{.C_RESET}}\n' "$CLUSTER" "$STATUS" "$((i*15))"; fi + sleep 15 + done + + ASSUME_DOC='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"capabilities.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' + + # --- ACK capability role (AssumeWorkloadRoles + ManageIRSARoles) --- + printf '{{.C_STEP}}▸ [%s] Ensuring ACK capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$ACK_ROLE_NAME" + aws iam get-role --role-name "$ACK_ROLE_NAME" >/dev/null 2>&1 || \ + aws iam create-role --role-name "$ACK_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null + aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name AssumeWorkloadRoles --policy-document \ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sts:AssumeRole\",\"sts:TagSession\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${PREFIX}-cluster-mgmt-*\"]}]}" >/dev/null + aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name ManageIRSARoles --policy-document \ + "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"iam:GetRole\",\"iam:CreateRole\",\"iam:DeleteRole\",\"iam:TagRole\",\"iam:UntagRole\",\"iam:UpdateRole\",\"iam:UpdateAssumeRolePolicy\",\"iam:AttachRolePolicy\",\"iam:DetachRolePolicy\",\"iam:ListAttachedRolePolicies\",\"iam:ListRolePolicies\",\"iam:ListRoleTags\",\"iam:ListInstanceProfilesForRole\",\"iam:PutRolePolicy\",\"iam:DeleteRolePolicy\",\"iam:GetRolePolicy\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${CLUSTER}-*\"]}]}" >/dev/null + ACK_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${ACK_ROLE_NAME}" + + # --- KRO capability role (AmazonEKSClusterPolicy) --- + printf '{{.C_STEP}}▸ [%s] Ensuring KRO capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$KRO_ROLE_NAME" + aws iam get-role --role-name "$KRO_ROLE_NAME" >/dev/null 2>&1 || \ + aws iam create-role --role-name "$KRO_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null + aws iam attach-role-policy --role-name "$KRO_ROLE_NAME" --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy >/dev/null 2>&1 || true + KRO_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${KRO_ROLE_NAME}" + + # --- Create the capabilities (idempotent) --- + for cap in "kro:KRO:$KRO_ROLE_ARN" "ack:ACK:$ACK_ROLE_ARN"; do + NAME="${cap%%:*}"; rest="${cap#*:}"; TYPE="${rest%%:*}"; ROLE_ARN="${rest#*:}" + STATUS=$(aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null || echo "NOT_FOUND") + if [ "$STATUS" = "NOT_FOUND" ]; then + printf '{{.C_STEP}}▸ [%s] Creating %s capability...{{.C_RESET}}\n' "$CLUSTER" "$TYPE" + # The capability role was just created/updated above; EKS may reject + # CreateCapability with "trust policy is invalid" until IAM has + # propagated the role's trust policy. Retry on that transient error. + j=0 + until aws eks create-capability --region "$REGION" --cluster-name "$CLUSTER" --capability-name "$NAME" --type "$TYPE" --role-arn "$ROLE_ARN" --delete-propagation-policy RETAIN >/dev/null 2>/tmp/create-cap-err; do + j=$((j+1)) + if [ "$j" -ge 12 ]; then + printf '{{.C_ERR}}⚠ [%s] %s capability creation failed after retries:{{.C_RESET}}\n' "$CLUSTER" "$TYPE"; cat /tmp/create-cap-err; exit 1 + fi + if grep -q "trust policy" /tmp/create-cap-err; then + printf '{{.C_INFO}} [%s] Waiting for IAM trust policy propagation (attempt %s)...{{.C_RESET}}\n' "$CLUSTER" "$j" + sleep 10 + else + cat /tmp/create-cap-err; exit 1 + fi + done + else + printf '{{.C_INFO}} [%s] %s capability already exists (%s).{{.C_RESET}}\n' "$CLUSTER" "$TYPE" "$STATUS" + fi + done + + # --- Wait for both ACTIVE --- + for NAME in kro ack; do + i=0 + until aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null | grep -q ACTIVE; do + i=$((i+1)); [ "$i" -ge 40 ] && { printf '{{.C_ERR}}⚠ [%s] %s capability not ACTIVE after timeout.{{.C_RESET}}\n' "$CLUSTER" "$NAME"; break; } + sleep 15 + done + done + printf '{{.C_OK}}✓ [%s] KRO + ACK capabilities created.{{.C_RESET}}\n' "$CLUSTER" + + spokes:enable-kro: desc: "Enable a spoke cluster via KRO (usage: task spokes:enable-kro -- )" vars: From c5d3504399d28cfbf9ee7bef695d8431a6e39e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 11:18:24 +0200 Subject: [PATCH 189/231] feat(crossplane): native EKS Capability MRs (provider-aws-eks v2.6.1) + async spokes + tolerant spoke gate Replace the imperative EKS Capability creation with declarative Crossplane Capability MRs, make Crossplane spokes provision fully async, and add a non-fatal final gate that verifies spokes before install completes. - Bump provider-aws-eks (+family/iam/ec2/rds/dynamodb/amp/grafana) v2.5.3->v2.6.1 (registry, kind-crossplane bootstrap, terraform helm.tf). v2.6.1 serves a cluster-scoped eks.aws.upbound.io/v1beta1 Capability, so it composes into the existing legacy composition with no v2/namespaced migration. - platform-cluster XRD/composition: native Capability MRs (kro/ack/argocd) + capability IAM roles, matching the kro-ack RGD (capabilities.eks.amazonaws.com trust; KRO=AmazonEKSClusterPolicy; ACK=inline AssumeWorkloadRoles+ManageIRSARoles), CEL-gated on spec.capabilities..enabled; deletePropagationPolicy RETAIN. Adds spec.accountId + spec.capabilities.* to the XRD and claim chart. - kind-crossplane: spokes:enable-crossplane sets capabilities+accountId; drop the imperative spokes:create-capabilities and its ~19min foreground EKS-ACTIVE wait (spokes now async). Hub claim sets kro/ack/argocd(+IDC); hub:seed waits on the Capability MRs instead of the create-capability.yaml Job. Delete argocd:capability / argocd:delete-capability tasks + create/delete-capability.yaml (per ITERATION_PLAN.md). - kind-kro-ack: remove the dead, unreferenced spokes:create-capabilities copy. - workshop:install: add tolerant, non-fatal wait-for-spokes gate before setup-env (EKS ACTIVE -> ArgoCD cluster secrets -> spoke apps synced within tolerance, with backoff nudge), overlapping the ray/IDC/model tail. --- .../kind-crossplane/Taskfile.yaml | 174 +++--------- .../manifests/argocd/create-capability.yaml | 162 ------------ .../manifests/argocd/delete-capability.yaml | 66 ----- cluster-providers/kind-kro-ack/Taskfile.yaml | 88 ------ cluster-providers/terraform/helm.tf | 10 +- cluster-providers/workshop/Taskfile.yaml | 88 ++++++ .../platform-cluster/templates/claim.yaml | 9 + .../templates/composition.yaml | 250 +++++++++++++++++- .../platform-cluster/templates/xrd.yaml | 38 +++ .../crossplane/platform-cluster/values.yaml | 3 + gitops/addons/registry/platform.yaml | 16 +- 11 files changed, 432 insertions(+), 472 deletions(-) delete mode 100644 cluster-providers/kind-crossplane/manifests/argocd/create-capability.yaml delete mode 100644 cluster-providers/kind-crossplane/manifests/argocd/delete-capability.yaml diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index bd321ec37..85afe1d9e 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -248,10 +248,12 @@ tasks: install:phase2-spoke-dev: internal: true cmds: + # Capabilities (KRO+ACK) are now created declaratively by the platform-cluster + # Composition (spec.capabilities.* set in the crossplane-clusters overlay by + # spokes:enable-crossplane). No imperative spokes:create-capabilities / foreground + # EKS-ACTIVE wait — the spoke (cluster + capabilities) provisions async via ArgoCD. - task: spokes:enable-crossplane vars: { CLI_ARGS: "{{.RESOURCE_PREFIX}}-spoke-dev" } - - task: spokes:create-capabilities - vars: { CLI_ARGS: "{{.RESOURCE_PREFIX}}-spoke-dev" } install:phase2-spoke-prod: internal: true @@ -386,12 +388,12 @@ tasks: --set functions.patchAndTransform.version=v0.10.0 --set functions.environmentConfigs.version=v0.3.0 --set functions.celFilter.version=v0.2.0 - --set providers.family.version=v2.5.3 - --set providers.iam.version=v2.5.3 + --set providers.family.version=v2.6.1 + --set providers.iam.version=v2.6.1 --set providers.iam.createIdentity=false - --set providers.eks.version=v2.5.3 + --set providers.eks.version=v2.6.1 --set providers.eks.createIdentity=false - --set providers.ec2.version=v2.5.3 + --set providers.ec2.version=v2.6.1 --set providers.ec2.createIdentity=false --set providers.kubernetes.version={{.K8S_PROVIDER_VERSION}} --set providers.kubernetes.serviceAccount=provider-kubernetes @@ -458,6 +460,13 @@ tasks: --set clusters.hub.kubernetesVersion={{.HUB_K8S_VERSION}} --set clusters.hub.vpcCidr={{.HUB_VPC_CIDR}} --set clusters.hub.resourcePrefix={{.RESOURCE_PREFIX}} + --set-string clusters.hub.accountId={{.AWS_ACCOUNT_ID}} + --set clusters.hub.capabilities.kro.enabled=true + --set clusters.hub.capabilities.ack.enabled=true + --set clusters.hub.capabilities.argocd.enabled=true + --set-string clusters.hub.capabilities.argocd.idcInstanceArn={{.IDC_INSTANCE_ARN}} + --set-string clusters.hub.capabilities.argocd.idcRegion={{.IDC_REGION}} + --set-string clusters.hub.capabilities.argocd.adminGroupId={{.IDC_ADMIN_GROUP_ID}} {{.MNG_SETS}} | kubectl apply -f - - sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g" claims/argocd-capability-role.yaml | kubectl apply -f - @@ -477,8 +486,8 @@ tasks: --set aws.region={{.AWS_REGION}} --set aws.clusterName={{.HUB_CLUSTER_NAME}} --set aws.accountId={{.AWS_ACCOUNT_ID}} - --set providers.iam.version=v2.5.3 - --set providers.eks.version=v2.5.3 + --set providers.iam.version=v2.6.1 + --set providers.eks.version=v2.6.1 | kubectl apply -f - - helm template crossplane-pod-identity {{.GITOPS_ROOT}}/addons/charts/crossplane-pod-identity --set aws.region={{.AWS_REGION}} @@ -536,7 +545,13 @@ tasks: - printf '{{.C_STEP}}▸ Waiting for pod identity resources...{{.C_RESET}}\n' - kubectl -n crossplane-system wait --for=condition=Ready --timeout=300s roles.iam.aws.upbound.io --all - kubectl -n crossplane-system wait --for=condition=Ready --timeout=600s podidentityassociations.eks.aws.upbound.io --all - - task: argocd:capability + # EKS Capabilities (argocd/kro/ack) are now created declaratively by the + # platform-cluster Composition (spec.capabilities.* on the hub claim), reconciled + # by Crossplane once the hub EKS cluster is ACTIVE. Wait for them to be Ready here + # so the ArgoCD capability CRDs are served before hub:apply-root-appset — replaces + # the imperative argocd:capability Job (create-capability.yaml). + - printf '{{.C_STEP}}▸ Waiting for EKS Capabilities (argocd/kro/ack) to be Ready...{{.C_RESET}}\n' + - kubectl wait --for=condition=Ready --timeout=900s capabilities.eks.aws.upbound.io --all - task: hub:create-mgmt-roles - task: secrets-manager:seed - task: secrets-manager:seed-keycloak @@ -1163,6 +1178,12 @@ tasks: kubernetesVersion: "{{.K8S_VERSION}}" autoMode: true resourcePrefix: {{.RESOURCE_PREFIX}} + accountId: "{{.AWS_ACCOUNT_ID}}" + capabilities: + kro: + enabled: true + ack: + enabled: true $ADMIN_ARN_LINE $ADMIN_INSTANCE_ARN_LINE EOF @@ -1211,92 +1232,6 @@ tasks: fi printf '{{.C_OK}}✓ Spoke {{.CLUSTER}} enabled via Crossplane in fleet-config overlay.{{.C_RESET}}\n' - spokes:create-capabilities: - desc: "Create KRO + ACK EKS Capabilities on a Crossplane-provisioned spoke (usage: task spokes:create-capabilities -- )" - # KRO-path spokes get their kro/ACK capabilities declaratively from the EksclusterWithVpc - # RGD. Crossplane-path spokes do NOT, so create them here the same way the hub does - # (aws eks create-capability), with per-spoke capability IAM roles. Idempotent. - vars: - CLUSTER: '{{.CLI_ARGS}}' - cmds: - - | - set -e - if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:create-capabilities -- "; exit 1; fi - CLUSTER="{{.CLUSTER}}" - REGION="{{.AWS_REGION}}" - ACCOUNT="{{.AWS_ACCOUNT_ID}}" - PREFIX="{{.RESOURCE_PREFIX}}" - # CLUSTER already carries the resource prefix (e.g. peeks-spoke-staging), so the - # capability role name is just --capability-role — prepending PREFIX - # again would double it (peeks-peeks-...). PREFIX is still used below for the - # cluster-mgmt-* assume-role resource ARNs. - ACK_ROLE_NAME="${CLUSTER}-ack-capability-role" - KRO_ROLE_NAME="${CLUSTER}-kro-capability-role" - - # Wait for the Crossplane-provisioned spoke EKS cluster to be ACTIVE (up to ~90 min). - printf '{{.C_STEP}}▸ [%s] Waiting for spoke EKS cluster ACTIVE...{{.C_RESET}}\n' "$CLUSTER" - i=0 - until aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null | grep -q ACTIVE; do - i=$((i+1)); [ "$i" -ge 360 ] && { printf '{{.C_ERR}}⚠ %s not ACTIVE after timeout — skipping capability creation.{{.C_RESET}}\n' "$CLUSTER"; exit 0; } - sleep 15 - done - - ASSUME_DOC='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"capabilities.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' - - # --- ACK capability role (AssumeWorkloadRoles + ManageIRSARoles) --- - printf '{{.C_STEP}}▸ [%s] Ensuring ACK capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$ACK_ROLE_NAME" - aws iam get-role --role-name "$ACK_ROLE_NAME" >/dev/null 2>&1 || \ - aws iam create-role --role-name "$ACK_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null - aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name AssumeWorkloadRoles --policy-document \ - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sts:AssumeRole\",\"sts:TagSession\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${PREFIX}-cluster-mgmt-*\"]}]}" >/dev/null - aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name ManageIRSARoles --policy-document \ - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"iam:GetRole\",\"iam:CreateRole\",\"iam:DeleteRole\",\"iam:TagRole\",\"iam:UntagRole\",\"iam:UpdateRole\",\"iam:UpdateAssumeRolePolicy\",\"iam:AttachRolePolicy\",\"iam:DetachRolePolicy\",\"iam:ListAttachedRolePolicies\",\"iam:ListRolePolicies\",\"iam:ListRoleTags\",\"iam:ListInstanceProfilesForRole\",\"iam:PutRolePolicy\",\"iam:DeleteRolePolicy\",\"iam:GetRolePolicy\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${CLUSTER}-*\"]}]}" >/dev/null - ACK_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${ACK_ROLE_NAME}" - - # --- KRO capability role (AmazonEKSClusterPolicy) --- - printf '{{.C_STEP}}▸ [%s] Ensuring KRO capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$KRO_ROLE_NAME" - aws iam get-role --role-name "$KRO_ROLE_NAME" >/dev/null 2>&1 || \ - aws iam create-role --role-name "$KRO_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null - aws iam attach-role-policy --role-name "$KRO_ROLE_NAME" --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy >/dev/null 2>&1 || true - KRO_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${KRO_ROLE_NAME}" - - # --- Create the capabilities (idempotent) --- - for cap in "kro:KRO:$KRO_ROLE_ARN" "ack:ACK:$ACK_ROLE_ARN"; do - NAME="${cap%%:*}"; rest="${cap#*:}"; TYPE="${rest%%:*}"; ROLE_ARN="${rest#*:}" - STATUS=$(aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null || echo "NOT_FOUND") - if [ "$STATUS" = "NOT_FOUND" ]; then - printf '{{.C_STEP}}▸ [%s] Creating %s capability...{{.C_RESET}}\n' "$CLUSTER" "$TYPE" - # The capability role was just created/updated above; EKS may reject - # CreateCapability with "trust policy is invalid" until IAM has - # propagated the role's trust policy. Retry on that transient error. - j=0 - until aws eks create-capability --region "$REGION" --cluster-name "$CLUSTER" --capability-name "$NAME" --type "$TYPE" --role-arn "$ROLE_ARN" --delete-propagation-policy RETAIN >/dev/null 2>/tmp/create-cap-err; do - j=$((j+1)) - if [ "$j" -ge 12 ]; then - printf '{{.C_ERR}}⚠ [%s] %s capability creation failed after retries:{{.C_RESET}}\n' "$CLUSTER" "$TYPE"; cat /tmp/create-cap-err; exit 1 - fi - if grep -q "trust policy" /tmp/create-cap-err; then - printf '{{.C_INFO}} [%s] Waiting for IAM trust policy propagation (attempt %s)...{{.C_RESET}}\n' "$CLUSTER" "$j" - sleep 10 - else - cat /tmp/create-cap-err; exit 1 - fi - done - else - printf '{{.C_INFO}} [%s] %s capability already exists (%s).{{.C_RESET}}\n' "$CLUSTER" "$TYPE" "$STATUS" - fi - done - - # --- Wait for both ACTIVE --- - for NAME in kro ack; do - i=0 - until aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null | grep -q ACTIVE; do - i=$((i+1)); [ "$i" -ge 40 ] && { printf '{{.C_ERR}}⚠ [%s] %s capability not ACTIVE after timeout.{{.C_RESET}}\n' "$CLUSTER" "$NAME"; break; } - sleep 15 - done - done - printf '{{.C_OK}}✓ [%s] KRO + ACK capabilities created.{{.C_RESET}}\n' "$CLUSTER" - spokes:enable-kro: desc: "Enable a spoke cluster via KRO (usage: task spokes:enable-kro -- )" vars: @@ -2259,40 +2194,6 @@ tasks: kubectl rollout restart deploy/grafana-operator -n grafana-operator 2>/dev/null || true echo "Grafana ExternalSecrets resynced and grafana-operator restarted to pick up seeded credentials." - argocd:capability: - desc: Create EKS ArgoCD Capability (skips if done) - status: - - kubectl -n crossplane-system get job create-argocd-capability -o jsonpath='{.status.succeeded}' 2>/dev/null | grep -q 1 - vars: - CAPABILITY_ROLE_ARN: - sh: kubectl -n crossplane-system get roles.iam.aws.upbound.io argocd-capability-role -o jsonpath='{.status.atProvider.arn}' 2>/dev/null || echo "" - cmds: - - kubectl -n crossplane-system create configmap argocd-capability-config - --from-literal=CLUSTER_NAME={{.HUB_CLUSTER_NAME}} - --from-literal=AWS_REGION={{.AWS_REGION}} - --from-literal=AWS_ACCOUNT_ID={{.AWS_ACCOUNT_ID}} - --from-literal=CAPABILITY_NAME={{.CAPABILITY_NAME}} - --from-literal=CAPABILITY_ROLE_ARN={{.CAPABILITY_ROLE_ARN}} - --from-literal=IDC_INSTANCE_ARN={{.IDC_INSTANCE_ARN}} - --from-literal=IDC_REGION={{.IDC_REGION}} - --from-literal=IDC_ADMIN_GROUP_ID={{.IDC_ADMIN_GROUP_ID}} - --dry-run=client -o yaml | kubectl apply -f - - - kubectl delete job create-argocd-capability -n crossplane-system 2>/dev/null || true - - kubectl apply -f manifests/argocd/create-capability.yaml - - kubectl -n crossplane-system wait --for=condition=Complete --timeout=2400s job/create-argocd-capability - - argocd:delete-capability: - desc: Delete EKS ArgoCD Capability via Job - cmds: - - kubectl -n crossplane-system create configmap argocd-capability-config - --from-literal=CLUSTER_NAME={{.HUB_CLUSTER_NAME}} - --from-literal=AWS_REGION={{.AWS_REGION}} - --from-literal=CAPABILITY_NAME={{.CAPABILITY_NAME}} - --dry-run=client -o yaml | kubectl apply -f - - - kubectl delete job delete-argocd-capability -n crossplane-system 2>/dev/null || true - - kubectl apply -f manifests/argocd/delete-capability.yaml - - kubectl -n crossplane-system wait --for=condition=Complete --timeout=2400s job/delete-argocd-capability - status: desc: Check bootstrap status cmds: @@ -2503,10 +2404,10 @@ tasks: --set aws.region={{.AWS_REGION}} \ --set aws.clusterName={{.HUB_CLUSTER_NAME}} \ --set aws.accountId={{.AWS_ACCOUNT_ID}} \ - --set providers.family.version=v2.5.3 \ - --set providers.iam.version=v2.5.3 \ - --set providers.eks.version=v2.5.3 \ - --set providers.ec2.version=v2.5.3 \ + --set providers.family.version=v2.6.1 \ + --set providers.iam.version=v2.6.1 \ + --set providers.eks.version=v2.6.1 \ + --set providers.ec2.version=v2.6.1 \ | kubectl delete --ignore-not-found --wait=false -f - 2>/dev/null || true kubectl -n crossplane-system wait --for=delete podidentityassociations.eks.aws.upbound.io --all --timeout=120s 2>/dev/null || true kubectl -n crossplane-system wait --for=delete roles.iam.aws.upbound.io -l platform.gitops.io/role --timeout=120s 2>/dev/null || true @@ -2697,6 +2598,13 @@ tasks: --set clusters.hub.kubernetesVersion={{.HUB_K8S_VERSION}} --set clusters.hub.vpcCidr={{.HUB_VPC_CIDR}} --set clusters.hub.resourcePrefix={{.RESOURCE_PREFIX}} + --set-string clusters.hub.accountId={{.AWS_ACCOUNT_ID}} + --set clusters.hub.capabilities.kro.enabled=true + --set clusters.hub.capabilities.ack.enabled=true + --set clusters.hub.capabilities.argocd.enabled=true + --set-string clusters.hub.capabilities.argocd.idcInstanceArn={{.IDC_INSTANCE_ARN}} + --set-string clusters.hub.capabilities.argocd.idcRegion={{.IDC_REGION}} + --set-string clusters.hub.capabilities.argocd.adminGroupId={{.IDC_ADMIN_GROUP_ID}} {{.MNG_SETS}} | kubectl apply -f - - kubectl apply -f claims/ diff --git a/cluster-providers/kind-crossplane/manifests/argocd/create-capability.yaml b/cluster-providers/kind-crossplane/manifests/argocd/create-capability.yaml deleted file mode 100644 index 14ff3991d..000000000 --- a/cluster-providers/kind-crossplane/manifests/argocd/create-capability.yaml +++ /dev/null @@ -1,162 +0,0 @@ -# Job: Creates EKS ArgoCD capability after cluster is ACTIVE. -# Env vars injected from argocd-capability-config ConfigMap -# (created by Taskfile from config.yaml + Crossplane outputs). -apiVersion: batch/v1 -kind: Job -metadata: - name: create-argocd-capability - namespace: crossplane-system -spec: - backoffLimit: 3 - ttlSecondsAfterFinished: 600 - template: - spec: - serviceAccountName: crossplane - restartPolicy: OnFailure - volumes: - - name: aws-creds - secret: - secretName: aws-credentials - containers: - - name: create-capability - image: amazon/aws-cli:latest - envFrom: - - configMapRef: - name: argocd-capability-config - env: - - name: AWS_SHARED_CREDENTIALS_FILE - value: /tmp/.aws/credentials - volumeMounts: - - name: aws-creds - mountPath: /tmp/.aws - readOnly: true - command: ["/bin/bash", "-c"] - args: - - | - set -euo pipefail - - echo "=== Wait for EKS cluster ACTIVE ===" - aws eks wait cluster-active \ - --name "$CLUSTER_NAME" --region "$AWS_REGION" - echo "Cluster $CLUSTER_NAME is ACTIVE" - - echo "=== Create ArgoCD capability ===" - EXISTING=$(aws eks describe-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "$CAPABILITY_NAME" \ - --region "$AWS_REGION" \ - --query 'capability.status' \ - --output text 2>/dev/null || echo "NOT_FOUND") - - if [ "$EXISTING" = "NOT_FOUND" ]; then - aws eks create-capability \ - --region "$AWS_REGION" \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "$CAPABILITY_NAME" \ - --type ARGOCD \ - --role-arn "$CAPABILITY_ROLE_ARN" \ - --delete-propagation-policy RETAIN \ - --configuration "{ - \"argoCd\": { - \"awsIdc\": { - \"idcInstanceArn\": \"$IDC_INSTANCE_ARN\", - \"idcRegion\": \"$IDC_REGION\" - }, - \"rbacRoleMappings\": [{ - \"role\": \"ADMIN\", - \"identities\": [{ - \"id\": \"$IDC_ADMIN_GROUP_ID\", - \"type\": \"SSO_GROUP\" - }] - }] - } - }" - else - echo "Capability exists: $EXISTING" - fi - - echo "=== Wait for ACTIVE ===" - for i in $(seq 1 60); do - STATUS=$(aws eks describe-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "$CAPABILITY_NAME" \ - --region "$AWS_REGION" \ - --query 'capability.status' \ - --output text 2>/dev/null || echo "UNKNOWN") - echo "Status: $STATUS ($i/60)" - [ "$STATUS" = "ACTIVE" ] && break - sleep 30 - done - [ "$STATUS" != "ACTIVE" ] && exit 1 - - echo "=== Associate access policy ===" - aws eks associate-access-policy \ - --region "$AWS_REGION" \ - --cluster-name "$CLUSTER_NAME" \ - --principal-arn "$CAPABILITY_ROLE_ARN" \ - --policy-arn arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy \ - --access-scope type=cluster 2>/dev/null || true - - echo "=== Create KRO capability ===" - KRO_STATUS=$(aws eks describe-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "kro" \ - --region "$AWS_REGION" \ - --query 'capability.status' \ - --output text 2>/dev/null || echo "NOT_FOUND") - if [ "$KRO_STATUS" = "NOT_FOUND" ]; then - aws eks create-capability \ - --region "$AWS_REGION" \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "kro" \ - --type KRO \ - --role-arn "$CAPABILITY_ROLE_ARN" \ - --delete-propagation-policy RETAIN - echo "KRO capability created" - else - echo "KRO capability exists: $KRO_STATUS" - fi - - echo "=== Create ACK capability ===" - ACK_STATUS=$(aws eks describe-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "ack" \ - --region "$AWS_REGION" \ - --query 'capability.status' \ - --output text 2>/dev/null || echo "NOT_FOUND") - if [ "$ACK_STATUS" = "NOT_FOUND" ]; then - aws eks create-capability \ - --region "$AWS_REGION" \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "ack" \ - --type ACK \ - --role-arn "$CAPABILITY_ROLE_ARN" \ - --delete-propagation-policy RETAIN - echo "ACK capability created" - else - echo "ACK capability exists: $ACK_STATUS" - fi - - echo "=== Wait for KRO and ACK ACTIVE ===" - for CAP in kro ack; do - for i in $(seq 1 40); do - S=$(aws eks describe-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "$CAP" \ - --region "$AWS_REGION" \ - --query 'capability.status' \ - --output text 2>/dev/null || echo "UNKNOWN") - [ "$S" = "ACTIVE" ] && echo "$CAP: ACTIVE" && break - echo "$CAP: $S ($i/40)" - sleep 15 - done - done - - echo "=== Done ===" - echo "ARN: arn:aws:eks:${AWS_REGION}:${AWS_ACCOUNT_ID}:cluster/${CLUSTER_NAME}" - aws eks describe-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "$CAPABILITY_NAME" \ - --region "$AWS_REGION" \ - --query 'capability.configuration.argoCd.serverUrl' \ - --output text diff --git a/cluster-providers/kind-crossplane/manifests/argocd/delete-capability.yaml b/cluster-providers/kind-crossplane/manifests/argocd/delete-capability.yaml deleted file mode 100644 index 686f43c19..000000000 --- a/cluster-providers/kind-crossplane/manifests/argocd/delete-capability.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: delete-argocd-capability - namespace: crossplane-system -spec: - backoffLimit: 3 - ttlSecondsAfterFinished: 600 - template: - spec: - serviceAccountName: crossplane - restartPolicy: OnFailure - volumes: - - name: aws-creds - secret: - secretName: aws-credentials - containers: - - name: delete-capability - image: amazon/aws-cli:latest - envFrom: - - configMapRef: - name: argocd-capability-config - env: - - name: AWS_SHARED_CREDENTIALS_FILE - value: /tmp/.aws/credentials - volumeMounts: - - name: aws-creds - mountPath: /tmp/.aws - readOnly: true - command: ["/bin/bash", "-c"] - args: - - | - set -euo pipefail - - echo "=== Delete ArgoCD capability ===" - STATUS=$(aws eks describe-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "$CAPABILITY_NAME" \ - --region "$AWS_REGION" \ - --query 'capability.status' \ - --output text 2>/dev/null || echo "NOT_FOUND") - - if [ "$STATUS" = "NOT_FOUND" ]; then - echo "Capability not found — nothing to delete" - exit 0 - fi - - aws eks delete-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "$CAPABILITY_NAME" \ - --region "$AWS_REGION" - - echo "=== Wait for deletion ===" - for i in $(seq 1 60); do - STATUS=$(aws eks describe-capability \ - --cluster-name "$CLUSTER_NAME" \ - --capability-name "$CAPABILITY_NAME" \ - --region "$AWS_REGION" \ - --query 'capability.status' \ - --output text 2>/dev/null || echo "DELETED") - echo "Status: $STATUS ($i/60)" - [ "$STATUS" = "DELETED" ] && break - sleep 30 - done - - echo "=== Done ===" diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index a8af5c8eb..446bcbf02 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1891,94 +1891,6 @@ tasks: printf '{{.C_OK}}✓ Spoke {{.CLUSTER}} enabled via Crossplane in fleet-config overlay.{{.C_RESET}}\n' - spokes:create-capabilities: - desc: "Create KRO + ACK EKS Capabilities on a Crossplane-provisioned spoke (usage: task spokes:create-capabilities -- )" - # KRO-path spokes get their kro/ACK capabilities declaratively from the EksclusterWithVpc - # RGD. Crossplane-path spokes do NOT, so create them here the same way the hub does - # (aws eks create-capability), with per-spoke capability IAM roles. Idempotent. - vars: - CLUSTER: '{{.CLI_ARGS}}' - cmds: - - | - set -e - if [ -z "{{.CLUSTER}}" ]; then echo "Usage: task spokes:create-capabilities -- "; exit 1; fi - CLUSTER="{{.CLUSTER}}" - REGION="{{.AWS_REGION}}" - ACCOUNT="{{.AWS_ACCOUNT_ID}}" - PREFIX="{{.RESOURCE_PREFIX}}" - # CLUSTER already carries the resource prefix (e.g. peeks-spoke-staging), so the - # capability role name is just --capability-role — prepending PREFIX - # again would double it (peeks-peeks-...). PREFIX is still used below for the - # cluster-mgmt-* assume-role resource ARNs. - ACK_ROLE_NAME="${CLUSTER}-ack-capability-role" - KRO_ROLE_NAME="${CLUSTER}-kro-capability-role" - - # Wait for the Crossplane-provisioned spoke EKS cluster to be ACTIVE (up to ~90 min). - printf '{{.C_STEP}}▸ [%s] Waiting for spoke EKS cluster ACTIVE...{{.C_RESET}}\n' "$CLUSTER" - i=0 - until aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null | grep -q ACTIVE; do - i=$((i+1)); [ "$i" -ge 360 ] && { printf '{{.C_ERR}}⚠ %s not ACTIVE after timeout — skipping capability creation.{{.C_RESET}}\n' "$CLUSTER"; exit 0; } - if [ $((i % 8)) -eq 0 ]; then STATUS=$(aws eks describe-cluster --name "$CLUSTER" --region "$REGION" --query 'cluster.status' --output text 2>/dev/null || echo "NOT_FOUND"); printf '{{.C_INFO}} [%s] status: %s (%ds){{.C_RESET}}\n' "$CLUSTER" "$STATUS" "$((i*15))"; fi - sleep 15 - done - - ASSUME_DOC='{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"capabilities.eks.amazonaws.com"},"Action":["sts:AssumeRole","sts:TagSession"]}]}' - - # --- ACK capability role (AssumeWorkloadRoles + ManageIRSARoles) --- - printf '{{.C_STEP}}▸ [%s] Ensuring ACK capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$ACK_ROLE_NAME" - aws iam get-role --role-name "$ACK_ROLE_NAME" >/dev/null 2>&1 || \ - aws iam create-role --role-name "$ACK_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null - aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name AssumeWorkloadRoles --policy-document \ - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"sts:AssumeRole\",\"sts:TagSession\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${PREFIX}-cluster-mgmt-*\"]}]}" >/dev/null - aws iam put-role-policy --role-name "$ACK_ROLE_NAME" --policy-name ManageIRSARoles --policy-document \ - "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"iam:GetRole\",\"iam:CreateRole\",\"iam:DeleteRole\",\"iam:TagRole\",\"iam:UntagRole\",\"iam:UpdateRole\",\"iam:UpdateAssumeRolePolicy\",\"iam:AttachRolePolicy\",\"iam:DetachRolePolicy\",\"iam:ListAttachedRolePolicies\",\"iam:ListRolePolicies\",\"iam:ListRoleTags\",\"iam:ListInstanceProfilesForRole\",\"iam:PutRolePolicy\",\"iam:DeleteRolePolicy\",\"iam:GetRolePolicy\"],\"Resource\":[\"arn:aws:iam::${ACCOUNT}:role/${CLUSTER}-*\"]}]}" >/dev/null - ACK_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${ACK_ROLE_NAME}" - - # --- KRO capability role (AmazonEKSClusterPolicy) --- - printf '{{.C_STEP}}▸ [%s] Ensuring KRO capability role %s...{{.C_RESET}}\n' "$CLUSTER" "$KRO_ROLE_NAME" - aws iam get-role --role-name "$KRO_ROLE_NAME" >/dev/null 2>&1 || \ - aws iam create-role --role-name "$KRO_ROLE_NAME" --assume-role-policy-document "$ASSUME_DOC" >/dev/null - aws iam attach-role-policy --role-name "$KRO_ROLE_NAME" --policy-arn arn:aws:iam::aws:policy/AmazonEKSClusterPolicy >/dev/null 2>&1 || true - KRO_ROLE_ARN="arn:aws:iam::${ACCOUNT}:role/${KRO_ROLE_NAME}" - - # --- Create the capabilities (idempotent) --- - for cap in "kro:KRO:$KRO_ROLE_ARN" "ack:ACK:$ACK_ROLE_ARN"; do - NAME="${cap%%:*}"; rest="${cap#*:}"; TYPE="${rest%%:*}"; ROLE_ARN="${rest#*:}" - STATUS=$(aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null || echo "NOT_FOUND") - if [ "$STATUS" = "NOT_FOUND" ]; then - printf '{{.C_STEP}}▸ [%s] Creating %s capability...{{.C_RESET}}\n' "$CLUSTER" "$TYPE" - # The capability role was just created/updated above; EKS may reject - # CreateCapability with "trust policy is invalid" until IAM has - # propagated the role's trust policy. Retry on that transient error. - j=0 - until aws eks create-capability --region "$REGION" --cluster-name "$CLUSTER" --capability-name "$NAME" --type "$TYPE" --role-arn "$ROLE_ARN" --delete-propagation-policy RETAIN >/dev/null 2>/tmp/create-cap-err; do - j=$((j+1)) - if [ "$j" -ge 12 ]; then - printf '{{.C_ERR}}⚠ [%s] %s capability creation failed after retries:{{.C_RESET}}\n' "$CLUSTER" "$TYPE"; cat /tmp/create-cap-err; exit 1 - fi - if grep -q "trust policy" /tmp/create-cap-err; then - printf '{{.C_INFO}} [%s] Waiting for IAM trust policy propagation (attempt %s)...{{.C_RESET}}\n' "$CLUSTER" "$j" - sleep 10 - else - cat /tmp/create-cap-err; exit 1 - fi - done - else - printf '{{.C_INFO}} [%s] %s capability already exists (%s).{{.C_RESET}}\n' "$CLUSTER" "$TYPE" "$STATUS" - fi - done - - # --- Wait for both ACTIVE --- - for NAME in kro ack; do - i=0 - until aws eks describe-capability --cluster-name "$CLUSTER" --capability-name "$NAME" --region "$REGION" --query 'capability.status' --output text 2>/dev/null | grep -q ACTIVE; do - i=$((i+1)); [ "$i" -ge 40 ] && { printf '{{.C_ERR}}⚠ [%s] %s capability not ACTIVE after timeout.{{.C_RESET}}\n' "$CLUSTER" "$NAME"; break; } - sleep 15 - done - done - printf '{{.C_OK}}✓ [%s] KRO + ACK capabilities created.{{.C_RESET}}\n' "$CLUSTER" - - spokes:enable-kro: desc: "Enable a spoke cluster via KRO (usage: task spokes:enable-kro -- )" vars: diff --git a/cluster-providers/terraform/helm.tf b/cluster-providers/terraform/helm.tf index e0a7c62fb..0bc25377e 100644 --- a/cluster-providers/terraform/helm.tf +++ b/cluster-providers/terraform/helm.tf @@ -23,19 +23,19 @@ locals { # (createIdentity: false in registry); others get identity from ArgoCD post-bootstrap. crossplane_bootstrap_providers = { iam = { - package = "xpkg.upbound.io/upbound/provider-aws-iam:v2.5.3" + package = "xpkg.upbound.io/upbound/provider-aws-iam:v2.6.1" service_account = "provider-aws-iam" } eks = { - package = "xpkg.upbound.io/upbound/provider-aws-eks:v2.5.3" + package = "xpkg.upbound.io/upbound/provider-aws-eks:v2.6.1" service_account = "provider-aws-eks" } ec2 = { - package = "xpkg.upbound.io/upbound/provider-aws-ec2:v2.5.3" + package = "xpkg.upbound.io/upbound/provider-aws-ec2:v2.6.1" service_account = "provider-aws-ec2" } secretsmanager = { - package = "xpkg.upbound.io/upbound/provider-aws-secretsmanager:v2.5.3" + package = "xpkg.upbound.io/upbound/provider-aws-secretsmanager:v2.6.1" service_account = "provider-aws-secretsmanager" } } @@ -121,7 +121,7 @@ resource "kubectl_manifest" "crossplane_provider_family" { name = "provider-family-aws" } spec = { - package = "xpkg.upbound.io/upbound/provider-family-aws:v2.5.3" + package = "xpkg.upbound.io/upbound/provider-family-aws:v2.6.1" } }) diff --git a/cluster-providers/workshop/Taskfile.yaml b/cluster-providers/workshop/Taskfile.yaml index 3786ff2fc..145cdf579 100644 --- a/cluster-providers/workshop/Taskfile.yaml +++ b/cluster-providers/workshop/Taskfile.yaml @@ -80,10 +80,98 @@ tasks: - task: ":{{.PROVIDER}}:ray:wait-image" # Download ML models to S3 - task: ":{{.PROVIDER}}:ray:prestage-models" + # Final tolerant gate: spokes are enabled asynchronously during {{.PROVIDER}}:install + # (git push only), so their EKS clusters + ArgoCD apps finish provisioning while the + # ray/IDC/model tail above runs. Confirm they landed before we declare success. + # Non-fatal: never fails the install (spoke apps may legitimately still be settling). + - task: wait-for-spokes + - 'printf "{{.C_INFO}} [%ds] wait-for-spokes done{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' # Write platform URLs + credentials to ~/.bashrc.d/platform.sh for IDE use - task: setup-env - 'printf "{{.C_OK}}✓ Workshop install complete in %ds.{{.C_RESET}}\n" "$(($(date +%s) - {{.INSTALL_START}}))"' + # ============================================================ + # FINAL SPOKE GATE (tolerant, non-fatal) + # ============================================================ + wait-for-spokes: + desc: "Wait for spoke EKS clusters ACTIVE + their ArgoCD cluster secrets + spoke apps synced (tolerant, non-fatal)" + vars: + SPOKES: "{{.RESOURCE_PREFIX}}-spoke-dev {{.RESOURCE_PREFIX}}-spoke-prod" + MAX_OUTOFSYNC: '{{.MAX_OUTOFSYNC | default "5"}}' + cmds: + - | + # Strictly non-fatal: disable errexit so no internal check can fail the install. + set +e + KC=$(mktemp) + if ! aws eks update-kubeconfig --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --kubeconfig "$KC" >/dev/null 2>&1; then + printf '{{.C_ERR}}⚠ wait-for-spokes: cannot get hub kubeconfig — skipping (non-fatal).{{.C_RESET}}\n' + rm -f "$KC"; exit 0 + fi + export KUBECONFIG="$KC" + + # 1) Spoke EKS clusters ACTIVE (~20min budget each, tolerant) + for S in {{.SPOKES}}; do + printf '{{.C_STEP}}▸ [%s] waiting for EKS cluster ACTIVE...{{.C_RESET}}\n' "$S" + OK=false + for i in $(seq 1 80); do + ST=$(aws eks describe-cluster --name "$S" --region {{.AWS_REGION}} --query 'cluster.status' --output text 2>/dev/null || echo "NONE") + if [ "$ST" = "ACTIVE" ]; then OK=true; break; fi + sleep 15 + done + if [ "$OK" = "true" ]; then + printf '{{.C_OK}} ✓ %s ACTIVE{{.C_RESET}}\n' "$S" + else + printf '{{.C_ERR}} ⚠ %s not ACTIVE after ~20min — continuing (non-fatal).{{.C_RESET}}\n' "$S" + fi + done + + # 2) Spoke ArgoCD cluster secrets registered on the hub (needed for CD ApplicationSets) + for S in {{.SPOKES}}; do + for i in $(seq 1 40); do + if kubectl get secret -n argocd -l argocd.argoproj.io/secret-type=cluster \ + -o custom-columns=NAME:.metadata.name --no-headers 2>/dev/null | grep -qx "$S"; then + break + fi + if [ "$i" = "40" ]; then + printf '{{.C_ERR}} ⚠ %s ArgoCD cluster secret not registered — continuing (non-fatal).{{.C_RESET}}\n' "$S" + fi + sleep 15 + done + done + + # 3) Spoke-targeted apps synced within tolerance, nudging stuck ones (resets ArgoCD backoff) + printf '{{.C_STEP}}▸ Waiting for spoke apps to sync (tolerance %s, non-fatal)...{{.C_RESET}}\n' "{{.MAX_OUTOFSYNC}}" + SPOKE_RE=$(echo "{{.SPOKES}}" | tr ' ' '|') + TIMEOUT=1200; INTERVAL=30; ELAPSED=0; LAST_NUDGE=0 + while [ $ELAPSED -lt $TIMEOUT ]; do + ALL=$(kubectl get applications.argoproj.io -n argocd \ + -o custom-columns=NAME:.metadata.name,SYNC:.status.sync.status,DEST:.spec.destination.name --no-headers 2>/dev/null || echo "") + APPS=$(echo "$ALL" | awk -v re="$SPOKE_RE" '$3 ~ ("^("re")$")') + TOTAL=$(echo "$APPS" | awk 'NF>0' | wc -l | tr -d ' ') + OOS=$(echo "$APPS" | awk '$2!="Synced" && NF>0' | wc -l | tr -d ' ') + printf '{{.C_INFO}} spoke apps: %s total, %s out-of-sync (%ss elapsed){{.C_RESET}}\n' "$TOTAL" "$OOS" "$ELAPSED" + if [ "$TOTAL" -gt 0 ] && [ "$OOS" -le {{.MAX_OUTOFSYNC}} ]; then + printf '{{.C_OK}}✓ spoke apps synced within tolerance.{{.C_RESET}}\n' + break + fi + # After a 3min grace period, nudge out-of-sync apps every 5min to reset stuck backoff + if [ $ELAPSED -ge 180 ] && [ $((ELAPSED - LAST_NUDGE)) -ge 300 ]; then + for a in $(echo "$APPS" | awk '$2!="Synced" && NF>0 {print $1}'); do + kubectl patch application "$a" -n argocd --type merge \ + -p '{"operation":{"initiatedBy":{"username":"admin"},"sync":{"revision":"HEAD"}}}' >/dev/null 2>&1 || true + done + LAST_NUDGE=$ELAPSED + printf '{{.C_INFO}} ↻ nudged out-of-sync spoke apps{{.C_RESET}}\n' + fi + sleep $INTERVAL + ELAPSED=$((ELAPSED + INTERVAL)) + done + if [ $ELAPSED -ge $TIMEOUT ]; then + printf '{{.C_ERR}}⚠ spoke apps still syncing after %ss — continuing (non-fatal).{{.C_RESET}}\n' "$TIMEOUT" + fi + rm -f "$KC" + - printf '{{.C_OK}}✓ Spoke wait-gate complete.{{.C_RESET}}\n' + # ============================================================ # PRE-REQUISITES # Run before task install to prepare workshop-specific resources diff --git a/gitops/abstractions/crossplane/platform-cluster/templates/claim.yaml b/gitops/abstractions/crossplane/platform-cluster/templates/claim.yaml index a9a7d388d..fb6db3a5c 100644 --- a/gitops/abstractions/crossplane/platform-cluster/templates/claim.yaml +++ b/gitops/abstractions/crossplane/platform-cluster/templates/claim.yaml @@ -33,6 +33,15 @@ spec: {{- else if $.Values.global.adminInstanceRoleArn }} adminInstanceRoleArn: {{ $.Values.global.adminInstanceRoleArn }} {{- end }} + {{- if $cluster.accountId }} + accountId: {{ $cluster.accountId | quote }} + {{- else if $.Values.global.accountId }} + accountId: {{ $.Values.global.accountId | quote }} + {{- end }} + {{- if $cluster.capabilities }} + capabilities: + {{- toYaml $cluster.capabilities | nindent 4 }} + {{- end }} {{- if and $cluster.managedNodeGroup $cluster.managedNodeGroup.enabled }} managedNodeGroup: enabled: true diff --git a/gitops/abstractions/crossplane/platform-cluster/templates/composition.yaml b/gitops/abstractions/crossplane/platform-cluster/templates/composition.yaml index ab3521d52..129105166 100644 --- a/gitops/abstractions/crossplane/platform-cluster/templates/composition.yaml +++ b/gitops/abstractions/crossplane/platform-cluster/templates/composition.yaml @@ -847,16 +847,228 @@ spec: fromFieldPath: spec.argoCDRoleArn toFieldPath: spec.forProvider.principalArn - # NOTE: KRO + ACK EKS Capabilities for Crossplane-provisioned spokes are - # created out-of-band by the IDE Taskfile task `spokes:create-capabilities` - # (install phase2-spoke-*), which authenticates with the IDE instance role - # and provisions per-spoke capability IAM roles - # (--kro/ack-capability-role). A previous in-composition - # `eks-capabilities-job` (an aws-cli Job on the `crossplane` SA) was removed: - # it assumed the crossplane SA had a Pod Identity association - # (CrossplaneCapabilitiesRole) that is never created on the hub, so it always - # failed with `NoCredentials` — a redundant, noisy second path while the - # Taskfile path already brings both capabilities to ACTIVE. + # ===== EKS CAPABILITIES (native provider-aws-eks Capability MRs) ===== + # Replaces the imperative spokes:create-capabilities task (Crossplane spokes) + # and create-capability.yaml Job (hub). Each capability + its IAM role is + # gated by spec.capabilities..enabled via the function-cel-filter step. + # Roles mirror the kro-ack RGD (rg-eks.yaml) exactly: capabilities.eks.amazonaws.com + # trust; KRO=AmazonEKSClusterPolicy; ACK=inline AssumeWorkloadRoles+ManageIRSARoles. + # The provider (AdministratorAccess pod identity) creates these once the Cluster + # is ACTIVE — no foreground wait, fully declarative/async. + + # --- KRO capability role (managed AmazonEKSClusterPolicy) --- + - name: kro-capability-role + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: Role + metadata: + labels: + capability: kro + spec: + forProvider: + assumeRolePolicy: | + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "capabilities.eks.amazonaws.com"}, + "Action": ["sts:AssumeRole", "sts:TagSession"] + }] + } + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: string + string: + type: Format + fmt: "%s-kro-capability-role" + - type: PatchSet + patchSetName: tags + + - name: kro-capability-policy + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicyAttachment + spec: + forProvider: + policyArn: "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy" + roleSelector: + matchControllerRef: true + matchLabels: + capability: kro + + - name: kro-capability + base: + apiVersion: eks.aws.upbound.io/v1beta1 + kind: Capability + spec: + forProvider: + capabilityName: kro + type: KRO + deletePropagationPolicy: RETAIN + clusterNameSelector: + matchControllerRef: true + roleArnSelector: + matchControllerRef: true + matchLabels: + capability: kro + patches: + - type: PatchSet + patchSetName: common + + # --- ACK capability role (inline AssumeWorkloadRoles + ManageIRSARoles) --- + - name: ack-capability-role + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: Role + metadata: + labels: + capability: ack + spec: + forProvider: + assumeRolePolicy: | + { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "capabilities.eks.amazonaws.com"}, + "Action": ["sts:AssumeRole", "sts:TagSession"] + }] + } + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: string + string: + type: Format + fmt: "%s-ack-capability-role" + - type: PatchSet + patchSetName: tags + + # inline AssumeWorkloadRoles: sts:AssumeRole/TagSession on -cluster-mgmt-* + - name: ack-capability-policy-assume + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicy + metadata: + labels: + capability: ack + spec: + forProvider: + roleSelector: + matchControllerRef: true + matchLabels: + capability: ack + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: string + string: + type: Format + fmt: "%s-ack-AssumeWorkloadRoles" + - type: CombineFromComposite + combine: + variables: + - fromFieldPath: spec.accountId + - fromFieldPath: spec.resourcePrefix + strategy: string + string: + fmt: '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["sts:AssumeRole","sts:TagSession"],"Resource":["arn:aws:iam::%s:role/%s-cluster-mgmt-*"]}]}' + toFieldPath: spec.forProvider.policy + + # inline ManageIRSARoles: IRSA verbs on -* + - name: ack-capability-policy-irsa + base: + apiVersion: iam.aws.upbound.io/v1beta1 + kind: RolePolicy + metadata: + labels: + capability: ack + spec: + forProvider: + roleSelector: + matchControllerRef: true + matchLabels: + capability: ack + patches: + - type: FromCompositeFieldPath + fromFieldPath: spec.clusterName + toFieldPath: metadata.annotations[crossplane.io/external-name] + transforms: + - type: string + string: + type: Format + fmt: "%s-ack-ManageIRSARoles" + - type: CombineFromComposite + combine: + variables: + - fromFieldPath: spec.accountId + - fromFieldPath: spec.clusterName + strategy: string + string: + fmt: '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["iam:GetRole","iam:CreateRole","iam:DeleteRole","iam:TagRole","iam:UntagRole","iam:UpdateRole","iam:UpdateAssumeRolePolicy","iam:AttachRolePolicy","iam:DetachRolePolicy","iam:ListAttachedRolePolicies","iam:ListRolePolicies","iam:ListRoleTags","iam:ListInstanceProfilesForRole","iam:PutRolePolicy","iam:DeleteRolePolicy","iam:GetRolePolicy"],"Resource":["arn:aws:iam::%s:role/%s-*"]}]}' + toFieldPath: spec.forProvider.policy + + - name: ack-capability + base: + apiVersion: eks.aws.upbound.io/v1beta1 + kind: Capability + spec: + forProvider: + capabilityName: ack + type: ACK + deletePropagationPolicy: RETAIN + clusterNameSelector: + matchControllerRef: true + roleArnSelector: + matchControllerRef: true + matchLabels: + capability: ack + patches: + - type: PatchSet + patchSetName: common + + # --- ArgoCD capability (hub only) — reuses the existing argocd-capability-role + # (spec.argoCDRoleArn); needs IDC config for SSO ADMIN RBAC mapping. --- + - name: argocd-capability + base: + apiVersion: eks.aws.upbound.io/v1beta1 + kind: Capability + spec: + forProvider: + capabilityName: argocd + type: ARGOCD + deletePropagationPolicy: RETAIN + clusterNameSelector: + matchControllerRef: true + configuration: + argoCd: + awsIdc: {} + rbacRoleMapping: + - role: ADMIN + identity: + - type: SSO_GROUP + patches: + - type: PatchSet + patchSetName: common + - type: FromCompositeFieldPath + fromFieldPath: spec.argoCDRoleArn + toFieldPath: spec.forProvider.roleArn + - type: FromCompositeFieldPath + fromFieldPath: spec.capabilities.argocd.idcInstanceArn + toFieldPath: spec.forProvider.configuration.argoCd.awsIdc.idcInstanceArn + - type: FromCompositeFieldPath + fromFieldPath: spec.capabilities.argocd.idcRegion + toFieldPath: spec.forProvider.configuration.argoCd.awsIdc.idcRegion + - type: FromCompositeFieldPath + fromFieldPath: spec.capabilities.argocd.adminGroupId + toFieldPath: spec.forProvider.configuration.argoCd.rbacRoleMapping[0].identity[0].id # EKS Cluster with Auto Mode enabled - name: eks-cluster @@ -1159,4 +1371,22 @@ spec: expression: >- has(observed.composite.resource.spec.adminInstanceRoleArn) && observed.composite.resource.spec.adminInstanceRoleArn != "" + # EKS Capabilities — only compose each capability (+ its IAM role/policies) + # when spec.capabilities..enabled is true. Hub gets kro+ack+argocd; + # Crossplane spokes get kro+ack; unset/false => not composed. + - name: "^kro-capability(-role|-policy)?$" + expression: >- + has(observed.composite.resource.spec.capabilities) && + has(observed.composite.resource.spec.capabilities.kro) && + observed.composite.resource.spec.capabilities.kro.enabled == true + - name: "^ack-capability(-role|-policy-assume|-policy-irsa)?$" + expression: >- + has(observed.composite.resource.spec.capabilities) && + has(observed.composite.resource.spec.capabilities.ack) && + observed.composite.resource.spec.capabilities.ack.enabled == true + - name: "^argocd-capability$" + expression: >- + has(observed.composite.resource.spec.capabilities) && + has(observed.composite.resource.spec.capabilities.argocd) && + observed.composite.resource.spec.capabilities.argocd.enabled == true {{- end }} diff --git a/gitops/abstractions/crossplane/platform-cluster/templates/xrd.yaml b/gitops/abstractions/crossplane/platform-cluster/templates/xrd.yaml index d834dbb81..c0f4c8f66 100644 --- a/gitops/abstractions/crossplane/platform-cluster/templates/xrd.yaml +++ b/gitops/abstractions/crossplane/platform-cluster/templates/xrd.yaml @@ -48,6 +48,44 @@ spec: resourcePrefix: type: string description: Prefix for resource tagging and identification + accountId: + type: string + description: AWS account ID (used to build capability-role inline policy ARNs, e.g. -cluster-mgmt-*) + capabilities: + type: object + description: | + Native EKS Capabilities (KRO / ACK / ArgoCD) created for this cluster via + provider-aws-eks Capability MRs. Replaces the imperative + `spokes:create-capabilities` / create-capability.yaml Job path. + properties: + kro: + type: object + properties: + enabled: + type: boolean + default: false + ack: + type: object + properties: + enabled: + type: boolean + default: false + argocd: + type: object + description: ArgoCD capability (hub only). Needs IDC config for SSO RBAC. + properties: + enabled: + type: boolean + default: false + idcInstanceArn: + type: string + description: IAM Identity Center instance ARN + idcRegion: + type: string + description: IAM Identity Center region + adminGroupId: + type: string + description: IDC group id mapped to the ArgoCD ADMIN role argoCDRoleArn: type: string description: ARN of the hub ArgoCD capability role for spoke access diff --git a/gitops/abstractions/crossplane/platform-cluster/values.yaml b/gitops/abstractions/crossplane/platform-cluster/values.yaml index 8d2026105..51cf9c770 100644 --- a/gitops/abstractions/crossplane/platform-cluster/values.yaml +++ b/gitops/abstractions/crossplane/platform-cluster/values.yaml @@ -4,6 +4,9 @@ global: argoCDRoleArn: "" + # AWS account ID, used to build ACK capability-role inline-policy ARNs + # (-cluster-mgmt-* and -*). Per-cluster accountId takes precedence. + accountId: "" # Default AWS region for clusters that don't set their own. # Injected by the bootstrap ApplicationSet from the hub cluster's # aws_region annotation. Per-cluster `region` in diff --git a/gitops/addons/registry/platform.yaml b/gitops/addons/registry/platform.yaml index 72de3ee3e..77544d071 100644 --- a/gitops/addons/registry/platform.yaml +++ b/gitops/addons/registry/platform.yaml @@ -48,27 +48,27 @@ crossplane-base: version: "v0.2.0" providers: family: - version: "v2.5.3" + version: "v2.6.1" iam: - version: "v2.5.3" + version: "v2.6.1" createIdentity: false ec2: - version: "v2.5.3" + version: "v2.6.1" createIdentity: true eks: - version: "v2.5.3" + version: "v2.6.1" createIdentity: false rds: - version: "v2.5.3" + version: "v2.6.1" createIdentity: true dynamodb: - version: "v2.5.3" + version: "v2.6.1" createIdentity: true amp: - version: "v2.5.3" + version: "v2.6.1" createIdentity: true grafana: - version: "v2.5.3" + version: "v2.6.1" createIdentity: true kubernetes: version: "v1.2.1" From ddef5d7186636190d961b2966d69d8eb0477a74d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 11:22:33 +0200 Subject: [PATCH 190/231] docs+fix(crossplane): document declarative EKS Capabilities; drop dead argocd:delete-capability call in destroy - README/ITERATION_PLAN/steering: reflect native Capability MRs instead of the create-capability.yaml Job (mark ITERATION_PLAN item 10 done). - destroy Phase 3: remove the now-broken '- task: argocd:delete-capability' reference (task deleted in prior commit). The AWS-API fallback already force-deletes capabilities before the cluster delete; capability IAM role cleanup retained. --- ITERATION_PLAN.md | 14 ++++++++------ cluster-providers/kind-crossplane/README.md | 5 ++--- cluster-providers/kind-crossplane/Taskfile.yaml | 8 +++----- .../steering/workshop-infrastructure-context.md | 4 +++- 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/ITERATION_PLAN.md b/ITERATION_PLAN.md index 669dd4244..5d8cc9633 100644 --- a/ITERATION_PLAN.md +++ b/ITERATION_PLAN.md @@ -162,17 +162,19 @@ The root Taskfile delegates to providers but each provider implements tasks diff --- -### 10. Remove ArgoCD capability create/delete Job when Crossplane supports it +### 10. Remove ArgoCD capability create/delete Job when Crossplane supports it ✅ DONE **Priority:** Low **Labels:** enhancement, tech-debt **Ref:** https://github.com/crossplane-contrib/provider-upjet-aws/pull/2015 -The `create-capability.yaml` and `delete-capability.yaml` Jobs use AWS CLI to manage the EKS ArgoCD Capability because Crossplane's EKS provider doesn't support it yet. Once `provider-upjet-aws` adds `EKSCapability` support: -- Replace Jobs with Crossplane managed resources -- Remove `manifests/argocd/create-capability.yaml` and `delete-capability.yaml` -- Remove `argocd:capability` and `argocd:delete-capability` tasks -- Add capability to the PlatformCluster composition or as a separate claim +**Done** (provider-aws-eks v2.6.1 adds the `Capability` MR). The EKS Capabilities +(KRO/ACK/ArgoCD) are now created declaratively: +- Crossplane path: native `Capability` MRs in the `platform-cluster` Composition, + gated by `spec.capabilities..enabled` (hub + spokes). +- The imperative `create-capability.yaml` / `delete-capability.yaml` Jobs and the + `argocd:capability` / `argocd:delete-capability` / `spokes:create-capabilities` + tasks were removed; `hub:seed` waits for the Capability MRs to be Ready. --- diff --git a/cluster-providers/kind-crossplane/README.md b/cluster-providers/kind-crossplane/README.md index d296148a5..74bebb207 100644 --- a/cluster-providers/kind-crossplane/README.md +++ b/cluster-providers/kind-crossplane/README.md @@ -36,10 +36,10 @@ task install a. crossplane:helm Install Crossplane (version from addons/registry/platform.yaml) b. crossplane:providers Render crossplane-base chart (providers + functions only, no ProviderConfig) c. crossplane:provider-config Apply bootstrap ProviderConfig (aws-credentials secret) - d. crossplane:claims Apply XRD/Composition, PlatformCluster claim, pod identities (see below) + d. crossplane:claims Apply XRD/Composition (incl. KRO/ACK/ArgoCD Capability MRs), PlatformCluster claim, pod identities (see below) 6. hub:seed a. Wait for EKS cluster, IAM roles, and pod identities to become Ready - b. argocd:capability Create EKS ArgoCD Capability via Job + b. (wait capabilities) Wait for the KRO/ACK/ArgoCD Capability MRs (created by the Composition) to be Ready c. secrets-manager:seed Write hub config to AWS Secrets Manager d. secrets-manager:seed-keycloak Seed keycloak passwords into Secrets Manager e. hub:install-eso Helm install External Secrets on the hub @@ -102,7 +102,6 @@ Supporting manifests in `manifests/`: | Path | Purpose | |------|---------| -| `argocd/create-capability.yaml` | Job that calls the EKS API to create the ArgoCD Capability | | `argocd/appproject.yaml` | ArgoCD AppProject definition | | `crossplane/provider-config-bootstrap.yaml` | ProviderConfig using the `aws-credentials` secret (Kind-only, not used on hub) | | `external-secrets/cluster-secret-store.yaml` | ClusterSecretStore for AWS Secrets Manager | diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index 85afe1d9e..cd20f4fd0 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -2345,11 +2345,9 @@ tasks: kubectl -n crossplane-system wait --for=delete policies.iam.aws.upbound.io -l platform.gitops.io/role --timeout=120s 2>/dev/null || true kubectl -n crossplane-system wait --for=delete roles.iam.aws.upbound.io -l platform.gitops.io/role --timeout=120s 2>/dev/null || true ignore_error: true - # Phase 3: Delete ArgoCD capability via Job, then its IAM role (needs IAM provider alive) - # Use `task:` instead of `cmd: task ...` so the call doesn't go through gosh, - # which mis-parses some yq-driven dynamic vars in credentials:setup. - - task: argocd:delete-capability - ignore_error: true + # Phase 3: Delete the capability IAM role. The EKS Capabilities are managed as + # Crossplane Capability MRs (deletePropagationPolicy RETAIN); the AWS-API fallback + # below force-deletes them so the cluster delete in Phase 4 isn't blocked. - cmd: | sed "s|CLUSTER_NAME|{{.HUB_CLUSTER_NAME}}|g" claims/argocd-capability-role.yaml | kubectl delete --ignore-not-found --wait=false -f - 2>/dev/null || true kubectl -n crossplane-system wait --for=delete roles.iam.aws.upbound.io/argocd-capability-role --timeout=120s 2>/dev/null || true diff --git a/hack/.kiro/steering/workshop-infrastructure-context.md b/hack/.kiro/steering/workshop-infrastructure-context.md index 2f647ec44..f50d7f192 100644 --- a/hack/.kiro/steering/workshop-infrastructure-context.md +++ b/hack/.kiro/steering/workshop-infrastructure-context.md @@ -19,7 +19,9 @@ Architecture and deployment context for the appmod-blueprints platform on the `f - **ArgoCD** — managed GitOps, configured with IDC for SSO (ADMIN mapped to IDC group) - **KRO** — managed Kube Resource Orchestrator, provides ResourceGraphDefinitions - **ACK** — managed AWS Controllers for Kubernetes, provisions AWS resources from K8s -- All three created in `manifests/argocd/create-capability.yaml` Job +- Created declaratively: crossplane path via native `Capability` MRs in the platform-cluster + Composition (gated by `spec.capabilities..enabled`); kro-ack path via the + `EksclusterWithVpc` KRO RGD - They are NOT pods — they're EKS managed services ### Exposure Mode: CloudFront From 51bb045e296e3423a032f15b69db2e70d1f7897d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 14:54:13 +0200 Subject: [PATCH 191/231] fix(kro-ack): make hub crossplane bootstrap create-only (drop fatal restart/wait) hub:bootstrap-crossplane-identity aborted the entire install (exit 201): the wait loop's 'grep -c True' returns exit 1 on zero Ready PIAs, which under go-task's set:[errexit,pipefail] failed the task. It also ran before crossplane-base is deployed (No resources found on the pod restart), making the restart premature. Strip the task to PIA creation only. Provider pods start after these PIAs exist (credentialed at startup); the phase1 hub:restart-identity-pods task already waits for all PIAs Ready and restarts providers non-fatally. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 27 +++++--------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 446bcbf02..44c391d50 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -915,26 +915,13 @@ tasks: echo " PIA crossplane-system/$SA already exists" fi done - - task: hub:kubeconfig - - printf '{{.C_STEP}}▸ Restarting Crossplane providers to pick up creds (pass 1)...{{.C_RESET}}\n' - - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl delete pod -n crossplane-system -l pkg.crossplane.io/revision --force --grace-period=0 2>/dev/null || true - - | - # Once iam/eks are credentialed they reconcile the downstream provider roles/PIAs - # (amp/rds/grafana/dynamodb/ec2). Wait up to 5 min for those PIAs to be Ready, then - # restart again so those providers pick up THEIR freshly-created credentials. - KC={{.ROOT_DIR}}/private/hub-kubeconfig - for i in $(seq 1 20); do - TOTAL=$(KUBECONFIG=$KC kubectl get podidentityassociations.eks.aws.upbound.io -A --no-headers 2>/dev/null | wc -l | tr -d ' ') - READY=$(KUBECONFIG=$KC kubectl get podidentityassociations.eks.aws.upbound.io -A \ - -o jsonpath='{range .items[*]}{.status.conditions[?(@.type=="Ready")].status}{"\n"}{end}' 2>/dev/null | grep -c True) - printf '{{.C_INFO}} [%s] crossplane provider PIAs Ready: %s/%s{{.C_RESET}}\n' "$i" "$READY" "$TOTAL" - [ "$TOTAL" -gt 0 ] && [ "$READY" = "$TOTAL" ] && break - sleep 15 - done - - printf '{{.C_STEP}}▸ Restarting Crossplane providers to pick up creds (pass 2)...{{.C_RESET}}\n' - - KUBECONFIG={{.ROOT_DIR}}/private/hub-kubeconfig kubectl delete pod -n crossplane-system -l pkg.crossplane.io/revision --force --grace-period=0 2>/dev/null || true - - rm -f {{.ROOT_DIR}}/private/hub-kubeconfig - - printf '{{.C_OK}}✓ Crossplane provider credentials bootstrapped.{{.C_RESET}}\n' + # NOTE: We deliberately do NOT restart the Crossplane providers here. + # This task runs before crossplane-base is deployed, so the provider pods + # don't exist yet — they will start AFTER these PIAs exist and therefore + # pick up credentials at startup. The downstream providers (amp/rds/etc.) + # and any that started early are restarted (non-fatally) by the phase1 + # task hub:restart-identity-pods, which waits for all PIAs Ready first. + - printf '{{.C_OK}}✓ Crossplane provider pod identities created (providers credentialed on startup/phase1 restart).{{.C_RESET}}\n' hub:wait-for-sync: desc: Wait for hub ArgoCD apps to sync From a21582dab838a3548ca458e358b1eb58fe2dbb55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 15:58:05 +0200 Subject: [PATCH 192/231] fix(kro): grant hub kro/ack capability role ACK RBAC for kro-provisioned spokes Provisioning a spoke via the KRO path creates the EksclusterWithVpc claim on the HUB, so the HUB's kro capability (peeks-hub-kro-capability-role/KRO) renders the ACK vpc/subnet/eks CRs. The eks-capabilities-rbac ClusterRole/Binding that grants those ACK API groups only targeted enable_kro_manifests (spokes); the hub uses enable_kro_manifests_hub, so it never got the RBAC. The hub's kro cap role is not cluster-admin, so KRO was RBAC-denied: vpcs.ec2.services.k8s.aws is forbidden: User .../peeks-hub-kro-capability-role/KRO cannot get resource vpcs in ec2.services.k8s.aws in namespace peeks-spoke-prod Add eks-capabilities-rbac-hub, gated on enable_kro_manifests_hub, mirroring the kro-manifests/kro-manifests-hub split. No cluster has both labels, so no collision. --- gitops/addons/registry/platform.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/gitops/addons/registry/platform.yaml b/gitops/addons/registry/platform.yaml index 77544d071..d9643f4b2 100644 --- a/gitops/addons/registry/platform.yaml +++ b/gitops/addons/registry/platform.yaml @@ -223,6 +223,25 @@ eks-capabilities-rbac: accountId: '{{.metadata.annotations.aws_account_id}}' clusterName: '{{.metadata.annotations.aws_cluster_name}}' +# Hub variant: the hub reconciles kro spoke claims (EksclusterWithVpc), so its own +# kro/ack capability role needs the same ACK RBAC. The hub uses enable_kro_manifests_hub +# (not enable_kro_manifests) to avoid the path-excluded spoke kro-manifests app, so the +# spoke-scoped eks-capabilities-rbac above never lands on the hub. Bind it here too. +eks-capabilities-rbac-hub: + namespace: kro + path: '{{.metadata.annotations.addonsRepoBasepath}}addons/charts/eks-capabilities-rbac' + annotationsAppSet: + argocd.argoproj.io/sync-wave: '-3' + selector: + matchExpressions: + - key: enable_kro_manifests_hub + operator: In + values: ['true'] + valuesObject: + global: + accountId: '{{.metadata.annotations.aws_account_id}}' + clusterName: '{{.metadata.annotations.aws_cluster_name}}' + multi-acct: namespace: kro path: '{{.metadata.annotations.addonsRepoBasepath}}addons/charts/multi-acct' From 5a088e7d62cfa334812379f1e36ba59c91bb0e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 16:12:31 +0200 Subject: [PATCH 193/231] docs(troubleshooting): hub kro/ack capability RBAC gap + manual ACK-object delete recovery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the 'vpcs.ec2.services.k8s.aws forbidden' error for kro-provisioned spokes (hub kro cap not cluster-admin, eks-capabilities-rbac only on spokes), the eks-capabilities-rbac-hub fix, and the key manual step: KRO won't reconcile over the ACK object left half-created during the denied window — delete it so KRO recreates it cleanly. --- .kiro/steering/troubleshooting.md | 40 +++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/.kiro/steering/troubleshooting.md b/.kiro/steering/troubleshooting.md index b8d697364..e57410161 100644 --- a/.kiro/steering/troubleshooting.md +++ b/.kiro/steering/troubleshooting.md @@ -396,6 +396,46 @@ The `services.k8s.aws/force-reconcile` annotation does NOT always work (especial **Fix**: Delete the CRD manually (`kubectl delete crd .kro.run`), then sync to let KRO recreate it. **Warning**: This deletes all instances of that CRD — may trigger resource deletion in AWS. Only do this when safe. +### Hub kro/ack capability RBAC-denied on ACK resources (kro-provisioned spokes) +**Symptoms**: A spoke provisioned via the **KRO path** never gets its EKS cluster. The +`EksclusterWithVpc` instance on the hub shows a reconcile error like: +``` +vpcs.ec2.services.k8s.aws "-vpc" is forbidden: User +"arn:aws:sts:::assumed-role/-kro-capability-role/KRO" cannot get resource +"vpcs" in API group "ec2.services.k8s.aws" in the namespace "" +``` + +**Root cause**: A kro-provisioned spoke's `EksclusterWithVpc` claim is reconciled by the +**hub's** kro capability (`-kro-capability-role/KRO`), which renders the ACK +VPC/subnet/EKS CRs. The hub's kro/ack capability roles are NOT cluster-admin (they get +`AmazonEKSClusterPolicy` / inline ACK policies, mirroring the kro-ack RGD). The +`eks-capabilities-rbac` ClusterRole that grants the ACK API groups only targeted +`enable_kro_manifests` (spokes); the hub uses `enable_kro_manifests_hub`, so the hub never +got the RBAC. The chart comment "redundant on the hub because the capability role has +ClusterAdminPolicy" is wrong — the hub kro/ack cap is not cluster-admin in either the +crossplane or kro-ack flow. This is a **shared latent gap**; it surfaces on whichever flow +first exercises a KRO-provisioned spoke (crossplane: `spoke-prod`). + +**Fix (GitOps)**: `eks-capabilities-rbac-hub` registry entry (`gitops/addons/registry/platform.yaml`), +gated on `enable_kro_manifests_hub`, deploys the same `eks-capabilities-kro` ClusterRole+Binding +on the hub, bound to `-kro-capability-role/KRO` and `-ack-capability-role/ACK`. + +**Applying to an already-running hub**: the hub's `addonsRepoRevision` tracks the branch, so a +hard-refresh of the `cluster-addons` ApplicationSet regenerates it and syncs +`eks-capabilities-rbac-hub-` automatically: +```bash +kubectl -n argocd annotate applicationset cluster-addons argocd.argoproj.io/refresh=hard --overwrite +``` +(Or apply the rendered ClusterRole/Binding directly for an immediate unblock.) + +**Manual recovery — KRO does NOT self-heal the stuck ACK object**: after the RBAC is fixed, +KRO will not reconcile *over* the ACK resource that was left half-created during the denied +window. Delete the stuck ACK object so KRO recreates it cleanly: +```bash +kubectl --context delete vpcs.ec2.services.k8s.aws -vpc -n +# KRO recreates it (now permitted) and the spoke provisioning resumes. +``` + ## Crossplane Issues ### NAT Gateway reference resolution race condition From 6cd745398983b498f0f91ea79b4ce90ae40e6cf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 16:19:51 +0200 Subject: [PATCH 194/231] perf(kro-ack): enable spokes before Crossplane provider bring-up KRO-provisioned spokes are created by the hub's ACK EKS capability, not the Crossplane providers, so spoke EKS creation does not depend on phase1 restart / wait-for-providers. Move set-overlay-repo + install:phase2-parallel ahead of the provider bring-up so the ~25min spoke build starts earlier and overlaps with the provider restart, observability seeding, ray image build and idc. kro-ack only; kind-crossplane spoke-dev genuinely uses the Crossplane path and is unchanged. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 44c391d50..4d48142ba 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -154,13 +154,21 @@ tasks: # downstream provider PIAs (createIdentity:false for iam/eks) — nothing else seeds the first # credential, so without this the whole crossplane chain stays credential-less. - task: hub:bootstrap-crossplane-identity + # Enable spokes EARLY (kro-ack-specific optimization): KRO-provisioned spokes are + # created by the hub's ACK EKS capability, NOT the Crossplane providers, so spoke + # EKS creation (~25min, both in parallel) does not depend on the provider bring-up + # below. Wire the overlay + push spoke enablement now so the long-pole spoke build + # overlaps with the provider restart, observability seeding, ray image build and idc + # that follow — this removes the provider-wait from the spoke critical path. + - task: hub:set-overlay-repo + - task: install:phase2-parallel + # Crossplane provider bring-up for the hub's OWN observability (AMP/RDS/Grafana). + # Runs while the spokes provision asynchronously in the background. - task: install:phase1-parallel - task: hub:wait-for-full-sync - # Wait for ALL Crossplane providers (including ec2) to be Healthy before spoke provisioning + # Wait for ALL Crossplane providers (including ec2) to be Healthy. - task: hub:wait-for-providers - task: install:phase1b-parallel - - task: hub:set-overlay-repo - - task: install:phase2-parallel - | LOCAL_FLEET="$HOME/environment/fleet-config" if [ -d "$LOCAL_FLEET/.git" ]; then From 8e15fa1be746b28af013a6d4b01c041e3ee093d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 16:35:35 +0200 Subject: [PATCH 195/231] fix(kro-spokes): provider-aware ArgoCD hub capability role name in clusters-kro The clusters-kro ApplicationSet hardcoded argoCdHubRoleArn/argoCdCapabilityRoleArn to -argocd-capability-role (kebab). That's correct on kro-ack (its RGD creates the role kebab-case) but wrong on the crossplane hub, whose ArgoCD capability role is -ArgoCDCapabilityRole (PascalCase). A KRO-provisioned spoke on the crossplane hub then built an argocd-role trust policy referencing a non-existent principal, and ACK IAM went terminal: MalformedPolicyDocument: Invalid principal in policy: AWS: arn:...:role/peeks-hub-argocd-capability-role Key the suffix off the hub secret's provider label (kro-ack => kebab; crossplane/ no-label => PascalCase) so the trust policy references an existing role. --- gitops/bootstrap/clusters-kro.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/gitops/bootstrap/clusters-kro.yaml b/gitops/bootstrap/clusters-kro.yaml index b28ff5af9..7b0344c82 100644 --- a/gitops/bootstrap/clusters-kro.yaml +++ b/gitops/bootstrap/clusters-kro.yaml @@ -65,8 +65,14 @@ spec: ignoreMissingValueFiles: true valuesObject: global: - argoCdHubRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-argocd-capability-role' - argoCdCapabilityRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-argocd-capability-role' + # The hub's ArgoCD capability role name differs by provisioner: the kro-ack + # RGD creates it kebab-case (-argocd-capability-role); the crossplane + # composition creates it PascalCase (-ArgoCDCapabilityRole). Key off the + # hub secret's `provider` label (kro-ack stamps it; crossplane has none) so the + # spoke argocd-role trust policy references a role that actually exists — otherwise + # ACK IAM rejects it (MalformedPolicyDocument: Invalid principal). + argoCdHubRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}{{ if and (hasKey .metadata.labels "provider") (eq (index .metadata.labels "provider") "kro-ack") }}-argocd-capability-role{{ else }}-ArgoCDCapabilityRole{{ end }}' + argoCdCapabilityRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}{{ if and (hasKey .metadata.labels "provider") (eq (index .metadata.labels "provider") "kro-ack") }}-argocd-capability-role{{ else }}-ArgoCDCapabilityRole{{ end }}' defaultRegion: '{{ .metadata.annotations.aws_region }}' targetCluster: '{{ .clusterName }}' # Propagate the deploy branch from the control-plane so kro spokes From 47ef237e6bc511e5baa0f9c13a0d1ccaad558c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 17:00:30 +0200 Subject: [PATCH 196/231] fix(kro-eks): only trust -argocd-capability-role when argocdCapability enabled The spoke argocd-role trust policy unconditionally listed -argocd-capability-role as a principal, but that role is only created when argocdCapability.enabled==true (hub only). On any spoke (enabled=false) the principal doesn't exist, so IAM rejected the whole trust: MalformedPolicyDocument: Invalid principal in policy: AWS: arn:...:role/peeks-spoke-prod-argocd-capability-role Make the second principal conditional: capability role ARN when enabled (hub), otherwise duplicate argoCdHubRoleArn (IAM dedupes) so there is no dangling principal. Fixes KRO-provisioned spokes on both hubs (surfaced on the crossplane hub where spoke-prod is the KRO path). --- .../addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index 6cf7e1275..428a3dccc 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -429,7 +429,7 @@ spec: "Principal": { "AWS": [ "${schema.spec.argoCdHubRoleArn}", - "arn:${schema.spec.aws_partition}:iam::${schema.spec.accountId}:role/${schema.spec.name}-argocd-capability-role" + "${schema.spec.argocdCapability.enabled == 'true' ? 'arn:' + schema.spec.aws_partition + ':iam::' + schema.spec.accountId + ':role/' + schema.spec.name + '-argocd-capability-role' : schema.spec.argoCdHubRoleArn}" ] }, "Action": [ From 8148cb61735535b47debc5e104410f59a66a5c7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 17:02:21 +0200 Subject: [PATCH 197/231] fix(clusters-kro): drop hasKey guard that broke on map[string]string labels The ApplicationSet clusters generator exposes .metadata.labels as map[string]string, but sprig hasKey expects map[string]interface{} -> 'wrong type for value' template error, leaving clusters-kro Degraded and blocking spoke generation. Use a plain index lookup instead: index on map[string]string returns "" for a missing key without tripping missingkey=error (which only affects .field access), so crossplane hubs (no provider label) correctly fall through to the PascalCase suffix. --- gitops/bootstrap/clusters-kro.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gitops/bootstrap/clusters-kro.yaml b/gitops/bootstrap/clusters-kro.yaml index 7b0344c82..33a7851b6 100644 --- a/gitops/bootstrap/clusters-kro.yaml +++ b/gitops/bootstrap/clusters-kro.yaml @@ -71,8 +71,8 @@ spec: # hub secret's `provider` label (kro-ack stamps it; crossplane has none) so the # spoke argocd-role trust policy references a role that actually exists — otherwise # ACK IAM rejects it (MalformedPolicyDocument: Invalid principal). - argoCdHubRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}{{ if and (hasKey .metadata.labels "provider") (eq (index .metadata.labels "provider") "kro-ack") }}-argocd-capability-role{{ else }}-ArgoCDCapabilityRole{{ end }}' - argoCdCapabilityRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}{{ if and (hasKey .metadata.labels "provider") (eq (index .metadata.labels "provider") "kro-ack") }}-argocd-capability-role{{ else }}-ArgoCDCapabilityRole{{ end }}' + argoCdHubRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}{{ if eq (index .metadata.labels "provider") "kro-ack" }}-argocd-capability-role{{ else }}-ArgoCDCapabilityRole{{ end }}' + argoCdCapabilityRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}{{ if eq (index .metadata.labels "provider") "kro-ack" }}-argocd-capability-role{{ else }}-ArgoCDCapabilityRole{{ end }}' defaultRegion: '{{ .metadata.annotations.aws_region }}' targetCluster: '{{ .clusterName }}' # Propagate the deploy branch from the control-plane so kro spokes From 524ee2755c0ba3ad0518fbc5238110f948e9c4b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 18:14:37 +0200 Subject: [PATCH 198/231] fix(kro-ack): seed /config vpc as JSON object so devlake RDS provisions secrets-manager:seed wrote the config secret's `vpc` property as a bare VPC-id string (--arg vpc '{{.VPC_ID}}'). But the aws-resources ExternalSecret reads that property into peeks-hub-vpc-secret.vpc_data, and the init-env-config Job parses .id/.subnet_ids/.cluster_security_group_id from it to build the vpc-config EnvironmentConfig the devlake RDS composition consumes. A bare string makes every jq lookup empty -> empty vpc-config -> RDS security group gets vpcId=null and the subnet group gets empty subnetIds -> RDS Instance never created -> devlake stuck waiting for its MySQL endpoint secret. Resolve PRIVATE_SUBNET_IDS + CLUSTER_SG and write `vpc` as the JSON object {id,subnet_ids,cluster_security_group_id} (stored as a JSON string via --arg, matching kind-crossplane). Sole consumer of the property is that ExternalSecret; bare-id consumers use the separate aws_vpc_id metadata field, so no other reader is affected. --- cluster-providers/kind-kro-ack/Taskfile.yaml | 21 +++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index 4d48142ba..ca9317d67 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -709,6 +709,15 @@ tasks: sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.arn' --output text 2>/dev/null VPC_ID: sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.vpcId' --output text + # Private subnets + cluster SG for the devlake RDS `vpc` object (see cmds). Mirrors the + # kind-crossplane seed so the aws-resources ExternalSecret / init-env-config Job can + # populate a non-empty vpc-config EnvironmentConfig. + PRIVATE_SUBNET_IDS: + sh: | + SUBNETS=$(aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.subnetIds' --output text | tr '\t' '\n') + aws ec2 describe-subnets --subnet-ids $SUBNETS --region {{.AWS_REGION}} --query 'Subnets[?MapPublicIpOnLaunch==`false`].SubnetId' --output text | tr '\t' ',' + CLUSTER_SG: + sh: aws eks describe-cluster --name {{.HUB_CLUSTER_NAME}} --region {{.AWS_REGION}} --query 'cluster.resourcesVpcConfig.clusterSecurityGroupId' --output text # Resolve the platform CloudFront domain at TASK-RUN time (task-level var, evaluated # when this task runs — after cloudfront:setup-exposure created it). The taskfile-level # CLOUDFRONT_DOMAIN global is frozen empty at load, which would make ingress_domain_name @@ -725,12 +734,22 @@ tasks: - | SECRET_KEY="{{.HUB_CLUSTER_NAME}}/config" GITLAB_CF_DOMAIN=$(cat {{.ROOT_DIR}}/private/gitlab-cloudfront-domain 2>/dev/null || yq '.cloudfront.gitlabDomain // ""' {{.CONFIG_FILE}} || true) + # The aws-resources ExternalSecret reads /config property `vpc` into + # peeks-hub-vpc-secret.vpc_data; the init-env-config Job then parses .id / .subnet_ids / + # .cluster_security_group_id from it to build the vpc-config EnvironmentConfig that the + # devlake RDS composition consumes. `vpc` MUST be a JSON object (a bare VPC-id string makes + # every jq lookup empty -> empty vpc-config -> RDS never provisions). Stored as a JSON + # STRING (--arg), matching kind-crossplane, so ESO's property extraction round-trips. + PRIVATE_SUBNETS_JSON=$(echo "{{.PRIVATE_SUBNET_IDS}}" | tr ',' '\n' | grep -v '^$' | jq -R . | jq -s .) + VPC_JSON=$(jq -n --arg id "{{.VPC_ID}}" --argjson subnet_ids "$PRIVATE_SUBNETS_JSON" \ + --arg cluster_security_group_id "{{.CLUSTER_SG}}" \ + '{id:$id, subnet_ids:$subnet_ids, cluster_security_group_id:$cluster_security_group_id}' | jq -c .) SECRET_VALUE=$(jq -n \ --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}","exposure_mode":"{{if .CLOUDFRONT_DOMAIN}}cloudfront{{else}}domain{{end}}","admin_role_name":"{{.ADMIN_ROLE_NAME}}","gitlab_domain_name":"$GITLAB_CF_DOMAIN","git_token":"{{.USER_PASSWORD}}","aws_argocd_url":""}' \ --arg config '{"tlsClientConfig":{"insecure":false}}' \ --arg server '{{.CLUSTER_ARN}}' \ --arg addons '{"provider":"kro-ack"}' \ - --arg vpc '{{.VPC_ID}}' \ + --arg vpc "$VPC_JSON" \ '{metadata: $metadata, config: $config, server: $server, addons: $addons, vpc: $vpc}') aws secretsmanager restore-secret --secret-id "$SECRET_KEY" --region {{.AWS_REGION}} 2>/dev/null || true aws secretsmanager create-secret \ From a85b36318b530bc2e69b4bc4878486e2aa551074 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Fri, 3 Jul 2026 18:23:51 +0200 Subject: [PATCH 199/231] fix(crossplane-spokes): use PascalCase ArgoCDCapabilityRole for spoke access entry clusters-crossplane set argoCDRoleArn to -argocd-capability-role (kebab), but the crossplane hub's ArgoCD capability role is -ArgoCDCapabilityRole (PascalCase). The composition therefore created the spoke's argocd AccessEntry for a non-existent principal, so the hub ArgoCD (connecting as the real PascalCase role) failed with 'failed to verify the access entry' and could not load state / sync any app on the crossplane-provisioned spoke (spoke-dev). Correct to PascalCase; this appset only runs on the crossplane hub. --- gitops/bootstrap/clusters-crossplane.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gitops/bootstrap/clusters-crossplane.yaml b/gitops/bootstrap/clusters-crossplane.yaml index 88d6bdd77..7f03b14ba 100644 --- a/gitops/bootstrap/clusters-crossplane.yaml +++ b/gitops/bootstrap/clusters-crossplane.yaml @@ -62,7 +62,13 @@ spec: xrd: enabled: false global: - argoCDRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-argocd-capability-role' + # Crossplane hub's ArgoCD capability role is PascalCase (-ArgoCDCapabilityRole, + # per claims/argocd-capability-role.yaml external-name). Using the kebab name here made + # the composition create the spoke's argocd access entry for a NON-EXISTENT principal, + # so the hub ArgoCD (which connects as the PascalCase capability role) got + # "failed to verify the access entry" and could not manage the spoke. clusters-crossplane + # only runs on the crossplane hub, so PascalCase is always correct here. + argoCDRoleArn: 'arn:aws:iam::{{ .metadata.annotations.aws_account_id }}:role/{{ .metadata.annotations.aws_cluster_name }}-ArgoCDCapabilityRole' defaultRegion: '{{ .metadata.annotations.aws_region }}' valueFiles: - '$values/{{ .metadata.annotations.fleetRepoBasepath }}fleet/spoke-values/default/crossplane-clusters/values.yaml' From 9c5659a285d4b0bb5f93662211b9ff0b5c58adc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 4 Jul 2026 15:48:13 +0200 Subject: [PATCH 200/231] fix(eks-capabilities-rbac): add ACK S3 API group to kro/ack ClusterRole The eks-capabilities-kro ClusterRole granted ec2/eks/iam/ecr/secretsmanager/ dynamodb ACK groups but NOT s3.services.k8s.aws. Creating an S3 bucket via the Backstage ACK/KRO template (workshop module 10.6) was RBAC-denied for KRO/ACK on s3.services.k8s.aws. Add the S3 group (covers both spoke eks-capabilities-rbac and hub eks-capabilities-rbac-hub, same chart). --- .../addons/charts/eks-capabilities-rbac/templates/rbac.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml index 0916a5f43..200b263bd 100644 --- a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml +++ b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml @@ -76,6 +76,10 @@ rules: - apiGroups: ["dynamodb.services.k8s.aws"] resources: ["*"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# ACK S3 +- apiGroups: ["s3.services.k8s.aws"] + resources: ["*"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # ACK Services (IAMRoleSelector and other cross-service CRDs) - apiGroups: ["services.k8s.aws"] resources: ["*"] From 87cdbda745c43efb6f796f604a34b8d8f4c69058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Sat, 4 Jul 2026 23:21:11 +0200 Subject: [PATCH 201/231] fix(kro-ack): expand GITLAB_CF_DOMAIN in seed + use platform ingressClass Two workshop-breaking blockers found by kro-ack testing: #3 Backstage 'Invalid GitLab integration config, $GITLAB_CF_DOMAIN is not a valid host': the kro-ack secrets seed wrote the config metadata as a single-quoted jq --arg, so the bash var $GITLAB_CF_DOMAIN was NOT expanded and the cluster-secret gitlab_domain_name annotation held the literal '$GITLAB_CF_DOMAIN'. That flows to the Backstage gitlab_domain_name Helm value -> dynamic-catalog gitlab_hostname -> integrations.gitlab host, breaking EVERY Backstage template. Break out of the single quotes so the real GitLab CloudFront host is substituted (crossplane already did this via jq --arg). #4 kro AppmodService/RayService hardcoded ingressClassName: alb, but EKS Auto Mode provides the 'platform' IngressClass (controller eks.amazonaws.com/alb) and there is no 'alb' class -> every app Ingress failed with 'ingressClass alb not found', the ALB address never populated, curl to /hello-world etc. failed. Align appmod-service.yaml (x2) and ray-service.yaml to 'platform' (cicd-pipeline.yaml already used it). --- cluster-providers/kind-kro-ack/Taskfile.yaml | 2 +- .../charts/kro/resource-groups/manifests/appmod-service.yaml | 4 ++-- .../resource-groups/manifests/ray-service/ray-service.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index ca9317d67..cda152721 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -745,7 +745,7 @@ tasks: --arg cluster_security_group_id "{{.CLUSTER_SG}}" \ '{id:$id, subnet_ids:$subnet_ids, cluster_security_group_id:$cluster_security_group_id}' | jq -c .) SECRET_VALUE=$(jq -n \ - --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}","exposure_mode":"{{if .CLOUDFRONT_DOMAIN}}cloudfront{{else}}domain{{end}}","admin_role_name":"{{.ADMIN_ROLE_NAME}}","gitlab_domain_name":"$GITLAB_CF_DOMAIN","git_token":"{{.USER_PASSWORD}}","aws_argocd_url":""}' \ + --arg metadata '{"addonsRepoURL":"{{.REPO_URL}}","addonsRepoRevision":"{{.REPO_REVISION}}","addonsRepoBasepath":"{{.REPO_BASEPATH}}","fleetRepoURL":"{{.REPO_URL}}","fleetRepoRevision":"{{.REPO_REVISION}}","fleetRepoBasepath":"{{.REPO_BASEPATH}}","aws_region":"{{.AWS_REGION}}","aws_account_id":"{{.AWS_ACCOUNT_ID}}","aws_cluster_name":"{{.HUB_CLUSTER_NAME}}","aws_vpc_id":"{{.VPC_ID}}","alb_controller_mode":"oss","ingress_domain_name":"{{if .CLOUDFRONT_DOMAIN}}{{.CLOUDFRONT_DOMAIN}}{{else}}{{.DOMAIN}}{{end}}","ingress_name":"{{.HUB_CLUSTER_NAME}}-platform","ingress_security_groups":"{{.INGRESS_SECURITY_GROUPS}}","resource_prefix":"{{.RESOURCE_PREFIX}}","exposure_mode":"{{if .CLOUDFRONT_DOMAIN}}cloudfront{{else}}domain{{end}}","admin_role_name":"{{.ADMIN_ROLE_NAME}}","gitlab_domain_name":"'"$GITLAB_CF_DOMAIN"'","git_token":"{{.USER_PASSWORD}}","aws_argocd_url":""}' \ --arg config '{"tlsClientConfig":{"insecure":false}}' \ --arg server '{{.CLUSTER_ARN}}' \ --arg addons '{"provider":"kro-ack"}' \ diff --git a/gitops/addons/charts/kro/resource-groups/manifests/appmod-service.yaml b/gitops/addons/charts/kro/resource-groups/manifests/appmod-service.yaml index d7eef7a91..a47c94351 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/appmod-service.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/appmod-service.yaml @@ -536,7 +536,7 @@ spec: controller: false annotations: '${{"argocd.argoproj.io/tracking-id": schema.metadata.?annotations["argocd.argoproj.io/tracking-id"].orValue(""), "alb.ingress.kubernetes.io/scheme": "internet-facing", "alb.ingress.kubernetes.io/target-type": "ip", "alb.ingress.kubernetes.io/healthcheck-path": schema.spec.ingress.healthcheckPath, "alb.ingress.kubernetes.io/listen-ports": "[{\"HTTP\": 80}]", "alb.ingress.kubernetes.io/transforms." + schema.metadata.name: "[{\"type\":\"url-rewrite\",\"urlRewriteConfig\":{\"rewrites\":[{\"regex\":\"^" + schema.spec.ingress.path + "/?(.*)$\",\"replace\":\"/$1\"}]}}]"}}' spec: - ingressClassName: alb + ingressClassName: platform rules: - host: ${schema.spec.ingress.host} http: @@ -572,7 +572,7 @@ spec: alb.ingress.kubernetes.io/healthcheck-path: ${schema.spec.ingress.healthcheckPath} alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}]' spec: - ingressClassName: alb + ingressClassName: platform rules: - host: ${schema.spec.ingress.host} http: diff --git a/gitops/addons/charts/kro/resource-groups/manifests/ray-service/ray-service.yaml b/gitops/addons/charts/kro/resource-groups/manifests/ray-service/ray-service.yaml index f2b39a094..b560eb900 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/ray-service/ray-service.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/ray-service/ray-service.yaml @@ -603,7 +603,7 @@ spec: name: ${schema.metadata.name} uid: ${schema.metadata.uid} spec: - ingressClassName: alb + ingressClassName: platform rules: - host: ${schema.spec.ingressHostname} http: From 04c5f8371e107064aef17568f209b1b1a59b5159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 6 Jul 2026 14:30:28 +0200 Subject: [PATCH 202/231] fix(backstage/ingress/rbac): stop clobbering GitLab PAT; platform ingressClass; kro RBAC escalate Root cause of kro-ack tester blockers #1/#2/#3 (Backstage scaffolder + catalog + CI git-checkout all 401): secrets-manager:seed-observability re-seeded git_token = $USER_PASS on every run (jq merge), OVERWRITING the real glpat- PAT that gitlab:init-ec2 writes. Worse, the peeks-observability-seed systemd timer re-runs seed-observability every ~20 min, so even a corrected git_token got clobbered again. gitlab:init logs confirm it DID write the PAT (user1_pat=glpat-...; 'Updated peeks-hub/secrets.git_token with real GitLab PAT'), then seed-observability reverted it to the password (works for git basic-auth but GitLab rejects as PRIVATE-TOKEN -> 401 on all Backstage/CI API calls). Remove the git_token key from the seed-observability jq merge in BOTH providers; git_token is owned by gitlab:init (+ CFN GitTokenSeed placeholder, which now preserves glpat-). #5: KubeVela ingress + path-based-ingress traits defaulted class to 'alb', but EKS Auto Mode only has the 'platform' IngressClass -> ALB never provisioned. Default to 'platform'. #4: kro/ack capability ClusterRole could not create Roles/RoleBindings granting argoproj.io perms (CICDPipeline ERROR) due to k8s privilege-escalation prevention. Add escalate + bind verbs. --- cluster-providers/kind-crossplane/Taskfile.yaml | 3 +-- cluster-providers/kind-kro-ack/Taskfile.yaml | 3 +-- .../addons/charts/eks-capabilities-rbac/templates/rbac.yaml | 6 +++++- gitops/addons/charts/kubevela/templates/traits/ingress.yaml | 2 +- .../kubevela/templates/traits/path-based-ingress.yaml | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/cluster-providers/kind-crossplane/Taskfile.yaml b/cluster-providers/kind-crossplane/Taskfile.yaml index cd20f4fd0..43b68d15e 100644 --- a/cluster-providers/kind-crossplane/Taskfile.yaml +++ b/cluster-providers/kind-crossplane/Taskfile.yaml @@ -2024,11 +2024,10 @@ tasks: --arg grafana_api_key "$AMG_API_KEY" \ --arg grafana_url "$AMG_ENDPOINT" \ --arg user_password "$USER_PASS" \ - --arg git_token "$USER_PASS" \ --arg backstage_postgres_password "$BS_PG_PASS" \ --arg user_password_hash "$KARGO_HASH" \ --arg user_password_key "$KARGO_KEY" \ - '. + {amp_endpoint_url: $amp_query_url, amp_region: $amp_region, grafana_api_key: $grafana_api_key, grafana_url: $grafana_url, user_password: $user_password, git_token: $git_token, backstage_postgres_password: $backstage_postgres_password, user_password_hash: $user_password_hash, user_password_key: $user_password_key}') + '. + {amp_endpoint_url: $amp_query_url, amp_region: $amp_region, grafana_api_key: $grafana_api_key, grafana_url: $grafana_url, user_password: $user_password, backstage_postgres_password: $backstage_postgres_password, user_password_hash: $user_password_hash, user_password_key: $user_password_key}') # Generate the grafana_mysql_password the grafana-dashboards # ExternalSecret expects, but only if it isn't already there # (so re-runs don't rotate it out from under live consumers). diff --git a/cluster-providers/kind-kro-ack/Taskfile.yaml b/cluster-providers/kind-kro-ack/Taskfile.yaml index cda152721..38c21fdb4 100644 --- a/cluster-providers/kind-kro-ack/Taskfile.yaml +++ b/cluster-providers/kind-kro-ack/Taskfile.yaml @@ -1542,11 +1542,10 @@ tasks: --arg grafana_api_key "$AMG_API_KEY" \ --arg grafana_url "$AMG_ENDPOINT" \ --arg user_password "$USER_PASS" \ - --arg git_token "$USER_PASS" \ --arg backstage_postgres_password "$BS_PG_PASS" \ --arg user_password_hash "$KARGO_HASH" \ --arg user_password_key "$KARGO_KEY" \ - '. + {amp_endpoint_url: $amp_query_url, amp_region: $amp_region, grafana_api_key: $grafana_api_key, grafana_url: $grafana_url, user_password: $user_password, git_token: $git_token, backstage_postgres_password: $backstage_postgres_password, user_password_hash: $user_password_hash, user_password_key: $user_password_key}') + '. + {amp_endpoint_url: $amp_query_url, amp_region: $amp_region, grafana_api_key: $grafana_api_key, grafana_url: $grafana_url, user_password: $user_password, backstage_postgres_password: $backstage_postgres_password, user_password_hash: $user_password_hash, user_password_key: $user_password_key}') # Generate the grafana_mysql_password the grafana-dashboards # ExternalSecret expects, but only if it isn't already there # (so re-runs don't rotate it out from under live consumers). diff --git a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml index 200b263bd..0c1573b94 100644 --- a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml +++ b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml @@ -31,7 +31,11 @@ rules: # RBAC resources - apiGroups: ["rbac.authorization.k8s.io"] resources: ["roles", "rolebindings", "clusterroles", "clusterrolebindings"] - verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + # escalate + bind let kro/ack create Roles/RoleBindings that grant permissions for + # the resources an AppmodService/CICDPipeline renders (argoproj.io, etc.). Without + # them Kubernetes' privilege-escalation prevention blocks the capability from + # creating those Roles and the CICDPipeline goes ERROR. + verbs: ["get", "list", "watch", "create", "update", "patch", "delete", "escalate", "bind"] # Networking resources - apiGroups: ["networking.k8s.io"] resources: ["ingresses"] diff --git a/gitops/addons/charts/kubevela/templates/traits/ingress.yaml b/gitops/addons/charts/kubevela/templates/traits/ingress.yaml index efd3cc96a..0d8a2e149 100644 --- a/gitops/addons/charts/kubevela/templates/traits/ingress.yaml +++ b/gitops/addons/charts/kubevela/templates/traits/ingress.yaml @@ -18,7 +18,7 @@ spec: parameter: { domain: string http: [string]: int - class: *"alb" | string + class: *"platform" | string targetType: *"ip" | string type: *"internet-facing" | "internal" createService: *false | bool diff --git a/gitops/addons/charts/kubevela/templates/traits/path-based-ingress.yaml b/gitops/addons/charts/kubevela/templates/traits/path-based-ingress.yaml index c96ad98a0..ee456f3f6 100644 --- a/gitops/addons/charts/kubevela/templates/traits/path-based-ingress.yaml +++ b/gitops/addons/charts/kubevela/templates/traits/path-based-ingress.yaml @@ -18,7 +18,7 @@ spec: parameter: { domain: string http: [string]: int - class: *"alb" | string + class: *"platform" | string rewritePath: *true | bool createService: *false | bool } From 0e4d0a52871bb1293bc530261a903ef0f4033930 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 6 Jul 2026 15:13:55 +0200 Subject: [PATCH 203/231] fix(eks-capabilities-rbac): add pods/events, apps/deployments, workflowtaskresults Complete the kro/ack capability RBAC for CICDPipeline/AppmodService: the capability renders Deployments (apps), and the CI/CD workflows produce events/pods and Argo workflowtaskresults. Without these the pipeline Role/rendering was incomplete (tester saw missing events/pods/apps/deployments/workflowtaskresults alongside the escalate/bind gap). --- .../addons/charts/eks-capabilities-rbac/templates/rbac.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml index 0c1573b94..b149e3b70 100644 --- a/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml +++ b/gitops/addons/charts/eks-capabilities-rbac/templates/rbac.yaml @@ -22,7 +22,11 @@ metadata: rules: # Core Kubernetes resources - apiGroups: [""] - resources: ["services", "configmaps", "secrets", "serviceaccounts"] + resources: ["services", "configmaps", "secrets", "serviceaccounts", "events", "pods"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# Apps resources (Deployments/ReplicaSets rendered by AppmodService/CICDPipeline) +- apiGroups: ["apps"] + resources: ["deployments", "replicasets", "statefulsets"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # Batch resources - apiGroups: ["batch"] From 70ee218b9617b65901ed55f086ac06512992b76f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Allamand?= Date: Mon, 6 Jul 2026 17:01:33 +0200 Subject: [PATCH 204/231] fix(spokes): platform IngressClass = OSS LBC everywhere (+ #2 provider creds, #5 healthcheck) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The platform IngressClass must ALWAYS be backed by the OSS AWS Load Balancer Controller (ingress.k8s.aws/alb), never EKS Auto Mode's native controller (eks.amazonaws.com/alb): the workshop relies on LBC-only features (transforms URL-rewrite) not yet supported by Auto Mode. On spokes the ingress-class-alb chart rendered platform in 'auto' mode because alb_controller_mode:oss was only seeded on the hub, never propagated to spoke cluster secrets. - fleet-secret chart (crossplane/member spokes): add alb_controller_mode annotation (default oss via hub.albControllerMode). - rg-eks RGD (kro spokes): add alb_controller_mode:oss annotation + enable_aws_load_balancer_controller:true label (LBC role/policy/PIA already present in the RGD). - hub-config.yaml: align spokes to enable_ingress_class_alb:true + enable_aws_load_balancer_controller:true. - crossplane-base: PostSync hook to restart credential-less provider pods so they pick up their Pod Identity token once the PIA is active — spoke-side equivalent of hub:restart-identity-pods (#2 DynamoDB provider 'no IMDS role'). - kubevela ingress/path-based-ingress traits: healthcheckPath is now a configurable parameter (default '/'); apps serving only sub-paths override it so OSS LBC target health checks don't 404 (#5). - eks-capabilities-rbac: complete kro capability RBAC (workflowtaskresults). NOTE: IngressClass .spec.controller is immutable. Fresh events create platform as OSS directly. Existing events need a one-time 'kubectl delete ingressclass platform' per spoke so ArgoCD recreates it OSS. crossplane-pod-identity addon (not the composition) provisions the crossplane spoke LBC PIA. --- .../provider-credential-restart.yaml | 104 ++++++++++++++++++ .../addons/charts/crossplane-base/values.yaml | 5 + .../eks-capabilities-rbac/templates/rbac.yaml | 2 +- .../resource-groups/manifests/eks/rg-eks.yaml | 11 ++ .../kubevela/templates/traits/ingress.yaml | 4 + .../templates/traits/path-based-ingress.yaml | 7 +- .../templates/cluster-secret.yaml | 7 ++ platform-charts/fleet-secret/values.yaml | 4 + platform/infra/terraform/hub-config.yaml | 6 +- 9 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 gitops/addons/charts/crossplane-base/templates/provider-credential-restart.yaml diff --git a/gitops/addons/charts/crossplane-base/templates/provider-credential-restart.yaml b/gitops/addons/charts/crossplane-base/templates/provider-credential-restart.yaml new file mode 100644 index 000000000..dbb45edf8 --- /dev/null +++ b/gitops/addons/charts/crossplane-base/templates/provider-credential-restart.yaml @@ -0,0 +1,104 @@ +{{- /* + Provider Pod Identity credential bootstrap (#2). + + EKS Pod Identity injects credentials into a pod's environment via a webhook at + pod CREATION time only. The Crossplane AWS provider pods (provider-aws-dynamodb, + -rds, -ec2, ...) are started by the package manager at wave 3, but their + PodIdentityAssociations (iam.yaml, same chart) only become active a bit later. + A provider pod that started before its PIA was active runs credential-less + ("cannot retrieve the AWS credentials: no EC2 IMDS role found") and every + managed resource it owns (e.g. a DynamoDB Table) stays Synced=False forever. + + The hub fixes this with `task hub:restart-identity-pods`, but spokes are pure + GitOps with no Taskfile run against them. This PostSync hook is the spoke-side + equivalent: it deletes any provider pod that is missing the injected + AWS_CONTAINER_CREDENTIALS_FULL_URI env var so the Deployment recreates it and + the webhook injects the (now active) credentials. + + NOTE: the shared pod-identity-restart-hook cannot be reused here — it does + `kubectl rollout restart deployment -l