diff --git a/gitops/addons/charts/kro/resource-groups/examples/backup-policy/README.md b/gitops/addons/charts/kro/resource-groups/examples/backup-policy/README.md new file mode 100644 index 000000000..8b493e680 --- /dev/null +++ b/gitops/addons/charts/kro/resource-groups/examples/backup-policy/README.md @@ -0,0 +1,24 @@ +# BackupPolicy examples + +Sample `kro.run/v1alpha1` `BackupPolicy` instances for manual testing of the +RGD defined in `../manifests/backup-policy/`. + +These are **NOT** picked up by Argo CD (they live outside the `manifests/` +directory consumed by the `kro-manifests-hub` ApplicationSet). Apply them by +hand once the RGD is `Active` and the underlying ACK Backup CRDs +(`backup.services.k8s.aws/{BackupVault,BackupPlan,BackupSelection}`) exist on +the hub: + +```bash +kubectl --context hub apply -f instance-bronze.yaml +kubectl --context hub get backupvault,backupplan,backupselection -A +``` + +| File | Tier | Region scope | Use case | +|--------------------------|--------|-----------------------|----------------------| +| `instance-bronze.yaml` | bronze | same-region | dev / sandbox | +| `instance-silver.yaml` | silver | + cross-region copy | staging | +| `instance-gold.yaml` | gold | + cross-region copy | prod / DR coverage | + +Update `iamRoleARN` to the actual `AWSBackupDefaultServiceRole` ARN of the +target account before applying. diff --git a/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-bronze.yaml b/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-bronze.yaml new file mode 100644 index 000000000..7db72917a --- /dev/null +++ b/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-bronze.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kro.run/v1alpha1 +kind: BackupPolicy +metadata: + name: spoke-prod-euw1-bronze + namespace: default +spec: + tier: bronze + spokeName: spoke-prod-euw1 + region: eu-west-1 + # bronze = same-region only, no copyDestinationRegion / copyDestinationVaultARN + iamRoleARN: arn:aws:iam::586794472760:role/AWSBackupDefaultServiceRole diff --git a/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-gold.yaml b/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-gold.yaml new file mode 100644 index 000000000..b59998089 --- /dev/null +++ b/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-gold.yaml @@ -0,0 +1,15 @@ +--- +apiVersion: kro.run/v1alpha1 +kind: BackupPolicy +metadata: + name: spoke-prod-euw1-gold + namespace: default +spec: + tier: gold + spokeName: spoke-prod-euw1 + region: eu-west-1 + copyDestinationRegion: eu-west-3 + # Pre-created destination vault ARN in eu-west-3 for cross-region copy + copyDestinationVaultARN: arn:aws:backup:eu-west-3:000000000000:backup-vault:peeks-spoke-prod-euw1-gold-dr + # Pre-created AWS Backup service role + iamRoleARN: arn:aws:iam::000000000000:role/AWSBackupDefaultServiceRole diff --git a/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-silver.yaml b/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-silver.yaml new file mode 100644 index 000000000..5c9fd60d1 --- /dev/null +++ b/gitops/addons/charts/kro/resource-groups/examples/backup-policy/instance-silver.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: kro.run/v1alpha1 +kind: BackupPolicy +metadata: + name: spoke-prod-euw1-silver + namespace: default +spec: + tier: silver + spokeName: spoke-prod-euw1 + region: eu-west-1 + copyDestinationRegion: eu-west-3 + copyDestinationVaultARN: arn:aws:backup:eu-west-3:000000000000:backup-vault:peeks-spoke-prod-euw1-silver-dr + iamRoleARN: arn:aws:iam::000000000000:role/AWSBackupDefaultServiceRole diff --git a/gitops/addons/charts/kro/resource-groups/manifests/backup-policy/README.md b/gitops/addons/charts/kro/resource-groups/manifests/backup-policy/README.md new file mode 100644 index 000000000..1a89fa66d --- /dev/null +++ b/gitops/addons/charts/kro/resource-groups/manifests/backup-policy/README.md @@ -0,0 +1,138 @@ +# BackupPolicy RGD + +Kro `ResourceGraphDefinition` that exposes a single `BackupPolicy` custom resource +and expands it into an ACK-managed AWS Backup stack: + +- `BackupVault` (created in the policy's region) +- `BackupPlan` with tier-specific lifecycle & schedule +- `BackupSelection` that matches resources by AWS tag + +## Why AWS Backup in a GitOps platform + +The platform uses GitOps (Argo CD + kro + ACK) as the source of truth for the +**control plane** — cluster shape, capabilities, IAM, workload manifests — and +AWS Backup as the source of truth for the **data plane** — persistent volumes +and any Kubernetes kind that cannot be reconstructed from Git. + +This split enables three concrete use cases that share the same primitive: + +- **Disaster Recovery (cross-region)** — a spoke in `eu-west-1` is destroyed; + kro + ACK recreates an equivalent spoke in `eu-west-3` via the normal + `SpokeCluster` path, Argo CD re-syncs every workload from Git, and AWS Backup + restores the PVs from cross-region copies (produced by the `gold`/`silver` + plans defined here). The DR spoke is never "adopted post-facto" — it is born + through the standard provisioning path with its data rehydrated on top. + +- **Blue/green cluster migrations** — stand up a new spoke alongside the live + one, restore data from the latest backup into the new spoke, flip traffic + when workloads report ready, decommission the old spoke. Same mechanism as + DR, planned instead of reactive. + +- **Environment cloning** — clone a stateful environment (prod snapshot into a + staging/dev spoke) for testing, without a bespoke export/import pipeline. + AWS Backup restore jobs target the destination spoke in `existingCluster` + mode; Argo CD already deployed the empty workload shells. + +In all three cases, the `BackupPlan` and `BackupSelection` produced by this +RGD are what makes a snapshot available at the moment it is needed. Retention +and copy policy are chosen per-tier to match the recovery objective of each +workload class, not a one-size-fits-all vault. + +## Tiers + +| Tier | Schedule | RPO | Retention | Cold storage | Cross-region copy | +| ------ | ---------------------- | ---- | --------- | ------------ | ----------------- | +| gold | `cron(0 */1 * * ? *)` | 1 h | 30 d | after 90 d | yes | +| silver | `cron(0 2 * * ? *)` | 24 h | 14 d | no | yes | +| bronze | `cron(0 3 * * ? *)` | 24 h | 7 d | no | no | + +Gold and silver require `copyDestinationRegion` and `copyDestinationVaultARN` +(a BackupVault pre-created in the DR region). Bronze stays same-region — it +covers workloads whose recovery policy is "redeploy fresh if the region is +lost", not full geo-DR. + +## Usage + +The expected pattern is **one `BackupPolicy` instance per tier per spoke** +(3 instances max per spoke, typically declared by the `SpokeCluster` RGD so +the platform team doesn't hand-roll them). + +```yaml +apiVersion: kro.run/v1alpha1 +kind: BackupPolicy +metadata: + name: spoke-prod-euw1-bronze +spec: + tier: bronze + spokeName: spoke-prod-euw1 + region: eu-west-1 + iamRoleARN: arn:aws:iam:::role/AWSBackupDefaultServiceRole +``` + +See `../../examples/backup-policy/instance-{gold,silver,bronze}.yaml`. + +## Developer contract + +A workload opts into backup by putting the right tags on its resources: + +- `peeks.io/backup-tier=` (gold | silver | bronze) +- `peeks.io/spoke=` + +The recommended way on EKS is to label the namespace or the PVC; the EBS CSI +driver propagates those tags onto the underlying EBS volumes, and the +`BackupSelection.conditions.stringEquals` block matches them. No central +ConfigMap or platform-team ticket needed — opt-in is visible in the app's Git +manifests and auditable with a single `kubectl get ns -l peeks.io/backup-tier=`. + +Included resource ARNs (for the selection): + +- `arn:aws:ec2:*:*:volume/*` — EBS volumes +- `arn:aws:eks:*:*:cluster/*` — EKS add-on backup + +## What this RGD does NOT cover + +Documented explicitly because the boundary matters for DR correctness: + +- **Secrets** — handled by External Secrets Operator (source of truth: AWS + Secrets Manager, replicated cross-region). Never restored from a K8s-level + snapshot. +- **Anything derivable from Git** — Deployments, Services, ConfigMaps, + Ingresses, PVC templates. Argo CD re-syncs them on the recovery spoke. +- **Heavy stateful workloads (Postgres, Kafka)** — AWS Backup gives a + crash-consistent filesystem snapshot, not a logically consistent database + backup. Those workloads need an application-level backup on top + (`pg_basebackup` + WAL archiving to S3, Kafka tiered storage, etc.) — see + task 2.2 in the DR plan. +- **PVC restore targeting** — the actual mechanism that attaches restored + volumes to the new pods in an `existingCluster` restore is handled by the + companion `RestoreSelection` RGD (same DR kit, separate concern). This RGD + only produces the backups; restoring them lives next to this directory. + +## Validation + +```sh +# syntax +yq eval '.' rgd-backup-policy.yaml + +# kro server-side validation +kubectl --context hub apply --dry-run=server -f rgd-backup-policy.yaml + +# apply & check +kubectl --context hub apply -f rgd-backup-policy.yaml +kubectl --context hub get rgd backuppolicy.kro.run + +# end-to-end: instance expands into BackupVault + BackupPlan + BackupSelection +kubectl --context hub apply -f ../../examples/backup-policy/instance-bronze.yaml +kubectl --context hub get backupvault,backupplan,backupselection -A +``` + +## Known limits (v1) + +- No KMS key rotation / custom encryption — uses the default vault key. +- Heavy PVCs (e.g. Postgres on large EBS) need an app-level snapshot workflow + in addition to AWS Backup. +- `copyDestinationVaultARN` must be pre-provisioned in the DR region. A future + iteration could nest that vault in the same RGD. +- Restore side (how the PVs land in a target cluster) is the responsibility of + the companion `RestoreSelection` RGD — shipped alongside this one as part of + the same DR kit. diff --git a/gitops/addons/charts/kro/resource-groups/manifests/backup-policy/rgd-backup-policy.yaml b/gitops/addons/charts/kro/resource-groups/manifests/backup-policy/rgd-backup-policy.yaml new file mode 100644 index 000000000..e267bf44d --- /dev/null +++ b/gitops/addons/charts/kro/resource-groups/manifests/backup-policy/rgd-backup-policy.yaml @@ -0,0 +1,236 @@ +# yamllint disable rule:line-length +--- +apiVersion: kro.run/v1alpha1 +kind: ResourceGraphDefinition +metadata: + name: backuppolicy.kro.run + annotations: + argocd.argoproj.io/sync-wave: "0" + argocd.argoproj.io/sync-options: SkipDryRunOnMissingResource=true +spec: + schema: + apiVersion: v1alpha1 + kind: BackupPolicy + spec: + # Tier of the backup policy: gold | silver | bronze + tier: string | required=true enum="gold,silver,bronze" + # Name of the spoke cluster this policy targets (used for naming & tag matching) + spokeName: string | required=true + # AWS region where the BackupVault / BackupPlan are created + region: string | required=true + # Name of the BackupVault (defaults to peeks--) + backupVaultName: string | default="" + # Destination region for cross-region copy (required for gold & silver) + copyDestinationRegion: string | default="" + # ARN of the destination BackupVault (required for gold & silver cross-region copy) + copyDestinationVaultARN: string | default="" + # ARN of the IAM role used by AWS Backup for BackupSelection + iamRoleARN: string | required=true + status: + vaultStatus: ${backupvault.status.conditions} + resources: + # ----------------------------------------------------------------------- + # BackupVault - created for every tier + # ----------------------------------------------------------------------- + - id: backupvault + readyWhen: + - ${backupvault.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: backup.services.k8s.aws/v1alpha1 + kind: BackupVault + metadata: + name: '${schema.spec.backupVaultName != "" ? schema.spec.backupVaultName : "peeks-" + schema.spec.spokeName + "-" + schema.spec.tier}' + spec: + name: '${schema.spec.backupVaultName != "" ? schema.spec.backupVaultName : "peeks-" + schema.spec.spokeName + "-" + schema.spec.tier}' + tags: + peeks-dr-test: "true" + peeks.io/spoke: ${schema.spec.spokeName} + peeks.io/tier: ${schema.spec.tier} + + # ----------------------------------------------------------------------- + # BackupPlan - GOLD tier: RPO 1h, retention 30d, cold 90d, cross-region copy + # ----------------------------------------------------------------------- + - id: backupplangold + includeWhen: + - ${schema.spec.tier == "gold"} + readyWhen: + - ${backupplangold.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: backup.services.k8s.aws/v1alpha1 + kind: BackupPlan + metadata: + name: peeks-${schema.spec.spokeName}-gold + spec: + name: peeks-${schema.spec.spokeName}-gold + tags: + peeks-dr-test: "true" + peeks.io/spoke: ${schema.spec.spokeName} + peeks.io/tier: gold + rules: + - ruleName: gold-hourly + targetBackupVaultName: ${backupvault.spec.name} + scheduleExpression: cron(0 */1 * * ? *) + startWindowMinutes: 60 + completionWindowMinutes: 180 + lifecycle: + deleteAfterDays: 30 + moveToColdStorageAfterDays: 90 + copyActions: + - destinationBackupVaultARN: ${schema.spec.copyDestinationVaultARN} + lifecycle: + deleteAfterDays: 30 + moveToColdStorageAfterDays: 90 + recoveryPointTags: + peeks-dr-test: "true" + peeks.io/spoke: ${schema.spec.spokeName} + peeks.io/tier: gold + + # ----------------------------------------------------------------------- + # BackupPlan - SILVER tier: RPO 24h, retention 14d, cross-region copy + # ----------------------------------------------------------------------- + - id: backupplansilver + includeWhen: + - ${schema.spec.tier == "silver"} + readyWhen: + - ${backupplansilver.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: backup.services.k8s.aws/v1alpha1 + kind: BackupPlan + metadata: + name: peeks-${schema.spec.spokeName}-silver + spec: + name: peeks-${schema.spec.spokeName}-silver + tags: + peeks-dr-test: "true" + peeks.io/spoke: ${schema.spec.spokeName} + peeks.io/tier: silver + rules: + - ruleName: silver-daily + targetBackupVaultName: ${backupvault.spec.name} + scheduleExpression: cron(0 2 * * ? *) + startWindowMinutes: 60 + completionWindowMinutes: 360 + lifecycle: + deleteAfterDays: 14 + copyActions: + - destinationBackupVaultARN: ${schema.spec.copyDestinationVaultARN} + lifecycle: + deleteAfterDays: 14 + recoveryPointTags: + peeks-dr-test: "true" + peeks.io/spoke: ${schema.spec.spokeName} + peeks.io/tier: silver + + # ----------------------------------------------------------------------- + # BackupPlan - BRONZE tier: RPO 24h, retention 7d, same-region only + # ----------------------------------------------------------------------- + - id: backupplanbronze + includeWhen: + - ${schema.spec.tier == "bronze"} + readyWhen: + - ${backupplanbronze.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: backup.services.k8s.aws/v1alpha1 + kind: BackupPlan + metadata: + name: peeks-${schema.spec.spokeName}-bronze + spec: + name: peeks-${schema.spec.spokeName}-bronze + tags: + peeks-dr-test: "true" + peeks.io/spoke: ${schema.spec.spokeName} + peeks.io/tier: bronze + rules: + - ruleName: bronze-daily + targetBackupVaultName: ${backupvault.spec.name} + scheduleExpression: cron(0 3 * * ? *) + startWindowMinutes: 60 + completionWindowMinutes: 360 + lifecycle: + deleteAfterDays: 7 + recoveryPointTags: + peeks-dr-test: "true" + peeks.io/spoke: ${schema.spec.spokeName} + peeks.io/tier: bronze + + # ----------------------------------------------------------------------- + # BackupSelection - one per tier (variant activated via includeWhen) + # Matches resources tagged with peeks.io/backup-tier + peeks.io/spoke. + # ----------------------------------------------------------------------- + - id: backupselectiongold + includeWhen: + - ${schema.spec.tier == "gold"} + readyWhen: + - ${backupselectiongold.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: backup.services.k8s.aws/v1alpha1 + kind: BackupSelection + metadata: + name: peeks-${schema.spec.spokeName}-gold-sel + spec: + name: peeks-${schema.spec.spokeName}-gold-sel + backupPlanRef: + from: + name: peeks-${schema.spec.spokeName}-gold + iamRoleARN: ${schema.spec.iamRoleARN} + resources: + - "arn:aws:ec2:*:*:volume/*" + - "arn:aws:eks:*:*:cluster/*" + conditions: + stringEquals: + - conditionKey: "aws:ResourceTag/peeks.io/backup-tier" + conditionValue: gold + - conditionKey: "aws:ResourceTag/peeks.io/spoke" + conditionValue: ${schema.spec.spokeName} + + - id: backupselectionsilver + includeWhen: + - ${schema.spec.tier == "silver"} + readyWhen: + - ${backupselectionsilver.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: backup.services.k8s.aws/v1alpha1 + kind: BackupSelection + metadata: + name: peeks-${schema.spec.spokeName}-silver-sel + spec: + name: peeks-${schema.spec.spokeName}-silver-sel + backupPlanRef: + from: + name: peeks-${schema.spec.spokeName}-silver + iamRoleARN: ${schema.spec.iamRoleARN} + resources: + - "arn:aws:ec2:*:*:volume/*" + - "arn:aws:eks:*:*:cluster/*" + conditions: + stringEquals: + - conditionKey: "aws:ResourceTag/peeks.io/backup-tier" + conditionValue: silver + - conditionKey: "aws:ResourceTag/peeks.io/spoke" + conditionValue: ${schema.spec.spokeName} + + - id: backupselectionbronze + includeWhen: + - ${schema.spec.tier == "bronze"} + readyWhen: + - ${backupselectionbronze.status.conditions.exists(x, x.type == 'ACK.ResourceSynced' && x.status == "True")} + template: + apiVersion: backup.services.k8s.aws/v1alpha1 + kind: BackupSelection + metadata: + name: peeks-${schema.spec.spokeName}-bronze-sel + spec: + name: peeks-${schema.spec.spokeName}-bronze-sel + backupPlanRef: + from: + name: peeks-${schema.spec.spokeName}-bronze + iamRoleARN: ${schema.spec.iamRoleARN} + resources: + - "arn:aws:ec2:*:*:volume/*" + - "arn:aws:eks:*:*:cluster/*" + conditions: + stringEquals: + - conditionKey: "aws:ResourceTag/peeks.io/backup-tier" + conditionValue: bronze + - conditionKey: "aws:ResourceTag/peeks.io/spoke" + conditionValue: ${schema.spec.spokeName} diff --git a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml index 062b2168f..32d6b20d3 100644 --- a/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml +++ b/gitops/addons/charts/kro/resource-groups/manifests/eks/rg-eks.yaml @@ -1172,6 +1172,42 @@ spec: - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryFullAccess - arn:aws:iam::aws:policy/AmazonS3FullAccess - arn:aws:iam::aws:policy/AmazonDynamoDBFullAccess + inlinePolicies: + ManagePodIdentityAssociations: | + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "PassWorkloadRolesToEKSPodIdentity", + "Effect": "Allow", + "Action": ["iam:PassRole"], + "Resource": ["arn:aws:iam::${schema.spec.accountId}:role/${schema.spec.name}-*"], + "Condition": { + "StringEquals": { + "iam:PassedToService": "pods.eks.amazonaws.com" + } + } + }, + { + "Sid": "ManagePodIdentityAssociations", + "Effect": "Allow", + "Action": [ + "eks:CreatePodIdentityAssociation", + "eks:DeletePodIdentityAssociation", + "eks:DescribePodIdentityAssociation", + "eks:ListPodIdentityAssociations", + "eks:UpdatePodIdentityAssociation", + "eks:TagResource", + "eks:UntagResource", + "eks:ListTagsForResource" + ], + "Resource": [ + "arn:aws:eks:*:${schema.spec.accountId}:cluster/${schema.spec.name}", + "arn:aws:eks:*:${schema.spec.accountId}:podidentityassociation/${schema.spec.name}/*" + ] + } + ] + } assumeRolePolicyDocument: | { "Version": "2012-10-17", diff --git a/gitops/addons/charts/multi-acct/templates/eks-capabilities-rbac.yaml b/gitops/addons/charts/multi-acct/templates/eks-capabilities-rbac.yaml index 840ae1f96..a47a1499e 100644 --- a/gitops/addons/charts/multi-acct/templates/eks-capabilities-rbac.yaml +++ b/gitops/addons/charts/multi-acct/templates/eks-capabilities-rbac.yaml @@ -47,6 +47,10 @@ rules: - apiGroups: ["ecr.services.k8s.aws"] resources: ["*"] verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +# ACK Backup (AWS Backup: BackupVault, BackupPlan, BackupSelection) +- apiGroups: ["backup.services.k8s.aws"] + resources: ["*"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] # Kro custom resources - apiGroups: ["kro.run"] resources: ["*"] @@ -61,6 +65,20 @@ roleRef: kind: ClusterRole name: eks-capabilities-kro subjects: +# Native EKS capability identity (used for most internal calls). - apiGroup: rbac.authorization.k8s.io kind: User name: capabilities.eks.amazonaws.com +{{- /* + When the KRO capability assumes its IAM role to act on K8s resources + (e.g. reading iam.services.k8s.aws/Policy while reconciling a RGD that + declares Policy/Role children, like the `podidentity` RGD), the API server + receives the STS username, not the synthetic `capabilities.eks.amazonaws.com`. + We must bind that username explicitly. Wildcards are not supported in RBAC + subjects, so we render the literal ARN. +*/ -}} +{{- if and .Values.global (and .Values.global.accountId .Values.global.resourcePrefix .Values.global.clusterName) }} +- apiGroup: rbac.authorization.k8s.io + kind: User + name: "arn:aws:sts::{{ .Values.global.accountId }}:assumed-role/{{ .Values.global.resourcePrefix }}-{{ .Values.global.clusterName }}-kro-capability-role/KRO" +{{- end }} diff --git a/gitops/addons/charts/platform-manifests-bootstrap/README-storageclasses.md b/gitops/addons/charts/platform-manifests-bootstrap/README-storageclasses.md new file mode 100644 index 000000000..2738d3c26 --- /dev/null +++ b/gitops/addons/charts/platform-manifests-bootstrap/README-storageclasses.md @@ -0,0 +1,85 @@ +# storageclass-resources + +Helm chart that provisions cluster `StorageClass` resources, both EFS and EBS. + +## What's new — Peeks DR-aware EBS StorageClasses + +Three EBS StorageClasses ship by default, one per backup tier: + +| StorageClass | Tier | RPO | Retention | Cross-region copy | +|---|---|---|---|---| +| `peeks-gold-gp3` | gold | 1 h | 30 d | yes | +| `peeks-silver-gp3` | silver | 24 h | 14 d | yes | +| `peeks-bronze-gp3` | bronze | 24 h | 7 d | no | + +Every EBS volume provisioned by one of these SCs inherits, as an AWS resource +tag: + +- `peeks.io/backup-tier=` +- `peeks.io/spoke=` (from the `spokeName` Helm value) +- `peeks.io/managed-by=peeks-platform` + +These tags are what the `BackupSelection` resource produced by the companion +`BackupPolicy` RGD matches on — so a workload opts into a backup tier simply +by picking the right StorageClass in its PVC manifest. No central ConfigMap, +no platform ticket. + +### Developer contract + +```yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pgdata +spec: + storageClassName: peeks-gold-gp3 # <-- declares the backup tier + accessModes: [ReadWriteOnce] + resources: + requests: + storage: 20Gi +``` + +That's it. The EBS CSI driver stamps the volume with the three tags at +provisioning time, the tier-level `BackupSelection` includes the volume in +its next snapshot cycle, and everything downstream (cross-region copy, +restore targeting) follows from there. + +### Why not "just label the namespace"? + +The EBS CSI driver's `tagSpecification_` parameter only expands the +tokens `${pvc.name}`, `${pvc.namespace}`, and `${pv.name}` — it does **not** +propagate arbitrary namespace or PVC labels onto the underlying AWS tags. +A mutating admission webhook could watch namespaces and inject the right +StorageClass or rewrite tags on the fly, but that's an extra piece of +infrastructure to operate. + +The per-SC strategy is the explicit contract in v1: it's visible in every +PVC manifest, requires no custom controllers, and reviewable in a pull +request. + +### Custom tags per StorageClass + +Any EBS SC can carry arbitrary tags via the `tags:` map: + +```yaml +storageClasses: + ebs: + my-custom-sc: + volumeType: gp3 + reclaimPolicy: Retain + tags: + cost-center: "1234" + env: prod +``` + +These are rendered as `tagSpecification_` on the SC's `parameters`. +The `peeks.io/spoke` tag from `spokeName` is merged in automatically. + +## Usage + +```sh +helm template . --set spokeName=spoke-prod-euw1 | kubectl apply -f - +``` + +Or, in a gitops context, set `spokeName` via the Argo CD ApplicationSet +parameter so each spoke gets its own tag value. diff --git a/gitops/addons/charts/platform-manifests-bootstrap/templates/peeks-storageclass.yaml b/gitops/addons/charts/platform-manifests-bootstrap/templates/peeks-storageclass.yaml new file mode 100644 index 000000000..45675e48e --- /dev/null +++ b/gitops/addons/charts/platform-manifests-bootstrap/templates/peeks-storageclass.yaml @@ -0,0 +1,57 @@ +{{- $fileSystemId := "" -}} +{{- if .Values.storageClasses.efs }} + {{- $fileSystemId = .Values.storageClasses.efs.fileSystemId | default "" -}} +{{- end }} + +{{- range $storageClassType, $storageClasses := .Values.storageClasses }} + {{- range $storageClassName, $storageClass := $storageClasses }} + {{- if ne $storageClassName "fileSystemId" }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ $storageClassName }} + annotations: + storageclass.kubernetes.io/is-default-class: "false" +provisioner: {{ if eq $storageClassType "efs" }}efs.csi.aws.com{{ else }}ebs.csi.aws.com{{ end }} +{{- if and (eq $storageClassType "efs") $fileSystemId }} +parameters: + fileSystemId: {{ $fileSystemId }} + directoryPerms: "{{ $storageClass.directoryPerms | default "700" }}" + provisioningMode: {{ $storageClass.provisioningMode | default "efs-ap" }} + basePath: {{ $storageClass.basePath | default "/" }} +mountOptions: +{{- range $storageClass.mountOptions }} + - {{ . }} +{{- end }} +{{- else if eq $storageClassType "ebs" }} +parameters: + type: {{ $storageClass.volumeType }} + fsType: ext4 + iopsPerGiB: "{{ $storageClass.iops | default "3000" }}" + throughput: "{{ $storageClass.throughput | default "125" }}" +{{- /* + Tag propagation: any tag declared under a StorageClass's `tags:` map is + rendered as a `tagSpecification_` parameter. The EBS CSI driver + translates these into AWS tags applied to every volume provisioned by + this StorageClass. Combined with `peeks.io/backup-tier` and + `peeks.io/spoke` tags, this lets an AWS Backup `BackupSelection` + match volumes by tag without any central ConfigMap. +*/}} +{{- $defaultTags := dict }} +{{- if $.Values.spokeName }} + {{- $_ := set $defaultTags "peeks.io/spoke" $.Values.spokeName }} +{{- end }} +{{- $scTags := merge (dict) ($storageClass.tags | default dict) $defaultTags }} +{{- $idx := 1 }} +{{- range $tagKey, $tagValue := $scTags }} + tagSpecification_{{ $idx }}: "{{ $tagKey }}={{ $tagValue }}" + {{- $idx = add $idx 1 }} +{{- end }} +{{- end }} +reclaimPolicy: {{ $storageClass.reclaimPolicy | default "Delete" }} +allowVolumeExpansion: true +volumeBindingMode: WaitForFirstConsumer +--- + {{- end }} + {{- end }} +{{- end }} diff --git a/gitops/addons/charts/platform-manifests-bootstrap/values.yaml b/gitops/addons/charts/platform-manifests-bootstrap/values.yaml new file mode 100644 index 000000000..b84328560 --- /dev/null +++ b/gitops/addons/charts/platform-manifests-bootstrap/values.yaml @@ -0,0 +1,82 @@ +# Defaults for nodepool toggles (consumed by gpu-nodepool.yaml, custom-nodepools.yaml). +# These are normally overridden by the ApplicationSet at deploy time. +gpu: + enabled: false +customNodepools: + enabled: false + +# Optional `spokeName` is rendered as a `peeks.io/spoke=` AWS tag +# on every volume provisioned by the EBS StorageClasses below. Useful for +# cost attribution and inventory. +# +# spokeName: spoke-prod-euw1 + +storageClasses: + # efs: + # fileSystemId: fs-12345678 + # efs-sc: + # reclaimPolicy: Delete + # directoryPerms: "700" + # basePath: /data + # mountOptions: + # - nfsvers=4.1 + + ebs: + # Legacy default — unchanged, kept for backwards compatibility. + ebs-sc-gp3: + reclaimPolicy: Retain + volumeType: gp3 + size: 20Gi + iops: 3000 + throughput: 125 + + # PeEKS default StorageClass — performance-tuned, no backup tier. + # Workloads that need AWS Backup coverage should use one of the + # tier-labeled StorageClasses below (peeks-gold-gp3 / peeks-silver-gp3 / + # peeks-bronze-gp3) so the volume is tagged with `peeks.io/backup-tier`, + # which the BackupSelection in the BackupPolicy RGD matches on. + peeks-gp3: + reclaimPolicy: Retain + volumeType: gp3 + iops: 3000 + throughput: 125 + tags: + peeks.io/managed-by: peeks-platform + + # PeEKS DR-aware StorageClasses — workloads opt into a backup tier by + # picking the matching SC in their PVC manifest. The `peeks.io/backup-tier` + # tag is propagated to the EBS volume by the EBS CSI driver and is what + # the BackupSelection (BackupPolicy RGD) uses to gate inclusion in a + # BackupPlan. No central ConfigMap, no platform ticket — the workload + # author chooses its own tier at PVC creation time. + # + # Tier policies (defined in the BackupPolicy RGD): + # gold — RPO 1h, retention 30d, cross-region copy + # silver — RPO 24h, retention 14d, cross-region copy + # bronze — RPO 24h, retention 7d, single-region + peeks-gold-gp3: + reclaimPolicy: Retain + volumeType: gp3 + iops: 3000 + throughput: 125 + tags: + peeks.io/managed-by: peeks-platform + peeks.io/backup-tier: gold + + peeks-silver-gp3: + reclaimPolicy: Retain + volumeType: gp3 + iops: 3000 + throughput: 125 + tags: + peeks.io/managed-by: peeks-platform + peeks.io/backup-tier: silver + + peeks-bronze-gp3: + reclaimPolicy: Delete + volumeType: gp3 + iops: 3000 + throughput: 125 + tags: + peeks.io/managed-by: peeks-platform + peeks.io/backup-tier: bronze diff --git a/gitops/addons/registry/platform.yaml b/gitops/addons/registry/platform.yaml index a9bb0111c..690802275 100644 --- a/gitops/addons/registry/platform.yaml +++ b/gitops/addons/registry/platform.yaml @@ -187,6 +187,8 @@ multi-acct: valuesObject: global: clusterName: '{{.metadata.annotations.aws_cluster_name}}' + accountId: '{{.metadata.annotations.aws_account_id}}' + resourcePrefix: '{{.metadata.annotations.resource_prefix}}' kubevela: namespace: vela-system @@ -314,6 +316,65 @@ ack-iam: jsonPointers: - /metadata/annotations/controller-gen.kubebuilder.io~1version +ack-backup: + # ack-backup-controller is in PREVIEW (v0.1.1) and is NOT yet bundled in + # the EKS 'ACK' capability as of May 2026. It is wired here as a regular + # Helm addon and gets its IAM via the kro+ACK pod-identity path + # (enableACKKRO=true), instead of relying on the capability role. + namespace: ack-system + chartName: backup-chart + chartNamespace: aws-controllers-k8s + chartRepository: public.ecr.aws + defaultVersion: '0.1.1' + enableACKKRO: true + annotationsAppSet: + argocd.argoproj.io/sync-wave: '-1' + selector: + matchExpressions: + - key: enable_ack_backup + operator: In + values: ['true'] + valuesObject: + aws: + region: '{{.metadata.annotations.aws_region}}' + serviceAccount: + name: ack-backup-controller + podIdentity: + piNamespace: ack-system + serviceAccounts: + - ack-backup-controller + # Least-privilege policy for the backup-controller. iam:PassRole is + # scoped to role/* (any role in the account) so the controller can + # pass backup service roles regardless of naming conventions. + policyDocument: + - actions: + - backup:* + - backup-storage:* + resourceName: '*' + - actions: + - iam:PassRole + customArn: arn:aws:iam::*:role/* + - actions: + - kms:Decrypt + - kms:Encrypt + - kms:GenerateDataKey + - kms:GenerateDataKeyWithoutPlaintext + - kms:DescribeKey + - kms:CreateGrant + resourceName: '*' + ignoreDifferences: + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + jsonPointers: + - /metadata/annotations/controller-gen.kubebuilder.io~1version + jqPathExpressions: + - '.spec.versions[].schema.openAPIV3Schema | .. | .generateName?' + - '.spec.versions[].schema.openAPIV3Schema | .. | .description?' + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + managedFieldsManagers: + - kube-controller-manager + ack-eks: namespace: ack-system chartName: eks-chart diff --git a/platform-charts/appset-chart/templates/_pod_identity_kro.tpl b/platform-charts/appset-chart/templates/_pod_identity_kro.tpl new file mode 100644 index 000000000..319fc77ea --- /dev/null +++ b/platform-charts/appset-chart/templates/_pod_identity_kro.tpl @@ -0,0 +1,32 @@ +{{/* +Template to generate pod-identity configuration via kro + ACK. +Uses the kro-pi-instance chart (gitops/addons/charts/kro/instances/pod-identity) +which expands to ACK iam.Policy + iam.Role + eks.PodIdentityAssociation via the +RGD podidentity.kro.run (gitops/addons/charts/kro/resource-groups/manifests/pod-identity). + +Adapted from upstream aws-samples/appmod-blueprints PR #631 to fit the +appset-chart structure on the feature/platform-cluster-kro-ack base (PR #642). +*/}} +{{- define "application-sets.pod-identity-kro" -}} +{{- $chartName := .chartName -}} +{{- $chartConfig := .chartConfig -}} +{{- $valueFiles := .valueFiles -}} +{{- $values := .values -}} +{{- $pi := $chartConfig.podIdentity -}} +- repoURL: '{{ $values.repoURLGit }}' + targetRevision: '{{ $values.repoURLGitRevision }}' + path: 'gitops/addons/charts/kro/instances/pod-identity' + helm: + releaseName: '{{`{{ .name }}`}}-{{ $chartConfig.chartName | default $chartName }}-pi' + valuesObject: + name: '{{`{{ .name }}`}}-{{ $chartConfig.chartName | default $chartName }}' + clusterName: '{{`{{ .name }}`}}' + region: '{{`{{ .metadata.annotations.aws_region }}`}}' + accountId: '{{`{{ .metadata.annotations.aws_account_id }}`}}' + piNamespace: {{ default $chartConfig.namespace $pi.piNamespace }} + serviceAccounts: + {{- toYaml $pi.serviceAccounts | nindent 6 }} + policyDocument: + {{- toYaml $pi.policyDocument | nindent 6 }} + ignoreMissingValueFiles: true +{{- end }} diff --git a/platform-charts/appset-chart/templates/application-set.yaml b/platform-charts/appset-chart/templates/application-set.yaml index 9145b7219..7bd52d811 100644 --- a/platform-charts/appset-chart/templates/application-set.yaml +++ b/platform-charts/appset-chart/templates/application-set.yaml @@ -130,12 +130,21 @@ spec: - repoURL: {{ $repoURLGit | squote}} targetRevision: {{ $repoURLGitRevision | squote }} ref: values + {{- if and (eq (toString $chartConfig.enableAckPodIdentity) "true") (eq (toString $chartConfig.enableACKKRO) "true") }} + {{- fail (printf "Addon %s: enableAckPodIdentity (Crossplane path) and enableACKKRO (kro+ACK path) are mutually exclusive. Pick one." $nameNormalize) }} + {{- end }} {{- if eq (toString $chartConfig.enableAckPodIdentity) "true" }} {{ include "application-sets.pod-identity" (dict "chartName" ($chartConfig.chartName | default $nameNormalize) "valueFiles" $valueFiles "chartConfig" $chartConfig "values" $values ) | nindent 6 }} {{- end }} + {{- if eq (toString $chartConfig.enableACKKRO) "true" }} + {{ include "application-sets.pod-identity-kro" (dict + "chartName" ($chartConfig.chartName | default $nameNormalize) + "valueFiles" $valueFiles + "chartConfig" $chartConfig "values" $values ) | nindent 6 }} + {{- end }} {{- if $chartConfig.path }} - repoURL: {{ $repoURLGit | squote }} path: {{$chartConfig.path | squote }} diff --git a/platform/infra/terraform/hub-config.yaml b/platform/infra/terraform/hub-config.yaml index 2b07e69fa..58726bb00 100644 --- a/platform/infra/terraform/hub-config.yaml +++ b/platform/infra/terraform/hub-config.yaml @@ -39,6 +39,7 @@ clusters: enable_ack_efs: false # EKS Managed capability enable_ack_s3: false # EKS Managed capability enable_ack_dynamodb: false # EKS Managed capability + enable_ack_backup: true # PREVIEW (v0.1.1), wired via kro+ACK pod-identity enable_crossplane: true enable_crossplane_aws: true enable_kro: false # EKS Managed capability @@ -86,6 +87,7 @@ clusters: enable_ack_s3: false # EKS Managed capability enable_ack_efs: false # EKS Managed capability enable_ack_dynamodb: false # EKS Managed capability + enable_ack_backup: false # not needed on dev spoke enable_crossplane: true enable_crossplane_aws: true enable_external_secrets: true @@ -135,6 +137,7 @@ clusters: enable_ack_efs: false # EKS Managed capability enable_ack_s3: false # EKS Managed capability enable_ack_dynamodb: false # EKS Managed capability + enable_ack_backup: false # not needed on prod spoke enable_kro: false # EKS Managed capability enable_kro_manifests: true enable_crossplane: true