diff --git a/pkg/argocd/foundational.go b/pkg/argocd/foundational.go index b58b05ff..b856b4ee 100644 --- a/pkg/argocd/foundational.go +++ b/pkg/argocd/foundational.go @@ -32,6 +32,14 @@ const ( // NebariFoundationalPartOf is the value of the app.kubernetes.io/part-of label for foundational resources. NebariFoundationalPartOf = "nebari-foundational" + + // labelPartOf and labelManagedBy are the standard Kubernetes recommended + // label keys applied to foundational resources NIC creates. + labelPartOf = "app.kubernetes.io/part-of" + labelManagedBy = "app.kubernetes.io/managed-by" + + // labelManagedByValue marks resources as managed by NIC. + labelManagedByValue = "nebari-infrastructure-core" ) // FoundationalConfig holds configuration for foundational services @@ -140,6 +148,23 @@ func InstallFoundationalServices(ctx context.Context, cfg *config.NebariConfig, return fmt.Errorf("failed to create Keycloak secrets: %w", err) } + // Render the embedded keycloak-config-cli defaults and write them + // to the in-cluster Secret the realm-setup Job will mount. The + // gitops repo never holds realm content under this design. + domain := cfg.Domain + if domain == "" { + domain = "nebari.local" + } + importContent, err := RenderKeycloakDefaults(ctx, TemplateData{Domain: domain}) + if err != nil { + span.RecordError(err) + return fmt.Errorf("failed to render keycloak-config-cli defaults: %w", err) + } + if err := createKeycloakImportSecret(ctx, k8sClient, importContent); err != nil { + span.RecordError(err) + return fmt.Errorf("failed to create %s secret: %w", KeycloakImportSecretName, err) + } + // Create namespace for Nebari system services if err := createNamespace(ctx, k8sClient, NebariSystemNamespace); err != nil { span.RecordError(err) @@ -287,8 +312,8 @@ func createKeycloakSecrets(ctx context.Context, client kubernetes.Interface, key Name: "nebari-realm-admin-credentials", Namespace: namespace, Labels: map[string]string{ - "app.kubernetes.io/part-of": NebariFoundationalPartOf, - "app.kubernetes.io/managed-by": "nebari-infrastructure-core", + labelPartOf: NebariFoundationalPartOf, + labelManagedBy: labelManagedByValue, }, }, Type: corev1.SecretTypeOpaque, @@ -308,8 +333,8 @@ func createKeycloakSecrets(ctx context.Context, client kubernetes.Interface, key Name: "argocd-oidc-client-secret", Namespace: namespace, Labels: map[string]string{ - "app.kubernetes.io/part-of": NebariFoundationalPartOf, - "app.kubernetes.io/managed-by": "nebari-infrastructure-core", + labelPartOf: NebariFoundationalPartOf, + labelManagedBy: labelManagedByValue, }, }, Type: corev1.SecretTypeOpaque, @@ -324,6 +349,33 @@ func createKeycloakSecrets(ctx context.Context, client kubernetes.Interface, key return nil } +// createKeycloakImportSecret writes the keycloak-config-cli input to an +// in-cluster Secret named `keycloak-config-import` in the `keycloak` +// namespace. The realm-setup Job mounts this Secret read-only and applies +// the file against Keycloak. +// +// The Secret carrier (rather than a gitops-repo ConfigMap) keeps realm +// structure and any inline credentials out of git: only the Job manifest +// lives in the gitops repo. Phase 1a writes only the embedded defaults; +// Phase 1b will overwrite or replace this content with the user's +// keycloak.yaml. +func createKeycloakImportSecret(ctx context.Context, client kubernetes.Interface, importContent []byte) error { + return createSecret(ctx, client, &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: KeycloakImportSecretName, + Namespace: KeycloakDefaultNamespace, + Labels: map[string]string{ + labelPartOf: NebariFoundationalPartOf, + labelManagedBy: labelManagedByValue, + }, + }, + Type: corev1.SecretTypeOpaque, + Data: map[string][]byte{ + KeycloakImportSecretKey: importContent, + }, + }) +} + // createLandingPageSecrets creates the required secrets for the nebari-landing service func createLandingPageSecrets(ctx context.Context, client kubernetes.Interface, landingCfg LandingPageConfig) error { namespace := NebariSystemNamespace @@ -335,8 +387,8 @@ func createLandingPageSecrets(ctx context.Context, client kubernetes.Interface, Name: NebariLandingRedisSecretName, Namespace: namespace, Labels: map[string]string{ - "app.kubernetes.io/part-of": NebariFoundationalPartOf, - "app.kubernetes.io/managed-by": "nebari-infrastructure-core", + labelPartOf: NebariFoundationalPartOf, + labelManagedBy: labelManagedByValue, }, }, Type: corev1.SecretTypeOpaque, diff --git a/pkg/argocd/keycloak_defaults.go b/pkg/argocd/keycloak_defaults.go new file mode 100644 index 00000000..b0709105 --- /dev/null +++ b/pkg/argocd/keycloak_defaults.go @@ -0,0 +1,56 @@ +package argocd + +import ( + "bytes" + "context" + _ "embed" + "fmt" + "text/template" + + "go.opentelemetry.io/otel" +) + +// keycloakDefaultsRaw is the default keycloak-config-cli input for the nebari +// realm. It is embedded as a Go template and rendered with TemplateData at +// deploy time before being written to the in-cluster Secret. +// +//go:embed keycloak_defaults.yaml +var keycloakDefaultsRaw []byte + +const ( + // KeycloakImportSecretName is the in-cluster Secret carrying the kcc + // input that the realm-setup Job applies. The content lives only inside + // the cluster's Secret store; the gitops repo never holds realm + // structure or inline credentials. + KeycloakImportSecretName = "keycloak-config-import" //nolint:gosec // secret name reference + + // KeycloakImportSecretKey is the key inside KeycloakImportSecretName + // under which the rendered kcc input lives. + KeycloakImportSecretKey = "realm.yaml" +) + +// RenderKeycloakDefaults renders the embedded default kcc input with the +// supplied TemplateData. Only the Domain field is substituted today (used +// to build the argocd client's redirect URI); future fields can be wired +// in as the schema grows. +// +// missingkey=error makes a stray template field (e.g. a future {{ .Typo }}) +// fail loudly here rather than silently emitting "" into the realm +// config, where it would surface only as a misconfigured Keycloak at runtime. +func RenderKeycloakDefaults(ctx context.Context, data TemplateData) ([]byte, error) { + tracer := otel.Tracer("nebari-infrastructure-core") + _, span := tracer.Start(ctx, "argocd.RenderKeycloakDefaults") + defer span.End() + + tmpl, err := template.New("keycloak_defaults.yaml").Option("missingkey=error").Parse(string(keycloakDefaultsRaw)) + if err != nil { + span.RecordError(err) + return nil, fmt.Errorf("parse keycloak defaults template: %w", err) + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + span.RecordError(err) + return nil, fmt.Errorf("render keycloak defaults: %w", err) + } + return buf.Bytes(), nil +} diff --git a/pkg/argocd/keycloak_defaults.yaml b/pkg/argocd/keycloak_defaults.yaml new file mode 100644 index 00000000..3fa9ce01 --- /dev/null +++ b/pkg/argocd/keycloak_defaults.yaml @@ -0,0 +1,115 @@ +# Default keycloak-config-cli input for the nebari realm. +# +# Phase 1a of the declarative Keycloak configuration approach proposed in +# nebari-infrastructure-core#154. This file is embedded into the NIC binary +# via go:embed (see +# keycloak_defaults.go), rendered with TemplateData at deploy time, and +# written verbatim to the in-cluster Secret `keycloak-config-import`. The +# kcc Job mounts that Secret read-only and applies the file against +# Keycloak. +# +# The gitops repo never sees this content. Realm structure and any +# inline credentials (none currently; secret-bearing fields use kcc's +# variable-substitution syntax) stay inside the cluster's Secret store. +# +# kcc resolves placeholders via Apache Commons StringSubstitutor BEFORE +# parsing the file as YAML. That means even comments are scanned for +# unresolved variables, so do NOT write the literal placeholder shape +# inside any comment in this file or kcc will fail with +# "Cannot resolve variable" before applying anything. The actual +# placeholders live in field values below. +# {{ .Domain }} is rendered by Go's text/template at NIC deploy time. +realm: nebari +enabled: true +displayName: Nebari +sslRequired: external +registrationAllowed: false +loginWithEmailAllowed: true +resetPasswordAllowed: true +bruteForceProtected: true + +# kcc applies list fields as a full replace, not a merge. We must +# include the built-in default scopes (Keycloak auto-creates these on +# realm creation) here so they survive the import — otherwise tokens +# would be missing email/profile/roles/etc. claims. +defaultDefaultClientScopes: + - basic + - profile + - email + - roles + - web-origins + - acr + - groups + +roles: + realm: + - name: admin + description: Administrator role + - name: user + description: Regular user role + +groups: + - name: argocd-admins + - name: argocd-viewers + +users: + - username: admin + enabled: true + emailVerified: true + firstName: Admin + lastName: User + email: admin@nebari.local + realmRoles: + - admin + - user + credentials: + - type: password + value: $(env:REALM_ADMIN_PASSWORD) + groups: + - /argocd-admins + +clientScopes: + - name: groups + protocol: openid-connect + attributes: + include.in.token.scope: "true" + display.on.consent.screen: "true" + protocolMappers: + - name: group-membership + protocol: openid-connect + protocolMapper: oidc-group-membership-mapper + config: + full.path: "false" + introspection.token.claim: "true" + userinfo.token.claim: "true" + multivalued: "true" + id.token.claim: "true" + access.token.claim: "true" + claim.name: groups + +clients: + - clientId: argocd + enabled: true + protocol: openid-connect + publicClient: false + secret: $(env:ARGOCD_CLIENT_SECRET) + redirectUris: + - https://argocd.{{ .Domain }}/auth/callback + webOrigins: + - https://argocd.{{ .Domain }} + directAccessGrantsEnabled: false + standardFlowEnabled: true + # The realm's defaultDefaultClientScopes auto-apply only at client + # CREATION time, not on subsequent updates. So we must explicitly + # list the full set of default scopes here — otherwise kcc's + # replace semantics on this field would leave older clients with + # whatever they had at creation. Same content as the realm + # default-defaults above so kcc converges to the expected state. + defaultClientScopes: + - basic + - profile + - email + - roles + - web-origins + - acr + - groups diff --git a/pkg/argocd/keycloak_defaults_test.go b/pkg/argocd/keycloak_defaults_test.go new file mode 100644 index 00000000..7acb9a70 --- /dev/null +++ b/pkg/argocd/keycloak_defaults_test.go @@ -0,0 +1,87 @@ +package argocd + +import ( + "context" + "strings" + "testing" +) + +// TestRenderKeycloakDefaults asserts the load-bearing fields of the +// rendered kcc input that NIC writes to the keycloak-config-import +// Secret. The rendered content is what makes the realm-setup Job +// produce a valid Keycloak state; regressions here are silent (the YAML +// renders fine but Keycloak gets misconfigured), so we check structure. +func TestRenderKeycloakDefaults(t *testing.T) { + rendered, err := RenderKeycloakDefaults(context.Background(), TemplateData{Domain: "test.example.com"}) + if err != nil { + t.Fatalf("RenderKeycloakDefaults error: %v", err) + } + output := string(rendered) + + // kcc realm payload — load-bearing fields the realm-setup-job depends on + for _, want := range []string{ + "realm: nebari", + "defaultDefaultClientScopes:", + // kcc applies list fields as full-replace, so the built-in default + // scopes must be present here or tokens lose email/profile/roles. + "- basic", + "- profile", + "- email", + "- roles", + "- web-origins", + "- acr", + "- groups", + "- name: argocd-admins", + "- name: argocd-viewers", + "username: admin", + "$(env:REALM_ADMIN_PASSWORD)", + "- /argocd-admins", + "protocolMapper: oidc-group-membership-mapper", + "claim.name: groups", + "clientId: argocd", + "$(env:ARGOCD_CLIENT_SECRET)", + "https://argocd.test.example.com/auth/callback", + // Realm security posture — silent-but-damaging if a regression flips these. + "sslRequired: external", + "registrationAllowed: false", + "bruteForceProtected: true", + // Realm roles assigned to the bootstrap admin user. + "- name: admin", + "- name: user", + // Group-membership mapper emits a multivalued groups claim. + `multivalued: "true"`, + // argocd client security flags + CORS origin (webOrigins is distinct + // from redirectUris and Domain-templated, so assert it renders too). + "publicClient: false", + "standardFlowEnabled: true", + "directAccessGrantsEnabled: false", + "https://argocd.test.example.com\n", + } { + if !strings.Contains(output, want) { + t.Errorf("expected %q in rendered realm config, got:\n%s", want, output) + } + } + + // The argocd client must explicitly list the full set of default + // scopes. Keycloak's realm default-defaults auto-apply only at client + // creation, not on subsequent updates — so we need this redundancy to + // keep kcc's replace semantics from stripping scopes on existing + // clients during a re-run. + if !strings.Contains(output, " defaultClientScopes:\n - basic") { + t.Errorf("argocd client must explicitly list defaultClientScopes starting with built-ins, got:\n%s", output) + } + + // The rendered content must not contain any unresolved Go-template + // markers. If a future edit introduces a new {{ . }} reference and + // forgets to add the field, this catches it instead of letting it + // land in Keycloak as a literal. + if strings.Contains(output, "{{") || strings.Contains(output, "}}") { + t.Errorf("rendered output contains unresolved template markers, got:\n%s", output) + } + + // missingkey=error should make a stray field error out, but guard against + // a rendered "" leaking into the realm config regardless. + if strings.Contains(output, "") { + t.Errorf("rendered output contains \"\" from an unresolved field, got:\n%s", output) + } +} diff --git a/pkg/argocd/templates/apps/keycloak.yaml b/pkg/argocd/templates/apps/keycloak.yaml index 4fef05d2..dd0be294 100644 --- a/pkg/argocd/templates/apps/keycloak.yaml +++ b/pkg/argocd/templates/apps/keycloak.yaml @@ -21,6 +21,11 @@ spec: helm: releaseName: keycloak values: | + # Pin the Keycloak image. The realm-setup-job uses a + # keycloak-config-cli build that's verified against this exact + # Keycloak version; bumping one means bumping the other. + image: + tag: "26.5.4" args: - start replicas: 1 diff --git a/pkg/argocd/templates/manifests/keycloak/realm-setup-job.yaml b/pkg/argocd/templates/manifests/keycloak/realm-setup-job.yaml index d303ba23..fa5b137e 100644 --- a/pkg/argocd/templates/manifests/keycloak/realm-setup-job.yaml +++ b/pkg/argocd/templates/manifests/keycloak/realm-setup-job.yaml @@ -8,7 +8,14 @@ metadata: app.kubernetes.io/managed-by: nebari-infrastructure-core annotations: argocd.argoproj.io/hook: PostSync - argocd.argoproj.io/hook-delete-policy: HookSucceeded + # `BeforeHookCreation` deletes any existing Job before creating a new one + # on each sync. Without it, a Job that fails (bad image, network blip, + # kcc bug) sits in the cluster forever — Kubernetes Jobs are immutable + # post-create, so ArgoCD can't update the Job's pod template in place. + # This makes upgrades safe even from a stuck/failed previous run. + # `HookSucceeded` additionally deletes the Job after a successful run + # to avoid Job clutter in steady state. + argocd.argoproj.io/hook-delete-policy: HookSucceeded,BeforeHookCreation spec: ttlSecondsAfterFinished: 300 template: @@ -16,137 +23,71 @@ spec: restartPolicy: OnFailure containers: - name: realm-setup - image: quay.io/keycloak/keycloak:24.0 + # Pinned to a kcc release verified in CI against the deployed + # Keycloak version (currently 26.5.4, set in + # pkg/argocd/templates/apps/keycloak.yaml). Bumping the Keycloak + # image requires bumping this tag in lock-step. See: + # https://redirect.github.com/adorsys/keycloak-config-cli/blob/v6.5.0/.github/workflows/ci.yaml + image: quay.io/adorsys/keycloak-config-cli:6.5.0-26.5.4 env: - - name: KEYCLOAK_ADMIN_PASSWORD + # Keycloak connection + - name: KEYCLOAK_URL + value: {{ .KeycloakServiceURL }} + - name: KEYCLOAK_USER + value: admin + - name: KEYCLOAK_PASSWORD valueFrom: secretKeyRef: name: {{ .KeycloakAdminSecretName }} key: admin-password + - name: KEYCLOAK_AVAILABILITYCHECK_ENABLED + value: "true" + # Matches the 5-minute readiness window of the previous shell + # script. Keycloak's first-start Quarkus augmentation can push + # past 2 minutes on slower runners. + - name: KEYCLOAK_AVAILABILITYCHECK_TIMEOUT + value: 300s + + # kcc import behavior + - name: IMPORT_FILES_LOCATIONS + value: /config/realm.yaml + - name: IMPORT_VARSUBSTITUTION_ENABLED + value: "true" + - name: IMPORT_VARSUBSTITUTION_UNDEFINEDISERROR + value: "true" + # Default is true since kcc v6.x; explicit here for review + # clarity. kcc tracks resources it created in a Keycloak realm + # attribute, so re-runs only touch resources kcc itself manages. + - name: IMPORT_REMOTESTATE_ENABLED + value: "true" + # Belt-and-braces: never delete clients on re-run, even ones + # that disappear from the input. Protects operator-managed + # OAuth2 clients (created at runtime via NebariApplication CRDs) + # if the input config ever changes shape. + - name: IMPORT_MANAGED_CLIENT + value: "no-delete" + + # Substituted into realm.yaml at kcc apply time - name: REALM_ADMIN_PASSWORD valueFrom: secretKeyRef: name: nebari-realm-admin-credentials key: password - - name: KEYCLOAK_URL - value: {{ .KeycloakServiceURL }} - name: ARGOCD_CLIENT_SECRET valueFrom: secretKeyRef: name: argocd-oidc-client-secret key: client-secret - - name: DOMAIN - value: {{ .Domain }} - command: - - /bin/bash - - -c - - | - set -e - KCADM="/opt/keycloak/bin/kcadm.sh" - - echo "Waiting for Keycloak to be ready..." - for i in $(seq 1 60); do - if $KCADM config credentials --server "$KEYCLOAK_URL" --realm master --user admin --password "$KEYCLOAK_ADMIN_PASSWORD" 2>/dev/null; then - echo "Keycloak is ready" - break - fi - echo "Keycloak not ready, waiting... ($i/60)" - sleep 5 - done - - echo "Creating nebari realm..." - $KCADM create realms \ - -s realm=nebari \ - -s enabled=true \ - -s displayName="Nebari" \ - -s sslRequired=external \ - -s registrationAllowed=false \ - -s loginWithEmailAllowed=true \ - -s resetPasswordAllowed=true \ - -s bruteForceProtected=true || echo "Realm may already exist" - - echo "Creating realm roles..." - $KCADM create roles -r nebari -s name=admin -s description="Administrator role" || true - $KCADM create roles -r nebari -s name=user -s description="Regular user role" || true - - echo "Creating admin user in nebari realm..." - $KCADM create users -r nebari \ - -s username=admin \ - -s enabled=true \ - -s emailVerified=true \ - -s firstName=Admin \ - -s lastName=User \ - -s email=admin@nebari.local || echo "User may already exist" - - echo "Setting admin user password..." - $KCADM set-password -r nebari \ - --username admin \ - --new-password "$REALM_ADMIN_PASSWORD" - - echo "Assigning roles to admin user..." - $KCADM add-roles -r nebari --uusername admin --rolename admin || true - $KCADM add-roles -r nebari --uusername admin --rolename user || true - - echo "Configuring groups client scope with group-membership mapper..." - # Look up the groups client scope ID using grep/sed (no python3 in keycloak image) - GROUPS_SCOPE_ID=$($KCADM get client-scopes -r nebari --fields id,name | \ - grep -B1 '"name" *: *"groups"' | sed -n 's/.*"id" *: *"\([^"]*\)".*/\1/p') - - if [ -z "$GROUPS_SCOPE_ID" ]; then - echo "Creating groups client scope..." - $KCADM create client-scopes -r nebari \ - -s name=groups \ - -s protocol=openid-connect \ - -s 'attributes={"include.in.token.scope":"true","display.on.consent.screen":"true"}' || true - GROUPS_SCOPE_ID=$($KCADM get client-scopes -r nebari --fields id,name | \ - grep -B1 '"name" *: *"groups"' | sed -n 's/.*"id" *: *"\([^"]*\)".*/\1/p') - fi - - if [ -n "$GROUPS_SCOPE_ID" ]; then - echo "Adding group-membership mapper to groups scope (id=$GROUPS_SCOPE_ID)..." - $KCADM create client-scopes/$GROUPS_SCOPE_ID/protocol-mappers/models -r nebari \ - -s name=group-membership \ - -s protocol=openid-connect \ - -s protocolMapper=oidc-group-membership-mapper \ - -s 'config={"full.path":"false","introspection.token.claim":"true","userinfo.token.claim":"true","id.token.claim":"true","access.token.claim":"true","claim.name":"groups"}' || echo "Mapper may already exist" - - echo "Ensuring groups is a realm default client scope..." - $KCADM update realms/nebari/default-default-client-scopes/$GROUPS_SCOPE_ID -r nebari || true - fi - - echo "Creating ArgoCD OIDC client..." - $KCADM create clients -r nebari \ - -s clientId=argocd \ - -s enabled=true \ - -s protocol=openid-connect \ - -s publicClient=false \ - -s "secret=$ARGOCD_CLIENT_SECRET" \ - -s "redirectUris=[\"https://argocd.$DOMAIN/auth/callback\"]" \ - -s directAccessGrantsEnabled=false \ - -s standardFlowEnabled=true || echo "Client may already exist" - - # Add groups scope to argocd client as a default scope - ARGOCD_CLIENT_ID=$($KCADM get clients -r nebari --fields id,clientId | \ - grep -B1 '"clientId" *: *"argocd"' | sed -n 's/.*"id" *: *"\([^"]*\)".*/\1/p') - - if [ -n "$ARGOCD_CLIENT_ID" ] && [ -n "$GROUPS_SCOPE_ID" ]; then - echo "Adding groups scope to argocd client..." - $KCADM update clients/$ARGOCD_CLIENT_ID/default-client-scopes/$GROUPS_SCOPE_ID -r nebari || true - fi - - echo "Creating ArgoCD access groups..." - $KCADM create groups -r nebari -s name=argocd-admins || echo "Group may already exist" - $KCADM create groups -r nebari -s name=argocd-viewers || echo "Group may already exist" - - echo "Adding admin user to argocd-admins group..." - ADMIN_USER_ID=$($KCADM get users -r nebari -q username=admin --fields id | \ - sed -n 's/.*"id" *: *"\([^"]*\)".*/\1/p') - ADMINS_GROUP_ID=$($KCADM get groups -r nebari --fields id,name | \ - grep -B1 '"name" *: *"argocd-admins"' | sed -n 's/.*"id" *: *"\([^"]*\)".*/\1/p') - - if [ -n "$ADMIN_USER_ID" ] && [ -n "$ADMINS_GROUP_ID" ]; then - $KCADM update users/$ADMIN_USER_ID/groups/$ADMINS_GROUP_ID -r nebari \ - -s realm=nebari -s userId=$ADMIN_USER_ID -s groupId=$ADMINS_GROUP_ID -n || true - fi - - echo "Realm setup complete!" + volumeMounts: + - name: realm-config + mountPath: /config + readOnly: true + volumes: + # The kcc input lives in an in-cluster Secret created by NIC at + # bootstrap (see pkg/argocd/foundational.go createKeycloakImportSecret). + # Using a Secret rather than a ConfigMap keeps realm structure and any + # inline credentials out of the gitops repo entirely; only this Job + # manifest sits there. + - name: realm-config + secret: + secretName: keycloak-config-import