diff --git a/diracx/Chart.yaml b/diracx/Chart.yaml index ec22d166..43a27fcc 100644 --- a/diracx/Chart.yaml +++ b/diracx/Chart.yaml @@ -11,7 +11,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: "1.0.19" +version: "1.0.20" # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/diracx/templates/_helpers.tpl b/diracx/templates/_helpers.tpl index 54eb1c25..bb010a92 100644 --- a/diracx/templates/_helpers.tpl +++ b/diracx/templates/_helpers.tpl @@ -205,3 +205,10 @@ reduce collisions. {{- $rand := randAlphaNum 3 | lower }} {{- printf "%s-%d-%s" $name .Release.Revision $rand | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Return the fullname template for the cleanupAuthDB cronjob. +*/}} +{{- define "cleanupAuthDB.fullname" -}} +{{- printf "%s-cleanup-authdb" .Release.Name -}} +{{- end -}} diff --git a/diracx/templates/diracx/cleanup-authdb/_cleanup-authdb.sh.tpl b/diracx/templates/diracx/cleanup-authdb/_cleanup-authdb.sh.tpl new file mode 100644 index 00000000..b71d74e0 --- /dev/null +++ b/diracx/templates/diracx/cleanup-authdb/_cleanup-authdb.sh.tpl @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +python -m diracx.logic cleanup-authdb diff --git a/diracx/templates/diracx/cleanup-authdb/configmap.yaml b/diracx/templates/diracx/cleanup-authdb/configmap.yaml new file mode 100644 index 00000000..9ec5c4dd --- /dev/null +++ b/diracx/templates/diracx/cleanup-authdb/configmap.yaml @@ -0,0 +1,10 @@ +{{- if .Values.cleanupAuthDB.enabled -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "cleanupAuthDB.fullname" . }} + namespace: {{ .Release.Namespace }} +data: + cleanup-authdb: | + {{- include (print $.Template.BasePath "/diracx/cleanup-authdb/_cleanup-authdb.sh.tpl") . | nindent 4 }} +{{- end -}} diff --git a/diracx/templates/diracx/cleanup-authdb/cronjob.yaml b/diracx/templates/diracx/cleanup-authdb/cronjob.yaml new file mode 100644 index 00000000..2aaab402 --- /dev/null +++ b/diracx/templates/diracx/cleanup-authdb/cronjob.yaml @@ -0,0 +1,83 @@ +{{- if .Values.cleanupAuthDB.enabled -}} + +{{/* Define common volume mounts for reusability */}} +{{- $commonVolumeMounts := list }} +{{- $commonVolumeMounts = append $commonVolumeMounts (dict "mountPath" "/entrypoint.sh" "name" "container-entrypoint" "subPath" "entrypoint.sh") }} +{{- if and .Values.developer.enabled .Values.developer.enableCoverage }} +{{- $commonVolumeMounts = append $commonVolumeMounts (dict "mountPath" "/diracx-coveragerc" "name" "container-entrypoint" "subPath" "coveragerc") }} +{{- $commonVolumeMounts = append $commonVolumeMounts (dict "mountPath" "/coverage-reports" "name" "coverage-data" "readOnly" false) }} +{{- end }} +{{- if and .Values.developer.enabled .Values.developer.mountedPythonModulesToInstall }} +{{- $commonVolumeMounts = append $commonVolumeMounts (dict "mountPath" .Values.developer.sourcePath "name" "diracx-code-mount" "readOnly" true) }} +{{- range $module := .Values.developer.mountedPythonModulesToInstall }} +{{- if $.Values.developer.editableMountedPythonModules }} +{{- $commonVolumeMounts = append $commonVolumeMounts (dict "mountPath" (printf "%s/%s/src/%s.egg-info" $.Values.developer.sourcePath $module (replace "-" "_" (base $module))) "name" (printf "%s-editable-install" (base $module | lower)) "readOnly" false) }} +{{- else }} +{{- $commonVolumeMounts = append $commonVolumeMounts (dict "mountPath" (printf "%s/%s/build" $.Values.developer.sourcePath $module (replace "-" "_" (base $module))) "name" (printf "%s-editable-install" (base $module | lower)) "readOnly" false) }} +{{- end }} +{{- end }} +{{- end }} + +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ template "cleanupAuthDB.fullname" . }} + namespace: {{ .Release.Namespace }} +spec: + schedule: {{ $.Values.cleanupAuthDB.schedule | quote }} + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: {{ .Chart.Name }}-cleanup + image: "{{ include "diracx.servicesImage" . }}" + imagePullPolicy: {{ .Values.global.imagePullPolicy }} + command: ["/bin/bash", "/entrypoint.sh"] + args: ["/bin/bash", "/scripts/cleanup-authdb"] + envFrom: + - secretRef: + name: diracx-secrets + - secretRef: + name: diracx-sql-connection-urls + - secretRef: + name: diracx-dynamic-secrets + volumeMounts: + {{ toYaml $commonVolumeMounts | nindent 16 }} + - name: keystore + mountPath: /keystore/jwks.json + subPath: jwks.json + - name: scripts + mountPath: /scripts + resources: + {{- toYaml .Values.cleanupAuthDB.resources | nindent 16 }} + volumes: + - name: keystore + secret: + secretName: diracx-jwks + items: + - key: jwks.json + path: jwks.json + - name: scripts + configMap: + name: {{ template "cleanupAuthDB.fullname" . }} + {{- if and .Values.developer.enabled .Values.developer.mountedPythonModulesToInstall }} + - name: diracx-code-mount + persistentVolumeClaim: + claimName: pvc-diracx-code + {{- range $module := .Values.developer.mountedPythonModulesToInstall }} + - name: {{ base $module | lower }}-editable-install + emptyDir: + sizeLimit: 5Mi + {{- end }} + {{- end }} + {{- if and .Values.developer.enabled .Values.developer.enableCoverage }} + - name: coverage-data + persistentVolumeClaim: + claimName: pvc-coverage + {{- end }} + - name: container-entrypoint + configMap: + name: diracx-container-entrypoint +{{- end -}} diff --git a/diracx/values.yaml b/diracx/values.yaml index 39ac52e2..b147fb5d 100644 --- a/diracx/values.yaml +++ b/diracx/values.yaml @@ -104,6 +104,12 @@ initOs: initKeyStore: enabled: true +cleanupAuthDB: + # -- Enable the AuthDB cleanup CronJob + enabled: true + # -- Cron schedule for AuthDB cleanup (default: monthly on the 1st) + schedule: "0 0 1 * *" + developer: enabled: false # -- Make it possible to launch the demo without having an internet connection @@ -150,6 +156,10 @@ diracx: DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE: "file:///keystore/jwks.json" DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS: '["http://anything:8000/docs/oauth2-redirect"]' + # -- Tune the AuthDB cleanup cronjob retention periods + # -- DIRACX_SERVICE_AUTH_REVOKED_REFRESH_TOKEN_RETENTION_MINUTES: + # -- DIRACX_SERVICE_AUTH_COMPLETED_FLOW_RETENTION_MINUTES: + # -- legacy exchange key for DIRAC legacy (see https://github.com/DIRACGrid/diracx/blob/7f766158a674fde0eed011cd2745d359e507f846/diracx-routers/src/diracx/routers/auth/token.py#L264) # -- DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY: diff --git a/docs/admin/explanations/architecture_diagram.md b/docs/admin/explanations/architecture_diagram.md index a5f855aa..8c5afebc 100644 --- a/docs/admin/explanations/architecture_diagram.md +++ b/docs/admin/explanations/architecture_diagram.md @@ -5,13 +5,15 @@ config: flowchart TD subgraph k8s_instance ["K8s Instance: diracx"] - subgraph helm_chart ["Helm Chart: diracx 1.0.19"] + subgraph helm_chart ["Helm Chart: diracx 1.0.20"] subgraph k8s_app ["K8s Application: diracx"] ing_diracx{{"ing: diracx"}} svc_diracx(("svc: diracx")) svc_diracx_web(("svc: diracx-web")) deploy_diracx["deploy: diracx"] deploy_diracx_web["deploy: diracx-web"] + cronjob_diracx_cleanup_authdb(["cronjob: diracx-cleanup-authdb"]) + cm_diracx_cleanup_authdb[("cm: diracx-cleanup-authdb")] cm_mysql_init_diracx_dbs[("cm: mysql-init-diracx-dbs")] secret_diracx_secrets>"secret: diracx-secrets"] sa_diracx[["sa: diracx"]] @@ -51,6 +53,9 @@ flowchart TD job_diracx_validate_config -->|"mounts"| cm_diracx_validate_config job_diracx_validate_config -->|"mounts"| cm_diracx_container_entrypoint job_diracx_validate_config -->|"env"| secret_diracx_secrets + cronjob_diracx_cleanup_authdb -->|"mounts"| cm_diracx_cleanup_authdb + cronjob_diracx_cleanup_authdb -->|"mounts"| cm_diracx_container_entrypoint + cronjob_diracx_cleanup_authdb -->|"env"| secret_diracx_secrets %% Styling classDef ing fill:#4a90d9,stroke:#2563eb,color:#ffffff @@ -62,10 +67,12 @@ flowchart TD classDef cm fill:#7ec8e3,stroke:#2563eb,color:#1a1a2e classDef secret fill:#a78bfa,stroke:#2563eb,color:#ffffff classDef sa fill:#94a3b8,stroke:#2563eb,color:#ffffff + class cm_diracx_cleanup_authdb cm class cm_diracx_container_entrypoint cm class cm_diracx_init_keystore cm class cm_diracx_validate_config cm class cm_mysql_init_diracx_dbs cm + class cronjob_diracx_cleanup_authdb cronjob class deploy_diracx deploy class deploy_diracx_web deploy class ing_diracx ing diff --git a/docs/admin/reference/values.md b/docs/admin/reference/values.md index 8c24c671..76e1bdc9 100644 --- a/docs/admin/reference/values.md +++ b/docs/admin/reference/values.md @@ -24,6 +24,8 @@ | cert-manager.enabled | bool | `true` | | | cert-manager.installCRDs | bool | `true` | | | cert-manager.startupapicheck.enabled | bool | `true` | | +| cleanupAuthDB.enabled | bool | `true` | Enable the AuthDB cleanup CronJob | +| cleanupAuthDB.schedule | string | `"0 0 1 * *"` | Cron schedule for AuthDB cleanup (default: monthly on the 1st) | | developer.autoReload | bool | `true` | Enable automatic reloading inside uvicorn when the sources change Used by the integration tests for running closer to prod setup | | developer.editableMountedPythonModules | bool | `true` | Use pip install -e for mountedPythonModulesToInstall This is used by the integration tests because editable install might behave differently | | developer.enableCoverage | bool | `false` | Enable collection of coverage reports (intended for CI usage only) |