docs: settings documentation using settings-doc markdown generator (#698)

* docs: settings doc

fix: annoying wrapper script

docs: fix task doc

fix: add ci skip settings-doc-check

* docs: env variable / settings documentation generator

fix: cleanups

* fix: use separate tooling env for settings-doc

fix: well that didn't work

fix: update tasks

fix: format settings doc files

Author: ryuwd

.github/workflows/generate_pixi_tasks_doc.py

@@ -18,6 +18,8 @@ def get_task_group(task_name):
         return "Client Generation"
     if task_name == "shellcheck":
         return "Shellcheck"
+    if task_name.startswith("generate-settings-"):
+        return "Settings"
     return "Default"
 
 

.gitignore

@@ -96,3 +96,7 @@ docs/source/_build
 .pixi
 pixi.lock
 *.egg-info
+docs/templates/_builtin_markdown.jinja
+
+# docs site
+site

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ default_language_version:
   python: python3
 
 ci:
-  skip: [generate-pixi-docs]
+  skip: [generate-pixi-docs, settings-doc-check]
 
 default_stages: [pre-commit]
 
@@ -85,3 +85,12 @@ repos:
         language: system
         pass_filenames: false
         files: ^pixi\.toml$|^pixi\.lock$   # only run if pixi files change
+
+  - repo: local
+    hooks:
+      - id: settings-doc-check
+        name: Generate settings documentation
+        entry: pixi run -e default python scripts/generate_settings_docs.py
+        language: system
+        pass_filenames: false
+        files: ^(diracx-.*/src/diracx/.*/settings\.py|docs/.*\.j2|docs/templates/.*\.jinja|scripts/generate_settings_docs\.py)$

diracx-core/src/diracx/core/settings.py

@@ -132,11 +132,16 @@ async def lifetime_function(self) -> AsyncIterator[None]:
 class DevelopmentSettings(ServiceSettingsBase):
     """Settings for the Development Configuration that can influence run time."""
 
-    model_config = SettingsConfigDict(env_prefix="DIRACX_DEV_")
+    model_config = SettingsConfigDict(
+        env_prefix="DIRACX_DEV_", use_attribute_docstrings=True
+    )
 
-    # When then to true (only for demo/CI), crash if an access policy isn't
-    # called
     crash_on_missed_access_policy: bool = False
+    """When set to true (only for demo/CI), crash if an access policy isn't called.
+
+    This is useful for development and testing to ensure all endpoints have proper
+    access control policies defined.
+    """
 
     @classmethod
     def create(cls) -> Self:
@@ -146,39 +151,123 @@ def create(cls) -> Self:
 class AuthSettings(ServiceSettingsBase):
     """Settings for the authentication service."""
 
-    model_config = SettingsConfigDict(env_prefix="DIRACX_SERVICE_AUTH_")
+    model_config = SettingsConfigDict(
+        env_prefix="DIRACX_SERVICE_AUTH_", use_attribute_docstrings=True
+    )
 
     dirac_client_id: str = "myDIRACClientID"
-    # TODO: This should be taken dynamically
-    # ["http://pclhcb211:8000/docs/oauth2-redirect"]
+    """OAuth2 client identifier for DIRAC services.
+
+    This should match the client ID registered with the identity provider.
+    """
+
     allowed_redirects: list[str] = []
+    """List of allowed redirect URLs for OAuth2 authorization flow.
+
+    These URLs must be pre-registered and should match the redirect URIs
+    configured in the OAuth2 client registration.
+    Example: ["http://localhost:8000/docs/oauth2-redirect"]
+    """
+
     device_flow_expiration_seconds: int = 600
+    """Expiration time in seconds for device flow authorization requests.
+
+    After this time, the device code becomes invalid and users must restart
+    the device flow process. Default: 10 minutes.
+    """
+
     authorization_flow_expiration_seconds: int = 300
+    """Expiration time in seconds for authorization code flow.
+
+    The time window during which the authorization code remains valid
+    before it must be exchanged for tokens. Default: 5 minutes.
+    """
 
-    # State key is used to encrypt/decrypt the state dict passed to the IAM
     state_key: FernetKey
+    """Encryption key used to encrypt/decrypt the state parameter passed to the IAM.
+
+    This key ensures the integrity and confidentiality of state information
+    during OAuth2 flows. Must be a valid Fernet key.
+    """
 
     token_issuer: str
+    """The issuer identifier for JWT tokens.
+
+    This should be a URI that uniquely identifies the token issuer and
+    matches the 'iss' claim in issued JWT tokens.
+    """
+
     token_keystore: TokenSigningKeyStore
+    """Keystore containing the cryptographic keys used for signing JWT tokens.
+
+    This includes both public and private keys for token signature
+    generation and verification.
+    """
+
     token_allowed_algorithms: list[str] = ["RS256", "EdDSA"]  # noqa: S105
+    """List of allowed cryptographic algorithms for JWT token signing.
+
+    Supported algorithms include RS256 (RSA with SHA-256) and EdDSA
+    (Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"]
+    """
+
     access_token_expire_minutes: int = 20
+    """Expiration time in minutes for access tokens.
+
+    After this duration, access tokens become invalid and must be refreshed
+    or re-obtained. Default: 20 minutes.
+    """
+
     refresh_token_expire_minutes: int = 60
+    """Expiration time in minutes for refresh tokens.
+
+    The maximum lifetime of refresh tokens before they must be re-issued
+    through a new authentication flow. Default: 60 minutes.
+    """
 
     available_properties: set[SecurityProperty] = Field(
         default_factory=SecurityProperty.available_properties
     )
+    """Set of security properties available in this DIRAC installation.
+
+    These properties define various authorization capabilities and are used
+    for access control decisions. Defaults to all available security properties.
+    """
 
 
 class SandboxStoreSettings(ServiceSettingsBase):
     """Settings for the sandbox store."""
 
-    model_config = SettingsConfigDict(env_prefix="DIRACX_SANDBOX_STORE_")
+    model_config = SettingsConfigDict(
+        env_prefix="DIRACX_SANDBOX_STORE_", use_attribute_docstrings=True
+    )
 
     bucket_name: str
+    """Name of the S3 bucket used for storing job sandboxes.
+
+    This bucket will contain input and output sandbox files for DIRAC jobs.
+    The bucket must exist or auto_create_bucket must be enabled.
+    """
+
     s3_client_kwargs: dict[str, Any]
+    """Configuration parameters passed to the S3 client."""
+
     auto_create_bucket: bool = False
+    """Whether to automatically create the S3 bucket if it doesn't exist."""
+
     url_validity_seconds: int = 5 * 60
+    """Validity duration in seconds for pre-signed S3 URLs.
+
+    This determines how long generated download/upload URLs remain valid
+    before expiring. Default: 300 seconds (5 minutes).
+    """
+
     se_name: str = "SandboxSE"
+    """Logical name of the Storage Element for the sandbox store.
+
+    This name is used within DIRAC to refer to this sandbox storage
+    endpoint in job descriptions and file catalogs.
+    """
     _client: S3Client = PrivateAttr()
 
     @contextlib.asynccontextmanager

docs/admin/reference/env-variables.md

@@ -1,39 +1,173 @@
 # List of environment variables
 
-## Core
-
-- `DIRACX_CONFIG_BACKEND_URL`: The URL of the configuration backend.
-
-## Services:
-
-- `DIRACX_SERVICE_AUTH_TOKEN_ISSUER`: The issuer for the auth tokens.
-- `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS`: A JSON-encoded list of allowed redirect URIs for the authorization code
-    flow.
-- `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS`: The expiration time for the device flow in seconds.
-- `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS`: The expiration time for the authorization flow in
-    seconds.
-- `DIRACX_SERVICE_AUTH_STATE_KEY`: The key used to encrypt the state in the authorization code flow.
-- `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE`: The path to the JWKS file containing the token signing keys.
-- `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS`: A JSON-encoded list of allowed algorithms for token signing.
-- `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES`: The expiration time for the access token in minutes.
-- `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES`: The expiration time for the refresh token in minutes.
-- `DIRACX_SANDBOX_STORE_BUCKET_NAME`: The name of the S3 bucket for the sandbox store.
-- `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS`: A JSON-encoded dictionary of keyword arguments for the S3 client.
-- `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET`: Whether to automatically create the S3 bucket if it doesn't exist.
-- `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS`: The validity of the presigned URLs for the sandbox store in seconds.
-- `DIRACX_SANDBOX_STORE_SE_NAME`: The name of the storage element for the sandbox store.
-- `DIRACX_LEGACY_EXCHANGE_HASHED_API_KEY`: The hashed API key for the legacy exchange endpoint.
-- `DIRACX_SERVICE_JOBS_ENABLED`: Whether the jobs service is enabled.
-
-## Databases:
-
-- `DIRACX_DB_URL_<db_name>`: The URL for the SQL database `<db_name>`.
-- `DIRACX_OS_DB_<db_name>`: A JSON-encoded dictionary of connection keyword arguments for the OpenSearch database `<db_name>`
-
-## OTEL:
-
-- `DIRACX_OTEL_ENABLED`: Whether OpenTelemetry is enabled.
-- `DIRACX_OTEL_APPLICATION_NAME`: The name of the application for OpenTelemetry.
-- `DIRACX_OTEL_GRPC_ENDPOINT`: The gRPC endpoint for the OpenTelemetry collector.
-- `DIRACX_OTEL_GRPC_INSECURE`: Whether to use an insecure gRPC connection for the OpenTelemetry collector.
-- `DIRACX_OTEL_HEADERS`: A JSON-encoded dictionary of headers to pass to the OpenTelemetry collector.
+*This page is auto-generated from the settings classes in `diracx.core.settings`.*
+
+## AuthSettings
+
+Settings for the authentication service.
+
+### `DIRACX_SERVICE_AUTH_DIRAC_CLIENT_ID`
+
+*Optional*, default value: `myDIRACClientID`
+
+OAuth2 client identifier for DIRAC services.
+
+This should match the client ID registered with the identity provider.
+
+### `DIRACX_SERVICE_AUTH_ALLOWED_REDIRECTS`
+
+*Optional*, default value: `[]`
+
+List of allowed redirect URLs for OAuth2 authorization flow.
+
+These URLs must be pre-registered and should match the redirect URIs
+configured in the OAuth2 client registration.
+Example: ["http://localhost:8000/docs/oauth2-redirect"]
+
+### `DIRACX_SERVICE_AUTH_DEVICE_FLOW_EXPIRATION_SECONDS`
+
+*Optional*, default value: `600`
+
+Expiration time in seconds for device flow authorization requests.
+
+After this time, the device code becomes invalid and users must restart
+the device flow process. Default: 10 minutes.
+
+### `DIRACX_SERVICE_AUTH_AUTHORIZATION_FLOW_EXPIRATION_SECONDS`
+
+*Optional*, default value: `300`
+
+Expiration time in seconds for authorization code flow.
+
+The time window during which the authorization code remains valid
+before it must be exchanged for tokens. Default: 5 minutes.
+
+### `DIRACX_SERVICE_AUTH_STATE_KEY`
+
+**Required**
+
+Encryption key used to encrypt/decrypt the state parameter passed to the IAM.
+
+This key ensures the integrity and confidentiality of state information
+during OAuth2 flows. Must be a valid Fernet key.
+
+### `DIRACX_SERVICE_AUTH_TOKEN_ISSUER`
+
+**Required**
+
+The issuer identifier for JWT tokens.
+
+This should be a URI that uniquely identifies the token issuer and
+matches the 'iss' claim in issued JWT tokens.
+
+### `DIRACX_SERVICE_AUTH_TOKEN_KEYSTORE`
+
+**Required**
+
+Keystore containing the cryptographic keys used for signing JWT tokens.
+
+This includes both public and private keys for token signature
+generation and verification.
+
+### `DIRACX_SERVICE_AUTH_TOKEN_ALLOWED_ALGORITHMS`
+
+*Optional*, default value: `['RS256', 'EdDSA']`
+
+List of allowed cryptographic algorithms for JWT token signing.
+
+Supported algorithms include RS256 (RSA with SHA-256) and EdDSA
+(Edwards-curve Digital Signature Algorithm). Default: ["RS256", "EdDSA"]
+
+### `DIRACX_SERVICE_AUTH_ACCESS_TOKEN_EXPIRE_MINUTES`
+
+*Optional*, default value: `20`
+
+Expiration time in minutes for access tokens.
+
+After this duration, access tokens become invalid and must be refreshed
+or re-obtained. Default: 20 minutes.
+
+### `DIRACX_SERVICE_AUTH_REFRESH_TOKEN_EXPIRE_MINUTES`
+
+*Optional*, default value: `60`
+
+Expiration time in minutes for refresh tokens.
+
+The maximum lifetime of refresh tokens before they must be re-issued
+through a new authentication flow. Default: 60 minutes.
+
+### `DIRACX_SERVICE_AUTH_AVAILABLE_PROPERTIES`
+
+*Optional*
+
+Set of security properties available in this DIRAC installation.
+
+These properties define various authorization capabilities and are used
+for access control decisions. Defaults to all available security properties.
+
+## SandboxStoreSettings
+
+Settings for the sandbox store.
+
+### `DIRACX_SANDBOX_STORE_BUCKET_NAME`
+
+**Required**
+
+Name of the S3 bucket used for storing job sandboxes.
+
+This bucket will contain input and output sandbox files for DIRAC jobs.
+The bucket must exist or auto_create_bucket must be enabled.
+
+### `DIRACX_SANDBOX_STORE_S3_CLIENT_KWARGS`
+
+**Required**
+
+Configuration parameters passed to the S3 client.
+
+### `DIRACX_SANDBOX_STORE_AUTO_CREATE_BUCKET`
+
+*Optional*, default value: `False`
+
+Whether to automatically create the S3 bucket if it doesn't exist.
+
+### `DIRACX_SANDBOX_STORE_URL_VALIDITY_SECONDS`
+
+*Optional*, default value: `300`
+
+Validity duration in seconds for pre-signed S3 URLs.
+
+This determines how long generated download/upload URLs remain valid
+before expiring. Default: 300 seconds (5 minutes).
+
+### `DIRACX_SANDBOX_STORE_SE_NAME`
+
+*Optional*, default value: `SandboxSE`
+
+Logical name of the Storage Element for the sandbox store.
+
+This name is used within DIRAC to refer to this sandbox storage
+endpoint in job descriptions and file catalogs.
+
+## OTELSettings
+
+Settings for the Open Telemetry Configuration.
+
+### `DIRACX_OTEL_ENABLED`
+
+*Optional*, default value: `False`
+
+### `DIRACX_OTEL_APPLICATION_NAME`
+
+*Optional*, default value: `diracx`
+
+### `DIRACX_OTEL_GRPC_ENDPOINT`
+
+*Optional*, default value: \`\`
+
+### `DIRACX_OTEL_GRPC_INSECURE`
+
+*Optional*, default value: `True`
+
+### `DIRACX_OTEL_HEADERS`
+
+*Optional*, default value: `None`