Fix PostgreSQL credential security issues and naming inconsistencies

The previous PostgreSQL setup had several security and consistency
problems:

Security issues:
1. POSTGRESQL_ADMIN_PASSWORD was set, making the postgres superuser
   accessible remotely (not just locally via `oc rsh` + `psql`). [1]
2. Both lightspeed-stack and llama-stack were connecting as the postgres
   superuser rather than a dedicated non-admin user.
3. The postgres superuser username usage was hardcoded and credential
   paths into the app and db pods were inconsistent.

Naming/consistency issues:
4. lightspeed-stack used database "postgres" while llama-stack used
   "llamastack", with no explicit database name in llama-stack's config.
5. POSTGRES_USER/POSTGRES_PASSWORD env var names did not follow the
   POSTGRESQL_* convention of the PostgreSQL container image and also
   the usage of these env vars was insonsistent across the operator
code.
6. Bootstrap SQL was embedded in postgres_bootstrap.sh, making it hard
   to read and maintain.

Fixes:
- Remove POSTGRESQL_ADMIN_PASSWORD. The postgres superuser is now only
  accessible locally via `oc rsh` + `psql` (no password required).
- Introduce a dedicated non-admin user "lightspeed-app-user", stored in
  lightspeed-postgres-secret alongside the password. Both apps now
  connect as this user via POSTGRESQL_USER and POSTGRESQL_PASSWORD.
- Rename the lightspeed-stack database from "postgres" to
  "lightspeed-stack" and make llama-stack's "llamastack" database
  selection explicit in its config.
- Rename POSTGRES_USER/POSTGRES_PASSWORD ->
POSTGRESQL_USER/POSTGRESQL_PASSWORD
  throughout to match the container image's naming convention.
- Extract bootstrap SQL into postgres_bootstrap.sql to enable syntax
  highlighting and improve maintainability.

[1]
https://github.com/sclorg/postgresql-container/blob/master/src/root/usr/share/container-scripts/postgresql/README.md#postgresql-admin-account

Author: lpiwowar

internal/controller/assets/postgres_bootstrap.sh

@@ -1,25 +1,23 @@
 #!/bin/bash
-
+# This script along with the postgres_bootstrap.sql script is responsible for preparing
+# the database for consumption by the lightspeed-stack and llama-stack (OGX) apps.
+# Things worth mentioning:
+#
+# *  This script explicitly creates database for llama-stack (OGX). Database creation
+#    for lightspeed-stack is handled by the builtin scripts in the container image
+#    (it reads POSTGRESQL_DATABASE value):
+#        - POSTGRESQL_DATABASE:             database used by lightspeed-stack
+#        - POSTGRESQL_LLAMA_STACK_DATABASE: database used by llama-stack (explicitly
+#                                           created by this script)
 set -e
 
 cat /var/lib/pgsql/data/userdata/postgresql.conf
 
-echo "attempting to create llama-stack database and pg_trgm extension if they do not exist"
-
-_psql () { psql --set ON_ERROR_STOP=1 "$@" ; }
-
-# Create database for llama-stack conversation storage
-DB_NAME="llamastack"
-
-echo "SELECT 'CREATE DATABASE $DB_NAME' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '$DB_NAME')\gexec" | _psql -d "$POSTGRESQL_DATABASE"
-
-# Create pg_trgm extension in default database (for OpenStack Lightspeed conversation cache)
-echo "CREATE EXTENSION IF NOT EXISTS pg_trgm;" | _psql -d "$POSTGRESQL_DATABASE"
-
-# Create pg_trgm extension in llama-stack database (for text search if needed)
-echo "CREATE EXTENSION IF NOT EXISTS pg_trgm;" | _psql -d $DB_NAME
+echo "Bootstrapping PostgreSQL databases and permissions"
 
-# Create schemas for isolating different components' data
-echo "CREATE SCHEMA IF NOT EXISTS lcore;" | _psql -d "$POSTGRESQL_DATABASE"
-echo "CREATE SCHEMA IF NOT EXISTS quota;" | _psql -d "$POSTGRESQL_DATABASE"
-echo "CREATE SCHEMA IF NOT EXISTS conversation_cache;" | _psql -d "$POSTGRESQL_DATABASE"
+psql \
+    -v ON_ERROR_STOP=1 \
+    -v postgresql_user="$POSTGRESQL_USER" \
+    -v postgresql_lightspeed_stack_database="$POSTGRESQL_DATABASE" \
+    -v postgresql_llama_stack_database="$POSTGRESQL_LLAMA_STACK_DATABASE" \
+    -f "$POSTGRESQL_BOOTSTRAP_SQL_FILE"

internal/controller/assets/postgres_bootstrap.sql

@@ -0,0 +1,29 @@
+-- 1) LIGHTSPEED STACK DATABASE CONFIGURATION ---------------------------------
+\c :postgresql_lightspeed_stack_database
+
+-- PostgreSQL 15+ removed the default CREATE/USAGE grants on the public schema.
+-- Therefore we have to explicitly grant the permissions to the user.
+GRANT USAGE, CREATE ON SCHEMA public TO :"postgresql_user";
+
+-- lightspeed-stack requires rights to create additional schemas under its database
+GRANT CREATE ON DATABASE :"postgresql_lightspeed_stack_database" TO :"postgresql_user";
+
+-- pg_trgm is the trigram similarity extension for PostgreSQL (enables e.g. fuzzy text search)
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+-------------------------------------------------------------------------------
+
+-- 2) LLAMA STACK DATABASE CONFIGURATION --------------------------------------
+-- Create postgresql_llama_stack_database.
+SELECT format('CREATE DATABASE %I', :'postgresql_llama_stack_database')
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = :'postgresql_llama_stack_database')\gexec
+
+-- Connect and configure postgresql_llama_stack_database
+\c :postgresql_llama_stack_database
+
+-- PostgreSQL 15+ removed the default CREATE/USAGE grants on the public schema.
+-- Therefore we have to explicitly grant the permissions to the user.
+GRANT USAGE, CREATE ON SCHEMA public TO :"postgresql_user";
+
+-- pg_trgm is the trigram similarity extension for PostgreSQL (enables e.g. fuzzy text search)
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+-------------------------------------------------------------------------------

internal/controller/common.go

@@ -82,6 +82,21 @@ func getConfigMapResourceVersion(ctx context.Context, h *common_helper.Helper, n
 	return cm.ResourceVersion, nil
 }
 
+// getSecretResourceVersion retrieves the resource version of a Secret.
+func getSecretResourceVersion(ctx context.Context, h *common_helper.Helper, name string, namespace string) (string, error) {
+	rawClient, err := getRawClient(h)
+	if err != nil {
+		return "", fmt.Errorf("failed to get raw client: %w", err)
+	}
+
+	secret := &corev1.Secret{}
+	err = rawClient.Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret)
+	if err != nil {
+		return "", fmt.Errorf("failed to get secret %s: %w", name, err)
+	}
+	return secret.ResourceVersion, nil
+}
+
 // providerNameToEnvVarName converts a provider name to a valid environment variable name.
 // It uppercases the string and replaces hyphens and dots with underscores.
 func providerNameToEnvVarName(providerName string) string {

internal/controller/constants.go

@@ -50,23 +50,32 @@ const (
 	PostgresConfigMapName                        = "lightspeed-postgres-conf"
 	PostgresNetworkPolicyName                    = "lightspeed-postgres-server"
 	PostgresServicePort                          = int32(5432)
-	PostgresDefaultUser                          = "postgres"
-	PostgresDefaultDbName                        = "postgres"
+	PostgresLightspeedStackDbName                = "lightspeed-stack"
+	PostgresLlamaStackDbName                     = "llamastack"
 	PostgresSharedBuffers                        = "256MB"
 	PostgresMaxConnections                       = 100
 	OpenStackLightspeedComponentPasswordFileName = "password"
-	PostgresExtensionScript                      = "create-extensions.sh"
+	PostgresUsernameSecretKey                    = "username"
+	PostgresBootstrapScript                      = "postgres_bootstrap.sh"
+	PostgresBootstrapSQLScript                   = "postgres_bootstrap.sql"
 	PostgresConfigKey                            = "postgresql.conf.sample"
-	PostgresBootstrapVolumeMountPath             = "/usr/share/container-scripts/postgresql/start/create-extensions.sh"
+	PostgresBootstrapVolumeMountPath             = "/usr/share/container-scripts/postgresql/start/postgres_bootstrap.sh"
+	PostgresBootstrapSQLVolumeMountPath          = "/usr/share/container-scripts/postgresql/start/postgres_bootstrap.sql"
 	PostgresConfigVolumeMountPath                = "/usr/share/pgsql/postgresql.conf.sample"
 	PostgresDataVolume                           = "postgres-data"
 	PostgresDataVolumeMountPath                  = "/var/lib/pgsql"
 	PostgresDataPVCName                          = "openstack-lightspeed-database"
 	PostgresDataPVCDefaultSize                   = "1Gi"
 	PostgresVarRunVolumeName                     = "lightspeed-postgres-var-run"
 	PostgresVarRunVolumeMountPath                = "/var/run/postgresql"
-	TmpVolumeName                                = "tmp-writable-volume"
-	TmpVolumeMountPath                           = "/tmp"
+
+	// Non-admin user that should be usef by lightspeed-stack and llama-stack (OGX) to access
+	// the PostgreSQL database. This user gets created by the PostgreSQL container by setting
+	// the POSTGRESQL_USER and POSTGRESQL_PASSWORD environment variable.
+	PostgresSQLUsername = "lightspeed-app-user"
+
+	TmpVolumeName      = "tmp-writable-volume"
+	TmpVolumeMountPath = "/tmp"
 	// Health probe settings for the PostgreSQL container.
 	// Startup probe allows up to 300s for initialization (e.g. WAL recovery).
 	// Liveness uses a longer period/timeout to avoid killing a busy-but-healthy instance.
@@ -231,6 +240,7 @@ const (
 	// By recording the resource version of a ConfigMap in a Deployment, StatefulSet, or similar resource,
 	// changes to the referenced ConfigMaps can be detected and trigger rollouts or reconciliation in the operator.
 	PostgresConfigMapResourceVersionAnnotation   = "ols.openshift.io/postgres-configmap-version"
+	PostgresSecretResourceVersionAnnotation      = "ols.openshift.io/postgres-secret-version"
 	VectorDBScriptsConfigMapVersionAnnotation    = "ols.openshift.io/vector-db-scripts-configmap-version"
 	LlamaStackConfigMapResourceVersionAnnotation = "ols.openshift.io/llamastack-configmap-version"
 	LCoreConfigMapResourceVersionAnnotation      = "ols.openshift.io/lcore-configmap-version"
@@ -314,6 +324,11 @@ const (
 //go:embed assets/postgres_bootstrap.sh
 var PostgresBootStrapScriptContent string
 
+// PostgreSQL Bootstrap SQL - creates database, extensions, and schemas
+//
+//go:embed assets/postgres_bootstrap.sql
+var PostgresBootStrapSQLContent string
+
 // PostgreSQL Configuration - SSL and TLS settings
 //
 //go:embed assets/postgres.conf

internal/controller/lcore_config.go

@@ -131,14 +131,14 @@ func buildLCoreDatabaseConfig(h *common_helper.Helper, _ *apiv1beta1.OpenStackLi
 		"postgres": map[string]interface{}{
 			"host":         PostgresServiceName + "." + h.GetBeforeObject().GetNamespace() + ".svc",
 			"port":         PostgresServicePort,
-			"db":           PostgresDefaultDbName,
-			"user":         PostgresDefaultUser,
+			"db":           PostgresLightspeedStackDbName,
+			"user":         "${env.POSTGRESQL_USER}",
 			"ssl_mode":     PostgresDefaultSSLMode,
 			"gss_encmode":  "disable",
 			"ca_cert_path": CABundleMountPath,
 
 			// Environment variable substitution via llama_stack.core.stack.replace_env_vars
-			"password": "${env.POSTGRES_PASSWORD}",
+			"password": "${env.POSTGRESQL_PASSWORD}",
 
 			// Separate schema for LCore to avoid conflicts with App Server
 			"namespace": "lcore",
@@ -163,9 +163,9 @@ func buildLCoreConversationCacheConfig(h *common_helper.Helper, _ *apiv1beta1.Op
 		"postgres": map[string]interface{}{
 			"host":         PostgresServiceName + "." + h.GetBeforeObject().GetNamespace() + ".svc",
 			"port":         PostgresServicePort,
-			"db":           PostgresDefaultDbName,
-			"user":         PostgresDefaultUser,
-			"password":     "${env.POSTGRES_PASSWORD}",
+			"db":           PostgresLightspeedStackDbName,
+			"user":         "${env.POSTGRESQL_USER}",
+			"password":     "${env.POSTGRESQL_PASSWORD}",
 			"ssl_mode":     PostgresDefaultSSLMode,
 			"gss_encmode":  "disable",
 			"ca_cert_path": CABundleMountPath,

internal/controller/lcore_deployment.go

@@ -575,8 +575,9 @@ func buildLlamaStackEnvVars(h *common_helper.Helper, ctx context.Context, instan
 		}
 	}
 
-	// Postgres password for ${env.POSTGRES_PASSWORD} substitution in llama-stack config
-	envVars = append(envVars, buildPostgresPasswordEnvVar())
+	// Postgres credentials for ${env.POSTGRESQL_PASSWORD} and ${env.POSTGRESQL_USER}
+	// substitution in llama-stack config
+	envVars = append(envVars, buildPostgresCredsEnvVars()...)
 
 	// PostgreSQL SSL configuration for OGX (llama-stack).
 	// OGX's PostgresSqlStoreConfig does not support ssl_mode/ca_cert_path fields yet
@@ -615,16 +616,29 @@ func buildLlamaStackEnvVars(h *common_helper.Helper, ctx context.Context, instan
 	return envVars, nil
 }
 
-// buildPostgresPasswordEnvVar returns the POSTGRES_PASSWORD env var sourced from the postgres secret.
-func buildPostgresPasswordEnvVar() corev1.EnvVar {
-	return corev1.EnvVar{
-		Name: "POSTGRES_PASSWORD",
-		ValueFrom: &corev1.EnvVarSource{
-			SecretKeyRef: &corev1.SecretKeySelector{
-				LocalObjectReference: corev1.LocalObjectReference{
-					Name: PostgresSecretName,
+// buildPostgresCredsEnvVars returns the POSTGRES_PASSWORD env var sourced from the postgres secret.
+func buildPostgresCredsEnvVars() []corev1.EnvVar {
+	return []corev1.EnvVar{
+		{
+			Name: "POSTGRESQL_PASSWORD",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: PostgresSecretName,
+					},
+					Key: OpenStackLightspeedComponentPasswordFileName,
+				},
+			},
+		},
+		{
+			Name: "POSTGRESQL_USER",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: PostgresSecretName,
+					},
+					Key: PostgresUsernameSecretKey,
 				},
-				Key: OpenStackLightspeedComponentPasswordFileName,
 			},
 		},
 	}
@@ -637,12 +651,12 @@ func buildLightspeedStackEnvVars(instance *apiv1beta1.OpenStackLightspeed) []cor
 			Name:  "LIGHTSPEED_STACK_LOG_LEVEL",
 			Value: instance.Spec.Logging.LightspeedStackLogLevel,
 		},
-		buildPostgresPasswordEnvVar(),
 	}
 	envVars = append(envVars, corev1.EnvVar{
 		Name:  "RH_SERVER_OKP",
 		Value: fmt.Sprintf("http://%s.%s.svc:%d", OKPServiceName, instance.GetNamespace(), OKPServicePort),
 	})
+	envVars = append(envVars, buildPostgresCredsEnvVars()...)
 	return envVars
 }
 
@@ -754,5 +768,14 @@ func buildConfigMapAnnotations(h *common_helper.Helper, ctx context.Context) (ma
 		annotations[CABundleConfigMapVersionAnnotation] = caBundleVersion
 	}
 
+	postgresSecretVersion, err := getSecretResourceVersion(ctx, h, PostgresSecretName, h.GetBeforeObject().GetNamespace())
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return nil, fmt.Errorf("failed to get postgres secret resource version: %w", err)
+		}
+	} else {
+		annotations[PostgresSecretResourceVersionAnnotation] = postgresSecretVersion
+	}
+
 	return annotations, nil
 }

internal/controller/llama_stack_config.go

@@ -321,10 +321,9 @@ func buildLlamaStackStorage(_ *common_helper.Helper, instance *apiv1beta1.OpenSt
 			"type":     "sql_postgres",
 			"host":     fmt.Sprintf("lightspeed-postgres-server.%s.svc", instance.GetNamespace()),
 			"port":     PostgresServicePort,
-			"user":     "postgres",
-			"password": "${env.POSTGRES_PASSWORD}",
-			// Note: Database name is HARDCODED to "llamastack" in llama-stack's postgres adapter
-			// Not configurable - llama-stack ignores image_name for database selection
+			"user":     "${env.POSTGRESQL_USER}",
+			"password": "${env.POSTGRESQL_PASSWORD}",
+			"db":       PostgresLlamaStackDbName,
 		},
 	}
 

internal/controller/postgres_deployment.go

@@ -63,7 +63,13 @@ func buildPostgresPodTemplateSpec() corev1.PodTemplateSpec {
 	volumeMounts = append(volumeMounts, corev1.VolumeMount{
 		Name:      "secret-" + PostgresBootstrapSecretName,
 		MountPath: PostgresBootstrapVolumeMountPath,
-		SubPath:   PostgresExtensionScript,
+		SubPath:   PostgresBootstrapScript,
+		ReadOnly:  true,
+	})
+	volumeMounts = append(volumeMounts, corev1.VolumeMount{
+		Name:      "secret-" + PostgresBootstrapSecretName,
+		MountPath: PostgresBootstrapSQLVolumeMountPath,
+		SubPath:   PostgresBootstrapSQLScript,
 		ReadOnly:  true,
 	})
 
@@ -120,6 +126,30 @@ func buildPostgresPodTemplateSpec() corev1.PodTemplateSpec {
 		MountPath: TmpVolumeMountPath,
 	})
 
+	envVars := []corev1.EnvVar{
+		{
+			Name:  "POSTGRESQL_DATABASE",
+			Value: PostgresLightspeedStackDbName,
+		},
+		{
+			Name:  "POSTGRESQL_LLAMA_STACK_DATABASE",
+			Value: PostgresLlamaStackDbName,
+		},
+		{
+			Name:  "POSTGRESQL_SHARED_BUFFERS",
+			Value: PostgresSharedBuffers,
+		},
+		{
+			Name:  "POSTGRESQL_MAX_CONNECTIONS",
+			Value: strconv.Itoa(PostgresMaxConnections),
+		},
+		{
+			Name:  "POSTGRESQL_BOOTSTRAP_SQL_FILE",
+			Value: PostgresBootstrapSQLVolumeMountPath,
+		},
+	}
+	envVars = append(envVars, buildPostgresCredsEnvVars()...)
+
 	return corev1.PodTemplateSpec{
 		ObjectMeta: metav1.ObjectMeta{
 			Labels:      generatePostgresSelectorLabels(),
@@ -156,42 +186,7 @@ func buildPostgresPodTemplateSpec() corev1.PodTemplateSpec {
 							corev1.ResourceMemory: resource.MustParse("2Gi"),
 						},
 					},
-					Env: []corev1.EnvVar{
-						{
-							Name:  "POSTGRESQL_USER",
-							Value: PostgresDefaultUser,
-						},
-						{
-							Name:  "POSTGRESQL_DATABASE",
-							Value: PostgresDefaultDbName,
-						},
-						{
-							Name:  "POSTGRESQL_SHARED_BUFFERS",
-							Value: PostgresSharedBuffers,
-						},
-						{
-							Name:  "POSTGRESQL_MAX_CONNECTIONS",
-							Value: strconv.Itoa(PostgresMaxConnections),
-						},
-						{
-							Name: "POSTGRESQL_ADMIN_PASSWORD",
-							ValueFrom: &corev1.EnvVarSource{
-								SecretKeyRef: &corev1.SecretKeySelector{
-									LocalObjectReference: corev1.LocalObjectReference{Name: PostgresSecretName},
-									Key:                  OpenStackLightspeedComponentPasswordFileName,
-								},
-							},
-						},
-						{
-							Name: "POSTGRESQL_PASSWORD",
-							ValueFrom: &corev1.EnvVarSource{
-								SecretKeyRef: &corev1.SecretKeySelector{
-									LocalObjectReference: corev1.LocalObjectReference{Name: PostgresSecretName},
-									Key:                  OpenStackLightspeedComponentPasswordFileName,
-								},
-							},
-						},
-					},
+					Env: envVars,
 				},
 			},
 			Volumes: volumes,
@@ -203,7 +198,7 @@ func buildPostgresProbe(period, timeout, failure, initialDelay int32) *corev1.Pr
 	return &corev1.Probe{
 		ProbeHandler: corev1.ProbeHandler{
 			Exec: &corev1.ExecAction{
-				Command: []string{"pg_isready", "-U", PostgresDefaultUser, "-d", PostgresDefaultDbName},
+				Command: []string{"/bin/sh", "-c", "pg_isready -U $POSTGRESQL_USER -d $POSTGRESQL_DATABASE"},
 			},
 		},
 		InitialDelaySeconds: initialDelay,