Fix PostgreSQL credential security and extract random string generation

internal/controller/assets/postgres_bootstrap.sh

@@ -1,25 +1,22 @@
 #!/bin/bash
-
+# This script prepares databases for lightspeed-stack and llama-stack (OGX) with
+# postgres_bootstrap.sql.
+#
+# Note:
+# - lightspeed-stack database: Auto-created by container image via POSTGRESQL_DATABASE.
+# - llama-stack database: Explicitly created by this script via POSTGRESQL_LLAMA_STACK_DATABASE.
+# - POSTGRESQL_ADMIN_PASSWORD is intentionally not set. The postgres superuser has no password
+#   by default, which restricts it to local connections only — a deliberate security improvement.
+#   Setting POSTGRESQL_ADMIN_PASSWORD would enable remote login for the postgres account.
 set -e
 
 cat /var/lib/pgsql/data/userdata/postgresql.conf
 
-echo "attempting to create llama-stack database and pg_trgm extension if they do not exist"
-
-_psql () { psql --set ON_ERROR_STOP=1 "$@" ; }
-
-# Create database for llama-stack conversation storage
-DB_NAME="llamastack"
-
-echo "SELECT 'CREATE DATABASE $DB_NAME' WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '$DB_NAME')\gexec" | _psql -d "$POSTGRESQL_DATABASE"
-
-# Create pg_trgm extension in default database (for OpenStack Lightspeed conversation cache)
-echo "CREATE EXTENSION IF NOT EXISTS pg_trgm;" | _psql -d "$POSTGRESQL_DATABASE"
-
-# Create pg_trgm extension in llama-stack database (for text search if needed)
-echo "CREATE EXTENSION IF NOT EXISTS pg_trgm;" | _psql -d $DB_NAME
+echo "Bootstrapping PostgreSQL databases and permissions"
 
-# Create schemas for isolating different components' data
-echo "CREATE SCHEMA IF NOT EXISTS lcore;" | _psql -d "$POSTGRESQL_DATABASE"
-echo "CREATE SCHEMA IF NOT EXISTS quota;" | _psql -d "$POSTGRESQL_DATABASE"
-echo "CREATE SCHEMA IF NOT EXISTS conversation_cache;" | _psql -d "$POSTGRESQL_DATABASE"
+psql \
+    -v ON_ERROR_STOP=1 \
+    -v postgresql_user="$POSTGRESQL_USER" \
+    -v postgresql_lightspeed_stack_database="$POSTGRESQL_DATABASE" \
+    -v postgresql_llama_stack_database="$POSTGRESQL_LLAMA_STACK_DATABASE" \
+    -f "$POSTGRESQL_BOOTSTRAP_SQL_FILE"

internal/controller/assets/postgres_bootstrap.sql

@@ -0,0 +1,29 @@
+-- 1) LIGHTSPEED STACK DATABASE CONFIGURATION ---------------------------------
+\c :postgresql_lightspeed_stack_database
+
+-- PostgreSQL 15+ removed the default CREATE/USAGE grants on the public schema.
+-- Therefore we have to explicitly grant the permissions to the user.
+GRANT USAGE, CREATE ON SCHEMA public TO :"postgresql_user";
+
+-- lightspeed-stack requires rights to create additional schemas under its database
+GRANT CREATE ON DATABASE :"postgresql_lightspeed_stack_database" TO :"postgresql_user";
+
+-- pg_trgm is the trigram similarity extension for PostgreSQL (enables e.g. fuzzy text search)
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+-------------------------------------------------------------------------------
+
+-- 2) LLAMA STACK DATABASE CONFIGURATION --------------------------------------
+-- Create postgresql_llama_stack_database.
+SELECT format('CREATE DATABASE %I', :'postgresql_llama_stack_database')
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = :'postgresql_llama_stack_database')\gexec
+
+-- Connect and configure postgresql_llama_stack_database
+\c :postgresql_llama_stack_database
+
+-- PostgreSQL 15+ removed the default CREATE/USAGE grants on the public schema.
+-- Therefore we have to explicitly grant the permissions to the user.
+GRANT USAGE, CREATE ON SCHEMA public TO :"postgresql_user";
+
+-- pg_trgm is the trigram similarity extension for PostgreSQL (enables e.g. fuzzy text search)
+CREATE EXTENSION IF NOT EXISTS pg_trgm;
+-------------------------------------------------------------------------------

internal/controller/common.go

@@ -18,7 +18,9 @@ package controller
 
 import (
 	"context"
+	"crypto/rand"
 	_ "embed"
+	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -82,6 +84,16 @@ func getConfigMapResourceVersion(ctx context.Context, h *common_helper.Helper, n
 	return cm.ResourceVersion, nil
 }
 
+// getSecretResourceVersion retrieves the resource version of a Secret.
+func getSecretResourceVersion(ctx context.Context, h *common_helper.Helper, name string, namespace string) (string, error) {
+	secret := &corev1.Secret{}
+	err := h.GetClient().Get(ctx, types.NamespacedName{Name: name, Namespace: namespace}, secret)
+	if err != nil {
+		return "", fmt.Errorf("failed to get secret %s: %w", name, err)
+	}
+	return secret.ResourceVersion, nil
+}
+
 // providerNameToEnvVarName converts a provider name to a valid environment variable name.
 // It uppercases the string and replaces hyphens and dots with underscores.
 func providerNameToEnvVarName(providerName string) string {
@@ -174,3 +186,17 @@ func getDeployment(ctx context.Context, h *common_helper.Helper, name string, na
 
 	return deployment, nil
 }
+
+// generateRandomString generates a random hex string of the given length.
+func generateRandomString(secretLength int) (string, error) {
+	// Ceiling division: hex.EncodeToString doubles the byte count, so we need ceil(secretLength/2) bytes.
+	randDataLen := (secretLength + 1) / 2
+
+	b := make([]byte, randDataLen)
+	if _, err := rand.Read(b); err != nil {
+		return "", fmt.Errorf("failed to generate secret: %w", err)
+	}
+
+	randString := hex.EncodeToString(b)
+	return randString[:secretLength], nil
+}

internal/controller/common_test.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2026.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"testing"
+)
+
+func TestGenerateRandomStringLength(t *testing.T) {
+	tests := []struct {
+		name   string
+		length int
+	}{
+		{
+			name:   "Zero length",
+			length: 0,
+		},
+		{
+			name:   "Odd length 1",
+			length: 1,
+		},
+		{
+			name:   "Even length 2",
+			length: 2,
+		},
+		{
+			name:   "Odd length 7",
+			length: 7,
+		},
+		{
+			name:   "Even length 8",
+			length: 8,
+		},
+		{
+			name:   "Odd length 15",
+			length: 15,
+		},
+		{
+			name:   "Even length 16",
+			length: 16,
+		},
+		{
+			name:   "Even length 32",
+			length: 32,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := generateRandomString(tt.length)
+			if err != nil {
+				t.Errorf("generateRandomString(%d) unexpected error: %v", tt.length, err)
+			}
+			if len(result) != tt.length {
+				t.Errorf("generateRandomString(%d) returned length %d, want %d", tt.length, len(result), tt.length)
+			}
+		})
+	}
+}
+
+func TestGenerateRandomStringHexCharacters(t *testing.T) {
+	result, err := generateRandomString(32)
+	if err != nil {
+		t.Fatalf("generateRandomString(32) unexpected error: %v", err)
+	}
+	for i, c := range result {
+		if (c < '0' || c > '9') && (c < 'a' || c > 'f') {
+			t.Errorf("generateRandomString(32) character at index %d is %q, not a lowercase hex character", i, c)
+		}
+	}
+}
+
+func TestGenerateRandomStringUniqueness(t *testing.T) {
+	const length = 16
+	a, err := generateRandomString(length)
+	if err != nil {
+		t.Fatalf("first call unexpected error: %v", err)
+	}
+	b, err := generateRandomString(length)
+	if err != nil {
+		t.Fatalf("second call unexpected error: %v", err)
+	}
+	if a == b {
+		t.Errorf("generateRandomString(%d) returned identical values across two calls: %q", length, a)
+	}
+}

internal/controller/constants.go

@@ -50,23 +50,32 @@ const (
 	PostgresConfigMapName                        = "lightspeed-postgres-conf"
 	PostgresNetworkPolicyName                    = "lightspeed-postgres-server"
 	PostgresServicePort                          = int32(5432)
-	PostgresDefaultUser                          = "postgres"
-	PostgresDefaultDbName                        = "postgres"
+	PostgresLightspeedStackDbName                = "lightspeed-stack"
+	PostgresLlamaStackDbName                     = "llamastack"
 	PostgresSharedBuffers                        = "256MB"
 	PostgresMaxConnections                       = 100
 	OpenStackLightspeedComponentPasswordFileName = "password"
-	PostgresExtensionScript                      = "create-extensions.sh"
+	PostgresUsernameSecretKey                    = "username"
+	PostgresBootstrapScript                      = "postgres_bootstrap.sh"
+	PostgresBootstrapSQLScript                   = "postgres_bootstrap.sql"
 	PostgresConfigKey                            = "postgresql.conf.sample"
-	PostgresBootstrapVolumeMountPath             = "/usr/share/container-scripts/postgresql/start/create-extensions.sh"
+	PostgresBootstrapVolumeMountPath             = "/usr/share/container-scripts/postgresql/start/postgres_bootstrap.sh"
+	PostgresBootstrapSQLVolumeMountPath          = "/usr/share/container-scripts/postgresql/start/postgres_bootstrap.sql"
 	PostgresConfigVolumeMountPath                = "/usr/share/pgsql/postgresql.conf.sample"
 	PostgresDataVolume                           = "postgres-data"
 	PostgresDataVolumeMountPath                  = "/var/lib/pgsql"
 	PostgresDataPVCName                          = "openstack-lightspeed-database"
 	PostgresDataPVCDefaultSize                   = "1Gi"
 	PostgresVarRunVolumeName                     = "lightspeed-postgres-var-run"
 	PostgresVarRunVolumeMountPath                = "/var/run/postgresql"
-	TmpVolumeName                                = "tmp-writable-volume"
-	TmpVolumeMountPath                           = "/tmp"
+
+	// Non-admin user that should be used by lightspeed-stack and llama-stack (OGX) to access
+	// the PostgreSQL database. This user gets created by the PostgreSQL container by setting
+	// the POSTGRESQL_USER and POSTGRESQL_PASSWORD environment variable.
+	PostgresSQLUsername = "lightspeed-app-user"
+
+	TmpVolumeName      = "tmp-writable-volume"
+	TmpVolumeMountPath = "/tmp"
 	// Health probe settings for the PostgreSQL container.
 	// Startup probe allows up to 300s for initialization (e.g. WAL recovery).
 	// Liveness uses a longer period/timeout to avoid killing a busy-but-healthy instance.
@@ -231,6 +240,7 @@ const (
 	// By recording the resource version of a ConfigMap in a Deployment, StatefulSet, or similar resource,
 	// changes to the referenced ConfigMaps can be detected and trigger rollouts or reconciliation in the operator.
 	PostgresConfigMapResourceVersionAnnotation   = "ols.openshift.io/postgres-configmap-version"
+	PostgresSecretResourceVersionAnnotation      = "ols.openshift.io/postgres-secret-version" // #nosec G101 -- annotation key, not a credential
 	VectorDBScriptsConfigMapVersionAnnotation    = "ols.openshift.io/vector-db-scripts-configmap-version"
 	LlamaStackConfigMapResourceVersionAnnotation = "ols.openshift.io/llamastack-configmap-version"
 	LCoreConfigMapResourceVersionAnnotation      = "ols.openshift.io/lcore-configmap-version"
@@ -314,6 +324,11 @@ const (
 //go:embed assets/postgres_bootstrap.sh
 var PostgresBootStrapScriptContent string
 
+// PostgreSQL Bootstrap SQL - creates database, extensions, and schemas
+//
+//go:embed assets/postgres_bootstrap.sql
+var PostgresBootStrapSQLContent string
+
 // PostgreSQL Configuration - SSL and TLS settings
 //
 //go:embed assets/postgres.conf

internal/controller/lcore_config.go

@@ -131,14 +131,14 @@ func buildLCoreDatabaseConfig(h *common_helper.Helper, _ *apiv1beta1.OpenStackLi
 		"postgres": map[string]interface{}{
 			"host":         PostgresServiceName + "." + h.GetBeforeObject().GetNamespace() + ".svc",
 			"port":         PostgresServicePort,
-			"db":           PostgresDefaultDbName,
-			"user":         PostgresDefaultUser,
+			"db":           PostgresLightspeedStackDbName,
+			"user":         "${env.POSTGRESQL_USER}",
 			"ssl_mode":     PostgresDefaultSSLMode,
 			"gss_encmode":  "disable",
 			"ca_cert_path": CABundleMountPath,
 
 			// Environment variable substitution via llama_stack.core.stack.replace_env_vars
-			"password": "${env.POSTGRES_PASSWORD}",
+			"password": "${env.POSTGRESQL_PASSWORD}",
 
 			// Separate schema for LCore to avoid conflicts with App Server
 			"namespace": "lcore",
@@ -163,9 +163,9 @@ func buildLCoreConversationCacheConfig(h *common_helper.Helper, _ *apiv1beta1.Op
 		"postgres": map[string]interface{}{
 			"host":         PostgresServiceName + "." + h.GetBeforeObject().GetNamespace() + ".svc",
 			"port":         PostgresServicePort,
-			"db":           PostgresDefaultDbName,
-			"user":         PostgresDefaultUser,
-			"password":     "${env.POSTGRES_PASSWORD}",
+			"db":           PostgresLightspeedStackDbName,
+			"user":         "${env.POSTGRESQL_USER}",
+			"password":     "${env.POSTGRESQL_PASSWORD}",
 			"ssl_mode":     PostgresDefaultSSLMode,
 			"gss_encmode":  "disable",
 			"ca_cert_path": CABundleMountPath,

internal/controller/lcore_deployment.go

@@ -557,8 +557,9 @@ func buildLlamaStackEnvVars(h *common_helper.Helper, ctx context.Context, instan
 		}
 	}
 
-	// Postgres password for ${env.POSTGRES_PASSWORD} substitution in llama-stack config
-	envVars = append(envVars, buildPostgresPasswordEnvVar())
+	// Postgres credentials for ${env.POSTGRESQL_PASSWORD} and ${env.POSTGRESQL_USER}
+	// substitution in llama-stack config
+	envVars = append(envVars, buildPostgresCredsEnvVars()...)
 
 	// PostgreSQL SSL configuration for OGX (llama-stack).
 	// OGX's PostgresSqlStoreConfig does not support ssl_mode/ca_cert_path fields yet
@@ -597,16 +598,30 @@ func buildLlamaStackEnvVars(h *common_helper.Helper, ctx context.Context, instan
 	return envVars, nil
 }
 
-// buildPostgresPasswordEnvVar returns the POSTGRES_PASSWORD env var sourced from the postgres secret.
-func buildPostgresPasswordEnvVar() corev1.EnvVar {
-	return corev1.EnvVar{
-		Name: "POSTGRES_PASSWORD",
-		ValueFrom: &corev1.EnvVarSource{
-			SecretKeyRef: &corev1.SecretKeySelector{
-				LocalObjectReference: corev1.LocalObjectReference{
-					Name: PostgresSecretName,
+// buildPostgresCredsEnvVars returns the POSTGRESQL_PASSWORD and POSTGRESQL_USER env var sourced from
+// the postgres secret generated by the operator.
+func buildPostgresCredsEnvVars() []corev1.EnvVar {
+	return []corev1.EnvVar{
+		{
+			Name: "POSTGRESQL_PASSWORD",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: PostgresSecretName,
+					},
+					Key: OpenStackLightspeedComponentPasswordFileName,
+				},
+			},
+		},
+		{
+			Name: "POSTGRESQL_USER",
+			ValueFrom: &corev1.EnvVarSource{
+				SecretKeyRef: &corev1.SecretKeySelector{
+					LocalObjectReference: corev1.LocalObjectReference{
+						Name: PostgresSecretName,
+					},
+					Key: PostgresUsernameSecretKey,
 				},
-				Key: OpenStackLightspeedComponentPasswordFileName,
 			},
 		},
 	}
@@ -619,12 +634,12 @@ func buildLightspeedStackEnvVars(instance *apiv1beta1.OpenStackLightspeed) []cor
 			Name:  "LIGHTSPEED_STACK_LOG_LEVEL",
 			Value: instance.Spec.Logging.LightspeedStackLogLevel,
 		},
-		buildPostgresPasswordEnvVar(),
 	}
 	envVars = append(envVars, corev1.EnvVar{
 		Name:  "RH_SERVER_OKP",
 		Value: fmt.Sprintf("http://%s.%s.svc:%d", OKPServiceName, instance.GetNamespace(), OKPServicePort),
 	})
+	envVars = append(envVars, buildPostgresCredsEnvVars()...)
 	return envVars
 }
 
@@ -736,5 +751,14 @@ func buildConfigMapAnnotations(h *common_helper.Helper, ctx context.Context) (ma
 		annotations[CABundleConfigMapVersionAnnotation] = caBundleVersion
 	}
 
+	postgresSecretVersion, err := getSecretResourceVersion(ctx, h, PostgresSecretName, h.GetBeforeObject().GetNamespace())
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return nil, fmt.Errorf("failed to get postgres secret resource version: %w", err)
+		}
+	} else {
+		annotations[PostgresSecretResourceVersionAnnotation] = postgresSecretVersion
+	}
+
 	return annotations, nil
 }