functions

#!/bin/bash

# set root repo relatively to a test dir
ROOT_REPO=${ROOT_REPO:-$(realpath ../../..)}
CERT_MANAGER_VER="1.20.0"
CHAOS_MESH_VER="2.5.1"
BUSYBOX_VER="1.36"
test_name=$(basename "$(pwd)")
source "${ROOT_REPO}/e2e-tests/vars.sh"

if oc get projects 2>/dev/null; then
	OPENSHIFT=4
fi

init_temp_dir() {
	rm -rf "$TEMP_DIR"
	mkdir -p "$TEMP_DIR"
}

create_namespace() {
	local namespace=$1

	if [[ $OPENSHIFT ]]; then
		set -o pipefail
		if [[ $OPERATOR_NS ]] && (oc get project "$OPERATOR_NS" -o json >/dev/null 2>&1 | jq -r '.metadata.name' >/dev/null 2>&1); then
			oc delete --grace-period=0 --force=true project "$namespace" && sleep 120 || :
		else
			oc delete project "$namespace" && sleep 40 || :
		fi
		wait_for_delete "project/$namespace"

		oc new-project "$namespace"
		oc project "$namespace"
		oc adm policy add-scc-to-user hostaccess -z default || :
	else
		kubectl delete namespace $namespace --ignore-not-found || :
		kubectl wait --for=delete namespace "$namespace" || :
		kubectl create namespace $namespace
	fi
}

check_operator_panic() {
	local operator_pod=$(get_operator_pod)
	local panic_log
	panic_log=$(kubectl logs -n "${OPERATOR_NS:-$NAMESPACE}" "$operator_pod" -c operator | grep -A 100 "Observed a panic" || true)
	if [ -n "$panic_log" ]; then
		echo "Detected panic in operator:"
		echo "$panic_log"
		exit 1
	fi
}

deploy_operator() {
	local cw_prefix=""

	destroy_operator

	if [[ $OPERATOR_NS ]]; then
		create_namespace $OPERATOR_NS
		cw_prefix="cw-"
	fi
	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/crd.yaml"
	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/${cw_prefix}rbac.yaml"

	local disable_telemetry=true
	if [ "${test_name}" == "telemetry-transfer" ]; then
		disable_telemetry=false
	fi
	yq eval '.spec.template.spec.containers[0].image = "'${IMAGE}'"' "${DEPLOY_DIR}/${cw_prefix}operator.yaml" \
		| yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "'${disable_telemetry}'"' - \
		| yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="LOG_LEVEL") | .value) = "DEBUG"' - \
		| yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="PGO_FEATURE_GATES") | .value) = "'${PGO_FEATURE_GATES}'"' - \
		| kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f -
}

update_operator() {
	local cw_prefix=""

	if [[ $OPERATOR_NS ]]; then
		cw_prefix="cw-"
	fi

	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/crd.yaml"
	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "${DEPLOY_DIR}/${cw_prefix}rbac.yaml"

	local disable_telemetry=true
	if [ "${test_name}" == "telemetry-transfer" ]; then
		disable_telemetry=false
	fi

	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" patch deployment percona-postgresql-operator -p \
		'{"spec":{"template":{"spec":{"containers":[{"name":"operator","image":"'${IMAGE}'"}]}}}}'
}

deploy_operator_gh() {
	local git_tag="$1"
	local cw_prefix=""

	destroy_operator

	if [[ $OPERATOR_NS ]]; then
		create_namespace $OPERATOR_NS
		cw_prefix="cw-"
	fi
	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "https://raw.githubusercontent.com/percona/percona-postgresql-operator/${git_tag}/deploy/crd.yaml"
	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply --server-side --force-conflicts -f "https://raw.githubusercontent.com/percona/percona-postgresql-operator/${git_tag}/deploy/${cw_prefix}rbac.yaml"

	curl -s "https://raw.githubusercontent.com/percona/percona-postgresql-operator/${git_tag}/deploy/${cw_prefix}operator.yaml" >"${TEMP_DIR}/${cw_prefix}operator_${git_tag}.yaml"

	local docker_hub_org=$(echo "$IMAGE" | $sed -E 's|(.*/)?([^/]+)/[^/]+(:.*)?|\2|')
	yq eval '.spec.template.spec.containers[0].image = "'${REGISTRY_NAME_FULL}''$docker_hub_org'/percona-postgresql-operator:'${git_tag#v}'"' \
		"${TEMP_DIR}/${cw_prefix}operator_${git_tag}.yaml" \
		| kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f -
}

remove_all_finalizers() {
	resource_types=("pg-restore" "pg-backup" "pg")

	for resource in "${resource_types[@]}"; do
		echo "removing all finalizers for $resource resources"

		kubectl -n "${NAMESPACE}" get "$resource" -o json | jq '.items[] | .metadata.name' -r | while IFS= read -r name; do
			kubectl -n "${NAMESPACE}" delete "$resource" "$name" --wait=0

			if [[ $(kubectl -n "${NAMESPACE}" get "$resource" "$name" -o yaml | yq '.metadata.finalizers | length') == "0" ]]; then
				continue
			fi
			kubectl -n "${NAMESPACE}" patch "$resource" "$name" --type=json -p='[{"op": "remove", "path": "/metadata/finalizers"}]'
		done
	done
}

wait_for_ready_containers() {
	local pod_prefix="$1"
	local target_count="$2"
	local namespace="${NAMESPACE}"
	local max_wait_seconds=300
	local check_interval=5
	local elapsed_time=0

	if [[ -z $pod_prefix || -z $target_count || -z $namespace ]]; then
		echo "Error: Missing arguments." >&2
		echo "Usage: wait_for_ready_containers <pod_name_prefix> <target_ready_count> <namespace>" >&2
		return 1
	fi

	echo "Waiting for pods starting with '$pod_prefix' in namespace '$namespace' to have $target_count ready containers (Max ${max_wait_seconds}s)..."

	while [[ $elapsed_time -lt $max_wait_seconds ]]; do
		local target_pods
		# Get pods that match the prefix AND are running
		target_pods=$(kubectl get pods -n "$namespace" --field-selector=status.phase=Running --output=json \
			| jq -r ".items[] | select(.metadata.name | startswith(\"$pod_prefix\")) | .metadata.name")
		# If no running pods match the prefix, something might be wrong, but we'll keep waiting.
		if [[ -z $target_pods ]]; then
			echo "No running pods found with prefix '$pod_prefix'. Waiting..."
			sleep "$check_interval"
			elapsed_time=$((elapsed_time + check_interval))
			continue
		fi

		local ready_count=0
		local total_matches=0

		# Check each pod individually
		for pod_name in $target_pods; do
			total_matches=$((total_matches + 1))
			current_ready=$(kubectl get pod "$pod_name" -n "$namespace" -o json 2>/dev/null \
				| jq '.status.containerStatuses | map(select(.ready == true)) | length')

			if [[ $current_ready -eq $target_count ]]; then
				ready_count=$((ready_count + 1))
			fi
		done

		if [[ $ready_count -eq $total_matches ]]; then
			echo "Success: All $total_matches pods now have $target_count ready containers."
			return 0
		fi

		echo "Current status: $ready_count of $total_matches pods have $target_count ready containers. Waiting ${check_interval}s..."

		sleep "$check_interval"
		elapsed_time=$((elapsed_time + check_interval))
	done

	echo "Error: Timeout reached! After ${max_wait_seconds} seconds, not all pods reached $target_count ready containers." >&2
	return 1
}

destroy_operator() {
	kubectl -n "${OPERATOR_NS:-$NAMESPACE}" delete deployment percona-postgresql-operator --force --grace-period=0 || true
	if [[ $OPERATOR_NS ]]; then
		kubectl delete namespace $OPERATOR_NS --force --grace-period=0 || true
	fi
}

get_operator_pod() {
	echo $(kubectl get pods -n "${OPERATOR_NS:-$NAMESPACE}" --selector=app.kubernetes.io/name=percona-postgresql-operator -o jsonpath='{.items[].metadata.name}')
}

retry() {
	local max=$1
	local delay=$2
	shift 2 # cut delay and max args
	local n=1

	until "$@"; do
		if [[ $n -ge $max ]]; then
			echo "The command ${*} has failed after $n attempts."
			exit 1
		fi
		((n++))
		sleep $delay
	done
}

deploy_minio() {
	local access_key
	local secret_key
	access_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)"
	secret_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)"

	helm uninstall -n "${NAMESPACE}" minio-service || :
	helm repo remove minio || :
	helm repo add minio https://charts.min.io/
	retry 10 60 helm install minio-service \
		-n "${NAMESPACE}" \
		--version "${MINIO_VER}" \
		--set replicas=1 \
		--set mode=standalone \
		--set resources.requests.memory=256Mi \
		--set rootUser=rootuser \
		--set rootPassword=rootpass123 \
		--set "users[0].accessKey"="$(printf '%q' "$(printf '%q' "$access_key")")" \
		--set "users[0].secretKey"="$(printf '%q' "$(printf '%q' "$secret_key")")" \
		--set "users[0].policy"=consoleAdmin \
		--set service.type=ClusterIP \
		--set configPathmc=/tmp/.minio/ \
		--set persistence.size=2G \
		--set securityContext.enabled=false \
		minio/minio
	MINIO_POD=$(kubectl -n "${NAMESPACE}" get pods --selector=release=minio-service -o 'jsonpath={.items[].metadata.name}')
	wait_pod $MINIO_POD

	# create bucket
	kubectl -n "${NAMESPACE}" run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
		bash -c "AWS_ACCESS_KEY_ID='$access_key' AWS_SECRET_ACCESS_KEY='$secret_key' AWS_DEFAULT_REGION=us-east-1 \
		/usr/bin/aws --endpoint-url http://minio-service:9000 s3 mb s3://operator-testing"
}

get_repo_auth() {
	local repo=$1
	local type=$2
	local key=$3
	local secret=$4

	auth="${repo}-${type}-key=${key}\n${repo}-${type}-key-secret=${secret}"
	if [[ "$type" == "azure" ]]; then
		auth="${repo}-${type}-account=${key}\n${repo}-${type}-key=${secret}"
	elif [[ "$type" == "gcs" ]]; then
		auth="${repo}-${type}-key=${key}"
	fi

	echo "\n$auth"
}

deploy_s3_secrets() {
	set +o xtrace

	local platform=${1:-$(detect_k8s_platform)}
	local repo_auth=""
	local secret_args=()

	if [[ $platform == "digitalocean" ]]; then
		key=$(yq eval 'select(.metadata.name=="*spaces*").data.ACCESS_KEY_ID' "$TESTS_CONFIG_DIR/cloud-secret.yml" | base64 -d)
		secret=$(yq eval 'select(.metadata.name=="*spaces*").data.SECRET_ACCESS_KEY' "$TESTS_CONFIG_DIR/cloud-secret.yml" | base64 -d)
		repo_auth+=$(get_repo_auth repo1 s3 $key $secret)

		case $test_name in
			"demand-backup")
				repo_auth+=$(get_repo_auth repo3 s3 $key $secret)
			;;
			"scheduled-backup")
				repo_auth+=$(get_repo_auth repo2 s3 $key $secret)
				repo_auth+=$(get_repo_auth repo3 s3 $key $secret)
			;;
		esac
	else
		key=$(yq eval 'select(.metadata.name=="*s3*").data.AWS_ACCESS_KEY_ID' "$TESTS_CONFIG_DIR/cloud-secret.yml" | base64 -d)
		secret=$(yq eval 'select(.metadata.name=="*s3*").data.AWS_SECRET_ACCESS_KEY' "$TESTS_CONFIG_DIR/cloud-secret.yml" | base64 -d)
		azure_key=$(yq eval 'select(.metadata.name=="azure*").data.AZURE_STORAGE_ACCOUNT_NAME' "$TESTS_CONFIG_DIR/cloud-secret.yml" | base64 -d)
		azure_secret=$(yq eval 'select(.metadata.name=="azure*").data.AZURE_STORAGE_ACCOUNT_KEY' "$TESTS_CONFIG_DIR/cloud-secret.yml" | base64 -d)
		repo_auth+=$(get_repo_auth repo1 s3 $key $secret)

		case $test_name in
			"demand-backup")
				repo_auth+=$(get_repo_auth repo3 azure $azure_key $azure_secret)
			;;
			"scheduled-backup")
				repo_auth+=$(get_repo_auth repo2 gcs "/etc/pgbackrest/conf.d/gcs-key.json")
				repo_auth+=$(get_repo_auth repo3 azure $azure_key $azure_secret)
				yq eval '.stringData["credentials.json"]' "$TESTS_CONFIG_DIR/cloud-secret-minio-gw.yml" > "$TEMP_DIR/gcs-key.json"
				secret_args+=(--from-file=gcs-key.json="$TEMP_DIR/gcs-key.json")
			;;
		esac
	fi

	printf "[global]%b" "$repo_auth" > "$TEMP_DIR/pgbackrest-secret.ini"
	secret_args+=(--from-file=cloud.conf="$TEMP_DIR/pgbackrest-secret.ini")
	kubectl -n "$NAMESPACE" create secret generic "${test_name}-pgbackrest-secrets" "${secret_args[@]}"

	case $test_name in
		"custom-extensions" | "builtin-extensions" | major-upgrade* )
			kubectl -n "$NAMESPACE" apply -f "$TESTS_CONFIG_DIR/cloud-secret.yml"
			kubectl -n "$NAMESPACE" apply -f "$TESTS_CONFIG_DIR/minio-secret.yml"
			;;
	esac

	set -o xtrace
}

deploy_client() {
	kubectl -n "${NAMESPACE}" apply -f "${TESTS_CONFIG_DIR}/client.yaml"
}

deploy_cmctl() {
	envsubst < "${TESTS_CONFIG_DIR}/cmctl.yml" | kubectl -n "${NAMESPACE}" apply -f -
	kubectl -n "${NAMESPACE}" wait deployment cmctl --for=condition=available --timeout=60s
}

get_client_pod() {
	kubectl -n ${NAMESPACE} get pods --selector=name=pg-client -o 'jsonpath={.items[].metadata.name}'
}

get_cr() {
	local cr_name=$1
	local repo_path=$2
	local source_path=$3
	if [ -z "$cr_name" ]; then
		cr_name=${test_name}
	fi

	local platform="$(detect_k8s_platform)"
	local cr_file="$TEST_CONFIG_DIR/${cr_name}.yaml"
	local spaces_cr_file="$TEST_CONFIG_DIR/${cr_name}-spaces.yaml"
	local aks_cr_file="$TEST_CONFIG_DIR/${cr_name}-aks.yaml"
	if [[ $platform == "digitalocean" && -f "$spaces_cr_file" ]]; then
		cr_file=$spaces_cr_file
	elif [[ $platform == "aks" && -f "$aks_cr_file" ]]; then	
		cr_file=$aks_cr_file
	fi

	local crs=("$DEPLOY_DIR/cr.yaml")
	if [[ -f "$cr_file" ]]; then
		crs+=($cr_file)
	fi

	yq eval-all '
		select(fileIndex == 0) * (select(fileIndex == 1) // {}) |
		.metadata.name = "'${cr_name}'" |
		.metadata.labels = {"e2e":"'${cr_name}'"} |
		.spec.postgresVersion = '$PG_VER' |
		.spec.users += [{"name":"postgres","password":{"type":"AlphaNumeric"}}] |
		.spec.users += [{"name":"'${cr_name}'","password":{"type":"AlphaNumeric"}}] |
		.spec.image = "'$IMAGE_POSTGRESQL'" |
		.spec.initContainer.image = "'$IMAGE'" |
		.spec.backups.pgbackrest.image = "'$IMAGE_BACKREST'" |
		.spec.proxy.pgBouncer.image = "'$IMAGE_PGBOUNCER'" |
		.spec.pmm.image = "'$IMAGE_PMM_CLIENT'" |
		.spec.pmm.secret = "'${cr_name}'-pmm-secret" |
		.spec.pmm.customClusterName = "'${cr_name}'-pmm-custom-name" |
		.spec.pmm.postgresParams = "--environment=dev-postgres"
	' "${crs[@]}" > "$TEMP_DIR/cr.yaml"

	$sed -i "s|<repo-path>|$repo_path|g" "$TEMP_DIR/cr.yaml"
	$sed -i "s|<source-path>|$source_path|g" "$TEMP_DIR/cr.yaml"
	$sed -i "s|<bucket>|$BUCKET|g" "$TEMP_DIR/cr.yaml"
	$sed -i "s|<image>|$IMAGE|g" "$TEMP_DIR/cr.yaml"

	if [[ $OPENSHIFT ]]; then
		yq eval -i '.spec.openshift = true' "$TEMP_DIR/cr.yaml"
	fi

	cat "$TEMP_DIR/cr.yaml"
}

run_comand_on_pod() {
	local command=${1}

	kubectl -n ${NAMESPACE} exec $(get_client_pod) -- \
		bash -c "$command"
}

run_psql_local() {
	local command=${1}
	local uri=${2}
	local driver=${3:-postgres}

	kubectl -n ${NAMESPACE} exec $(get_client_pod) -- \
		bash -c "printf '$command\n' | psql -v ON_ERROR_STOP=1 -t -q $driver://'$uri'"
}

run_psql() {
	local command=${1}
	local uri=${2}
	local password=${3}

	kubectl -n ${NAMESPACE} exec $(get_client_pod) -- \
		bash -c "printf '$command\n' | PGPASSWORD="\'$password\'" psql -v ON_ERROR_STOP=1 -t -q $uri"
}

get_psql_user_pass() {
	local secret_name=${1}

	kubectl -n ${NAMESPACE} get "secret/${secret_name}" --template='{{.data.password | base64decode}}'
}

get_pgbouncer_host() {
	local secret_name=${1}

	kubectl -n ${NAMESPACE} get "secret/${secret_name}" -o jsonpath={.data.pgbouncer-host} | base64 -d
}

get_psql_user_host() {
	local secret_name=${1}

	kubectl -n ${NAMESPACE} get "secret/${secret_name}" --template='{{.data.host | base64decode }}'
}

get_aws_access_key() {
	local secret_name=${1}

	kubectl -n ${NAMESPACE} get "secret/${secret_name}" --template='{{.data.AWS_SECRET_ACCESS_KEY | base64decode }}'
}

get_aws_access_key_id() {
	local secret_name=${1}

	kubectl -n ${NAMESPACE} get "secret/${secret_name}" --template='{{.data.AWS_ACCESS_KEY_ID | base64decode }}'
}

get_psql_user_host() {
	local secret_name=${1}

	kubectl -n ${NAMESPACE} get "secret/${secret_name}" --template='{{.data.host | base64decode }}'
}
get_instance_set_pods() {
	local instance=${1:-instance1}

	kubectl get pods -n ${NAMESPACE} --selector postgres-operator.crunchydata.com/instance-set=${instance} -o custom-columns='NAME:.metadata.name' --no-headers
}

copy_custom_extensions_form_aws() {
	set +o xtrace

	access_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_ACCESS_KEY_ID}' | base64 -d)"
	secret_key="$(kubectl -n "${NAMESPACE}" get secret minio-secret -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 -d)"

	kubectl -n "${NAMESPACE}" run -i --rm aws-cli \
		--image=perconalab/awscli \
		--restart=Never -- \
		bash -c "
		AWS_ACCESS_KEY_ID=$(get_aws_access_key_id aws-s3-secret) \
		AWS_SECRET_ACCESS_KEY=$(get_aws_access_key aws-s3-secret) \
		AWS_DEFAULT_REGION=eu-central-1 \
		/usr/bin/aws --endpoint-url https://s3.amazonaws.com s3 cp s3://pg-extensions/ /tmp/ --recursive &&

		AWS_ACCESS_KEY_ID='${access_key}' \
		AWS_SECRET_ACCESS_KEY='${secret_key}' \
		AWS_DEFAULT_REGION=us-east-1 \
		/usr/bin/aws --endpoint-url http://minio-service:9000 s3 cp /tmp/ s3://operator-testing/ --recursive
	  "
	set -o xtrace
}

get_psql_pod_host() {
	local pod=${1}

	echo "${pod}.${test_name}-pods.${NAMESPACE}.svc"
}

wait_pod() {
	local pod=$1

	set +o xtrace
	retry=0
	echo -n $pod
	until kubectl get pod/$pod -n "${NAMESPACE}" -o jsonpath='{.status.containerStatuses[0].ready}' 2>/dev/null | grep 'true'; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 360 ]; then
			kubectl describe pod/$pod -n "${NAMESPACE}"
			kubectl logs $pod -n "${NAMESPACE}"
			kubectl logs $(get_operator_pod) ${OPERATOR_NS:+-n $OPERATOR_NS} \
				| grep -v 'level=info' \
				| grep -v 'level=debug' \
				| grep -v 'Getting tasks for pod' \
				| grep -v 'Getting pods from source' \
				| tail -100
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	set -o xtrace
}

get_service_ip() {
	local service=$1
	while (kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.spec.type}' 2>&1 || :) | grep -q NotFound; do
		sleep 1
	done
	if [ "$(kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.spec.type}')" = "ClusterIP" ]; then
		kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.spec.clusterIP}'
		return
	fi
	until kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[]}' 2>&1 | egrep -q "hostname|ip"; do
		sleep 1
	done
	kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[].ip}'
	kubectl get service/$service -n "${NAMESPACE}" -o 'jsonpath={.status.loadBalancer.ingress[].hostname}'
}

get_version_images() {
	local cr_version="$1"
	local component="$2"
	local release_versions_file="${TEMP_DIR}/release_version_${cr_version}"

	if [[ ! -f $release_versions_file ]]; then
		curl -s "https://raw.githubusercontent.com/percona/percona-postgresql-operator/refs/tags/v${cr_version}/e2e-tests/release_versions" >$release_versions_file
	fi

	grep -i ${component} $release_versions_file | cut -d'=' -f2
}

get_release_image() {
	local key="$1"
	local release_versions_file="${ROOT_REPO}/e2e-tests/release_versions"

	grep -E "^${key}=" "$release_versions_file" | cut -d'=' -f2
}

wait_for_delete() {
	local res="$1"

	echo -n "$res - "
	retry=0
	until (kubectl get $res -n "${NAMESPACE}" || :) 2>&1 | grep NotFound; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 120 ]; then
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
}

deploy_pmm_server() {
	helm uninstall -n "${NAMESPACE}" pmm || :
	if [[ $OPENSHIFT ]]; then
		platform=openshift
		oc create sa pmm-server -n "$NAMESPACE"
		oc adm policy add-scc-to-user privileged -z pmm-server -n "$NAMESPACE"

		if [[ $OPERATOR_NS ]]; then
			timeout 30 oc delete clusterrolebinding $(kubectl get clusterrolebinding | grep 'pmm-pg-operator-' | awk '{print $1}') || :
			oc create clusterrolebinding pmm-pg-operator-cluster-wide --clusterrole=percona-postgresql-operator --serviceaccount=$NAMESPACE:pmm-server -n "$NAMESPACE"
			oc patch clusterrole/percona-postgresql-operator --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' ${OPERATOR_NS:+-n $OPERATOR_NS}
		else
			oc create rolebinding pmm-pg-operator-namespace-only --role percona-postgresql-operator --serviceaccount=$NAMESPACE:pmm-server -n "${NAMESPACE}"
			oc patch role/percona-postgresql-operator --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' -n "$NAMESPACE"
		fi
		helm install monitoring --set imageTag=${IMAGE_PMM_SERVER#*:} --set imageRepo=${IMAGE_PMM_SERVER%:*} --set platform=$platform --set sa=pmm-server --set supresshttp2=false https://percona-charts.storage.googleapis.com/pmm-server-${PMM_SERVER_VERSION}.tgz -n "$NAMESPACE"
	else
		platform=kubernetes
		helm install monitoring -n "$NAMESPACE" --set imageTag=${IMAGE_PMM_SERVER#*:} --set service.type="LoadBalancer" \
			--set imageRepo=${IMAGE_PMM_SERVER%:*} --set platform="$platform" "https://percona-charts.storage.googleapis.com/pmm-server-${PMM_SERVER_VERSION}.tgz"
	fi
}

deploy_pmm3_server() {
	helm uninstall -n "${NAMESPACE}" monitoring || :
	helm repo remove percona || :
	kubectl delete clusterrole monitoring --ignore-not-found
	kubectl delete clusterrolebinding monitoring --ignore-not-found
	helm repo add percona https://percona.github.io/percona-helm-charts/
	helm repo update

	if [[ $OPENSHIFT ]]; then
		oc create sa pmm-server -n "$NAMESPACE"
		oc adm policy add-scc-to-user privileged -z pmm-server -n "$NAMESPACE"

		if [[ $OPERATOR_NS ]]; then
			timeout 30 oc delete clusterrolebinding $(kubectl get clusterrolebinding | grep 'pmm-pg-operator-' | awk '{print $1}') || :
			oc create clusterrolebinding pmm-pg-operator-cluster-wide --clusterrole=percona-postgresql-operator --serviceaccount=$NAMESPACE:pmm-server -n "$NAMESPACE"
			oc patch clusterrole/percona-postgresql-operator --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' ${OPERATOR_NS:+-n $OPERATOR_NS}
		else
			oc create rolebinding pmm-pg-operator-namespace-only --role percona-postgresql-operator --serviceaccount=$NAMESPACE:pmm-server -n "${NAMESPACE}"
			oc patch role/percona-postgresql-operator --type json -p='[{"op":"add","path": "/rules/-","value":{"apiGroups":["security.openshift.io"],"resources":["securitycontextconstraints"],"verbs":["use"],"resourceNames":["privileged"]}}]' -n "$NAMESPACE"
		fi
		local additional_params="--set platform=openshift --set supresshttp2=false --set serviceAccount.create=false --set serviceAccount.name=pmm-server"
	fi

	retry 10 60 helm install monitoring percona/pmm -n "${NAMESPACE}" \
		--set fullnameOverride=monitoring \
		--set image.tag=${IMAGE_PMM3_SERVER#*:} \
		--set image.repository=${IMAGE_PMM3_SERVER%:*} \
		--set service.type=LoadBalancer \
		$additional_params \
		--force
}

retry() {
	local max=$1
	local delay=$2
	shift 2 # cut delay and max args
	local n=1

	until "$@"; do
		if [[ $n -ge $max ]]; then
			echo "The command '$@' has failed after $n attempts."
			exit 1
		fi
		((n++))
		sleep $delay
	done
}

generate_pmm_api_key() {
	local ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" exec monitoring-0 -- bash -c "printenv | grep ADMIN_PASSWORD | cut -d '=' -f2")
	local PMM_SERVICE_IP=$(get_service_ip monitoring-service)
	curl \
		--insecure \
		-X POST \
		-H "Content-Type: application/json" \
		-d '{"name":"'${RANDOM}'", "role": "Admin"}' \
		--user "admin:${ADMIN_PASSWORD}" \
		"https://${PMM_SERVICE_IP}/graph/api/auth/keys" \
		| jq -r .key
}

generate_pmm3_server_token() {
	local key_name=$RANDOM

	local ADMIN_PASSWORD
	ADMIN_PASSWORD=$(kubectl -n "${NAMESPACE}" get secret pmm-secret -o jsonpath="{.data.PMM_ADMIN_PASSWORD}" | base64 --decode)

	if [[ -z $ADMIN_PASSWORD ]]; then
		echo "Error: ADMIN_PASSWORD is empty or not found!" >&2
		return 1
	fi

	local create_response create_status_code create_json_response
	create_response=$(curl --insecure -s -X POST -H 'Content-Type: application/json' -H 'Accept: application/json' \
		-d "{\"name\":\"${key_name}\", \"role\":\"Admin\", \"isDisabled\":false}" \
		--user "admin:${ADMIN_PASSWORD}" \
		"https://$(get_service_ip monitoring-service)/graph/api/serviceaccounts" \
		-w "\n%{http_code}")

	create_status_code=$(echo "$create_response" | tail -n1)
	create_json_response=$(echo "$create_response" | $sed '$ d')

	if [[ $create_status_code -ne 201 ]]; then
		echo "Error: Failed to create PMM service account. HTTP Status: $create_status_code" >&2
		echo "Response: $create_json_response" >&2
		return 1
	fi

	local service_account_id
	service_account_id=$(echo "$create_json_response" | jq -r '.id')

	if [[ -z $service_account_id || $service_account_id == "null" ]]; then
		echo "Error: Failed to extract service account ID!" >&2
		return 1
	fi

	local token_response token_status_code token_json_response
	token_response=$(curl --insecure -s -X POST -H 'Content-Type: application/json' \
		-d "{\"name\":\"${key_name}\"}" \
		--user "admin:${ADMIN_PASSWORD}" \
		"https://$(get_service_ip monitoring-service)/graph/api/serviceaccounts/${service_account_id}/tokens" \
		-w "\n%{http_code}")

	token_status_code=$(echo "$token_response" | tail -n1)
	token_json_response=$(echo "$token_response" | $sed '$ d')

	if [[ $token_status_code -ne 200 ]]; then
		echo "Error: Failed to create token. HTTP Status: $token_status_code" >&2
		echo "Response: $token_json_response" >&2
		return 1
	fi

	echo "$token_json_response" | jq -r '.key'
}

get_metric_values() {
	local metric=$1
	local instance=$2
	local token=$3
	local start=$($date -u "+%s" -d "-5 minute")
	local end=$($date -u "+%s")
	local endpoint=$(get_service_ip monitoring-service)

	local wait_count=20
	local retry=0
	until [[ $(curl -s -k -H "Authorization: Bearer ${token}" "https://$endpoint/graph/api/datasources/proxy/1/api/v1/query_range?query=min%28$metric%7Bnode_name%3D%7E%22$instance%22%7d%20or%20$metric%7Bnode_name%3D%7E%22$instance%22%7D%29&start=$start&end=$end&step=60" \
		| jq '.data.result[0].values[][1]' \
		| grep '^"[0-9]') ]]; do
		sleep 2
		local start=$($date -u "+%s" -d "-5 minute")
		local end=$($date -u "+%s")
		let retry+=1
		if [[ $retry -ge $wait_count ]]; then
			exit 1
		fi
	done
}

get_qan20_values() {
	local instance=$1
	local api_key=$2
	local start=$($date -u "+%Y-%m-%dT%H:%M:%S" -d "-30 minute")
	local end=$($date -u "+%Y-%m-%dT%H:%M:%S")
	local endpoint=$(get_service_ip monitoring-service)

	cat >payload.json <<EOF
{
   "columns":[
	  "load",
	  "num_queries",
	  "query_time"
   ],
   "first_seen": false,
   "group_by": "queryid",
   "include_only_fields": [],
   "keyword": "",
   "labels": [
	   {
		   "key": "cluster",
		   "value": ["postgresql"]
   }],
   "limit": 10,
   "offset": 0,
   "order_by": "-load",
   "main_metric": "load",
   "period_start_from": "$($date -u -d '-12 hour' '+%Y-%m-%dT%H:%M:%S%:z')",
   "period_start_to": "$($date -u '+%Y-%m-%dT%H:%M:%S%:z')"
}
EOF

	curl -s -k -H "Authorization: Bearer ${api_key}" -XPOST -d @payload.json "https://$endpoint/v0/qan/GetReport" \
		| jq '.rows[].sparkline'
	rm -f payload.json
}

get_qan20_values_pmm3() {
	local instance=$1
	local token=$2
	local start=$($date -u "+%Y-%m-%dT%H:%M:%S" -d "-30 minute")
	local end=$($date -u "+%Y-%m-%dT%H:%M:%S")
	local endpoint=$(get_service_ip monitoring-service)

	cat >payload.json <<EOF
{
   "columns":[
	  "load",
	  "num_queries",
	  "query_time"
   ],
   "first_seen": false,
   "group_by": "queryid",
   "include_only_fields": [],
   "keyword": "",
   "labels": [
	   {
		   "key": "cluster",
		   "value": ["postgresql"]
   }],
   "limit": 10,
   "offset": 0,
   "order_by": "-load",
   "main_metric": "load",
   "period_start_from": "$($date -u -d '-12 hour' '+%Y-%m-%dT%H:%M:%S%:z')",
   "period_start_to": "$($date -u '+%Y-%m-%dT%H:%M:%S%:z')"
}
EOF

	curl -s -k -H "Authorization: Bearer ${token}" -XPOST -d @payload.json "https://$endpoint/v1/qan/metrics:getReport" \
		| jq '.rows[].sparkline'
	rm -f payload.json
}

deploy_chaos_mesh() {
	destroy_chaos_mesh

	helm repo add chaos-mesh https://charts.chaos-mesh.org
	helm install chaos-mesh chaos-mesh/chaos-mesh --namespace=${NAMESPACE} --set chaosDaemon.runtime=containerd --set chaosDaemon.socketPath=/run/containerd/containerd.sock --set dashboard.create=false --version ${CHAOS_MESH_VER}
	if [[ $OPENSHIFT ]]; then
		oc adm policy add-scc-to-user privileged -z chaos-daemon --namespace=${NAMESPACE}
	fi
	sleep 10
}

destroy_chaos_mesh() {
	local chaos_mesh_ns=$(helm list --all-namespaces --filter chaos-mesh | tail -n1 | awk -F' ' '{print $2}' | $sed 's/NAMESPACE//')

	if [ -n "${chaos_mesh_ns}" ]; then
		helm uninstall --wait --timeout 60s chaos-mesh --namespace ${chaos_mesh_ns} || :
	fi
	timeout 30 kubectl delete MutatingWebhookConfiguration $(kubectl get MutatingWebhookConfiguration | grep 'chaos-mesh' | awk '{print $1}') || :
	timeout 30 kubectl delete ValidatingWebhookConfiguration $(kubectl get ValidatingWebhookConfiguration | grep 'chaos-mesh' | awk '{print $1}') || :
	timeout 30 kubectl delete ValidatingWebhookConfiguration $(kubectl get ValidatingWebhookConfiguration | grep 'validate-auth' | awk '{print $1}') || :
	for i in $(kubectl api-resources | grep chaos-mesh | awk '{print $1}'); do
		kubectl get ${i} --all-namespaces --no-headers -o custom-columns=Kind:.kind,Name:.metadata.name,NAMESPACE:.metadata.namespace \
			| while read -r line; do
				local kind=$(echo "$line" | awk '{print $1}')
				local name=$(echo "$line" | awk '{print $2}')
				local namespace=$(echo "$line" | awk '{print $3}')
				kubectl patch $kind $name -n $namespace --type=merge -p '{"metadata":{"finalizers":[]}}' || :
			done
		timeout 30 kubectl delete ${i} --all --all-namespaces || :
	done
	timeout 30 kubectl delete crd $(kubectl get crd | grep 'chaos-mesh.org' | awk '{print $1}') || :
	timeout 30 kubectl delete clusterrolebinding $(kubectl get clusterrolebinding | grep 'chaos-mesh' | awk '{print $1}') || :
	timeout 30 kubectl delete clusterrole $(kubectl get clusterrole | grep 'chaos-mesh' | awk '{print $1}') || :
}

kill_pods() {
	local ns=$1
	local selector=$2
	local pod_label=$3
	local label_value=$4

	if [ "${selector}" == "pod" ]; then
		yq eval '
			.metadata.name = "chaos-pod-kill-'${RANDOM}'" |
			del(.spec.selector.pods.test-namespace) |
			.spec.selector.pods.'${ns}'[0] = "'${pod_label}'"' ${TESTS_CONFIG_DIR}/chaos-pod-kill.yml \
			| kubectl apply --namespace ${ns} -f -
	elif [ "${selector}" == "label" ]; then
		yq eval '
			.metadata.name = "chaos-kill-label-'${RANDOM}'" |
			.spec.mode = "all" |
			del(.spec.selector.pods) |
			.spec.selector.labelSelectors."'${pod_label}'" = "'${label_value}'"' ${TESTS_CONFIG_DIR}/chaos-pod-kill.yml \
			| kubectl apply --namespace ${ns} -f -
	fi
	sleep 5
}

failure_pod() {
	local ns=$1
	local pod=$2

	yq eval '
        .metadata.name = "chaos-pod-failure-'${RANDOM}'" |
        del(.spec.selector.pods.test-namespace) |
        .spec.selector.pods.'${ns}'[0] = "'${pod}'"' ${TESTS_CONFIG_DIR}/chaos-pod-failure.yml \
		| kubectl apply --namespace ${ns} -f -
	sleep 5
}

network_loss() {
	local ns=$1
	local pod=$2

	yq eval '
        .metadata.name = "chaos-pod-network-loss-'${RANDOM}'" |
        del(.spec.selector.pods.test-namespace) |
        .spec.selector.pods.'${ns}'[0] = "'${pod}'"' ${TESTS_CONFIG_DIR}/chaos-network-loss.yml \
		| kubectl apply --namespace ${ns} -f -
	sleep 5
}

wait_deployment() {
	local name=$1
	local target_namespace=${2:-"$namespace"}

	sleep 10
	set +o xtrace
	retry=0
	echo -n $name
	until [ -n "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.replicas}')" \
		-a "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.replicas}')" \
		== "$(kubectl -n ${target_namespace} get deployment $name -o jsonpath='{.status.readyReplicas}')" ]; do
		sleep 1
		echo -n .
		let retry+=1
		if [ $retry -ge 360 ]; then
			kubectl logs $(get_operator_pod) -c operator \
				| grep -v 'level=info' \
				| grep -v 'level=debug' \
				| tail -100
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
	done
	echo
	set -o xtrace
}

get_pod_by_role() {
	local cluster=${1}
	local role=${2}
	local parameter=${3}

	case ${parameter} in
		'name')
			local jsonpath="{.items[].metadata.name}"
			;;
		'IP')
			local jsonpath="{.items[].status.podIP}"
			;;
	esac

	echo "$(kubectl get pods --namespace ${NAMESPACE} --selector=postgres-operator.crunchydata.com/role=${role},postgres-operator.crunchydata.com/cluster=${cluster} -o 'jsonpath='${jsonpath}'')"
}

check_passwords_leak() {
	local secrets
	local passwords
	local pods

	secrets=$(
		kubectl -n "${NAMESPACE}" get secrets -o json | jq -r '.items[] | select(.data."password"? != null) | .data."password"'
		kubectl -n "${NAMESPACE}" get secrets -o json | jq -r '.items[] | select(.data."pgbouncer-password"? != null) | .data."pgbouncer-password"'
	)

	passwords="$(for i in $secrets; do
		base64 -d <<<$i
		echo
	done) $secrets"
	pods=$(kubectl -n "${NAMESPACE}" get pods -o name | awk -F "/" '{print $2}')

	collect_logs() {
		local containers
		local count

		NS=$1
		for p in $pods; do
			containers=$(kubectl -n "$NS" get pod $p -o jsonpath='{.spec.containers[*].name}')
			for c in $containers; do
				# temporary, because of: https://jira.percona.com/browse/PMM-8357
				if [[ $c =~ "pmm" ]]; then
					continue
				fi
				kubectl -n "$NS" logs $p -c $c >${TEMP_DIR}/logs_output-$p-$c.txt
				echo logs saved in: ${TEMP_DIR}/logs_output-$p-$c.txt
				for pass in $passwords; do
					count=$(grep -c --fixed-strings -- "$pass" ${TEMP_DIR}/logs_output-$p-$c.txt || :)
					if [[ $count != 0 ]]; then
						echo leaked passwords are found in log ${TEMP_DIR}/logs_output-$p-$c.txt
						false
					fi
				done
			done
			echo
		done
	}

	collect_logs $NAMESPACE
	if [ -n "$OPERATOR_NS" ]; then
		pods=$(kubectl -n "${OPERATOR_NS}" get pods -o name | awk -F "/" '{print $2}')
		collect_logs $OPERATOR_NS
	fi
}

get_backup_destination() {
	local cluster=$1
	local backup_name=$2

	local repo
	local storage_type

	repo=$(kubectl get pg-backup -n "$NAMESPACE" "$backup_name" -o yaml \
		| yq ".spec.repoName")

	if [[ $(kubectl get pg -n "$NAMESPACE" "$cluster" -o yaml \
		| yq ".spec.backups.pgbackrest.repos[] | select(.name==\"$repo\") | has(\"s3\")") == "true" ]]; then
		storage_type="s3"
	elif [[ $(kubectl get pg -n "$NAMESPACE" "$cluster" -o yaml \
		| yq ".spec.backups.pgbackrest.repos[] | select(.name==\"$repo\") | has(\"gcs\")") == "true" ]]; then
		storage_type="gcs"
	elif [[ $(kubectl get pg -n "$NAMESPACE" "$cluster" -o yaml \
		| yq ".spec.backups.pgbackrest.repos[] | select(.name==\"$repo\") | has(\"azure\")") == "true" ]]; then
		storage_type="azure"
	else
		echo "ERROR: unknown storage type"
		exit 1
	fi

	local repo_path
	local bucket

	repo_path=$(kubectl get pg -n "$NAMESPACE" "$cluster" -o yaml \
		| yq ".spec.backups.pgbackrest.global.$repo-path")
	bucket=$(kubectl get pg -n "$NAMESPACE" "$cluster" -o yaml \
		| yq ".spec.backups.pgbackrest.repos[] | select(.name==\"$repo\").$storage_type.bucket")

	if [[ $storage_type == "gcs" ]]; then
		storage_type="gs"
	fi

	echo -n "$storage_type://$bucket$repo_path"
}

get_backup_name_by_job() {
	local job_name=$1

	pgbackup_per_job=$(kubectl get pg-backup -n "$NAMESPACE" -o yaml \
		| yq "[.items[] | select(.status.jobName==\"$job_name\")]" \
		| yq '. | length')
	if [[ $pgbackup_per_job != 1 ]]; then
		echo "ERROR: pgbackup_per_job != 1"
		exit 1
	fi

	kubectl get pg-backup -n "$NAMESPACE" -o yaml \
		| yq ".items[] | select(.status.jobName==\"$job_name\").metadata.name"
}

check_jobs_and_pgbackups() {
	local cluster=$1

	job_names=()
	kubectl get job -n "$NAMESPACE" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | while IFS= read -r line; do
		job_names+=("$line")
	done

	for ((i = 0; i < ${#job_names[@]}; i++)); do
		job_name=${job_names[$i]}

		backup_name=$(get_backup_name_by_job "$job_name")

		dest=$(kubectl get pg-backup -n "$NAMESPACE" "$backup_name" -o yaml | yq ".status.destination")
		if [[ $dest != "$(get_backup_destination "$cluster" "$backup_name")" ]]; then
			echo "ERROR: $dest != $(get_backup_destination "$cluster" "$backup_name")"
			exit 1
		fi

		job_backup_type=$(kubectl get job "$job_name" -n "$NAMESPACE" -o yaml \
			| yq '.spec.template.metadata.labels."postgres-operator.crunchydata.com/pgbackrest-backup"')
		if [[ $job_backup_type == "null" ]]; then
			job_backup_type=$(kubectl get job "$job_name" -n "$NAMESPACE" -o yaml \
				| yq '.spec.template.metadata.labels."postgres-operator.crunchydata.com/pgbackrest-cronjob"')
		fi
		if [[ $job_backup_type == "replica-create" ]]; then
			job_backup_type="full"
		fi
		if [[ $job_backup_type == "manual" ]]; then
			job_backup_type="full"
		fi
		pg_backup_type=$(kubectl get pg-backup -n "$NAMESPACE" "$backup_name" -o yaml \
			| yq '.status.backupType')

		if [[ $job_backup_type != "$pg_backup_type" ]]; then
			echo "ERROR: backup type $job_backup_type != $pg_backup_type"
			exit 1
		fi
	done
}

restart_pg_pods() {
	local cluster=$1
	local instance=$2

	pods=$(kubectl get pods -n "${NAMESPACE}" --selector=postgres-operator.crunchydata.com/cluster="${cluster}",postgres-operator.crunchydata.com/instance-set="${instance}" -o jsonpath='{.items[*].metadata.name}')
	for pod in $pods; do
		kubectl delete pod -n "${NAMESPACE}" "${pod}"
		wait_pod "${pod}"
	done
}

get_updated_replicas() {
	kubectl get statefulset "$1" -n "$NAMESPACE" -o jsonpath='{.status.updatedReplicas}'
}

# Function to get the desired number of replicas
get_desired_replicas() {
	kubectl get statefulset "$1" -n "$NAMESPACE" -o jsonpath='{.spec.replicas}'
}

get_ready_replicas() {
	kubectl get sts "$1" -n "$NAMESPACE" -o jsonpath='{.status.readyReplicas}'
}

wait_sts_rollout() {
	local sts=$1
	local ready_replicas=$(get_ready_replicas $sts)
	local updated_replicas=$(get_updated_replicas $sts)
	local desired_replicas=$(get_desired_replicas $sts)
	echo "ready_replicas $ready_replicas desired_replicas $desired_replicas"

	until [[ $ready_replicas -eq $desired_replicas && $updated_replicas -eq $desired_replicas ]]; do
		ready_replicas=$(get_ready_replicas $sts)
		desired_replicas=$(get_desired_replicas $sts)
		updated_replicas=$(get_updated_replicas $sts)
		echo "Waiting for sts/$sts to update... $ready_replicas/$updated_replicas/$desired_replicas pods updated."
		sleep 10
	done
}

wait_cluster_consistency() {
	local cluster_name=$1
	local wait_time=${2:-32}

	retry=0
	sleep 7 # wait for two reconcile loops ;)  3 sec x 2 times + 1 sec = 7 seconds
	echo -n 'waiting for cluster readyness'
	until [[ "$(kubectl -n "${NAMESPACE}" get pg "${cluster_name}" -o jsonpath='{.status.state}')" == "ready" ]]; do
		let retry+=1
		if [ $retry -ge $wait_time ]; then
			echo max retry count $retry reached. something went wrong with operator or kubernetes cluster
			exit 1
		fi
		echo -n .
		sleep 10
	done

	sts=$(kubectl -n "${NAMESPACE}" get sts --selector=postgres-operator.crunchydata.com/cluster="${cluster_name}" -o jsonpath='{.items[*].metadata.name}' | grep "${cluster_name}")
	for st in $sts; do
		wait_sts_rollout "$st"
	done
}

deploy_cert_manager() {
	destroy_cert_manager

	echo 'deploy cert manager'
	kubectl create namespace cert-manager || :
	kubectl label namespace cert-manager certmanager.k8s.io/disable-validation=true || :
	kubectl apply -f "https://github.com/jetstack/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml" --validate=false >/dev/null 2>&1 || :

	kubectl wait deployment cert-manager cert-manager-webhook cert-manager-cainjector \
		--for=condition=available \
		--timeout=100s \
		--namespace cert-manager

	echo "Waiting for cert-manager webhook to be ready..."
	until kubectl get validatingwebhookconfiguration cert-manager-webhook -o jsonpath='{.webhooks[0].clientConfig.caBundle}' | grep -q '[A-Za-z0-9+/=]'; do
		sleep 5
	done
}

destroy_cert_manager() {
	kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml >/dev/null 2>&1 || :
	kubectl delete --grace-period=0 --force=true namespace cert-manager >/dev/null 2>&1 || :
}

get_container_image() {
	local component=$1
	local pgVersion=$2

	if [[ ${IMAGE} == percona/* || ${IMAGE} == docker.io/percona/* ]] || [[ -n ${FORCE_RELEASE_RUN} ]]; then
		local key
		case $component in
			pgbouncer)  key="IMAGE_PGBOUNCER${pgVersion}" ;;
			pgbackrest) key="IMAGE_BACKREST${pgVersion}" ;;
			postgis)    key="IMAGE_POSTGIS${pgVersion}" ;;
			*)          key="IMAGE_POSTGRESQL${pgVersion}" ;;
		esac
		get_release_image "$key"
		return
	fi

	local operatorVersion=${VERSION}

	# if operator version is not a proper version (i.e. 2.5.0) use main
	# to not force people to create bazillion number of images
	if [[ ! ${operatorVersion} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
		operatorVersion=main
	fi

	if [[ $component == 'pgbouncer' || $component == 'pgbackrest' ]]; then
		echo "${IMAGE_BASE}:${operatorVersion}-${component}${pgVersion}"
	else
		echo "${IMAGE_BASE}:${operatorVersion}-ppg${pgVersion}-${component}"
	fi
}

get_postgresql_logs() {
	local pgVersion=$1

	for pod in $(kubectl get pods -l postgres-operator.crunchydata.com/data=postgres --no-headers | awk '{print $1}'); do
		local phase=$(kubectl -n ${NAMESPACE} get ${pod} -o jsonpath={".status.phase"})
		if [[ ${phase} != "Running" ]]; then
			echo "Waiting for ${pod} to start running"
			continue
		fi
		echo "find /pgdata/pg${pgVersion}/log -type f -iname 'postgresql*.log' -exec tail -n 30 {} \;" \
			| kubectl -n ${NAMESPACE} exec -it ${pod} -- bash 2>/dev/null
	done
}

wait_for_generation() {
	local resource="$1"
	local target_generation="$2"

	echo "Waiting for $resource to reach generation $target_generation..."

	while true; do
		current_generation=$(kubectl -n ${NAMESPACE} get "$resource" -o jsonpath='{.metadata.generation}')

		if [ "$current_generation" -eq "$target_generation" ]; then
			echo "Resource $resource has reached generation $target_generation."
			break
		else
			echo "Resource $resource is at generation $current_generation. Waiting..."
			sleep 5
		fi
	done
}

# Reads all environment variables from a pod (single kubectl exec call)
get_envs_from_pod() {
  local namespace=$1
  local pod=$2
  local container=$3

  if [ -n "$container" ]; then
    kubectl exec -n "$namespace" "$pod" -c "$container" -- printenv 2>/dev/null || true
  else
    kubectl exec -n "$namespace" "$pod" -- printenv 2>/dev/null || true
  fi
}

# Verifies a single variable in given env content
check_env_in_pod() {
  local check_type=$1
  local pod=$2
  local var_name=$3
  local expected_value=$4
  local env_content=$5

  local actual_value
  actual_value=$(echo "$env_content" | grep -E "^${var_name}=" | cut -d'=' -f2- || true)

  if [[ "$check_type" == "add" ]]; then
    if [ "$actual_value" != "$expected_value" ]; then
      echo "ERROR: $var_name in $pod — expected '$expected_value', got '${actual_value:-<missing>}'"
      return 1
    else
      echo "OK: $var_name=$actual_value in $pod"
    fi
  elif [[ "$check_type" == "delete" ]]; then
    if [ -n "$actual_value" ]; then
      echo "ERROR: $var_name exists in $pod (should not exist)"
      return 1
    else
      echo "OK: $var_name deleted in $pod"
    fi
  else
    echo "ERROR: unknown check type '$check_type'"
    return 1
  fi
}

# Checks multiple env vars in one or more components
check_envs_for_component() {
  local check_type=$1      # add | delete
  local component=$2       # instance | pgbouncer | repohost
  local vars=("${@:3}")    # everything after the 2nd argument

  case "$component" in
    instance)
      POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/instance-set=instance1 -o 'jsonpath={.items[0].metadata.name}')
      CONTAINER=""
      ;;
    pgbouncer)
      POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/role=pgbouncer -o 'jsonpath={.items[0].metadata.name}')
      CONTAINER="pgbouncer"
      ;;
    repohost)
      POD=$(kubectl get -n "${NAMESPACE}" pod -l postgres-operator.crunchydata.com/data=pgbackrest -o 'jsonpath={.items[0].metadata.name}')
      CONTAINER="pgbackrest"
      ;;
    *)
      echo "ERROR: unknown component '$component'"
      return 1
      ;;
  esac

  echo "Fetching environment variables for $component pod $POD..."
  local env_content
  env_content=$(get_envs_from_pod "${NAMESPACE}" "$POD" "$CONTAINER")

  local errors=0
  for var_entry in "${vars[@]}"; do
    if [[ "$check_type" == "add" ]]; then
      local var_name="${var_entry%%=*}"
      local var_expected="${var_entry#*=}"
      check_env_in_pod add "$POD" "$var_name" "$var_expected" "$env_content" || errors=$((errors+1))
    else
      check_env_in_pod delete "$POD" "$var_entry" "" "$env_content" || errors=$((errors+1))
    fi
  done

  if (( errors > 0 )); then
    echo "$errors environment check(s) failed for component '$component'"
    return 1
  else
    echo "All environment checks passed for component '$component'"
  fi
}

detect_k8s_platform() {
	local platform=""

	if kubectl get nodes -o json | jq -r '.items[0].spec.providerID' | grep -q "gce://"; then
		platform="gke"

	elif oc get projects >/dev/null 2>&1; then
		platform="openshift"

	elif kubectl get nodes -o json | jq -r '.items[0].spec.providerID' | grep -q "aws://"; then
		platform="eks"

	elif kubectl get nodes -o json | jq -r '.items[0].spec.providerID' | grep -q "digitalocean://"; then
		platform="digitalocean"

	elif kubectl get nodes -o json | jq -r '.items[0].spec.providerID' | grep -q "azure://"; then
		platform="aks"

	else
		platform="unknown"
	fi

	echo "${platform}"
}

ensure_ebs_gp3_storage_class() {
	local platform=${1:-$(detect_k8s_platform)}

	if [[ "$platform" != "eks" && "$platform" != "openshift" ]]; then
		return 0
	fi

	cat <<EOF | kubectl apply -f -
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: ebs-csi-gp3
provisioner: ebs.csi.aws.com
volumeBindingMode: WaitForFirstConsumer
allowVolumeExpansion: true
parameters:
  type: gp3
EOF
}

enable_hugepages() {
	local platform=$1
	local hugepage_count=${2:-1024}

	echo "Detected platform: ${platform}"

	case ${platform} in
		gke)
			enable_hugepages_gke "${hugepage_count}"
			;;
		openshift)
			enable_hugepages_openshift "${hugepage_count}"
			;;
		eks)
			enable_hugepages_eks "${hugepage_count}"
			;;
		aks)
			enable_hugepages_aks "${hugepage_count}"
			;;
		digitalocean)
			enable_hugepages_doks "${hugepage_count}"
			;;
		*)
			echo "Unknown platform, cannot configure hugepages"
			return 1
			;;
	esac
}

enable_hugepages_gke() {
  local hugepage_count=$1

  echo "Hugepages configuration is now handled during cluster creation"
  echo "Skipping runtime configuration for GKE"

  echo "Verifying hugepages configuration..."
  local node_name=$(kubectl get nodes -o jsonpath='{.items[0].metadata.name}')

  if kubectl get node ${node_name} -o jsonpath='{.status.allocatable.hugepages-2Mi}' | grep -q '[0-9]'; then
    echo "✓ Hugepages are already configured on nodes"
    return 0
  else
    echo "✗ Warning: Hugepages not found in node allocatable resources"
    return 1
  fi
}

enable_hugepages_eks() {

	echo "Configuring hugepages on EKS"

	if verify_hugepages_on_nodes; then
		echo "Hugepages already configured"
		return 0
	fi
}

enable_hugepages_doks() {
	local hugepage_count=$1
  
  if verify_hugepages_on_nodes; then
		echo "Hugepages already configured"
		return 0
	fi

	cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: hugepages
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: hugepages
  template:
    metadata:
      labels:
        app: hugepages
    spec:
      hostPID: true
      containers:
      - name: hugepages
        image: busybox
        securityContext:
          privileged: true
        command:
        - sh
        - -c
        - |
          echo "Setting hugepages..."
          echo 1024 > /proc/sys/vm/nr_hugepages
          cat /proc/sys/vm/nr_hugepages
          echo "Restarting Kubelet..."
          nsenter -t 1 -m -u -i -n -p systemctl restart kubelet
          echo "Done, sleeping..."
          sleep infinity
EOF

	kubectl -n kube-system rollout status daemonset/hugepages --timeout=15m || {
		echo "DOKS: DaemonSet rollout did not complete. Skipping test."
		return 1
	}

	if verify_hugepages_on_nodes; then
		echo "Hugepages successfully configured"
	else
		echo "Failed to configure hugepages"
		return 1
	fi
}

enable_hugepages_aks() {
	local hugepage_count="${1:-2048}"
	local nodepool_name="${AKS_HUGEPAGES_NODEPOOL:-hp2m}"
	local vm_size="${AKS_HUGEPAGES_VM_SIZE:-Standard_D4_v2}"
	local node_count="${AKS_HUGEPAGES_NODE_COUNT:-1}"

	echo "Checking hugepages on AKS..."

	if verify_hugepages_on_nodes 2>/dev/null; then
		echo "Hugepages already configured"
		return 0
	fi

	local cluster_name
	node_rg="$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.kubernetes\.azure\.com/cluster}' 2>/dev/null || true)"
	[[ -n $node_rg ]] || return 1

	aks_id="$(az aks list --subscription "${AZURE_SUBSCRIPTION:-eng-cloud-dev}" \
		--query "[?nodeResourceGroup=='$node_rg'] | [0].id" \
		-o tsv 2>/dev/null || true)"
	[[ -n $aks_id ]] || return 1

	cluster_name="$(echo "$aks_id" | $sed -E 's|.*/managedClusters/([^/]+)$|\1|I')"
	[[ -n $cluster_name ]] || return 1

	if [[ -z ${cluster_name} ]]; then
		echo "AKS: can't detect cluster name from node labels."
		echo "Skipping hugepages test."
		return 1
	fi

	echo "AKS cluster detected: ${cluster_name}"

	if ! az aks nodepool show -g percona-operators --cluster-name "${cluster_name}" -n "${nodepool_name}" >/dev/null 2>&1; then
		echo "Creating nodepool '${nodepool_name}' (${vm_size}, count=${node_count}) with label hugepages=enabled..."
		az aks nodepool add \
			--subscription "${AZURE_SUBSCRIPTION:-eng-cloud-dev}" \
			-g "${AKS_RG:-percona-operators}" \
			--cluster-name "${cluster_name}" \
			-n "${nodepool_name}" \
			--node-count "${node_count}" \
			--node-vm-size "${vm_size}" \
			--labels hugepages=enabled \
			--mode User
	else
		echo "Nodepool '${nodepool_name}' already exists."
	fi

	echo "Waiting for hugepages node(s) to be Ready..."
	kubectl wait node -l hugepages=enabled --for=condition=Ready --timeout=15m || {
		echo "AKS: hugepages node(s) not ready in time. Skipping test."
		return 1
	}

	echo "Applying DaemonSet to allocate hugepages on hugepages=enabled nodes (HP_2M=${hugepage_count})..."
	cat <<EOF | kubectl apply -f -
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: hugepages-2mi-enable
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: hugepages-2mi-enable
  template:
    metadata:
      labels:
        app: hugepages-2mi-enable
    spec:
      nodeSelector:
        hugepages: "enabled"
      hostPID: true
      tolerations:
        - operator: Exists
      containers:
        - name: enable
          image: busybox:${BUSYBOX_VER}
          securityContext:
            privileged: true
          env:
            - name: HP_2M
              value: "${hugepage_count}"
          command: ["/bin/sh","-c"]
          args:
            - |
              set -eux
              echo "Allocating \${HP_2M} hugepages (2Mi) on host..."
              echo "\${HP_2M}" > /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages
              mkdir -p /host/etc/sysctl.d
              echo "vm.nr_hugepages=\${HP_2M}" > /host/etc/sysctl.d/60-hugepages.conf
              chroot /host /bin/sh -c "sysctl -w vm.nr_hugepages=\${HP_2M} || true; systemctl restart kubelet || true"
              echo "Done. Sleeping..."
              sleep infinity
          volumeMounts:
            - name: host-root
              mountPath: /host
            - name: sys
              mountPath: /sys
      volumes:
        - name: host-root
          hostPath:
            path: /
        - name: sys
          hostPath:
            path: /sys
EOF

	kubectl -n kube-system rollout status ds/hugepages-2mi-enable --timeout=10m || {
		echo "AKS: DaemonSet rollout did not complete. Skipping test."
		return 1
	}

	echo "Re-checking hugepages on nodes..."
	if verify_hugepages_on_nodes 2>/dev/null; then
		echo "Hugepages configured successfully"
		return 0
	fi

	echo "AKS: Hugepages still not visible after configuration."
	echo "Skipping hugepages test."
	return 1
}

enable_hugepages_openshift() {
	local hugepage_count=$1

	echo "Configuring hugepages on OpenShift"

	if verify_hugepages_on_nodes; then
		echo "Hugepages already configured"
		return 0
	fi

	echo "Creating MachineConfig..."

	cat <<EOF | kubectl apply -f -
apiVersion: machineconfiguration.openshift.io/v1
kind: MachineConfig
metadata:
  labels:
    machineconfiguration.openshift.io/role: worker
  name: 99-hugepages-worker
spec:
  config:
    ignition:
      version: 3.2.0
    storage:
      files:
      - contents:
          source: data:text/plain;charset=utf-8;base64,$(echo "vm.nr_hugepages = ${hugepage_count}" | base64 -w0)
        mode: 0644
        path: /etc/sysctl.d/99-hugepages.conf
EOF

    [ $? -eq 0 ] || return 1

    echo "MachineConfig created"
    echo "Waiting for worker pool to update (~10 minutes)..."

    kubectl wait --for=condition=Updated mcp/worker --timeout=900s 2>/dev/null || {
        echo "Update taking longer than expected"
        return 1
    }

    echo "Worker pool updated"

    sleep 10
    verify_hugepages_on_nodes
}

verify_hugepages_on_nodes() {
    echo "Verifying hugepages on nodes"

    # Get first worker node, fallback to first non-master, fallback to any node
    local node_name=$(
        kubectl get nodes -l node-role.kubernetes.io/worker -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || \
        kubectl get nodes -l '!node-role.kubernetes.io/master,!node-role.kubernetes.io/control-plane' -o jsonpath='{.items[0].metadata.name}' 2>/dev/null || \
        kubectl get nodes -o jsonpath='{.items[0].metadata.name}'
    )

    if [ -z "${node_name}" ]; then
        echo "No nodes found"
        return 1
    fi

    echo "Checking node: ${node_name}"

    local hugepages_capacity=$(kubectl get node ${node_name} \
        -o jsonpath='{.status.capacity.hugepages-2Mi}')

    if [ -n "${hugepages_capacity}" ] && [ "${hugepages_capacity}" != "0" ]; then
        echo "Node has hugepages capacity: ${hugepages_capacity}"
        return 0
    else
        echo "No hugepages capacity found on node ${node_name}"
        return 1
    fi
}

verify_hugepages_in_pod() {
    local pod_name=$1
    local namespace=$2
    local container=${3:-postgres}

    echo "Verifying hugepages in pod ${pod_name}"

    # Check /proc/meminfo
    local hugepages_total=$(kubectl exec ${pod_name} -n ${namespace} -c ${container} -- \
        grep HugePages_Total /proc/meminfo | awk '{print $2}')

    local hugepages_free=$(kubectl exec ${pod_name} -n ${namespace} -c ${container} -- \
        grep HugePages_Free /proc/meminfo | awk '{print $2}')

    echo "HugePages_Total: ${hugepages_total}"
    echo "HugePages_Free: ${hugepages_free}"

    if [ "${hugepages_total}" -gt 0 ]; then
        echo "Hugepages are available in pod"
        return 0
    else
        echo "No hugepages in pod"
        return 1
    fi
}

verify_postgresql_hugepages_setting() {
    local cluster_name=$1
    local expected_value=${2:-try}

    echo "Verifying PostgreSQL huge_pages setting..."

    local huge_pages=$(run_psql_local \
        "SHOW huge_pages;" \
        "postgres:$(get_psql_user_pass ${cluster_name}-pguser-postgres)@$(get_psql_user_host ${cluster_name}-pguser-postgres)")

    echo "huge_pages: ${huge_pages}"

    if [[ "${huge_pages}" == *"${expected_value}"* ]]; then
        echo "PostgreSQL huge_pages is set to '${expected_value}'"
        return 0
    else
        echo "PostgreSQL huge_pages not set to '${expected_value}' (value: ${huge_pages})"
        return 1
    fi
}

verify_hugepages_usage() {
    local pod_name=$1
    local namespace=$2
    local container=${3:-database}

    echo "Checking hugepages usage..."

    kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \
        grep HugePages /proc/meminfo

    local hugepages_total=$(kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \
        grep HugePages_Total /proc/meminfo | awk '{print $2}')

    local hugepages_free=$(kubectl -n ${namespace} exec ${pod_name} -c ${container} -- \
        grep HugePages_Free /proc/meminfo | awk '{print $2}')

    local hugepages_used=$((hugepages_total - hugepages_free))

    echo ""
    echo "HugePages usage:"
    echo "  Total: ${hugepages_total}"
    echo "  Used:  ${hugepages_used}"

    if [ "${hugepages_used}" -gt 0 ]; then
        echo "PostgreSQL is using hugepages"
        return 0
    else
        echo "Hugepages available but NOT being used by PostgreSQL"
        return 1
    fi
}