Merge pull request #1576 from percona/K8SPG-984-k8s-upgrade

K8SPG-984 add k8s-upgrade test for gke and doks

Author: valmiranogueira

e2e-tests/functions

@@ -1744,4 +1744,237 @@ verify_hugepages_usage() {
         echo "Hugepages available but NOT being used by PostgreSQL"
         return 1
     fi
-}
+}
+
+get_available_k8s_versions() {
+	local offset="${1:-1}"
+	local platform="$2"
+	local region="$3"
+	local versions=""
+
+	case "$platform" in
+		digitalocean)
+			versions=$(doctl kubernetes options versions -o json \
+				| jq -r '.[].slug')
+			;;
+
+		gke)
+			versions=$(
+				gcloud container get-server-config \
+					--region "${region}" \
+					--format="value(validMasterVersions)" 2>/dev/null \
+					| tr ';' '\n' \
+					| awk -F'.' '!seen[$1"."$2]++'
+			)
+			;;
+
+		*)
+			echo "ERROR: unsupported platform '$platform'"
+			return 1
+			;;
+	esac
+
+	versions_sorted=$(echo "$versions" | sort -u -V)
+	echo "$versions_sorted" | tail -n "$offset" | head -n 1
+}
+
+create_k8s_cluster() {
+	local name="$1"
+	local version="$2"
+	local platform="$3"
+	local region="$4"
+	local ttl="${5:-"1"}"
+
+	case "$platform" in
+
+		digitalocean)
+			echo "Creating DOKS cluster: $name ($version)"
+			creation_time=$(date +%s)
+			doctl kubernetes cluster create "$name" \
+				--region "$region" \
+				--version "$version" \
+				--size "s-4vcpu-16gb-amd" \
+				--count "3" \
+				--tag "delete-cluster-after-hours:$ttl" \
+				--tag "creation-time:$creation_time" \
+				--tag "team:cloud" \
+				--tag "product:pg-operator" \
+				--wait
+
+			if [[ -n $DOKS_PROJECT ]]; then
+				project_id=$(doctl projects list --output json | jq -r --arg name "${DOKS_PROJECT}" '.[] | select(.name == $name) | .id')
+				cluster_id=$(doctl kubernetes cluster list --output json | jq -r --arg name "$name" '.[] | select(.name == $name) | .id')
+				urn=do:kubernetes:$cluster_id
+				doctl projects resources assign "$project_id" --resource "$urn"
+			fi
+			;;
+
+		gke)
+			echo "Creating GKE cluster: $name ($version)"
+			gcloud container clusters create "$name" \
+				--zone "$region" \
+				--cluster-ipv4-cidr=/21 \
+				--cluster-version "$version" \
+				--num-nodes "3" \
+				--machine-type "n1-standard-4" \
+				--labels="delete-cluster-after-hours=$ttl,team=cloud,product=pg-operator" \
+				--no-enable-autoupgrade \
+				--monitoring=NONE \
+				--logging=NONE \
+				--no-enable-managed-prometheus \
+				--quiet
+
+			kubectl create clusterrolebinding cluster-admin-binding1 \
+				--clusterrole=cluster-admin \
+				--user="$(gcloud config get-value core/account)"
+			;;
+
+		*)
+			echo "ERROR: unsupported platform '$platform'"
+			return 1
+			;;
+	esac
+
+	kubectl create namespace "${NAMESPACE}"
+	echo "Cluster $name created successfully"
+}
+
+upgrade_k8s_cluster() {
+	local name="$1"
+	local version="$2"
+	local platform="$(detect_k8s_platform)"
+
+	case "$platform" in
+		digitalocean)
+			echo "Upgrading DOKS cluster: $name to $version"
+			doctl kubernetes cluster upgrade "$name" \
+				--version "$version"
+			;;
+		gke)
+			echo "Upgrading GKE cluster: $name to $version"
+			location=$(gcloud container clusters list \
+				--filter="name=$name" \
+				--format="value(location)")
+			gcloud container clusters upgrade "$name" \
+				--master \
+				--cluster-version "$version" \
+				--zone "$location" \
+				--quiet
+			gcloud container clusters upgrade "$name" \
+				--node-pool default-pool \
+				--cluster-version "$version" \
+				--zone "$location" \
+				--quiet
+			;;
+		*)
+			echo "ERROR: unsupported platform '$platform'"
+			return 1
+			;;
+	esac
+
+	echo "Cluster $name upgraded to $version successfully"
+}
+
+delete_k8s_cluster() {
+	local name="$1"
+	local platform="$2"
+	local region="$3"
+
+	case "$platform" in
+		digitalocean)
+			echo "Deleting DOKS cluster: $name"
+			cluster_id=$(doctl kubernetes cluster list --output json | jq -r --arg name "$name" '.[] | select(.name == $name) | .id')
+			# Dangerous includes LoadBalancers, Volumes and all resources related to the cluster
+			doctl kubernetes cluster delete "$cluster_id" --force --dangerous
+			;;
+		gke)
+			echo "Deleting GKE cluster: $name"
+			gcloud container clusters delete "$name" \
+				--zone "$region" \
+				--quiet \
+				--async
+			;;
+		*)
+			echo "ERROR: unsupported platform '$platform'"
+			return 1
+			;;
+	esac
+
+	echo "Cluster $name deleted successfully"
+}
+
+get_k8s_cluster_name() {
+	local platform="${1:-$(detect_k8s_platform)}"
+
+	case "$platform" in
+		digitalocean)
+			local nodepool_id="$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.doks\.digitalocean\.com/node-pool-id}')"
+			doctl kubernetes cluster list -o json \
+				| jq -r --arg id "$nodepool_id" \
+					'.[] | select(.node_pools[]?.id == $id) | .name' 2>/dev/null
+			;;
+		gke)
+			kubectl get nodes -o jsonpath='{.items[0].spec.providerID}' \
+				| sed -E 's|.*gke-(.*)-default-pool.*|\1|'
+			;;
+		*)
+			echo "ERROR: unsupported platform '$platform'" >&2
+			return 1
+			;;
+	esac
+}
+
+get_k8s_cluster_version() {
+	local name="$1"
+	local platform="${2:-$(detect_k8s_platform)}"
+
+	case "$platform" in
+		digitalocean)
+			doctl kubernetes cluster get "$name" --output json \
+				| jq -r '.[0].version'
+			;;
+		gke)
+			location=$(gcloud container clusters list \
+				--filter="name=$name" \
+				--format="value(location)")
+			gcloud container clusters describe "$name" \
+				--zone "$location" \
+				--format="value(currentMasterVersion)"
+			;;
+		*)
+			echo "ERROR: unsupported platform '$platform'" >&2
+			return 1
+			;;
+	esac
+}
+
+get_k8s_nodes_version() {
+	kubectl get nodes -o json \
+		| jq -r '.items[].status.nodeInfo.kubeletVersion' \
+		| sort -u
+}
+
+verify_k8s_cluster_version() {
+	local cluster_name="$1"
+	local expected_version="$2"
+	local current_version="$(get_k8s_cluster_version "$cluster_name")"
+
+	if [[ $expected_version != "$current_version" ]]; then
+		echo "Error current cluster version $current_version is not the same as expected version $expected_version"
+		return 1
+	fi
+}
+
+verify_k8s_nodes_version() {
+	local expected_version="$1"
+	local nodes_version="$(get_k8s_nodes_version)"
+	local expected_clean="$(echo "$expected_version" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" # Gets only Kubernetes version xx.xx.xx
+
+	for node_version in $nodes_version; do
+		node_clean=$(echo "$node_version" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')
+		if [[ $node_clean != "$expected_clean" ]]; then
+			echo "Error current node version $node_clean is not the same as expected version $expected_clean"
+			return 1
+		fi
+	done
+}

e2e-tests/run-release.csv

@@ -10,6 +10,7 @@ demand-backup-offline-snapshot
 dynamic-configuration
 finalizers
 init-deploy
+k8s-upgrade
 huge-pages
 major-upgrade-13-to-14
 major-upgrade-14-to-15

e2e-tests/tests/k8s-upgrade/00-assert.yaml

@@ -0,0 +1,35 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 300
+commands:
+  - script: |
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      expected_cluster_name=$(kubectl get configmap cluster-name -o jsonpath='{.data.name}' -n ${NAMESPACE})
+      expected_version=$(kubectl get configmap initial-cluster-version -o jsonpath='{.data.version}' -n ${NAMESPACE})
+
+      until verify_k8s_cluster_version "$expected_cluster_name" "$expected_version"; do
+        echo "Waiting for the cluster to state the version $expected_version..."
+        sleep 10
+      done
+
+      until verify_k8s_nodes_version "$expected_version"; do
+        echo "Waiting for all nodes to state the version $expected_version..."
+        kubectl get nodes
+        sleep 10
+      done
+
+      until [[ -z "$(kubectl get nodes --no-headers | awk '$2 != "Ready" {print}')" ]]; do
+        echo "Waiting for all nodes to be Ready..."
+        kubectl get nodes
+        sleep 5
+      done
+
+      cluster_name=$(get_k8s_cluster_name "${K8S_UPGRADE_PLATFORM}")
+      if [[ "$expected_cluster_name" != "$cluster_name" ]]; then
+        echo "Error current cluster $cluster_name is not the same as expected cluster name $expected_cluster_name"
+        exit 1
+      fi

e2e-tests/tests/k8s-upgrade/00-create-k8s-cluster.yaml

@@ -0,0 +1,41 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - timeout: 1000
+    script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      name="k8s-upgrade-$RANDOM"
+
+      if [[ "${K8S_UPGRADE_PLATFORM}" != "gke" && "${K8S_UPGRADE_PLATFORM}" != "digitalocean" ]]; then
+        echo "Unsupported platform: ${K8S_UPGRADE_PLATFORM}"
+        exit 1
+      fi
+
+      if [[ -n "${K8S_UPGRADE_INITIAL_VERSION}" && -z "${K8S_UPGRADE_FINAL_VERSION}" ]] ||
+        [[ -z "${K8S_UPGRADE_INITIAL_VERSION}" && -n "${K8S_UPGRADE_FINAL_VERSION}" ]]; then
+        echo "K8S_UPGRADE_INITIAL_VERSION and K8S_UPGRADE_FINAL_VERSION are both required when one is configured"
+        exit 1
+      fi
+
+      # By defautt, we will use the two most recent Kubernetes versions available for the specified platform and region
+      initial_version="${K8S_UPGRADE_INITIAL_VERSION:-$(get_available_k8s_versions 2 "${K8S_UPGRADE_PLATFORM}" "${K8S_UPGRADE_REGION}")}"
+      final_version="${K8S_UPGRADE_FINAL_VERSION:-$(get_available_k8s_versions 1 "${K8S_UPGRADE_PLATFORM}" "${K8S_UPGRADE_REGION}")}"
+
+      create_k8s_cluster "$name" "$initial_version" "${K8S_UPGRADE_PLATFORM}" "${K8S_UPGRADE_REGION}"
+
+      # Saving cluster information and available versions for the next test steps
+      kubectl create configmap cluster-name \
+        --from-literal=name="$name" \
+        --namespace "${NAMESPACE}"
+
+      kubectl create configmap initial-cluster-version \
+        --from-literal=version="$initial_version" \
+        --namespace "${NAMESPACE}"
+
+      kubectl create configmap final-cluster-version \
+        --from-literal=version="$final_version" \
+        --namespace "${NAMESPACE}"

e2e-tests/tests/k8s-upgrade/01-assert.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+metadata:
+  name: check-operator-deploy-status
+timeout: 120
+commands:
+  - script: |
+      set -euo pipefail
+
+      kubectl assert exist-enhanced crd perconapgclusters.pgv2.percona.com
+      kubectl assert exist-enhanced crd perconapgbackups.pgv2.percona.com
+      kubectl assert exist-enhanced crd perconapgrestores.pgv2.percona.com
+      kubectl assert exist-enhanced crd perconapgupgrades.pgv2.percona.com
+      kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1

e2e-tests/tests/k8s-upgrade/01-deploy-operator.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+      init_temp_dir # do this only in the first TestStep
+
+      deploy_operator
+      deploy_client
+      deploy_s3_secrets

e2e-tests/tests/k8s-upgrade/02-assert.yaml

@@ -0,0 +1,33 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 600
+commands:
+- script: |
+    verify_cluster() {
+      kubectl assert exist-enhanced statefulset k8s-upgrade-repo-host -n ${NAMESPACE} \
+        --field-selector status.readyReplicas=1 \
+        --field-selector status.observedGeneration=1
+
+      for sts in $(kubectl get statefulset -n ${NAMESPACE} \
+        -l postgres-operator.crunchydata.com/instance-set=instance1 \
+        -o jsonpath='{.items[*].metadata.name}'); do
+        kubectl assert exist-enhanced statefulset ${sts} -n ${NAMESPACE} \
+          --field-selector status.readyReplicas=1 \
+          --field-selector status.observedGeneration=1
+      done
+
+      kubectl assert exist-enhanced deployment k8s-upgrade-pgbouncer -n ${NAMESPACE} \
+        --field-selector status.readyReplicas=3
+
+      kubectl assert exist-enhanced postgrescluster k8s-upgrade -n ${NAMESPACE} \
+        --field-selector status.proxy.pgBouncer.readyReplicas=3
+
+      kubectl assert exist-enhanced perconapgcluster k8s-upgrade -n ${NAMESPACE} \
+        --field-selector status.state=ready \
+        --field-selector metadata.generation=1
+    }
+
+    until verify_cluster; do
+      echo "Waiting for cluster to be ready..."
+      sleep 10
+    done