Skip to content

Commit 2e89eba

Browse files
valmiranogueiravalmiranogueira
authored
Merge branch 'main' into release-3.0.0
2 parents dbee68a + 8c38aaf commit 2e89eba

25 files changed

Lines changed: 691 additions & 0 deletions

‎e2e-tests/functions‎

Lines changed: 233 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1777,3 +1777,236 @@ verify_hugepages_usage() {
17771777
return 1
17781778
fi
17791779
}
1780+
1781+
get_available_k8s_versions() {
1782+
local offset="${1:-1}"
1783+
local platform="$2"
1784+
local region="$3"
1785+
local versions=""
1786+
1787+
case "$platform" in
1788+
digitalocean)
1789+
versions=$(doctl kubernetes options versions -o json \
1790+
| jq -r '.[].slug')
1791+
;;
1792+
1793+
gke)
1794+
versions=$(
1795+
gcloud container get-server-config \
1796+
--region "${region}" \
1797+
--format="value(validMasterVersions)" 2>/dev/null \
1798+
| tr ';' '\n' \
1799+
| awk -F'.' '!seen[$1"."$2]++'
1800+
)
1801+
;;
1802+
1803+
*)
1804+
echo "ERROR: unsupported platform '$platform'"
1805+
return 1
1806+
;;
1807+
esac
1808+
1809+
versions_sorted=$(echo "$versions" | sort -u -V)
1810+
echo "$versions_sorted" | tail -n "$offset" | head -n 1
1811+
}
1812+
1813+
create_k8s_cluster() {
1814+
local name="$1"
1815+
local version="$2"
1816+
local platform="$3"
1817+
local region="$4"
1818+
local ttl="${5:-"1"}"
1819+
1820+
case "$platform" in
1821+
1822+
digitalocean)
1823+
echo "Creating DOKS cluster: $name ($version)"
1824+
creation_time=$(date +%s)
1825+
doctl kubernetes cluster create "$name" \
1826+
--region "$region" \
1827+
--version "$version" \
1828+
--size "s-4vcpu-16gb-amd" \
1829+
--count "3" \
1830+
--tag "delete-cluster-after-hours:$ttl" \
1831+
--tag "creation-time:$creation_time" \
1832+
--tag "team:cloud" \
1833+
--tag "product:pg-operator" \
1834+
--wait
1835+
1836+
if [[ -n $DOKS_PROJECT ]]; then
1837+
project_id=$(doctl projects list --output json | jq -r --arg name "${DOKS_PROJECT}" '.[] | select(.name == $name) | .id')
1838+
cluster_id=$(doctl kubernetes cluster list --output json | jq -r --arg name "$name" '.[] | select(.name == $name) | .id')
1839+
urn=do:kubernetes:$cluster_id
1840+
doctl projects resources assign "$project_id" --resource "$urn"
1841+
fi
1842+
;;
1843+
1844+
gke)
1845+
echo "Creating GKE cluster: $name ($version)"
1846+
gcloud container clusters create "$name" \
1847+
--zone "$region" \
1848+
--cluster-ipv4-cidr=/21 \
1849+
--cluster-version "$version" \
1850+
--num-nodes "3" \
1851+
--machine-type "n1-standard-4" \
1852+
--labels="delete-cluster-after-hours=$ttl,team=cloud,product=pg-operator" \
1853+
--no-enable-autoupgrade \
1854+
--monitoring=NONE \
1855+
--logging=NONE \
1856+
--no-enable-managed-prometheus \
1857+
--quiet
1858+
1859+
kubectl create clusterrolebinding cluster-admin-binding1 \
1860+
--clusterrole=cluster-admin \
1861+
--user="$(gcloud config get-value core/account)"
1862+
;;
1863+
1864+
*)
1865+
echo "ERROR: unsupported platform '$platform'"
1866+
return 1
1867+
;;
1868+
esac
1869+
1870+
kubectl create namespace "${NAMESPACE}"
1871+
echo "Cluster $name created successfully"
1872+
}
1873+
1874+
upgrade_k8s_cluster() {
1875+
local name="$1"
1876+
local version="$2"
1877+
local platform="$(detect_k8s_platform)"
1878+
1879+
case "$platform" in
1880+
digitalocean)
1881+
echo "Upgrading DOKS cluster: $name to $version"
1882+
doctl kubernetes cluster upgrade "$name" \
1883+
--version "$version"
1884+
;;
1885+
gke)
1886+
echo "Upgrading GKE cluster: $name to $version"
1887+
location=$(gcloud container clusters list \
1888+
--filter="name=$name" \
1889+
--format="value(location)")
1890+
gcloud container clusters upgrade "$name" \
1891+
--master \
1892+
--cluster-version "$version" \
1893+
--zone "$location" \
1894+
--quiet
1895+
gcloud container clusters upgrade "$name" \
1896+
--node-pool default-pool \
1897+
--cluster-version "$version" \
1898+
--zone "$location" \
1899+
--quiet
1900+
;;
1901+
*)
1902+
echo "ERROR: unsupported platform '$platform'"
1903+
return 1
1904+
;;
1905+
esac
1906+
1907+
echo "Cluster $name upgraded to $version successfully"
1908+
}
1909+
1910+
delete_k8s_cluster() {
1911+
local name="$1"
1912+
local platform="$2"
1913+
local region="$3"
1914+
1915+
case "$platform" in
1916+
digitalocean)
1917+
echo "Deleting DOKS cluster: $name"
1918+
cluster_id=$(doctl kubernetes cluster list --output json | jq -r --arg name "$name" '.[] | select(.name == $name) | .id')
1919+
# Dangerous includes LoadBalancers, Volumes and all resources related to the cluster
1920+
doctl kubernetes cluster delete "$cluster_id" --force --dangerous
1921+
;;
1922+
gke)
1923+
echo "Deleting GKE cluster: $name"
1924+
gcloud container clusters delete "$name" \
1925+
--zone "$region" \
1926+
--quiet \
1927+
--async
1928+
;;
1929+
*)
1930+
echo "ERROR: unsupported platform '$platform'"
1931+
return 1
1932+
;;
1933+
esac
1934+
1935+
echo "Cluster $name deleted successfully"
1936+
}
1937+
1938+
get_k8s_cluster_name() {
1939+
local platform="${1:-$(detect_k8s_platform)}"
1940+
1941+
case "$platform" in
1942+
digitalocean)
1943+
local nodepool_id="$(kubectl get nodes -o jsonpath='{.items[0].metadata.labels.doks\.digitalocean\.com/node-pool-id}')"
1944+
doctl kubernetes cluster list -o json \
1945+
| jq -r --arg id "$nodepool_id" \
1946+
'.[] | select(.node_pools[]?.id == $id) | .name' 2>/dev/null
1947+
;;
1948+
gke)
1949+
kubectl get nodes -o jsonpath='{.items[0].spec.providerID}' \
1950+
| sed -E 's|.*gke-(.*)-default-pool.*|\1|'
1951+
;;
1952+
*)
1953+
echo "ERROR: unsupported platform '$platform'" >&2
1954+
return 1
1955+
;;
1956+
esac
1957+
}
1958+
1959+
get_k8s_cluster_version() {
1960+
local name="$1"
1961+
local platform="${2:-$(detect_k8s_platform)}"
1962+
1963+
case "$platform" in
1964+
digitalocean)
1965+
doctl kubernetes cluster get "$name" --output json \
1966+
| jq -r '.[0].version'
1967+
;;
1968+
gke)
1969+
location=$(gcloud container clusters list \
1970+
--filter="name=$name" \
1971+
--format="value(location)")
1972+
gcloud container clusters describe "$name" \
1973+
--zone "$location" \
1974+
--format="value(currentMasterVersion)"
1975+
;;
1976+
*)
1977+
echo "ERROR: unsupported platform '$platform'" >&2
1978+
return 1
1979+
;;
1980+
esac
1981+
}
1982+
1983+
get_k8s_nodes_version() {
1984+
kubectl get nodes -o json \
1985+
| jq -r '.items[].status.nodeInfo.kubeletVersion' \
1986+
| sort -u
1987+
}
1988+
1989+
verify_k8s_cluster_version() {
1990+
local cluster_name="$1"
1991+
local expected_version="$2"
1992+
local current_version="$(get_k8s_cluster_version "$cluster_name")"
1993+
1994+
if [[ $expected_version != "$current_version" ]]; then
1995+
echo "Error current cluster version $current_version is not the same as expected version $expected_version"
1996+
return 1
1997+
fi
1998+
}
1999+
2000+
verify_k8s_nodes_version() {
2001+
local expected_version="$1"
2002+
local nodes_version="$(get_k8s_nodes_version)"
2003+
local expected_clean="$(echo "$expected_version" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')" # Gets only Kubernetes version xx.xx.xx
2004+
2005+
for node_version in $nodes_version; do
2006+
node_clean=$(echo "$node_version" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+')
2007+
if [[ $node_clean != "$expected_clean" ]]; then
2008+
echo "Error current node version $node_clean is not the same as expected version $expected_clean"
2009+
return 1
2010+
fi
2011+
done
2012+
}

‎e2e-tests/run-release.csv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ demand-backup-offline-snapshot
99
dynamic-configuration
1010
finalizers
1111
init-deploy
12+
k8s-upgrade
1213
huge-pages
1314
major-upgrade-14-to-15
1415
major-upgrade-15-to-16

‎e2e-tests/tests/k8s-upgrade/00-assert.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestAssert
3+
timeout: 300
4+
commands:
5+
- script: |
6+
set -o errexit
7+
set -o xtrace
8+
9+
source ../../functions
10+
11+
expected_cluster_name=$(kubectl get configmap cluster-name -o jsonpath='{.data.name}' -n ${NAMESPACE})
12+
expected_version=$(kubectl get configmap initial-cluster-version -o jsonpath='{.data.version}' -n ${NAMESPACE})
13+
14+
until verify_k8s_cluster_version "$expected_cluster_name" "$expected_version"; do
15+
echo "Waiting for the cluster to state the version $expected_version..."
16+
sleep 10
17+
done
18+
19+
until verify_k8s_nodes_version "$expected_version"; do
20+
echo "Waiting for all nodes to state the version $expected_version..."
21+
kubectl get nodes
22+
sleep 10
23+
done
24+
25+
until [[ -z "$(kubectl get nodes --no-headers | awk '$2 != "Ready" {print}')" ]]; do
26+
echo "Waiting for all nodes to be Ready..."
27+
kubectl get nodes
28+
sleep 5
29+
done
30+
31+
cluster_name=$(get_k8s_cluster_name "${K8S_UPGRADE_PLATFORM}")
32+
if [[ "$expected_cluster_name" != "$cluster_name" ]]; then
33+
echo "Error current cluster $cluster_name is not the same as expected cluster name $expected_cluster_name"
34+
exit 1
35+
fi

‎e2e-tests/tests/k8s-upgrade/00-create-k8s-cluster.yaml‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestStep
3+
commands:
4+
- timeout: 1000
5+
script: |-
6+
set -o errexit
7+
set -o xtrace
8+
9+
source ../../functions
10+
11+
name="k8s-upgrade-$RANDOM"
12+
13+
if [[ "${K8S_UPGRADE_PLATFORM}" != "gke" && "${K8S_UPGRADE_PLATFORM}" != "digitalocean" ]]; then
14+
echo "Unsupported platform: ${K8S_UPGRADE_PLATFORM}"
15+
exit 1
16+
fi
17+
18+
if [[ -n "${K8S_UPGRADE_INITIAL_VERSION}" && -z "${K8S_UPGRADE_FINAL_VERSION}" ]] ||
19+
[[ -z "${K8S_UPGRADE_INITIAL_VERSION}" && -n "${K8S_UPGRADE_FINAL_VERSION}" ]]; then
20+
echo "K8S_UPGRADE_INITIAL_VERSION and K8S_UPGRADE_FINAL_VERSION are both required when one is configured"
21+
exit 1
22+
fi
23+
24+
# By defautt, we will use the two most recent Kubernetes versions available for the specified platform and region
25+
initial_version="${K8S_UPGRADE_INITIAL_VERSION:-$(get_available_k8s_versions 2 "${K8S_UPGRADE_PLATFORM}" "${K8S_UPGRADE_REGION}")}"
26+
final_version="${K8S_UPGRADE_FINAL_VERSION:-$(get_available_k8s_versions 1 "${K8S_UPGRADE_PLATFORM}" "${K8S_UPGRADE_REGION}")}"
27+
28+
create_k8s_cluster "$name" "$initial_version" "${K8S_UPGRADE_PLATFORM}" "${K8S_UPGRADE_REGION}"
29+
30+
# Saving cluster information and available versions for the next test steps
31+
kubectl create configmap cluster-name \
32+
--from-literal=name="$name" \
33+
--namespace "${NAMESPACE}"
34+
35+
kubectl create configmap initial-cluster-version \
36+
--from-literal=version="$initial_version" \
37+
--namespace "${NAMESPACE}"
38+
39+
kubectl create configmap final-cluster-version \
40+
--from-literal=version="$final_version" \
41+
--namespace "${NAMESPACE}"

‎e2e-tests/tests/k8s-upgrade/01-assert.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestAssert
3+
metadata:
4+
name: check-operator-deploy-status
5+
timeout: 120
6+
commands:
7+
- script: |
8+
set -euo pipefail
9+
10+
kubectl assert exist-enhanced crd perconapgclusters.pgv2.percona.com
11+
kubectl assert exist-enhanced crd perconapgbackups.pgv2.percona.com
12+
kubectl assert exist-enhanced crd perconapgrestores.pgv2.percona.com
13+
kubectl assert exist-enhanced crd perconapgupgrades.pgv2.percona.com
14+
kubectl assert exist-enhanced deployment percona-postgresql-operator -n ${OPERATOR_NS:-$NAMESPACE} --field-selector status.readyReplicas=1

‎e2e-tests/tests/k8s-upgrade/01-deploy-operator.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestStep
3+
timeout: 10
4+
commands:
5+
- script: |-
6+
set -o errexit
7+
set -o xtrace
8+
9+
source ../../functions
10+
init_temp_dir # do this only in the first TestStep
11+
12+
deploy_operator
13+
deploy_client
14+
deploy_s3_secrets

‎e2e-tests/tests/k8s-upgrade/02-assert.yaml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestAssert
3+
timeout: 600
4+
commands:
5+
- script: |
6+
verify_cluster() {
7+
kubectl assert exist-enhanced statefulset k8s-upgrade-repo-host -n ${NAMESPACE} \
8+
--field-selector status.readyReplicas=1 \
9+
--field-selector status.observedGeneration=1
10+
11+
for sts in $(kubectl get statefulset -n ${NAMESPACE} \
12+
-l postgres-operator.crunchydata.com/instance-set=instance1 \
13+
-o jsonpath='{.items[*].metadata.name}'); do
14+
kubectl assert exist-enhanced statefulset ${sts} -n ${NAMESPACE} \
15+
--field-selector status.readyReplicas=1 \
16+
--field-selector status.observedGeneration=1
17+
done
18+
19+
kubectl assert exist-enhanced deployment k8s-upgrade-pgbouncer -n ${NAMESPACE} \
20+
--field-selector status.readyReplicas=3
21+
22+
kubectl assert exist-enhanced postgrescluster k8s-upgrade -n ${NAMESPACE} \
23+
--field-selector status.proxy.pgBouncer.readyReplicas=3
24+
25+
kubectl assert exist-enhanced perconapgcluster k8s-upgrade -n ${NAMESPACE} \
26+
--field-selector status.state=ready \
27+
--field-selector metadata.generation=1
28+
}
29+
30+
until verify_cluster; do
31+
echo "Waiting for cluster to be ready..."
32+
sleep 10
33+
done

‎e2e-tests/tests/k8s-upgrade/02-create-cluster.yaml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
apiVersion: kuttl.dev/v1beta1
2+
kind: TestStep
3+
commands:
4+
- timeout: 60
5+
script: |-
6+
set -o errexit
7+
set -o xtrace
8+
9+
source ../../functions
10+
11+
get_cr "k8s-upgrade" ${RANDOM} | kubectl -n "${NAMESPACE}" apply -f -

0 commit comments

Comments
 (0)