Skip to content
This repository was archived by the owner on Jan 23, 2026. It is now read-only.

Commit de021ef

Browse files
gitbudaantejavor
gitbuda
and
antejavor
authored
Add v3.4 smoke tests and basic HA cluster upgrade test (#607)
The main challenge with the HA cluster upgrade tests is that there are multiple different things to wait for (helm, port-forward, data/coordinator getting ready) -> it gets very complex very quickly. ## v3.4 - [x] Nested indices - [x] Vector search on edges - [x] Implement the k8s helm chart upgrade scenario - [x] Resolve memgraph/memgraph#3023 (any reuse of the PVCs failed) - [x] memgraph/best-practices#43 -> almost exactly the same, look at how the script is waiting for pods - [x] Make upgrades more robust -> https://www.notion.so/memgraph/How-to-make-upgrade-more-robust-1c26b158b98180a1a76cd8998e2e7ca4#1c36b158b98180f286d4eed383d1451a - [x] Add some other feature tests (expand test cases) -> skipped - [x] Put a timeout to the wait_for_memgraph function (default is 10s) - [x] Validate results / make sure to exit on errors / properly name / refactor Python scripts - [x] Test for if main is elected specifically before running any CREATE query on the cluster NEXT >> #643 --------- Co-authored-by: antejavor <ante.javor@memgraph.io>
1 parent 383cfed commit de021ef

11 files changed

Lines changed: 347 additions & 36 deletions

File tree

‎.github/workflows/reusable_smoke_tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,7 +160,7 @@ jobs:
160160
-e MEMGRAPH_LAST_DOCKERHUB_IMAGE="$MEMGRAPH_LAST_DOCKERHUB_IMAGE" \
161161
-e MEMGRAPH_NEXT_DOCKERHUB_IMAGE="$MEMGRAPH_NEXT_DOCKERHUB_IMAGE" \
162162
"$CONTAINER_NAME" \
163-
bash -c "cd /mage/smoke-release-testing && ./test.bash"
163+
bash -c "cd /mage/smoke-release-testing && ./test_single_mage.bash"
164164
165165
- name: Clean up inner Docker state
166166
if: always()

‎smoke-release-testing/experiment.bash‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ echo "Waiting for memgraph to initialize..."
1010
wait_for_memgraph $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
1111
echo "Memgraph is up and running!"
1212

13-
source ./mgconsole/regex.bash
14-
test_regex $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
13+
source ./mgconsole/vector_search.bash
14+
test_vector_search $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
1515

1616
# NOTE: Test what's the exit status of the script by using `echo $?`:
1717
# * if it's == 0 -> all good

‎smoke-release-testing/k8s/run.bash‎

Lines changed: 148 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash
1+
#!/bin/bash -e
22
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
33
source "$SCRIPT_DIR/../utils.bash"
44
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
@@ -16,31 +16,165 @@ export PATH="$(go env GOPATH)/bin:$PATH"
1616
# * It takes some time to delete all PVs, check with `kubectl get pv`.
1717
# * If you want more details or helm dry run just append `--debug` of
1818
# `--dry-run`.
19+
BOLT_SERVER="localhost:10000" # Just tmp value -> each coordinator should have a different value.
20+
# E.g. if kubectl port-foward is used, the configured host values should be passed as `bolt_server`.
21+
22+
setup_coordinator() {
23+
local i=$1
24+
echo "ADD COORDINATOR $i WITH CONFIG {\"bolt_server\": \"$BOLT_SERVER\", \"management_server\": \"memgraph-coordinator-$i.default.svc.cluster.local:10000\", \"coordinator_server\": \"memgraph-coordinator-$i.default.svc.cluster.local:12000\"};" | $MEMGRAPH_CONSOLE_BINARY --port 17687
25+
echo "coordinator $i DONE"
26+
}
27+
setup_replica() {
28+
local i=$1
29+
echo "REGISTER INSTANCE instance_$i WITH CONFIG {\"bolt_server\": \"$BOLT_SERVER\", \"management_server\": \"memgraph-data-$i.default.svc.cluster.local:10000\", \"replication_server\": \"memgraph-data-$i.default.svc.cluster.local:20000\"};" | $MEMGRAPH_CONSOLE_BINARY --port 17687
30+
echo "replica $i DONE"
31+
}
32+
setup_main() {
33+
local i=$1
34+
echo "SET INSTANCE instance_$i TO MAIN;" | $MEMGRAPH_CONSOLE_BINARY --port 17687
35+
echo "main DONE"
36+
}
37+
setup_cluster() {
38+
with_kubectl_portforward memgraph-coordinator-1-0 17687:7687 'wait_for_memgraph_coordinator localhost 17687 5' -- \
39+
'setup_coordinator 1' \
40+
'setup_coordinator 2' \
41+
'setup_coordinator 3' \
42+
'setup_replica 0' \
43+
'setup_main 0'
44+
}
45+
46+
execute_query_against_main() {
47+
query="$1"
48+
with_kubectl_portforward memgraph-coordinator-1-0 17687:7687 'wait_for_memgraph_coordinator localhost 17687 5' -- \
49+
"MAIN_INSTANCE=\$(echo \"SHOW INSTANCES;\" | $MEMGRAPH_CONSOLE_BINARY --port 17687 --output-format=csv | python3 $SCRIPT_DIR/../validator.py get_main_parser)" \
50+
"echo \"NOTE: MAIN instance is \$MAIN_INSTANCE\"" \
51+
"echo \"MG_MAIN=\$MAIN_INSTANCE\" > $SCRIPT_DIR/mg_main.out" # Couldn't get export to move the info -> used file instead.
52+
source $SCRIPT_DIR/mg_main.out
53+
# NOTE: Waiting for MAIN is required because sometimes all instances are up, but MAIN is not yet fully configured.
54+
with_kubectl_portforward "$MG_MAIN-0" 17687:7687 'wait_for_memgraph localhost 17687 5' -- \
55+
"wait_for_memgraph_main localhost 17687 10" \
56+
"echo \"$query\" | $MEMGRAPH_CONSOLE_BINARY --port 17687"
57+
}
58+
59+
validate_nodes_against_main() {
60+
expected=$1
61+
with_kubectl_portforward memgraph-coordinator-1-0 17687:7687 'wait_for_memgraph_coordinator localhost 17687 5' -- \
62+
"MAIN_INSTANCE=\$(echo \"SHOW INSTANCES;\" | $MEMGRAPH_CONSOLE_BINARY --port 17687 --output-format=csv | python3 $SCRIPT_DIR/../validator.py get_main_parser)" \
63+
"echo \"NOTE: MAIN instance is \$MAIN_INSTANCE\"" \
64+
"echo \"MG_MAIN=\$MAIN_INSTANCE\" > $SCRIPT_DIR/mg_main.out" # Couldn't get export to move the info -> used file instead.
65+
source $SCRIPT_DIR/mg_main.out
66+
with_kubectl_portforward "$MG_MAIN-0" 17687:7687 'wait_for_memgraph localhost 17687 5' -- \
67+
"echo \"MATCH (n) RETURN n;\" | $MEMGRAPH_CONSOLE_BINARY --port 17687 --output-format=csv | python3 $SCRIPT_DIR/../validator.py validate_number_of_results -e $expected"
68+
echo "validate_nodes_against_main PASSED"
69+
}
1970

2071
test_k8s_single() {
2172
echo "Test k8s single memgraph instance using image: $MEMGRAPH_NEXT_DOCKERHUB_IMAGE"
2273
kind load docker-image $MEMGRAPH_NEXT_DOCKERHUB_IMAGE -n smoke-release-testing
2374
MEMGRAPH_NEXT_DOCKERHUB_TAG="${MEMGRAPH_NEXT_DOCKERHUB_IMAGE##*:}"
2475
helm install memgraph-single-smoke memgraph/memgraph \
25-
-f "$SCRIPT_DIR/values-single.yaml" --set "image.tag=$MEMGRAPH_NEXT_DOCKERHUB_TAG"
76+
-f "$SCRIPT_DIR/values-single.yaml" \
77+
--set "image.tag=$MEMGRAPH_NEXT_DOCKERHUB_TAG"
2678
kubectl wait --for=condition=Ready pod/memgraph-single-smoke-0 --timeout=120s
27-
kubectl port-forward memgraph-single-smoke-0 17687:7687 &
28-
PF_PID=$!
29-
wait_for_memgraph localhost 17687
30-
echo "CREATE ();" | $MEMGRAPH_CONSOLE_BINARY --port 17687
31-
echo "MATCH (n) RETURN n;" | $MEMGRAPH_CONSOLE_BINARY --port 17687
32-
kill $PF_PID
33-
wait $PF_PID 2>/dev/null || true
79+
80+
with_kubectl_portforward memgraph-single-smoke-0 17687:7687 "wait_for_memgraph localhost 17687 5" -- \
81+
"echo \"CREATE ();\" | $MEMGRAPH_CONSOLE_BINARY --port 17687" \
82+
"echo \"MATCH (n) RETURN n;\" | $MEMGRAPH_CONSOLE_BINARY --port 17687"
83+
3484
helm uninstall memgraph-single-smoke
3585
}
3686

37-
test_k8s_ha() {
38-
helm install myhadb memgraph/memgraph-high-availability \
87+
helm_install_myhadb() {
88+
chart_path="$1"
89+
image_tag="$2"
90+
helm install myhadb $chart_path \
3991
--set env.MEMGRAPH_ENTERPRISE_LICENSE=$MEMGRAPH_ENTERPRISE_LICENSE,env.MEMGRAPH_ORGANIZATION_NAME=$MEMGRAPH_ORGANIZATION_NAME \
40-
-f $SCRIPT_DIR/values-ha.yaml
41-
# TODO(gitbuda): Setup cluster commands + routing + test query.
92+
-f "$SCRIPT_DIR/values-ha.yaml" \
93+
--set "image.tag=$image_tag"
94+
}
95+
96+
test_k8s_help() {
97+
echo "usage: test_k8s_ha LAST|NEXT [-p|--chart-path PATH]"
98+
echo " [-s|--skip-cluster-setup] [-u|--skip-helm-uninstall] [-c|--skip-cleanup]"
99+
echo " [-n|--expected-nodes-no]"
100+
echo " [-h|--help]"
101+
exit 1
102+
}
103+
104+
cleanup_k8s_all() {
105+
# NOTE: An attempt to cleanup any leftovers from kubectl port-forward...
106+
kill -9 $(pgrep kubectl) || true
107+
if helm status myhadb > /dev/null 2>&1; then
108+
helm uninstall myhadb
109+
fi
110+
if helm status myhadb > /dev/null 2>&1; then
111+
helm uninstall memgraph-single-smoke
112+
fi
113+
kubectl delete pvc --all
114+
}
115+
116+
test_k8s_ha() {
117+
if [ "$#" -lt 1 ]; then
118+
test_k8s_help
119+
fi
120+
WHICH="$1"
121+
WHICH_TMP="MEMGRAPH_${WHICH}_DOCKERHUB_IMAGE"
122+
WHICH_IMAGE="${!WHICH_TMP}"
123+
MEMGRAPH_DOCKERHUB_TAG="${WHICH_IMAGE##*:}"
124+
shift
125+
CHART_PATH="memgraph/memgraph-high-availability"
126+
SKIP_CLUSTER_SETUP=false
127+
SKIP_CLEANUP=false
128+
SKIP_HELM_UNINSTALL=false
129+
EXPECTED_NODES_COUNT=1
130+
while [ "$#" -gt 0 ]; do
131+
case $1 in
132+
-p|--chart-path) CHART_PATH="$2"; shift 2 ;;
133+
-s|--skip-cluster-setup) SKIP_CLUSTER_SETUP=true; shift ;;
134+
-u|--skip-helm-uninstall) SKIP_HELM_UNINSTALL=true; shift ;;
135+
-c|--skip-cleanup) SKIP_CLEANUP=true; shift ;;
136+
-n|--expected-nodes-no) EXPECTED_NODES_COUNT="$2"; shift 2 ;;
137+
-h|--help) test_k8s_help; ;;
138+
*) shift; break ;;
139+
esac
140+
done
141+
echo "Test k8s HA memgraph cluster using image:"
142+
echo " * image: $WHICH_IMAGE"
143+
echo " * tag: $MEMGRAPH_DOCKERHUB_TAG"
144+
echo " * chart: $CHART_PATH"
145+
echo " * expected nodes number: $EXPECTED_NODES_COUNT"
146+
echo " * skip cluster setup: $SKIP_CLUSTER_SETUP"
147+
echo " * skip helm uninstall: $SKIP_HELM_UNINSTALL"
148+
echo " * skip cleanup: $SKIP_CLEANUP"
149+
150+
kind load docker-image $WHICH_IMAGE -n smoke-release-testing
151+
helm_install_myhadb $CHART_PATH $MEMGRAPH_DOCKERHUB_TAG
152+
sleep 1 # NOTE: Sometimes there is an Error from Server -> pod XYZ not found...
153+
kubectl wait --for=condition=Ready pod -l role=coordinator --timeout=120s
154+
kubectl wait --for=condition=Ready pod -l role=data --timeout=120s
155+
156+
if [ "$SKIP_CLUSTER_SETUP" = false ]; then
157+
setup_cluster
158+
fi
159+
execute_query_against_main "SHOW VERSION;"
160+
execute_query_against_main "CREATE ();"
161+
validate_nodes_against_main $EXPECTED_NODES_COUNT
162+
if [ "$SKIP_HELM_UNINSTALL" = false ]; then
163+
helm uninstall myhadb
164+
fi
165+
if [ "$SKIP_CLEANUP" = false ]; then
166+
kubectl delete pvc --all
167+
fi
42168
}
43169

44170
if [ "${BASH_SOURCE[0]}" -ef "$0" ]; then
45-
test_k8s_single
171+
# NOTE: Developing workflow: download+load required images and defined MEMGRAPH_NEXT_DOCKERHUB_IMAGE.
172+
echo "Running $0 directly..."
173+
174+
# test_k8s_single
175+
# test_k8s_ha LAST -c # Skip cleanup because we want to recover from existing PVCs.
176+
# test_k8s_ha NEXT -s # Skip cluster setup because that should be recovered from PVCs.
177+
178+
# How to inject local version of the helm chart because we want to test any local fixes upfront.
179+
# test_k8s_ha NEXT ~/Workspace/code/memgraph/helm-charts/charts/memgraph-high-availability
46180
fi

‎smoke-release-testing/k8s/values-ha.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ image:
22
repository: memgraph/memgraph
33
# It is a bad practice to set the image tag name to latest as it can trigger automatic upgrade of the charts
44
# With some of the pullPolicy values. Please consider fixing the tag to a specific Memgraph version
5-
tag: 3.2.1
5+
tag: 3.4.0
66
pullPolicy: IfNotPresent
77

88
env:

‎smoke-release-testing/k8s/values-single.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
image:
2-
repository: memgraph/memgraph-mage
2+
repository: memgraph/memgraph
33
# Overrides the image tag whose default is v{{ .Chart.AppVersion }}
44
# It is a bad practice to set the image tag name to latest as it can trigger automatic upgrade of the charts
55
# With some of the pullPolicy values. Please consider fixing the tag to a specific Memgraph version

‎smoke-release-testing/mgconsole/indices.bash‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,14 @@ test_composite_indices() {
1111
echo "CREATE (:Label {prop1:0, prop2: 1});" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
1212
echo "EXPLAIN MATCH (n:Label {prop1:0, prop2: 1}) RETURN n;" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port | grep -E "ScanAllByLabelProperties \(n :Label \{prop1, prop2\}\)"
1313
}
14+
15+
test_nested_indices() {
16+
__host="$1"
17+
__port="$2"
18+
echo "FEATURE: Nested Indices"
19+
20+
echo "CREATE INDEX ON :Project(delivery.status.due_date);" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
21+
echo "CREATE (:Project {delivery: {status: {due_date: date('2025-06-04'), milestone: 'v3.14'}}});" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
22+
echo "EXPLAIN MATCH (proj:Project) WHERE proj.delivery.status.due_date = date('2025-06-04') RETURN *;" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port | grep -E "ScanAllByLabelProperties"
23+
24+
}

‎smoke-release-testing/mgconsole/vector_search.bash‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ test_vector_search() {
88
echo "FEATURE: Vector Search"
99

1010
echo "CREATE VECTOR INDEX vsi ON :Label(embedding) WITH CONFIG {\"dimension\":2, \"capacity\": 10};" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
11+
echo "CREATE VECTOR EDGE INDEX etvsi ON :EdgeType(embedding) WITH CONFIG {\"dimension\": 256, \"capacity\": 1000};" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
12+
1113
echo "CALL vector_search.show_index_info() YIELD * RETURN *;" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
1214
echo "SHOW VECTOR INDEX INFO;" | $MEMGRAPH_CONSOLE_BINARY --host $__host --port $__port
1315
}

‎smoke-release-testing/test_ha_memgraph.bash‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash -e
2+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
3+
source "$SCRIPT_DIR/utils.bash"
4+
5+
source $SCRIPT_DIR/k8s/run.bash
6+
cleanup_k8s_all
7+
8+
test_k8s_single
9+
10+
# Test the upgrade scenario. PVCs are preserved between the two runs.
11+
test_k8s_ha LAST -n 1 -c # Skip cleanup because we want to recover from existing PVCs.
12+
test_k8s_ha NEXT -n 2 -s # Skip cluster setup because that should be recovered from PVCs.

smoke-release-testing/test.bash renamed to smoke-release-testing/test_single_mage.bash

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,8 @@
22
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
33
source "$SCRIPT_DIR/utils.bash"
44

5-
# TODO(gitbuda): Measure the total execution time, it should be under ~10s.
6-
# TODO(gitbuda): Test docker compose.
7-
85
# NOTE: 1st arg is how to pull LAST image, 2nd arg is how to pull NEXT image.
9-
spinup_and_cleanup_memgraph_dockers DockerHub RC
6+
spinup_and_cleanup_memgraph_dockers Dockerhub RC
107
echo "Waiting for memgraph to initialize..."
118
wait_for_memgraph $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_LAST_DATA_BOLT_PORT
129
wait_for_memgraph $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
@@ -44,11 +41,8 @@ test_edge_type_operations $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
4441
test_composite_indices $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
4542
test_monitoring $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
4643
test_multi_tenancy $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
44+
test_nested_indices $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
4745
test_or_expression_for_labels $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
4846
# NOTE: If the testing container is NOT restarted, all the auth test have to
4947
# come after all tests that assume there are no users.
5048
test_impersonate_user $MEMGRAPH_DEFAULT_HOST $MEMGRAPH_NEXT_DATA_BOLT_PORT
51-
52-
# k8s is a special case, because it requires extra setup.
53-
source $SCRIPT_DIR/k8s/run.bash
54-
test_k8s_single

0 commit comments

Comments
 (0)