Skip to content

Commit 92cfcef

Browse files
ajfloederajfloeder
authored
Add gfs2 fence test option (#141)
* Add gfs2 fence test option Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com> * Add copyrights Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com> --------- Signed-off-by: Anthony Floeder <anthony.floeder@hpe.com>
1 parent 6728fcf commit 92cfcef

6 files changed

Lines changed: 304 additions & 1 deletion

File tree

‎Makefile‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ dm:
5959
container:
6060
${GINKGO_RUN} --label-filter='container' .
6161

62+
# Run gfs2 fence test
63+
.PHONY: gfs2_fence
64+
gfs2_fence:
65+
${GINKGO_RUN} --label-filter='gfs2_fence' .
66+
6267
.PHONY: .version
6368
.version: ## Uses the git-version-gen script to generate a tag version
6469
./git-version-gen --fallback `git rev-parse HEAD` > .version

‎int_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ package test
2121

2222
import (
2323
"fmt"
24+
"time"
2425

2526
. "github.com/NearNodeFlash/nnf-integration-test/internal"
2627

@@ -77,13 +78,21 @@ var tests = []*T{
7778
// External Computes
7879
MakeTest("Lustre External", "#DW jobdw type=lustre name=lustre capacity=50GB").WithExternalComputes().WithLabels(ExternalLustre),
7980

81+
// GFS2 Fence
82+
MakeTest("GFS2 Fence", "#DW jobdw type=gfs2 name=gfs2-fence capacity=50GB").WithLabels(GFS2Fence).
83+
DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second). // start pacemaker
84+
DelayInState(dwsv1alpha7.StatePreRun, 60*time.Second). // fence node(s)
85+
DelayInState(dwsv1alpha7.StateDataOut, 15*time.Second), // stop pacemaker on surviving node(s)
86+
8087
// Storage Profiles
8188
MakeTest("XFS with Storage Profile",
8289
"#DW jobdw type=xfs name=xfs-storage-profile capacity=50GB profile=my-xfs-storage-profile").
8390
WithStorageProfile(),
8491
MakeTest("GFS2 with Storage Profile",
8592
"#DW jobdw type=gfs2 name=gfs2-storage-profile capacity=50GB profile=my-gfs2-storage-profile").
8693
WithStorageProfile(),
94+
// WithStorageProfile().DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second).DelayInState(dwsv1alpha7.StateDataOut, 15*time.Second).Focused(), // Useful for debugging
95+
// WithStorageProfile().DelayInState(dwsv1alpha7.StateDataIn, 15*time.Second).StopAfter(dwsv1alpha7.StatePreRun).Focused(),
8796
MakeTest("XFS with Storage Profile and LV Create",
8897
"#DW jobdw type=xfs name=xfs-storage-profile capacity=14TB profile=my-xfs-storage-profile").
8998
WithStorageProfileLvCreate("--zero n --activate y --type raid5 --nosync --extents $PERCENT_VG --stripes $DEVICE_NUM-1 --stripesize=64KiB --name $LV_NAME $VG_NAME"),

‎internal/internal.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ func (t *T) Workflow() *dwsv1alpha7.Workflow {
153153
const (
154154
Simple = "simple"
155155
ExternalLustre = "external_lustre"
156+
GFS2Fence = "gfs2_fence"
156157
)
157158

158159
func (t *T) WithLabels(labels ...string) *T { t.labels = append(t.labels, labels...); return t }

‎internal/options.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,7 +401,7 @@ func (t *T) Prepare(ctx context.Context, k8sClient client.Client) error {
401401
profile.Data.RetryLimit = int32(*opt.RetryLimit)
402402
}
403403
if opt.NoStorage {
404-
for i, _ := range profile.Data.Storages {
404+
for i := range profile.Data.Storages {
405405
storage := &profile.Data.Storages[i]
406406
storage.Optional = true
407407
}

‎scripts/run_test_and_manage_cluster.sh‎

Lines changed: 235 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,235 @@
1+
#!/bin/bash
2+
3+
# Copyright 2025 Hewlett Packard Enterprise Development LP
4+
# Other additional copyright holders may be indicated within.
5+
#
6+
# The entirety of this work is licensed under the Apache License,
7+
# Version 2.0 (the "License"); you may not use this file except
8+
# in compliance with the License.
9+
#
10+
# You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
# See the License for the specific language governing permissions and
18+
# limitations under the License.
19+
20+
# This script automates starting and stopping cluster services during a 'make test' run.
21+
# It randomly selects nodes to fence during PreRun and manages services accordingly.
22+
23+
# --- Configuration ---
24+
# All compute nodes available
25+
ALL_NODES=(rabbit-compute-2 rabbit-compute-3 rabbit-compute-4 rabbit-compute-5)
26+
27+
# Map compute nodes to their rabbit nodes
28+
# Function to get the rabbit node for a given compute node
29+
get_rabbit_for_compute() {
30+
local compute_node="$1"
31+
case "$compute_node" in
32+
rabbit-compute-2) echo "rabbit-node-1" ;;
33+
rabbit-compute-3) echo "rabbit-node-1" ;;
34+
rabbit-compute-4) echo "rabbit-node-2" ;;
35+
rabbit-compute-5) echo "rabbit-node-2" ;;
36+
*) echo "" ;;
37+
esac
38+
}
39+
40+
# Percentage of nodes to fence (0-100)
41+
FENCE_PERCENTAGE=0
42+
43+
# The command to start the services
44+
START_CMD='sudo systemctl start corosync pacemaker && sleep 5 && sudo pcs node unstandby'
45+
46+
# The command to stop the services
47+
STOP_CMD='sudo pcs node standby && sudo systemctl stop pacemaker corosync'
48+
49+
# The absolute path to the clush command
50+
# CLUSH_CMD="/Users/afloeder/dev2/nnf-deploy/nnf-integration-test/.venv/bin/clush"
51+
CLUSH_CMD="clush"
52+
53+
# --- Randomly select nodes to fence (balanced across rabbits) ---
54+
NUM_NODES=${#ALL_NODES[@]}
55+
NUM_TO_FENCE=$(( NUM_NODES * FENCE_PERCENTAGE / 100 ))
56+
if [ $NUM_TO_FENCE -eq 0 ] && [ $FENCE_PERCENTAGE -gt 0 ]; then
57+
NUM_TO_FENCE=1 # Fence at least one node if percentage > 0
58+
fi
59+
60+
# Group nodes by rabbit
61+
RABBIT1_NODES=()
62+
RABBIT2_NODES=()
63+
for node in "${ALL_NODES[@]}"; do
64+
rabbit=$(get_rabbit_for_compute "$node")
65+
if [ "$rabbit" = "rabbit-node-1" ]; then
66+
RABBIT1_NODES+=("$node")
67+
elif [ "$rabbit" = "rabbit-node-2" ]; then
68+
RABBIT2_NODES+=("$node")
69+
fi
70+
done
71+
72+
# Calculate how many to fence from each rabbit (balance the load)
73+
NUM_FROM_RABBIT1=$(( NUM_TO_FENCE / 2 ))
74+
NUM_FROM_RABBIT2=$(( NUM_TO_FENCE - NUM_FROM_RABBIT1 ))
75+
76+
# If we need more from one rabbit than available, adjust
77+
if [ $NUM_FROM_RABBIT1 -gt ${#RABBIT1_NODES[@]} ]; then
78+
NUM_FROM_RABBIT1=${#RABBIT1_NODES[@]}
79+
NUM_FROM_RABBIT2=$(( NUM_TO_FENCE - NUM_FROM_RABBIT1 ))
80+
fi
81+
if [ $NUM_FROM_RABBIT2 -gt ${#RABBIT2_NODES[@]} ]; then
82+
NUM_FROM_RABBIT2=${#RABBIT2_NODES[@]}
83+
NUM_FROM_RABBIT1=$(( NUM_TO_FENCE - NUM_FROM_RABBIT2 ))
84+
fi
85+
86+
# Randomly select nodes from each rabbit
87+
NODES_TO_FENCE=()
88+
if [ $NUM_FROM_RABBIT1 -gt 0 ]; then
89+
NODES_TO_FENCE+=($(printf '%s\n' "${RABBIT1_NODES[@]}" | awk 'BEGIN{srand()}{print rand()"\t"$0}' | sort -n | cut -f2- | head -n $NUM_FROM_RABBIT1))
90+
fi
91+
if [ $NUM_FROM_RABBIT2 -gt 0 ]; then
92+
NODES_TO_FENCE+=($(printf '%s\n' "${RABBIT2_NODES[@]}" | awk 'BEGIN{srand()}{print rand()"\t"$0}' | sort -n | cut -f2- | head -n $NUM_FROM_RABBIT2))
93+
fi
94+
95+
NODES_NOT_FENCED=($(printf '%s\n' "${ALL_NODES[@]}" "${NODES_TO_FENCE[@]}" | sort | uniq -u))
96+
97+
echo "=========================================="
98+
echo "Fence Test Configuration:"
99+
echo "=========================================="
100+
echo "Total nodes: ${#ALL_NODES[@]}"
101+
echo " rabbit-node-1 nodes: ${RABBIT1_NODES[*]}"
102+
echo " rabbit-node-2 nodes: ${RABBIT2_NODES[*]}"
103+
echo "Number to fence: $NUM_TO_FENCE (${FENCE_PERCENTAGE}%)"
104+
echo " From rabbit-node-1: $NUM_FROM_RABBIT1"
105+
echo " From rabbit-node-2: $NUM_FROM_RABBIT2"
106+
echo "Nodes to fence: ${NODES_TO_FENCE[*]}"
107+
echo "Nodes to manage normally: ${NODES_NOT_FENCED[*]}"
108+
echo "=========================================="
109+
echo
110+
111+
# Convert arrays to clush-compatible format
112+
ALL_NODES_PATTERN=$(IFS=,; echo "${ALL_NODES[*]}")
113+
FENCE_NODES_PATTERN=$(IFS=,; echo "${NODES_TO_FENCE[*]}")
114+
NORMAL_NODES_PATTERN=$(IFS=,; echo "${NODES_NOT_FENCED[*]}")
115+
116+
# --- Script ---
117+
118+
echo "Starting 'make gfs2_fence' and monitoring for state changes..."
119+
120+
# Use process substitution to read from the 'make test' output line by line.
121+
# The output of 'make test' is also sent to the terminal.
122+
while IFS= read -r LINE; do
123+
echo "$LINE"
124+
125+
# Check for the "DataIn" state - Start services on all nodes
126+
if echo "$LINE" | grep -q "Delaying in state DataIn"; then
127+
echo
128+
echo ">>> 'DataIn' state detected. Starting services on ALL nodes <<<"
129+
$CLUSH_CMD -w "$ALL_NODES_PATTERN" "$START_CMD"
130+
echo
131+
fi
132+
133+
# Check for the "PreRun" state - Fence selected nodes
134+
if echo "$LINE" | grep -q "Delaying in state PreRun"; then
135+
echo
136+
echo "=========================================="
137+
echo ">>> 'PreRun' state detected. Fencing nodes! <<<"
138+
echo "=========================================="
139+
140+
if [ ${#NODES_TO_FENCE[@]} -gt 0 ]; then
141+
# Fence each compute node via its rabbit
142+
for compute_node in "${NODES_TO_FENCE[@]}"; do
143+
rabbit_node=$(get_rabbit_for_compute "$compute_node")
144+
if [ -z "$rabbit_node" ]; then
145+
echo "[ERROR] No rabbit mapping found for $compute_node, skipping..."
146+
continue
147+
fi
148+
149+
echo "[FENCE] Fencing $compute_node via $rabbit_node..."
150+
$CLUSH_CMD -w "$rabbit_node" "sudo pcs stonith fence $compute_node" &
151+
done
152+
153+
# Wait for fencing operations to complete
154+
wait
155+
echo
156+
157+
# Verify fencing occurred
158+
echo "[VERIFY] Checking fence history for fenced nodes..."
159+
for compute_node in "${NODES_TO_FENCE[@]}"; do
160+
rabbit_node=$(get_rabbit_for_compute "$compute_node")
161+
if [ -n "$rabbit_node" ]; then
162+
echo " History for $compute_node (via $rabbit_node):"
163+
$CLUSH_CMD -w "$rabbit_node" "sudo pcs stonith history $compute_node | tail -10"
164+
fi
165+
done
166+
echo
167+
168+
echo "[INFO] Fenced nodes will remain fenced"
169+
echo
170+
171+
# Check node status from each rabbit
172+
echo "[STATUS] Checking cluster status from rabbits..."
173+
# Get unique rabbit nodes from the fenced compute nodes using simple deduplication
174+
unique_rabbits=""
175+
for compute_node in "${NODES_TO_FENCE[@]}"; do
176+
rabbit_node=$(get_rabbit_for_compute "$compute_node")
177+
if [ -n "$rabbit_node" ]; then
178+
# Check if we've already added this rabbit
179+
if ! echo "$unique_rabbits" | grep -q "$rabbit_node"; then
180+
unique_rabbits="$unique_rabbits $rabbit_node"
181+
fi
182+
fi
183+
done
184+
185+
for rabbit_node in $unique_rabbits; do
186+
echo " Status from $rabbit_node:"
187+
$CLUSH_CMD -w "$rabbit_node" "sudo pcs status nodes" | head -20
188+
done
189+
echo
190+
else
191+
echo "[INFO] No nodes selected for fencing in this run"
192+
fi
193+
194+
echo "=========================================="
195+
echo
196+
fi
197+
198+
# Check for the "DataOut" state - Stop services on non-fenced nodes only
199+
if echo "$LINE" | grep -q "Delaying in state DataOut"; then
200+
echo
201+
echo "=========================================="
202+
echo ">>> 'DataOut' state detected. Managing node services <<<"
203+
echo "=========================================="
204+
205+
if [ ${#NODES_NOT_FENCED[@]} -gt 0 ]; then
206+
echo "[STOP] Stopping services on non-fenced nodes: ${NODES_NOT_FENCED[*]}"
207+
$CLUSH_CMD -w "$NORMAL_NODES_PATTERN" "$STOP_CMD"
208+
else
209+
echo "[INFO] All nodes were fenced, no services to stop"
210+
fi
211+
212+
if [ ${#NODES_TO_FENCE[@]} -gt 0 ]; then
213+
echo "[INFO] Fenced nodes (${NODES_TO_FENCE[*]}) should be rebooting/recovering"
214+
echo "[CHECK] Checking if fenced nodes are back online..."
215+
for node in "${NODES_TO_FENCE[@]}"; do
216+
if ping -c 1 -W 2 "$node" &>/dev/null; then
217+
echo " ✓ $node is responding to ping"
218+
else
219+
echo " ✗ $node is still offline"
220+
fi
221+
done
222+
fi
223+
224+
echo "=========================================="
225+
echo
226+
fi
227+
done < <(make gfs2_fence 2>&1)
228+
229+
echo
230+
echo "=========================================="
231+
echo "Test finished."
232+
echo "=========================================="
233+
echo "Fenced nodes: ${NODES_TO_FENCE[*]:-none}"
234+
echo "Normally managed nodes: ${NODES_NOT_FENCED[*]:-none}"
235+
echo "=========================================="

‎scripts/teardown.sh‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
#!/usr/bin/env bash
2+
3+
# Copyright 2025 Hewlett Packard Enterprise Development LP
4+
# Other additional copyright holders may be indicated within.
5+
#
6+
# The entirety of this work is licensed under the Apache License,
7+
# Version 2.0 (the "License"); you may not use this file except
8+
# in compliance with the License.
9+
#
10+
# You may obtain a copy of the License at
11+
#
12+
# http://www.apache.org/licenses/LICENSE-2.0
13+
#
14+
# Unless required by applicable law or agreed to in writing, software
15+
# distributed under the License is distributed on an "AS IS" BASIS,
16+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
# See the License for the specific language governing permissions and
18+
# limitations under the License.
19+
20+
set -euo pipefail
21+
NS=default
22+
echo "Patching all workflows in namespace $NS to Teardown..."
23+
for wf in $(kubectl -n "$NS" get workflows -o jsonpath='{.items[*].metadata.name}'); do
24+
cur=$(kubectl -n "$NS" get workflow "$wf" -o jsonpath='{.spec.desiredState}')
25+
if [ "$cur" = "Teardown" ]; then
26+
echo "[=] $wf already Teardown"
27+
continue
28+
fi
29+
echo "[>] $wf -> Teardown"
30+
kubectl -n "$NS" patch workflow "$wf" --type=merge -p '{"spec":{"desiredState":"Teardown"}}'
31+
done
32+
33+
echo "Waiting for all to report status.state=Teardown (up to 10m each)..."
34+
for wf in $(kubectl -n "$NS" get workflows -o jsonpath='{.items[*].metadata.name}'); do
35+
kubectl -n "$NS" wait --for=jsonpath='{.status.state}'=Teardown workflow/"$wf" --timeout=10m || echo "[!] Timeout $wf"
36+
done
37+
38+
echo "Waiting for all workflows to be ready (up to 5m each)..."
39+
for wf in $(kubectl -n "$NS" get workflows -o jsonpath='{.items[*].metadata.name}'); do
40+
echo "[.] Waiting for $wf to be ready..."
41+
kubectl -n "$NS" wait --for=jsonpath='{.status.ready}'=true workflow/"$wf" --timeout=5m || echo "[!] Timeout waiting for ready: $wf"
42+
done
43+
44+
echo "Deleting all workflows..."
45+
for wf in $(kubectl -n "$NS" get workflows -o jsonpath='{.items[*].metadata.name}'); do
46+
echo "[-] Deleting $wf"
47+
kubectl -n "$NS" delete workflow "$wf" --wait=false
48+
done
49+
50+
echo "Waiting for all workflows to be deleted..."
51+
kubectl -n "$NS" wait --for=delete workflows --all --timeout=5m 2>/dev/null || echo "[!] Some workflows may still be deleting"
52+
53+
echo "Teardown complete."

0 commit comments

Comments
 (0)