Skip to content

Commit c5d09b6

Browse files
eggfoobareggfoobar
committed
upkeep: rename and validate flow for sno to 3node
Signed-off-by: ehila <ehila@redhat.com>
1 parent cede990 commit c5d09b6

21 files changed

Lines changed: 278 additions & 30 deletions

‎deploy/Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ full-clean:
5252
clean-spoke:
5353
@./openshift-clusters/scripts/clean-spoke.sh
5454

55+
clean-mutable-topology:
56+
@./openshift-clusters/scripts/clean-mutable-topology.sh
57+
5558
ssh:
5659
@./aws-hypervisor/scripts/ssh.sh
5760

@@ -140,6 +143,7 @@ help:
140143
@echo " clean - Clean OpenShift cluster using dev-scripts clean target"
141144
@echo " full-clean - Fully clean instance cache and OpenShift cluster using dev-scripts realclean target"
142145
@echo " clean-spoke - Clean spoke cluster resources (VMs, network, auth) from assisted installer"
146+
@echo " clean-mutable-topology - Remove master-1/2 VMs, disks, DHCP/DNS entries (sno-to-3node cleanup)"
143147
@echo " patch-nodes - Build resource-agents RPM and patch cluster nodes (default version: 4.11)"
144148
@echo ""
145149
@echo "Cluster Utilities:"

‎deploy/openshift-clusters/clean-mutable-topology.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
---
2+
- hosts: metal_machine
3+
gather_facts: yes
4+
5+
pre_tasks:
6+
- name: Confirm mutable topology cleanup
7+
ansible.builtin.pause:
8+
prompt: >-
9+
This will destroy the master-1 and master-2 VMs, their disks, DHCP reservations,
10+
and DNS entries. master-0 (if still present) will be unaffected.
11+
Press Enter to proceed or Ctrl+C to abort.
12+
delegate_to: localhost
13+
run_once: true
14+
when: interactive_mode | default(true) | bool
15+
16+
- name: Detect cluster domain from kubeconfig (best-effort)
17+
shell: |
18+
KUBECONFIG={{ dev_scripts_path | default('openshift-metal3/dev-scripts') }}/ocp/{{ sno_cluster_name | default('ostest') }}/auth/kubeconfig \
19+
oc get infrastructure cluster -o jsonpath='{.status.apiServerInternalURI}' 2>/dev/null \
20+
| sed 's|^https://api-int\.||; s|:6443$||'
21+
register: detected_domain
22+
changed_when: false
23+
failed_when: false
24+
ignore_errors: true
25+
26+
- name: Set cluster domain (use detected, variable override, or default)
27+
set_fact:
28+
sno_cluster_domain: >-
29+
{{ sno_cluster_domain
30+
if (sno_cluster_domain is defined and sno_cluster_domain)
31+
else (detected_domain.stdout | trim
32+
if (detected_domain.stdout is defined and detected_domain.stdout | trim)
33+
else 'ostest.test.metalkube.org') }}
34+
35+
tasks:
36+
- name: Run mutable topology cleanup
37+
import_role:
38+
name: mutable-topology/sno-to-3node
39+
tasks_from: clean.yml
40+
41+
- name: Cleanup complete
42+
ansible.builtin.debug:
43+
msg: "Mutable topology cleanup complete. master-1 and master-2 VMs have been removed."

deploy/openshift-clusters/roles/sno-expand/sno-to-3node/defaults/main.yml renamed to deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/defaults/main.yml

File renamed without changes.

deploy/openshift-clusters/roles/sno-expand/sno-to-3node/tasks/boot-nodes.yml renamed to deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/boot-nodes.yml

Lines changed: 45 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -67,28 +67,65 @@
6767
--os-variant rhel9.0 \
6868
--graphics none \
6969
--noautoconsole \
70-
--boot hd,cdrom
70+
--boot loader=/usr/share/edk2/ovmf/OVMF_CODE.fd,loader_ro=yes,loader_type=pflash,nvram_template=/usr/share/edk2/ovmf/OVMF_VARS.fd,loader_secure=no \
71+
--boot hd,cdrom \
72+
--tpm none
7173
loop: "{{ sno_new_nodes }}"
7274

73-
- name: "[boot] Verify VMs are running"
75+
- name: "[boot] Verify VMs are running (initial install boot)"
7476
shell: |
7577
sudo virsh domstate {{ item.name }}
7678
register: vm_state
7779
loop: "{{ sno_new_nodes }}"
7880
changed_when: false
7981
failed_when: "'running' not in vm_state.stdout"
8082

81-
- name: "[boot] Set on_reboot to restart (not destroy)"
83+
- name: "[boot] Wait for coreos-installer to complete (VM will power off)"
84+
# coreos-installer runs ExecStartPost=systemctl reboot, but RHCOS live issues
85+
# an ACPI poweroff rather than a reset. libvirt fires on_poweroff=destroy so
86+
# the VM shuts off. We poll until shut off, then boot from disk below.
8287
shell: |
83-
sudo virsh dumpxml {{ item.name }} | grep -q '<on_reboot>destroy</on_reboot>' && \
84-
sudo virt-xml {{ item.name }} --edit --events on_reboot=restart 2>/dev/null || true
88+
for i in $(seq 50); do
89+
STATE=$(sudo virsh domstate {{ item.name }} 2>/dev/null || echo "unknown")
90+
if echo "$STATE" | grep -q "shut off"; then
91+
echo "{{ item.name }} shut off after $((i * 20))s - install complete"
92+
exit 0
93+
fi
94+
sleep 20
95+
done
96+
echo "Timeout: {{ item.name }} did not shut off within 1000s"
97+
exit 1
98+
loop: "{{ sno_new_nodes }}"
99+
changed_when: false
100+
101+
- name: "[boot] Remove CDROM from boot order after install"
102+
# Prevent coreos-installer loop: strip the cdrom boot entry so UEFI only
103+
# tries the hard disk on subsequent boots.
104+
shell: |
105+
TMPXML=$(mktemp /tmp/vm-XXXXXX.xml)
106+
sudo virsh dumpxml {{ item.name }} > "$TMPXML"
107+
sudo sed -i "/<boot dev='cdrom'\/>/d" "$TMPXML"
108+
sudo virsh define "$TMPXML"
109+
sudo rm -f "$TMPXML"
110+
loop: "{{ sno_new_nodes }}"
111+
changed_when: true
112+
113+
- name: "[boot] Start VMs to boot from installed RHCOS"
114+
shell: |
115+
sudo virsh start {{ item.name }}
85116
loop: "{{ sno_new_nodes }}"
86-
failed_when: false
87117
changed_when: true
88118

119+
- name: "[boot] Verify VMs are running from installed disk"
120+
shell: |
121+
sudo virsh domstate {{ item.name }}
122+
register: vm_state_disk
123+
loop: "{{ sno_new_nodes }}"
124+
changed_when: false
125+
failed_when: "'running' not in vm_state_disk.stdout"
126+
89127
- name: "[boot] VMs booted with RHCOS"
90128
debug:
91129
msg: >-
92-
{{ sno_new_nodes | length }} VMs booted with RHCOS ISO (ignition embedded).
93-
coreos-installer will write to disk and reboot automatically.
130+
{{ sno_new_nodes | length }} VMs installed and started from disk.
94131
Waiting for nodes to join the cluster...

‎deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/clean.yml‎

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
---
2+
- name: "[clean] Stop and undefine new VMs"
3+
shell: |
4+
sudo virsh destroy {{ item.name }} 2>/dev/null || true
5+
sudo virsh undefine {{ item.name }} --remove-all-storage 2>/dev/null || true
6+
loop: "{{ sno_new_nodes }}"
7+
failed_when: false
8+
changed_when: true
9+
10+
- name: "[clean] Remove disk images"
11+
file:
12+
path: "{{ sno_vm_image_dir }}/{{ item.name }}.qcow2"
13+
state: absent
14+
loop: "{{ sno_new_nodes }}"
15+
become: true
16+
failed_when: false
17+
18+
- name: "[clean] Remove NVRAM files"
19+
file:
20+
path: "/var/lib/libvirt/qemu/nvram/{{ item.name }}_VARS.fd"
21+
state: absent
22+
loop: "{{ sno_new_nodes }}"
23+
become: true
24+
failed_when: false
25+
26+
- name: "[clean] Remove per-node RHCOS ISOs"
27+
file:
28+
path: "/var/lib/libvirt/images/rhcos-{{ item.hostname }}.iso"
29+
state: absent
30+
loop: "{{ sno_new_nodes }}"
31+
become: true
32+
failed_when: false
33+
34+
- name: "[clean] Remove DHCP reservations from libvirt network"
35+
shell: |
36+
NET_XML=$(sudo virsh net-dumpxml {{ sno_libvirt_network }} 2>/dev/null || true)
37+
EXISTING_MAC=$(echo "$NET_XML" | python3 -c "
38+
import xml.etree.ElementTree as ET, sys
39+
root = ET.parse(sys.stdin).getroot()
40+
for host in root.findall('.//dhcp/host'):
41+
if host.get('ip') == '{{ item.ip }}':
42+
print(host.get('mac',''))
43+
" 2>/dev/null || true)
44+
45+
if [ -n "$EXISTING_MAC" ]; then
46+
sudo virsh net-update {{ sno_libvirt_network }} delete ip-dhcp-host \
47+
"<host mac='${EXISTING_MAC}' ip='{{ item.ip }}'/>" --live --config 2>/dev/null || true
48+
echo "Removed DHCP reservation for {{ item.ip }} (mac=${EXISTING_MAC})"
49+
else
50+
echo "No DHCP reservation found for {{ item.ip }}"
51+
fi
52+
loop: "{{ sno_new_nodes }}"
53+
failed_when: false
54+
changed_when: true
55+
56+
- name: "[clean] Remove DNS host entries from libvirt network"
57+
shell: |
58+
sudo virsh net-update {{ sno_libvirt_network }} delete dns-host \
59+
"<host ip='{{ item.ip }}'><hostname>{{ item.hostname }}.{{ sno_cluster_domain }}</hostname></host>" \
60+
--live --config 2>/dev/null || true
61+
sudo virsh net-update {{ sno_libvirt_network }} delete dns-host \
62+
"<host ip='{{ item.ip }}'><hostname>{{ item.hostname }}</hostname></host>" \
63+
--live --config 2>/dev/null || true
64+
loop: "{{ sno_new_nodes }}"
65+
failed_when: false
66+
changed_when: true
67+
68+
- name: "[clean] Remove api-int from libvirt network DNS"
69+
shell: |
70+
sudo virsh net-update {{ sno_libvirt_network }} delete dns-host \
71+
"<host ip='{{ sno_master0_ip }}'><hostname>api-int.{{ sno_cluster_domain }}</hostname></host>" \
72+
--live --config 2>/dev/null || true
73+
failed_when: false
74+
changed_when: true
75+
76+
- name: "[clean] Remove entries from addnhosts"
77+
lineinfile:
78+
path: "{{ sno_addnhosts_path }}"
79+
regexp: "^{{ item.ip }}\\s"
80+
state: absent
81+
loop: "{{ sno_new_nodes }}"
82+
become: true
83+
failed_when: false
84+
85+
- name: "[clean] Remove hostname entries from addnhosts"
86+
lineinfile:
87+
path: "{{ sno_addnhosts_path }}"
88+
regexp: "\\s{{ item.hostname }}\\.{{ sno_cluster_domain }}(\\s|$)"
89+
state: absent
90+
loop: "{{ sno_new_nodes }}"
91+
become: true
92+
failed_when: false
93+
94+
- name: "[clean] Flush DHCP leases for new node IPs"
95+
shell: |
96+
python3 -c "
97+
import json, os
98+
lf = '{{ sno_lease_file }}'
99+
if not os.path.exists(lf) or os.path.getsize(lf) == 0:
100+
exit(0)
101+
with open(lf) as f:
102+
leases = json.load(f)
103+
reserved = {{ [sno_master1_ip, sno_master2_ip] | to_json }}
104+
before = len(leases)
105+
leases = [l for l in leases if l.get('ip-address') not in reserved]
106+
with open(lf, 'w') as f:
107+
json.dump(leases, f, indent=2)
108+
print(f'Flushed {before - len(leases)} leases')
109+
"
110+
become: true
111+
failed_when: false
112+
changed_when: true
113+
114+
- name: "[clean] Reload libvirt dnsmasq"
115+
shell: |
116+
sudo kill -HUP $(cat /run/libvirt/network/{{ sno_libvirt_bridge }}.pid) 2>/dev/null || true
117+
changed_when: true
118+
failed_when: false
119+
120+
- name: "[clean] Cleanup complete"
121+
debug:
122+
msg: >-
123+
Removed VMs: {{ sno_new_nodes | map(attribute='name') | list | join(', ') }}.
124+
Disk images, NVRAM, ISOs, DHCP reservations, and DNS entries removed.

deploy/openshift-clusters/roles/sno-expand/sno-to-3node/tasks/create-vms.yml renamed to deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/create-vms.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,9 @@
112112
--os-variant rhel9.0 \
113113
--graphics none \
114114
--noautoconsole \
115+
--boot loader=/usr/share/edk2/ovmf/OVMF_CODE.fd,loader_ro=yes,loader_type=pflash,nvram_template=/usr/share/edk2/ovmf/OVMF_VARS.fd,loader_secure=no \
115116
--boot hd \
117+
--tpm none \
116118
--noreboot \
117119
--import \
118120
--print-xml | sudo virsh define /dev/stdin

deploy/openshift-clusters/roles/sno-expand/sno-to-3node/tasks/ignition.yml renamed to deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/ignition.yml

File renamed without changes.

deploy/openshift-clusters/roles/sno-expand/sno-to-3node/tasks/main.yml renamed to deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/main.yml

File renamed without changes.

deploy/openshift-clusters/roles/sno-expand/sno-to-3node/tasks/mco-rollout.yml renamed to deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/mco-rollout.yml

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -30,32 +30,37 @@
3030
print(conds.get('Degraded','Unknown'))
3131
")
3232
33-
NODE_SCHED=$(oc get nodes --no-headers -o custom-columns=SCHED:.spec.unschedulable 2>/dev/null | head -1)
33+
CORDONED_NODES=$(oc get nodes --no-headers 2>/dev/null | grep SchedulingDisabled | awk '{print $1}' || true)
3434
35-
echo "STATUS=updated:${UPDATED},updating:${UPDATING},degraded:${DEGRADED},cordoned:${NODE_SCHED}"
35+
echo "STATUS=updated:${UPDATED},updating:${UPDATING},degraded:${DEGRADED},cordoned:${CORDONED_NODES:-none}"
3636
3737
if [ "$UPDATED" = "True" ] && [ "$UPDATING" = "False" ] && [ "$DEGRADED" = "False" ]; then
3838
echo "RESULT=done"
3939
exit 0
4040
fi
4141
42-
# Auto-fix drain deadlock if node is cordoned and auto-fix is enabled
43-
if [ "$NODE_SCHED" = "true" ] && [ "{{ sno_auto_fix_drain | default(true) | bool }}" = "True" ]; then
44-
NODE_NAME=$(oc get nodes --no-headers -o custom-columns=NAME:.metadata.name | head -1)
45-
echo "DRAIN_FIX: Node $NODE_NAME is cordoned, applying drain deadlock fix..."
42+
# Auto-fix drain deadlock for any cordoned node.
43+
# On SNO→3node, MCD cordons a node before draining. If MCC cannot reach
44+
# the node (api-int flap, RBAC timing) the drain annotation never advances
45+
# and the node stays cordoned forever. Fix: uncordon, set lastAppliedDrain
46+
# to the drain-<desired> value MCC is waiting for, restart MCC.
47+
if [ -n "$CORDONED_NODES" ] && [ "{{ sno_auto_fix_drain | default(true) | bool }}" = "True" ]; then
48+
DESIRED=$(oc get mcp master -o jsonpath='{.spec.configuration.name}' 2>/dev/null || echo "")
49+
for NODE_NAME in $CORDONED_NODES; do
50+
echo "DRAIN_FIX: Fixing drain deadlock on $NODE_NAME (desired=${DESIRED})..."
4651
47-
# Uncordon
48-
oc adm uncordon "$NODE_NAME" 2>&1 || true
52+
oc adm uncordon "$NODE_NAME" 2>&1 || true
4953
50-
# Set lastAppliedDrain annotation
51-
DESIRED=$(oc get node "$NODE_NAME" -o jsonpath='{.metadata.annotations.machine\.openshift\.io/desired-drain}' 2>/dev/null || echo "")
52-
if [ -n "$DESIRED" ]; then
53-
oc annotate node "$NODE_NAME" "machine.openshift.io/last-applied-drain=$DESIRED" --overwrite 2>&1 || true
54-
echo "DRAIN_FIX: Set lastAppliedDrain=$DESIRED"
55-
fi
54+
if [ -n "$DESIRED" ]; then
55+
oc annotate node "$NODE_NAME" \
56+
"machineconfiguration.openshift.io/lastAppliedDrain=drain-${DESIRED}" \
57+
--overwrite 2>&1 || true
58+
echo "DRAIN_FIX: Set lastAppliedDrain=drain-${DESIRED} on $NODE_NAME"
59+
fi
60+
done
5661
57-
# Restart MCC pod
58-
oc delete pod -n openshift-machine-config-operator -l k8s-app=machine-config-controller 2>&1 || true
62+
oc delete pod -n openshift-machine-config-operator \
63+
-l k8s-app=machine-config-controller 2>&1 || true
5964
echo "DRAIN_FIX: Restarted MCC pod"
6065
fi
6166

deploy/openshift-clusters/roles/sno-expand/sno-to-3node/tasks/preflight.yml renamed to deploy/openshift-clusters/roles/mutable-topology/sno-to-3node/tasks/preflight.yml

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -126,19 +126,31 @@
126126
changed_when: false
127127
failed_when: false
128128

129-
- name: "[preflight] Add api-int to addnhosts if missing"
129+
- name: "[preflight] Add api-int to libvirt network DNS"
130+
# Using virsh net-update (not lineinfile) so that api-int survives addnhosts
131+
# regeneration when create-vms.yml calls virsh net-update for master-1/2.
132+
# libvirt regenerates ostestbm.addnhosts from the network XML on every net-update.
130133
shell: |
131-
echo "{{ sno_master0_ip }} api-int.{{ sno_cluster_domain }} api-int" | \
132-
sudo tee -a {{ sno_addnhosts_path }}
133-
sudo kill -HUP $(cat /run/libvirt/network/{{ sno_libvirt_bridge }}.pid) 2>/dev/null || true
134+
# Remove any stale api-int entry (idempotent, ignore errors)
135+
sudo virsh net-update {{ sno_libvirt_network }} delete dns-host \
136+
"<host ip='{{ sno_master0_ip }}'><hostname>api-int.{{ sno_cluster_domain }}</hostname></host>" \
137+
--live --config 2>/dev/null || true
138+
# Add the correct entry - triggers addnhosts regeneration with api-int included
139+
sudo virsh net-update {{ sno_libvirt_network }} add dns-host \
140+
"<host ip='{{ sno_master0_ip }}'><hostname>api-int.{{ sno_cluster_domain }}</hostname></host>" \
141+
--live --config
134142
when: dns_apiint.stdout | trim == "" or sno_master0_ip not in dns_apiint.stdout
143+
changed_when: true
135144

136145
- name: "[preflight] Verify api-int DNS after fix"
137146
shell: |
138147
dig +short api-int.{{ sno_cluster_domain }} @192.168.111.1
139148
register: dns_apiint_verify
140149
changed_when: false
141150
failed_when: dns_apiint_verify.stdout | trim == ""
151+
retries: 3
152+
delay: 2
153+
until: dns_apiint_verify.stdout | trim != ""
142154
when: dns_apiint.stdout | trim == "" or sno_master0_ip not in dns_apiint.stdout
143155

144156
- name: "[preflight] Preflight checks passed"

0 commit comments

Comments
 (0)