[cifmw_backup_restore] Fix post-restore validation and cleanup

- Wait for compute services and network agents to be ready with
  retry loops before proceeding to workload validation, preventing
  tempest from running against a partially recovered control plane
- Delete test-operator CRs (Tempest, Tobiko, AnsibleTest, HorizonTest)
  at the beginning of cleanup while controllers and dependencies are
  still running, so finalizers get processed properly
- Wait for test-operator pods to terminate after CR deletion
- Adapt GaleraRestore pod discovery to the shortened resource names
  from mariadb-operator which drops the galera instance name prefix
  from generated resources (restore-<name> instead of
  <galera>-restore-<name>). Uses the galerarestore/name label selector
  when available, with fallback to the old naming convention so this
  change can land independently of the mariadb-operator PR
- Increase control plane ready timeout from 10m to 30m
- Fix loop_var collision with _delete_all_of_kind.yml

Related-To: openstack-k8s-operators/mariadb-operator#463

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Signed-off-by: Martin Schuppert <mschuppert@redhat.com>

Author: 2 people

roles/cifmw_backup_restore/defaults/main.yml

@@ -66,11 +66,15 @@ cifmw_backup_restore_ovn_db_ready_timeout: 5m
 cifmw_backup_restore_restore_timeout: 900
 cifmw_backup_restore_edpm_deploy_timeout: 40m
 cifmw_backup_restore_infra_ready_timeout: 20m
-cifmw_backup_restore_ctlplane_ready_timeout: 10m
+cifmw_backup_restore_ctlplane_ready_timeout: 30m
 cifmw_backup_restore_strict_restore: true
 cifmw_backup_restore_restore_content: data
 cifmw_backup_restore_pin_pvcs: false
 
+# Post-restore service readiness
+cifmw_backup_restore_service_retry_count: 30
+cifmw_backup_restore_service_retry_delay: 10
+
 # Cleanup
 cifmw_backup_restore_cleanup_ctlplane: true
 cifmw_backup_restore_cleanup_dataplane: true

roles/cifmw_backup_restore/tasks/cleanup.yml

@@ -71,6 +71,38 @@
     msg: "Cleanup cancelled by user"
   when: not (cifmw_backup_restore_auto_ack | bool) and _delete_confirm.user_input != "yes"
 
+# ========================================
+# Test-operator Cleanup (must happen first while controllers and
+# their dependencies are still running, so finalizers get processed)
+# ========================================
+- name: Delete test-operator CRs
+  ansible.builtin.include_tasks: _delete_all_of_kind.yml
+  vars:
+    _resource_api_version: test.openstack.org/v1beta1
+    _resource_kind: "{{ _test_cr_kind }}"
+    _resource_wait: true
+  loop:
+    - Tempest
+    - Tobiko
+    - AnsibleTest
+    - HorizonTest
+  loop_control:
+    loop_var: _test_cr_kind
+  when: cifmw_backup_restore_cleanup_ctlplane | bool
+
+- name: Wait for test-operator pods to terminate
+  kubernetes.core.k8s_info:
+    api_version: v1
+    kind: Pod
+    namespace: "{{ cifmw_backup_restore_namespace }}"
+    label_selectors:
+      - operator=test-operator
+  register: _test_pods
+  until: _test_pods.resources | length == 0
+  retries: 12
+  delay: 5
+  when: cifmw_backup_restore_cleanup_ctlplane | bool
+
 # ========================================
 # DataPlane Cleanup
 # ========================================

roles/cifmw_backup_restore/tasks/e2e.yml

@@ -131,29 +131,51 @@
     _os_exec: >-
       oc exec -t openstackclient -n {{ cifmw_backup_restore_namespace }} --
   block:
-    - name: Verify compute services are up
+    - name: Wait for compute services to be up
       ansible.builtin.shell: |
         set -o pipefail
         {{ _os_exec }} openstack compute service list -f json | \
-          jq -r '.[] | "\(.Binary) \(.Host) \(.State)"'
+          jq -e '[.[] | select(.State != "up")] | length == 0'
       register: _compute_services
       changed_when: false
+      retries: "{{ cifmw_backup_restore_service_retry_count }}"
+      delay: "{{ cifmw_backup_restore_service_retry_delay }}"
+      until: _compute_services.rc == 0
+
+    - name: Display compute services
+      ansible.builtin.shell: |
+        set -o pipefail
+        {{ _os_exec }} openstack compute service list -f json | \
+          jq -r '.[] | "\(.Binary) \(.Host) \(.State)"'
+      register: _compute_services_display
+      changed_when: false
 
     - name: Display compute services
       ansible.builtin.debug:
-        msg: "{{ _compute_services.stdout_lines }}"
+        msg: "{{ _compute_services_display.stdout_lines }}"
 
-    - name: Verify network agents are up
+    - name: Wait for network agents to be alive
       ansible.builtin.shell: |
         set -o pipefail
         {{ _os_exec }} openstack network agent list -f json | \
-          jq -r '.[] | "\(.["Agent Type"]) \(.Host) \(.Alive)"'
+          jq -e '[.[] | select(.Alive != true)] | length == 0'
       register: _network_agents
       changed_when: false
+      retries: "{{ cifmw_backup_restore_service_retry_count }}"
+      delay: "{{ cifmw_backup_restore_service_retry_delay }}"
+      until: _network_agents.rc == 0
+
+    - name: Display network agents
+      ansible.builtin.shell: |
+        set -o pipefail
+        {{ _os_exec }} openstack network agent list -f json | \
+          jq -r '.[] | "\(.["Agent Type"]) \(.Host) \(.Alive)"'
+      register: _network_agents_display
+      changed_when: false
 
     - name: Display network agents
       ansible.builtin.debug:
-        msg: "{{ _network_agents.stdout_lines }}"
+        msg: "{{ _network_agents_display.stdout_lines }}"
 
     - name: Get instance info
       ansible.builtin.shell: |

roles/cifmw_backup_restore/tasks/restore.yml

@@ -325,19 +325,28 @@
 
 - name: Wait for GaleraRestore pods to be ready
   ansible.builtin.shell: |
-    RESTORE_NAME="{{ item }}restore"
-    BACKUP_SOURCE="{{ item }}"
-    POD_NAME="${BACKUP_SOURCE}-restore-${RESTORE_NAME}"
-    oc wait --for=condition=Ready pod/${POD_NAME} -n {{ cifmw_backup_restore_namespace }} --timeout=120s
+    POD_NAME=$(oc get pod \
+      -l galerarestore/name={{ item }} \
+      -n {{ cifmw_backup_restore_namespace }} \
+      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+    if [ -z "${POD_NAME}" ]; then
+      POD_NAME="{{ item }}-restore-{{ item }}"
+    fi
+    oc wait --for=condition=Ready pod/${POD_NAME} \
+      -n {{ cifmw_backup_restore_namespace }} --timeout=120s
   loop: "{{ _galerabackup_list }}"
   changed_when: false
   when: _galerabackup_list | length > 0
 
 - name: Execute database restore for each GaleraRestore
   ansible.builtin.shell: |
-    RESTORE_NAME="{{ item }}restore"
-    BACKUP_SOURCE="{{ item }}"
-    POD_NAME="${BACKUP_SOURCE}-restore-${RESTORE_NAME}"
+    POD_NAME=$(oc get pod \
+      -l galerarestore/name={{ item }} \
+      -n {{ cifmw_backup_restore_namespace }} \
+      -o jsonpath='{.items[0].metadata.name}' 2>/dev/null)
+    if [ -z "${POD_NAME}" ]; then
+      POD_NAME="{{ item }}-restore-{{ item }}"
+    fi
     TIMESTAMP="{{ cifmw_backup_restore_backup_timestamp }}"
     RESTORE_PATTERN="/backup/data/*_${TIMESTAMP}.sql.gz"
     oc exec -n {{ cifmw_backup_restore_namespace }} ${POD_NAME} -- \
@@ -348,7 +357,7 @@
 
 - name: List GaleraRestore CRs kept for validation
   ansible.builtin.debug:
-    msg: "GaleraRestore CR '{{ item }}restore' kept for post-restore validation (cleaned up by cleanup step)"
+    msg: "GaleraRestore CR '{{ item }}' kept for post-restore validation (cleaned up by cleanup step)"
   loop: "{{ _galerabackup_list }}"
   when: _galerabackup_list | length > 0
 

roles/cifmw_backup_restore/templates/06a-galerarestore.yaml.j2

@@ -4,7 +4,7 @@
 apiVersion: mariadb.openstack.org/v1beta1
 kind: GaleraRestore
 metadata:
-  name: {{ backup_name }}restore
+  name: {{ backup_name }}
   namespace: {{ cifmw_backup_restore_namespace }}
 spec:
   backupSource: {{ backup_name }}