[cleanup_openstack] Fix namespace stuck on cleanup by separating deletion phases

Split CRs deletion into two phases to prevent namespace getting stuck
during cleanup:
- Phase 1: Delete application resources (DataPlane, ControlPlane, BMH, etc.)
- Phase 2: Delete operators (NMState, MetalLB, OpenStack operators, OLM)
- Wait between phases to ensure Phase 1 resources are fully deleted

Each resource type (DataPlaneDeployment, DataPlaneNodeSet, ControlPlane,
RabbitmqCluster) is waited on individually in dependency order. If a
resource gets stuck, its finalizers are force-removed before moving to
the next type. This replaces the previous RabbitmqCluster-specific
workaround with a generic mechanism.

This prevents race condition where operator controller is deleted before
it can process finalizers from application resources, which causes the
namespace to hang indefinitely.

Signed-off-by: Miguel Angel Nieto Jimenez <mnietoji@redhat.com>
Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: 2 people

roles/cleanup_openstack/tasks/main.yaml

@@ -51,34 +51,7 @@
   ansible.builtin.import_tasks: detach_bmh.yaml
   when: cifmw_cleanup_openstack_detach_bmh
 
-- name: Work around for stale rabbit cluster deletion
-  block:
-    - name: Check if rabbitmq-cell1 exists
-      kubernetes.core.k8s_info:
-        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
-        api_key: "{{ cifmw_openshift_token | default(omit) }}"
-        context: "{{ cifmw_openshift_context | default(omit) }}"
-        namespace: "{{ cifmw_openstack_namespace }}"
-        kind: RabbitmqCluster
-        name: rabbitmq-cell1
-      register: rabbitmq_cell1_info
-      failed_when: false
-
-    - name: Patch rabbitmq-cell1 finalizers
-      kubernetes.core.k8s_json_patch:
-        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
-        api_key: "{{ cifmw_openshift_token | default(omit) }}"
-        context: "{{ cifmw_openshift_context | default(omit) }}"
-        namespace: "{{ cifmw_openstack_namespace }}"
-        kind: RabbitmqCluster
-        name: rabbitmq-cell1
-        patch:
-          - op: replace
-            path: /metadata/finalizers
-            value: []
-      when: rabbitmq_cell1_info.resources | default([]) | length > 0
-
-- name: Delete deployment CRs
+- name: Delete deployment CRs (Phase 1 - Application Resources)
   vars:
     _stages_crs: >-
       {{
@@ -99,11 +72,6 @@
     _external_dns_crs:
       - "{{ cifmw_manifests_dir }}/cifmw_external_dns/ceph-local-dns.yml"
       - "{{ cifmw_manifests_dir }}/cifmw_external_dns/ceph-local-cert.yml"
-    _operators_crs:
-      - "{{ cifmw_kustomize_deploy_nmstate_dest_file }}"
-      - "{{ cifmw_kustomize_deploy_metallb_dest_file }}"
-      - "{{ cifmw_kustomize_deploy_kustomizations_dest_dir }}/openstack.yaml"
-      - "{{ cifmw_kustomize_deploy_olm_dest_file }}"
     _bmh_crs: >-
       {{
         bmh_crs.files |
@@ -121,11 +89,37 @@
         _external_dns_crs +
         _stages_crs_path +
         _bmh_crs +
-        _bmh_secrets_crs +
-        _operators_crs
+        _bmh_secrets_crs
       }}
   ansible.builtin.import_tasks: cleanup_crs.yaml
 
+- name: Ensure Phase 1 resources are removed before deleting operators
+  ansible.builtin.include_tasks: wait_and_cleanup_resource.yaml
+  loop:
+    - kind: OpenStackDataPlaneDeployment
+      api_version: dataplane.openstack.org/v1beta1
+    - kind: OpenStackDataPlaneNodeSet
+      api_version: dataplane.openstack.org/v1beta1
+    - kind: OpenStackControlPlane
+      api_version: core.openstack.org/v1beta1
+    - kind: RabbitmqCluster
+      api_version: rabbitmq.com/v1beta1
+  loop_control:
+    loop_var: cifmw_cleanup_resource
+  vars:
+    cifmw_cleanup_resource_kind: "{{ cifmw_cleanup_resource.kind }}"
+    cifmw_cleanup_resource_api_version: "{{ cifmw_cleanup_resource.api_version }}"
+
+- name: Delete deployment CRs (Phase 2 - Operators)
+  vars:
+    _operators_crs:
+      - "{{ cifmw_kustomize_deploy_nmstate_dest_file }}"
+      - "{{ cifmw_kustomize_deploy_metallb_dest_file }}"
+      - "{{ cifmw_kustomize_deploy_kustomizations_dest_dir }}/openstack.yaml"
+      - "{{ cifmw_kustomize_deploy_olm_dest_file }}"
+    _crs_to_delete: "{{ _operators_crs }}"
+  ansible.builtin.import_tasks: cleanup_crs.yaml
+
 - name: Get artifacts scripts
   ansible.builtin.find:
     path: "{{ cifmw_kustomize_deploy_basedir }}/artifacts"

roles/cleanup_openstack/tasks/wait_and_cleanup_resource.yaml

@@ -0,0 +1,33 @@
+---
+- name: "Ensure resources are removed - {{ cifmw_cleanup_resource_kind }}"
+  block:
+    - name: "Wait for resource to be deleted - {{ cifmw_cleanup_resource_kind }}"
+      kubernetes.core.k8s_info:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        api_key: "{{ cifmw_openshift_token | default(omit) }}"
+        context: "{{ cifmw_openshift_context | default(omit) }}"
+        namespace: "{{ cifmw_openstack_namespace }}"
+        kind: "{{ cifmw_cleanup_resource_kind }}"
+        api_version: "{{ cifmw_cleanup_resource_api_version }}"
+      register: cifmw_cleanup_resource_result
+      until: cifmw_cleanup_resource_result.resources | default([]) | length == 0
+      retries: 60
+      delay: 10
+  rescue:
+    - name: "Force-remove finalizers from stuck resource - {{ cifmw_cleanup_resource_kind }}"
+      kubernetes.core.k8s_json_patch:
+        kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+        api_key: "{{ cifmw_openshift_token | default(omit) }}"
+        context: "{{ cifmw_openshift_context | default(omit) }}"
+        namespace: "{{ cifmw_openstack_namespace }}"
+        kind: "{{ cifmw_cleanup_resource_kind }}"
+        api_version: "{{ cifmw_cleanup_resource_api_version }}"
+        name: "{{ item.metadata.name }}"
+        patch:
+          - op: replace
+            path: /metadata/finalizers
+            value: []
+      loop: "{{ cifmw_cleanup_resource_result.resources | default([]) }}"
+      loop_control:
+        label: "{{ item.metadata.name }}"
+      failed_when: false