Enable cleanup for test operator

This patch is meant to allow independent cleanup of resources created by the test operator.

- Use files to manage CRs
- Add cleanup tasks to remove CRs and resources created by test operator
- Reduce redundant evaluations in test_operator role

Author: eshulman2

clean_openstack_deployment.yaml

@@ -1,6 +1,11 @@
 - name: Clean OpenStack deployment
   hosts: "{{ target_host | default('localhost') }}"
   tasks:
+    - name: Clean up testing resources
+      ansible.builtin.include_role:
+        name: test_operator
+        tasks_from: cleanup
+
     - name: Clean up OpenStack operators
       vars:
         cifmw_kustomize_deploy_keep_generated_crs: false

roles/test_operator/README.md

@@ -13,6 +13,24 @@ Execute tests via the [test-operator](https://openstack-k8s-operators.github.io/
 * `cifmw_test_operator_concurrency`: (Integer) Tempest concurrency value. NOTE: This parameter is deprecated, please use `cifmw_test_operator_tempest_concurrency` instead. Default value: `8`
 * `cifmw_test_operator_cleanup`: (Bool) Delete all resources created by the role at the end of the testing. Default value: `false`
 * `cifmw_test_operator_tempest_cleanup`: (Bool) Run tempest cleanup after test execution (tempest run) to delete any resources created by tempest that may have been left out.
+* `cifmw_test_operator_crs_path`: (String) The path into which the tests CRs file will be created in. Default value: `{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/artifacts/test-operator-crs`
+* `cifmw_test_operator_log_pod_definition`: (Object) The CR definition template for creating the test log pod. Default value:
+```
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_instance_name }}"
+    namespace: "{{ cifmw_test_operator_namespace }}"
+  spec:
+    containers:
+      - name: test-operator-logs-container
+        image: "{{ cifmw_test_operator_logs_image }}"
+        command: ["sleep"]
+        args: ["infinity"]
+        volumeMounts: "{{ volume_mounts }}"
+    volumes: "{{ volumes }}"
+    tolerations: "{{ cifmw_test_operator_tolerations | default(omit) }}"
+```
 * `cifmw_test_operator_default_groups`: (List) List of groups in the include list to search for tests to be executed. Default value: `[ 'default' ]`
 * `cifmw_test_operator_default_jobs`: (List) List of jobs in the exclude list to search for tests to be excluded. Default value: `[ 'default' ]`
 * `cifmw_test_operator_dry_run`: (Boolean) Whether test-operator should run or not. Default value: `false`

roles/test_operator/defaults/main.yml

@@ -42,6 +42,22 @@ cifmw_test_operator_storage_class: "{{ cifmw_test_operator_storage_class_prefix
 cifmw_test_operator_delete_logs_pod: false
 cifmw_test_operator_privileged: true
 cifmw_test_operator_selinux_level: "s0:c478,c978"
+cifmw_test_operator_crs_path: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/artifacts/test-operator-crs"
+cifmw_test_operator_log_pod_definition:
+  apiVersion: v1
+  kind: Pod
+  metadata:
+    name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_instance_name }}"
+    namespace: "{{ cifmw_test_operator_namespace }}"
+  spec:
+    containers:
+      - name: test-operator-logs-container
+        image: "{{ cifmw_test_operator_logs_image }}"
+        command: ["sleep"]
+        args: ["infinity"]
+        volumeMounts: "{{ volume_mounts }}"
+    volumes: "{{ volumes }}"
+    tolerations: "{{ cifmw_test_operator_tolerations | default(omit) }}"
 # default test framework registry, namespace and tag can be overridden per test framework (tempest, tobiko, horizontest and ansibletest)
 cifmw_test_operator_default_registry: quay.io
 cifmw_test_operator_default_namespace: podified-antelope-centos9

roles/test_operator/tasks/cleanup-run.yaml

@@ -0,0 +1,35 @@
+- name: Delete {{ run_test_fw }}
+  kubernetes.core.k8s:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit)}}"
+    context: "{{ cifmw_openshift_context | default(omit)}}"
+    state: absent
+    src: "{{ cifmw_test_operator_crs_path }}/{{ test_operator_instance_name }}.yaml"
+    wait: true
+    wait_timeout: 600
+
+- name: Delete CRD for {{ run_test_fw }}
+  kubernetes.core.k8s:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit)}}"
+    context: "{{ cifmw_openshift_context | default(omit)}}"
+    kind: CustomResourceDefinition
+    state: absent
+    api_version: v1
+    name: "{{ test_operator_crd_name }}"
+    namespace: "{{ cifmw_test_operator_namespace }}"
+    wait: true
+    wait_timeout: 600
+
+- name: Delete test-operator-logs-pod
+  kubernetes.core.k8s:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit)}}"
+    context: "{{ cifmw_openshift_context | default(omit)}}"
+    state: absent
+    api_version: v1
+    src: "{{ cifmw_test_operator_crs_path }}/{{ test_operator_instance_name }}-log-pod.yaml"
+    wait: true
+    wait_timeout: 600
+  when:
+    - cifmw_test_operator_delete_logs_pod | bool or cifmw_test_operator_cleanup | bool

roles/test_operator/tasks/cleanup.yaml

@@ -0,0 +1,24 @@
+---
+- name: List all CR files in the test operator CRs path
+  ansible.builtin.find:
+    paths: "{{ cifmw_test_operator_crs_path }}"
+    patterns: "*.yaml"
+  register: test_operator_cr_files
+
+- name: Delete all CRs in OCP
+  kubernetes.core.k8s:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit)}}"
+    context: "{{ cifmw_openshift_context | default(omit)}}"
+    state: absent
+    src: "{{ item.path }}"
+    wait: true
+    wait_timeout: 600
+  loop: "{{ test_operator_cr_files.files }}"
+  failed_when: false
+
+- name: Delete test operator CRs files
+  ansible.builtin.file:
+    path: "{{ item.path }}"
+    state: absent
+  loop: "{{ test_operator_cr_files.files }}"

roles/test_operator/tasks/collect-logs.yaml

@@ -0,0 +1,81 @@
+- name: Reset volumes and volume_mounts to an empty list
+  ansible.builtin.set_fact:
+    volumes: []
+    volume_mounts: []
+
+- name: Get information about PVCs that store the logs
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit)}}"
+    context: "{{ cifmw_openshift_context | default(omit)}}"
+    namespace: "{{ cifmw_test_operator_namespace }}"
+    kind: PersistentVolumeClaim
+    label_selectors:
+      - "instanceName={{ test_operator_instance_name }}"
+  register: logsPVCs
+
+- name: Set up volume mounts and volumes for all PVCs
+  ansible.builtin.set_fact:
+    volume_mounts: >
+      {{
+        (volume_mounts | default([])) + [{
+          'name': "logs-volume-" ~ index,
+          'mountPath': "/mnt/logs-{{ test_operator_instance_name }}-step-" ~ index
+        }]
+      }}
+    volumes: >
+      {{
+        (volumes | default([])) + [{
+          'name': "logs-volume-" ~ index,
+          'persistentVolumeClaim': {
+            'claimName': pvc.metadata.name
+          }
+        }]
+      }}
+  loop: "{{ logsPVCs.resources }}"
+  loop_control:
+    loop_var: pvc
+    index_var: index
+
+- name: Write log pod definition to file
+  ansible.builtin.copy:
+    content: "{{ cifmw_test_operator_log_pod_definition }}"
+    dest: "{{ cifmw_test_operator_crs_path }}/{{ test_operator_instance_name }}-log-pod.yaml"
+    mode: '0644'
+
+- name: Start test-operator-logs-pod
+  kubernetes.core.k8s:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit)}}"
+    context: "{{ cifmw_openshift_context | default(omit)}}"
+    state: present
+    wait: true
+    src: "{{ cifmw_test_operator_crs_path }}/{{ test_operator_instance_name }}-log-pod.yaml"
+
+- name: Ensure that the test-operator-logs-pod is Running
+  kubernetes.core.k8s_info:
+    kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
+    api_key: "{{ cifmw_openshift_token | default(omit) }}"
+    context: "{{ cifmw_openshift_context | default(omit) }}"
+    namespace: "{{ cifmw_test_operator_namespace }}"
+    kind: Pod
+    name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_instance_name }}"
+    wait: true
+  register: logs_pod
+  until: logs_pod.resources[0].status.phase == "Running"
+  delay: 10
+  retries: 20
+
+- name: Get logs from test-operator-logs-pod
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path }}"
+  vars:
+    pod_path: mnt/logs-{{ test_operator_instance_name }}-step-{{ index }}
+  ansible.builtin.shell: >
+    oc cp -n {{ cifmw_test_operator_namespace }}
+    test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_instance_name }}:{{ pod_path }}
+    {{ cifmw_test_operator_artifacts_basedir }}
+  loop: "{{ logsPVCs.resources }}"
+  loop_control:
+    index_var: index