ansible test: fix race condition in operator update

test/ansible/.ansible-lint-ignore

@@ -0,0 +1,5 @@
+# This file contains ignores rule violations for ansible-lint
+tasks/cleanup_environment.yaml yaml[line-length] skip
+tasks/install_operator.yaml yaml[line-length] skip
+tasks/prepare_environment.yaml yaml[line-length] skip
+tasks/update_operator.yaml yaml[line-length] skip

test/ansible/README.md

@@ -33,13 +33,13 @@ ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags deploy,operato
 ### Only update operator with pod disruption -> verify galera is restarted correctly
 
 ```bash
-ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update
+ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update,operator
 ```
 
 ### Only update part without pod disruption
 
 ```bash
-ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update -e disruption=false
+ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update,operator -e disruption=false
 ```
 
 ### Full end-to-end run with 1-node Galera

test/ansible/playbooks/mariadb-operator-update.yaml

@@ -40,6 +40,12 @@
       tags:
         - always
 
+    - name: Prepare environment
+      ansible.builtin.include_tasks:
+        file: ../tasks/prepare_environment.yaml
+      tags:
+        - deploy
+
     - name: Cleanup existing mariadb-operator and galera CR
       ansible.builtin.include_tasks:
         file: ../tasks/cleanup_environment.yaml

test/ansible/tasks/deploy_resources.yaml

@@ -1,14 +1,8 @@
 ---
-- name: Deploy mariadb-operator with initial image
-  community.general.make:
-    chdir: "{{ install_yamls_dir }}"
-    target: mariadb
-  environment:
-    MARIADB_IMG: "{{ mariadb_operator_image_initial }}"
-    OPERATOR_NAMESPACE: "{{ operator_namespace }}"
-    NAMESPACE: "{{ cr_namespace }}"
-    OUT: "{{ tmp_dir }}"
-    TIMEOUT: "{{ deployment_timeout }}"
+- name: Install a default mariadb-operator
+  ansible.builtin.include_tasks: install_operator.yaml
+  vars:
+    mariadb_operator_index: "{{ mariadb_operator_image_initial }}"
   tags:
     - operator
 

test/ansible/tasks/install_operator.yaml

@@ -0,0 +1,66 @@
+---
+- name: Clean up existing mariadb-operator
+  community.general.make:
+    chdir: "{{ install_yamls_dir }}"
+    target: mariadb_cleanup
+  environment:
+    MARIADB_IMG: "{{ mariadb_operator_index }}"
+    OPERATOR_NAMESPACE: "{{ operator_namespace }}"
+    NAMESPACE: "{{ cr_namespace }}"
+    OUT: "{{ tmp_dir }}"
+    TIMEOUT: "{{ deployment_timeout }}"
+  tags:
+    - operator
+
+- name: Wait until the catalogsource associated with the old mariadb-operator is deleted
+  ansible.builtin.shell: >
+    oc -n {{ operator_namespace }} wait --for=delete --timeout={{ deployment_timeout }}
+    catalogsource/mariadb-operator-index
+  changed_when: false
+  tags:
+    - operator
+
+- name: Remove the leftover install plans associated with the old mariadb-operator
+  ansible.builtin.shell: |
+    set -o pipefail
+    oc -n {{ operator_namespace }} get installplans -o name | xargs -r oc -n {{ operator_namespace }} delete
+  changed_when: false
+  tags:
+    - operator
+
+- name: Prepare the resource required to install a new mariadb-operator
+  community.general.make:
+    chdir: "{{ install_yamls_dir }}"
+    target: mariadb_prep
+  environment:
+    MARIADB_IMG: "{{ mariadb_operator_index }}"
+    OPERATOR_NAMESPACE: "{{ operator_namespace }}"
+    NAMESPACE: "{{ cr_namespace }}"
+    OUT: "{{ tmp_dir }}"
+    TIMEOUT: "{{ deployment_timeout }}"
+  tags:
+    - operator
+
+- name: Install the catalogsource for the new mariadb-operator
+  # Apply CatalogSource first and wait for OLM to have fetch the bundle information, otherwise
+  # the old bundle info might be reused when creating the subscription CR
+  # Note: several mariadb-operator-index pods are created, but only one gets ready, so wait accordingly
+  ansible.builtin.shell: |
+    set -e -o pipefail
+    oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/catalogsource.yaml
+    until oc -n {{ operator_namespace }} get pods -l olm.catalogSource=mariadb-operator-index -o jsonpath='{range .items[*]}{.metadata.name} {.status.containerStatuses[*].ready}{end}' | grep -w -m1 true; do sleep 2; done
+    index_pod=$(oc -n {{ operator_namespace }} get pod -l olm.catalogSource=mariadb-operator-index -o name)
+    test -n "${index_pod}"
+    timeout {{ deployment_timeout | community.general.to_seconds | int }} bash -c "oc -n {{ operator_namespace }} logs '${index_pod}' -f | grep -m1 'grpc.code=OK.*GetBundleForChannel'"
+  changed_when: false
+  tags:
+    - operator
+
+- name: Apply subscriptions and operatorgroup to deploy the new mariadb-operator
+  ansible.builtin.shell: |
+    set -e
+    oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/operatorgroup.yaml
+    oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/subscription.yaml
+  changed_when: false
+  tags:
+    - operator

test/ansible/tasks/prepare_environment.yaml

@@ -0,0 +1,13 @@
+---
+- name: Disable openstack-operator-controller-init to allow custom mariadb-operator image
+  ansible.builtin.shell: |
+    set -o pipefail
+    if ! oc -n {{ operator_namespace }} get deployment/openstack-operator-controller-init &>/dev/null; then exit 0; fi
+    SAVED_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r '.metadata.labels["test.mariadb-operator/saved-replicas"]')
+    if [ -n "$SAVED_REPLICAS" ] && [ "$SAVED_REPLICAS" != "null" ]; then exit 0; fi
+    CURRENT_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r .spec.replicas)
+    oc -n {{ operator_namespace }} label deployment/openstack-operator-controller-init test.mariadb-operator/saved-replicas="$CURRENT_REPLICAS"
+    oc -n {{ operator_namespace }} patch deployment/openstack-operator-controller-init --type='json' -p='[{"op":"replace","path":"/spec/replicas","value":0}]'
+  changed_when: false
+  tags:
+    - always

test/ansible/tasks/update_operator.yaml

@@ -1,44 +1,8 @@
 ---
-- name: Uninstall the old mariadb-operator
-  community.general.make:
-    chdir: "{{ install_yamls_dir }}"
-    target: mariadb_cleanup
-  environment:
-    OPERATOR_NAMESPACE: "{{ operator_namespace }}"
-    NAMESPACE: "{{ cr_namespace }}"
-    TIMEOUT: "{{ update_timeout }}"
-  tags:
-    - update
-
-- name: Wait for the mariadb-operator to be uninstalled
-  ansible.builtin.shell: >
-    set -o pipefail && echo $(oc -n {{ operator_namespace }} get csv -l operators.coreos.com/mariadb-operator.openstack-operators -o name)
-    catalogsource/mariadb-operator-index
-    subscription/mariadb-operator
-    | xargs oc -n {{ operator_namespace }} wait --for=delete --timeout={{ update_timeout }}
-  changed_when: false
-  tags:
-    - update
-
-- name: Disable openstack-operator-controller-init to allow custom mariadb-operator image
-  ansible.builtin.shell: |
-    set -o pipefail
-    PREV_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r .spec.replicas)
-    oc -n {{ operator_namespace }} label deployment/openstack-operator-controller-init test.mariadb-operator/saved-replicas="$PREV_REPLICAS"
-    oc -n {{ operator_namespace }} patch deployment/openstack-operator-controller-init --type='json' -p='[{"op":"replace","path":"/spec/replicas","value":0}]'
-  changed_when: false
-  tags:
-    - update
-
-- name: Update mariadb-operator to new image
-  community.general.make:
-    chdir: "{{ install_yamls_dir }}"
-    target: mariadb
-  environment:
-    MARIADB_IMG: "{{ mariadb_operator_image_updated }}"
-    OPERATOR_NAMESPACE: "{{ operator_namespace }}"
-    NAMESPACE: "{{ cr_namespace }}"
-    TIMEOUT: "{{ deployment_timeout }}"
+- name: Install the updated mariadb-operator
+  ansible.builtin.include_tasks: install_operator.yaml
+  vars:
+    mariadb_operator_index: "{{ mariadb_operator_image_updated }}"
   tags:
     - update
 

test/ansible/vars/mariadb-operator-vars.yaml

@@ -16,7 +16,7 @@ cr_namespace: "openstack"
 operator_namespace: "openstack-operators"
 
 # Galera cluster configuration
-galera_cr:  "{{ tmp_dir }}/operator/mariadb-operator/config/samples/mariadb_v1beta1_galera.yaml"
+galera_cr: "{{ tmp_dir }}/operator/mariadb-operator/config/samples/mariadb_v1beta1_galera.yaml"
 replicas: 3
 db_service: "galera"