From 5d906c660d9a54339931a0b9b380f7ab6d401ffb Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Fri, 5 Jun 2026 16:43:41 +0200 Subject: [PATCH] ansible test: fix race condition in operator update The usual way of installing a new mariadb-operator is to use install_yamls and `make mariadb`. However this one creates the subscription CR right after the catalog CR has been created. If a previous operator was present, this might prevent OLM to retrieve the bundle data for the new operator, which would cause the previous version of the operator to be reinstalled. Fix that race condition by first installing the catalogsource, and wait until OLM refreshed its cache with the new bundle information, at which point it is safe to create the new subscription CR. --- test/ansible/.ansible-lint-ignore | 5 ++ test/ansible/README.md | 4 +- .../playbooks/mariadb-operator-update.yaml | 6 ++ test/ansible/tasks/deploy_resources.yaml | 14 ++-- test/ansible/tasks/install_operator.yaml | 66 +++++++++++++++++++ test/ansible/tasks/prepare_environment.yaml | 13 ++++ test/ansible/tasks/update_operator.yaml | 44 ++----------- test/ansible/vars/mariadb-operator-vars.yaml | 2 +- 8 files changed, 101 insertions(+), 53 deletions(-) create mode 100644 test/ansible/.ansible-lint-ignore create mode 100644 test/ansible/tasks/install_operator.yaml create mode 100644 test/ansible/tasks/prepare_environment.yaml diff --git a/test/ansible/.ansible-lint-ignore b/test/ansible/.ansible-lint-ignore new file mode 100644 index 00000000..3ab70fa0 --- /dev/null +++ b/test/ansible/.ansible-lint-ignore @@ -0,0 +1,5 @@ +# This file contains ignores rule violations for ansible-lint +tasks/cleanup_environment.yaml yaml[line-length] skip +tasks/install_operator.yaml yaml[line-length] skip +tasks/prepare_environment.yaml yaml[line-length] skip +tasks/update_operator.yaml yaml[line-length] skip diff --git a/test/ansible/README.md b/test/ansible/README.md index 38fe0f90..09b5ca94 100644 --- a/test/ansible/README.md +++ b/test/ansible/README.md @@ -33,13 +33,13 @@ ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags deploy,operato ### Only update operator with pod disruption -> verify galera is restarted correctly ```bash -ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update +ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update,operator ``` ### Only update part without pod disruption ```bash -ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update -e disruption=false +ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update,operator -e disruption=false ``` ### Full end-to-end run with 1-node Galera diff --git a/test/ansible/playbooks/mariadb-operator-update.yaml b/test/ansible/playbooks/mariadb-operator-update.yaml index 84760788..251f6dc6 100644 --- a/test/ansible/playbooks/mariadb-operator-update.yaml +++ b/test/ansible/playbooks/mariadb-operator-update.yaml @@ -40,6 +40,12 @@ tags: - always + - name: Prepare environment + ansible.builtin.include_tasks: + file: ../tasks/prepare_environment.yaml + tags: + - deploy + - name: Cleanup existing mariadb-operator and galera CR ansible.builtin.include_tasks: file: ../tasks/cleanup_environment.yaml diff --git a/test/ansible/tasks/deploy_resources.yaml b/test/ansible/tasks/deploy_resources.yaml index 0eb55412..0e430963 100644 --- a/test/ansible/tasks/deploy_resources.yaml +++ b/test/ansible/tasks/deploy_resources.yaml @@ -1,14 +1,8 @@ --- -- name: Deploy mariadb-operator with initial image - community.general.make: - chdir: "{{ install_yamls_dir }}" - target: mariadb - environment: - MARIADB_IMG: "{{ mariadb_operator_image_initial }}" - OPERATOR_NAMESPACE: "{{ operator_namespace }}" - NAMESPACE: "{{ cr_namespace }}" - OUT: "{{ tmp_dir }}" - TIMEOUT: "{{ deployment_timeout }}" +- name: Install a default mariadb-operator + ansible.builtin.include_tasks: install_operator.yaml + vars: + mariadb_operator_index: "{{ mariadb_operator_image_initial }}" tags: - operator diff --git a/test/ansible/tasks/install_operator.yaml b/test/ansible/tasks/install_operator.yaml new file mode 100644 index 00000000..b758d602 --- /dev/null +++ b/test/ansible/tasks/install_operator.yaml @@ -0,0 +1,66 @@ +--- +- name: Clean up existing mariadb-operator + community.general.make: + chdir: "{{ install_yamls_dir }}" + target: mariadb_cleanup + environment: + MARIADB_IMG: "{{ mariadb_operator_index }}" + OPERATOR_NAMESPACE: "{{ operator_namespace }}" + NAMESPACE: "{{ cr_namespace }}" + OUT: "{{ tmp_dir }}" + TIMEOUT: "{{ deployment_timeout }}" + tags: + - operator + +- name: Wait until the catalogsource associated with the old mariadb-operator is deleted + ansible.builtin.shell: > + oc -n {{ operator_namespace }} wait --for=delete --timeout={{ deployment_timeout }} + catalogsource/mariadb-operator-index + changed_when: false + tags: + - operator + +- name: Remove the leftover install plans associated with the old mariadb-operator + ansible.builtin.shell: | + set -o pipefail + oc -n {{ operator_namespace }} get installplans -o name | xargs -r oc -n {{ operator_namespace }} delete + changed_when: false + tags: + - operator + +- name: Prepare the resource required to install a new mariadb-operator + community.general.make: + chdir: "{{ install_yamls_dir }}" + target: mariadb_prep + environment: + MARIADB_IMG: "{{ mariadb_operator_index }}" + OPERATOR_NAMESPACE: "{{ operator_namespace }}" + NAMESPACE: "{{ cr_namespace }}" + OUT: "{{ tmp_dir }}" + TIMEOUT: "{{ deployment_timeout }}" + tags: + - operator + +- name: Install the catalogsource for the new mariadb-operator + # Apply CatalogSource first and wait for OLM to have fetch the bundle information, otherwise + # the old bundle info might be reused when creating the subscription CR + # Note: several mariadb-operator-index pods are created, but only one gets ready, so wait accordingly + ansible.builtin.shell: | + set -e -o pipefail + oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/catalogsource.yaml + until oc -n {{ operator_namespace }} get pods -l olm.catalogSource=mariadb-operator-index -o jsonpath='{range .items[*]}{.metadata.name} {.status.containerStatuses[*].ready}{end}' | grep -w -m1 true; do sleep 2; done + index_pod=$(oc -n {{ operator_namespace }} get pod -l olm.catalogSource=mariadb-operator-index -o name) + test -n "${index_pod}" + timeout {{ deployment_timeout | community.general.to_seconds | int }} bash -c "oc -n {{ operator_namespace }} logs '${index_pod}' -f | grep -m1 'grpc.code=OK.*GetBundleForChannel'" + changed_when: false + tags: + - operator + +- name: Apply subscriptions and operatorgroup to deploy the new mariadb-operator + ansible.builtin.shell: | + set -e + oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/operatorgroup.yaml + oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/subscription.yaml + changed_when: false + tags: + - operator diff --git a/test/ansible/tasks/prepare_environment.yaml b/test/ansible/tasks/prepare_environment.yaml new file mode 100644 index 00000000..310cf261 --- /dev/null +++ b/test/ansible/tasks/prepare_environment.yaml @@ -0,0 +1,13 @@ +--- +- name: Disable openstack-operator-controller-init to allow custom mariadb-operator image + ansible.builtin.shell: | + set -o pipefail + if ! oc -n {{ operator_namespace }} get deployment/openstack-operator-controller-init &>/dev/null; then exit 0; fi + SAVED_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r '.metadata.labels["test.mariadb-operator/saved-replicas"]') + if [ -n "$SAVED_REPLICAS" ] && [ "$SAVED_REPLICAS" != "null" ]; then exit 0; fi + CURRENT_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r .spec.replicas) + oc -n {{ operator_namespace }} label deployment/openstack-operator-controller-init test.mariadb-operator/saved-replicas="$CURRENT_REPLICAS" + oc -n {{ operator_namespace }} patch deployment/openstack-operator-controller-init --type='json' -p='[{"op":"replace","path":"/spec/replicas","value":0}]' + changed_when: false + tags: + - always diff --git a/test/ansible/tasks/update_operator.yaml b/test/ansible/tasks/update_operator.yaml index 1dd2b859..658702cd 100644 --- a/test/ansible/tasks/update_operator.yaml +++ b/test/ansible/tasks/update_operator.yaml @@ -1,44 +1,8 @@ --- -- name: Uninstall the old mariadb-operator - community.general.make: - chdir: "{{ install_yamls_dir }}" - target: mariadb_cleanup - environment: - OPERATOR_NAMESPACE: "{{ operator_namespace }}" - NAMESPACE: "{{ cr_namespace }}" - TIMEOUT: "{{ update_timeout }}" - tags: - - update - -- name: Wait for the mariadb-operator to be uninstalled - ansible.builtin.shell: > - set -o pipefail && echo $(oc -n {{ operator_namespace }} get csv -l operators.coreos.com/mariadb-operator.openstack-operators -o name) - catalogsource/mariadb-operator-index - subscription/mariadb-operator - | xargs oc -n {{ operator_namespace }} wait --for=delete --timeout={{ update_timeout }} - changed_when: false - tags: - - update - -- name: Disable openstack-operator-controller-init to allow custom mariadb-operator image - ansible.builtin.shell: | - set -o pipefail - PREV_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r .spec.replicas) - oc -n {{ operator_namespace }} label deployment/openstack-operator-controller-init test.mariadb-operator/saved-replicas="$PREV_REPLICAS" - oc -n {{ operator_namespace }} patch deployment/openstack-operator-controller-init --type='json' -p='[{"op":"replace","path":"/spec/replicas","value":0}]' - changed_when: false - tags: - - update - -- name: Update mariadb-operator to new image - community.general.make: - chdir: "{{ install_yamls_dir }}" - target: mariadb - environment: - MARIADB_IMG: "{{ mariadb_operator_image_updated }}" - OPERATOR_NAMESPACE: "{{ operator_namespace }}" - NAMESPACE: "{{ cr_namespace }}" - TIMEOUT: "{{ deployment_timeout }}" +- name: Install the updated mariadb-operator + ansible.builtin.include_tasks: install_operator.yaml + vars: + mariadb_operator_index: "{{ mariadb_operator_image_updated }}" tags: - update diff --git a/test/ansible/vars/mariadb-operator-vars.yaml b/test/ansible/vars/mariadb-operator-vars.yaml index e98882da..2b83d1a8 100644 --- a/test/ansible/vars/mariadb-operator-vars.yaml +++ b/test/ansible/vars/mariadb-operator-vars.yaml @@ -16,7 +16,7 @@ cr_namespace: "openstack" operator_namespace: "openstack-operators" # Galera cluster configuration -galera_cr: "{{ tmp_dir }}/operator/mariadb-operator/config/samples/mariadb_v1beta1_galera.yaml" +galera_cr: "{{ tmp_dir }}/operator/mariadb-operator/config/samples/mariadb_v1beta1_galera.yaml" replicas: 3 db_service: "galera"