Skip to content

Commit cf54000

Browse files
dciabrindciabrin
committed
ansible test: fix race condition in operator update
The usual way of installing a new mariadb-operator is to use install_yamls and `make mariadb`. However this one creates the subscription CR right after the catalog CR has been created. If a previous operator was present, this might prevent OLM to retrieve the bundle data for the new operator, which would cause the previous version of the operator to be reinstalled. Fix that race condition by first installing the catalogsource, and wait until OLM refreshed its cache with the new bundle information, at which point it is safe to create the new subscription CR.
1 parent e7a4f0a commit cf54000

8 files changed

Lines changed: 101 additions & 53 deletions

‎test/ansible/.ansible-lint-ignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# This file contains ignores rule violations for ansible-lint
2+
tasks/cleanup_environment.yaml yaml[line-length] skip
3+
tasks/install_operator.yaml yaml[line-length] skip
4+
tasks/prepare_environment.yaml yaml[line-length] skip
5+
tasks/update_operator.yaml yaml[line-length] skip

‎test/ansible/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags deploy,operato
3333
### Only update operator with pod disruption -> verify galera is restarted correctly
3434

3535
```bash
36-
ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update
36+
ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update,operator
3737
```
3838

3939
### Only update part without pod disruption
4040

4141
```bash
42-
ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update -e disruption=false
42+
ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update,operator -e disruption=false
4343
```
4444

4545
### Full end-to-end run with 1-node Galera

‎test/ansible/playbooks/mariadb-operator-update.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@
4040
tags:
4141
- always
4242

43+
- name: Prepare environment
44+
ansible.builtin.include_tasks:
45+
file: ../tasks/prepare_environment.yaml
46+
tags:
47+
- deploy
48+
4349
- name: Cleanup existing mariadb-operator and galera CR
4450
ansible.builtin.include_tasks:
4551
file: ../tasks/cleanup_environment.yaml

‎test/ansible/tasks/deploy_resources.yaml‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,8 @@
11
---
2-
- name: Deploy mariadb-operator with initial image
3-
community.general.make:
4-
chdir: "{{ install_yamls_dir }}"
5-
target: mariadb
6-
environment:
7-
MARIADB_IMG: "{{ mariadb_operator_image_initial }}"
8-
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
9-
NAMESPACE: "{{ cr_namespace }}"
10-
OUT: "{{ tmp_dir }}"
11-
TIMEOUT: "{{ deployment_timeout }}"
2+
- name: Install a default mariadb-operator
3+
ansible.builtin.include_tasks: install_operator.yaml
4+
vars:
5+
mariadb_operator_index: "{{ mariadb_operator_image_initial }}"
126
tags:
137
- operator
148

‎test/ansible/tasks/install_operator.yaml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
---
2+
- name: Clean up existing mariadb-operator
3+
community.general.make:
4+
chdir: "{{ install_yamls_dir }}"
5+
target: mariadb_cleanup
6+
environment:
7+
MARIADB_IMG: "{{ mariadb_operator_index }}"
8+
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
9+
NAMESPACE: "{{ cr_namespace }}"
10+
OUT: "{{ tmp_dir }}"
11+
TIMEOUT: "{{ deployment_timeout }}"
12+
tags:
13+
- operator
14+
15+
- name: Wait until the catalogsource associated with the old mariadb-operator is deleted
16+
ansible.builtin.shell: >
17+
oc -n {{ operator_namespace }} wait --for=delete --timeout={{ deployment_timeout }}
18+
catalogsource/mariadb-operator-index
19+
changed_when: false
20+
tags:
21+
- operator
22+
23+
- name: Remove the leftover install plans associated with the old mariadb-operator
24+
ansible.builtin.shell: |
25+
set -o pipefail
26+
oc -n {{ operator_namespace }} get installplans -o name | xargs -r oc -n {{ operator_namespace }} delete
27+
changed_when: false
28+
tags:
29+
- operator
30+
31+
- name: Prepare the resource required to install a new mariadb-operator
32+
community.general.make:
33+
chdir: "{{ install_yamls_dir }}"
34+
target: mariadb_prep
35+
environment:
36+
MARIADB_IMG: "{{ mariadb_operator_index }}"
37+
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
38+
NAMESPACE: "{{ cr_namespace }}"
39+
OUT: "{{ tmp_dir }}"
40+
TIMEOUT: "{{ deployment_timeout }}"
41+
tags:
42+
- operator
43+
44+
- name: Install the catalogsource for the new mariadb-operator
45+
# Apply CatalogSource first and wait for OLM to have fetch the bundle information, otherwise
46+
# the old bundle info might be reused when creating the subscription CR
47+
# Note: several mariadb-operator-index pods are created, but only one gets ready, so wait accordingly
48+
ansible.builtin.shell: |
49+
set -e -o pipefail
50+
oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/catalogsource.yaml
51+
until oc -n {{ operator_namespace }} get pods -l olm.catalogSource=mariadb-operator-index -o jsonpath='{range .items[*]}{.metadata.name} {.status.containerStatuses[*].ready}{end}' | grep -w -m1 true; do sleep 2; done
52+
index_pod=$(oc -n {{ operator_namespace }} get pod -l olm.catalogSource=mariadb-operator-index -o name)
53+
test -n "${index_pod}"
54+
timeout {{ deployment_timeout | community.general.to_seconds | int }} bash -c "oc -n {{ operator_namespace }} logs '${index_pod}' -f | grep -m1 'grpc.code=OK.*GetBundleForChannel'"
55+
changed_when: false
56+
tags:
57+
- operator
58+
59+
- name: Apply subscriptions and operatorgroup to deploy the new mariadb-operator
60+
ansible.builtin.shell: |
61+
set -e
62+
oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/operatorgroup.yaml
63+
oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/subscription.yaml
64+
changed_when: false
65+
tags:
66+
- operator

‎test/ansible/tasks/prepare_environment.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
- name: Disable openstack-operator-controller-init to allow custom mariadb-operator image
3+
ansible.builtin.shell: |
4+
set -o pipefail
5+
if ! oc -n {{ operator_namespace }} get deployment/openstack-operator-controller-init &>/dev/null; then exit 0; fi
6+
SAVED_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r '.metadata.labels["test.mariadb-operator/saved-replicas"]')
7+
if [ -n "$SAVED_REPLICAS" ] && [ "$SAVED_REPLICAS" != "null" ]; then exit 0; fi
8+
CURRENT_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r .spec.replicas)
9+
oc -n {{ operator_namespace }} label deployment/openstack-operator-controller-init test.mariadb-operator/saved-replicas="$CURRENT_REPLICAS"
10+
oc -n {{ operator_namespace }} patch deployment/openstack-operator-controller-init --type='json' -p='[{"op":"replace","path":"/spec/replicas","value":0}]'
11+
changed_when: false
12+
tags:
13+
- always

‎test/ansible/tasks/update_operator.yaml‎

Lines changed: 4 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,8 @@
11
---
2-
- name: Uninstall the old mariadb-operator
3-
community.general.make:
4-
chdir: "{{ install_yamls_dir }}"
5-
target: mariadb_cleanup
6-
environment:
7-
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
8-
NAMESPACE: "{{ cr_namespace }}"
9-
TIMEOUT: "{{ update_timeout }}"
10-
tags:
11-
- update
12-
13-
- name: Wait for the mariadb-operator to be uninstalled
14-
ansible.builtin.shell: >
15-
set -o pipefail && echo $(oc -n {{ operator_namespace }} get csv -l operators.coreos.com/mariadb-operator.openstack-operators -o name)
16-
catalogsource/mariadb-operator-index
17-
subscription/mariadb-operator
18-
| xargs oc -n {{ operator_namespace }} wait --for=delete --timeout={{ update_timeout }}
19-
changed_when: false
20-
tags:
21-
- update
22-
23-
- name: Disable openstack-operator-controller-init to allow custom mariadb-operator image
24-
ansible.builtin.shell: |
25-
set -o pipefail
26-
PREV_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r .spec.replicas)
27-
oc -n {{ operator_namespace }} label deployment/openstack-operator-controller-init test.mariadb-operator/saved-replicas="$PREV_REPLICAS"
28-
oc -n {{ operator_namespace }} patch deployment/openstack-operator-controller-init --type='json' -p='[{"op":"replace","path":"/spec/replicas","value":0}]'
29-
changed_when: false
30-
tags:
31-
- update
32-
33-
- name: Update mariadb-operator to new image
34-
community.general.make:
35-
chdir: "{{ install_yamls_dir }}"
36-
target: mariadb
37-
environment:
38-
MARIADB_IMG: "{{ mariadb_operator_image_updated }}"
39-
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
40-
NAMESPACE: "{{ cr_namespace }}"
41-
TIMEOUT: "{{ deployment_timeout }}"
2+
- name: Install the updated mariadb-operator
3+
ansible.builtin.include_tasks: install_operator.yaml
4+
vars:
5+
mariadb_operator_index: "{{ mariadb_operator_image_updated }}"
426
tags:
437
- update
448

‎test/ansible/vars/mariadb-operator-vars.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ cr_namespace: "openstack"
1616
operator_namespace: "openstack-operators"
1717

1818
# Galera cluster configuration
19-
galera_cr: "{{ tmp_dir }}/operator/mariadb-operator/config/samples/mariadb_v1beta1_galera.yaml"
19+
galera_cr: "{{ tmp_dir }}/operator/mariadb-operator/config/samples/mariadb_v1beta1_galera.yaml"
2020
replicas: 3
2121
db_service: "galera"
2222

0 commit comments

Comments
 (0)