Skip to content

Commit 00dfff2

Browse files
openshift-merge-bot[bot]openshift-merge-bot[bot]
authored
Merge pull request #475 from dciabrin/ansible-update-race
ansible test: fix race condition in operator update
2 parents e7a4f0a + cf54000 commit 00dfff2

8 files changed

Lines changed: 101 additions & 53 deletions

‎test/ansible/.ansible-lint-ignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# This file contains ignores rule violations for ansible-lint
2+
tasks/cleanup_environment.yaml yaml[line-length] skip
3+
tasks/install_operator.yaml yaml[line-length] skip
4+
tasks/prepare_environment.yaml yaml[line-length] skip
5+
tasks/update_operator.yaml yaml[line-length] skip

‎test/ansible/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,13 +33,13 @@ ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags deploy,operato
3333
### Only update operator with pod disruption -> verify galera is restarted correctly
3434

3535
```bash
36-
ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update
36+
ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update,operator
3737
```
3838

3939
### Only update part without pod disruption
4040

4141
```bash
42-
ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update -e disruption=false
42+
ansible-playbook -v playbooks/mariadb-operator-update.yaml --tags update,operator -e disruption=false
4343
```
4444

4545
### Full end-to-end run with 1-node Galera

‎test/ansible/playbooks/mariadb-operator-update.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,12 @@
4040
tags:
4141
- always
4242

43+
- name: Prepare environment
44+
ansible.builtin.include_tasks:
45+
file: ../tasks/prepare_environment.yaml
46+
tags:
47+
- deploy
48+
4349
- name: Cleanup existing mariadb-operator and galera CR
4450
ansible.builtin.include_tasks:
4551
file: ../tasks/cleanup_environment.yaml

‎test/ansible/tasks/deploy_resources.yaml‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,8 @@
11
---
2-
- name: Deploy mariadb-operator with initial image
3-
community.general.make:
4-
chdir: "{{ install_yamls_dir }}"
5-
target: mariadb
6-
environment:
7-
MARIADB_IMG: "{{ mariadb_operator_image_initial }}"
8-
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
9-
NAMESPACE: "{{ cr_namespace }}"
10-
OUT: "{{ tmp_dir }}"
11-
TIMEOUT: "{{ deployment_timeout }}"
2+
- name: Install a default mariadb-operator
3+
ansible.builtin.include_tasks: install_operator.yaml
4+
vars:
5+
mariadb_operator_index: "{{ mariadb_operator_image_initial }}"
126
tags:
137
- operator
148

‎test/ansible/tasks/install_operator.yaml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
---
2+
- name: Clean up existing mariadb-operator
3+
community.general.make:
4+
chdir: "{{ install_yamls_dir }}"
5+
target: mariadb_cleanup
6+
environment:
7+
MARIADB_IMG: "{{ mariadb_operator_index }}"
8+
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
9+
NAMESPACE: "{{ cr_namespace }}"
10+
OUT: "{{ tmp_dir }}"
11+
TIMEOUT: "{{ deployment_timeout }}"
12+
tags:
13+
- operator
14+
15+
- name: Wait until the catalogsource associated with the old mariadb-operator is deleted
16+
ansible.builtin.shell: >
17+
oc -n {{ operator_namespace }} wait --for=delete --timeout={{ deployment_timeout }}
18+
catalogsource/mariadb-operator-index
19+
changed_when: false
20+
tags:
21+
- operator
22+
23+
- name: Remove the leftover install plans associated with the old mariadb-operator
24+
ansible.builtin.shell: |
25+
set -o pipefail
26+
oc -n {{ operator_namespace }} get installplans -o name | xargs -r oc -n {{ operator_namespace }} delete
27+
changed_when: false
28+
tags:
29+
- operator
30+
31+
- name: Prepare the resource required to install a new mariadb-operator
32+
community.general.make:
33+
chdir: "{{ install_yamls_dir }}"
34+
target: mariadb_prep
35+
environment:
36+
MARIADB_IMG: "{{ mariadb_operator_index }}"
37+
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
38+
NAMESPACE: "{{ cr_namespace }}"
39+
OUT: "{{ tmp_dir }}"
40+
TIMEOUT: "{{ deployment_timeout }}"
41+
tags:
42+
- operator
43+
44+
- name: Install the catalogsource for the new mariadb-operator
45+
# Apply CatalogSource first and wait for OLM to have fetch the bundle information, otherwise
46+
# the old bundle info might be reused when creating the subscription CR
47+
# Note: several mariadb-operator-index pods are created, but only one gets ready, so wait accordingly
48+
ansible.builtin.shell: |
49+
set -e -o pipefail
50+
oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/catalogsource.yaml
51+
until oc -n {{ operator_namespace }} get pods -l olm.catalogSource=mariadb-operator-index -o jsonpath='{range .items[*]}{.metadata.name} {.status.containerStatuses[*].ready}{end}' | grep -w -m1 true; do sleep 2; done
52+
index_pod=$(oc -n {{ operator_namespace }} get pod -l olm.catalogSource=mariadb-operator-index -o name)
53+
test -n "${index_pod}"
54+
timeout {{ deployment_timeout | community.general.to_seconds | int }} bash -c "oc -n {{ operator_namespace }} logs '${index_pod}' -f | grep -m1 'grpc.code=OK.*GetBundleForChannel'"
55+
changed_when: false
56+
tags:
57+
- operator
58+
59+
- name: Apply subscriptions and operatorgroup to deploy the new mariadb-operator
60+
ansible.builtin.shell: |
61+
set -e
62+
oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/operatorgroup.yaml
63+
oc -n {{ operator_namespace }} apply -f {{ tmp_dir }}/openstack-operators/mariadb/op/subscription.yaml
64+
changed_when: false
65+
tags:
66+
- operator

‎test/ansible/tasks/prepare_environment.yaml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
---
2+
- name: Disable openstack-operator-controller-init to allow custom mariadb-operator image
3+
ansible.builtin.shell: |
4+
set -o pipefail
5+
if ! oc -n {{ operator_namespace }} get deployment/openstack-operator-controller-init &>/dev/null; then exit 0; fi
6+
SAVED_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r '.metadata.labels["test.mariadb-operator/saved-replicas"]')
7+
if [ -n "$SAVED_REPLICAS" ] && [ "$SAVED_REPLICAS" != "null" ]; then exit 0; fi
8+
CURRENT_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r .spec.replicas)
9+
oc -n {{ operator_namespace }} label deployment/openstack-operator-controller-init test.mariadb-operator/saved-replicas="$CURRENT_REPLICAS"
10+
oc -n {{ operator_namespace }} patch deployment/openstack-operator-controller-init --type='json' -p='[{"op":"replace","path":"/spec/replicas","value":0}]'
11+
changed_when: false
12+
tags:
13+
- always

‎test/ansible/tasks/update_operator.yaml‎

Lines changed: 4 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,44 +1,8 @@
11
---
2-
- name: Uninstall the old mariadb-operator
3-
community.general.make:
4-
chdir: "{{ install_yamls_dir }}"
5-
target: mariadb_cleanup
6-
environment:
7-
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
8-
NAMESPACE: "{{ cr_namespace }}"
9-
TIMEOUT: "{{ update_timeout }}"
10-
tags:
11-
- update
12-
13-
- name: Wait for the mariadb-operator to be uninstalled
14-
ansible.builtin.shell: >
15-
set -o pipefail && echo $(oc -n {{ operator_namespace }} get csv -l operators.coreos.com/mariadb-operator.openstack-operators -o name)
16-
catalogsource/mariadb-operator-index
17-
subscription/mariadb-operator
18-
| xargs oc -n {{ operator_namespace }} wait --for=delete --timeout={{ update_timeout }}
19-
changed_when: false
20-
tags:
21-
- update
22-
23-
- name: Disable openstack-operator-controller-init to allow custom mariadb-operator image
24-
ansible.builtin.shell: |
25-
set -o pipefail
26-
PREV_REPLICAS=$(oc -n {{ operator_namespace }} get -o json deployment/openstack-operator-controller-init | jq -r .spec.replicas)
27-
oc -n {{ operator_namespace }} label deployment/openstack-operator-controller-init test.mariadb-operator/saved-replicas="$PREV_REPLICAS"
28-
oc -n {{ operator_namespace }} patch deployment/openstack-operator-controller-init --type='json' -p='[{"op":"replace","path":"/spec/replicas","value":0}]'
29-
changed_when: false
30-
tags:
31-
- update
32-
33-
- name: Update mariadb-operator to new image
34-
community.general.make:
35-
chdir: "{{ install_yamls_dir }}"
36-
target: mariadb
37-
environment:
38-
MARIADB_IMG: "{{ mariadb_operator_image_updated }}"
39-
OPERATOR_NAMESPACE: "{{ operator_namespace }}"
40-
NAMESPACE: "{{ cr_namespace }}"
41-
TIMEOUT: "{{ deployment_timeout }}"
2+
- name: Install the updated mariadb-operator
3+
ansible.builtin.include_tasks: install_operator.yaml
4+
vars:
5+
mariadb_operator_index: "{{ mariadb_operator_image_updated }}"
426
tags:
437
- update
448

‎test/ansible/vars/mariadb-operator-vars.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ cr_namespace: "openstack"
1616
operator_namespace: "openstack-operators"
1717

1818
# Galera cluster configuration
19-
galera_cr: "{{ tmp_dir }}/operator/mariadb-operator/config/samples/mariadb_v1beta1_galera.yaml"
19+
galera_cr: "{{ tmp_dir }}/operator/mariadb-operator/config/samples/mariadb_v1beta1_galera.yaml"
2020
replicas: 3
2121
db_service: "galera"
2222

0 commit comments

Comments
 (0)