Skip to content

Commit 5f17bc5

Browse files
sathlansathlan
committed
Fix continuous control plane testing during update
Create a new container run by podman so that the update process won't interfere with the commands triggered on the OpenStack plateform. Closes: [OSPRH-16001](https://issues.redhat.com/browse/OSPRH-16001)
1 parent 22adb5a commit 5f17bc5

4 files changed

Lines changed: 139 additions & 50 deletions

File tree

‎roles/update/tasks/collect_openstackclient_config.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
---
2+
- name: Collect file from openstackclient container
3+
kubernetes.core.k8s_exec:
4+
kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
5+
namespace: "openstack"
6+
api_key: "{{ cifmw_openshift_token | default(omit) }}"
7+
context: "{{ cifmw_openshift_context | default(omit) }}"
8+
pod: "openstackclient"
9+
container: "openstackclient"
10+
command: "/usr/bin/cat /home/cloud-admin/.config/openstack/{{ item }}"
11+
register: file_content
12+
changed_when: false
13+
14+
- name: Save file locally
15+
ansible.builtin.copy:
16+
content: "{{ file_content.stdout }}"
17+
dest: "{{ cifmw_update_artifacts_basedir }}/{{ item }}"
18+
mode: '0644'
19+
changed_when: false

‎roles/update/tasks/create_local_openstackclient.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
---
2+
- name: Gather NodeSet resource information
3+
kubernetes.core.k8s_info:
4+
kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
5+
namespace: "openstack"
6+
api_key: "{{ cifmw_openshift_token | default(omit) }}"
7+
context: "{{ cifmw_openshift_context | default(omit) }}"
8+
kind: "OpenStackDataPlaneNodeSet"
9+
api_version: "dataplane.openstack.org/v1beta1"
10+
register: _cifmw_update_osdpns_all_info
11+
12+
- name: Fail if no OSDPNS resources are found
13+
ansible.builtin.fail:
14+
msg: "No OSDPNS resources found in the 'openstack' namespace!"
15+
when: _cifmw_update_osdpns_all_info.resources | length == 0
16+
17+
- name: Choose the first OSDPNS resource
18+
ansible.builtin.set_fact:
19+
_cifmw_update_osdpns_info: "{{ _cifmw_update_osdpns_all_info.resources[0] }}"
20+
21+
- name: Display which osdpns we're using
22+
ansible.builtin.debug:
23+
msg: "Found OSDPNS named: '{{ _cifmw_update_osdpns_info.metadata.name }}'"
24+
25+
- name: Determine registry
26+
ansible.builtin.set_fact:
27+
cifmw_update_local_registry: >-
28+
{{
29+
(cifmw_ci_gen_kustomize_values_ooi_image.split('/')[0])
30+
if cifmw_ci_gen_kustomize_values_ooi_image is defined
31+
else 'quay.io'
32+
| trim
33+
}}
34+
35+
- name: Check if credentials exist
36+
ansible.builtin.set_fact:
37+
brew_username: "{{ login_username }}"
38+
brew_password: "{{ login_dict[login_username] }}"
39+
vars:
40+
login_dict: >-
41+
{{
42+
_cifmw_update_osdpns_info.spec.nodeTemplate.ansible.ansibleVars.
43+
edpm_container_registry_logins[cifmw_update_local_registry]
44+
}}
45+
login_username: "{{ login_dict.keys()|list|first }}"
46+
when:
47+
- _cifmw_update_osdpns_info.spec.nodeTemplate.ansible.ansibleVars.edpm_container_registry_logins is defined
48+
- login_dict is defined
49+
- login_dict|length > 0
50+
- cifmw_update_local_registry != 'quay.io'
51+
52+
- name: Log in to registry when needed
53+
containers.podman.podman_login:
54+
# Hardcoded for now
55+
registry: "registry.redhat.io"
56+
username: "{{ brew_username }}"
57+
password: "{{ brew_password }}"
58+
when:
59+
- brew_username is defined
60+
- brew_password is defined
61+
- cifmw_update_local_registry != 'quay.io'
62+
63+
- name: Retrieve the openstackclient Pod
64+
kubernetes.core.k8s_info:
65+
kubeconfig: "{{ cifmw_openshift_kubeconfig }}"
66+
namespace: "openstack"
67+
api_key: "{{ cifmw_openshift_token | default(omit) }}"
68+
context: "{{ cifmw_openshift_context | default(omit) }}"
69+
kind: "Pod"
70+
name: "openstackclient"
71+
register: _cifmw_update_openstackclient_pod
72+
73+
- name: Fail if openstackclient Pod is not found
74+
ansible.builtin.fail:
75+
msg: "No openstackclient Pod found in the openstack namespace!"
76+
when: _cifmw_update_openstackclient_pod.resources | length == 0
77+
78+
- name: Set the openstackclient image fact
79+
ansible.builtin.set_fact:
80+
openstackclient_image: "{{ _cifmw_update_openstackclient_pod.resources[0].spec.containers[0].image }}"
81+
82+
- name: Collect and save OpenStack config files
83+
ansible.builtin.include_tasks: collect_openstackclient_config.yml
84+
loop:
85+
- 'clouds.yaml'
86+
- 'secure.yaml'
87+
loop_control:
88+
label: "{{ item }}"
89+
90+
- name: Create local openstack wrapper script
91+
ansible.builtin.copy:
92+
dest: "{{ cifmw_update_artifacts_basedir }}/openstack"
93+
mode: '0755'
94+
content: |
95+
#!/usr/bin/env bash
96+
set -euo pipefail
97+
OS_CLOUD=default /usr/bin/openstack --insecure "$@"
98+
99+
- name: Ensure lopenstackclient container is running
100+
containers.podman.podman_container:
101+
name: lopenstackclient
102+
image: "{{ openstackclient_image }}"
103+
state: started
104+
net: host
105+
volumes:
106+
- "{{ cifmw_update_artifacts_basedir }}/clouds.yaml:/home/cloud-admin/.config/openstack/clouds.yaml:ro,Z"
107+
- "{{ cifmw_update_artifacts_basedir }}/secure.yaml:/home/cloud-admin/.config/openstack/secure.yaml:ro,Z"
108+
- "{{ cifmw_update_artifacts_basedir }}/openstack:/home/cloud-admin/.local/bin/openstack:ro,Z"
109+
command: ['/usr/bin/sleep', 'infinity']

‎roles/update/tasks/main.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,12 @@
2929
- name: Start ping test
3030
ansible.builtin.include_tasks: l3_agent_connectivity_check_start.yml
3131

32+
- name: Create local openstackclient
33+
when:
34+
- cifmw_update_control_plane_check | bool
35+
- not cifmw_update_run_dryrun | bool
36+
ansible.builtin.include_tasks: create_local_openstackclient.yml
37+
3238
- name: Trigger the continuous control plane test
3339
when:
3440
- cifmw_update_control_plane_check | bool

‎roles/update/templates/workload_launch_k8s.sh.j2‎

Lines changed: 5 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,8 @@
11
#!/usr/bin/bash
2-
set +x
32

4-
export KUBECONFIG="{{ cifmw_openshift_kubeconfig }}"
5-
export PATH="{{ cifmw_path }}"
6-
7-
OS_POD_TIMEOUT={{ cifmw_update_openstackclient_pod_timeout }}
8-
WAIT=0
9-
10-
# Temporary file where to put the error message, if any.
11-
ERROR_FILE=/tmp/cifmw_update_ctl_testing_current_ouput.txt
12-
rm -f "${ERROR_FILE}"
13-
14-
while [ $((WAIT++)) -lt ${OS_POD_TIMEOUT} ]; do
15-
set -o pipefail # Make sure we get the failure, as tee
3+
set -e
4+
set -o pipefail # Make sure we get the failure, as tee
165
# will always succeed.
17-
cat "{{ cifmw_update_artifacts_basedir }}/workload_launch.sh" | \
18-
oc rsh -n openstack openstackclient env WKL_MODE=sanityfast bash 2>&1 | tee "${ERROR_FILE}"
19-
RC=$?
20-
set +o pipefail
21-
if [ "${RC}" -eq 137 ]; then
22-
# When the command is interrupted by the restart of the
23-
# OSclient, we have this returns code. We just retry.
24-
sleep 1
25-
continue
26-
fi
27-
# If there's an error and the error file was created we check for
28-
# the error message.
29-
if [ "${RC}" -ne 0 ]; then
30-
if [ ! -e "${ERROR_FILE}" ]; then
31-
# no error file, rethrow the error.
32-
exit $RC
33-
fi
34-
# Fragile as it depends on the exact output message.
35-
if grep -F 'error: unable to upgrade connection: container not found' \
36-
"${ERROR_FILE}"; then
37-
# Openstackclient was not able to start as it's being
38-
# restarted, retry.
39-
sleep 1
40-
continue
41-
fi
42-
# Error is not related to the the openstackclient not being
43-
# available. We rethrow it.
44-
exit ${RC}
45-
fi
46-
# No error.
47-
exit 0
48-
done
49-
50-
# We only reach this code if we reach timeout while retrying to
51-
# trigger the openstackclient.
52-
echo "OpenstackClient Pod unavalaible, giving up after ${OS_POD_TIMEOUT} seconds" >&2
53-
exit 127
6+
cat "{{ cifmw_update_artifacts_basedir }}/workload_launch.sh" | \
7+
podman exec -i lopenstackclient \
8+
env WKL_MODE=sanityfast bash -i 2>&1

0 commit comments

Comments
 (0)