diff --git a/roles/shiftstack/README.md b/roles/shiftstack/README.md index 8d76b8ef5..dfc335859 100644 --- a/roles/shiftstack/README.md +++ b/roles/shiftstack/README.md @@ -14,13 +14,15 @@ Role for triggering Openshift on Openstack QA automation (installation and tests * `cifmw_shiftstack_client_pvc_manifest`: (*string*) The file name for the shiftstackclient pvc manifest. Defaults to `"{{ cifmw_shiftstack_client_pod_name }}_pvc.yml"`. * `cifmw_shiftstack_cluster_name`: (*string*) The Openshift cluster name. Defaults to `ostest`. * `cifmw_shiftstack_hypervisor`: (*string*) The hypervisor where RHOSO is deployed. Defaults to `"{{ hostvars[hostvars['controller-0']['cifmw_hypervisor_host'] | default ('')]['ansible_host'] | default('') }}"`. +* `cifmw_shiftstack_exec_delay`: (*integer*) Seconds between polling attempts when waiting for a command to complete in the pod. Defaults to `5`. +* `cifmw_shiftstack_exec_retries`: (*integer*) Maximum polling attempts when waiting for a command to complete in the pod. With the default delay of 5s, 5760 retries gives an 8-hour timeout. Defaults to `5760`. * `cifmw_shiftstack_exclude_artifacts_regex`: (*string*) Regex that will be passed on `oc rsync` command as `--exclude` param, so the role does not gather the artifacts matching it. * `cifmw_shiftstack_installation_dir`: (*string*) Directory to place installation files. Defaults to `"{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/installation"`. * `cifmw_shiftstack_manifests_dir`: (*string*) Directory name for the role generated Openshift manifests. Defaults to `"{{ cifmw_shiftstack_basedir }}/manifests"`. * `cifmw_shiftstack_project_name`: (*string*) The Openstack project name. Defaults to `shiftstack`. * `cifmw_shiftstack_proxy`: (*string*) The proxy url that should be used to reach the underlying OCP. Defaults to omit. -* `cifmw_shiftstack_qa_gerrithub_change`: (*string*) The gerrithub change to fetch from the `cifmw_shiftstack_qa_repo` repository (i.e. 'refs/changes/29/1188429/50)'. Defaults to ''. -* `cifmw_shiftstack_qa_repo`: (*string*) The repository containing the Openshift on Openstack QA automation. Defaults to `https://review.gerrithub.io/shiftstack/shiftstack-qa`. +* `cifmw_shiftstack_qa_change_ref`: (*string*) A git ref to fetch and checkout from the `cifmw_shiftstack_qa_repo` repository. Supports GitHub PR refs (e.g. `refs/pull/5/head`) and GerritHub change refs (e.g. `refs/changes/29/1188429/50`). Defaults to `''`. +* `cifmw_shiftstack_qa_repo`: (*string*) The repository containing the Openshift on Openstack QA automation. Defaults to `https://github.com/shiftstack/shiftstack-qa`. * `cifmw_shiftstack_run_playbook`: (*string*) The playbook to be run from the `cifmw_shiftstack_qa_repo` repository. Defaults to `ocp_testing.yaml`. * `cifmw_shiftstack_sc`: (*string*) The storage class to be used for PVC for the shiftstackclient pod. Defaults to `local-storage`. * `cifmw_shiftstack_shiftstackclient_artifacts_dir`: (*string*) The artifacts directory path for the shiftstackclient pod. Defaults to `/home/cloud-admin/artifacts`. @@ -40,7 +42,7 @@ cifmw_run_test_role: shiftstack cifmw_run_test_shiftstack_testconfig: - 4.17_ovnkubernetes_ipi.yaml - 4.17_ovnkubernetes_upi.yaml -cifmw_shiftstack_qa_gerrithub_change: refs/changes/29/1188429/50 #optional +cifmw_shiftstack_qa_change_ref: refs/pull/5/head #optional -$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_test_role=shiftstack cifmw_test_operator_stages=[] cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_gerrithub_change=refs/changes/29/1188429/50" +$ ansible-playbook deploy-edpm.yml --extra-vars "cifmw_run_tests=true cifmw_run_test_role=shiftstack cifmw_test_operator_stages=[] cifmw_openshift_kubeconfig={{ ansible_user_dir }}/.kube/config cifmw_run_test_shiftstack_testconfig=4.15_ovnkubernetes_ipi_va1.yaml cifmw_shiftstack_qa_change_ref=refs/pull/5/head" ``` diff --git a/roles/shiftstack/defaults/main.yml b/roles/shiftstack/defaults/main.yml index b77c8a558..4d69265e4 100644 --- a/roles/shiftstack/defaults/main.yml +++ b/roles/shiftstack/defaults/main.yml @@ -28,13 +28,15 @@ cifmw_shiftstack_client_pod_name: "shiftstackclient-{{ cifmw_shiftstack_project_ cifmw_shiftstack_client_pod_namespace: "openstack" cifmw_shiftstack_client_pvc_manifest: "{{ cifmw_shiftstack_client_pod_name }}_pvc.yml" cifmw_shiftstack_cluster_name: "ostest" +cifmw_shiftstack_exec_delay: 5 +cifmw_shiftstack_exec_retries: 5760 cifmw_shiftstack_exclude_artifacts_regex: "openshift-install" cifmw_shiftstack_hypervisor: "{{ hostvars[hostvars['controller-0']['cifmw_hypervisor_host'] | default ('')]['ansible_host'] | default('') }}" cifmw_shiftstack_installation_dir: "{{ cifmw_shiftstack_basedir }}/installation" cifmw_shiftstack_manifests_dir: "{{ cifmw_shiftstack_basedir }}/manifests" cifmw_shiftstack_project_name: "shiftstack" -cifmw_shiftstack_qa_gerrithub_change: "" -cifmw_shiftstack_qa_repo: "https://review.gerrithub.io/shiftstack/shiftstack-qa" +cifmw_shiftstack_qa_change_ref: "" +cifmw_shiftstack_qa_repo: "https://github.com/shiftstack/shiftstack-qa" cifmw_shiftstack_run_playbook: "ocp_testing.yaml" cifmw_shiftstack_sc: "local-storage" cifmw_shiftstack_shiftstackclient_artifacts_dir: "/home/cloud-admin/artifacts" diff --git a/roles/shiftstack/tasks/exec_command_in_pod.yml b/roles/shiftstack/tasks/exec_command_in_pod.yml index 5ef3f416c..e0d66e2da 100644 --- a/roles/shiftstack/tasks/exec_command_in_pod.yml +++ b/roles/shiftstack/tasks/exec_command_in_pod.yml @@ -16,23 +16,130 @@ - name: Command execution block block: - - name: Execute in the pod the command '{{ command }}' + - name: Kill any previous background command and clear markers + kubernetes.core.k8s_exec: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + namespace: "{{ namespace }}" + pod: "{{ pod_name }}" + command: >- + bash -c 'if [ -f /tmp/cifmw_cmd_pid ]; then + kill $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; + wait $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; fi; + rm -f /tmp/cifmw_cmd_rc /tmp/cifmw_cmd_pid + /tmp/cifmw_cmd_script.sh /tmp/cifmw_cmd_output.log' + changed_when: false + failed_when: false + + - name: Copy the command script into the pod + kubernetes.core.k8s_cp: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + namespace: "{{ namespace }}" + pod: "{{ pod_name }}" + remote_path: /tmp/cifmw_cmd_script.sh + content: | + #!/bin/bash + {{ command }} + state: to_pod + + - name: Execute detached in the pod - {{ command }} + kubernetes.core.k8s_exec: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + namespace: "{{ namespace }}" + pod: "{{ pod_name }}" + command: >- + bash -c 'nohup bash -c "bash /tmp/cifmw_cmd_script.sh; + echo \$? > /tmp/cifmw_cmd_rc" + > /tmp/cifmw_cmd_output.log 2>&1 & + echo $! > /tmp/cifmw_cmd_pid && cat /tmp/cifmw_cmd_pid' + register: pod_command_result + + - name: Wait for the command to complete in the pod environment: PATH: "{{ cifmw_path }}" KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" ansible.builtin.command: - cmd: "oc rsh -n {{ namespace }} {{ pod_name }} bash -c '{{ command }}'" - register: pod_command_result + cmd: >- + oc exec -n {{ namespace }} {{ pod_name }} -- + bash -c 'if cat /tmp/cifmw_cmd_rc 2>/dev/null; then exit 0; + elif [ -f /tmp/cifmw_cmd_pid ] && kill -0 $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; then exit 1; + else exit 2; fi' + register: pod_exit_code + changed_when: false + failed_when: false + until: pod_exit_code.rc == 0 or pod_exit_code.rc == 2 + retries: "{{ cifmw_shiftstack_exec_retries }}" + delay: "{{ cifmw_shiftstack_exec_delay }}" + + - name: Fail if the background process died without writing the marker + ansible.builtin.fail: + msg: >- + The background process in pod {{ pod_name }} exited without writing a result. + The command may have been killed by OOM or a signal. + when: pod_exit_code.rc == 2 + + - name: Fail if polling timed out + ansible.builtin.fail: + msg: >- + Timed out waiting for "{{ command }}" to complete + in pod {{ pod_name }} after {{ cifmw_shiftstack_exec_retries }} attempts. + when: pod_exit_code.rc != 0 and pod_exit_code.rc != 2 + + - name: Fail if the command failed inside the pod + ansible.builtin.fail: + msg: >- + "{{ command }}" failed inside the pod with + exit code {{ pod_exit_code.stdout | trim }}. + when: + - pod_exit_code.rc == 0 + - (pod_exit_code.stdout | default('') | trim) != '0' rescue: - - name: Fail when the command module fails + - name: Kill background process on failure + kubernetes.core.k8s_exec: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + namespace: "{{ namespace }}" + pod: "{{ pod_name }}" + command: >- + bash -c 'if [ -f /tmp/cifmw_cmd_pid ]; then + kill $(cat /tmp/cifmw_cmd_pid) 2>/dev/null; fi' + failed_when: false + changed_when: false + + - name: Retrieve command output from pod + kubernetes.core.k8s_exec: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + namespace: "{{ namespace }}" + pod: "{{ pod_name }}" + command: >- + bash -c 'tail -c 8192 /tmp/cifmw_cmd_output.log 2>/dev/null + || echo "(output not available)"' + register: pod_output_log + failed_when: false + changed_when: false + + - name: Fail with diagnostic information ansible.builtin.fail: - msg: '"{{ command }}" command execution failed with exit code "{{ pod_command_result.rc }}" and error: "{{ pod_command_result.stderr }}".' + msg: >- + "{{ command }}" failed. + pod exit code: {{ pod_exit_code.stdout | default('N/A') | trim }}, + output (last 8KB): {{ pod_output_log.stdout | default('(not retrieved)') }} always: - name: Command execution log saving block when: log_file_name | default(false) block: + - name: Retrieve command output for log + kubernetes.core.k8s_exec: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + namespace: "{{ namespace }}" + pod: "{{ pod_name }}" + command: >- + bash -c 'tail -c 16384 /tmp/cifmw_cmd_output.log 2>/dev/null + || echo "(output not available)"' + register: pod_output_for_log + failed_when: false + changed_when: false + - name: Get current date and time ansible.builtin.command: cmd: date +"%Y_%m_%d-%H_%M_%S" @@ -44,7 +151,9 @@ namespace: "{{ namespace }}" pod_name: "{{ pod_name }}" command: "{{ command }}" - pod_command_result: "{{ pod_command_result }}" + pod_command_result: "{{ pod_command_result | default('N/A') }}" + pod_exit_code: "{{ pod_exit_code | default('N/A') }}" + pod_output: "{{ pod_output_for_log.stdout | default('N/A') }}" ansible.builtin.copy: content: "{{ command_info | to_nice_yaml }}" dest: "{{ cifmw_shiftstack_ansible_command_logs_dir }}/{{ current_date.stdout }}.{{ log_file_name }}" diff --git a/roles/shiftstack/tasks/test_shiftstack.yml b/roles/shiftstack/tasks/test_shiftstack.yml index 25f0e36e3..67a42ad5d 100644 --- a/roles/shiftstack/tasks/test_shiftstack.yml +++ b/roles/shiftstack/tasks/test_shiftstack.yml @@ -23,18 +23,18 @@ log_file_name: "clone_shiftstack_qa_repo.log" ansible.builtin.include_tasks: exec_command_in_pod.yml -- name: Fetch the gerrithub change '{{ cifmw_shiftstack_qa_gerrithub_change }}' +- name: Fetch the change ref '{{ cifmw_shiftstack_qa_change_ref }}' vars: namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" pod_name: "{{ cifmw_shiftstack_client_pod_name }}" command: >- cd shiftstack-qa && - git fetch {{ cifmw_shiftstack_qa_repo }} {{ cifmw_shiftstack_qa_gerrithub_change }} && + git fetch {{ cifmw_shiftstack_qa_repo }} {{ cifmw_shiftstack_qa_change_ref }} && git checkout FETCH_HEAD - log_file_name: "fetch_shiftstack_qa_gerrithub_change.log" + log_file_name: "fetch_shiftstack_qa_change_ref.log" when: - - cifmw_shiftstack_qa_gerrithub_change is defined - - cifmw_shiftstack_qa_gerrithub_change != '' + - cifmw_shiftstack_qa_change_ref is defined + - cifmw_shiftstack_qa_change_ref != '' ansible.builtin.include_tasks: exec_command_in_pod.yml - name: Install the ansible collections