[multiple] Accept CA certificate if expired when CRC is used

It can happen that the image used by the CRC has expired certificate,
so it require to approve new generated certificate before making test.
The CRC log shows a log message:

    level=info msg="Kubelet serving certificate has expired, waiting for automatic renewal... [will take up to 5 minutes]"
    Failed to renew TLS certificates: please check if a newer CRC release is available:
    Temporary error: certificate /var/lib/kubelet/pki/kubelet-server-current.pem still expired (x59)

Wait for the cluster to be stable when CA cert is expired.
Also move all crc command to single playbook, for easier
maintenance.

Signed-off-by: Daniel Pawlik <dpawlik@redhat.com>

Author: danpawlik

roles/cert_manager/molecule/default/prepare.yml

@@ -20,6 +20,7 @@
   roles:
     - role: test_deps
   tasks:
-    - name: Ensure CRC is started
-      ansible.builtin.command:
-        cmd: crc start
+    - name: Start CRC
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml

roles/ci_local_storage/molecule/default/prepare.yml

@@ -20,9 +20,10 @@
   roles:
     - role: test_deps
   tasks:
-    - name: Ensure CRC is started
-      ansible.builtin.command:
-        cmd: crc start
+    - name: Start CRC
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml
 
     - name: Inject crc hostname/IP in hosts
       become: true

roles/cifmw_helpers/README.md

@@ -3,6 +3,7 @@
 That role was created to replace nested Ansible (Ansible that execute
 ansible or ansible-playbook binary using command/shell module) execution in
 this project.
+Role might contain tasks, required for test jobs, like molecule job.
 
 ## Helper for Zuul executor cifmw general collection
 
@@ -389,3 +390,30 @@ it parse all yaml files available in the directory.
         msg: |
           {{ noop_helper_var }}
 ```
+
+## Test project helpers
+
+### Start CRC
+
+In some CI jobs, we are using [crc](https://crc.dev/) to verify the role functionality.
+Example usage:
+
+* Setup crc with default resources
+
+```yaml
+    - name: Start CRC
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml
+```
+
+* Setup crc with limited resources, like memory, disk, cpu - [more](https://crc.dev/docs/installing/)
+
+```yaml
+    - name: Start CRC
+      vars:
+        cifmw_helpers_crc_additional_params: "--memory 14000 --disk-size 80 --cpus 6"
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml
+```

roles/cifmw_helpers/tasks/crc_start.yml

@@ -0,0 +1,20 @@
+---
+- name: Ensure CRC is started
+  ansible.builtin.command:
+    cmd: >
+      crc start {{ cifmw_helpers_crc_additional_params | default('') }}
+  register: _crc_output
+  ignore_errors: true  # noqa: ignore-errors
+
+- name: Login to the OpenShift when certificate is expired
+  when: "'Kubelet serving certificate has expired' in _crc_output.stderr"
+  ansible.builtin.command: >
+    oc login
+    -u kubeadmin
+    https://api.crc.testing:6443
+    --insecure-skip-tls-verify
+  register: _openshift_login
+  until: _openshift_login.rc == 0
+  retries: 90
+  delay: 10
+  changed_when: false

roles/env_op_images/molecule/default/prepare.yml

@@ -24,6 +24,9 @@
     - role: ci_setup
     - role: install_yamls
   tasks:
-    - name: Ensure CRC is started
-      ansible.builtin.command:
-        cmd: crc start --memory 14000 --disk-size 80 --cpus 6
+    - name: Start CRC
+      vars:
+        cifmw_helpers_crc_additional_params: "--memory 14000 --disk-size 80 --cpus 6"
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml

roles/install_openstack_ca/molecule/default/prepare.yml

@@ -21,26 +21,16 @@
     cifmw_path: "{{ ansible_user_dir }}/.crc/bin:{{ ansible_user_dir }}/.crc/bin/oc:{{ ansible_user_dir }}/bin:{{ ansible_env.PATH }}"
     cifmw_openshift_kubeconfig: "{{ ansible_user_dir  }}/.crc/machines/crc/kubeconfig"
   pre_tasks:
-    - name: Ensure CRC is started
-      async: 1800
-      poll: 0
-      register: _crc_start
-      ansible.builtin.command:
-        cmd: crc start
+    - name: Start CRC
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml
   roles:
     - role: test_deps
     - role: ci_setup
     - role: install_yamls
     - role: ci_local_storage
   tasks:
-    - name: Check for CRC status
-      ansible.builtin.async_status:
-        jid: "{{ _crc_start.ansible_job_id  }}"
-      register: _crc_status
-      until: _crc_status.finished
-      retries: 100
-      delay: 10
-
     - name: Install install_yamls dependencies
       ansible.builtin.include_role:
         name: 'install_yamls_makes'

roles/manage_secrets/molecule/dataplane_ssh/prepare.yml

@@ -17,21 +17,11 @@
 - name: Prepare
   hosts: all
   pre_tasks:
-    - name: Ensure CRC is started
-      async: 1800
-      poll: 0
-      register: _crc_start
-      ansible.builtin.command:
-        cmd: crc start
+    - name: Start CRC
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml
   roles:
     - role: test_deps
     - role: ci_setup
     - role: install_yamls
-  tasks:
-    - name: Check for CRC status
-      ansible.builtin.async_status:
-        jid: "{{ _crc_start.ansible_job_id }}"
-      register: _crc_status
-      until: _crc_status.finished
-      retries: 100
-      delay: 10

roles/manage_secrets/molecule/nova_migration/prepare.yml

@@ -17,21 +17,11 @@
 - name: Prepare
   hosts: all
   pre_tasks:
-    - name: Ensure CRC is started
-      async: 1800
-      poll: 0
-      register: _crc_start
-      ansible.builtin.command:
-        cmd: crc start
+    - name: Start CRC
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml
   roles:
     - role: test_deps
     - role: ci_setup
     - role: install_yamls
-  tasks:
-    - name: Check for CRC status
-      ansible.builtin.async_status:
-        jid: "{{ _crc_start.ansible_job_id }}"
-      register: _crc_status
-      until: _crc_status.finished
-      retries: 100
-      delay: 10

roles/manage_secrets/molecule/osp_secrets/prepare.yml

@@ -17,21 +17,11 @@
 - name: Prepare
   hosts: all
   pre_tasks:
-    - name: Ensure CRC is started
-      async: 1800
-      poll: 0
-      register: _crc_start
-      ansible.builtin.command:
-        cmd: crc start
+    - name: Start CRC
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml
   roles:
     - role: test_deps
     - role: ci_setup
     - role: install_yamls
-  tasks:
-    - name: Check for CRC status
-      ansible.builtin.async_status:
-        jid: "{{ _crc_start.ansible_job_id }}"
-      register: _crc_status
-      until: _crc_status.finished
-      retries: 100
-      delay: 10

roles/openshift_login/molecule/default/prepare.yml

@@ -23,6 +23,7 @@
     - role: install_yamls
 
   tasks:
-    - name: Ensure CRC is started
-      ansible.builtin.command:
-        cmd: crc start
+    - name: Start CRC
+      ansible.builtin.include_role:
+        name: cifmw_helpers
+        tasks_from: crc_start.yml