Support Application Credentials in shiftstack-qa automation

collection/stages/roles/day2ops/defaults/main.yml

@@ -2,3 +2,6 @@
 # defaults file for day2ops
 day2ops_steps: []
 day2ops_report_filename: shiftstack-qa-day2ops-results.xml
+
+# Application Credentials rotation
+app_credential_name: "AppCreds-{{ user_cloud }}"

collection/stages/roles/day2ops/tasks/procedures/rotate_app_creds.yml

@@ -0,0 +1,45 @@
+---
+# Source: https://github.com/shiftstack/installer/blob/master/docs/user/openstack/README.md#openstack-credentials-update
+- name: Restore user/password auth to ensure full permissions for app credential creation
+  ansible.builtin.include_role:
+    name: shiftstack.stages.prepare
+    tasks_from: clouds.yml
+
+- name: Create application credentials and update clouds.yaml
+  ansible.builtin.include_role:
+    name: shiftstack.stages.prepare
+    tasks_from: app_creds.yml
+
+- name: Rotate OpenShift Cloud Credentials
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    cat {{ clouds_yaml_file_path }} | sed 's/{{ user_cloud }}:/openstack:/' | \
+    oc set data -n kube-system secret/openstack-credentials clouds.yaml=-
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  changed_when: true
+
+- name: Get OpenStack Credentials from OCP cluster
+  ansible.builtin.shell: |
+    set -o pipefail && \
+    oc get secret -n kube-system openstack-credentials -o json | jq -r '.data."clouds.yaml"' | base64 -d
+  environment:
+    KUBECONFIG: "{{ kubeconfig }}"
+  register: ocp_creds_output
+  changed_when: false
+
+- name: Parse OCP credentials
+  ansible.builtin.set_fact:
+    ocp_creds: "{{ ocp_creds_output.stdout | from_yaml }}"
+
+- name: Verify credentials rotated to application credentials
+  ansible.builtin.assert:
+    that:
+      - ocp_creds.clouds.openstack.auth_type == 'v3applicationcredential'
+    fail_msg: "Credential rotation failed — auth_type is not v3applicationcredential"
+    success_msg: "Credential rotation verified — auth_type is v3applicationcredential"
+
+- name: Wait until the Cluster Operators are healthy
+  ansible.builtin.include_role:
+    name: tools_cluster_checks
+    tasks_from: wait_until_cluster_operators_ready.yml

collection/stages/roles/prepare/defaults/main.yml

@@ -21,6 +21,10 @@ project:
     load_balancers: 1000
     load_balancer_listeners: 5000
     load_balancer_pools: 5000
+# Application Credentials
+use_application_credentials: false
+app_credential_name: "AppCreds-{{ user_cloud }}"
+
 ocp_api_description: "API {{ ocp_cluster_name }}.{{ ocp_base_domain }}"
 ocp_apps_description: "APPS {{ ocp_cluster_name }}.{{ ocp_base_domain }}"
 ocp_bootstrap_fip_description: "Bootstrap {{ ocp_cluster_name }}.{{ ocp_base_domain }}"

collection/stages/roles/prepare/tasks/app_creds.yml

@@ -0,0 +1,76 @@
+---
+- name: Delete stale temporary Application Credential from interrupted previous run
+  ansible.builtin.shell: |
+    openstack application credential delete {{ app_credential_name }}-new
+  environment:
+    OS_CLOUD: "{{ user_cloud }}"
+  failed_when: false
+  changed_when: false
+
+- name: Create new Application Credential
+  ansible.builtin.shell: |
+    openstack application credential create \
+      --description "App Creds - All roles" \
+      "{{ app_credential_name }}-new" -f yaml
+  environment:
+    OS_CLOUD: "{{ user_cloud }}"
+  register: app_cred_output
+  changed_when: true
+
+- name: Parse Application Credential output
+  ansible.builtin.set_fact:
+    app_cred_info: "{{ app_cred_output.stdout | from_yaml }}"
+
+- name: Read current clouds.yaml
+  ansible.builtin.slurp:
+    src: "{{ clouds_yaml_file_path }}"
+  register: clouds_yaml_file
+
+- name: Set clouds.yaml fact
+  ansible.builtin.set_fact:
+    clouds_yaml_params: "{{ clouds_yaml_file.content | b64decode | from_yaml }}"
+
+- name: Build updated cloud entry with application credentials
+  ansible.builtin.set_fact:
+    updated_cloud_entry:
+      auth:
+        auth_url: "{{ clouds_yaml_params.clouds[user_cloud].auth.auth_url }}"
+        application_credential_id: "{{ app_cred_info.id }}"
+        application_credential_secret: "{{ app_cred_info.secret }}"
+      auth_type: v3applicationcredential
+      identity_api_version: "{{ clouds_yaml_params.clouds[user_cloud].identity_api_version | default('3') }}"
+      region_name: "{{ clouds_yaml_params.clouds[user_cloud].region_name | default(omit) }}"
+
+- name: Add cacert to updated cloud entry
+  ansible.builtin.set_fact:
+    updated_cloud_entry: "{{ updated_cloud_entry | combine({'cacert': clouds_yaml_params.clouds[user_cloud].cacert}, recursive=True) }}"
+  when: clouds_yaml_params.clouds[user_cloud].cacert is defined
+
+- name: Update clouds.yaml with application credentials
+  ansible.builtin.set_fact:
+    clouds_yaml_params: "{{ {'clouds': (clouds_yaml_params.clouds | combine({user_cloud: updated_cloud_entry}))} }}"
+
+- name: Write updated clouds.yaml
+  ansible.builtin.copy:
+    content: "{{ clouds_yaml_params | to_nice_yaml(indent=4) }}"
+    dest: "{{ clouds_yaml_file_path }}"
+    mode: u=rw,g=rw,o=r
+
+- name: Update clouds.yaml copy in osp_config_dir
+  ansible.builtin.copy:
+    content: "{{ clouds_yaml_params | to_nice_yaml(indent=4) }}"
+    dest: "{{ osp_config_dir }}/clouds.yaml"
+    mode: u=rw,g=rw,o=r
+  when: osp_config_dir is defined
+
+- name: Validate new application credentials work
+  openstack.cloud.auth:
+    cloud: "{{ user_cloud }}"
+
+- name: Delete previous Application Credential
+  ansible.builtin.shell: |
+    openstack application credential delete {{ app_credential_name }}
+  environment:
+    OS_CLOUD: "{{ user_cloud }}"
+  failed_when: false
+  changed_when: false

collection/stages/roles/prepare/tasks/main.yml

@@ -32,6 +32,10 @@
 - name: Update clouds.yml file with new Project
   ansible.builtin.include_tasks: clouds.yml
 
+- name: Configure Application Credentials for OpenStack authentication
+  ansible.builtin.include_tasks: app_creds.yml
+  when: use_application_credentials | default(false)
+
 - name: Restricted Network Preparations
   ansible.builtin.include_tasks: restricted_network.yml
   when:

collection/tools/roles/tools_get_openshift_release/defaults/main.yml

@@ -2,6 +2,5 @@
 # defaults file for tools_get_openshift_release
 openshift_releasestream_url: "https://openshift-release.apps.ci.l2s4.p1.openshiftapps.com/api/v1/releasestream"
 release_name: "{{ openshift_release_build_name | default('') }}"
-openshift_download_url: "{{ 'https://openshift-release-artifacts.apps.ci.l2s4.p1.openshiftapps.com' + '/' + release_name }}"
 openshift_mirror_url: "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp"
 ocp_build_info_file: "{{ controller_home_dir }}/latest_build.json"

collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_binaries.yml

@@ -1,44 +1,153 @@
 ---
+# Extract OCP installer and/or client binaries directly from the release image
+# using `oc adm release extract --tools` instead of the release-controller's
+# file-cache (openshift-release-artifacts), which has no SLA and can get stuck
+# indefinitely during tool extraction.
+#
+# The pull secret is extracted from the host cluster via the Kubernetes API
+# using the kubeconfig's client certificate. If `oc` is not already present
+# in the pod (cold-start), a stable client is bootstrapped from
+# mirror.openshift.com before running `oc adm release extract --tools`.
 - name: Get the OCP installer and/or client binaries
   vars:
-    installer_url: "{{ openshift_download_url }}/openshift-install-linux-{{ release_name }}.tar.gz"
-    client_url: "{{ openshift_download_url }}/openshift-client-linux-{{ release_name }}.tar.gz"
+    installer_tarball: "openshift-install-linux-{{ release_name }}.tar.gz"
+    client_tarball: "openshift-client-linux-{{ release_name }}.tar.gz"
+    pull_secret_file: "{{ home_dir }}/pull-secret.json"
+    bootstrap_oc_dir: "{{ home_dir }}/bootstrap-oc"
+    bootstrap_oc_url: "{{ openshift_mirror_url }}/stable/openshift-client-linux.tar.gz"
   block:
     - name: Fail if release_name var is not defined
       ansible.builtin.fail:
         msg: "'release_name' variable must be defined and cannot be empty"
       when: release_name == ''
 
-    - name: Wait for content to come up on {{ openshift_download_url }}
-      ansible.builtin.uri:
-        url: "{{ openshift_download_url }}"
-        method: GET
-        return_content: yes
-        status_code: 200
-        body_format: json
-      register: result
-      until: result.content.find("openshift-install-linux") != -1
-      retries: 20
-      delay: 60
+    - name: Fail if openshift_release_pull_spec is not defined
+      ansible.builtin.fail:
+        msg: "'openshift_release_pull_spec' must be set by get_openshift_release_build_name.yml"
+      when: openshift_release_pull_spec is not defined or openshift_release_pull_spec == ''
+
+    - name: Extract pull secret from host cluster via Kubernetes API
+      ansible.builtin.shell: |
+        python3 << 'PYEOF'
+        import yaml, json, base64, subprocess, os, sys, tempfile
+
+        kubeconfig_path = "{{ rhoso_kubeconfig }}"
+        output_path = "{{ pull_secret_file }}"
+
+        with open(kubeconfig_path) as f:
+            kc = yaml.safe_load(f)
+
+        server = kc['clusters'][0]['cluster']['server']
+        user = kc['users'][0]['user']
+
+        try:
+            cert_data = user['client-certificate-data']
+            key_data = user['client-key-data']
+        except KeyError:
+            print(f"rhoso_kubeconfig must use client-certificate auth, "
+                  f"found auth keys: {list(user.keys())}", file=sys.stderr)
+            sys.exit(1)
+
+        with tempfile.TemporaryDirectory() as tmpdir:
+            ca_path = os.path.join(tmpdir, 'ca.crt')
+            cert_path = os.path.join(tmpdir, 'client.crt')
+            key_path = os.path.join(tmpdir, 'client.key')
+
+            with open(ca_path, 'wb') as f:
+                f.write(base64.b64decode(kc['clusters'][0]['cluster']['certificate-authority-data']))
+            with open(cert_path, 'wb') as f:
+                f.write(base64.b64decode(cert_data))
+            with open(key_path, 'wb') as f:
+                f.write(base64.b64decode(key_data))
+
+            result = subprocess.run([
+                'curl', '-s', '--fail',
+                '--cacert', ca_path,
+                '--cert', cert_path,
+                '--key', key_path,
+                f'{server}/api/v1/namespaces/openshift-config/secrets/pull-secret'
+            ], capture_output=True, text=True)
+
+        if result.returncode != 0:
+            print(f"Failed to fetch pull secret from {server}: {result.stderr}", file=sys.stderr)
+            sys.exit(1)
+
+        data = json.loads(result.stdout)
+        decoded = base64.b64decode(data['data']['.dockerconfigjson']).decode()
+        auths = json.loads(decoded)
+
+        with open(output_path, 'w') as f:
+            f.write(decoded)
+
+        print(f"Pull secret extracted: {len(auths.get('auths', {}))} registries")
+        PYEOF
+      register: _pull_secret_result
+      no_log: true
+
+    - name: Verify pull secret file is valid
+      ansible.builtin.shell: >-
+        python3 -c "import json; d=json.load(open('{{ pull_secret_file }}'));
+        print(len(d.get('auths',{})), 'registries found')"
+      register: _pull_secret_verify
+      changed_when: false
+
+    - name: Check if oc is already available
+      ansible.builtin.command: which oc
+      register: _oc_available
+      ignore_errors: true
+      changed_when: false
+
+    - name: Bootstrap oc client from {{ bootstrap_oc_url }}
+      when: _oc_available is failed
+      block:
+        - name: Create bootstrap directory
+          ansible.builtin.file:
+            path: "{{ bootstrap_oc_dir }}"
+            state: directory
+            mode: u=rwx,g=rw,o=r
+
+        - name: Download stable oc client from mirror
+          ansible.builtin.unarchive:
+            src: "{{ bootstrap_oc_url }}"
+            dest: "{{ bootstrap_oc_dir }}"
+            remote_src: yes
+          register: _bootstrap_download
+          until: _bootstrap_download is not failed
+          retries: 3
+          delay: 10
+
+    - name: Set oc binary path
+      ansible.builtin.set_fact:
+        _oc_bin: "{{ (bootstrap_oc_dir + '/oc') if _oc_available is failed else 'oc' }}"
 
     - name: Create the installer directory
       ansible.builtin.file:
         path: "{{ home_dir }}/{{ release_name }}"
         state: directory
         mode: u=rwx,g=rw,o=r
 
+    - name: Extract OCP tools from release image {{ openshift_release_pull_spec }}
+      ansible.builtin.command:
+        cmd: >-
+          timeout 900
+          {{ _oc_bin }} adm release extract
+          --tools
+          --registry-config={{ pull_secret_file }}
+          --to={{ home_dir }}/{{ release_name }}
+          {{ openshift_release_pull_spec }}
+      register: extract_result
+      until: extract_result is not failed
+      retries: 3
+      delay: 30
+
     - name: Get the installer binary and create a symlink
       when: "'installer' in binaries"
       block:
-        - name: Download and unarchive the installer from {{ installer_url }}
+        - name: Unarchive the installer from {{ installer_tarball }}
           ansible.builtin.unarchive:
-            src: "{{ installer_url }}"
+            src: "{{ home_dir }}/{{ release_name }}/{{ installer_tarball }}"
             dest: "{{ home_dir }}/{{ release_name }}"
             remote_src: yes
-          register: result
-          until: result is not failed
-          retries: 3
-          delay: 10
 
         - name: Create a symlink to the openshift-install binary from /usr/local/bin
           ansible.builtin.file:
@@ -47,18 +156,14 @@
             state: link
           become: true
 
-    - name: Get the installer binary and create symlinks
+    - name: Get the client binary and create symlinks
       when: "'client' in binaries"
       block:
-        - name: Download and unarchive the client from {{ client_url }}
+        - name: Unarchive the client from {{ client_tarball }}
           ansible.builtin.unarchive:
-            src: "{{ client_url }}"
+            src: "{{ home_dir }}/{{ release_name }}/{{ client_tarball }}"
             dest: "{{ home_dir }}/{{ release_name }}"
             remote_src: yes
-          register: result
-          until: result is not failed
-          retries: 3
-          delay: 10
 
         - name: Create a symlink to the oc binary from /usr/local/bin
           ansible.builtin.file:
@@ -73,3 +178,14 @@
             dest: /usr/bin/kubectl
             state: link
           become: true
+
+  always:
+    - name: Remove pull secret file
+      ansible.builtin.file:
+        path: "{{ pull_secret_file }}"
+        state: absent
+
+    - name: Remove bootstrap oc directory
+      ansible.builtin.file:
+        path: "{{ bootstrap_oc_dir }}"
+        state: absent