Merge branch 'master' into alexy_ck_job_6

Author: ayefimov-1

roles/telemetry_chargeback/README.md

@@ -41,6 +41,9 @@ These variables can be overridden when importing the role or set at the play lev
 | `cloudkitty_debug_dir` | `"{{ (cloudkitty_debug \| bool) \| ternary(artifacts_dir_zuul + '/debug_ck_db', '') }}"` | Directory for debug output (auto-set based on debug flag) |
 | `logs_dir_zuul` | `"{{ cifmw_basedir }}/logs"` | Directory for log files |
 | `artifacts_dir_zuul` | `"{{ cifmw_basedir }}/artifacts"` | Directory for generated artifacts and test output |
+| `cert_dir` | `"{{ cifmw_basedir }}/ck-certs"` | Directory for CloudKitty client certificates |
+| `local_cert_dir` | `"{{ cifmw_basedir }}/flush_certs"` | Local directory for flush certificates (cleaned up after run) |
+| `remote_cert_dir` | `"osp-certs"` | Remote directory inside OpenStack pod for certificates |
 | `cert_secret_name` | `"cert-cloudkitty-client-internal"` | OpenShift secret name for client certificates |
 | `client_secret` | `"secret/cloudkitty-lokistack-gateway-client-http"` | Secret for flush client certificates |
 | `ca_configmap` | `"cm/cloudkitty-lokistack-ca-bundle"` | ConfigMap for CA bundle |
@@ -49,7 +52,6 @@ These variables can be overridden when importing the role or set at the play lev
 | `openstackpod` | `"openstackclient"` | OpenStack client pod name for exec/cp operations |
 | `lookback` | `6` | Days to look back for Loki query time range |
 | `limit` | `50` | Limit for Loki query results |
-| `cloudkitty_test_scenarios` | `[]` | List of test scenario files to run (empty = auto-discover) |
 
 How It Works
 ------------
@@ -67,8 +69,7 @@ The role executes the following workflow:
    - Configures Loki push/query URLs
 
 3. **Test Scenario Discovery**
-   - **Auto-discovery** (default): Finds all `test_*.yml` files in `files/` directory
-   - **User-provided**: Uses scenarios from `cloudkitty_test_scenarios` variable
+   - Automatically finds all `test_*.yml` files in `files/` directory
 
 4. **Scenario Execution Loop** (for each discovered scenario)
    - Generates synthetic Loki log data (`gen_synth_loki_data.py`)
@@ -79,9 +80,6 @@ The role executes the following workflow:
    - Removes temporary certificate directories
    - Always runs (even on failure) via block/rescue/always structure
 
-### Loop Variable
-
-The role uses `{{ scenario_name }}` as the loop variable when processing multiple test scenarios, making it easy to track which scenario is currently executing.
 
 Python Scripts
 --------------

roles/telemetry_chargeback/tasks/ingest_loki_data.yml

@@ -1,23 +1,32 @@
 ---
 # Ingest data generated by gen_synth_loki_data.yml to Loki
 
-- name: "TEST Push data to Loki"
-  ansible.builtin.uri:
-    ## loki_push_url will be used in future role developement
-    url: "{{ loki_push_url }}"
-    method: POST
-    src: "{{ cloudkitty_data_file }}"
-    body_format: json
-    client_cert: "{{ cert_dir }}/tls.crt"
-    client_key: "{{ cert_dir }}/tls.key"
-    validate_certs: false
-    status_code: 204
-    return_content: true
-  register: loki_response
-  ignore_errors: true
-  failed_when: loki_response.status != 204
+- name: "Ingest data log to Loki via API"
+  block:
 
-- name: "Ingest Status Message"
-  ansible.builtin.debug:
-    msg: "Ingestion Successful"
-  when: loki_response.status | default(0) == 204
+    - name: "Push data to Loki"
+      ansible.builtin.uri:
+        ## loki_push_url will be used in future role developement
+        url: "{{ loki_push_url }}"
+        method: POST
+        src: "{{ cloudkitty_data_file }}"
+        body_format: json
+        client_cert: "{{ cert_dir }}/tls.crt"
+        client_key: "{{ cert_dir }}/tls.key"
+        validate_certs: false
+        status_code: 204
+        return_content: true
+      register: loki_response
+      ignore_errors: false
+      failed_when: loki_response.status != 204
+
+    # Success
+    - name: "Confirm ingestion success"
+      ansible.builtin.debug:
+        msg: "Ingestion Successful!"
+
+  rescue:
+    # Rescue block
+    - name: "Debug failure"
+      ansible.builtin.debug:
+        msg: "Ingest failed. Loki Status: {{ loki_response.status | default('N/A') }}"

roles/telemetry_chargeback/tasks/load_loki_data.yml

@@ -2,8 +2,4 @@
 - name: "Ingest CloudKitty data log for {{ scenario_name }}"
   ansible.builtin.include_tasks: ingest_loki_data.yml
 
-- name: "Flush data to Loki storage for {{ scenario_name }}"
-  ansible.builtin.include_tasks: flush_loki_data.yml
-
-- name: "Retrieve data log from Loki for {{ scenario_name }}"
-  ansible.builtin.include_tasks: retrieve_loki_data.yml
+  ### more task files in future PRs

roles/telemetry_chargeback/tasks/main.yml

@@ -5,34 +5,24 @@
 - name: "Setup Loki Environment"
   ansible.builtin.include_tasks: "setup_loki_env.yml"
 
-- name: "Auto-discover test scenarios"
-  when: cloudkitty_test_scenarios | length == 0
-  block:
-    - name: "Find test files"
-      ansible.builtin.find:
-        paths: "{{ cloudkitty_scenario_dir }}"
-        patterns: "test_*.yml"
-      register: test_files_raw
-
-    - name: "Set scenario list from discovered files"
-      ansible.builtin.set_fact:
-        cloudkitty_test_scenario_names: "{{ test_files_raw.files | map(attribute='path') | map('basename') | map('regex_replace', '\\.yml$', '') | list }}"
+- name: "Find test files"
+  ansible.builtin.find:
+    paths: "{{ cloudkitty_scenario_dir }}"
+    patterns: "test_*.yml"
+  register: found_files_raw
 
-- name: "Set scenario list from user-provided variable"
+- name: "Extract only the filenames into a clean list"
   ansible.builtin.set_fact:
-    cloudkitty_test_scenario_names: "{{ cloudkitty_test_scenarios }}"
-  when: cloudkitty_test_scenarios | length > 0
+    found_files: "{{ found_files_raw.files | map(attribute='path') | map('basename') | map('regex_replace', '\\.yml$', '') | list }}"
 
 - name: "Run scenario file through workflow"
-  when: cloudkitty_test_scenario_names | length > 0
-
   block:
     - name: "Process and Loop if files exist"
       ansible.builtin.include_tasks: run_test_scenarios.yml
-      loop: "{{ cloudkitty_test_scenario_names }}"
+      loop: "{{ found_files }}"
       loop_control:
         loop_var: scenario_name
-      when: cloudkitty_test_scenario_names | length > 0
+      when: found_files | length > 0
 
   rescue:
     - name: "Log failure"

roles/telemetry_chargeback/tasks/setup_loki_env.yml

@@ -30,15 +30,12 @@
       - "Flush URL:  {{ ingester_flush_url }}"
       - "Query URL:  {{ loki_query_url }}"
 
-- name: "Ensure Certificate Directories Exist"
+# Certs to Ingest & Retrieve data to/from Loki
+- name: "Ensure Local Certificate Directory Exists"
   ansible.builtin.file:
-    path: "{{ item }}"
+    path: "{{ cert_dir }}"
     state: directory
-    mode: '0700'  # Restricted for security
-  loop:
-    - "{{ cert_dir }}"
-    - "{{ local_cert_dir }}"
-  when: item is defined
+    mode: '0755'
 
 - name: "Extract Certificates from OpenShift Secret"
   ansible.builtin.command:

roles/telemetry_chargeback/vars/main.yml

@@ -1,6 +1,4 @@
 ---
-# Scenario and script paths (using role_path)
-cloudkitty_scenario_dir: "{{ role_path }}/files"
 cloudkitty_synth_script: "{{ role_path }}/files/gen_synth_loki_data.py"
 cloudkitty_data_template: "{{ role_path }}/templates/loki_data_templ.j2"
 cloudkitty_summary_script: "{{ role_path }}/files/gen_db_summary.py"