diff --git a/roles/telemetry_chargeback/tasks/gen_synth_loki_data.yml b/roles/telemetry_chargeback/tasks/gen_synth_loki_data.yml index ff3aa98b..3d4f939e 100644 --- a/roles/telemetry_chargeback/tasks/gen_synth_loki_data.yml +++ b/roles/telemetry_chargeback/tasks/gen_synth_loki_data.yml @@ -35,3 +35,10 @@ when: not file_preexists.stat.exists | bool changed_when: cloudkitty_synth_totals_file_output.rc == 0 any_errors_fatal: true + +# Load synthetic metrics summary for validation +- name: "Load synthetic metrics summary for {{ scenario_name }}" + ansible.builtin.include_vars: + file: "{{ cloudkitty_synth_totals_file }}" + name: synth_data_rates + when: cloudkitty_synth_totals_file_output is defined diff --git a/roles/telemetry_chargeback/tasks/load_loki_data.yml b/roles/telemetry_chargeback/tasks/load_loki_data.yml index 37c2aedb..9376dd78 100644 --- a/roles/telemetry_chargeback/tasks/load_loki_data.yml +++ b/roles/telemetry_chargeback/tasks/load_loki_data.yml @@ -4,3 +4,6 @@ - name: "Flush data to Loki storage for {{ scenario_name }}" ansible.builtin.include_tasks: flush_loki_data.yml + +- name: "Retrieve data log from Loki for {{ scenario_name }}" + ansible.builtin.include_tasks: retrieve_loki_data.yml diff --git a/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml b/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml new file mode 100644 index 00000000..56b139a7 --- /dev/null +++ b/roles/telemetry_chargeback/tasks/retrieve_loki_data.yml @@ -0,0 +1,71 @@ +--- +- name: "Expected Count {{ scenario_name }}" + ansible.builtin.debug: + msg: "Input file has {{ synth_data_rates.data_summary.log_count }} data entries that Loki has to return" + +# Query Loki +- name: "Retrieve Logs from Loki via API {{ scenario_name }}" + block: + - name: "Query Loki API" + ansible.builtin.uri: + url: "{{ loki_query_url }}?query={{ logql_query | urlencode }}&start={{ synth_data_rates.time.begin_step.nanosec }}&limit={{ limit }}" + method: GET + client_cert: "{{ cert_dir }}/tls.crt" + client_key: "{{ cert_dir }}/tls.key" + ca_path: "{{ cert_dir }}/ca.crt" + # TODO: Enable validate_certs in production environments + validate_certs: false + return_content: true + body_format: json + register: loki_response + # Wait for Loki to return all expected log entries + until: + - loki_response.status == 200 + - loki_response.json.status == 'success' + - loki_response.json.data.result | length > 0 + # Verify Loki returned all expected log entries + - (loki_response.json.data.result | map(attribute='values') | map('length') | sum) >= (synth_data_rates.data_summary.log_count | int) + retries: 20 + delay: 60 + + - name: "Save Loki Data to JSON file" + ansible.builtin.copy: + content: "{{ loki_response.json | to_json }}" + dest: "{{ artifacts_dir_zuul }}/{{ scenario_name }}{{ cloudkitty_loki_data_suffix }}" + mode: '0644' + + # Validate + - name: "Verify Data Integrity {{ scenario_name }}" + vars: + actual_count: "{{ loki_response.json.data.result | map(attribute='values') | map('length') | sum }}" + ansible.builtin.assert: + that: + - loki_response.json.status == 'success' + - loki_response.json.data.result | length > 0 + - actual_count | int == (synth_data_rates.data_summary.log_count | int) + fail_msg: >- + Query did not return all data entries. Expected + {{ synth_data_rates.data_summary.log_count }} log entries, but Loki + only returned {{ actual_count }} + success_msg: "Query returned all data entries. Input file had {{ synth_data_rates.data_summary.log_count }} entries and Loki returned {{ actual_count }}" + + # Only runs if retrieval was successful + - name: "Generate chargeback stats from Loki-retrieved data file: {{ scenario_name }}" + ansible.builtin.command: + cmd: > + python3 "{{ cloudkitty_summary_script }}" + -j "{{ artifacts_dir_zuul }}/{{ scenario_name }}{{ cloudkitty_loki_data_suffix }}" + -o "{{ artifacts_dir_zuul }}/{{ scenario_name }}{{ cloudkitty_loki_totals_metrics_suffix }}" + --debug "{{ cloudkitty_debug }}" + --debug_dir "{{ cloudkitty_debug_dir }}" + register: synth_rating_info + changed_when: synth_rating_info.rc == 0 + + rescue: + - name: "Report Retrieval Failure" + ansible.builtin.fail: + msg: + - "Failed to retrieve Loki data for {{ scenario_name }}" + - "Status: {{ loki_response.status | default('Unknown') }}" + - "Body: {{ loki_response.content | default('No Content') }}" + - "Message: {{ loki_response.msg | default('Request failed') }}"