Add periodic job with volume pool metrics tests (#287)

To check volume pool metrics, we need a correct environment
configured with a storage backend - in this case ceph.
The job takes abet 20 minutes longer than our usual jobs,
so it was decided to run it as part of the periodic pipeline
instead of running it with each PR in the check pipeline.

Author: vyzigold

.zuul.yaml

@@ -127,8 +127,45 @@
     required-projects: *required_projects
     extra-vars: *functional_autoscaling_extra_vars
 
+- job:
+    name: functional-periodic-telemetry-with-ceph
+    parent: podified-multinode-hci-deployment-crc-1comp-backends
+    dependencies: ["telemetry-openstack-meta-content-provider-master"]
+    description: |
+      Deploy OpenStack with Telemetry and Ceph. Run metric-verification with volume pool metric tests
+    extra-vars: *functional_autoscaling_extra_vars
+    vars:
+      patch_observabilityclient: true
+      cifmw_update_containers: false
+      telemetry_verify_metrics_metric_sources_to_test:
+        - ceilometer_compute_agent
+        - ceilometer_central_agent
+        - ksm
+        - node_exporter
+        - podman_exporter
+        - rabbitmq
+        - openstack_network_exporter
+        - kepler
+        - mariadb
+        - volume_pool
+      cifmw_extras:
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/multinode-ci.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/ci-framework'].src_dir  }}/scenarios/centos-9/hci_ceph_backends.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-telemetry-with-ceph.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-logging-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/openstack-k8s-operators/telemetry-operator'].src_dir }}/ci/vars-autoscaling-tempest.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-functional-test.yml"
+        - "@{{ ansible_user_dir }}/{{ zuul.projects['github.com/infrawatch/feature-verification-tests'].src_dir }}/ci/vars-use-master-containers.yml"
+    irrelevant-files: *irrelevant_files
+    required-projects: *required_projects
+
 - project:
     name: infrawatch/feature-verification-tests
+    periodic:
+      jobs:
+        - telemetry-openstack-meta-content-provider-master:
+            override-checkout: main
+        - functional-periodic-telemetry-with-ceph
     github-check:
       jobs:
         - telemetry-openstack-meta-content-provider-master:
@@ -169,3 +206,11 @@
               - ci/vars-graphing-test.yml
               - ci/run_graphing_test.yml
               - ci/report_result.yml
+        - functional-periodic-telemetry-with-ceph:
+            files:
+              # Run this job for changes to the volume_pool_metrics test changes as this is
+              # the only job, where those tests get executed and for changes to zuul.yaml in
+              # case the job definition changes.
+              - roles/telemetry_verify_metrics/tasks/verify_ceilometer_volume_pool_metrics.yml
+              - .zuul.yaml
+

roles/telemetry_verify_metrics/README.md

@@ -35,7 +35,7 @@ Tests:
 Role Variables
 --------------
 openstack\_cmd - command to access openstack cli. For example: "oc rsh openstackclient openstack"
-telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, ceilometer\_ipmi_\_agent, node\_exporter, rabbitmq, kepler, openstack_network_exporter
+telemetry\_verify\_metrics\_metric\_sources\_to\_test - List of sources to test. Current set of possible sources: ceilometer\_compute\_agent, ceilometer\_central\_agent, ceilometer\_ipmi_\_agent, node\_exporter, rabbitmq, kepler, openstack\_network\_exporter, volume\_pool
 
 Example Playbook
 ----------------

roles/telemetry_verify_metrics/defaults/main.yml

@@ -12,3 +12,5 @@ telemetry_verify_metrics_metric_sources_to_test:
   - mariadb
   # NOTE: IPMI metrics can only be verified when compute nodes are baremetal
   # - ceilometer_ipmi_agent
+  # NOTE: volume pool metrics can only be verified when a pool with a correct backend is configured
+  # - volume_pool

roles/telemetry_verify_metrics/tasks/main.yml

@@ -108,3 +108,9 @@
   when: '"kepler_vm" in telemetry_verify_metrics_metric_sources_to_test and "ACTIVE" in kepler_vm_status.stdout'
   vars:
     metrics_cmd_prefix: "{{ openstack_cmd }} metric show --disable-rbac -c value -f value"
+
+- name: Verify Ceilometer volume pool metrics are being exposed and stored
+  ansible.builtin.include_tasks:
+    file: verify_ceilometer_volume_pool_metrics.yml
+  tags: test
+  when: '"volume_pool" in telemetry_verify_metrics_metric_sources_to_test'

roles/telemetry_verify_metrics/tasks/verify_ceilometer_volume_pool_metrics.yml

@@ -0,0 +1,40 @@
+- name: Verify ceilometer scrapeconfig exists
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_cr_list:
+      - kind: scrapeconfigs.monitoring.rhobs
+        name: telemetry-ceilometer
+
+- name: Verify ceilometer central agent is running
+  ansible.builtin.include_role:
+    name: common
+  vars:
+    common_pod_status_str: "Running"
+    common_pod_nspace: openstack
+    common_pod_list:
+      - ceilometer-0
+
+- name: |
+    TEST Use openstack CLI to verify a cinder volume is configured
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} volume backend pool list
+  register: result
+  until: result.rc == 0 and result.stdout_lines | length > 0
+  changed_when: false
+
+- name: |
+    TEST Use openstack observabilityclient to verify ceilometer volume pool metrics are stored in prometheus
+  vars:
+    volume_pool_metrics:
+      - ceilometer_volume_provider_pool_capacity_allocated
+      - ceilometer_volume_provider_pool_capacity_free
+      - ceilometer_volume_provider_pool_capacity_total
+  ansible.builtin.shell: |
+    {{ openstack_cmd }} metric show {{ item }}
+  register: result
+  delay: 30
+  retries: 10
+  until: result.rc == 0 and item in result.stdout
+  changed_when: false
+  loop: "{{ volume_pool_metrics }}"