Fix issues

Signed-off-by: Joshua Yao <yi.a.yao@intel.com>

Author: joshuayao

AudioQnA/docker_compose/intel/cpu/xeon/prometheus.yaml

@@ -17,11 +17,13 @@ scrape_configs:
     metrics_path: /metrics
     static_configs:
       - targets: ["tgi-service:80"]
-  - job_name: "docsum-backend-server"
+  - job_name: "audioqna-backend-server"
     metrics_path: /metrics
     static_configs:
       - targets: ["audioqna-xeon-backend-server:8888"]
   - job_name: "prometheus-node-exporter"
+    scrape_interval: 30s
+    scrape_timeout: 25s
     metrics_path: /metrics
     static_configs:
       - targets: ["node-exporter:9100"]

AudioQnA/docker_compose/intel/cpu/xeon/set_env.sh

@@ -27,3 +27,5 @@ export BACKEND_SERVICE_ENDPOINT=http://${host_ip}:3008/v1/audioqna
 pushd "${SCRIPT_DIR}/grafana/dashboards" > /dev/null
 source download_opea_dashboard.sh
 popd > /dev/null
+
+export no_proxy="${no_proxy},localhost,127.0.0.1,${host_ip},node-exporter,opea_prometheus,grafana"

AudioQnA/docker_compose/intel/hpu/gaudi/compose.monitoring.yaml

@@ -47,7 +47,7 @@ services:
     command:
       - '--path.procfs=/host/proc'
       - '--path.sysfs=/host/sys'
-      - --collector.filesystem.ignored-mount-points
+      - --collector.filesystem.mount-points-exclude
       - "^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)"
     environment:
       no_proxy: ${no_proxy}
@@ -57,3 +57,18 @@ services:
     restart: always
     deploy:
       mode: global
+
+  gaudi-metrics-exporter:
+    image: vault.habana.ai/gaudi-metric-exporter/metric-exporter:latest
+    privileged: true
+    container_name: gaudi-metrics-exporter
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+      - /dev:/dev
+    deploy:
+      mode: global
+    ports:
+      - 41611:41611
+    restart: unless-stopped

AudioQnA/docker_compose/intel/hpu/gaudi/grafana/dashboards/download_opea_dashboard.sh

@@ -2,11 +2,12 @@
 # Copyright (C) 2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-# if ls *.json 1> /dev/null 2>&1; then
-#     rm *.json
-# fi
+if ls *.json 1> /dev/null 2>&1; then
+    rm *.json
+fi
 
 wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/vllm_grafana.json
 wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/tgi_grafana.json
-# wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/audioqna_megaservice_grafana.json
+wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/gaudi_grafana_v2.json
+wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/audioqna_megaservice_grafana.json
 wget https://raw.githubusercontent.com/opea-project/GenAIEval/refs/heads/main/evals/benchmark/grafana/node_grafana.json

AudioQnA/docker_compose/intel/hpu/gaudi/prometheus.yaml

@@ -12,16 +12,23 @@ scrape_configs:
   - job_name: "vllm"
     metrics_path: /metrics
     static_configs:
-      - targets: ["vllm-service:80"]
+      - targets: ["vllm-gaudi-service:80"]
   - job_name: "tgi"
     metrics_path: /metrics
     static_configs:
-      - targets: ["tgi-service:80"]
-  - job_name: "docsum-backend-server"
+      - targets: ["tgi-gaudi-service:80"]
+  - job_name: "audioqna-backend-server"
     metrics_path: /metrics
     static_configs:
       - targets: ["audioqna-gaudi-backend-server:8888"]
   - job_name: "prometheus-node-exporter"
+    scrape_interval: 30s
+    scrape_timeout: 25s
     metrics_path: /metrics
     static_configs:
       - targets: ["node-exporter:9100"]
+  - job_name: "gaudi-metrics-exporter"
+    scrape_interval: 30s
+    metrics_path: /metrics
+    static_configs:
+      - targets: ["gaudi-metrics-exporter:41611"]

AudioQnA/docker_compose/intel/hpu/gaudi/set_env.sh

@@ -32,3 +32,5 @@ export BACKEND_SERVICE_ENDPOINT=http://${host_ip}:3008/v1/audioqna
 pushd "${SCRIPT_DIR}/grafana/dashboards" > /dev/null
 source download_opea_dashboard.sh
 popd > /dev/null
+
+export no_proxy="${no_proxy},localhost,127.0.0.1,${host_ip},node-exporter,opea_prometheus,grafana"

AudioQnA/tests/test_compose_on_gaudi.sh

@@ -43,7 +43,7 @@ function start_services() {
     docker compose -f compose.yaml -f compose.monitoring.yaml up -d > ${LOG_PATH}/start_services_with_compose.log
     n=0
     until [[ "$n" -ge 200 ]]; do
-       docker logs vllm-gaudi-service > $LOG_PATH/vllm_service_start.log 2>&1
+       docker logs vllm-gaudi-service 2>&1| tee $LOG_PATH/vllm_service_start.log 
        if grep -q complete $LOG_PATH/vllm_service_start.log; then
            break
        fi
@@ -53,8 +53,8 @@ function start_services() {
 
     n=0
     until [[ "$n" -ge 100 ]]; do
-       docker logs whisper-service > $LOG_PATH/whisper_service_start.log
-       if grep -q "Uvicorn server setup on port" $LOG_PATH/whisper_service_start.log; then
+       docker logs whisper-service 2>&1| tee $LOG_PATH/whisper_service_start.log
+       if grep -q "Uvicorn running on" $LOG_PATH/whisper_service_start.log; then
            break
        fi
        sleep 5s
@@ -65,16 +65,15 @@ function start_services() {
 
 function validate_megaservice() {
     response=$(http_proxy="" curl http://${ip_address}:3008/v1/audioqna -XPOST -d '{"audio": "UklGRigAAABXQVZFZm10IBIAAAABAAEARKwAAIhYAQACABAAAABkYXRhAgAAAAEA", "max_tokens":64}' -H 'Content-Type: application/json')
-    # always print the log
-    docker logs whisper-service > $LOG_PATH/whisper-service.log
-    docker logs speecht5-service > $LOG_PATH/tts-service.log
-    docker logs vllm-gaudi-service > $LOG_PATH/vllm-gaudi-service.log
-    docker logs audioqna-gaudi-backend-server > $LOG_PATH/audioqna-gaudi-backend-server.log
     echo "$response" | sed 's/^"//;s/"$//' | base64 -d > speech.mp3
 
     if [[ $(file speech.mp3) == *"RIFF"* ]]; then
         echo "Result correct."
     else
+        docker logs whisper-service > $LOG_PATH/whisper-service.log
+        docker logs speecht5-service > $LOG_PATH/tts-service.log
+        docker logs vllm-gaudi-service > $LOG_PATH/vllm-gaudi-service.log
+        docker logs audioqna-gaudi-backend-server > $LOG_PATH/audioqna-gaudi-backend-server.log    
         echo "Result wrong."
         exit 1
     fi

AudioQnA/tests/test_compose_tgi_on_gaudi.sh

@@ -52,7 +52,7 @@ function start_services() {
     n=0
     until [[ "$n" -ge 100 ]]; do
        docker logs whisper-service > $LOG_PATH/whisper_service_start.log
-       if grep -q "Uvicorn server setup on port" $LOG_PATH/whisper_service_start.log; then
+       if grep -q "Uvicorn running on" $LOG_PATH/whisper_service_start.log; then
            break
        fi
        sleep 5s