Skip to content

Commit 51a399e

Browse files
committed
fix: enable Ascend NPU container-level metrics in exporter and chart display
Replace hardcoded return 0, nil with real PromQL queries (npu_chip_info_utilization, npu_chip_info_hbm_used_memory) for Ascend taskCoreUsed/taskMemoryUsed. Add AscendGPUDevice case to GenerateContainerMetrics switch for core unit conversion. Remove NVIDIA/MXC type guards in task detail view so all vendors show trend charts. Closes #86
1 parent c59f776 commit 51a399e

3 files changed

Lines changed: 115 additions & 17 deletions

File tree

packages/web/projects/vgpu/views/task/admin/Detail.vue

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -150,23 +150,18 @@
150150
</block-box>
151151

152152
<trend-time-filter
153-
v-if="detail.type && (detail.type.startsWith('NVIDIA') || detail.type.startsWith('MXC'))"
153+
v-if="detail.type"
154154
v-model="times"
155155
/>
156156

157157
<div class="task-trend-row">
158158
<block-box v-for="{ title, data } in lineConfigView" :key="title" :title="title">
159159
<div class="trend-chart">
160-
<template v-if="detail.type && !detail.type.startsWith('NVIDIA') && !detail.type.startsWith('MXC')">
161-
<el-empty :description="$t('task.noMonitorSupport')" :image-size="60" />
162-
</template>
163-
<template v-else>
164-
<VChart
165-
:option="getLineOptions({ data, seriesName: $t('dashboard.usageRateLegend'), animation: false })"
166-
:autoresize="true"
167-
class="trend-vchart"
168-
/>
169-
</template>
160+
<VChart
161+
:option="getLineOptions({ data, seriesName: $t('dashboard.usageRateLegend'), animation: false })"
162+
:autoresize="true"
163+
class="trend-vchart"
164+
/>
170165
</div>
171166
</block-box>
172167
</div>

server/internal/exporter/exporter.go

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -399,10 +399,13 @@ func (s *MetricsGenerator) GenerateContainerMetrics(ctx context.Context) error {
399399
case biz.HygonGPUDevice:
400400
used = float64(taskCoreUsed)
401401
util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core))
402-
case metax.MetaxSGPUDevice:
403-
used = float64(taskCoreUsed)
404-
util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core))
405-
default:
402+
case biz.AscendGPUDevice:
403+
used = float64(taskCoreUsed) / 100 * float64(core)
404+
util = float64(taskCoreUsed)
405+
case metax.MetaxSGPUDevice:
406+
used = float64(taskCoreUsed)
407+
util = roundToOneDecimal(100 * float64(taskCoreUsed) / float64(core))
408+
default:
406409
}
407410
cardCoreUtil, err := s.deviceCoreUtil(ctx, provider, device.Id)
408411
if err == nil && used != 0 && cardCoreUtil > 95 {
@@ -540,7 +543,7 @@ func (s *MetricsGenerator) taskCoreUsed(ctx context.Context, provider, namespace
540543
case biz.CambriconGPUDevice:
541544
query = fmt.Sprintf("avg(mlu_utilization * on(uuid) group_right mlu_container{namespace=\"%s\",pod=\"%s\",container=\"%s\",type=\"mlu370.smlu.vcore\"})", namespace, pod, container)
542545
case biz.AscendGPUDevice:
543-
return 0, nil
546+
query = fmt.Sprintf("avg(npu_chip_info_utilization{vdie_id=\"%s\"})", deviceUUID)
544547
case biz.HygonGPUDevice:
545548
query = fmt.Sprintf("avg(vdcu_percent{pod_uuid=\"%s\", container_name=\"%s\"})", podUUID, container)
546549
case biz.MetaxGPUDevice, metax.MetaxGPUDevice:
@@ -563,7 +566,7 @@ func (s *MetricsGenerator) taskMemoryUsed(ctx context.Context, provider, namespa
563566
case biz.CambriconGPUDevice:
564567
query = fmt.Sprintf("avg(mlu_memory_utilization * on(uuid) group_right mlu_container{namespace=\"%s\",pod=\"%s\",container=\"%s\",type=\"mlu370.smlu.vmemory\"})", namespace, pod, container)
565568
case biz.AscendGPUDevice:
566-
return 0, nil
569+
query = fmt.Sprintf("avg(npu_chip_info_hbm_used_memory{vdie_id=\"%s\"})", deviceUUID)
567570
case biz.HygonGPUDevice:
568571
query = fmt.Sprintf("avg(vdcu_usage_memory_size{pod_uuid=\"%s\", container_name=\"%s\"})", podUUID, container)
569572
case metax.MetaxGPUDevice:
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package exporter
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"io"
7+
"net/http"
8+
"net/http/httptest"
9+
"strings"
10+
"testing"
11+
"time"
12+
13+
"vgpu/internal/biz"
14+
"vgpu/internal/data/prom"
15+
"vgpu/internal/service"
16+
17+
"github.com/go-kratos/kratos/v2/log"
18+
)
19+
20+
// testPromHandler returns a handler that acts as a fake Prometheus /api/v1/query
21+
// endpoint. When value is non-empty, it returns a single-sample vector with that
22+
// value. When value is empty, it returns an empty result vector.
23+
func testPromHandler(t *testing.T, value string) http.HandlerFunc {
24+
return func(w http.ResponseWriter, r *http.Request) {
25+
if r.Method != http.MethodPost && r.Method != http.MethodGet {
26+
t.Errorf("unexpected method: %s", r.Method)
27+
}
28+
if !strings.HasSuffix(r.URL.Path, "/api/v1/query") {
29+
t.Errorf("unexpected path: %s", r.URL.Path)
30+
}
31+
w.Header().Set("Content-Type", "application/json")
32+
if value == "" {
33+
fmt.Fprint(w, `{"status":"success","data":{"resultType":"vector","result":[]}}`)
34+
return
35+
}
36+
fmt.Fprintf(w, `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1719500000,%q]}]}}`, value)
37+
}
38+
}
39+
40+
// newTestGenerator starts an httptest server with the given handler, creates a
41+
// prom.Client pointed at it, wraps it in a MonitorService, and returns a bare
42+
// MetricsGenerator (other fields left nil). The returned close func shuts down
43+
// the test server.
44+
func newTestGenerator(t *testing.T, handler http.HandlerFunc) (*MetricsGenerator, func()) {
45+
srv := httptest.NewServer(handler)
46+
47+
promClient, err := prom.NewClient(srv.URL, time.Second, "")
48+
if err != nil {
49+
srv.Close()
50+
t.Fatalf("prom.NewClient: %v", err)
51+
}
52+
53+
monitorSvc := service.NewMonitorService(promClient, nil, nil)
54+
55+
gen := &MetricsGenerator{
56+
monitorService: monitorSvc,
57+
log: log.NewHelper(log.NewStdLogger(io.Discard)),
58+
}
59+
60+
return gen, srv.Close
61+
}
62+
63+
func TestMetricsGenerator_taskCoreUsed_Ascend(t *testing.T) {
64+
gen, cleanup := newTestGenerator(t, testPromHandler(t, "42.5"))
65+
defer cleanup()
66+
67+
result, err := gen.taskCoreUsed(context.Background(), biz.AscendGPUDevice, "ns", "pod", "ctr", "pod-uuid", "dev-uuid", "host", 0)
68+
if err != nil {
69+
t.Fatalf("taskCoreUsed failed: %v", err)
70+
}
71+
if result != 42.5 {
72+
t.Errorf("expected 42.5, got %v", result)
73+
}
74+
}
75+
76+
func TestMetricsGenerator_taskMemoryUsed_Ascend(t *testing.T) {
77+
gen, cleanup := newTestGenerator(t, testPromHandler(t, "8589934592"))
78+
defer cleanup()
79+
80+
result, err := gen.taskMemoryUsed(context.Background(), biz.AscendGPUDevice, "ns", "pod", "ctr", "pod-uuid", "dev-uuid", "host", 0)
81+
if err != nil {
82+
t.Fatalf("taskMemoryUsed failed: %v", err)
83+
}
84+
if result != 8589934592 {
85+
t.Errorf("expected 8589934592, got %v", result)
86+
}
87+
}
88+
89+
func TestMetricsGenerator_taskCoreUsed_Ascend_EmptyResult(t *testing.T) {
90+
gen, cleanup := newTestGenerator(t, testPromHandler(t, ""))
91+
defer cleanup()
92+
93+
result, err := gen.taskCoreUsed(context.Background(), biz.AscendGPUDevice, "ns", "pod", "ctr", "pod-uuid", "dev-uuid", "host", 0)
94+
if err != nil {
95+
t.Fatalf("taskCoreUsed failed: %v", err)
96+
}
97+
if result != 0 {
98+
t.Errorf("expected 0, got %v", result)
99+
}
100+
}

0 commit comments

Comments
 (0)