fix: eliminate per-scrape Kubernetes API call causing 15s+ metrics latency

Collect() was calling containerLister.Update() on every Prometheus scrape,
which issues a synchronous cluster-wide Pods().List() against the API server.
The watchAndFeedback goroutine already refreshes container state every 5s,
so Collect() only needs to read cached data.

Also remove nvml.Init() from Collect() — NVML is already initialized in
watchAndFeedback and should only be called once at startup.

Fix a data race in ListContainers(): Update() holds the mutex while writing
the containers map, but the previous ListContainers() returned the raw map
with no lock. It now returns a snapshot under the mutex.

Co-Authored-By: charford <casey@caseyharford.com>
Signed-off-by: charford <casey@caseyharford.com>

Author: charford

cmd/vgpu-monitor/metrics.go

@@ -139,14 +139,7 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
 func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
 	klog.Info("Starting to collect metrics for vGPUMonitor")
 	containerLister := cc.ClusterManager.containerLister
-	if err := containerLister.Update(); err != nil {
-		klog.Error("Update container error: %s", err.Error())
-	}
 
-	nvret := config.Nvml().Init()
-	if nvret != nvml.SUCCESS {
-		klog.Errorf("nvml Init err= %v", nvret)
-	}
 	devnum, nvret := config.Nvml().DeviceGetCount()
 	if nvret != nvml.SUCCESS {
 		klog.Errorf("nvml GetDeviceCount err= %v", nvret)

pkg/monitor/nvidia/cudevshr.go

@@ -112,7 +112,13 @@ func (l *ContainerLister) UnLock() {
 }
 
 func (l *ContainerLister) ListContainers() map[string]*ContainerUsage {
-	return l.containers
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	snapshot := make(map[string]*ContainerUsage, len(l.containers))
+	for k, v := range l.containers {
+		snapshot[k] = v
+	}
+	return snapshot
 }
 
 func (l *ContainerLister) Clientset() *kubernetes.Clientset {