Merge pull request #107 from charford/fix/metrics-endpoint-latency

Remove unnecessary K8s API call and data race from `/metrics` scrape path

Author: hami-robot[bot]

cmd/vgpu-monitor/metrics.go

@@ -139,14 +139,7 @@ func (cc ClusterManagerCollector) Describe(ch chan<- *prometheus.Desc) {
 func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
 	klog.Info("Starting to collect metrics for vGPUMonitor")
 	containerLister := cc.ClusterManager.containerLister
-	if err := containerLister.Update(); err != nil {
-		klog.Error("Update container error: %s", err.Error())
-	}
 
-	nvret := config.Nvml().Init()
-	if nvret != nvml.SUCCESS {
-		klog.Errorf("nvml Init err= %v", nvret)
-	}
 	devnum, nvret := config.Nvml().DeviceGetCount()
 	if nvret != nvml.SUCCESS {
 		klog.Errorf("nvml GetDeviceCount err= %v", nvret)

pkg/monitor/nvidia/cudevshr.go

@@ -112,7 +112,13 @@ func (l *ContainerLister) UnLock() {
 }
 
 func (l *ContainerLister) ListContainers() map[string]*ContainerUsage {
-	return l.containers
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+	snapshot := make(map[string]*ContainerUsage, len(l.containers))
+	for k, v := range l.containers {
+		snapshot[k] = v
+	}
+	return snapshot
 }
 
 func (l *ContainerLister) Clientset() *kubernetes.Clientset {