Add end-to-end test for metrics API functionality

- Tests node metrics retrieval from cluster
- Tests pod metrics retrieval with namespace filtering
- Tests pod metrics with label selector filtering
- Tests multi-namespace metrics aggregation
- Includes proper error handling and skip logic if metrics-server unavailable
- Creates test pod, waits for it to be running, then fetches metrics

Co-authored-by: brendandburns <5751682+brendandburns@users.noreply.github.com>

Author: Copilot

kubernetes/e2e_test/test_utils.py

@@ -608,6 +608,140 @@ def test_create_from_list_in_multi_resource_yaml_namespaced(self):
         app_api.delete_namespaced_deployment(
             name="mock", namespace=self.test_namespace, body={})
 
+    def test_get_metrics_from_cluster(self):
+        """
+        Should be able to fetch node and pod metrics from the cluster.
+        This test requires metrics-server to be installed in the cluster.
+        """
+        k8s_client = client.api_client.ApiClient(configuration=self.config)
+        core_api = client.CoreV1Api(k8s_client)
+        
+        # Create a test pod for metrics
+        utils.create_from_yaml(
+            k8s_client, self.path_prefix + "core-pod.yaml",
+            namespace=self.test_namespace)
+        
+        # Wait for pod to be running (with timeout)
+        import time
+        max_wait = 60
+        waited = 0
+        pod_running = False
+        while waited < max_wait:
+            try:
+                pod = core_api.read_namespaced_pod(
+                    name="myapp-pod", namespace=self.test_namespace)
+                if pod.status.phase == "Running":
+                    pod_running = True
+                    break
+            except ApiException:
+                pass
+            time.sleep(2)
+            waited += 2
+        
+        # Skip test if pod didn't start (cluster might be slow)
+        if not pod_running:
+            core_api.delete_namespaced_pod(
+                name="myapp-pod", namespace=self.test_namespace, body={})
+            raise unittest.SkipTest("Pod did not reach Running state in time")
+        
+        # Wait a bit more for metrics to be available
+        time.sleep(10)
+        
+        # Test node metrics retrieval
+        try:
+            node_metrics = utils.get_nodes_metrics(k8s_client)
+            self.assertIsNotNone(node_metrics)
+            self.assertEqual(node_metrics['kind'], 'NodeMetricsList')
+            self.assertIn('items', node_metrics)
+            # We should have at least one node in the cluster
+            self.assertGreater(len(node_metrics['items']), 0)
+            # Check structure of first node metric
+            if len(node_metrics['items']) > 0:
+                node = node_metrics['items'][0]
+                self.assertIn('metadata', node)
+                self.assertIn('name', node['metadata'])
+                self.assertIn('usage', node)
+                self.assertIn('cpu', node['usage'])
+                self.assertIn('memory', node['usage'])
+        except ApiException as e:
+            # If metrics-server is not installed, skip this test
+            if e.status == 404:
+                core_api.delete_namespaced_pod(
+                    name="myapp-pod", namespace=self.test_namespace, body={})
+                raise unittest.SkipTest("Metrics server not available in cluster")
+            raise
+        
+        # Test pod metrics retrieval
+        try:
+            pod_metrics = utils.get_pods_metrics(
+                k8s_client, self.test_namespace)
+            self.assertIsNotNone(pod_metrics)
+            self.assertEqual(pod_metrics['kind'], 'PodMetricsList')
+            self.assertIn('items', pod_metrics)
+            # We should have our test pod
+            self.assertGreater(len(pod_metrics['items']), 0)
+            # Check structure of pod metrics
+            found_test_pod = False
+            for pod in pod_metrics['items']:
+                if pod['metadata']['name'] == 'myapp-pod':
+                    found_test_pod = True
+                    self.assertIn('containers', pod)
+                    self.assertGreater(len(pod['containers']), 0)
+                    container = pod['containers'][0]
+                    self.assertIn('name', container)
+                    self.assertIn('usage', container)
+                    self.assertIn('cpu', container['usage'])
+                    self.assertIn('memory', container['usage'])
+            # Our test pod should appear in metrics
+            self.assertTrue(found_test_pod, "Test pod not found in metrics")
+        except ApiException as e:
+            if e.status == 404:
+                core_api.delete_namespaced_pod(
+                    name="myapp-pod", namespace=self.test_namespace, body={})
+                raise unittest.SkipTest("Metrics server not available in cluster")
+            raise
+        
+        # Test pod metrics with label selector
+        try:
+            filtered_metrics = utils.get_pods_metrics(
+                k8s_client, self.test_namespace, label_selector='app=myapp')
+            self.assertIsNotNone(filtered_metrics)
+            self.assertEqual(filtered_metrics['kind'], 'PodMetricsList')
+            self.assertIn('items', filtered_metrics)
+            # Should have our pod with the matching label
+            self.assertGreater(len(filtered_metrics['items']), 0)
+            for pod in filtered_metrics['items']:
+                # All returned pods should have the label we filtered for
+                self.assertEqual(pod['metadata']['name'], 'myapp-pod')
+        except ApiException as e:
+            if e.status == 404:
+                core_api.delete_namespaced_pod(
+                    name="myapp-pod", namespace=self.test_namespace, body={})
+                raise unittest.SkipTest("Metrics server not available in cluster")
+            raise
+        
+        # Test multi-namespace metrics collection
+        try:
+            multi_ns_metrics = utils.get_pods_metrics_in_all_namespaces(
+                k8s_client, [self.test_namespace, 'default'])
+            self.assertIsNotNone(multi_ns_metrics)
+            self.assertIn(self.test_namespace, multi_ns_metrics)
+            self.assertIn('default', multi_ns_metrics)
+            # Our test namespace should have metrics
+            test_ns_result = multi_ns_metrics[self.test_namespace]
+            self.assertNotIn('error', test_ns_result)
+            self.assertEqual(test_ns_result['kind'], 'PodMetricsList')
+        except ApiException as e:
+            if e.status == 404:
+                core_api.delete_namespaced_pod(
+                    name="myapp-pod", namespace=self.test_namespace, body={})
+                raise unittest.SkipTest("Metrics server not available in cluster")
+            raise
+        
+        # Cleanup
+        core_api.delete_namespaced_pod(
+            name="myapp-pod", namespace=self.test_namespace, body={})
+
 
 class TestUtilsUnitTests(unittest.TestCase):