diff --git a/codecarbon/emissions_tracker.py b/codecarbon/emissions_tracker.py index 57c9786ae..862eba2b4 100644 --- a/codecarbon/emissions_tracker.py +++ b/codecarbon/emissions_tracker.py @@ -978,9 +978,10 @@ def _monitor_power(self) -> None: if isinstance(hardware, GPU): gpu_ids_to_monitor = hardware.gpu_ids gpu_details = hardware.devices.get_gpu_details() - for gpu_detail in gpu_details: + for gpu_index, gpu_detail in enumerate(gpu_details): + resolved_gpu_index = gpu_detail.get("gpu_index", gpu_index) if ( - gpu_detail["gpu_index"] in gpu_ids_to_monitor + resolved_gpu_index in gpu_ids_to_monitor and "gpu_utilization" in gpu_detail ): self._gpu_utilization_history.append( diff --git a/tests/test_emissions_tracker.py b/tests/test_emissions_tracker.py index 8309c57e8..ab4a0a275 100644 --- a/tests/test_emissions_tracker.py +++ b/tests/test_emissions_tracker.py @@ -117,6 +117,33 @@ def test_carbon_tracker_TWO_GPU_PRIVATE_INFRA_CANADA( self.assertIsInstance(emissions, float) self.assertAlmostEqual(emissions, 6.262572537957655e-05, places=2) + def test_monitor_power_uses_gpu_detail_position_when_gpu_index_is_missing( + self, + mock_cli_setup, + mock_log_values, + mocked_get_gpu_details, + mocked_env_cloud_details, + mocked_is_gpu_details_available, + mocked_is_nvidia_system, + ): + tracker = EmissionsTracker(measure_power_secs=1, save_to_file=False) + + mock_gpu = mock.MagicMock() + from codecarbon.external.hardware import GPU + + mock_gpu.__class__ = GPU + mock_gpu.gpu_ids = [0, 1] + mock_gpu.devices = mock.MagicMock() + mock_gpu.devices.get_gpu_details.return_value = [ + {"gpu_utilization": 10}, + {"gpu_index": 1, "gpu_utilization": 25}, + ] + tracker._hardware = [mock_gpu] + + tracker._monitor_power() + + self.assertEqual([10, 25], tracker._gpu_utilization_history) + @mock.patch("codecarbon.external.geography.requests.get") def test_carbon_tracker_timeout( self,