@@ -30,23 +30,19 @@ def _query_ecc_counters(gpu_index: int) -> dict:
3030 vol_sbe = 0
3131 vol_dbe = 0
3232 try :
33- vol_sbe = (
34- pynvml .nvmlDeviceGetTotalEccErrors (
35- handle ,
36- pynvml .NVML_SINGLE_BIT_ECC ,
37- pynvml .NVML_VOLATILE_ECC ,
38- )
33+ vol_sbe = pynvml .nvmlDeviceGetTotalEccErrors (
34+ handle ,
35+ pynvml .NVML_SINGLE_BIT_ECC ,
36+ pynvml .NVML_VOLATILE_ECC ,
3937 )
4038 except (pynvml .NVMLError , AttributeError ):
4139 vol_sbe = - 1 # Not supported
4240
4341 try :
44- vol_dbe = (
45- pynvml .nvmlDeviceGetTotalEccErrors (
46- handle ,
47- pynvml .NVML_DOUBLE_BIT_ECC ,
48- pynvml .NVML_VOLATILE_ECC ,
49- )
42+ vol_dbe = pynvml .nvmlDeviceGetTotalEccErrors (
43+ handle ,
44+ pynvml .NVML_DOUBLE_BIT_ECC ,
45+ pynvml .NVML_VOLATILE_ECC ,
5046 )
5147 except (pynvml .NVMLError , AttributeError ):
5248 vol_dbe = - 1
@@ -55,23 +51,19 @@ def _query_ecc_counters(gpu_index: int) -> dict:
5551 agg_sbe = 0
5652 agg_dbe = 0
5753 try :
58- agg_sbe = (
59- pynvml .nvmlDeviceGetTotalEccErrors (
60- handle ,
61- pynvml .NVML_SINGLE_BIT_ECC ,
62- pynvml .NVML_AGGREGATE_ECC ,
63- )
54+ agg_sbe = pynvml .nvmlDeviceGetTotalEccErrors (
55+ handle ,
56+ pynvml .NVML_SINGLE_BIT_ECC ,
57+ pynvml .NVML_AGGREGATE_ECC ,
6458 )
6559 except (pynvml .NVMLError , AttributeError ):
6660 agg_sbe = - 1
6761
6862 try :
69- agg_dbe = (
70- pynvml .nvmlDeviceGetTotalEccErrors (
71- handle ,
72- pynvml .NVML_DOUBLE_BIT_ECC ,
73- pynvml .NVML_AGGREGATE_ECC ,
74- )
63+ agg_dbe = pynvml .nvmlDeviceGetTotalEccErrors (
64+ handle ,
65+ pynvml .NVML_DOUBLE_BIT_ECC ,
66+ pynvml .NVML_AGGREGATE_ECC ,
7567 )
7668 except (pynvml .NVMLError , AttributeError ):
7769 agg_dbe = - 1
@@ -80,23 +72,19 @@ def _query_ecc_counters(gpu_index: int) -> dict:
8072 retired_sbe = 0
8173 retired_dbe = 0
8274 try :
83- retired_sbe = (
84- pynvml .nvmlDeviceGetRetiredPages (
85- handle ,
86- pynvml .NVML_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS ,
87- )
75+ retired_sbe = pynvml .nvmlDeviceGetRetiredPages (
76+ handle ,
77+ pynvml .NVML_PAGE_RETIREMENT_CAUSE_MULTIPLE_SINGLE_BIT_ECC_ERRORS ,
8878 )
8979 if isinstance (retired_sbe , (list , tuple )):
9080 retired_sbe = len (retired_sbe )
9181 except (pynvml .NVMLError , AttributeError ):
9282 retired_sbe = - 1
9383
9484 try :
95- retired_dbe = (
96- pynvml .nvmlDeviceGetRetiredPages (
97- handle ,
98- pynvml .NVML_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR ,
99- )
85+ retired_dbe = pynvml .nvmlDeviceGetRetiredPages (
86+ handle ,
87+ pynvml .NVML_PAGE_RETIREMENT_CAUSE_DOUBLE_BIT_ECC_ERROR ,
10088 )
10189 if isinstance (retired_dbe , (list , tuple )):
10290 retired_dbe = len (retired_dbe )
@@ -106,9 +94,7 @@ def _query_ecc_counters(gpu_index: int) -> dict:
10694 # Pending retired pages
10795 pending_retirement = False
10896 try :
109- pending = (
110- pynvml .nvmlDeviceGetRetiredPages_v2 (handle )
111- )
97+ pending = pynvml .nvmlDeviceGetRetiredPages_v2 (handle )
11298 if pending :
11399 pending_retirement = True
114100 except (pynvml .NVMLError , AttributeError ):
@@ -118,9 +104,7 @@ def _query_ecc_counters(gpu_index: int) -> dict:
118104 remapped_rows = {}
119105 try :
120106 remapped = pynvml .nvmlDeviceGetRemappedRows (handle )
121- correctable , uncorrectable , pending_remap , failure = (
122- remapped
123- )
107+ correctable , uncorrectable , pending_remap , failure = remapped
124108 remapped_rows = {
125109 "correctable" : correctable ,
126110 "uncorrectable" : uncorrectable ,
@@ -168,10 +152,7 @@ def _check_ecc_health(
168152 test_name = "telemetry.ecc_health" ,
169153 status = TestStatus .SKIP ,
170154 duration_seconds = time .time () - start ,
171- message = (
172- "ECC not supported on this GPU "
173- f"({ gpu .name } )"
174- ),
155+ message = (f"ECC not supported on this GPU ({ gpu .name } )" ),
175156 gpu_uuid = gpu .uuid ,
176157 details = {"ecc_mode" : gpu .ecc_mode },
177158 )
@@ -183,10 +164,7 @@ def _check_ecc_health(
183164 test_name = "telemetry.ecc_health" ,
184165 status = TestStatus .SKIP ,
185166 duration_seconds = time .time () - start ,
186- message = (
187- f"ECC query failed: "
188- f"{ counters .get ('error' , 'unknown' )} "
189- ),
167+ message = (f"ECC query failed: { counters .get ('error' , 'unknown' )} " ),
190168 gpu_uuid = gpu .uuid ,
191169 details = counters ,
192170 )
@@ -256,9 +234,7 @@ def _check_ecc_health(
256234 test_name = "telemetry.ecc_health" ,
257235 status = TestStatus .WARN ,
258236 duration_seconds = time .time () - start ,
259- message = (
260- "Pages pending retirement — reboot recommended"
261- ),
237+ message = ("Pages pending retirement — reboot recommended" ),
262238 gpu_uuid = gpu .uuid ,
263239 details = details ,
264240 )
@@ -282,6 +258,4 @@ def run_ecc_health_checks(
282258 profile : dict [str , Any ],
283259) -> list [TestResult ]:
284260 """Execute ECC health analysis on all GPUs."""
285- return [
286- _check_ecc_health (gpu , profile ) for gpu in gpu_infos
287- ]
261+ return [_check_ecc_health (gpu , profile ) for gpu in gpu_infos ]
0 commit comments