fix(fitness): address PR feedback on fitness checks system

- Use configurable GPU_BENCHMARK_TIMEOUT instead of hardcoded 100ms threshold
- Change test assertion from >= to == to prevent accidental check registration
- Remove empty TestFallbackExecution class (covered by integration tests)
- Make error message limit configurable via RUNPOD_GPU_MAX_ERROR_MESSAGES env var

All 83 fitness check tests pass with these changes.

Author: deanq

runpod/serverless/modules/rp_gpu_fitness.py

@@ -23,6 +23,7 @@
 
 # Configuration via environment variables
 TIMEOUT_SECONDS = int(os.environ.get("RUNPOD_GPU_TEST_TIMEOUT", "30"))
+MAX_ERROR_MESSAGES = int(os.environ.get("RUNPOD_GPU_MAX_ERROR_MESSAGES", "10"))
 
 
 def _get_gpu_test_binary_path() -> Optional[Path]:
@@ -151,7 +152,7 @@ async def _run_gpu_test_binary() -> Dict[str, Any]:
         if not result["success"]:
             error_msg = "GPU memory allocation test failed"
             if result["errors"]:
-                error_msg += f": {'; '.join(result['errors'][:3])}"  # Limit to 3 errors
+                error_msg += f": {'; '.join(result['errors'][:MAX_ERROR_MESSAGES])}"
             raise RuntimeError(error_msg)
 
         log.info(

runpod/serverless/modules/rp_system_fitness.py

@@ -379,15 +379,16 @@ async def _check_gpu_compute_benchmark() -> None:
         # Do computation
         A = torch.randn(size, size, device="cuda")
         B = torch.randn(size, size, device="cuda")
-        C = torch.matmul(A, B)
+        torch.matmul(A, B)
         torch.cuda.synchronize()  # Wait for GPU to finish
 
         elapsed_ms = (time.time() - start_time) * 1000
+        max_ms = GPU_BENCHMARK_TIMEOUT * 1000
 
-        if elapsed_ms > 100:
+        if elapsed_ms > max_ms:
             raise RuntimeError(
                 f"GPU compute too slow: Matrix multiply took {elapsed_ms:.0f}ms "
-                f"(max: 100ms)"
+                f"(max: {max_ms:.0f}ms)"
             )
 
         log.info(f"GPU compute benchmark passed: Matrix multiply completed in {elapsed_ms:.0f}ms")
@@ -407,15 +408,16 @@ async def _check_gpu_compute_benchmark() -> None:
 
         A = cp.random.randn(size, size)
         B = cp.random.randn(size, size)
-        C = cp.matmul(A, B)
+        cp.matmul(A, B)
         cp.cuda.Device().synchronize()
 
         elapsed_ms = (time.time() - start_time) * 1000
+        max_ms = GPU_BENCHMARK_TIMEOUT * 1000
 
-        if elapsed_ms > 100:
+        if elapsed_ms > max_ms:
             raise RuntimeError(
                 f"GPU compute too slow: Matrix multiply took {elapsed_ms:.0f}ms "
-                f"(max: 100ms)"
+                f"(max: {max_ms:.0f}ms)"
             )
 
         log.info(f"GPU compute benchmark passed: Matrix multiply completed in {elapsed_ms:.0f}ms")

tests/test_serverless/test_modules/test_gpu_fitness.py

@@ -268,18 +268,10 @@ async def test_binary_failure_output(self):
 
 
 # ============================================================================
-# Fallback Tests
+# Note: Fallback execution tests are covered by integration tests since
+# they involve subprocess calls that are difficult to mock cleanly.
 # ============================================================================
 
-class TestFallbackExecution:
-    """Tests for Python fallback GPU check.
-
-    Fallback tests are primarily covered by integration tests since
-    the fallback involves subprocess calls that are difficult to mock cleanly.
-    """
-    pass
-
-
 # ============================================================================
 # Health Check Logic Tests
 # ============================================================================

tests/test_serverless/test_modules/test_gpu_fitness_integration.py

@@ -48,7 +48,7 @@ def mock_gpu_test_binary():
 """)
         binary_path = f.name
 
-    os.chmod(binary_path, 0o755)
+    os.chmod(binary_path, 0o700)
     yield Path(binary_path)
 
     # Cleanup
@@ -70,7 +70,7 @@ def mock_gpu_test_binary_failure():
 """)
         binary_path = f.name
 
-    os.chmod(binary_path, 0o755)
+    os.chmod(binary_path, 0o700)
     yield Path(binary_path)
 
     # Cleanup
@@ -98,7 +98,7 @@ def mock_gpu_test_binary_multi_gpu():
 """)
         binary_path = f.name
 
-    os.chmod(binary_path, 0o755)
+    os.chmod(binary_path, 0o700)
     yield Path(binary_path)
 
     # Cleanup

tests/test_serverless/test_modules/test_system_fitness.py

@@ -501,7 +501,7 @@ def test_auto_register_all_checks_with_gpu(self, mock_gpu_available):
         mock_gpu_available.return_value = True
         auto_register_system_checks()
         # Should register: memory, disk, network, cuda_version, cuda_init, benchmark
-        assert len(_fitness_checks) >= 6
+        assert len(_fitness_checks) == 6
 
     @patch("runpod.serverless.modules.rp_system_fitness.gpu_available")
     def test_auto_register_cpu_only(self, mock_gpu_available):