raise exceptions instead of skipping, more defensively run scenario

Author: sid-rl

tests/smoketests/sdk/test_async_benchmark.py

@@ -33,7 +33,7 @@ async def test_benchmark_from_existing(self, async_sdk_client: AsyncRunloopSDK)
         benchmarks = benchmarks_page.benchmarks
 
         if not benchmarks:
-            pytest.skip("No benchmarks available to test")
+            raise Exception("No benchmarks available to test")
 
         benchmark_data = benchmarks[0]
 
@@ -69,9 +69,11 @@ async def test_benchmark_run_and_cancel(self, async_sdk_client: AsyncRunloopSDK)
         benchmarks = benchmarks_page.benchmarks
 
         if not benchmarks:
-            pytest.skip("No benchmarks available to test")
+            raise Exception("No benchmarks available to test")
 
         benchmark_data = benchmarks[0]
+        if not benchmark_data.scenario_ids:
+            raise Exception("No scenarios available to test")
 
         # Create AsyncBenchmark wrapper
         benchmark = AsyncBenchmark(
@@ -81,14 +83,8 @@ async def test_benchmark_run_and_cancel(self, async_sdk_client: AsyncRunloopSDK)
 
         # Start a run
         run = await benchmark.run(run_name="sdk-smoketest-async-benchmark-run")
-
-        # If the benchmark has scenarios, run one
-        scenario_runs: list[AsyncScenarioRun] = []
-        if benchmark_data.scenario_ids:
-            scenario = async_sdk_client.scenario.from_id(benchmark_data.scenario_ids[0])
-            scenario_runs.append(
-                await scenario.run(benchmark_run_id=run.id, run_name="sdk-smoketest-async-benchmark-run-scenario")
-            )
+        scenario = async_sdk_client.scenario.from_id(benchmark_data.scenario_ids[0])
+        scenario_run = None
 
         try:
             assert isinstance(run, AsyncBenchmarkRun)
@@ -100,30 +96,30 @@ async def test_benchmark_run_and_cancel(self, async_sdk_client: AsyncRunloopSDK)
             assert info.id == run.id
             assert info.state in ["running", "completed", "canceled"]
 
-            bench_scenario_runs = await run.list_scenario_runs()
-            assert isinstance(bench_scenario_runs, list)
-            assert len(bench_scenario_runs) == len(scenario_runs)
-            for bench_scenario_run in bench_scenario_runs:
-                assert isinstance(bench_scenario_run, AsyncScenarioRun)
-                assert bench_scenario_run.id == scenario_runs[0].id
-                assert bench_scenario_run.devbox_id == scenario_runs[0].devbox_id
+            # Start a scenario run
+            scenario_run = await scenario.run(
+                benchmark_run_id=run.id, run_name="sdk-smoketest-async-benchmark-run-scenario"
+            )
+            scenario_runs = await run.list_scenario_runs()
+            assert isinstance(scenario_runs, list)
+            assert len(scenario_runs) == 1
+            assert isinstance(scenario_runs[0], AsyncScenarioRun)
+            assert scenario_runs[0].id == scenario_run.id
+            assert scenario_runs[0].devbox_id == scenario_run.devbox_id
 
-                # Cancel the scenario run
-                scenario_result = await bench_scenario_run.cancel()
-                assert scenario_result.state in ["canceled", "completed"]
+            # Cancel the scenario run
+            scenario_result = await scenario_run.cancel()
+            assert scenario_result.state in ["canceled", "completed"]
 
             # Cancel the benchmark run
             result = await run.cancel()
             assert result.state in ["canceled", "completed"]
 
         except Exception:
             # Ensure cleanup on any error
-            try:
-                for scenario_run in scenario_runs:
-                    await async_sdk_client.api.scenarios.runs.cancel(scenario_run.id)
-                await async_sdk_client.api.benchmarks.runs.cancel(run.id)
-            except Exception:
-                pass
+            if scenario_run:
+                await async_sdk_client.api.scenarios.runs.cancel(scenario_run.id)
+            await async_sdk_client.api.benchmarks.runs.cancel(run.id)
             raise
 
 
@@ -144,7 +140,7 @@ async def test_list_runs(self, async_sdk_client: AsyncRunloopSDK) -> None:
         benchmarks = benchmarks_page.benchmarks
 
         if not benchmarks:
-            pytest.skip("No benchmarks available to test")
+            raise Exception("No benchmarks available to test")
 
         benchmark_data = benchmarks[0]
 

tests/smoketests/sdk/test_benchmark.py

@@ -32,7 +32,7 @@ def test_benchmark_from_existing(self, sdk_client: RunloopSDK) -> None:
         benchmarks = sdk_client.api.benchmarks.list_public(limit=1).benchmarks
 
         if not benchmarks:
-            pytest.skip("No benchmarks available to test")
+            raise Exception("No benchmarks available to test")
 
         benchmark_data = benchmarks[0]
 
@@ -67,9 +67,11 @@ def test_benchmark_run_lifecycle(self, sdk_client: RunloopSDK) -> None:
         benchmarks = sdk_client.api.benchmarks.list_public(limit=1).benchmarks
 
         if not benchmarks:
-            pytest.skip("No benchmarks available to test")
+            raise Exception("No benchmarks available to test")
 
         benchmark_data = benchmarks[0]
+        if not benchmark_data.scenario_ids:
+            raise Exception("No scenarios available to test")
 
         # Create Benchmark wrapper
         benchmark = Benchmark(
@@ -79,12 +81,8 @@ def test_benchmark_run_lifecycle(self, sdk_client: RunloopSDK) -> None:
 
         # Start a run
         run = benchmark.run(run_name="sdk-smoketest-benchmark-run")
-
-        # If the benchmark has scenarios, run one
-        scenario_runs: list[ScenarioRun] = []
-        if benchmark_data.scenario_ids:
-            scenario = sdk_client.scenario.from_id(benchmark_data.scenario_ids[0])
-            scenario_runs.append(scenario.run(benchmark_run_id=run.id, run_name="sdk-smoketest-benchmark-run-scenario"))
+        scenario = sdk_client.scenario.from_id(benchmark_data.scenario_ids[0])
+        scenario_run = None
 
         try:
             assert isinstance(run, BenchmarkRun)
@@ -96,30 +94,28 @@ def test_benchmark_run_lifecycle(self, sdk_client: RunloopSDK) -> None:
             assert info.id == run.id
             assert info.state in ["running", "completed", "canceled"]
 
-            bench_scenario_runs = run.list_scenario_runs()
-            assert isinstance(bench_scenario_runs, list)
-            assert len(bench_scenario_runs) == len(scenario_runs)
-            for bench_scenario_run in bench_scenario_runs:
-                assert isinstance(bench_scenario_run, ScenarioRun)
-                assert bench_scenario_run.id == scenario_runs[0].id
-                assert bench_scenario_run.devbox_id == scenario_runs[0].devbox_id
+            # Start a scenario run
+            scenario_run = scenario.run(benchmark_run_id=run.id, run_name="sdk-smoketest-benchmark-run-scenario")
+            scenario_runs = run.list_scenario_runs()
+            assert isinstance(scenario_runs, list)
+            assert len(scenario_runs) == 1
+            assert isinstance(scenario_runs[0], ScenarioRun)
+            assert scenario_runs[0].id == scenario_run.id
+            assert scenario_runs[0].devbox_id == scenario_run.devbox_id
 
-                # Cancel the scenario run
-                scenario_result = bench_scenario_run.cancel()
-                assert scenario_result.state in ["canceled", "completed"]
+            # Cancel the scenario run
+            scenario_result = scenario_run.cancel()
+            assert scenario_result.state in ["canceled", "completed"]
 
             # Cancel the benchmark run
             result = run.cancel()
             assert result.state in ["canceled", "completed"]
 
         except Exception:
             # Ensure cleanup on any error
-            try:
-                for scenario_run in scenario_runs:
-                    sdk_client.api.scenarios.runs.cancel(scenario_run.id)
-                sdk_client.api.benchmarks.runs.cancel(run.id)
-            except Exception:
-                pass
+            if scenario_run:
+                sdk_client.api.scenarios.runs.cancel(scenario_run.id)
+            sdk_client.api.benchmarks.runs.cancel(run.id)
             raise
 
 
@@ -139,7 +135,7 @@ def test_list_runs(self, sdk_client: RunloopSDK) -> None:
         benchmarks = sdk_client.api.benchmarks.list(limit=1).benchmarks
 
         if not benchmarks:
-            pytest.skip("No benchmarks available to test")
+            raise Exception("No benchmarks available to test")
 
         benchmark_data = benchmarks[0]