[NA] [SDK] [GHA] test: stabilize Gemini + agentic-judge integration tests (#7405)

Three related changes to stop provider-driven flakiness in the real-model
integration suites:

- Agentic LLM-judge tests: run the Anthropic lanes against claude-sonnet-4-6
  instead of claude-haiku-4-5. On the agentic path (tools in the request, so
  response_format is best-effort) Haiku wrapped verdicts in prose/```json
  fences; Sonnet follows the structured-output contract reliably (the one-shot
  suite already used Sonnet).
- GenAI + ADK test matrices: run only the oldest and newest supported Python
  versions (3.10, 3.14). These suites make live Vertex AI calls against the
  shared opik-sdk-tests GCP project; the full 3.10-3.14 matrix re-multiplied
  those calls into 429 RESOURCE_EXHAUSTED. Endpoints cover version-specific
  breakage at a fraction of the request volume; max-parallel cap kept.
- Remove the broken GenAI rate-limit retry helpers: _is_rate_limit_error
  checked exception.response.status_code, which raises AttributeError on
  google-genai's ClientError (no such attribute), so the retry crashed instead
  of retrying. With the matrix trim as the 429 mitigation, the dead helper and
  its decorators are removed rather than repaired.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: alexkuzmik

.github/workflows/lib-adk-tests.yml

@@ -29,12 +29,15 @@ jobs:
 
     strategy:
       fail-fast: true
-      # Throttle concurrent matrix jobs so we keep full Python coverage
-      # without re-multiplying live Vertex AI calls into 429 RESOURCE_EXHAUSTED
-      # against the shared opik-sdk-tests GCP project.
+      # This suite makes live Vertex AI calls against the shared
+      # opik-sdk-tests GCP project (shared with the GenAI suite). Running the
+      # full 3.10-3.14 matrix re-multiplied those calls into 429
+      # RESOURCE_EXHAUSTED, so we run only the oldest and newest supported
+      # Python versions — enough to catch version-specific breakage at a
+      # fraction of the request volume — and keep the concurrency cap.
       max-parallel: 2
       matrix:
-        python_version: ${{ fromJSON(vars.PYTHON_VERSIONS) }}
+        python_version: ["3.10", "3.14"]
 
     steps:
       - name: Check out code

.github/workflows/lib-genai-tests.yml

@@ -33,13 +33,15 @@ jobs:
 
     strategy:
       fail-fast: true
-      # Throttle concurrent matrix jobs so we keep full Python coverage
-      # without re-multiplying live Vertex AI calls into 429 RESOURCE_EXHAUSTED
-      # against the shared opik-sdk-tests GCP project (shared with the ADK
-      # suite, so any uncapped suite re-multiplies their contention too).
+      # This suite makes live Vertex AI calls against the shared
+      # opik-sdk-tests GCP project (shared with the ADK suite). Running the
+      # full 3.10-3.14 matrix re-multiplied those calls into 429
+      # RESOURCE_EXHAUSTED, so we run only the oldest and newest supported
+      # Python versions — enough to catch version-specific breakage at a
+      # fraction of the request volume — and keep the concurrency cap.
       max-parallel: 2
       matrix:
-        python_version: ${{ fromJSON(vars.PYTHON_VERSIONS) }}
+        python_version: ["3.10", "3.14"]
 
     steps:
       - name: Check out code

sdks/python/tests/library_integration/genai/test_genai.py

@@ -5,7 +5,6 @@
 import pytest
 from google import genai
 from google.genai.types import HttpOptions, GenerateContentConfig
-from google.genai import errors as genai_errors
 import opik
 from opik.config import OPIK_PROJECT_DEFAULT_NAME
 from opik.integrations.genai import track_genai
@@ -21,7 +20,6 @@
     assert_dict_has_keys,
     assert_equal,
 )
-import tenacity
 
 pytestmark = pytest.mark.usefixtures("ensure_vertexai_configured")
 
@@ -38,19 +36,6 @@
 )
 
 
-def _is_rate_limit_error(exception: Exception) -> bool:
-    if isinstance(exception, genai_errors.ClientError):
-        return exception.response.status_code == 429
-    return False
-
-
-retry_with_waiting_on_rate_limit_errors = tenacity.retry(
-    stop=tenacity.stop_after_attempt(3),
-    wait=tenacity.wait_incrementing(start=5, increment=5),
-    retry=tenacity.retry_if_exception(_is_rate_limit_error),
-)
-
-
 def _assert_metadata_contains_required_keys(metadata: Dict[str, Any]):
     REQUIRED_METADATA_KEYS = [
         "model",
@@ -61,7 +46,6 @@ def _assert_metadata_contains_required_keys(metadata: Dict[str, Any]):
     assert_dict_has_keys(metadata, REQUIRED_METADATA_KEYS)
 
 
-@retry_with_waiting_on_rate_limit_errors
 @pytest.mark.parametrize(
     "project_name, expected_project_name",
     [
@@ -131,7 +115,6 @@ def test_genai_client__generate_content__happyflow(
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 def test_genai_client__async_generate_content__happyflow(fake_backend):
     client = genai.Client(
         vertexai=True,
@@ -187,7 +170,6 @@ def test_genai_client__async_generate_content__happyflow(fake_backend):
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 @pytest.mark.asyncio
 async def test_genai_client__async_generate_content__opik_args__happyflow(fake_backend):
     client = genai.Client(
@@ -254,7 +236,6 @@ async def test_genai_client__async_generate_content__opik_args__happyflow(fake_b
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 @pytest.mark.parametrize(
     "project_name, expected_project_name",
     [
@@ -331,7 +312,6 @@ def f():
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 def test_genai_client__async_generate_content_called_inside_another_tracked_function__happyflow(
     fake_backend,
 ):
@@ -400,7 +380,6 @@ async def f():
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 def test_genai_client__generate_content_stream__happyflow(fake_backend):
     client = genai.Client(
         vertexai=True,
@@ -459,7 +438,6 @@ def test_genai_client__generate_content_stream__happyflow(fake_backend):
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 def test_genai_client__async_generate_content_stream__happyflow(fake_backend):
     client = genai.Client(
         vertexai=True,
@@ -520,7 +498,6 @@ async def stream_example():
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 def test_genai_client__generate_content_stream_called_inside_another_tracked_function__generations_started_after_the_parent_span_closed__llm_span_attached_to_a_parent_function_span(
     fake_backend,
 ):
@@ -595,7 +572,6 @@ def f():
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 def test_genai_client__async_generate_content_stream_called_inside_another_tracked_function__generations_started_after_the_parent_span_closed__llm_span_has_a_separate_trace(
     fake_backend,
 ):
@@ -673,7 +649,6 @@ async def stream_outside_of_parent_function_example():
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 @pytest.mark.parametrize(
     "project_name, expected_project_name",
     [
@@ -755,7 +730,6 @@ def test_genai_client__generate_content__opik_args__happyflow(
     _assert_metadata_contains_required_keys(llm_span_metadata)
 
 
-@retry_with_waiting_on_rate_limit_errors
 def test_genai_client__generate_content__cost_callback__sets_span_total_cost(
     fake_backend,
 ):

sdks/python/tests/library_integration/genai/test_genai_videos.py

@@ -27,7 +27,6 @@
     assert_equal,
     patch_environ,
 )
-import tenacity
 
 pytestmark = [
     pytest.mark.usefixtures("ensure_vertexai_configured"),
@@ -57,24 +56,10 @@ def use_us_central1_for_veo():
         yield
 
 
-def _is_rate_limit_error(exception: Exception) -> bool:
-    if isinstance(exception, genai_errors.ClientError):
-        return exception.response.status_code == 429
-    return False
-
-
-retry_on_rate_limit = tenacity.retry(
-    stop=tenacity.stop_after_attempt(3),
-    wait=tenacity.wait_incrementing(start=5, increment=5),
-    retry=tenacity.retry_if_exception(_is_rate_limit_error),
-)
-
-
 @pytest.mark.skipif(
     SKIP_EXPENSIVE_TESTS,
     reason="Expensive tests disabled. Set OPIK_TEST_EXPENSIVE=1 to enable.",
 )
-@retry_on_rate_limit
 def test_genai_client__generate_videos_and_save__sync__happyflow(fake_backend):
     """
     Test sync video generation workflow: create -> wait -> save.
@@ -283,7 +268,6 @@ def test_genai_client__generate_videos_and_save__sync__happyflow(fake_backend):
     reason="Expensive tests disabled. Set OPIK_TEST_EXPENSIVE=1 to enable.",
 )
 @pytest.mark.asyncio
-@retry_on_rate_limit
 async def test_genai_client__generate_videos_and_save__async__happyflow(fake_backend):
     """
     Test async video generation workflow: create -> wait -> save.
@@ -571,7 +555,6 @@ def test_genai_client__generate_videos__error_handling(fake_backend):
     SKIP_EXPENSIVE_TESTS,
     reason="Expensive tests disabled. Set OPIK_TEST_EXPENSIVE=1 to enable.",
 )
-@retry_on_rate_limit
 def test_genai_client__generate_videos_with_upload_videos_disabled__no_attachment(
     fake_backend,
 ):

sdks/python/tests/library_integration/metrics_with_llm_judge/agentic/conftest.py

@@ -32,11 +32,13 @@
 #   canonical "doesn't call tools" failure mode (see
 #   `SupportedJudgeProvider.java`), so don't swap it in here without
 #   re-validating tool-use tests by hand.
-# - Anthropic `claude-haiku-4-5` (native `AnthropicChatModel`): the
-#   cheap+fast Claude tier. Sonnet would also work but the per-call
-#   latency hurts CI throughput given the agentic loop's multi-round
-#   nature.
-# - Anthropic `claude-haiku-4-5` *via LiteLLM* (`litellm_anthropic`):
+# - Anthropic `claude-sonnet-4-6` (native `AnthropicChatModel`): the
+#   Claude tier we run the agentic judge against. Haiku is cheaper but
+#   on the agentic path (tools in the request, so `response_format` is
+#   best-effort) it narrates in prose and wraps the verdict in a
+#   ```json fence rather than emitting the bare object the parser
+#   expects; Sonnet follows the structured-output contract reliably.
+# - Anthropic `claude-sonnet-4-6` *via LiteLLM* (`litellm_anthropic`):
 #   identical model string as the native row, but routed through the
 #   LiteLLM adapter by forcing `_should_use_anthropic_native=False`.
 #   This is the only parametrize entry that exercises Anthropic-by-
@@ -51,11 +53,11 @@
 _JUDGE_MODEL_PARAMS: List[Tuple[str, List[str]]] = [
     (llm_constants.OPENAI_GPT_4O_MINI, ["_skip_unless_openai_configured"]),
     (
-        f"{llm_constants.ANTHROPIC_CLAUDE_HAIKU}",
+        f"{llm_constants.ANTHROPIC_CLAUDE_SONNET}",
         ["_skip_unless_anthropic_configured"],
     ),
     (
-        f"{llm_constants.LITELLM_ANTHROPIC_CLAUDE_HAIKU}",
+        f"{llm_constants.LITELLM_ANTHROPIC_CLAUDE_SONNET}",
         ["_skip_unless_anthropic_configured", "_force_litellm_path"],
     ),
 ]