Fix and improve integration tests (#1996)

* Fix docker builds for the local sandbox
* Disable raga ai e2e test as https://catalyst.raga.ai is currently down'
* Rename `test_tool_wrapper.py` to `test_lc_tool_wrapper.py` avoiding a pytest name clash
* Update agent tests to accept a list of acceptable answers (ex: allowing both "LLM" and "large language model")
* Workflow improvements:
  - Replacing older models with a nemotron model
  - Increasing max tokens and disabling thinking to prevent truncation

## By Submitting this PR I confirm:
- I am familiar with the [Contributing Guidelines](https://github.com/NVIDIA/NeMo-Agent-Toolkit/blob/develop/docs/source/resources/contributing/index.md).
- We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license.
  - Any contribution which contains commits that are not Signed-Off will not be accepted.
- When the PR is ready for review, new or existing tests cover these changes.
- When the PR is ready for review, the documentation is up to date with these changes.



## Summary by CodeRabbit

* **New Features**
  * N/A

* **Bug Fixes**
  * Skipped flaky external-service test to avoid CI failures

* **Chores**
  * Updated example configs and notebook to use a different recommended model, increase max tokens to 1024, and disable “thinking”
  * Adjusted CI image tag and JSON model counts
  * Made Docker upgrades non-interactive

* **Tests**
  * Enhanced test utilities and fixtures to accept and validate multiple acceptable answers



[![Review Change Stack](https://storage.googleapis.com/coderabbit_public_assets/review-stack-in-coderabbit-ui.svg)](https://app.coderabbit.ai/change-stack/NVIDIA/NeMo-Agent-Toolkit/pull/1996?utm_source=github_walkthrough&utm_medium=github&utm_campaign=change_stack)

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Yuchen Zhang (https://github.com/yczhang-nv)
  - Will Killian (https://github.com/willkill07)

URL: #1996

Author: dagardner-nv

.gitlab-ci.yml

@@ -134,7 +134,7 @@ test:python_tests:
       alias: phoenix
       pull_policy: if-not-present
 
-    - name: $CI_REGISTRY_IMAGE/nginx-rewrite-models:20260303
+    - name: $CI_REGISTRY_IMAGE/nginx-rewrite-models:20260529
       alias: nginx-rewrite-models
       pull_policy: if-not-present
 

ci/.nim_models_used.json

@@ -3,11 +3,11 @@
   "llms": [
     {
       "model": "meta/llama-3.1-70b-instruct",
-      "num_configs": 53
+      "num_configs": 51
     },
     {
       "model": "nvidia/nemotron-3-nano-30b-a3b",
-      "num_configs": 40
+      "num_configs": 42
     },
     {
       "model": "meta/llama-3.3-70b-instruct",

examples/agents/react/configs/config-reasoning.yml

@@ -17,9 +17,12 @@
 llms:
   nim_llm:
     _type: nim
-    model_name: meta/llama-3.1-70b-instruct
+    model_name: nvidia/nemotron-3-nano-30b-a3b
     temperature: 0.0
-    max_tokens: 250
+    max_tokens: 1024
+    chat_template_kwargs:
+      enable_thinking: false
+
   nemotron_model:
     _type: nim
     model_name: nvidia/llama-3.3-nemotron-super-49b-v1

examples/agents/react/configs/config.yml

@@ -19,7 +19,9 @@ llms:
     _type: nim
     model_name: nvidia/nemotron-3-nano-30b-a3b
     temperature: 0.0
-    max_tokens: 250
+    max_tokens: 1024
+    chat_template_kwargs:
+      enable_thinking: false
 
 functions:
   wikipedia_search:

examples/agents/tests/conftest.py

@@ -43,9 +43,9 @@ def fixture_question() -> str:
     return "What are LLMs"
 
 
-@pytest.fixture(name="answer", scope="session")
-def fixture_answer() -> str:
-    return "large language model"
+@pytest.fixture(name="expected_answers", scope="session")
+def fixture_expected_answers() -> list[str]:
+    return ["llm", "large language model"]
 
 
 @pytest.fixture(name="rewoo_data", scope="session")

examples/agents/tests/test_agents.py

@@ -35,10 +35,14 @@ def _extract_serve_response_text(response_json: dict) -> str:
     return "\n".join(combined)
 
 
-def _assert_expected_answer(result: str, expected_answer: str) -> None:
+def _assert_expected_answer(result: str, expected_answers: str | list[str]) -> None:
     """Assert that the expected answer appears in the result, normalizing whitespace and case."""
-    normalized = ' '.join(result.split())
-    assert expected_answer.lower() in normalized.lower(), f"Expected '{expected_answer}' in '{result}'"
+    normalized = ' '.join(result.split()).lower()
+    if isinstance(expected_answers, str):
+        expected_answers = [expected_answers]
+
+    assert any(expected.lower() in normalized for expected in expected_answers), \
+        f"Expected one of '{expected_answers}' in '{result}'"
 
 
 # ---------------------------------------------------------------------------
@@ -82,19 +86,21 @@ async def test_question(self, rewoo_nat_client, rewoo_data: list[dict], qa_idx:
 
 @pytest.mark.integration
 @pytest.mark.usefixtures("openai_api_key")
-async def test_tool_calling_responses_api(agents_dir: Path, question: str, answer: str):
+async def test_tool_calling_responses_api(agents_dir: Path, question: str, expected_answers: list[str]):
     await run_workflow(config_file=agents_dir / "tool_calling/configs/config-responses-api.yml",
                        question=question,
-                       expected_answer=answer)
+                       expected_answer=expected_answers)
 
 
 @pytest.mark.integration
 @pytest.mark.usefixtures("openai_api_key")
-async def test_nat_run_tool_calling_responses_api(tool_calling_responses_api_nat_client, question: str, answer: str):
+async def test_nat_run_tool_calling_responses_api(tool_calling_responses_api_nat_client,
+                                                  question: str,
+                                                  expected_answers: list[str]):
     resp = await tool_calling_responses_api_nat_client.post("/generate", json={"input_message": question})
     resp.raise_for_status()
     response_text = _extract_serve_response_text(resp.json())
-    _assert_expected_answer(response_text, answer)
+    _assert_expected_answer(response_text, expected_answers)
 
 
 # ---------------------------------------------------------------------------
@@ -107,23 +113,23 @@ async def test_nat_run_tool_calling_responses_api(tool_calling_responses_api_nat
 @pytest.mark.usefixtures("nvidia_api_key")
 class TestAgentNatRun:
 
-    async def test_question(self, agent_session_manager, question: str, answer: str):
+    async def test_question(self, agent_session_manager, question: str, expected_answers: list[str]):
         async with agent_session_manager.session() as session:
             async with session.run(question) as runner:
                 result = await runner.result(to_type=str)
-                _assert_expected_answer(result, answer)
+                _assert_expected_answer(result, expected_answers)
 
 
 @pytest.mark.slow
 @pytest.mark.integration
 @pytest.mark.usefixtures("nvidia_api_key")
 class TestAgentNatServe:
 
-    async def test_question(self, agent_nat_client, question: str, answer: str):
+    async def test_question(self, agent_nat_client, question: str, expected_answers: list[str]):
         resp = await agent_nat_client.post("/generate", json={"messages": [{"role": "user", "content": question}]})
         resp.raise_for_status()
         response_text = _extract_serve_response_text(resp.json())
-        _assert_expected_answer(response_text, answer)
+        _assert_expected_answer(response_text, expected_answers)
 
 
 # Code examples from `docs/source/resources/running-tests.md`

examples/agents/tool_calling/configs/config-reasoning.yml

@@ -19,7 +19,10 @@ llms:
     _type: nim
     model_name: nvidia/nemotron-3-nano-30b-a3b
     temperature: 0.0
-    max_tokens: 250
+    max_tokens: 1024
+    chat_template_kwargs:
+      enable_thinking: false
+
   reasoning_model:
     _type: nim
     model_name: nvidia/llama-3.3-nemotron-super-49b-v1

examples/agents/tool_calling/configs/config.yml

@@ -17,9 +17,11 @@
 llms:
   nim_llm:
     _type: nim
-    model_name: meta/llama-3.1-70b-instruct
+    model_name: nvidia/nemotron-3-nano-30b-a3b
     temperature: 0.0
-    max_tokens: 250
+    max_tokens: 1024
+    chat_template_kwargs:
+      enable_thinking: false
 
 functions:
   wikipedia_search:

examples/notebooks/hello_world.ipynb

@@ -179,7 +179,7 @@
     "   # Tell NeMo Agent Toolkit which LLM to use for the agent\n",
     "   nim_llm:\n",
     "      _type: nim\n",
-    "      model_name: meta/llama-3.1-70b-instruct\n",
+    "      model_name: nvidia/nemotron-3-nano-30b-a3b\n",
     "      temperature: 0.0\n",
     "\n",
     "workflow:\n",

examples/observability/simple_calculator_observability/tests/test_simple_calc_observability.py

@@ -242,6 +242,7 @@ async def test_galileo_full_workflow(config_dir: Path,
     assert len(spans.records) > 1
 
 
+@pytest.mark.skip(reason="https://catalyst.raga.ai appears to be having issues")
 @pytest.mark.integration
 @pytest.mark.usefixtures("catalyst_keys", "aiq_compatibility_span_prefix")
 async def test_catalyst_full_workflow(config_dir: Path,