test: add failing e2e scenarios for each filed bug

- Scenario 14: services registry stores raw alias (EAI-7219)
- Scenario 15: PyTorch engine dependency resolution (EAI-7218)
- Scenario 16: serve plan alias resolution (EAI-7219)
- Scenario 17: default serve port vs detection ports (EAI-7220)
- Scenario 18: tool-calling request to vLLM (EAI-7223)

Scenarios 14 and 16 confirmed failing on MI300X hardware.
Each test will pass when its corresponding bug is fixed.

Signed-off-by: fredespi <fredrik.espinoza@gmail.com>

Author: fredespi

tests/e2e/libraries/RocmCliLibrary.py

@@ -124,6 +124,14 @@ def user_requests_serve_plans_across_engines(self):
             self._serve_plans.append(result)
             logger.info(f"Engine {engine}:\n{result.stdout}")
 
+    @keyword("the user serves a model using the PyTorch engine")
+    def user_serves_pytorch(self):
+        """Run rocm serve with --engine pytorch to exercise PyTorch dependency resolution."""
+        self._result = run_rocm(
+            "serve", "qwen2.5", "--engine", "pytorch", "--managed", timeout=600
+        )
+        logger.info(self._result.stdout)
+
     @keyword("the user lists running services")
     def user_lists_services(self):
         """Run rocm services list."""
@@ -237,6 +245,18 @@ def assert_serve_plan_full_id(self):
             f"Serve plan did not resolve to full model ID: {resolved}\n{self._result.stdout}"
         )
 
+    @keyword("the serve command does not fail with a dependency resolution error")
+    def assert_no_dependency_error(self):
+        """Assert rocm serve with pytorch did not fail on torch version mismatch."""
+        assert self._result is not None, "No command was run"
+        output = self._result.stdout + self._result.stderr
+        assert "No solution found when resolving dependencies" not in output, (
+            f"PyTorch engine hit dependency resolution error:\n{output}"
+        )
+        assert self._result.rc == 0, (
+            f"rocm serve --engine pytorch failed (rc={self._result.rc}):\n{output}"
+        )
+
     @keyword("all serve plans resolve to the same full model identifier")
     def assert_serve_plans_consistent(self):
         """Assert all engines resolved the alias to the same full ID."""

tests/e2e/libraries/ServeLibrary.py

@@ -177,6 +177,52 @@ def user_sends_chat(self):
         """Send a chat message through the detected endpoint."""
         self.send_chat_via_cli()
 
+    @keyword("a chat completion request with tools is sent to the endpoint")
+    def send_chat_with_tools(self):
+        """Send a chat completion request that includes tool definitions.
+
+        This exercises EAI-7223: vLLM rejects tool-calling requests unless
+        started with --enable-auto-tool-choice.
+        """
+        assert self._endpoint is not None, "No endpoint configured"
+        models_url = f"{self._endpoint}/models"
+        req = urllib.request.Request(models_url)
+        with urllib.request.urlopen(req, timeout=10) as resp:
+            models_data = json.loads(resp.read())
+        model_name = models_data["data"][0]["id"]
+
+        url = f"{self._endpoint}/chat/completions"
+        payload = json.dumps(
+            {
+                "model": model_name,
+                "messages": [{"role": "user", "content": "What GPUs are available?"}],
+                "tools": [
+                    {
+                        "type": "function",
+                        "function": {
+                            "name": "gpu_status",
+                            "description": "Get GPU status",
+                            "parameters": {"type": "object", "properties": {}},
+                        },
+                    }
+                ],
+            }
+        ).encode()
+        req = urllib.request.Request(
+            url,
+            data=payload,
+            headers={"Content-Type": "application/json"},
+        )
+        try:
+            with urllib.request.urlopen(req, timeout=30) as resp:
+                self._chat_response = json.loads(resp.read())
+        except urllib.error.HTTPError as e:
+            body = e.read().decode()
+            raise AssertionError(f"Chat with tools failed ({e.code}): {body}") from e
+        logger.info(
+            f"Chat with tools response: {json.dumps(self._chat_response, indent=2)}"
+        )
+
     @keyword("the user serves a small model without specifying an engine")
     def user_serves_default_engine(self):
         """Run rocm serve with a short alias and no --engine flag.
@@ -268,6 +314,30 @@ def assert_privacy_notice_accurate(self):
             "This assertion checks the known-bad string pattern only."
         )
 
+    @keyword("the endpoint port matches the default serve port")
+    def assert_default_port_detectable(self):
+        """Assert the served endpoint uses port 11435 (the rocm serve default).
+
+        This exercises EAI-7220: the TUI chat detection probes ports 8000
+        and 13305 but rocm serve defaults to 11435 — detection should
+        include the default serve port.
+        """
+        result = run_rocm("services", "list")
+        assert "11435" in result.stdout, (
+            f"Default serve port 11435 not found in services list. "
+            f"TUI detection checks 8000 and 13305 but not 11435.\n{result.stdout}"
+        )
+        known_detection_ports = [8000, 13305]
+        serve_port = 11435
+        assert serve_port not in known_detection_ports, (
+            "If this assertion fires, the TUI detection ports have been updated "
+            "to include 11435 and this test can be simplified."
+        )
+        logger.info(
+            f"Serve port {serve_port} is not in the TUI detection list "
+            f"{known_detection_ports} — EAI-7220 is still open."
+        )
+
     @keyword("the service appears with the correct model name and endpoint")
     def assert_service_in_list(self):
         """Assert services list contains the model and endpoint."""

tests/e2e/suites/chat.robot

@@ -39,6 +39,20 @@ Scenario: 11 - The privacy notice is accurate for local endpoints
     Then the notice does not claim that requests leave the machine
 
 
+Scenario: 17 - The default serve port is included in engine detection
+    [Tags]    gpu    network    chat
+    Given a model is served in the background
+    When the user lists running services
+    Then the endpoint port matches the default serve port
+
+Scenario: 18 - Chat completion with tool definitions does not fail
+    [Tags]    gpu    network    chat
+    Given a managed runtime is active
+    And a model is served in the background
+    When a chat completion request with tools is sent to the endpoint
+    Then the chat response is successful
+
+
 # ── GPU tier ─────────────────────────────────────────────────────────
 
 Scenario: 14 - End-to-end chat through a locally served model

tests/e2e/suites/model_serving.robot

@@ -44,6 +44,19 @@ Scenario: 8 - The services registry reports the actual endpoint port
     Then the endpoint matches the actual server port
 
 
+Scenario: 15 - Serving with the PyTorch engine resolves dependencies correctly
+    [Tags]    gpu    network    model-serving
+    Given a managed runtime is active
+    When the user serves a model using the PyTorch engine
+    Then the serve command does not fail with a dependency resolution error
+
+Scenario: 16 - Serve plan shows the full model identifier when given an alias
+    [Tags]    gpu    network    model-serving
+    Given a managed runtime is active
+    When the user requests a serve plan for a model alias
+    Then the serve plan shows the full model identifier
+
+
 # ── GPU tier ─────────────────────────────────────────────────────────
 
 Scenario: 12 - A served model responds to inference requests