From c9848f5c6b435effa248022989557a37037553cc Mon Sep 17 00:00:00 2001 From: Radovan Fuchs Date: Wed, 11 Mar 2026 09:44:00 +0100 Subject: [PATCH] fix logic in e2e tests to handle all providers --- tests/e2e/features/query.feature | 2 +- tests/e2e/features/steps/common_http.py | 12 +++++++++--- tests/e2e/features/streaming_query.feature | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tests/e2e/features/query.feature b/tests/e2e/features/query.feature index ac43b786b..0f4fe1038 100644 --- a/tests/e2e/features/query.feature +++ b/tests/e2e/features/query.feature @@ -175,7 +175,7 @@ Scenario: Check if LLM responds for query request with error for missing query {"query": "Say hello", "model": "{MODEL}", "provider":"unknown"} """ Then The status code of the response is 404 - And The body of the response contains Model with ID gpt-4o-mini does not exist + And The body of the response contains Model with ID {MODEL} does not exist @skip-in-library-mode Scenario: Check if LLM responds for query request with error for inability to connect to llama-stack diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 8e64fe5fc..016791fb6 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -165,11 +165,17 @@ def check_response_body_schema(context: Context) -> None: @then("The body of the response contains {substring}") def check_response_body_contains(context: Context, substring: str) -> None: - """Check that response body contains a substring.""" + """Check that response body contains a substring. + + Supports {MODEL} and {PROVIDER} placeholders in the substring so + assertions work with any configured provider (e.g. unknown-provider + error message includes the actual model id). + """ assert context.response is not None, "Request needs to be performed first" + expected = replace_placeholders(context, substring) assert ( - substring in context.response.text - ), f"The response text '{context.response.text}' doesn't contain '{substring}'" + expected in context.response.text + ), f"The response text '{context.response.text}' doesn't contain '{expected}'" @then("The body of the response is the following") diff --git a/tests/e2e/features/streaming_query.feature b/tests/e2e/features/streaming_query.feature index 22b3255b9..3872e5343 100644 --- a/tests/e2e/features/streaming_query.feature +++ b/tests/e2e/features/streaming_query.feature @@ -133,7 +133,7 @@ Feature: streaming_query endpoint API tests {"query": "Say hello", "model": "{MODEL}", "provider":"unknown"} """ Then The status code of the response is 404 - And The body of the response contains Model with ID gpt-4o-mini does not exist + And The body of the response contains Model with ID {MODEL} does not exist And The token metrics should not have changed Scenario: Check if LLM responds properly when XML and JSON attachments are sent