Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/e2e/features/query.feature
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ Scenario: Check if LLM responds for query request with error for missing query
{"query": "Say hello", "model": "{MODEL}", "provider":"unknown"}
"""
Then The status code of the response is 404
And The body of the response contains Model with ID gpt-4o-mini does not exist
And The body of the response contains Model with ID {MODEL} does not exist

@skip-in-library-mode
Scenario: Check if LLM responds for query request with error for inability to connect to llama-stack
Expand Down
12 changes: 9 additions & 3 deletions tests/e2e/features/steps/common_http.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,11 +165,17 @@ def check_response_body_schema(context: Context) -> None:

@then("The body of the response contains {substring}")
def check_response_body_contains(context: Context, substring: str) -> None:
"""Check that response body contains a substring."""
"""Check that response body contains a substring.

Supports {MODEL} and {PROVIDER} placeholders in the substring so
assertions work with any configured provider (e.g. unknown-provider
error message includes the actual model id).
"""
assert context.response is not None, "Request needs to be performed first"
expected = replace_placeholders(context, substring)
assert (
substring in context.response.text
), f"The response text '{context.response.text}' doesn't contain '{substring}'"
expected in context.response.text
), f"The response text '{context.response.text}' doesn't contain '{expected}'"


@then("The body of the response is the following")
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/features/streaming_query.feature
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ Feature: streaming_query endpoint API tests
{"query": "Say hello", "model": "{MODEL}", "provider":"unknown"}
"""
Then The status code of the response is 404
And The body of the response contains Model with ID gpt-4o-mini does not exist
And The body of the response contains Model with ID {MODEL} does not exist
And The token metrics should not have changed

Scenario: Check if LLM responds properly when XML and JSON attachments are sent
Expand Down
Loading