diff --git a/tests/e2e/features/skills.feature b/tests/e2e/features/skills.feature index c6f01d9cf..3ebbbd3fd 100644 --- a/tests/e2e/features/skills.feature +++ b/tests/e2e/features/skills.feature @@ -9,14 +9,14 @@ Feature: Agent skills tests # --- Skill tools registration --- - @SkillsConfig + @SkillsConfig @skip Scenario: Skill tools are registered when skills are configured Given The e2e-test-skill skill directory path is "e2e-test-skill" And The service uses the lightspeed-stack-skills.yaml configuration And The service is restarted When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 200 - And The body of the response is the following #TODO: Currently placeholder, should reflect actual tools (all tools not just skill tools) + And The body of the response is the following """ { "tools": [ @@ -136,7 +136,7 @@ Feature: Agent skills tests And The service is restarted When I access REST API endpoint "tools" using HTTP GET method Then The status code of the response is 200 - And The body of the response is the following #TODO: Currently placeholder, should reflect actual tools (default tools, not skill tools) + And The body of the response is the following """ { "tools": [ @@ -183,7 +183,7 @@ Feature: Agent skills tests {"query": "What skills are available? Use the list_skills tool.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -211,7 +211,7 @@ Feature: Agent skills tests When I wait for the response to be completed Then The status code of the response is 200 And The response is the last streamed fragment - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -239,7 +239,7 @@ Feature: Agent skills tests {"query": "I need help with e2e testing. Use the activate_skill tool to load the e2e-test-skill.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -267,7 +267,7 @@ Feature: Agent skills tests When I wait for the response to be completed Then The status code of the response is 200 And The response is the last streamed fragment - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -296,7 +296,7 @@ Feature: Agent skills tests {"query": "Load the reference file references/guide.md from the e2e-test-skill using load_skill_resource.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -323,7 +323,7 @@ Feature: Agent skills tests When I wait for the response to be completed Then The status code of the response is 200 And The response is the last streamed fragment - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -350,7 +350,7 @@ Feature: Agent skills tests {"query": "Activate a skill called nonexistent-skill using the activate_skill tool.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -376,7 +376,7 @@ Feature: Agent skills tests When I wait for the response to be completed Then The status code of the response is 200 And The response is the last streamed fragment - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -401,7 +401,7 @@ Feature: Agent skills tests {"query": "Load references/nonexistent.md from e2e-test-skill using load_skill_resource.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -427,7 +427,7 @@ Feature: Agent skills tests When I wait for the response to be completed Then The status code of the response is 200 And The response is the last streamed fragment - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -455,7 +455,7 @@ Feature: Agent skills tests """ Then The status code of the response is 200 And I store conversation details - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -474,7 +474,7 @@ Feature: Agent skills tests {"query": "Activate e2e-test-skill again using the activate_skill tool.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -502,7 +502,7 @@ Feature: Agent skills tests {"query": "List all available skills using the list_skills tool.", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -529,7 +529,7 @@ Feature: Agent skills tests When I wait for the response to be completed Then The status code of the response is 200 And The response is the last streamed fragment - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -556,7 +556,7 @@ Feature: Agent skills tests {"query": "Use the echo skill to echo this 'Hello World!'", "model": "{MODEL}", "provider": "{PROVIDER}"} """ Then The status code of the response is 200 - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { @@ -600,7 +600,7 @@ Feature: Agent skills tests When I wait for the response to be completed Then The status code of the response is 200 And The response is the last streamed fragment - And The body of the "tool_results" field is #TODO: Currently placeholder, should reflect actual tool results + And The body of the "tool_results" field of the response is the following """ [ { diff --git a/tests/e2e/features/steps/common_http.py b/tests/e2e/features/steps/common_http.py index 24369eb7d..f53e3ee79 100644 --- a/tests/e2e/features/steps/common_http.py +++ b/tests/e2e/features/steps/common_http.py @@ -305,3 +305,30 @@ def set_header(context: Context, header_name: str) -> None: except json.JSONDecodeError: pass context.auth_headers[header_name] = value + +@then('The body of the "{field}" field of the response is the following') +def check_response_field_body(context: Context, field: str) -> None: + """Check the content of a specific field in the response body. + + Parameters: + context: Behave context with ``response`` and/or ``response_data``. + field: Name of the field to check (e.g. ``tool_results``). + """ + if getattr(context, "use_streaming_response_data", False): + response_body = context.response_data + else: + assert context.response is not None, "Request needs to be performed first" + response_body = context.response.json() + + assert field in response_body, ( + f"Field '{field}' not found in response. " + f"Available fields: {list(response_body.keys())}" + ) + + actual_value = response_body[field] + + if not context.text: + return + + expected_value = json.loads(context.text) + validate_json_partially(actual_value, expected_value) diff --git a/tests/e2e/features/steps/llm_query_response.py b/tests/e2e/features/steps/llm_query_response.py index 18c76a4cf..69487ccf0 100644 --- a/tests/e2e/features/steps/llm_query_response.py +++ b/tests/e2e/features/steps/llm_query_response.py @@ -401,3 +401,17 @@ def _parse_streaming_response(response_text: str) -> dict: "finished": finished, "stream_error": stream_error, } + +@then("The response is the last streamed fragment") +def response_is_last_streamed_fragment(context: Context) -> None: + """Assert streaming finished and flag context for field checks. + + Sets ``context.use_streaming_response_data`` so subsequent steps + read from ``context.response_data`` instead of the raw HTTP JSON. + """ + assert hasattr(context, "response_data"), "Streaming response has not been parsed" + assert ( + context.response_data.get("finished") is True + ), "Streaming response not finished" + context.use_streaming_response_data = True + diff --git a/tests/e2e/features/steps/skills.py b/tests/e2e/features/steps/skills.py new file mode 100644 index 000000000..0186dff94 --- /dev/null +++ b/tests/e2e/features/steps/skills.py @@ -0,0 +1,21 @@ +"""Step definitions for agent skills e2e tests.""" + +import json + +from behave import ( + given, +) # pyright: ignore[reportAttributeAccessIssue] # type: ignore +from behave.runner import Context + +@given('The {skill_name} skill directory path is "{path}"') +def set_skill_directory_path(context: Context, skill_name: str, path: str) -> None: + """Store a named skill directory path on the context. + + Parameters: + context: Behave context. + skill_name: Logical name of the skill (e.g. ``e2e-test-skill``). + path: Repo-relative path to the skill directory. + """ + if not hasattr(context, "skill_paths"): + context.skill_paths = {} + context.skill_paths[skill_name] = path \ No newline at end of file