LCORE-1270: Added e2e tests for responses endpoint

Author: Radovan Fuchs

tests/e2e/features/responses.feature

@@ -12,6 +12,7 @@
 from behave.runner import Context
 
 from tests.e2e.utils.utils import (
+    http_response_json_or_responses_sse_terminal,
     normalize_endpoint,
     replace_placeholders,
     validate_json,
@@ -171,11 +172,16 @@ def check_response_body_contains(context: Context, substring: str) -> None:
     Supports {MODEL} and {PROVIDER} placeholders in the substring so
     assertions work with any configured provider (e.g. unknown-provider
     error message includes the actual model id).
+
+    Matching is case-insensitive so LLM replies (e.g. ``Hello`` vs ``hello``)
+    do not fail otherwise-identical scenarios.
     """
     assert context.response is not None, "Request needs to be performed first"
     expected = replace_placeholders(context, substring)
+    haystack = context.response.text.lower()
+    needle = expected.lower()
     assert (
-        expected in context.response.text
+        needle in haystack
     ), f"The response text '{context.response.text}' doesn't contain '{expected}'"
 
 
@@ -425,7 +431,7 @@ def check_response_partially(context: Context) -> None:
     in `context.text`, ignoring extra keys or values not specified.
     """
     assert context.response is not None, "Request needs to be performed first"
-    body = context.response.json()
+    body = http_response_json_or_responses_sse_terminal(context.response)
     json_str = replace_placeholders(context, context.text or "{}")
     expected = json.loads(json_str)
     validate_json_partially(body, expected)

tests/e2e/features/responses_streaming.feature

@@ -58,6 +58,68 @@ def access_conversation_endpoint_get_specific(
     context.response = requests.get(url, headers=headers, timeout=DEFAULT_TIMEOUT)
 
 
+@step(
+    "I use REST API conversation endpoint with the forked responses conversation id using HTTP GET method"
+)
+def access_forked_responses_conversation_get(context: Context) -> None:
+    """GET /conversations/{id} using the id stored after a responses API fork."""
+    assert hasattr(
+        context, "responses_fork_conversation_id"
+    ), "responses_fork_conversation_id not set; store fork id first"
+    endpoint = "conversations"
+    base = f"http://{context.hostname}:{context.port}"
+    path = (
+        f"{context.api_prefix}/{endpoint}/{context.responses_fork_conversation_id}"
+    ).replace("//", "/")
+    url = base + path
+    headers = context.auth_headers if hasattr(context, "auth_headers") else {}
+    context.response = requests.get(url, headers=headers, timeout=DEFAULT_TIMEOUT)
+
+
+@step(
+    "I use REST API conversation endpoint with the responses multi-turn baseline conversation id using HTTP GET method"
+)
+def access_responses_baseline_conversation_get(context: Context) -> None:
+    """GET /conversations/{id} for the linear thread before a responses fork."""
+    assert hasattr(
+        context, "responses_multi_turn_baseline_conversation_id"
+    ), "responses_multi_turn_baseline_conversation_id not set"
+    endpoint = "conversations"
+    base = f"http://{context.hostname}:{context.port}"
+    cid = context.responses_multi_turn_baseline_conversation_id
+    path = f"{context.api_prefix}/{endpoint}/{cid}".replace("//", "/")
+    url = base + path
+    headers = context.auth_headers if hasattr(context, "auth_headers") else {}
+    context.response = requests.get(url, headers=headers, timeout=DEFAULT_TIMEOUT)
+
+
+@then("The GET conversation response id matches the forked responses conversation id")
+def check_get_matches_forked_responses_conversation_id(context: Context) -> None:
+    """Assert GET /conversations payload matches the fork id from /v1/responses."""
+    assert context.response is not None
+    assert hasattr(context, "responses_fork_conversation_id")
+    response_json = context.response.json()
+    assert response_json["conversation_id"] == context.responses_fork_conversation_id, (
+        f"expected conversation_id {context.responses_fork_conversation_id!r}, "
+        f"got {response_json.get('conversation_id')!r}"
+    )
+
+
+@then(
+    "The GET conversation response id matches the responses multi-turn baseline conversation id"
+)
+def check_get_matches_responses_baseline_conversation_id(context: Context) -> None:
+    """Assert GET /conversations payload matches the pre-fork thread id."""
+    assert context.response is not None
+    assert hasattr(context, "responses_multi_turn_baseline_conversation_id")
+    expected = context.responses_multi_turn_baseline_conversation_id
+    response_json = context.response.json()
+    assert response_json["conversation_id"] == expected, (
+        f"expected conversation_id {expected!r}, "
+        f"got {response_json.get('conversation_id')!r}"
+    )
+
+
 @when(
     "I use REST API conversation endpoint with conversation_id from above using HTTP DELETE method"
 )

tests/e2e/features/steps/common_http.py

@@ -58,7 +58,10 @@ def ask_question_authorized(context: Context, endpoint: str) -> None:
     json_str = replace_placeholders(context, context.text or "{}")
 
     data = json.loads(json_str)
-    if endpoint == "streaming_query":
+    use_sse = endpoint == "streaming_query" or (
+        endpoint == "responses" and bool(data.get("stream"))
+    )
+    if use_sse:
         resp = requests.post(
             url,
             json=data,

tests/e2e/features/steps/conversation.py

@@ -0,0 +1,80 @@
+"""Behave steps for POST /v1/responses (LCORE Responses API) multi-turn tests."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from behave import step  # pyright: ignore[reportAttributeAccessIssue]
+from behave.runner import Context
+
+from tests.e2e.utils.utils import http_response_json_or_responses_sse_terminal
+
+
+def _successful_responses_result_object(context: Context) -> dict[str, Any]:
+    """Return the LCORE responses payload for a successful call (JSON or SSE)."""
+    assert context.response is not None
+    assert context.response.status_code == 200, context.response.text
+    data = http_response_json_or_responses_sse_terminal(context.response)
+    assert "id" in data and "conversation" in data, data
+    return data
+
+
+@step("I store the first responses turn from the last response")
+def store_first_responses_turn(context: Context) -> None:
+    """Save response id and conversation from a successful responses call (JSON or SSE)."""
+    assert context.response is not None, "Request needs to be performed first"
+    data = _successful_responses_result_object(context)
+    context.responses_first_response_id = data["id"]
+    context.responses_conversation_id = data["conversation"]
+
+
+@step("I store the multi-turn baseline from the last responses response")
+def store_multi_turn_baseline(context: Context) -> None:
+    """Save conversation id after a continuation turn (thread before an optional fork)."""
+    assert context.response is not None, "Request needs to be performed first"
+    data = _successful_responses_result_object(context)
+    context.responses_multi_turn_baseline_conversation_id = data["conversation"]
+    context.responses_second_response_id = data["id"]
+
+
+@step("I store the forked responses conversation id from the last response")
+def store_forked_responses_conversation_id(context: Context) -> None:
+    """Save conversation id returned after branching from a non-latest ``previous_response_id``."""
+    assert context.response is not None, "Request needs to be performed first"
+    data = _successful_responses_result_object(context)
+    context.responses_fork_conversation_id = data["conversation"]
+
+
+@step("The responses conversation id is different from the multi-turn baseline")
+def responses_conversation_differs_from_baseline(context: Context) -> None:
+    """Assert fork applied: returned conversation is not the baseline thread id."""
+    assert context.response is not None
+    data = _successful_responses_result_object(context)
+    baseline = context.responses_multi_turn_baseline_conversation_id
+    current = data["conversation"]
+    assert (
+        current != baseline
+    ), f"Expected a new conversation after fork, got same as baseline {baseline!r}"
+
+
+@step("The responses conversation id matches the multi-turn baseline")
+def responses_conversation_matches_baseline(context: Context) -> None:
+    """Assert continuation on latest turn keeps the same normalized conversation id."""
+    assert context.response is not None
+    data = _successful_responses_result_object(context)
+    baseline = context.responses_multi_turn_baseline_conversation_id
+    current = data["conversation"]
+    assert current == baseline, f"Expected conversation {baseline!r}, got {current!r}"
+
+
+@step("The responses conversation id matches the first stored conversation")
+def responses_conversation_matches_first_stored(context: Context) -> None:
+    """Assert the response uses the same thread as ``I store the first responses turn``."""
+    assert context.response is not None
+    assert hasattr(
+        context, "responses_conversation_id"
+    ), "responses_conversation_id not set; run the first-store step first"
+    data = _successful_responses_result_object(context)
+    expected = context.responses_conversation_id
+    current = data["conversation"]
+    assert current == expected, f"Expected conversation {expected!r}, got {current!r}"

tests/e2e/features/steps/llm_query_response.py

@@ -10,13 +10,11 @@ features/feedback.feature
 features/health.feature
 features/info.feature
 features/responses.feature
+features/responses_streaming.feature
 features/query.feature
 features/rlsapi_v1.feature
 features/rlsapi_v1_errors.feature
 features/streaming_query.feature
 features/rest_api.feature
 features/mcp.feature
-features/mcp_servers_api.feature
-features/mcp_servers_api_auth.feature
-features/mcp_servers_api_no_config.feature
 features/models.feature

tests/e2e/features/steps/responses_steps.py

@@ -1,5 +1,6 @@
 """Unsorted utility functions to be used from other sources and test step definitions."""
 
+import json
 import os
 import shutil
 import subprocess
@@ -163,6 +164,50 @@ def validate_json_partially(actual: Any, expected: Any) -> None:
         assert actual == expected, f"Value mismatch: expected {expected}, got {actual}"
 
 
+RESPONSES_SSE_TERMINAL_EVENT_TYPES = frozenset(
+    {"response.completed", "response.incomplete", "response.failed"}
+)
+
+
+def parse_responses_sse_final_response_object(text: str) -> dict[str, Any]:
+    """Return the ``response`` object from the last terminal LCORE ``/responses`` SSE event."""
+    last: dict[str, Any] | None = None
+    for line in text.splitlines():
+        stripped = line.strip()
+        if not stripped.startswith("data:"):
+            continue
+        payload = stripped[5:].strip()
+        if payload == "[DONE]":
+            continue
+        try:
+            obj = json.loads(payload)
+        except json.JSONDecodeError:
+            continue
+        if obj.get("type") in RESPONSES_SSE_TERMINAL_EVENT_TYPES and isinstance(
+            obj.get("response"), dict
+        ):
+            last = obj["response"]
+    if last is None:
+        raise AssertionError(
+            "No terminal responses SSE event (completed/incomplete/failed) found in body"
+        )
+    return last
+
+
+def http_response_json_or_responses_sse_terminal(response: Any) -> Any:
+    """Decode JSON body or, for streaming ``/responses`` SSE, the terminal ``response`` object.
+
+    Non-SST responses use ``response.json()`` unchanged.
+    """
+    text = response.text
+    content_type = (response.headers.get("content-type") or "").lower()
+    if "text/event-stream" in content_type or (
+        "event:" in text and "data:" in text and "[DONE]" in text
+    ):
+        return parse_responses_sse_final_response_object(text)
+    return response.json()
+
+
 def switch_config(
     source_path: str, destination_path: str = "lightspeed-stack.yaml"
 ) -> None:
@@ -320,4 +365,17 @@ def replace_placeholders(context: Context, text: str) -> str:
     """
     result = text.replace("{MODEL}", context.default_model)
     result = result.replace("{PROVIDER}", context.default_provider)
-    return result.replace("{VECTOR_STORE_ID}", context.faiss_vector_store_id)
+    result = result.replace("{VECTOR_STORE_ID}", context.faiss_vector_store_id)
+    if hasattr(context, "responses_first_response_id"):
+        result = result.replace(
+            "{RESPONSES_FIRST_RESPONSE_ID}", context.responses_first_response_id
+        )
+    if hasattr(context, "responses_conversation_id"):
+        result = result.replace(
+            "{RESPONSES_CONVERSATION_ID}", context.responses_conversation_id
+        )
+    if hasattr(context, "responses_second_response_id"):
+        result = result.replace(
+            "{RESPONSES_SECOND_RESPONSE_ID}", context.responses_second_response_id
+        )
+    return result