Merge pull request #1992 from anik120/degraded-mode-tests

LCORE-1860: Add E2E tests for degraded mode startup scenarios

Author: tisnik

tests/e2e/configuration/server-mode/lightspeed-stack-degraded-mode.yaml

@@ -0,0 +1,25 @@
+name: Lightspeed Core Service (LCS) - Degraded Mode Test
+service:
+  host: 0.0.0.0
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  # Server mode - connects to separate llama-stack service
+  use_as_library_client: false
+  url: http://${env.E2E_LLAMA_HOSTNAME}:8321
+  api_key: xyzzy
+  # Enable degraded mode to allow startup without llama-stack
+  allow_degraded_mode: true
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: "/tmp/data/feedback"
+  transcripts_enabled: true
+  transcripts_storage: "/tmp/data/transcripts"
+authentication:
+  module: "noop"
+inference:
+  default_provider: openai
+  default_model: gpt-4o-mini

tests/e2e/features/degraded_mode_startup.feature

@@ -0,0 +1,40 @@
+@e2e_group_3 @skip-in-library-mode @Authorized
+Feature: Degraded mode startup
+
+  End-to-end scenarios that test LCORE startup behavior when llama-stack
+  is NOT available at startup time and allow_degraded_mode is enabled.
+
+  These tests verify that LCORE metrics correctly reflect startup state
+  in both healthy and degraded modes.
+
+  Background:
+    Given The service is started locally
+      And The system is in default state
+      And REST API service prefix is /v1
+      And the Lightspeed stack configuration directory is "tests/e2e/configuration"
+
+  Scenario: Degraded mode metric is set to 0.0 when started with llama-stack
+    Given The service uses the lightspeed-stack-degraded-mode.yaml configuration
+      And The service is restarted
+    When I access endpoint "metrics" using HTTP GET method
+    Then The status code of the response is 200
+    And The response body contains "ls_started_in_degraded_mode 0.0"
+
+  Scenario: Degraded mode metric is set to 1.0 when started without llama-stack
+    Given The llama-stack connection is disrupted
+      And The service uses the lightspeed-stack-degraded-mode.yaml configuration
+      And The service is restarted
+    When I access endpoint "metrics" using HTTP GET method
+    Then The status code of the response is 200
+    And The response body contains "ls_started_in_degraded_mode 1.0"
+
+  Scenario: Readiness endpoint reports degraded state when started without llama-stack
+    Given The llama-stack connection is disrupted
+      And The service uses the lightspeed-stack-degraded-mode.yaml configuration
+      And The service is restarted
+    When I access endpoint "readiness" using HTTP GET method
+    Then The status code of the response is 503
+    And The body of the response, ignoring the "providers" field, is the following
+    """
+    {"ready": false, "reason": "Cannot connect to backend service", "overall_status": "unhealthy", "impacts": ["LLM inference unavailable", "Provider health checks unavailable"]}
+    """

tests/e2e/features/steps/common_http.py

@@ -112,6 +112,19 @@ def check_response_body_does_not_contain(context: Context, substring: str) -> No
     ), f"The response text '{context.response.text}' contains '{substring}'"
 
 
+@then('The response body contains "{substring}"')
+def check_response_contains_substring(context: Context, substring: str) -> None:
+    """Check that response body contains a specific substring.
+
+    This step handles quoted strings and performs exact matching
+    (case-sensitive) for metrics and structured output validation.
+    """
+    assert context.response is not None, "Request needs to be performed first"
+    assert (
+        substring in context.response.text
+    ), f"Expected '{substring}' in response, but got: {context.response.text[:200]}"
+
+
 @then("The body of the response is the following")
 def check_prediction_result(context: Context) -> None:
     """Check the content of the response to be exactly the same.