Merge pull request #2221 from anik120/otel-intrumentation

LCORE-1801: Wrap lightspeed-stack startup with opentelemetry-instrument

Author: tisnik

Makefile

@@ -36,7 +36,7 @@ CONTAINER_RUNTIME ?= $(shell command -v podman 2>/dev/null || command -v docker
 	docs/models/database.svg
 
 run-stack: ## Run lightspeed-stack directly, without building dependent service/s
-	uv run src/lightspeed_stack.py -c $(CONFIG)
+	uv run opentelemetry-instrument python3.12 src/lightspeed_stack.py -c $(CONFIG)
 
 run: start-llama-stack-container ## Run the service locally with dependent services
 	@echo "Starting Lightspeed Core Stack..."

README.md

@@ -1130,7 +1130,7 @@ ENV PATH="/app-root/.venv/bin:$PATH"
 
 # Run the application
 EXPOSE 8080
-ENTRYPOINT ["python3.12", "src/lightspeed_stack.py"]
+ENTRYPOINT ["opentelemetry-instrument", "python3.12", "src/lightspeed_stack.py"]
 USER 1001
 ```
 

deploy/lightspeed-stack/Containerfile

@@ -135,7 +135,7 @@ ENV PYTHONPATH="/app-root"
 
 # Run the application
 EXPOSE 8080
-ENTRYPOINT ["python3.12", "src/lightspeed_stack.py"]
+ENTRYPOINT ["opentelemetry-instrument", "python3.12", "src/lightspeed_stack.py"]
 
 LABEL vendor="Red Hat, Inc." \
     name="lightspeed-core/lightspeed-stack-rhel9" \

docs/user_doc/deployment_guide.md

@@ -468,7 +468,7 @@ make run
 ```
 
 ```
-uv run src/lightspeed_stack.py
+uv run opentelemetry-instrument python3.12 src/lightspeed_stack.py
 [07/29/25 15:43:35] INFO     Initializing app                                                                                 main.py:19
                     INFO     Including routers                                                                                main.py:68
 INFO:     Started server process [1922983]
@@ -561,7 +561,7 @@ cp examples/lightspeed-stack-lls-library.yaml lightspeed-stack.yaml
     ```
 1. Check the output
     ```text
-    uv run src/lightspeed_stack.py
+    uv run opentelemetry-instrument python3.12 src/lightspeed_stack.py
     Using config run.yaml:
     apis:
     - agents

tests/e2e-prow/rhoai/manifests/lightspeed/lightspeed-stack.yaml

@@ -37,7 +37,7 @@ spec:
               key: key
               optional: true
       image: ${LIGHTSPEED_STACK_IMAGE}
-      command: ["/bin/sh", "-c", "mkdir -p /tmp/data && exec python3.12 src/lightspeed_stack.py"]
+      command: ["/bin/sh", "-c", "mkdir -p /tmp/data && exec opentelemetry-instrument python3.12 src/lightspeed_stack.py"]
       ports:
         - containerPort: 8080
       # TCP probes avoid HTTP/auth. LCS + Llama handshake and large images can take 60–120s before :8080 listens;

tests/e2e/utils/prow_utils.py

@@ -62,8 +62,12 @@ def run_e2e_ops(
     )
 
 
-def wait_for_pod_health(pod_name: str, max_attempts: int = 12) -> None:
-    """Wait for pod to be ready in OpenShift/Prow environment."""
+def wait_for_pod_health(pod_name: str, max_attempts: int = 20) -> None:
+    """Wait for pod to be ready in OpenShift/Prow environment.
+
+    Generous number of attempts to account for OpenTelemetry instrumentation
+    initialization overhead during service startup.
+    """
     actual_pod_name = get_pod_name(pod_name)
     try:
         result = run_e2e_ops("wait-for-pod", [actual_pod_name, str(max_attempts)])

tests/e2e/utils/utils.py

@@ -181,7 +181,7 @@ def validate_json(message: Any, schema: Any) -> None:
         assert False, "The provided schema is faulty:" + str(e)
 
 
-def wait_for_container_health(container_name: str, max_attempts: int = 6) -> None:
+def wait_for_container_health(container_name: str, max_attempts: int = 20) -> None:
     """Wait for container to be healthy.
 
     Polls a Docker container until its health status becomes `healthy` or the
@@ -192,14 +192,17 @@ def wait_for_container_health(container_name: str, max_attempts: int = 6) -> Non
     inspect errors or timeouts are ignored and retried; the function returns
     after the container is observed healthy or after all attempts complete.
 
+    OpenTelemetry instrumentation adds initialization overhead, so the default
+    has been set to 20 attempts (40 seconds) to prevent timeouts.
+
     Returns:
     -------
         None
 
     Parameters:
     ----------
         container_name (str): Docker container name or ID to check.
-        max_attempts (int): Maximum number of health check attempts (default 6).
+        max_attempts (int): Maximum number of health check attempts (default 20).
     """
     if is_prow_environment():
         wait_for_pod_health(container_name, max_attempts)
@@ -468,9 +471,10 @@ def restart_container(container_name: str) -> None:
 
     # Wait for container to be healthy.
     # Library mode embeds llama-stack, so the container takes longer to start
-    # (~45-60s vs ~10s in server mode).  Use a generous attempt count so
-    # MCP-auth scenarios that restart the container don't time out.
-    wait_for_container_health(container_name, max_attempts=12)
+    # (~45-60s vs ~10s in server mode). OpenTelemetry instrumentation adds
+    # initialization overhead. Use a generous attempt count so MCP-auth scenarios
+    # that restart the container don't time out.
+    wait_for_container_health(container_name, max_attempts=20)
 
     if container_name == "llama-stack":
         from tests.e2e.features.steps.health import (