Skip to content

Commit 8d9c478

Browse files
STHITAPRAJNASSTHITAPRAJNAS
committed
fix(tracing): instrument FastAPI ASGI layer for inbound traceparent propagation
Fixes #4767. When get_fast_api_app() is used in production behind an OTel-instrumented caller (e.g. a Next.js service using @opentelemetry/sdk-node), every inbound request carried a W3C traceparent header that ADK silently discarded. The TracerProvider and W3C propagator were already wired up correctly by _setup_telemetry(), but without an ASGI-level hook to extract the header, each request spawned a new trace root instead of continuing the caller's trace. The fix calls FastAPIInstrumentor.instrument_app(app) immediately after the FastAPI instance is created, but only when an OTel export pipeline is actually active (OTLP env vars or otel_to_cloud=True). The instrumentation is applied before the CORS and origin-check middleware are registered so that Starlette's reverse-registration order leaves the security wrappers outermost in the stack. The call is best-effort: if opentelemetry-instrumentation-fastapi is absent a debug-level message is emitted and the server starts normally. The package is added to the otel-gcp extras so users who install google-adk[otel-gcp] get end-to-end distributed tracing out of the box without any manual post-instrumentation step. opentelemetry-instrumentation-fastapi is also added to the test extras so the new unit tests can import it directly. #4767
1 parent fe41817 commit 8d9c478

3 files changed

Lines changed: 214 additions & 1 deletion

File tree

‎pyproject.toml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ test = [
129129
"litellm>=1.75.5, <=1.82.6", # For LiteLLM tests. Upper bound pinned: versions 1.82.7+ compromised in supply chain attack.
130130
"llama-index-readers-file>=0.4.0", # For retrieval tests
131131
"openai>=1.100.2", # For LiteLLM
132+
"opentelemetry-instrumentation-fastapi>=0.48b0, <1.0.0",
132133
"opentelemetry-instrumentation-google-genai>=0.3b0, <1.0.0",
133134
"pypika>=0.50.0", # For crewai->chromadb dependency
134135
"pytest-asyncio>=0.25.0",
@@ -168,7 +169,12 @@ extensions = [
168169
"toolbox-adk>=0.7.0, <0.8.0", # For tools.toolbox_toolset.ToolboxToolset
169170
]
170171

171-
otel-gcp = ["opentelemetry-instrumentation-google-genai>=0.6b0, <1.0.0"]
172+
otel-gcp = [
173+
# go/keep-sorted start
174+
"opentelemetry-instrumentation-fastapi>=0.48b0, <1.0.0",
175+
"opentelemetry-instrumentation-google-genai>=0.6b0, <1.0.0",
176+
# go/keep-sorted end
177+
]
172178

173179
toolbox = ["toolbox-adk>=0.7.0, <0.8.0"]
174180

‎src/google/adk/cli/adk_web_server.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -910,6 +910,27 @@ async def internal_lifespan(app: FastAPI):
910910
# Run the FastAPI server.
911911
app = FastAPI(lifespan=internal_lifespan)
912912

913+
# When an OTel pipeline is active, instrument the ASGI layer so that
914+
# inbound W3C traceparent headers are extracted and agent spans are
915+
# correctly parented to the caller's trace. Without this, every request
916+
# starts a new trace root even though the TracerProvider and W3C propagator
917+
# are already configured. Applied before other middleware so that
918+
# security/CORS wrappers are outermost in the stack (Starlette applies
919+
# middleware in reverse-registration order).
920+
if otel_to_cloud or _otel_env_vars_enabled():
921+
try:
922+
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor # pylint: disable=g-import-not-at-top
923+
924+
FastAPIInstrumentor.instrument_app(app)
925+
logger.debug("FastAPI OpenTelemetry instrumentation enabled.")
926+
except ImportError:
927+
logger.debug(
928+
"opentelemetry-instrumentation-fastapi is not installed; inbound"
929+
" traceparent headers will not be propagated into agent spans."
930+
" Install it alongside the OTel extras: pip install"
931+
" opentelemetry-instrumentation-fastapi"
932+
)
933+
913934
has_configured_allowed_origins = bool(allow_origins)
914935
if allow_origins:
915936
literal_origins, combined_regex = _parse_cors_origins(allow_origins)

‎tests/unittests/cli/test_fast_api.py‎

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1950,5 +1950,191 @@ async def run_async_session_not_found(self, **kwargs):
19501950
assert "Session not found" in response.json()["detail"]
19511951

19521952

1953+
# ---------------------------------------------------------------------------
1954+
# OpenTelemetry FastAPI instrumentation tests
1955+
# ---------------------------------------------------------------------------
1956+
1957+
_OTEL_ENV_ENABLED = "google.adk.cli.adk_web_server._otel_env_vars_enabled"
1958+
_SETUP_TELEMETRY = "google.adk.cli.adk_web_server._setup_telemetry"
1959+
_FASTAPI_INSTRUMENTOR = "opentelemetry.instrumentation.fastapi.FastAPIInstrumentor"
1960+
1961+
1962+
def _make_otel_test_client(
1963+
mock_session_service,
1964+
mock_artifact_service,
1965+
mock_memory_service,
1966+
mock_agent_loader,
1967+
mock_eval_sets_manager,
1968+
mock_eval_set_results_manager,
1969+
**app_kwargs,
1970+
):
1971+
"""Like _create_test_client but suppresses real OTel provider setup."""
1972+
defaults = dict(
1973+
agents_dir=".",
1974+
web=False,
1975+
session_service_uri="",
1976+
artifact_service_uri="",
1977+
memory_service_uri="",
1978+
a2a=False,
1979+
host="127.0.0.1",
1980+
port=8000,
1981+
)
1982+
defaults.update(app_kwargs)
1983+
with (
1984+
patch.object(signal, "signal", autospec=True, return_value=None),
1985+
patch.object(
1986+
fast_api_module,
1987+
"create_session_service_from_options",
1988+
autospec=True,
1989+
return_value=mock_session_service,
1990+
),
1991+
patch.object(
1992+
fast_api_module,
1993+
"create_artifact_service_from_options",
1994+
autospec=True,
1995+
return_value=mock_artifact_service,
1996+
),
1997+
patch.object(
1998+
fast_api_module,
1999+
"create_memory_service_from_options",
2000+
autospec=True,
2001+
return_value=mock_memory_service,
2002+
),
2003+
patch.object(
2004+
fast_api_module,
2005+
"AgentLoader",
2006+
autospec=True,
2007+
return_value=mock_agent_loader,
2008+
),
2009+
patch.object(
2010+
fast_api_module,
2011+
"LocalEvalSetsManager",
2012+
autospec=True,
2013+
return_value=mock_eval_sets_manager,
2014+
),
2015+
patch.object(
2016+
fast_api_module,
2017+
"LocalEvalSetResultsManager",
2018+
autospec=True,
2019+
return_value=mock_eval_set_results_manager,
2020+
),
2021+
# Suppress real OTel provider / exporter setup so tests stay isolated.
2022+
patch(_SETUP_TELEMETRY),
2023+
):
2024+
app = get_fast_api_app(**defaults)
2025+
return TestClient(app)
2026+
2027+
2028+
def test_fastapi_instrumented_when_otlp_env_var_set(
2029+
mock_session_service,
2030+
mock_artifact_service,
2031+
mock_memory_service,
2032+
mock_agent_loader,
2033+
mock_eval_sets_manager,
2034+
mock_eval_set_results_manager,
2035+
):
2036+
"""FastAPIInstrumentor.instrument_app is called when an OTLP env var is set."""
2037+
with (
2038+
patch(_OTEL_ENV_ENABLED, return_value=True),
2039+
patch(_FASTAPI_INSTRUMENTOR) as mock_instrumentor_cls,
2040+
):
2041+
_make_otel_test_client(
2042+
mock_session_service,
2043+
mock_artifact_service,
2044+
mock_memory_service,
2045+
mock_agent_loader,
2046+
mock_eval_sets_manager,
2047+
mock_eval_set_results_manager,
2048+
)
2049+
2050+
mock_instrumentor_cls.instrument_app.assert_called_once()
2051+
2052+
2053+
def test_fastapi_instrumented_when_otel_to_cloud_enabled(
2054+
mock_session_service,
2055+
mock_artifact_service,
2056+
mock_memory_service,
2057+
mock_agent_loader,
2058+
mock_eval_sets_manager,
2059+
mock_eval_set_results_manager,
2060+
):
2061+
"""FastAPIInstrumentor.instrument_app is called when otel_to_cloud=True."""
2062+
with (
2063+
# otel_to_cloud=True triggers the instrumentation regardless of env vars.
2064+
patch(_OTEL_ENV_ENABLED, return_value=False),
2065+
patch(_FASTAPI_INSTRUMENTOR) as mock_instrumentor_cls,
2066+
):
2067+
_make_otel_test_client(
2068+
mock_session_service,
2069+
mock_artifact_service,
2070+
mock_memory_service,
2071+
mock_agent_loader,
2072+
mock_eval_sets_manager,
2073+
mock_eval_set_results_manager,
2074+
otel_to_cloud=True,
2075+
)
2076+
2077+
mock_instrumentor_cls.instrument_app.assert_called_once()
2078+
2079+
2080+
def test_fastapi_not_instrumented_without_otel_config(
2081+
mock_session_service,
2082+
mock_artifact_service,
2083+
mock_memory_service,
2084+
mock_agent_loader,
2085+
mock_eval_sets_manager,
2086+
mock_eval_set_results_manager,
2087+
):
2088+
"""FastAPIInstrumentor.instrument_app is NOT called when OTel is not configured."""
2089+
with (
2090+
patch(_OTEL_ENV_ENABLED, return_value=False),
2091+
patch(_FASTAPI_INSTRUMENTOR) as mock_instrumentor_cls,
2092+
):
2093+
_make_otel_test_client(
2094+
mock_session_service,
2095+
mock_artifact_service,
2096+
mock_memory_service,
2097+
mock_agent_loader,
2098+
mock_eval_sets_manager,
2099+
mock_eval_set_results_manager,
2100+
otel_to_cloud=False,
2101+
)
2102+
2103+
mock_instrumentor_cls.instrument_app.assert_not_called()
2104+
2105+
2106+
def test_missing_fastapi_instrumentor_does_not_prevent_startup(
2107+
mock_session_service,
2108+
mock_artifact_service,
2109+
mock_memory_service,
2110+
mock_agent_loader,
2111+
mock_eval_sets_manager,
2112+
mock_eval_set_results_manager,
2113+
):
2114+
"""App starts normally when opentelemetry-instrumentation-fastapi is absent."""
2115+
import sys
2116+
2117+
# Simulate the package not being installed by removing it from sys.modules
2118+
# and making the import raise ImportError.
2119+
with (
2120+
patch(_OTEL_ENV_ENABLED, return_value=True),
2121+
patch.dict(
2122+
sys.modules,
2123+
{"opentelemetry.instrumentation.fastapi": None},
2124+
),
2125+
):
2126+
client = _make_otel_test_client(
2127+
mock_session_service,
2128+
mock_artifact_service,
2129+
mock_memory_service,
2130+
mock_agent_loader,
2131+
mock_eval_sets_manager,
2132+
mock_eval_set_results_manager,
2133+
)
2134+
2135+
response = client.get("/health")
2136+
assert response.status_code == 200
2137+
2138+
19532139
if __name__ == "__main__":
19542140
pytest.main(["-xvs", __file__])

0 commit comments

Comments
 (0)