fix(adk): set OTLP exporter timeout in milliseconds (#1639)

OTEL specified that timeout env vars should be in **milliseconds**:
https://opentelemetry.io/docs/languages/sdk-configuration/otlp-exporter/#timeout-configuration

Python SDK's exporter currently expects it in seconds, so this adds a
utility function to parse the env var into seconds. Go is not affected
because OTEL Go SDK will properly take up the new value in milliseconds
properly.

Without changing this to milliseconds, Go ADK was having a timeout
configured as 15ms which results in intermittent error like `traces
export: exporter export timeout: rpc error: code = DeadlineExceeded desc
= context deadline exceeded`

---------

Signed-off-by: Jet Chiang <pokyuen.jetchiang-ext@solo.io>

Author: supreme-gg-gg

helm/kagent/values.yaml

@@ -497,12 +497,12 @@ otel:
       otlp:
         endpoint: ""
         protocol: "grpc"
-        timeout: 15
+        timeout: 15000 # milliseconds
         insecure: true
   logging:
     enabled: false
     exporter:
       otlp:
         endpoint: ""
-        timeout: 15
+        timeout: 15000 # milliseconds
         insecure: true

python/packages/kagent-core/src/kagent/core/tracing/_utils.py

@@ -18,6 +18,39 @@
 from ._span_processor import KagentAttributesSpanProcessor
 
 
+def _resolve_otlp_timeout_seconds(signal: str) -> float:
+    """
+    Resolve OTLP timeout env vars (milliseconds) into seconds for exporters.
+    By default, Python OTLP exporter reads timeout env var as seconds.
+    However, OTEL spec defines timeout as milliseconds.
+    """
+    signal_timeout_env = f"OTEL_EXPORTER_OTLP_{signal}_TIMEOUT"
+    raw_timeout = os.getenv(signal_timeout_env) or os.getenv("OTEL_EXPORTER_OTLP_TIMEOUT")
+    if raw_timeout is None:
+        # OTEL spec default is 10000ms
+        return 10.0
+
+    try:
+        timeout_millis = float(raw_timeout)
+    except ValueError:
+        logging.warning(
+            "Invalid OTEL timeout value %r from %s; falling back to 10000ms",
+            raw_timeout,
+            signal_timeout_env,
+        )
+        return 10.0
+
+    if timeout_millis < 0:
+        logging.warning(
+            "Negative OTEL timeout value %r from %s; falling back to 10000ms",
+            raw_timeout,
+            signal_timeout_env,
+        )
+        return 10.0
+
+    return timeout_millis / 1000.0
+
+
 def _instrument_anthropic(event_logger_provider=None):
     """Instrument Anthropic SDK if available."""
     try:
@@ -69,11 +102,12 @@ def configure(name: str = "kagent", namespace: str = "kagent", fastapi_app: Fast
             or os.getenv("OTEL_TRACING_EXPORTER_OTLP_ENDPOINT")  # Backward compatibility
             or os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
         )
+        trace_timeout_seconds = _resolve_otlp_timeout_seconds("TRACES")
         logging.info("Trace endpoint: %s", trace_endpoint or "<default>")
         if trace_endpoint:
-            processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=trace_endpoint))
+            processor = BatchSpanProcessor(OTLPSpanExporter(endpoint=trace_endpoint, timeout=trace_timeout_seconds))
         else:
-            processor = BatchSpanProcessor(OTLPSpanExporter())
+            processor = BatchSpanProcessor(OTLPSpanExporter(timeout=trace_timeout_seconds))
 
         # Check if a TracerProvider already exists (e.g., set by CrewAI)
         current_provider = trace.get_tracer_provider()
@@ -107,13 +141,14 @@ def configure(name: str = "kagent", namespace: str = "kagent", fastapi_app: Fast
             or os.getenv("OTEL_LOGGING_EXPORTER_OTLP_ENDPOINT")  # Backward compatibility
             or os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT")
         )
+        log_timeout_seconds = _resolve_otlp_timeout_seconds("LOGS")
         logging.info("Log endpoint: %s", log_endpoint or "<default>")
 
         # Add OTLP exporter
         if log_endpoint:
-            log_processor = BatchLogRecordProcessor(OTLPLogExporter(endpoint=log_endpoint))
+            log_processor = BatchLogRecordProcessor(OTLPLogExporter(endpoint=log_endpoint, timeout=log_timeout_seconds))
         else:
-            log_processor = BatchLogRecordProcessor(OTLPLogExporter())
+            log_processor = BatchLogRecordProcessor(OTLPLogExporter(timeout=log_timeout_seconds))
         logger_provider.add_log_record_processor(log_processor)
 
         _logs.set_logger_provider(logger_provider)

python/packages/kagent-core/tests/test_tracing_configure.py

@@ -1,5 +1,6 @@
 from types import SimpleNamespace
 
+import pytest
 from opentelemetry.propagate import get_global_textmap
 from opentelemetry.trace import get_current_span
 
@@ -94,3 +95,28 @@ def test_otel_sdk_default_propagator_includes_w3c_tracecontext():
 
     ctx = get_global_textmap().extract(carrier)
     assert get_current_span(ctx).get_span_context().trace_id == trace_id
+
+
+@pytest.mark.parametrize(
+    ("signal", "env", "expected"),
+    [
+        ("TRACES", {}, 10.0),
+        ("TRACES", {"OTEL_EXPORTER_OTLP_TIMEOUT": "500"}, 0.5),
+        ("TRACES", {"OTEL_EXPORTER_OTLP_TRACES_TIMEOUT": "250"}, 0.25),
+        (
+            "LOGS",
+            {
+                "OTEL_EXPORTER_OTLP_TIMEOUT": "500",
+                "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT": "750",
+            },
+            0.75,
+        ),
+    ],
+)
+def test_resolve_otlp_timeout_seconds_uses_milliseconds(monkeypatch, signal, env, expected):
+    for key in ("OTEL_EXPORTER_OTLP_TIMEOUT", "OTEL_EXPORTER_OTLP_TRACES_TIMEOUT", "OTEL_EXPORTER_OTLP_LOGS_TIMEOUT"):
+        monkeypatch.delenv(key, raising=False)
+    for key, value in env.items():
+        monkeypatch.setenv(key, value)
+
+    assert _utils._resolve_otlp_timeout_seconds(signal) == expected