Skip to content

Commit 2fac870

Browse files
authored
fix(livekit): measure eou detection delay (#446)
Use LiveKit EOU delay metrics to anchor eou_detection spans instead of logging them as zero-duration events. Stop emitting a second metrics-derived vad_endpointing span so raw VAD endpointing remains distinct. Covered by the existing VCR-backed LiveKit voice turn assertions.
1 parent 6963b97 commit 2fac870

2 files changed

Lines changed: 18 additions & 33 deletions

File tree

py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -447,6 +447,10 @@ def _assert_eou_spans(logs):
447447
if eou_logs:
448448
_assert_any_span(eou_logs, lambda log: log.get("input", {}).get("text"))
449449
_assert_any_span(eou_logs, lambda log: "is_end_of_turn" in log.get("output", {}))
450+
_assert_any_span(
451+
eou_logs,
452+
lambda log: log.get("metrics", {}).get("end", 0) - log.get("metrics", {}).get("start", 0) > 0,
453+
)
450454

451455

452456
def _assert_stt_spans(logs, speech_text):

py/src/braintrust/integrations/livekit_agents/tracing.py

Lines changed: 14 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,6 @@ def on_metrics_collected(event: Any) -> None:
7777
parent=parent if isinstance(parent, str) else None,
7878
)
7979
elif metrics_type == "eou_metrics":
80-
_log_vad_endpointing_span(
81-
metrics_obj,
82-
parent=parent if isinstance(parent, str) else None,
83-
)
8480
_log_metric_span(
8581
"eou_detection",
8682
metrics_obj,
@@ -781,44 +777,29 @@ def _log_metric_span(
781777
event.update(_eou_detection_io(metrics_payload))
782778
if output is not None:
783779
event["output"] = output
784-
start_time, end_time = _metric_span_times(metrics_payload)
780+
if name == "eou_detection":
781+
start_time, end_time = _eou_detection_span_times(metrics_payload)
782+
else:
783+
start_time, end_time = _metric_span_times(metrics_payload)
785784
span = start_span(name=name, type=span_type, start_time=start_time, set_current=False, parent=parent, input=input)
786785
span.log(**event)
787786
span.end(end_time=end_time)
788787

789788

790-
def _log_vad_endpointing_span(metrics_obj: Any, parent: str | None = None) -> None:
791-
metrics_payload = _metrics_from_object(metrics_obj)
792-
endpointing_delay = _first_numeric_metric(
789+
def _eou_detection_span_times(metrics_payload: dict[str, Any]) -> tuple[float | None, float | None]:
790+
timestamp = metrics_payload.get("timestamp")
791+
if not isinstance(timestamp, (int, float)) or isinstance(timestamp, bool):
792+
return _metric_span_times(metrics_payload)
793+
eou_delay = _first_numeric_metric(
793794
metrics_payload,
794-
"endpointing_delay",
795-
"end_of_turn_delay",
796795
"end_of_utterance_delay",
796+
"end_of_turn_delay",
797+
"endpointing_delay",
797798
"eou_delay",
798799
)
799-
if endpointing_delay is None or endpointing_delay <= 0:
800-
return
801-
timestamp = metrics_payload.get("timestamp")
802-
end_time = timestamp if isinstance(timestamp, (int, float)) and not isinstance(timestamp, bool) else None
803-
start_time = end_time - endpointing_delay if end_time is not None else None
804-
metadata = _promoted_metadata(metrics_payload)
805-
metadata["livekit_metrics"] = {
806-
key: value
807-
for key, value in metrics_payload.items()
808-
if key
809-
not in {"metadata", "type", "endpointing_delay", "end_of_turn_delay", "end_of_utterance_delay", "eou_delay"}
810-
}
811-
if not metadata["livekit_metrics"]:
812-
metadata.pop("livekit_metrics")
813-
span = start_span(
814-
name="vad_endpointing",
815-
type=SpanTypeAttribute.TASK,
816-
start_time=start_time,
817-
set_current=False,
818-
parent=parent,
819-
)
820-
span.log(metadata=metadata)
821-
span.end(end_time=end_time)
800+
if eou_delay is None or eou_delay <= 0:
801+
return _metric_span_times(metrics_payload)
802+
return timestamp - eou_delay, timestamp
822803

823804

824805
def _first_numeric_metric(metrics_payload: dict[str, Any], *keys: str) -> float | None:

0 commit comments

Comments
 (0)