@@ -865,6 +865,192 @@ async def test_llm_span_output_tool_calls_payload_gating() -> None:
865865 ]
866866
867867
868+ # ---------------------------------------------------------------------------
869+ # Proposal 0067 — GenAI metrics (observability §11)
870+ # ---------------------------------------------------------------------------
871+
872+
873+ def _collect_metric_points (reader : Any ) -> list [tuple [str , float , int , dict [str , Any ]]]:
874+ """Flatten an InMemoryMetricReader's collected data into
875+ ``(instrument_name, point_sum, point_count, point_attributes)``
876+ tuples. Histogram observations with identical attribute sets
877+ aggregate into one data point (sum + count), so per-attempt tests
878+ assert on ``count``, not point cardinality."""
879+ data = reader .get_metrics_data ()
880+ points : list [tuple [str , float , int , dict [str , Any ]]] = []
881+ if data is None :
882+ return points
883+ for resource_metric in data .resource_metrics :
884+ for scope_metric in resource_metric .scope_metrics :
885+ for metric in scope_metric .metrics :
886+ for pt in metric .data .data_points :
887+ points .append ((metric .name , pt .sum , pt .count , dict (pt .attributes )))
888+ return points
889+
890+
891+ async def _drive_metrics_events (
892+ events : list [Any ],
893+ * ,
894+ enable_metrics : bool = True ,
895+ disable_llm_spans : bool = False ,
896+ ) -> tuple [list [tuple [str , float , int , dict [str , Any ]]], list [Any ]]:
897+ """Feed LlmRetryAttemptEvents through an OTelObserver wired to a
898+ private MeterProvider + InMemoryMetricReader; return the captured
899+ ``(metric_points, llm_complete_spans)``."""
900+ from opentelemetry .sdk .metrics import MeterProvider as SdkMeterProvider
901+ from opentelemetry .sdk .metrics .export import InMemoryMetricReader
902+
903+ from openarmature .observability .correlation import (
904+ _reset_invocation_id ,
905+ _set_invocation_id ,
906+ )
907+
908+ reader = InMemoryMetricReader ()
909+ meter_provider = SdkMeterProvider (metric_readers = [reader ])
910+ exporter = InMemorySpanExporter ()
911+ observer = OTelObserver (
912+ span_processor = SimpleSpanProcessor (exporter ),
913+ enable_metrics = enable_metrics ,
914+ disable_llm_spans = disable_llm_spans ,
915+ meter_provider = meter_provider ,
916+ )
917+ token = _set_invocation_id ("inv-metrics" )
918+ try :
919+ for event in events :
920+ await observer (event )
921+ finally :
922+ _reset_invocation_id (token )
923+ observer .shutdown ()
924+ llm_spans = [s for s in exporter .get_finished_spans () if s .name == "openarmature.llm.complete" ]
925+ return _collect_metric_points (reader ), llm_spans
926+
927+
928+ async def test_metrics_records_token_and_duration () -> None :
929+ # Proposal 0067 §11 (mirrors fixture 088): a successful LLM attempt
930+ # with usage {input 5, output 1} records two token.usage observations
931+ # (input + output) and one duration observation, with the §11.3
932+ # dimensions. Duration value is not asserted (§11.4).
933+ from openarmature .llm .response import Usage
934+ from tests ._helpers .typed_event import make_retry_attempt_event
935+
936+ event = make_retry_attempt_event (
937+ model = "test-model" ,
938+ provider = "openai" ,
939+ latency_ms = 12.0 ,
940+ usage = Usage (prompt_tokens = 5 , completion_tokens = 1 , total_tokens = 6 ),
941+ )
942+ points , _ = await _drive_metrics_events ([event ])
943+ token_points = [p for p in points if p [0 ] == "openarmature.gen_ai.client.token.usage" ]
944+ duration_points = [p for p in points if p [0 ] == "openarmature.gen_ai.client.operation.duration" ]
945+ by_type = {p [3 ]["openarmature.gen_ai.token.type" ]: p for p in token_points }
946+ assert by_type ["input" ][1 ] == 5
947+ assert by_type ["output" ][1 ] == 1
948+ for ttype in ("input" , "output" ):
949+ dims = by_type [ttype ][3 ]
950+ assert dims ["openarmature.gen_ai.operation" ] == "chat"
951+ assert dims ["gen_ai.request.model" ] == "test-model"
952+ assert dims ["gen_ai.system" ] == "openai"
953+ assert len (duration_points ) == 1
954+ ddims = duration_points [0 ][3 ]
955+ assert ddims ["openarmature.gen_ai.operation" ] == "chat"
956+ assert ddims ["gen_ai.request.model" ] == "test-model"
957+ assert ddims ["gen_ai.system" ] == "openai"
958+ assert "error.type" not in ddims
959+
960+
961+ async def test_metrics_records_duration_with_error_type_on_failure () -> None :
962+ # Proposal 0067 §11.2 / §11.3 (mirrors fixture 090): a failed attempt
963+ # records a duration observation carrying error.type, and NO
964+ # token.usage observation (a failed attempt returned no usage).
965+ from tests ._helpers .typed_event import make_retry_attempt_event
966+
967+ event = make_retry_attempt_event (
968+ model = "test-model" ,
969+ provider = "openai" ,
970+ latency_ms = 8.0 ,
971+ finish_reason = None ,
972+ usage = None ,
973+ error_category = "provider_unavailable" ,
974+ error_type = "ProviderUnavailable" ,
975+ error_message = "down" ,
976+ )
977+ points , _ = await _drive_metrics_events ([event ])
978+ token_points = [p for p in points if p [0 ] == "openarmature.gen_ai.client.token.usage" ]
979+ duration_points = [p for p in points if p [0 ] == "openarmature.gen_ai.client.operation.duration" ]
980+ assert token_points == []
981+ assert len (duration_points ) == 1
982+ assert duration_points [0 ][3 ]["error.type" ] == "provider_unavailable"
983+ assert duration_points [0 ][3 ]["openarmature.gen_ai.operation" ] == "chat"
984+
985+
986+ async def test_metrics_disabled_records_nothing () -> None :
987+ # Proposal 0067 §11.1 (mirrors fixture 091): enable_metrics off (the
988+ # default) creates no instrument and records nothing.
989+ from openarmature .llm .response import Usage
990+ from tests ._helpers .typed_event import make_retry_attempt_event
991+
992+ event = make_retry_attempt_event (usage = Usage (prompt_tokens = 5 , completion_tokens = 1 , total_tokens = 6 ))
993+ points , _ = await _drive_metrics_events ([event ], enable_metrics = False )
994+ assert points == []
995+
996+
997+ async def test_metrics_independent_of_disable_llm_spans () -> None :
998+ # Proposal 0067 §11.1: metrics record even with spans disabled — the
999+ # disable_llm_spans flag governs span emission only.
1000+ from openarmature .llm .response import Usage
1001+ from tests ._helpers .typed_event import make_retry_attempt_event
1002+
1003+ event = make_retry_attempt_event (usage = Usage (prompt_tokens = 5 , completion_tokens = 1 , total_tokens = 6 ))
1004+ points , llm_spans = await _drive_metrics_events ([event ], disable_llm_spans = True )
1005+ assert llm_spans == []
1006+ assert any (p [0 ] == "openarmature.gen_ai.client.operation.duration" for p in points )
1007+ assert any (p [0 ] == "openarmature.gen_ai.client.token.usage" for p in points )
1008+
1009+
1010+ async def test_metrics_record_once_per_attempt_under_retry () -> None :
1011+ # Proposal 0067 §11.2 "Call-level retry": the duration histogram
1012+ # records once per attempt (failed attempts carry error.type), and
1013+ # token.usage only for an attempt that returned usage. Two failed
1014+ # attempts + one success -> 3 duration observations (2 with
1015+ # error.type), 2 token.usage observations (the success's input +
1016+ # output). Observations with identical dimensions aggregate into one
1017+ # data point, so this asserts on histogram counts.
1018+ from openarmature .llm .response import Usage
1019+ from tests ._helpers .typed_event import make_retry_attempt_event
1020+
1021+ failed = [
1022+ make_retry_attempt_event (
1023+ llm_attempt_index = i ,
1024+ latency_ms = 5.0 ,
1025+ finish_reason = None ,
1026+ usage = None ,
1027+ error_category = "provider_unavailable" ,
1028+ error_type = "ProviderUnavailable" ,
1029+ error_message = "down" ,
1030+ )
1031+ for i in range (2 )
1032+ ]
1033+ success = make_retry_attempt_event (
1034+ llm_attempt_index = 2 ,
1035+ latency_ms = 7.0 ,
1036+ usage = Usage (prompt_tokens = 5 , completion_tokens = 1 , total_tokens = 6 ),
1037+ )
1038+ points , _ = await _drive_metrics_events ([* failed , success ])
1039+ duration_points = [p for p in points if p [0 ] == "openarmature.gen_ai.client.operation.duration" ]
1040+ token_points = [p for p in points if p [0 ] == "openarmature.gen_ai.client.token.usage" ]
1041+ # 3 duration observations total: 2 share the error dims (one
1042+ # aggregated point, count 2), the success is a separate point.
1043+ assert sum (p [2 ] for p in duration_points ) == 3
1044+ error_duration = [p for p in duration_points if p [3 ].get ("error.type" ) == "provider_unavailable" ]
1045+ assert sum (p [2 ] for p in error_duration ) == 2
1046+ success_duration = [p for p in duration_points if "error.type" not in p [3 ]]
1047+ assert sum (p [2 ] for p in success_duration ) == 1
1048+ # token.usage only from the success attempt: one input, one output.
1049+ by_type = {p [3 ]["openarmature.gen_ai.token.type" ]: p for p in token_points }
1050+ assert by_type ["input" ][1 ] == 5 and by_type ["input" ][2 ] == 1
1051+ assert by_type ["output" ][1 ] == 1 and by_type ["output" ][2 ] == 1
1052+
1053+
8681054async def test_llm_span_zero_duration_when_latency_missing () -> None :
8691055 # When the typed event omits latency_ms (None), the handler falls
8701056 # back to a zero-duration span at end_time rather than guessing
0 commit comments