From ba24bd02b3f113627a80cf9bf6228c05e92985c9 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Thu, 23 Jul 2026 11:37:57 -0700 Subject: [PATCH 1/2] Add 0082 Langfuse failed-Generation rendering PR C of proposal 0082, completing the 3-PR split with the Langfuse failed-Generation rendering. 0082 is now fully implemented. For a structured_output_invalid failure the bundled Langfuse observer renders the response-side surface on the ERROR Generation like a completion: output (payload-gated) + usage on the Generation, and finish_reason / response_id / response_model in metadata (the completion-only isinstance guard widened to include the category), alongside the existing level=ERROR + statusMessage=category. Every other failure category is unchanged. Fixture 123 is driven in the dedicated Langfuse harness (response_schema threaded + expected_error handled). conformance.toml marks 0082 implemented. Also folds in the whole-proposal adversarial review over all three PRs: - Cross-observer consistency: the failure event builders project empty output_content to None (matching the success path's content-or-None), so both observers omit empty content identically. The OpenAI wire cannot currently produce empty content on the structured path, so this is a defensive builder projection with a builder-direct test. - The Langfuse conformance harness now asserts observation statusMessage (previously a silent no-op, so 123's category assertion did not run; also affected fixtures 130/137/138). - Refreshed a stale _typed_event_metadata docstring, dropped a docstring spec ref, and completed the CHANGELOG response id/model listing. --- CHANGELOG.md | 2 +- conformance.toml | 10 +-- src/openarmature/llm/providers/openai.py | 9 ++- .../observability/langfuse/observer.py | 57 ++++++++++------ tests/conformance/test_observability.py | 7 +- .../test_observability_langfuse.py | 29 +++++++- tests/unit/test_llm_provider.py | 29 ++++++++ tests/unit/test_observability_langfuse.py | 66 +++++++++++++++++++ 8 files changed, 178 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9c67538..3966d89a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The - **PromptManager service-wide default cache TTL** (proposal 0086, prompt-management §6, spec v0.79.0). `PromptManager` construction accepts an optional `default_cache_ttl_seconds`, applied to any `fetch` or `get` that omits a per-call `cache_ttl_seconds`. Resolution follows a precedence chain: an explicit per-call value (including `0` force-fresh) wins; otherwise the manager default applies; otherwise nothing is forwarded and the backend's own caching governs. An omitted or explicit-`None` per-call value both select the default, so resolution does not depend on argument presence. A negative default is rejected at construction, and the per-call negative rejection is unchanged. `get` delegates to `fetch` and inherits the chain, and the bundled caching backends need no change (they already honor a resolved `cache_ttl_seconds`). Conformance fixture 036 is un-deferred. - **PromptGroup arity enforcement** (proposal 0080, prompt-management §10 / §11, spec v0.75.0). Constructing a `PromptGroup` with fewer than two members (an empty or single-member group) now raises a categorized `PromptGroupInvalid` at construction, before any render or LLM call. This replaces the previous bare `ValueError` (which pydantic folded into a `ValidationError` carrying no error category) and adds a `prompt_group_invalid` category to the prompt-management error set. `PromptGroupInvalid` is non-transient and is exported from `openarmature.prompts`. Conformance fixture 035 is un-deferred. -- **Structured-output failure diagnostics on the LLM failure event** (proposal 0082, graph-engine §6 + llm-provider §7, spec v0.77.0). A `structured_output_invalid` failure is a completion whose final validation gate failed, so the wire response is intact. `LlmFailedEvent` now carries the response-side surface for that one category (`None` for every other): `output_content` (the content that failed validation, payload-gated), `finish_reason` (the truncation-vs-malformed triage signal, `"length"` vs `"stop"`), `usage`, `response_id`, and `response_model`. The `StructuredOutputInvalid` error additionally exposes `finish_reason` and `usage`, so a caller's exception handler can make a truncation-aware retry decision. The bundled OTel observer renders this surface on the error `openarmature.llm.complete` span (`ERROR` status plus `finish_reason`, `usage`, and payload-gated `output.content`), and the opt-in `openarmature.gen_ai.client.token.usage` metric now records a structured-output failure's token usage (other failure categories still record none). Rendering the surface on the Langfuse failed Generation lands in a follow-on PR of the same proposal. +- **Structured-output failure diagnostics on the LLM failure event** (proposal 0082, graph-engine §6 + llm-provider §7, spec v0.77.0). A `structured_output_invalid` failure is a completion whose final validation gate failed, so the wire response is intact. `LlmFailedEvent` now carries the response-side surface for that one category (`None` for every other): `output_content` (the content that failed validation, payload-gated), `finish_reason` (the truncation-vs-malformed triage signal, `"length"` vs `"stop"`), `usage`, `response_id`, and `response_model`. The `StructuredOutputInvalid` error additionally exposes `finish_reason` and `usage`, so a caller's exception handler can make a truncation-aware retry decision. The bundled OTel observer renders this surface on the error `openarmature.llm.complete` span (`ERROR` status plus `finish_reason`, `usage`, payload-gated `output.content`, and the response id / model), and the opt-in `openarmature.gen_ai.client.token.usage` metric now records a structured-output failure's token usage (other failure categories still record none). The bundled Langfuse observer renders the same surface on the failed Generation: `level = "ERROR"` + `statusMessage` = the category, plus `output` (payload-gated), `usage`, `metadata.finish_reason`, and `metadata.response_id` / `metadata.response_model`. ### Fixed diff --git a/conformance.toml b/conformance.toml index 87aa0f53..72d45a94 100644 --- a/conformance.toml +++ b/conformance.toml @@ -816,13 +816,13 @@ since = "0.16.0" # Spec v0.77.0 (proposal 0082). Structured-output failure diagnostics # (graph-engine §6 -- LlmFailedEvent response-side surface for # structured_output_invalid + llm-provider §7 finish_reason / usage). -# Partial since 0.17.0: the event + error surface is implemented (PR A); -# the OTel / Langfuse / metrics RENDERING (observability fixtures 123-125) -# lands in later PRs of the split. +# Implemented since 0.17.0 across the 3-PR split: the event + error surface +# (PR A), the OTel error-span rendering + §11 token metric (PR B), and the +# Langfuse failed-Generation rendering (PR C). [proposals."0082"] -status = "partial" +status = "implemented" since = "0.17.0" -note = "The graph-engine §6 event surface + llm-provider §7 error surface (PR A). LlmFailedEvent gains five response-side fields (output_content / finish_reason / usage / response_id / response_model), populated ONLY for structured_output_invalid (None for every other category); output_content is payload-bearing, the other four are not gated. StructuredOutputInvalid gains finish_reason / usage (the §7-mandated pair) plus response_id / response_model (internal, for the event builder), attached at the OpenAI provider's parse/validate call site where the intact wire response is in hand; the failed-event builder reads them off the enriched error. Un-defers observability fixtures 120 (truncation, finish_reason=length) / 121 (schema-mismatch, finish_reason=stop) / 122 (null-on-non-body companion), driven via the typed-event collector with a new calls_llm.response_schema harness lever; and llm-provider fixtures 022 / 023, whose carries block now asserts finish_reason + usage (the carries reader gained subset matching for a mapping-valued field like usage). PR B adds the OTel rendering + the metric: the per-attempt LlmRetryAttemptEvent now carries the same response-side surface on a structured_output_invalid failed attempt, so the OTel error span renders it (a structured_output_invalid failure falls through the failed-attempt early-return to render finish_reason / usage / gated output.content / response id+model while keeping ERROR status + category) and the §11.2 token-usage histogram records the failure's usage (the metric already recorded whenever usage was present; a structured failure now carries it). Un-defers observability fixtures 124 (OTel error span, payload on/off) and 125 (token metric on the failure). PARTIAL: only the Langfuse failed-Generation rendering (123) is deferred to the final PR of the split -- it renders the surface the event carries onto a failed Generation." +note = "The graph-engine §6 event surface + llm-provider §7 error surface (PR A). LlmFailedEvent gains five response-side fields (output_content / finish_reason / usage / response_id / response_model), populated ONLY for structured_output_invalid (None for every other category); output_content is payload-bearing, the other four are not gated. StructuredOutputInvalid gains finish_reason / usage (the §7-mandated pair) plus response_id / response_model (internal, for the event builder), attached at the OpenAI provider's parse/validate call site where the intact wire response is in hand; the failed-event builder reads them off the enriched error. Un-defers observability fixtures 120 (truncation, finish_reason=length) / 121 (schema-mismatch, finish_reason=stop) / 122 (null-on-non-body companion), driven via the typed-event collector with a new calls_llm.response_schema harness lever; and llm-provider fixtures 022 / 023, whose carries block now asserts finish_reason + usage (the carries reader gained subset matching for a mapping-valued field like usage). PR B adds the OTel rendering + the metric: the per-attempt LlmRetryAttemptEvent now carries the same response-side surface on a structured_output_invalid failed attempt, so the OTel error span renders it (a structured_output_invalid failure falls through the failed-attempt early-return to render finish_reason / usage / gated output.content / response id+model while keeping ERROR status + category) and the §11.2 token-usage histogram records the failure's usage (the metric already recorded whenever usage was present; a structured failure now carries it). Un-defers observability fixtures 124 (OTel error span, payload on/off) and 125 (token metric on the failure). PR C completes it with the Langfuse failed-Generation rendering: _handle_typed_llm_failed renders output (payload-gated) + usage on the ERROR Generation and _typed_event_metadata renders finish_reason / response id+model for a structured_output_invalid failure (the isinstance(LlmCompletionEvent) guard widened to include it), alongside the existing level=ERROR + statusMessage=category. Un-defers observability fixture 123, driven in the dedicated test_observability_langfuse harness (calls_llm.response_schema threaded + expected_error handled). 0082 now fully implemented." # Spec v0.78.0 (proposal 0083). Per-prompt token-budget observability # (prompt-management §3 / graph-engine §6 / observability §5.5.15). diff --git a/src/openarmature/llm/providers/openai.py b/src/openarmature/llm/providers/openai.py index a5ef4159..548a05f7 100644 --- a/src/openarmature/llm/providers/openai.py +++ b/src/openarmature/llm/providers/openai.py @@ -764,7 +764,10 @@ def _build_llm_failed_event( response_model: str | None = None error_message = str(exc) if isinstance(exc, StructuredOutputInvalid): - output_content = exc.raw_content + # Empty content projects to None (as the success path does with + # ``content or None``), so both observers omit it identically + # rather than one rendering "" and the other dropping it. + output_content = exc.raw_content or None finish_reason = exc.finish_reason usage = exc.usage response_id = exc.response_id @@ -875,7 +878,9 @@ def _build_llm_retry_attempt_event( response_fields: dict[str, Any] = {} if isinstance(exc, StructuredOutputInvalid): response_fields = { - "output_content": exc.raw_content, + # Empty content -> None (as the success path projects), so both + # observers omit it identically. + "output_content": exc.raw_content or None, "finish_reason": exc.finish_reason, "usage": exc.usage, "response_id": exc.response_id, diff --git a/src/openarmature/observability/langfuse/observer.py b/src/openarmature/observability/langfuse/observer.py index afa0d816..4b5a4879 100644 --- a/src/openarmature/observability/langfuse/observer.py +++ b/src/openarmature/observability/langfuse/observer.py @@ -1757,9 +1757,11 @@ def _handle_typed_llm_completion(self, event: LlmCompletionEvent) -> None: def _handle_typed_llm_failed(self, event: LlmFailedEvent) -> None: """Open + close an ERROR-level Generation observation from the - typed LlmFailedEvent (failure path). Same shape as the success - path with ERROR level + error_category as the Generation - observation's statusMessage.""" + typed LlmFailedEvent (failure path): ERROR level + error_category + as the Generation's statusMessage. A structured_output_invalid + failure additionally carries the response-side surface (payload-gated + output, usage, metadata.finish_reason) like a completion, since its + wire response was intact.""" from openarmature.observability.correlation import ( current_correlation_id, current_invocation_id, @@ -1798,11 +1800,24 @@ def _handle_typed_llm_failed(self, event: LlmFailedEvent) -> None: metadata["error_message"] = event.error_message model_parameters: dict[str, Any] = dict(event.request_params or {}) input_value: Any = None + output_value: Any = None + end_kwargs: dict[str, Any] = {} if not self.disable_provider_payload: if event.input_messages: input_value = self._maybe_truncate_for_input(event.input_messages) if event.request_extras: metadata["request_extras"] = self._maybe_truncate_for_extras(dict(event.request_extras)) + # Proposal 0082: a structured_output_invalid failure has an intact wire + # response, so the failed Generation carries the response-side surface + # (output payload-gated, usage) like a completion, alongside the ERROR + # level + category. metadata.finish_reason is added for this category by + # _typed_event_metadata. + if event.error_category == "structured_output_invalid": + if not self.disable_provider_payload and event.output_content is not None: + output_value = self._maybe_truncate_for_output(event.output_content) + usage = self._usage_from_typed_event(event) + if usage is not None: + end_kwargs["usage"] = usage target_trace_id = self._trace_id_for(inv_state, event.namespace, event.fan_out_index) handle = self.client.generation( trace_id=target_trace_id, @@ -1810,6 +1825,7 @@ def _handle_typed_llm_failed(self, event: LlmFailedEvent) -> None: model=event.model, model_parameters=model_parameters, input=input_value, + output=output_value, metadata=metadata, parent_observation_id=parent_observation_id, prompt=self._resolve_prompt_link_from_typed_event(event), @@ -1820,6 +1836,7 @@ def _handle_typed_llm_failed(self, event: LlmFailedEvent) -> None: end_time=end_time, level="ERROR", status_message=event.error_category, + **end_kwargs, ) # Spec proposal 0063: dedicated Langfuse Tool observation (asType="tool"). @@ -2169,13 +2186,11 @@ def _typed_event_metadata( ) -> dict[str, Any]: """Build the Generation observation's metadata dict from a typed LLM event. Shared between the success path - (LlmCompletionEvent) and the failure path (LlmFailedEvent); - response-side metadata (finish_reason / response_model / - response_id) lands only on the success variant for now. The - failure variant carries those fields too (populated only for a - structured_output_invalid failure, proposal 0082), but rendering - them on a failed Generation is deferred to the failed-generation - rendering PR via the isinstance guard below.""" + (LlmCompletionEvent) and the failure path (LlmFailedEvent). + Response-side metadata (finish_reason / response_model / + response_id) renders for a completion and for a + structured_output_invalid failure (its wire response was intact); + every other failure category renders none, per the guard below.""" metadata: dict[str, Any] = {} if correlation_id is not None: metadata["correlation_id"] = correlation_id @@ -2201,14 +2216,18 @@ def _typed_event_metadata( metadata["prompt_group_name"] = active_group.group_name if event.caller_invocation_metadata is not None: _apply_caller_metadata(metadata, event.caller_invocation_metadata) - # Response-side rendering is success-path-only for now. The failure - # variant carries these fields too (populated only for a - # structured_output_invalid failure, proposal 0082), but this - # deliberate isinstance guard defers rendering them on a failed - # Generation to the failed-generation rendering PR; the failure-path - # metadata stays focused on the request-side + the failure-specific - # fields the caller adds separately. - if isinstance(event, LlmCompletionEvent): + # Response-side metadata. A completion always carries it; a + # structured_output_invalid failure also carries it (proposal 0082), + # since its wire response was intact, so finish_reason (the truncation + # signal) and the response identity render on the failed Generation too. + # Every other failure category received no response and renders none. + # In the second operand ``event`` narrows to LlmFailedEvent (the only + # non-completion in the union), so error_category is valid there and is + # never read for a completion (short-circuit). + renders_response_side = ( + isinstance(event, LlmCompletionEvent) or event.error_category == "structured_output_invalid" + ) + if renders_response_side: if event.finish_reason is not None: metadata["finish_reason"] = event.finish_reason if event.response_model is not None: @@ -2217,7 +2236,7 @@ def _typed_event_metadata( metadata["response_id"] = event.response_id return metadata - def _usage_from_typed_event(self, event: LlmCompletionEvent) -> LangfuseUsage | None: + def _usage_from_typed_event(self, event: LlmCompletionEvent | LlmFailedEvent) -> LangfuseUsage | None: """Map the typed event's Usage onto the Langfuse Usage record. Returns None when no usage was reported.""" # Spec observability §8.4.3 (Langfuse usage mapping). diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 2bb118f5..7a6599e4 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -355,10 +355,11 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: ), # Proposal 0082 (structured-output failure diagnostics, spec v0.77.0). # The event surface (120-122), the OTel error-span rendering (124) and the - # §11 token-usage metric (125) are implemented; only the Langfuse - # failed-Generation rendering (123) defers to the final PR of the 0082 split. + # §11 token-usage metric (125) run here; the Langfuse failed-Generation + # rendering (123) is driven in the dedicated test_observability_langfuse + # harness (its langfuse_trace shape lives there, like 022-024). "123-langfuse-failed-generation-renders-output-usage-finish-reason": ( - "structured-output failure Langfuse rendering (proposal 0082) not-yet implemented" + "Langfuse failed-Generation rendering; driven in test_observability_langfuse" ), # Proposal 0083 (per-prompt token-budget observability, spec v0.78.0). # The Prompt.token_budget advisory + budget-exceeded span attribute / diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py index 0f176315..b97d3b71 100644 --- a/tests/conformance/test_observability_langfuse.py +++ b/tests/conformance/test_observability_langfuse.py @@ -117,6 +117,11 @@ # cases 1 + 2 are TEMPORARILY deferred via _DEFERRED_CASES pending the # shared nested-dispatch-keying fix (see that note). "039-nested-lineage-augmentation", + # 123 (proposal 0082): a structured_output_invalid failure renders the + # response-side surface (payload-gated output, usage, metadata.finish_reason) + # on the ERROR failed Generation. Drives the real failure path via + # calls_llm.response_schema + expected_error. + "123-langfuse-failed-generation-renders-output-usage-finish-reason", } ) @@ -1148,7 +1153,17 @@ async def _run_case(case: Mapping[str, Any], *, fixture_stem: str | None = None) await provider.aclose() return - await graph.invoke(initial_state_factory(), **invoke_kwargs) + expected_error = cast("dict[str, Any] | None", case.get("expected_error")) + if expected_error is not None: + from openarmature.graph import NodeException # noqa: PLC0415 + + # A structured_output_invalid failure (fixture 123) raises out of the + # node; the failed Generation is still emitted (proposal 0058), and its + # ERROR level / statusMessage / response surface is asserted below. + with pytest.raises(NodeException): + await graph.invoke(initial_state_factory(), **invoke_kwargs) + else: + await graph.invoke(initial_state_factory(), **invoke_kwargs) await graph.drain() if provider is not None: await provider.aclose() @@ -1413,6 +1428,10 @@ async def _llm_node(_s: Any) -> dict[str, Any]: ) config_spec = cast("dict[str, Any] | None", (calls_llm_spec or {}).get("config")) stores_in = cast("str", (calls_llm_spec or {}).get("stores_response_in", "msg")) + # A calls_llm carrying a response_schema drives the structured-output + # path (proposal 0082 fixture 123): complete() raises + # structured_output_invalid on a parse/schema failure. + response_schema = cast("dict[str, Any] | None", (calls_llm_spec or {}).get("response_schema")) if renders_prompt_name is not None: assert prompt_manager is not None, "renders_prompt requires a prompt_backend block" prompt = await prompt_manager.fetch(renders_prompt_name) @@ -1422,12 +1441,14 @@ async def _llm_node(_s: Any) -> dict[str, Any]: response = await provider.complete( cast("Sequence[Any]", llm_messages), config=_runtime_config_from_spec(config_spec), + response_schema=response_schema, ) else: llm_messages = _materialize_messages(messages_spec or []) response = await provider.complete( cast("Sequence[Any]", llm_messages), config=_runtime_config_from_spec(config_spec), + response_schema=response_schema, ) return {stores_in: response.message.content or ""} @@ -1704,6 +1725,12 @@ def _assert_observation( assert actual.level == expected["level"], ( f"observation {actual.name!r} level: expected {expected['level']!r}, got {actual.level!r}" ) + if "statusMessage" in expected: + # §8.4.2 error-category mapping: statusMessage = the §7 error category. + assert actual.status_message == expected["statusMessage"], ( + f"observation {actual.name!r} statusMessage: " + f"expected {expected['statusMessage']!r}, got {actual.status_message!r}" + ) if "model" in expected: assert actual.model == expected["model"], ( f"observation {actual.name!r} model: expected {expected['model']!r}, got {actual.model!r}" diff --git a/tests/unit/test_llm_provider.py b/tests/unit/test_llm_provider.py index e4e5b531..9c87c244 100644 --- a/tests/unit/test_llm_provider.py +++ b/tests/unit/test_llm_provider.py @@ -1408,6 +1408,35 @@ def _handler(_req: httpx.Request) -> httpx.Response: assert ev.error_message != str(excinfo.value) +def test_structured_output_builder_projects_empty_content_to_none() -> None: + # Proposal 0082 cross-observer parity (defensive): a structured failure whose + # raw_content is empty projects output_content to None in the built event + # (like the success path's `content or None`), so both observers omit it + # identically rather than one rendering "" and the other dropping it on a + # truthiness gate. The OpenAI wire can't currently produce empty content on + # the structured path -- an empty assistant message with no tool calls fails + # earlier as provider_invalid_response, and a tool_calls response skips + # structured validation -- so this pins the projection at the builder. + from openarmature.llm import StructuredOutputInvalid + + provider = OpenAIProvider(base_url="http://test", model="m", api_key="k") + exc = StructuredOutputInvalid( + "empty content", response_schema={}, raw_content="", failure_description="empty" + ) + event = provider._build_llm_failed_event( # noqa: SLF001 + exc, + latency_ms=1.0, + call_id="cc", + input_messages=[], + request_params={}, + request_extras={}, + active_prompt=None, + active_prompt_group=None, + ) + assert event.error_category == "structured_output_invalid" + assert event.output_content is None + + async def test_complete_populates_output_tool_calls_on_typed_events() -> None: # Proposal 0076: provider.complete() populates output_tool_calls # (the ToolCall records) on BOTH the terminal LlmCompletionEvent and diff --git a/tests/unit/test_observability_langfuse.py b/tests/unit/test_observability_langfuse.py index 5c399f60..92b992ff 100644 --- a/tests/unit/test_observability_langfuse.py +++ b/tests/unit/test_observability_langfuse.py @@ -1384,6 +1384,72 @@ async def test_typed_llm_event_emits_generation_with_expected_fields() -> None: assert obs.ended is True +async def test_structured_output_failure_generation_renders_response_surface() -> None: + # Proposal 0082: a structured_output_invalid failure renders the response-side + # surface (output payload-gated, usage, metadata.finish_reason) on the ERROR + # Generation, alongside level=ERROR + statusMessage=category. + from openarmature.llm.response import Usage + from openarmature.observability.correlation import _reset_invocation_id, _set_invocation_id + from tests._helpers.typed_event import make_failed_event + + client = InMemoryLangfuseClient() + observer = LangfuseObserver(client=client, disable_provider_payload=False) + token = _set_invocation_id("inv-soi") + try: + await observer( + make_failed_event( + invocation_id="inv-soi", + error_category="structured_output_invalid", + error_type="StructuredOutputInvalid", + output_content='{"name":"Alice","age":', + finish_reason="length", + usage=Usage(prompt_tokens=20, completion_tokens=16, total_tokens=36), + response_id="cc-xyz", + response_model="gpt-test-v2", + ) + ) + finally: + _reset_invocation_id(token) + + gen = next(o for o in client.traces["inv-soi"].observations if o.type == "generation") + assert gen.level == "ERROR" + assert gen.status_message == "structured_output_invalid" + assert gen.output == '{"name":"Alice","age":' + assert gen.usage == LangfuseUsage(input=20, output=16, total=36) + assert gen.metadata.get("finish_reason") == "length" + assert gen.metadata.get("response_model") == "gpt-test-v2" + + +async def test_structured_output_failure_generation_redacts_output_when_payload_disabled() -> None: + # Payload-gated: with disable_provider_payload=True, output is redacted while + # usage / metadata.finish_reason / ERROR level stay (proposal 0082). + from openarmature.llm.response import Usage + from openarmature.observability.correlation import _reset_invocation_id, _set_invocation_id + from tests._helpers.typed_event import make_failed_event + + client = InMemoryLangfuseClient() + observer = LangfuseObserver(client=client, disable_provider_payload=True) + token = _set_invocation_id("inv-soi2") + try: + await observer( + make_failed_event( + invocation_id="inv-soi2", + error_category="structured_output_invalid", + output_content='{"name":"Alice","age":', + finish_reason="length", + usage=Usage(prompt_tokens=20, completion_tokens=16, total_tokens=36), + ) + ) + finally: + _reset_invocation_id(token) + + gen = next(o for o in client.traces["inv-soi2"].observations if o.type == "generation") + assert gen.level == "ERROR" + assert gen.output is None + assert gen.usage == LangfuseUsage(input=20, output=16, total=36) + assert gen.metadata.get("finish_reason") == "length" + + async def test_typed_llm_event_back_dates_generation_using_latency_ms() -> None: # Generation observation's start/end timestamps reflect the # adapter-boundary latency rather than the typed event's arrival From 25d90244cce4a960acb09f87598741eea4861eb4 Mon Sep 17 00:00:00 2001 From: chris-colinsky Date: Thu, 23 Jul 2026 12:05:47 -0700 Subject: [PATCH 2/2] Tighten 0082 Langfuse failure-path assertions Address CoPilot review threads on PR #232 (0082 Langfuse failed-Generation rendering): - Assert metadata.response_id on the failed-Generation unit test. The event set it but only response_model was pinned, so a regression dropping response_id would have passed. - Assert raised_from against the raised NodeException in the linear conformance path. Fixture 123 declares it but nothing checked it. Category is not asserted on the exception (its category is always node_exception); it rides on the Generation statusMessage. - Restructure renders_response_side to an explicit if/else so the failure branch is obvious. No behavior change. --- src/openarmature/observability/langfuse/observer.py | 11 +++++------ tests/conformance/test_observability_langfuse.py | 13 +++++++++++-- tests/unit/test_observability_langfuse.py | 1 + 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/openarmature/observability/langfuse/observer.py b/src/openarmature/observability/langfuse/observer.py index 4b5a4879..04430acf 100644 --- a/src/openarmature/observability/langfuse/observer.py +++ b/src/openarmature/observability/langfuse/observer.py @@ -2221,12 +2221,11 @@ def _typed_event_metadata( # since its wire response was intact, so finish_reason (the truncation # signal) and the response identity render on the failed Generation too. # Every other failure category received no response and renders none. - # In the second operand ``event`` narrows to LlmFailedEvent (the only - # non-completion in the union), so error_category is valid there and is - # never read for a completion (short-circuit). - renders_response_side = ( - isinstance(event, LlmCompletionEvent) or event.error_category == "structured_output_invalid" - ) + if isinstance(event, LlmCompletionEvent): + renders_response_side = True + else: + # event narrows to LlmFailedEvent (the only other union member). + renders_response_side = event.error_category == "structured_output_invalid" if renders_response_side: if event.finish_reason is not None: metadata["finish_reason"] = event.finish_reason diff --git a/tests/conformance/test_observability_langfuse.py b/tests/conformance/test_observability_langfuse.py index b97d3b71..35e91af0 100644 --- a/tests/conformance/test_observability_langfuse.py +++ b/tests/conformance/test_observability_langfuse.py @@ -1159,9 +1159,18 @@ async def _run_case(case: Mapping[str, Any], *, fixture_stem: str | None = None) # A structured_output_invalid failure (fixture 123) raises out of the # node; the failed Generation is still emitted (proposal 0058), and its - # ERROR level / statusMessage / response surface is asserted below. - with pytest.raises(NodeException): + # ERROR level / statusMessage / response surface is asserted below. The + # §7 error category rides on the Generation's statusMessage (checked in + # _assert_observation), NOT on the outer NodeException (whose category + # is always "node_exception"), so category is not re-asserted here. + with pytest.raises(NodeException) as exc_info: await graph.invoke(initial_state_factory(), **invoke_kwargs) + expected_raised_from = expected_error.get("raised_from") + if expected_raised_from is not None: + assert exc_info.value.node_name == expected_raised_from, ( + f"expected error raised_from: expected {expected_raised_from!r}, " + f"got {exc_info.value.node_name!r}" + ) else: await graph.invoke(initial_state_factory(), **invoke_kwargs) await graph.drain() diff --git a/tests/unit/test_observability_langfuse.py b/tests/unit/test_observability_langfuse.py index 92b992ff..90c3af5d 100644 --- a/tests/unit/test_observability_langfuse.py +++ b/tests/unit/test_observability_langfuse.py @@ -1418,6 +1418,7 @@ async def test_structured_output_failure_generation_renders_response_surface() - assert gen.usage == LangfuseUsage(input=20, output=16, total=36) assert gen.metadata.get("finish_reason") == "length" assert gen.metadata.get("response_model") == "gpt-test-v2" + assert gen.metadata.get("response_id") == "cc-xyz" async def test_structured_output_failure_generation_redacts_output_when_payload_disabled() -> None: