diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a2ada0f..d3b105fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). The - **PromptManager service-wide default cache TTL** (proposal 0086, prompt-management §6, spec v0.79.0). `PromptManager` construction accepts an optional `default_cache_ttl_seconds`, applied to any `fetch` or `get` that omits a per-call `cache_ttl_seconds`. Resolution follows a precedence chain: an explicit per-call value (including `0` force-fresh) wins; otherwise the manager default applies; otherwise nothing is forwarded and the backend's own caching governs. An omitted or explicit-`None` per-call value both select the default, so resolution does not depend on argument presence. A negative default is rejected at construction, and the per-call negative rejection is unchanged. `get` delegates to `fetch` and inherits the chain, and the bundled caching backends need no change (they already honor a resolved `cache_ttl_seconds`). Conformance fixture 036 is un-deferred. - **PromptGroup arity enforcement** (proposal 0080, prompt-management §10 / §11, spec v0.75.0). Constructing a `PromptGroup` with fewer than two members (an empty or single-member group) now raises a categorized `PromptGroupInvalid` at construction, before any render or LLM call. This replaces the previous bare `ValueError` (which pydantic folded into a `ValidationError` carrying no error category) and adds a `prompt_group_invalid` category to the prompt-management error set. `PromptGroupInvalid` is non-transient and is exported from `openarmature.prompts`. Conformance fixture 035 is un-deferred. +- **Structured-output failure diagnostics on the LLM failure event** (proposal 0082, graph-engine §6 + llm-provider §7, spec v0.77.0). A `structured_output_invalid` failure is a completion whose final validation gate failed, so the wire response is intact. `LlmFailedEvent` now carries the response-side surface for that one category (`None` for every other): `output_content` (the content that failed validation, payload-gated), `finish_reason` (the truncation-vs-malformed triage signal, `"length"` vs `"stop"`), `usage`, `response_id`, and `response_model`. The `StructuredOutputInvalid` error additionally exposes `finish_reason` and `usage`, so a caller's exception handler can make a truncation-aware retry decision. Observer rendering of the surface (OTel error span, Langfuse failed Generation) and the token-usage-on-failure metric land in follow-on PRs of the same proposal. ## [0.16.0] — 2026-07-18 diff --git a/conformance.toml b/conformance.toml index 57c8594b..5c8d0e86 100644 --- a/conformance.toml +++ b/conformance.toml @@ -816,10 +816,13 @@ since = "0.16.0" # Spec v0.77.0 (proposal 0082). Structured-output failure diagnostics # (graph-engine §6 -- LlmFailedEvent response-side surface for # structured_output_invalid + llm-provider §7 finish_reason / usage). -# Not-yet; llm-provider fixtures 022/023 and observability fixtures -# 120-125 defer with it. +# Partial since 0.17.0: the event + error surface is implemented (PR A); +# the OTel / Langfuse / metrics RENDERING (observability fixtures 123-125) +# lands in later PRs of the split. [proposals."0082"] -status = "not-yet" +status = "partial" +since = "0.17.0" +note = "The graph-engine §6 event surface + llm-provider §7 error surface (PR A). LlmFailedEvent gains five response-side fields (output_content / finish_reason / usage / response_id / response_model), populated ONLY for structured_output_invalid (None for every other category); output_content is payload-bearing, the other four are not gated. StructuredOutputInvalid gains finish_reason / usage (the §7-mandated pair) plus response_id / response_model (internal, for the event builder), attached at the OpenAI provider's parse/validate call site where the intact wire response is in hand; the failed-event builder reads them off the enriched error. Un-defers observability fixtures 120 (truncation, finish_reason=length) / 121 (schema-mismatch, finish_reason=stop) / 122 (null-on-non-body companion), driven via the typed-event collector with a new calls_llm.response_schema harness lever; and llm-provider fixtures 022 / 023, whose carries block now asserts finish_reason + usage (the carries reader gained subset matching for a mapping-valued field like usage). PARTIAL: the OTel error-span rendering (124), Langfuse failed-Generation rendering (123), and the §11.2 token-usage-on-failure metric (125) are deferred to the OTel+metrics and Langfuse PRs -- they render the surface the event now carries." # Spec v0.78.0 (proposal 0083). Per-prompt token-budget observability # (prompt-management §3 / graph-engine §6 / observability §5.5.15). diff --git a/docs/concepts/observability.md b/docs/concepts/observability.md index 3065cba6..e5b50487 100644 --- a/docs/concepts/observability.md +++ b/docs/concepts/observability.md @@ -1017,10 +1017,14 @@ async def my_llm_observer(event): The two variants are **mutually exclusive on a single `complete()` call** — implementations MUST NOT emit both for the same call. Conformance fixture 072 locks this down. The failure variant carries -the same identity + request-side fields as the completion variant, -minus the response-side fields (`response_id`, `response_model`, -`usage`, `output_content`, `finish_reason`) — there was no response -to record. +the same identity + request-side fields as the completion variant. The +response-side fields (`output_content`, `finish_reason`, `usage`, +`response_id`, `response_model`) are populated only for a +`structured_output_invalid` failure, where the model returned a +response whose content failed downstream parse or validation, so +`finish_reason` (`"length"` signals a truncation) and the token usage +genuinely exist; they are `None` for every other category, where there +was no response to record. A custom `Provider` that wants observers to see the same events dispatches `LlmCompletionEvent(...)` on success and diff --git a/src/openarmature/graph/events.py b/src/openarmature/graph/events.py index 0fdb3711..5047dfe4 100644 --- a/src/openarmature/graph/events.py +++ b/src/openarmature/graph/events.py @@ -629,9 +629,19 @@ class LlmFailedEvent: Identity / scoping / request-side field set mirrors ``LlmCompletionEvent`` 1:1 — same field semantics, same nullability - rules. Response-side fields (``response_id``, ``response_model``, - ``usage``, ``output_content``, ``finish_reason``) are ABSENT from - this variant — no response was received. + rules. The response-side fields (``output_content``, + ``finish_reason``, ``usage``, ``response_id``, ``response_model``) + are populated ONLY for a ``structured_output_invalid`` failure — the + one category where the provider returned a response (content that + failed downstream parse or validation), so this event is in effect a + completion whose final validation gate failed. They are ``None`` for + every other category (no response was received). ``output_content`` + is payload-bearing (gated observer-side by ``disable_provider_payload`` + like the success variant); the other four are not gated. + ``error_message`` carries the failure summary plus the failing-locator + description for this category, so observers read the diagnostic there + without a dedicated field. The validated value is not carried, as on + the completion event. Failure-specific fields: @@ -673,6 +683,17 @@ class LlmFailedEvent: error_message: str error_type: str | None = None caller_invocation_metadata: Mapping[str, AttributeValue] | None = None + # Response-side surface, populated only for structured_output_invalid + # (the failure is a downstream parse/validation step on an intact wire + # response). None for every other category. output_content is + # payload-bearing; the other four are not gated. Mirrors the same fields + # on LlmCompletionEvent, less output_tool_calls (a structured-output + # failure never carries tool calls). + output_content: str | None = None + finish_reason: str | None = None + usage: "Usage | None" = None + response_id: str | None = None + response_model: str | None = None # Python-internal per-attempt LLM event. NOT a spec-normative event type diff --git a/src/openarmature/llm/errors.py b/src/openarmature/llm/errors.py index 6673fd11..95de9807 100644 --- a/src/openarmature/llm/errors.py +++ b/src/openarmature/llm/errors.py @@ -16,7 +16,10 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from openarmature.llm.response import Usage # --------------------------------------------------------------------------- # Canonical category strings (llm-provider spec §7) @@ -199,12 +202,29 @@ class StructuredOutputInvalid(LlmProviderError): raw_content: The raw response content the model produced. failure_description: A description of the parse or validation failure. + finish_reason: The normalized finish reason of the response that + failed validation (``"length"`` signals a truncation, the key + retry signal; ``"stop"`` a clean-finish schema/parse failure). + usage: Token usage of the response that failed validation, for + cost attribution and truncation corroboration. + response_id: The provider's response identifier, when present. + response_model: The model identifier the provider reported, when + present. + + The failure is a downstream parse/validation step on an intact wire + response, so the response-side context genuinely exists. It is + attached at the parse/validate call site (which holds the parsed + Response); the four fields default to ``None`` until enriched there. """ category = STRUCTURED_OUTPUT_INVALID response_schema: dict[str, Any] raw_content: str failure_description: str + finish_reason: str | None + usage: Usage | None + response_id: str | None + response_model: str | None def __init__( self, @@ -212,11 +232,19 @@ def __init__( response_schema: dict[str, Any], raw_content: str, failure_description: str, + finish_reason: str | None = None, + usage: Usage | None = None, + response_id: str | None = None, + response_model: str | None = None, ) -> None: super().__init__(*args) self.response_schema = response_schema self.raw_content = raw_content self.failure_description = failure_description + self.finish_reason = finish_reason + self.usage = usage + self.response_id = response_id + self.response_model = response_model __all__ = [ diff --git a/src/openarmature/llm/providers/openai.py b/src/openarmature/llm/providers/openai.py index f0357b9d..e23ec2ff 100644 --- a/src/openarmature/llm/providers/openai.py +++ b/src/openarmature/llm/providers/openai.py @@ -752,6 +752,27 @@ def _build_llm_failed_event( caller_metadata: Mapping[str, AttributeValue] | None = None if self._populate_caller_metadata: caller_metadata = dict(current_invocation_metadata()) + # Response-side surface, populated only for structured_output_invalid + # (proposal 0082): the failure is a downstream validation step on an + # intact wire response, so the model's output genuinely exists and was + # attached to the error at the parse/validate call site. None for every + # other category (no response received). + output_content: str | None = None + finish_reason: str | None = None + usage: Usage | None = None + response_id: str | None = None + response_model: str | None = None + error_message = str(exc) + if isinstance(exc, StructuredOutputInvalid): + output_content = exc.raw_content + finish_reason = exc.finish_reason + usage = exc.usage + response_id = exc.response_id + response_model = exc.response_model + # error_message carries the failing locator (proposal 0082): the + # terse category summary alone drops the which-field detail, so + # combine it with the failure_description observers triage on. + error_message = f"{exc}: {exc.failure_description}" return LlmFailedEvent( invocation_id=invocation_id, correlation_id=current_correlation_id(), @@ -771,8 +792,13 @@ def _build_llm_failed_event( call_id=call_id, error_category=exc.category, error_type=type(exc).__name__, - error_message=str(exc), + error_message=error_message, caller_invocation_metadata=caller_metadata, + output_content=output_content, + finish_reason=finish_reason, + usage=usage, + response_id=response_id, + response_model=response_model, ) def _build_llm_retry_attempt_event( @@ -1065,23 +1091,36 @@ def _parse_response( except ValidationError as exc: raise ProviderInvalidResponse(f"invalid usage record: {exc}") from exc - # Structured-output parsing. parsed is absent when no schema - # was requested AND when the response is a tool-call response - # — the tool-call path and structured-content path are - # mutually exclusive at the response level. - parsed: ParsedValue = None - if schema_dict is not None and finish_reason_typed != "tool_calls": - parsed = _parse_and_validate(assistant_msg.content, schema_dict, schema_class) - # gen_ai.response.id / gen_ai.response.model semconv (spec # §5.5.3) read these off the Response. The wire fields are # optional — providers MAY omit either or both. ``None`` when - # absent or not a string. + # absent or not a string. Parsed before structured-output + # validation so a structured_output_invalid failure can carry the + # response identity (proposal 0082). response_id_raw = payload.get("id") response_id: str | None = response_id_raw if isinstance(response_id_raw, str) else None response_model_raw = payload.get("model") response_model: str | None = response_model_raw if isinstance(response_model_raw, str) else None + # Structured-output parsing. parsed is absent when no schema + # was requested AND when the response is a tool-call response + # — the tool-call path and structured-content path are + # mutually exclusive at the response level. + parsed: ParsedValue = None + if schema_dict is not None and finish_reason_typed != "tool_calls": + try: + parsed = _parse_and_validate(assistant_msg.content, schema_dict, schema_class) + except StructuredOutputInvalid as exc: + # The wire response is intact; attach its response-side + # context so the failed event / §7 error carries finish_reason + # (truncation triage), usage, and the response identity + # (proposal 0082). raw_content is already set by the helper. + exc.finish_reason = finish_reason_typed + exc.usage = usage + exc.response_id = response_id + exc.response_model = response_model + raise + return Response( message=assistant_msg, finish_reason=finish_reason_typed, diff --git a/src/openarmature/observability/langfuse/observer.py b/src/openarmature/observability/langfuse/observer.py index ec34451f..afa0d816 100644 --- a/src/openarmature/observability/langfuse/observer.py +++ b/src/openarmature/observability/langfuse/observer.py @@ -2171,8 +2171,11 @@ def _typed_event_metadata( typed LLM event. Shared between the success path (LlmCompletionEvent) and the failure path (LlmFailedEvent); response-side metadata (finish_reason / response_model / - response_id) lands only on the success variant since those - fields don't exist on LlmFailedEvent.""" + response_id) lands only on the success variant for now. The + failure variant carries those fields too (populated only for a + structured_output_invalid failure, proposal 0082), but rendering + them on a failed Generation is deferred to the failed-generation + rendering PR via the isinstance guard below.""" metadata: dict[str, Any] = {} if correlation_id is not None: metadata["correlation_id"] = correlation_id @@ -2198,11 +2201,13 @@ def _typed_event_metadata( metadata["prompt_group_name"] = active_group.group_name if event.caller_invocation_metadata is not None: _apply_caller_metadata(metadata, event.caller_invocation_metadata) - # Response-side fields are LlmCompletionEvent-only — absent - # from LlmFailedEvent. The type guard keeps the success-path - # metadata complete while the failure-path metadata stays - # focused on the request-side + the failure-specific fields - # the caller adds separately. + # Response-side rendering is success-path-only for now. The failure + # variant carries these fields too (populated only for a + # structured_output_invalid failure, proposal 0082), but this + # deliberate isinstance guard defers rendering them on a failed + # Generation to the failed-generation rendering PR; the failure-path + # metadata stays focused on the request-side + the failure-specific + # fields the caller adds separately. if isinstance(event, LlmCompletionEvent): if event.finish_reason is not None: metadata["finish_reason"] = event.finish_reason diff --git a/tests/conformance/harness/wire.py b/tests/conformance/harness/wire.py index 8cc7bd65..7c9cc394 100644 --- a/tests/conformance/harness/wire.py +++ b/tests/conformance/harness/wire.py @@ -181,12 +181,41 @@ def assert_error_carries(exc: BaseException, carries: Mapping[str, Any]) -> None ) else: actual = _get_carries_attr(exc, key) - if actual != expected: + if isinstance(expected, Mapping): + # A mapping-valued field (e.g. usage) is a subset match: each + # named key must equal the actual's value, reading a pydantic + # record's fields (Usage) or a plain mapping. Keys the fixture + # omits are ignored (the record MAY carry optional extras). + actual_map = _as_carries_mapping(actual) + if actual_map is None: + raise AssertionError( + f"carries check failed: {key!r} is not a mapping/record " + f"(got {type(actual).__name__}); cannot subset-match {expected!r}" + ) + for subkey, subval in cast("Mapping[str, Any]", expected).items(): + if actual_map.get(subkey) != subval: + raise AssertionError( + f"carries check failed: {key!r}[{subkey!r}] " + f"actual={actual_map.get(subkey)!r}, expected={subval!r}" + ) + elif actual != expected: raise AssertionError( f"carries check failed: {key!r} actual={actual!r}, expected={expected!r}" ) +def _as_carries_mapping(value: Any) -> Mapping[str, Any] | None: + """Coerce a carries attribute to a mapping for subset comparison: a plain + Mapping passes through; a pydantic record (e.g. Usage) is dumped to a dict; + anything else returns None (not comparable as a mapping).""" + if isinstance(value, Mapping): + return cast("Mapping[str, Any]", value) + dump = getattr(value, "model_dump", None) + if callable(dump): + return cast("Mapping[str, Any]", dump()) + return None + + def _get_carries_attr(exc: BaseException, name: str) -> Any: # Allow fixture-naming-friendly aliases for the carries block. The # spec fixtures use ``raw_response_content`` (the wire-side label); diff --git a/tests/conformance/test_fixture_parsing.py b/tests/conformance/test_fixture_parsing.py index d4ac69bd..6a3239e7 100644 --- a/tests/conformance/test_fixture_parsing.py +++ b/tests/conformance/test_fixture_parsing.py @@ -597,16 +597,18 @@ def _id(case: tuple[str, Path]) -> str: "observability/119-otel-callable-branch-attempt-index-under-node-retry": ( "Proposal 0075 callable-branch coverage round-out; harness shape not modelled" ), - # Proposal 0082 (structured-output failure diagnostics, v0.77.0) -- the - # LlmFailedEvent response-side directive shape. + # Proposal 0082 (structured-output failure diagnostics, v0.77.0) -- + # fixtures 120-122 share the same typed_event_collector shape as 069-073 + # that the cross-cap parser doesn't model; the response-side surface IS + # implemented and runtime-driven in test_observability. "observability/120-llm-failure-event-structured-output-truncation": ( - "Proposal 0082 structured-output failure diagnostics; not implemented" + "Proposal 0082 structured-output failure event; harness typed_event_collector schema pending" ), "observability/121-llm-failure-event-structured-output-schema-mismatch": ( - "Proposal 0082 structured-output failure diagnostics; not implemented" + "Proposal 0082 structured-output failure event; harness typed_event_collector schema pending" ), "observability/122-llm-failure-event-response-side-null-on-non-body-failure": ( - "Proposal 0082 structured-output failure diagnostics; not implemented" + "Proposal 0082 structured-output failure event; harness typed_event_collector schema pending" ), # Proposal 0083 (per-prompt token-budget observability, v0.78.0) -- the # token_budget directive shape + budget-exceeded expectations. diff --git a/tests/conformance/test_llm_provider.py b/tests/conformance/test_llm_provider.py index 3116742f..1efa2fd2 100644 --- a/tests/conformance/test_llm_provider.py +++ b/tests/conformance/test_llm_provider.py @@ -112,22 +112,6 @@ "056-call-level-retry-transient": ("per-attempt LLM spans; see test_observability_otel.py"), "057-call-level-retry-exhaustion": ("per-attempt LLM spans; see test_observability_otel.py"), "058-call-level-retry-non-transient-no-retry": ("per-attempt LLM spans; see test_observability_otel.py"), - # ----- v0.16.0 spec-pin bump (v0.70.1 -> v0.84.0) ------------------- - # Proposal 0082 (structured-output failure diagnostics, spec v0.77.0) - # extended fixtures 022/023 to additionally assert the now-required - # finish_reason + usage on the structured_output_invalid error. The - # base structured_output_invalid mapping still has coverage in - # tests/unit/test_structured_output.py:: - # test_pydantic_validation_failure_wraps_in_structured_output_invalid; - # only 0082's additive finish_reason / usage-on-error response-side - # fields are unimplemented, which is why the conformance fixtures defer - # until a later v0.16.0 PR lands 0082. - "022-structured-output-parse-failure": ( - "Proposal 0082 finish_reason/usage on structured_output_invalid; not implemented" - ), - "023-structured-output-validation-failure": ( - "Proposal 0082 finish_reason/usage on structured_output_invalid; not implemented" - ), # Proposal 0062 (LLM completion streaming, spec v0.71.0) -- the stream # flag on complete() + SSE wire handling + the streaming-unsupported # rejection. Unimplemented until a later v0.16.0 PR. diff --git a/tests/conformance/test_observability.py b/tests/conformance/test_observability.py index 7ba7873a..7a6ff6ef 100644 --- a/tests/conformance/test_observability.py +++ b/tests/conformance/test_observability.py @@ -184,6 +184,12 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "068-llm-completion-event-response-model-distinct-from-request", "071-llm-failure-event-call-id-distinct-from-completion-event", "072-llm-failure-event-mutual-exclusion-with-completion-event", + # proposal 0082: the LlmFailedEvent response-side surface on a + # structured_output_invalid failure (120/121) + the null-on-non-body + # companion (122). Rendering fixtures 123-125 stay deferred (later PRs). + "120-llm-failure-event-structured-output-truncation", + "121-llm-failure-event-structured-output-schema-mismatch", + "122-llm-failure-event-response-side-null-on-non-body-failure", # proposal 0052 attribution fixture (case 1) + proposal 0061 # (case 2: the §5.1 attribution lands on the detached trace's own # openarmature.invocation span). Wired together now that 0061 @@ -342,15 +348,12 @@ def _reset_otel_global_tracer_provider(restore_to: object) -> None: "case not yet wired -- harness gap, not unimplemented" ), # Proposal 0082 (structured-output failure diagnostics, spec v0.77.0). - # The LlmFailedEvent response-side surface (output_content / finish_reason - # / usage on structured_output_invalid) is unimplemented until a later - # v0.16.0 PR; the OTel / Langfuse / metrics rendering fixtures defer too. + # The event-level response-side surface (120-122) is implemented; the + # OTel / Langfuse / metrics RENDERING fixtures defer to later PRs of the + # 0082 split (they render the surface the event now carries). **{ - fixture_id: "structured-output failure diagnostics (proposal 0082) not-yet implemented" + fixture_id: "structured-output failure diagnostics rendering (proposal 0082) not-yet implemented" for fixture_id in ( - "120-llm-failure-event-structured-output-truncation", - "121-llm-failure-event-structured-output-schema-mismatch", - "122-llm-failure-event-response-side-null-on-non-body-failure", "123-langfuse-failed-generation-renders-output-usage-finish-reason", "124-otel-error-span-renders-output-usage-finish-reason", "125-metrics-token-usage-on-structured-output-failure", @@ -655,7 +658,12 @@ async def test_observability_fixture(fixture_path: Path) -> None: "068-llm-completion-event-response-model-distinct-from-request", }: await _run_typed_event_cases(spec) - elif fixture_id == "072-llm-failure-event-mutual-exclusion-with-completion-event": + elif fixture_id in { + "072-llm-failure-event-mutual-exclusion-with-completion-event", + "120-llm-failure-event-structured-output-truncation", + "121-llm-failure-event-structured-output-schema-mismatch", + "122-llm-failure-event-response-side-null-on-non-body-failure", + }: await _run_typed_event_cases(spec, expect_failure=True) elif fixture_id == "067-llm-completion-event-call-id-always-present-and-distinct": await _run_typed_event_chain_cases(spec) @@ -4799,9 +4807,15 @@ def _build_simple_llm_graph( else: messages = [] + response_schema = cast("dict[str, Any] | None", calls_llm_spec.get("response_schema")) + async def ask_body(_s: Any) -> dict[str, str]: response = await _complete_with_optional_prompt( - provider, messages, config=runtime_config, prompt_result=prompt_result + provider, + messages, + config=runtime_config, + prompt_result=prompt_result, + response_schema=response_schema, ) return {stores_in: response.message.content or ""} @@ -4941,9 +4955,14 @@ async def _complete_with_optional_prompt( *, config: Any, prompt_result: Any, + response_schema: Any = None, ) -> Any: """Call ``provider.complete`` inside the active-prompt context when the node rendered a prompt, otherwise call it directly. + + A ``response_schema`` drives the structured-output path (proposal 0082 + fixtures 120/121): complete() validates the content and raises + structured_output_invalid on a parse/schema failure. """ if prompt_result is not None: from openarmature.prompts import with_active_prompt @@ -4951,8 +4970,8 @@ async def _complete_with_optional_prompt( # Inside with_active_prompt so the provider stamps active_prompt onto # the typed event. with with_active_prompt(prompt_result): - return await provider.complete(messages, config=config) - return await provider.complete(messages, config=config) + return await provider.complete(messages, config=config, response_schema=response_schema) + return await provider.complete(messages, config=config, response_schema=response_schema) def _build_chain_llm_graph( diff --git a/tests/unit/test_llm_provider.py b/tests/unit/test_llm_provider.py index 9dd42259..e4e5b531 100644 --- a/tests/unit/test_llm_provider.py +++ b/tests/unit/test_llm_provider.py @@ -1335,6 +1335,77 @@ def _503(_req: httpx.Request) -> httpx.Response: assert len(failed_events) == 1 assert failed_events[0].error_category == "provider_unavailable" assert failed_events[0].error_type == "ProviderUnavailable" + # Proposal 0082: the response-side surface is null for a non-structured + # (no-response) failure category. + assert failed_events[0].output_content is None + assert failed_events[0].finish_reason is None + assert failed_events[0].usage is None + assert failed_events[0].response_id is None + assert failed_events[0].response_model is None + + +async def test_complete_structured_output_failure_event_carries_response_surface() -> None: + # Proposal 0082: a structured_output_invalid failure is a completion + # whose validation gate failed, so the LlmFailedEvent carries the + # response-side surface (finish_reason for retry triage, output_content, + # usage, response identity), and error_message carries the failing + # locator (the failure_description) rather than just the terse summary. + from openarmature.graph.events import LlmFailedEvent + from openarmature.llm import StructuredOutputInvalid + + schema = { + "type": "object", + "properties": {"name": {"type": "string"}, "age": {"type": "integer"}}, + "required": ["name", "age"], + "additionalProperties": False, + } + + def _handler(_req: httpx.Request) -> httpx.Response: + # Valid JSON missing the required 'age' -> schema-validation failure, + # finish_reason "stop" (a clean-finish malformed output, not a truncation). + return httpx.Response( + 200, + json={ + "id": "cc-xyz", + "object": "chat.completion", + "model": "gpt-test-v2", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": '{"name": "Alice"}'}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 20, "completion_tokens": 8, "total_tokens": 28}, + }, + ) + + events, token = _collecting_dispatch() + provider = OpenAIProvider( + base_url="http://test", model="m", api_key="k", transport=httpx.MockTransport(_handler) + ) + try: + with pytest.raises(StructuredOutputInvalid) as excinfo: + await provider.complete([UserMessage(content="hi")], response_schema=schema) + finally: + await provider.aclose() + _release_dispatch(token) + + failed = [e for e in events if isinstance(e, LlmFailedEvent)] + assert len(failed) == 1 + ev = failed[0] + assert ev.error_category == "structured_output_invalid" + assert ev.finish_reason == "stop" + assert ev.output_content == '{"name": "Alice"}' + assert ev.usage is not None + assert ev.usage.completion_tokens == 8 + assert ev.response_id == "cc-xyz" + assert ev.response_model == "gpt-test-v2" + # error_message carries the failing-field locator, not just the terse + # category summary (proposal 0082). + assert "age" in ev.error_message + assert str(excinfo.value) in ev.error_message + assert ev.error_message != str(excinfo.value) async def test_complete_populates_output_tool_calls_on_typed_events() -> None: @@ -1585,7 +1656,10 @@ def test_build_llm_failed_event_maps_category_and_type_per_exception( ) assert event.error_category == expected_category assert event.error_type == expected_cls_name - assert event.error_message == "boom" + # error_message is str(exc) for all categories except + # structured_output_invalid, which appends the failure_description + # locator (proposal 0082); startswith covers both. + assert event.error_message.startswith("boom") assert event.latency_ms == 12.0 assert event.call_id == "cc-test" diff --git a/tests/unit/test_structured_output.py b/tests/unit/test_structured_output.py index aaf8045c..c3a59cff 100644 --- a/tests/unit/test_structured_output.py +++ b/tests/unit/test_structured_output.py @@ -556,6 +556,14 @@ async def test_pydantic_validation_failure_wraps_in_structured_output_invalid() err = excinfo.value assert err.raw_content == '{"name":"Alice","age":"thirty"}' assert "age" in err.failure_description + # Proposal 0082: the error carries the intact response's response-side + # context (finish_reason for retry triage, usage, response identity), + # attached at the parse/validate call site. + assert err.finish_reason == "stop" + assert err.response_id == "test" + assert err.response_model == "test-model" + assert err.usage is not None + assert err.usage.completion_tokens == 5 async def test_pydantic_class_wire_body_matches_dict_form() -> None: