Skip to content

Commit 853b6d5

Browse files
test(unit): prompts subpackage + OTel attribute propagation
Adds tests/unit/test_prompts.py (25 tests) covering gaps the conformance fixtures don't exercise directly: - error categories match spec §10 strings; PROMPT_TRANSIENT_CATEGORIES contains only prompt_store_unavailable. - error attribute carriage (PromptNotFound name/label/backend, PromptRenderError name/version/label/variables/description). - template_hash / rendered_hash determinism, prefix, and length; divergence for different inputs. - Prompt extra-field rejection; PromptGroup 0/1-member rejection and 2+ acceptance. - PromptManager construction (zero-backend rejection). - Empty-string render output boundary wrap (the spec-agent's concern about Jinja2 cleanly rendering '' but UserMessage rejecting empty content — verified to surface as PromptRenderError). - Identity-field propagation from Prompt to PromptResult on render. - FilesystemPromptBackend disk I/O: success path, missing file raises PromptNotFound, OSError that isn't FileNotFoundError raises PromptStoreUnavailable. - Context-var propagation: with_active_prompt / _prompt_group set + reset, innermost-wins nesting, async-task visibility. - PromptManager fallback gaps: first-match short-circuits later backends; render returns a UserMessage carrying the rendered text. Adds two OTel observer tests under tests/unit/test_observability_otel.py: - Active prompt + active prompt group propagates the six openarmature.prompt.* span attributes (name, version, label, template_hash, rendered_hash, group_name) on the openarmature.llm.complete span. - Without an active prompt, the LLM-call span carries no openarmature.prompt.* attributes.
1 parent 9bdbcb7 commit 853b6d5

2 files changed

Lines changed: 516 additions & 0 deletions

File tree

tests/unit/test_observability_otel.py

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,140 @@ async def test_checkpoint_save_emits_zero_duration_span() -> None:
256256
# ---------------------------------------------------------------------------
257257

258258

259+
async def test_active_prompt_propagates_to_llm_span_attributes() -> None:
260+
"""Spec prompt-management §11: when an LLM call fires inside a
261+
``with_active_prompt`` context, the OTel observer MUST surface
262+
``openarmature.prompt.*`` attributes on the LLM-call span.
263+
``with_active_prompt_group`` adds ``openarmature.prompt.group_name``."""
264+
from datetime import UTC, datetime
265+
266+
from openarmature.graph.events import NodeEvent
267+
from openarmature.llm.messages import UserMessage
268+
from openarmature.llm.providers.openai import _LlmEventState
269+
from openarmature.observability.correlation import (
270+
_reset_invocation_id,
271+
_set_invocation_id,
272+
)
273+
from openarmature.prompts import (
274+
Prompt,
275+
PromptGroup,
276+
PromptResult,
277+
with_active_prompt,
278+
with_active_prompt_group,
279+
)
280+
281+
exporter = InMemorySpanExporter()
282+
observer = OTelObserver(span_processor=SimpleSpanProcessor(exporter))
283+
284+
now = datetime.now(UTC)
285+
prompt = Prompt(
286+
name="greeting",
287+
version="v1",
288+
label="production",
289+
template="Hello, {{ user }}!",
290+
template_hash="sha256:tpl",
291+
fetched_at=now,
292+
)
293+
result = PromptResult(
294+
name=prompt.name,
295+
version=prompt.version,
296+
label=prompt.label,
297+
template_hash=prompt.template_hash,
298+
rendered_hash="sha256:rendered",
299+
messages=[UserMessage(content="Hello, Alice!")],
300+
variables={"user": "Alice"},
301+
fetched_at=now,
302+
rendered_at=now,
303+
)
304+
group = PromptGroup(group_name="classifier_chain", members=[result, result])
305+
306+
token = _set_invocation_id("inv-1")
307+
try:
308+
with with_active_prompt(result), with_active_prompt_group(group):
309+
started = NodeEvent(
310+
node_name="openarmature.llm.complete",
311+
namespace=("openarmature.llm.complete",),
312+
step=-1,
313+
phase="started",
314+
pre_state=_LlmEventState(call_id="test-call-prompt", model="test-m"),
315+
post_state=None,
316+
error=None,
317+
parent_states=(),
318+
)
319+
completed = NodeEvent(
320+
node_name="openarmature.llm.complete",
321+
namespace=("openarmature.llm.complete",),
322+
step=-1,
323+
phase="completed",
324+
pre_state=_LlmEventState(call_id="test-call-prompt", model="test-m", finish_reason="stop"),
325+
post_state=None,
326+
error=None,
327+
parent_states=(),
328+
)
329+
await observer(started)
330+
await observer(completed)
331+
finally:
332+
_reset_invocation_id(token)
333+
334+
observer.shutdown()
335+
llm_spans = [s for s in exporter.get_finished_spans() if s.name == "openarmature.llm.complete"]
336+
assert len(llm_spans) == 1
337+
attrs = llm_spans[0].attributes or {}
338+
assert attrs.get("openarmature.prompt.name") == "greeting"
339+
assert attrs.get("openarmature.prompt.version") == "v1"
340+
assert attrs.get("openarmature.prompt.label") == "production"
341+
assert attrs.get("openarmature.prompt.template_hash") == "sha256:tpl"
342+
assert attrs.get("openarmature.prompt.rendered_hash") == "sha256:rendered"
343+
assert attrs.get("openarmature.prompt.group_name") == "classifier_chain"
344+
345+
346+
async def test_llm_span_has_no_prompt_attributes_when_no_active_prompt() -> None:
347+
"""Without ``with_active_prompt``, the LLM-call span MUST NOT carry
348+
``openarmature.prompt.*`` attributes."""
349+
from openarmature.graph.events import NodeEvent
350+
from openarmature.llm.providers.openai import _LlmEventState
351+
from openarmature.observability.correlation import (
352+
_reset_invocation_id,
353+
_set_invocation_id,
354+
)
355+
356+
exporter = InMemorySpanExporter()
357+
observer = OTelObserver(span_processor=SimpleSpanProcessor(exporter))
358+
359+
token = _set_invocation_id("inv-2")
360+
try:
361+
started = NodeEvent(
362+
node_name="openarmature.llm.complete",
363+
namespace=("openarmature.llm.complete",),
364+
step=-1,
365+
phase="started",
366+
pre_state=_LlmEventState(call_id="test-call-noprompt", model="test-m"),
367+
post_state=None,
368+
error=None,
369+
parent_states=(),
370+
)
371+
completed = NodeEvent(
372+
node_name="openarmature.llm.complete",
373+
namespace=("openarmature.llm.complete",),
374+
step=-1,
375+
phase="completed",
376+
pre_state=_LlmEventState(call_id="test-call-noprompt", model="test-m", finish_reason="stop"),
377+
post_state=None,
378+
error=None,
379+
parent_states=(),
380+
)
381+
await observer(started)
382+
await observer(completed)
383+
finally:
384+
_reset_invocation_id(token)
385+
observer.shutdown()
386+
387+
llm_spans = [s for s in exporter.get_finished_spans() if s.name == "openarmature.llm.complete"]
388+
assert len(llm_spans) == 1
389+
attrs = llm_spans[0].attributes or {}
390+
assert not any(k.startswith("openarmature.prompt.") for k in attrs)
391+
392+
259393
async def test_disable_llm_spans_skips_llm_provider_span() -> None:
260394
"""Spec §5.5: ``disable_llm_spans=True`` MUST suppress the
261395
LLM-provider span emission while leaving all other spans intact."""

0 commit comments

Comments
 (0)