Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion .claude/worktrees/agent-a663d9be60cbd74bf
Submodule agent-a663d9be60cbd74bf deleted from c7df17
1 change: 0 additions & 1 deletion .claude/worktrees/agent-af9dfbbffaac749ab
Submodule agent-af9dfbbffaac749ab deleted from c7df17
4 changes: 4 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ jobs:
- run: python -m pip install -e ".[dev]"
- run: python -m ruff check .
- run: python -m mypy
- name: Contract compatibility (pinned cas-contracts v0.1)
# Consumer-side gate: fails red if the pinned CAS contract version or the
# vendored v0.1 schema release drifts from what the Pydantic models emit.
run: python -m pytest tests/test_contract_registry.py -q --tb=short -o addopts=""
- run: python -m pytest
- run: python -m cas_reference_product.evidence

Expand Down
3 changes: 3 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,6 @@ build/
.env
.foundry/results/

# Claude Code session worktrees — never commit these
.claude/worktrees/

7 changes: 4 additions & 3 deletions tests/test_api.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from typing import Any
from unittest.mock import patch

from fastapi.testclient import TestClient
Expand All @@ -8,11 +9,11 @@


class FailingExternalService:
def run(self, envelope) -> str:
def run(self: "Any", envelope: "Any") -> str:
raise WorkflowAgentServiceError("sensitive provider detail")


def test_workflow_api_emits_canonical_events(envelope) -> None:
def test_workflow_api_emits_canonical_events(envelope: "Any") -> None:
client = TestClient(create_app(Settings()))

response = client.post("/api/v1/workflows", json=envelope.model_dump(mode="json"))
Expand Down Expand Up @@ -56,7 +57,7 @@ def test_invalid_foundry_endpoint_is_not_ready() -> None:
assert client.get("/health/ready").status_code == 503


def test_workflow_api_sanitizes_external_service_failures(envelope) -> None:
def test_workflow_api_sanitizes_external_service_failures(envelope: "Any") -> None:
with patch(
"cas_reference_product.app.build_workflow_agent_service",
return_value=FailingExternalService(),
Expand Down
57 changes: 52 additions & 5 deletions tests/test_contract_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,32 @@
from pathlib import Path
from typing import Any

import pytest
from jsonschema import Draft202012Validator
from referencing import Registry, Resource

from cas_reference_product.models import LifecycleMetadata
from cas_reference_product.workflow import LocalWorkflowAgentService, WorkflowOrchestrator

CONTRACT_ROOT = Path(__file__).parent / "contracts" / "cas-contracts" / "v0.1.0"
# The cas-contracts schema version this consumer pins to. Its Pydantic models
# (cas_reference_product.models.LifecycleMetadata.schemaVersion) emit this exact
# value, so the pin below and the models must stay in lockstep.
PINNED_SCHEMA_VERSION = "0.1.0"

CONTRACT_ROOT = Path(__file__).parent / "contracts" / "cas-contracts" / f"v{PINNED_SCHEMA_VERSION}"
# Sibling source-of-truth checkout (local polyrepo). Absent in isolated CI.
UPSTREAM_ROOT = (
Path(__file__).resolve().parents[2]
/ "cas-contracts"
/ "registry"
/ "releases"
/ f"v{PINNED_SCHEMA_VERSION}"
)


def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
from typing import cast
return cast(dict[str, Any], json.loads(path.read_text(encoding="utf-8")))


def contract_registry() -> Registry[Any]:
Expand All @@ -31,17 +47,48 @@ def assert_valid(schema_name: str, instance: dict[str, Any]) -> None:
def test_vendored_contract_release_matches_manifest_hashes() -> None:
manifest = load_json(CONTRACT_ROOT / "manifest.json")

assert manifest["version"] == "0.1.0"
assert manifest["version"] == PINNED_SCHEMA_VERSION
for entry in manifest["schemas"]:
content = (CONTRACT_ROOT / entry["path"]).read_bytes()
assert hashlib.sha256(content).hexdigest() == entry["sha256"]


def test_prompt_envelope_serialization_conforms_to_v010_registry(envelope) -> None:
def test_pinned_version_is_the_version_models_emit() -> None:
"""The version this consumer pins must equal both the schemaVersion const the
vendored schema enforces and the schemaVersion the Pydantic models emit.

Fails red if the vendored release is bumped without updating models.LifecycleMetadata.
"""
common = load_json(CONTRACT_ROOT / "common.schema.json")
schema_const = common["$defs"]["lifecycleMetadata"]["properties"]["schemaVersion"]["const"]
assert schema_const == PINNED_SCHEMA_VERSION

model_default = LifecycleMetadata.model_fields["schemaVersion"].default
assert model_default == PINNED_SCHEMA_VERSION


@pytest.mark.skipif(
not UPSTREAM_ROOT.exists(),
reason="sibling cas-contracts checkout not present (expected in isolated CI)",
)
def test_vendored_release_matches_upstream_source_of_truth() -> None:
"""Local-only drift guard: vendored copy must equal the sibling cas-contracts
release byte-for-byte. Skipped in isolated CI where the sibling is not checked out.
"""
for path in CONTRACT_ROOT.glob("*.json"):
upstream = UPSTREAM_ROOT / path.name
assert upstream.exists(), f"upstream missing {path.name}"
assert (
hashlib.sha256(path.read_bytes()).hexdigest()
== hashlib.sha256(upstream.read_bytes()).hexdigest()
), f"vendored {path.name} drifted from upstream cas-contracts {PINNED_SCHEMA_VERSION}"


def test_prompt_envelope_serialization_conforms_to_v010_registry(envelope: "Any") -> None:
assert_valid("prompt-envelope.schema.json", envelope.model_dump(mode="json"))


def test_run_event_serialization_conforms_to_v010_registry(envelope) -> None:
def test_run_event_serialization_conforms_to_v010_registry(envelope: "Any") -> None:
result = WorkflowOrchestrator(LocalWorkflowAgentService(), envelope.repo).execute(envelope)

for event in result.events:
Expand Down
3 changes: 2 additions & 1 deletion tests/test_function_boundary.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
import json
from typing import Any

import pytest

from cas_reference_product.ingress import InvalidIngressRequest, create_worker_message


def test_ingress_validates_and_serializes_canonical_envelope(envelope) -> None:
def test_ingress_validates_and_serializes_canonical_envelope(envelope: "Any") -> None:
message = create_worker_message(envelope.model_dump_json().encode())

assert json.loads(message)["runId"] == envelope.runId
Expand Down
12 changes: 8 additions & 4 deletions tests/test_models.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,20 @@
from typing import Any

import pytest
from pydantic import ValidationError

from cas_reference_product.models import Actor, PromptEnvelope, RunEvent, TraceContext


def test_prompt_envelope_rejects_extra_properties(envelope) -> None:
def test_prompt_envelope_rejects_extra_properties(envelope: "Any") -> None:
payload = envelope.model_dump()
payload["secret"] = "not-allowed"

with pytest.raises(ValidationError):
PromptEnvelope.model_validate(payload)


def test_prompt_envelope_matches_cas_contract_metadata(envelope) -> None:
def test_prompt_envelope_matches_cas_contract_metadata(envelope: "Any") -> None:
payload = envelope.model_dump(mode="json")

assert payload["kind"] == "PromptEnvelope"
Expand All @@ -28,7 +30,9 @@ def test_prompt_envelope_matches_cas_contract_metadata(envelope) -> None:
["No secrets", "No secrets"],
],
)
def test_prompt_envelope_enforces_cas_contract_constraints(envelope, constraints) -> None:
def test_prompt_envelope_enforces_cas_contract_constraints(
envelope: "Any", constraints: "Any"
) -> None:
payload = envelope.model_dump()
payload["constraints"] = constraints

Expand Down Expand Up @@ -67,6 +71,6 @@ def test_prompt_envelope_enforces_cas_contract_constraints(envelope, constraints
),
],
)
def test_contract_models_reject_explicit_null_optional_fields(model, payload) -> None:
def test_contract_models_reject_explicit_null_optional_fields(model: "Any", payload: "Any") -> None:
with pytest.raises(ValidationError):
model.model_validate(payload)
5 changes: 3 additions & 2 deletions tests/test_service_factory.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
from typing import Any
from unittest.mock import patch

import pytest
Expand Down Expand Up @@ -39,7 +40,7 @@ def test_foundry_service_rejects_invalid_project_endpoint() -> None:
FoundryWorkflowAgentService(settings)


def test_foundry_service_uses_next_gen_agent_reference(envelope) -> None:
def test_foundry_service_uses_next_gen_agent_reference(envelope: "Any") -> None:
settings = Settings(
environment="prod",
workflow_backend="foundry",
Expand Down Expand Up @@ -71,7 +72,7 @@ def test_foundry_service_uses_next_gen_agent_reference(envelope) -> None:
assert result == "Foundry result"


def test_foundry_service_sanitizes_sdk_failure(envelope) -> None:
def test_foundry_service_sanitizes_sdk_failure(envelope: "Any") -> None:
settings = Settings(
environment="prod",
workflow_backend="foundry",
Expand Down
26 changes: 17 additions & 9 deletions tests/test_telemetry.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Tests for Phase 2 — Telemetry Hardening (TEL-01 through TEL-04)."""

from typing import Any
from unittest.mock import MagicMock, patch

from fastapi.testclient import TestClient
Expand Down Expand Up @@ -110,12 +111,13 @@ def test_all_loop_stages_share_one_trace_without_prompt_or_output_attributes() -
pass

spans = exporter.get_finished_spans()
assert {span.attributes["cas.stage"] for span in spans} == {stage.value for stage in LoopStage}
stage_values = {stage.value for stage in LoopStage}
assert {span.attributes["cas.stage"] for span in spans if span.attributes} == stage_values
assert len({span.context.trace_id for span in spans}) == 1
assert all(
"prompt" not in key and "output" not in key
for span in spans
for key in span.attributes
for key in (span.attributes or {})
)


Expand All @@ -140,7 +142,9 @@ def test_install_propagator_sets_w3c_propagator() -> None:
# ---------------------------------------------------------------------------


def test_workflow_endpoint_creates_span(in_memory_exporter: InMemorySpanExporter, envelope) -> None:
def test_workflow_endpoint_creates_span(
in_memory_exporter: InMemorySpanExporter, envelope: "Any"
) -> None:
client = TestClient(create_app(Settings()))
response = client.post("/api/v1/workflows", json=envelope.model_dump(mode="json"))

Expand All @@ -157,12 +161,15 @@ def test_workflow_endpoint_creates_span(in_memory_exporter: InMemorySpanExporter
# ---------------------------------------------------------------------------


def test_workflow_span_attributes(in_memory_exporter: InMemorySpanExporter, envelope) -> None:
def test_workflow_span_attributes(
in_memory_exporter: InMemorySpanExporter, envelope: "Any"
) -> None:
client = TestClient(create_app(Settings()))
client.post("/api/v1/workflows", json=envelope.model_dump(mode="json"))

spans = in_memory_exporter.get_finished_spans()
api_span = next(s for s in spans if s.name == "cas.api.workflows.execute")
assert api_span.attributes
assert api_span.attributes.get("cas.correlation_id") == envelope.correlationId
assert api_span.attributes.get("cas.run_id") == envelope.runId
assert api_span.attributes.get("cas.intent") == envelope.intent
Expand All @@ -174,7 +181,7 @@ def test_workflow_span_attributes(in_memory_exporter: InMemorySpanExporter, enve


def test_workflow_span_events_started_and_completed(
in_memory_exporter: InMemorySpanExporter, envelope
in_memory_exporter: InMemorySpanExporter, envelope: "Any"
) -> None:
client = TestClient(create_app(Settings()))
client.post("/api/v1/workflows", json=envelope.model_dump(mode="json"))
Expand All @@ -188,12 +195,12 @@ def test_workflow_span_events_started_and_completed(


def test_workflow_span_events_started_and_failed(
in_memory_exporter: InMemorySpanExporter, envelope
in_memory_exporter: InMemorySpanExporter, envelope: "Any"
) -> None:
from cas_reference_product.workflow import WorkflowAgentServiceError

class FailingService:
def run(self, _env) -> str:
def run(self: "Any", _env: "Any") -> str:
raise WorkflowAgentServiceError("backend down")

with patch(
Expand All @@ -213,14 +220,15 @@ def run(self, _env) -> str:


def test_span_event_carries_correlation_id(
in_memory_exporter: InMemorySpanExporter, envelope
in_memory_exporter: InMemorySpanExporter, envelope: "Any"
) -> None:
client = TestClient(create_app(Settings()))
client.post("/api/v1/workflows", json=envelope.model_dump(mode="json"))

spans = in_memory_exporter.get_finished_spans()
api_span = next(s for s in spans if s.name == "cas.api.workflows.execute")
started_event = next(e for e in api_span.events if e.name == "workflow.started")
assert started_event.attributes
assert started_event.attributes.get("cas.correlation_id") == envelope.correlationId
assert started_event.attributes.get("cas.run_id") == envelope.runId

Expand All @@ -231,7 +239,7 @@ def test_span_event_carries_correlation_id(


def test_w3c_traceparent_propagated_inbound(
in_memory_exporter: InMemorySpanExporter, envelope
in_memory_exporter: InMemorySpanExporter, envelope: "Any"
) -> None:
"""Request with a W3C traceparent header links the API span as a child."""
incoming_traceparent = "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
Expand Down
9 changes: 5 additions & 4 deletions tests/test_workflow.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from datetime import UTC, datetime
from typing import Any
from unittest.mock import patch

import pytest
Expand All @@ -7,16 +8,16 @@


class SuccessfulService:
def run(self, envelope) -> str:
def run(self: "Any", envelope: "Any") -> str:
return f"processed:{envelope.promptId}"


class FailingService:
def run(self, envelope) -> str:
def run(self: "Any", envelope: "Any") -> str:
raise RuntimeError("expected")


def test_orchestrator_returns_traceable_events(envelope) -> None:
def test_orchestrator_returns_traceable_events(envelope: "Any") -> None:
fixed = datetime(2026, 6, 11, 10, 0, tzinfo=UTC)
# Patch current_traceparent so this unit test is provider-independent.
with patch(
Expand All @@ -36,6 +37,6 @@ def test_orchestrator_returns_traceable_events(envelope) -> None:
assert all(event.traceContext == envelope.traceContext for event in result.events)


def test_orchestrator_propagates_failure(envelope) -> None:
def test_orchestrator_propagates_failure(envelope: "Any") -> None:
with pytest.raises(RuntimeError, match="expected"):
WorkflowOrchestrator(FailingService(), envelope.repo).execute(envelope)
Loading