|
| 1 | +"""Opt-in deterministic adapter for the local CAS reference product.""" |
| 2 | + |
| 3 | +from __future__ import annotations |
| 4 | + |
| 5 | +import hashlib |
| 6 | +import json |
| 7 | +from collections.abc import Callable |
| 8 | +from pathlib import Path |
| 9 | +from typing import Any |
| 10 | +from urllib.error import HTTPError, URLError |
| 11 | +from urllib.parse import urlsplit |
| 12 | +from urllib.request import Request, urlopen |
| 13 | + |
| 14 | +from .contracts import CONTRACT_VERSION |
| 15 | +from .evaluator import DEFAULT_RELEASED_AT, _evaluate_case_with_evidence, lifecycle_metadata |
| 16 | + |
| 17 | +DEFAULT_REFERENCE_PRODUCT_URL = "http://127.0.0.1:8080/api/v1/workflows" |
| 18 | +REFERENCE_PRODUCT_TARGET = "cas-reference-product/api/v1/workflows" |
| 19 | +MAX_RESPONSE_BYTES = 2_000_000 |
| 20 | +Transport = Callable[[dict[str, Any]], dict[str, Any]] |
| 21 | + |
| 22 | + |
| 23 | +class ReferenceProductError(RuntimeError): |
| 24 | + """Raised when the reference-product contract is unavailable or invalid.""" |
| 25 | + |
| 26 | + |
| 27 | +def _digest_text(value: str) -> str: |
| 28 | + return f"sha256:{hashlib.sha256(value.encode('utf-8')).hexdigest()}" |
| 29 | + |
| 30 | + |
| 31 | +def _http_transport(endpoint: str, timeout_seconds: float) -> Transport: |
| 32 | + parsed = urlsplit(endpoint) |
| 33 | + if parsed.scheme not in {"http", "https"} or not parsed.netloc: |
| 34 | + raise ReferenceProductError("reference product endpoint must be an HTTP(S) URL") |
| 35 | + if timeout_seconds <= 0: |
| 36 | + raise ReferenceProductError("reference product timeout must be greater than zero") |
| 37 | + |
| 38 | + def post(envelope: dict[str, Any]) -> dict[str, Any]: |
| 39 | + request = Request( |
| 40 | + endpoint, |
| 41 | + data=json.dumps(envelope, sort_keys=True, separators=(",", ":")).encode("utf-8"), |
| 42 | + headers={"Content-Type": "application/json"}, |
| 43 | + method="POST", |
| 44 | + ) |
| 45 | + try: |
| 46 | + with urlopen(request, timeout=timeout_seconds) as response: |
| 47 | + payload = response.read(MAX_RESPONSE_BYTES + 1) |
| 48 | + except HTTPError as error: |
| 49 | + raise ReferenceProductError(f"reference product returned HTTP {error.code}") from None |
| 50 | + except (URLError, TimeoutError, OSError): |
| 51 | + raise ReferenceProductError("reference product is unavailable") from None |
| 52 | + if len(payload) > MAX_RESPONSE_BYTES: |
| 53 | + raise ReferenceProductError("reference product response exceeds the size limit") |
| 54 | + try: |
| 55 | + value = json.loads(payload.decode("utf-8")) |
| 56 | + except (UnicodeDecodeError, json.JSONDecodeError): |
| 57 | + raise ReferenceProductError("reference product returned invalid JSON") from None |
| 58 | + if not isinstance(value, dict): |
| 59 | + raise ReferenceProductError("reference product response must be an object") |
| 60 | + return value |
| 61 | + |
| 62 | + return post |
| 63 | + |
| 64 | + |
| 65 | +def _build_envelope(case: dict[str, Any], suite_id: str, released_at: str) -> dict[str, Any]: |
| 66 | + metadata = lifecycle_metadata(case["id"], suite_id, released_at) |
| 67 | + return { |
| 68 | + "kind": "PromptEnvelope", |
| 69 | + **metadata, |
| 70 | + "repo": "Coding-Autopilot-System/cas-evals", |
| 71 | + "actor": {"id": "cas-evals", "type": "service"}, |
| 72 | + "schemaVersion": CONTRACT_VERSION, |
| 73 | + "intent": case.get("capability", case["kind"]), |
| 74 | + "prompt": case["prompt"], |
| 75 | + "constraints": case.get("constraints", []), |
| 76 | + } |
| 77 | + |
| 78 | + |
| 79 | +def _validate_response(response: dict[str, Any], envelope: dict[str, Any]) -> tuple[str, list[dict[str, Any]]]: |
| 80 | + output = response.get("output") |
| 81 | + events = response.get("events") |
| 82 | + if ( |
| 83 | + response.get("runId") != envelope["runId"] |
| 84 | + or not isinstance(output, str) |
| 85 | + or not output |
| 86 | + or not isinstance(events, list) |
| 87 | + ): |
| 88 | + raise ReferenceProductError("reference product response contract is invalid") |
| 89 | + if not events: |
| 90 | + raise ReferenceProductError("reference product response contains no lifecycle events") |
| 91 | + |
| 92 | + expected = { |
| 93 | + "correlationId": envelope["correlationId"], |
| 94 | + "promptId": envelope["promptId"], |
| 95 | + "runId": envelope["runId"], |
| 96 | + "traceContext": envelope["traceContext"], |
| 97 | + } |
| 98 | + normalized_events = [] |
| 99 | + for event in events: |
| 100 | + if not isinstance(event, dict) or any(event.get(field) != value for field, value in expected.items()): |
| 101 | + raise ReferenceProductError("reference product did not preserve lifecycle metadata") |
| 102 | + normalized_events.append( |
| 103 | + { |
| 104 | + **expected, |
| 105 | + "eventType": event.get("eventType"), |
| 106 | + "sequence": event.get("sequence"), |
| 107 | + "status": event.get("status"), |
| 108 | + } |
| 109 | + ) |
| 110 | + return output, normalized_events |
| 111 | + |
| 112 | + |
| 113 | +def evaluate_reference_suite( |
| 114 | + path: str | Path, |
| 115 | + *, |
| 116 | + endpoint: str = DEFAULT_REFERENCE_PRODUCT_URL, |
| 117 | + timeout_seconds: float = 5.0, |
| 118 | + transport: Transport | None = None, |
| 119 | +) -> dict[str, Any]: |
| 120 | + """Evaluate a fixture suite against the local reference-product workflow endpoint.""" |
| 121 | + fixture_path = Path(path) |
| 122 | + suite = json.loads(fixture_path.read_text(encoding="utf-8")) |
| 123 | + released_at = suite.get("releasedAt", DEFAULT_RELEASED_AT) |
| 124 | + invoke = transport or _http_transport(endpoint, timeout_seconds) |
| 125 | + evaluated = [] |
| 126 | + |
| 127 | + for source_case in suite["cases"]: |
| 128 | + envelope = _build_envelope(source_case, suite["suiteId"], released_at) |
| 129 | + output, events = _validate_response(invoke(envelope), envelope) |
| 130 | + live_case = {**source_case, "response": output} |
| 131 | + evidence = { |
| 132 | + "adapter": "cas-reference-product", |
| 133 | + "target": REFERENCE_PRODUCT_TARGET, |
| 134 | + "lifecycle": { |
| 135 | + field: envelope[field] |
| 136 | + for field in ("correlationId", "promptId", "runId", "traceContext") |
| 137 | + }, |
| 138 | + "responseDigest": _digest_text(output), |
| 139 | + "events": events, |
| 140 | + "timing": { |
| 141 | + "latencyMs": float(source_case.get("observed", {}).get("latency_ms", 0.0)), |
| 142 | + "normalization": "fixture-observed", |
| 143 | + }, |
| 144 | + } |
| 145 | + evaluated.append( |
| 146 | + _evaluate_case_with_evidence( |
| 147 | + live_case, |
| 148 | + suite["suiteId"], |
| 149 | + released_at, |
| 150 | + source_case=source_case, |
| 151 | + metadata=envelope, |
| 152 | + execution_evidence=evidence, |
| 153 | + ) |
| 154 | + ) |
| 155 | + |
| 156 | + results = [result for result, _ in evaluated] |
| 157 | + return { |
| 158 | + "schemaVersion": "0.2.0", |
| 159 | + "suiteId": suite["suiteId"], |
| 160 | + "results": results, |
| 161 | + "evidence": [evidence for _, evidence in evaluated], |
| 162 | + "summary": { |
| 163 | + "total": len(results), |
| 164 | + "passed": sum(result["outcome"] == "passed" for result in results), |
| 165 | + "failed": sum(result["outcome"] != "passed" for result in results), |
| 166 | + }, |
| 167 | + } |
0 commit comments