|
| 1 | +"""Validate each evaluator's fixtures.json. |
| 2 | +
|
| 3 | +Two layers, mirroring eval-config: |
| 4 | + - shared structure: every case is {id, input, expected} per the shared |
| 5 | + evals/_schemas/fixtures.schema.json. |
| 6 | + - per-evaluator binding: each case's `input` must satisfy that evaluator's own |
| 7 | + input_schema.json, and each field in `expected` must satisfy the matching |
| 8 | + property in its output_schema.json. (`expected` is a subset of the full |
| 9 | + output — just the label(s) — so we validate present fields, not requireds.) |
| 10 | +""" |
| 11 | + |
| 12 | +from __future__ import annotations |
| 13 | + |
| 14 | +import json |
| 15 | +import os |
| 16 | + |
| 17 | +from jsonschema import Draft202012Validator |
| 18 | +from jsonschema.exceptions import SchemaError |
| 19 | + |
| 20 | +from .base import Check, Result, Violation, evaluator_configs, load_json, shared_schema_path |
| 21 | + |
| 22 | +_FIXTURES_SCHEMA = "fixtures.schema.json" |
| 23 | + |
| 24 | + |
| 25 | +class EvalFixtures(Check): |
| 26 | + name = "eval-fixtures" |
| 27 | + description = "Validate fixtures.json structure + bind each case's input/expected to the evaluator's schemas" |
| 28 | + |
| 29 | + def run(self, fix: bool) -> Result: |
| 30 | + result = Result(self.name) |
| 31 | + shared = Draft202012Validator(load_json(shared_schema_path(_FIXTURES_SCHEMA))) |
| 32 | + for cfg_path in evaluator_configs(): |
| 33 | + self._check_fixtures(cfg_path, shared, result) |
| 34 | + return result |
| 35 | + |
| 36 | + def _check_fixtures(self, cfg_path: str, shared: Draft202012Validator, result: Result) -> None: |
| 37 | + base = os.path.dirname(cfg_path) |
| 38 | + config = load_json(cfg_path) |
| 39 | + |
| 40 | + def fail(msg: str) -> None: |
| 41 | + result.violations.append(Violation(cfg_path, msg)) |
| 42 | + |
| 43 | + rel = config.get("fixtures", {}).get("path") |
| 44 | + if not rel: |
| 45 | + return |
| 46 | + path = os.path.join(base, rel) |
| 47 | + if not os.path.exists(path): |
| 48 | + return # eval-config reports the missing file |
| 49 | + try: |
| 50 | + fixtures = load_json(path) |
| 51 | + except json.JSONDecodeError as e: |
| 52 | + fail(f"{rel}: invalid JSON: {e}") |
| 53 | + return |
| 54 | + |
| 55 | + # Layer 1: shared structure. |
| 56 | + structurally_ok = True |
| 57 | + for err in sorted(shared.iter_errors(fixtures), key=lambda e: list(e.path)): |
| 58 | + structurally_ok = False |
| 59 | + fail(f"{rel}: {self._loc(err)}: {err.message}") |
| 60 | + if not structurally_ok: |
| 61 | + return # per-case binding below assumes well-formed cases |
| 62 | + |
| 63 | + # Layer 2: bind input/expected to this evaluator's own schemas. |
| 64 | + input_validator = self._validator_for(config, base, "input_schema") |
| 65 | + expected_validator = self._expected_validator(config, base) |
| 66 | + for case in fixtures: |
| 67 | + cid = case.get("id", "?") |
| 68 | + if input_validator: |
| 69 | + for err in input_validator.iter_errors(case.get("input", {})): |
| 70 | + fail(f"{rel}[{cid}].input: {self._loc(err)}: {err.message}") |
| 71 | + if expected_validator: |
| 72 | + for err in expected_validator.iter_errors(case.get("expected", {})): |
| 73 | + fail(f"{rel}[{cid}].expected: {self._loc(err)}: {err.message}") |
| 74 | + |
| 75 | + def _validator_for(self, config: dict, base: str, key: str): |
| 76 | + ref = config.get(key, {}) |
| 77 | + if not (isinstance(ref, dict) and "$ref" in ref): |
| 78 | + return None |
| 79 | + path = os.path.join(base, ref["$ref"]) |
| 80 | + if not os.path.exists(path): |
| 81 | + return None |
| 82 | + return self._safe_validator(load_json(path)) |
| 83 | + |
| 84 | + def _expected_validator(self, config: dict, base: str): |
| 85 | + """A partial validator: each expected field must match its output property. |
| 86 | +
|
| 87 | + Built from output_schema.properties (+ $defs for $ref resolution) with no |
| 88 | + `required`, so a fixture carrying only the label validates, and an unknown |
| 89 | + expected field is rejected. |
| 90 | + """ |
| 91 | + ref = config.get("output_schema", {}) |
| 92 | + if not (isinstance(ref, dict) and "$ref" in ref): |
| 93 | + return None |
| 94 | + path = os.path.join(base, ref["$ref"]) |
| 95 | + if not os.path.exists(path): |
| 96 | + return None |
| 97 | + out = load_json(path) |
| 98 | + schema = { |
| 99 | + "type": "object", |
| 100 | + "additionalProperties": False, |
| 101 | + "properties": out.get("properties", {}), |
| 102 | + } |
| 103 | + if "$defs" in out: |
| 104 | + schema["$defs"] = out["$defs"] |
| 105 | + return self._safe_validator(schema) |
| 106 | + |
| 107 | + @staticmethod |
| 108 | + def _safe_validator(schema: dict): |
| 109 | + """Build a validator only if `schema` is itself a valid JSON Schema. |
| 110 | +
|
| 111 | + A malformed input/output schema is reported by eval-schemas; here we just |
| 112 | + skip binding rather than crash on it. |
| 113 | + """ |
| 114 | + try: |
| 115 | + Draft202012Validator.check_schema(schema) |
| 116 | + except SchemaError: |
| 117 | + return None |
| 118 | + return Draft202012Validator(schema) |
| 119 | + |
| 120 | + @staticmethod |
| 121 | + def _loc(err) -> str: |
| 122 | + return "/".join(str(p) for p in err.path) or "(root)" |
0 commit comments