diff --git a/baselines/lcore_regression/current_baseline_summary.json b/baselines/lcore_regression/current_baseline_summary.json new file mode 100644 index 00000000..ae777acf --- /dev/null +++ b/baselines/lcore_regression/current_baseline_summary.json @@ -0,0 +1,114 @@ +{ + "timestamp": "2026-06-30T21:09:26.159970+00:00", + "total_evaluations": 432, + "summary_stats": { + "overall": { + "TOTAL": 432, + "PASS": 160, + "FAIL": 188, + "ERROR": 84, + "SKIPPED": 0, + "pass_rate": 37.03703703703704, + "fail_rate": 43.51851851851852, + "error_rate": 19.444444444444446 + }, + "by_metric": { + "ragas:context_precision_with_reference": { + "pass": 36, + "fail": 51, + "error": 21, + "skipped": 0, + "pass_rate": 33.33333333333333, + "fail_rate": 47.22222222222222, + "error_rate": 19.444444444444446, + "skipped_rate": 0.0, + "score_statistics": { + "count": 87, + "mean": 0.4434865900147031, + "median": 0.0, + "std": 0.47947104466841306, + "min": 0.0, + "max": 0.9999999999666667, + "confidence_interval": { + "low": 0.3438697317798228, + "mean": 0.4425287356098659, + "high": 0.5421455938391834, + "confidence_level": 95.0 + } + } + }, + "ragas:context_precision_without_reference": { + "pass": 50, + "fail": 37, + "error": 21, + "skipped": 0, + "pass_rate": 46.2962962962963, + "fail_rate": 34.25925925925926, + "error_rate": 19.444444444444446, + "skipped_rate": 0.0, + "score_statistics": { + "count": 87, + "mean": 0.6091954022673371, + "median": 0.9999999999, + "std": 0.46281465738414, + "min": 0.0, + "max": 0.9999999999666667, + "confidence_interval": { + "low": 0.5143678160670976, + "mean": 0.6101532566731322, + "high": 0.7059386972799353, + "confidence_level": 95.0 + } + } + }, + "ragas:context_recall": { + "pass": 26, + "fail": 61, + "error": 21, + "skipped": 0, + "pass_rate": 24.074074074074073, + "fail_rate": 56.481481481481474, + "error_rate": 19.444444444444446, + "skipped_rate": 0.0, + "score_statistics": { + "count": 87, + "mean": 0.4013409961685824, + "median": 0.0, + "std": 0.4450649459822112, + "min": 0.0, + "max": 1.0, + "confidence_interval": { + "low": 0.3065134099616859, + "mean": 0.4013409961685824, + "high": 0.4932950191570882, + "confidence_level": 95.0 + } + } + }, + "ragas:context_relevance": { + "pass": 48, + "fail": 39, + "error": 21, + "skipped": 0, + "pass_rate": 44.44444444444444, + "fail_rate": 36.11111111111111, + "error_rate": 19.444444444444446, + "skipped_rate": 0.0, + "score_statistics": { + "count": 87, + "mean": 0.6522988505747126, + "median": 1.0, + "std": 0.4059089746437468, + "min": 0.0, + "max": 1.0, + "confidence_interval": { + "low": 0.5689655172413792, + "mean": 0.6522988505747126, + "high": 0.7385057471264367, + "confidence_level": 95.0 + } + } + } + } + } +} diff --git a/config/lcore_regression/system-config-pr-gate.yaml b/config/lcore_regression/system-config-pr-gate.yaml new file mode 100644 index 00000000..39fdbaf0 --- /dev/null +++ b/config/lcore_regression/system-config-pr-gate.yaml @@ -0,0 +1,109 @@ +# System configuration for LCORE regression PR gate evaluation. +# +# This config tells the evaluation framework: +# - How to call the lightspeed-stack API (api section) +# - Which LLM judge to use for scoring (llm_pool + judge_panel) +# - Which embedding model to use for semantic metrics (embedding) +# - Default thresholds for each metric (metrics_metadata) +# - Where to write results (storage) +# +# The stack under test runs at localhost:8080 (started by docker-compose). +# The judge LLM is a separate OpenAI call (gpt-4o-mini) used only for scoring. + +core: + max_threads: 4 + fail_on_invalid_data: true + skip_on_failure: false + +llm_pool: + defaults: + cache_enabled: false + timeout: 120 + num_retries: 3 + parameters: + temperature: 0.0 + max_completion_tokens: 4096 + models: + judge_gpt_4o_mini: + provider: openai + model: gpt-4o-mini + +embedding: + provider: openai + model: text-embedding-3-small + cache_enabled: false + +api: + enabled: true + api_base: http://localhost:8080 + version: v1 + endpoint_type: query + timeout: 300 + provider: openai + model: gpt-4o-mini + cache_enabled: false + +metrics_metadata: + turn_level: + ragas:context_recall: + threshold: 0.8 + description: Did we fetch every fact the answer needs? + default: false + + ragas:context_precision_with_reference: + threshold: 0.7 + description: How precise the retrieved context is (with reference) + default: false + + ragas:context_precision_without_reference: + threshold: 0.7 + description: How precise the retrieved context is (without reference) + default: false + + ragas:context_relevance: + threshold: 0.7 + description: Is what we retrieved actually relevant to user query? + default: false + +storage: + - type: file + output_dir: ./eval_output/lcore_regression + base_filename: evaluation + enabled_outputs: + - csv + - json + - txt + csv_columns: + - conversation_group_id + - turn_id + - metric_identifier + - result + - score + - threshold + - reason + - query + - response + - contexts + - expected_response + - expected_keywords + - api_input_tokens + - api_output_tokens + +visualization: + figsize: + - 14 + - 10 + dpi: 150 + enabled_graphs: + - pass_rates + - score_distribution + - status_breakdown + +environment: + LITELLM_LOG: ERROR + +logging: + source_level: INFO + package_level: WARNING + log_format: "%(asctime)s - %(levelname)s - %(message)s" + show_timestamps: true diff --git a/script/regression/__init__.py b/script/regression/__init__.py new file mode 100644 index 00000000..103e2952 --- /dev/null +++ b/script/regression/__init__.py @@ -0,0 +1 @@ +"""Regression comparison scripts for evaluation gating.""" diff --git a/script/regression/compare_against_baseline.py b/script/regression/compare_against_baseline.py new file mode 100644 index 00000000..f929ca93 --- /dev/null +++ b/script/regression/compare_against_baseline.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +"""Baseline comparison script for OKP quality regression gating. + +Compares a current evaluation run against a baseline run and determines +whether regressions have occurred. Two modes: + + --check-only --> Print "regression" or "ok" (for OKP image comparison) + --fail-on-critical-regression --> Exit non-zero if critical metrics regressed (for PR gating) +""" + +import argparse +import json +import sys +from pathlib import Path +from typing import Any + +CRITICAL_METRICS = { + "ragas:context_recall", + "ragas:context_precision_with_reference", + "ragas:context_precision_without_reference", + "ragas:context_relevance", +} + + +def parse_args() -> argparse.Namespace: + """Parse command-line arguments.""" + parser = argparse.ArgumentParser( + description="Compare evaluation results against a baseline for regression detection.", + ) + parser.add_argument( + "--current", + required=True, + help="Path to the current run's output directory.", + ) + parser.add_argument( + "--baseline", + required=True, + help="Path to the baseline run's output directory.", + ) + parser.add_argument( + "--output", + help="Path to write the markdown regression summary.", + ) + parser.add_argument( + "--check-only", + action="store_true", + help='Print "regression" or "ok" to stdout. For OKP image comparison.', + ) + parser.add_argument( + "--fail-on-critical-regression", + action="store_true", + help="Exit non-zero if any critical metric regressed. For PR gating.", + ) + parser.add_argument( + "--critical-delta", + type=float, + default=0.03, + help="Allowed mean score drop for critical metrics (default: 0.03).", + ) + parser.add_argument( + "--warn-delta", + type=float, + default=0.03, + help="Score drop threshold for non-critical warnings (default: 0.03).", + ) + return parser.parse_args() + + +def find_and_load_summary(directory: str) -> dict[str, Any]: + """Find and load the summary JSON file from an evaluation output directory. + + Args: + directory: Path to an evaluation output directory. + + Returns: + Parsed JSON content of the summary file. + + Raises: + FileNotFoundError: If no summary JSON is found. + RuntimeError: If multiple summary JSONs are found. + """ + dir_path = Path(directory) + + if not dir_path.is_dir(): + raise FileNotFoundError(f"Directory not found: {directory}") + + summary_files = list(dir_path.glob("*_summary.json")) + + if len(summary_files) == 0: + raise FileNotFoundError(f"No *_summary.json file found in: {directory}") + + if len(summary_files) > 1: + raise RuntimeError( + f"Multiple summary files found in {directory}: " + f"{[f.name for f in summary_files]}. Expected exactly one." + ) + + with open(summary_files[0], "r", encoding="utf-8") as f: + return json.load(f) + + +def compute_metric_deltas( + baseline_data: dict[str, Any], + current_data: dict[str, Any], + critical_delta: float, + warn_delta: float, +) -> list[dict[str, Any]]: + """Compute per-metric deltas between baseline and current runs. + + Args: + baseline_data: Parsed baseline summary JSON. + current_data: Parsed current run summary JSON. + critical_delta: Allowed score drop for critical metrics. + warn_delta: Score drop threshold for non-critical warnings. + + Returns: + List of dicts, one per metric, with delta and status. + """ + baseline_metrics = baseline_data.get("summary_stats", {}).get("by_metric", {}) + current_metrics = current_data.get("summary_stats", {}).get("by_metric", {}) + + all_metrics = set(baseline_metrics.keys()) | set(current_metrics.keys()) + results = [] + + for metric in sorted(all_metrics): + baseline_stats = baseline_metrics.get(metric, {}) + current_stats = current_metrics.get(metric, {}) + + baseline_mean = baseline_stats.get("score_statistics", {}).get("mean") + current_mean = current_stats.get("score_statistics", {}).get("mean") + + baseline_pass_rate = baseline_stats.get("pass_rate") + current_pass_rate = current_stats.get("pass_rate") + + if baseline_mean is not None and current_mean is not None: + score_delta = current_mean - baseline_mean + else: + score_delta = None + + if baseline_pass_rate is not None and current_pass_rate is not None: + pass_rate_delta = current_pass_rate - baseline_pass_rate + else: + pass_rate_delta = None + + is_critical = metric in CRITICAL_METRICS + threshold = critical_delta if is_critical else warn_delta + + if baseline_mean is not None and current_mean is None: + status = "FAIL" if is_critical else "WARN" + elif score_delta is not None and score_delta < -threshold: + status = "FAIL" if is_critical else "WARN" + else: + status = "PASS" + + results.append( + { + "metric": metric, + "baseline_mean": baseline_mean, + "current_mean": current_mean, + "score_delta": score_delta, + "baseline_pass_rate": baseline_pass_rate, + "current_pass_rate": current_pass_rate, + "pass_rate_delta": pass_rate_delta, + "is_critical": is_critical, + "status": status, + } + ) + + return results + + +def generate_markdown_summary( + deltas: list[dict[str, Any]], + baseline_data: dict[str, Any], + current_data: dict[str, Any], +) -> str: + """Generate a markdown regression summary table.""" + lines = [ + "# Regression Comparison Summary", + "", + f"- **Baseline:** {baseline_data['total_evaluations']} evaluations " + f"({baseline_data['timestamp']})", + f"- **Current:** {current_data['total_evaluations']} evaluations " + f"({current_data['timestamp']})", + "", + "| Metric | Baseline | Current | Delta | Status |", + "|---|---:|---:|---:|---|", + ] + + for d in deltas: + bm = f"{d['baseline_mean']:.3f}" if d["baseline_mean"] is not None else "N/A" + cm = f"{d['current_mean']:.3f}" if d["current_mean"] is not None else "N/A" + sd = f"{d['score_delta']:+.3f}" if d["score_delta"] is not None else "N/A" + status = d["status"] + if status == "FAIL": + status_display = "**FAIL**" + elif status == "WARN": + status_display = "WARN" + else: + status_display = "PASS" + lines.append(f"| {d['metric']} | {bm} | {cm} | {sd} | {status_display} |") + + has_critical_fail = any(d["status"] == "FAIL" and d["is_critical"] for d in deltas) + has_warn = any(d["status"] == "WARN" for d in deltas) + + lines.append("") + if has_critical_fail: + lines.append("**Result: REGRESSION** (critical metrics degraded)") + elif has_warn: + lines.append("**Result: WARNING** (non-critical metrics degraded)") + else: + lines.append("**Result: PASS** (no regressions detected)") + + return "\n".join(lines) + "\n" + + +def main() -> int: + """Entry point.""" + args = parse_args() + + try: + baseline_data = find_and_load_summary(args.baseline) + current_data = find_and_load_summary(args.current) + except (FileNotFoundError, RuntimeError) as err: + print(f"Error: {err}", file=sys.stderr) + return 1 + + deltas = compute_metric_deltas( + baseline_data, + current_data, + args.critical_delta, + args.warn_delta, + ) + + has_critical_fail = any(d["status"] == "FAIL" and d["is_critical"] for d in deltas) + + if args.check_only: + print("regression" if has_critical_fail else "ok") + return 0 + + print( + f"Baseline: {baseline_data['total_evaluations']} evaluations " + f"({baseline_data['timestamp']})" + ) + print( + f"Current: {current_data['total_evaluations']} evaluations " + f"({current_data['timestamp']})" + ) + + print( + f"\n{'Metric':<50} {'Baseline':>10} {'Current':>10} " + f"{'Delta':>10} {'Status':>8}" + ) + print("-" * 92) + for d in deltas: + bm = f"{d['baseline_mean']:.3f}" if d["baseline_mean"] is not None else "N/A" + cm = f"{d['current_mean']:.3f}" if d["current_mean"] is not None else "N/A" + sd = f"{d['score_delta']:+.3f}" if d["score_delta"] is not None else "N/A" + print(f"{d['metric']:<50} {bm:>10} {cm:>10} {sd:>10} {d['status']:>8}") + + if args.output: + markdown = generate_markdown_summary(deltas, baseline_data, current_data) + output_path = Path(args.output) + output_path.write_text(markdown, encoding="utf-8") + print(f"\nMarkdown summary written to: {output_path}") + + if args.fail_on_critical_regression and has_critical_fail: + print("\nCritical regression detected — exiting with code 1.") + return 1 + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/script/conftest.py b/tests/script/conftest.py index ad002dee..8bade6a0 100644 --- a/tests/script/conftest.py +++ b/tests/script/conftest.py @@ -2,6 +2,7 @@ """Pytest configuration and fixtures for script tests.""" +import json from pathlib import Path from typing import Any @@ -156,7 +157,7 @@ def sample_evaluation_summary() -> dict[str, Any]: "error_rate": 0.0, }, "by_metric": { - "ragas:faithfulness": { + "ragas:context_recall": { "pass": 4, "fail": 0, "error": 0, @@ -172,7 +173,7 @@ def sample_evaluation_summary() -> dict[str, Any]: "count": 4, }, }, - "ragas:response_relevancy": { + "ragas:context_relevance": { "pass": 4, "fail": 2, "error": 0, @@ -194,7 +195,7 @@ def sample_evaluation_summary() -> dict[str, Any]: { "conversation_group_id": "conv1", "turn_id": "turn1", - "metric_identifier": "ragas:faithfulness", + "metric_identifier": "ragas:context_recall", "result": "PASS", "score": 0.95, "threshold": 0.8, @@ -203,7 +204,7 @@ def sample_evaluation_summary() -> dict[str, Any]: { "conversation_group_id": "conv1", "turn_id": "turn2", - "metric_identifier": "ragas:response_relevancy", + "metric_identifier": "ragas:context_relevance", "result": "PASS", "score": 0.85, "threshold": 0.7, @@ -212,3 +213,45 @@ def sample_evaluation_summary() -> dict[str, Any]: ] * 5, # Repeat to get 10 results } + + +def make_summary( + metrics: dict[str, tuple[float, float]], + timestamp: str = "2026-01-01T00:00:00", +) -> dict[str, Any]: + """Build a minimal summary dict matching the real evaluation output format. + + Args: + metrics: Mapping of metric name to (mean_score, pass_rate). + timestamp: Timestamp string for the summary. + + Returns: + A dict shaped like a real *_summary.json file. + """ + by_metric = {} + for name, (mean, pass_rate) in metrics.items(): + by_metric[name] = { + "pass_rate": pass_rate, + "score_statistics": {"mean": mean}, + } + + total = len(metrics) * 10 + return { + "timestamp": timestamp, + "total_evaluations": total, + "summary_stats": { + "overall": {"TOTAL": total, "PASS": total, "FAIL": 0}, + "by_metric": by_metric, + }, + } + + +def write_summary( + directory: Path, + data: dict[str, Any], + name: str = "evaluation_20260101_000000_summary.json", +) -> Path: + """Write a summary dict to a JSON file in the given directory.""" + path = directory / name + path.write_text(json.dumps(data), encoding="utf-8") + return path diff --git a/tests/script/test_compare_against_baseline.py b/tests/script/test_compare_against_baseline.py new file mode 100644 index 00000000..e75041ba --- /dev/null +++ b/tests/script/test_compare_against_baseline.py @@ -0,0 +1,317 @@ +"""Tests for the baseline comparison regression gating script.""" + +from pathlib import Path +from typing import Any + +import pytest + +from script.regression.compare_against_baseline import ( + compute_metric_deltas, + find_and_load_summary, + generate_markdown_summary, +) +from tests.script.conftest import make_summary, write_summary + +# --------------------------------------------------------------------------- +# Tests for find_and_load_summary +# --------------------------------------------------------------------------- + + +class TestFindAndLoadSummary: + """Tests for the find_and_load_summary function.""" + + def test_loads_single_summary(self, tmp_path: Path) -> None: + """Happy path: one summary file in the directory.""" + expected = make_summary({"ragas:context_recall": (0.9, 100.0)}) + write_summary(tmp_path, expected) + + result = find_and_load_summary(str(tmp_path)) + + assert result == expected + + def test_raises_on_no_summary(self, tmp_path: Path) -> None: + """No *_summary.json file should raise FileNotFoundError.""" + (tmp_path / "other_file.txt").write_text("not a summary") + + with pytest.raises(FileNotFoundError, match="No \\*_summary.json"): + find_and_load_summary(str(tmp_path)) + + def test_raises_on_multiple_summaries(self, tmp_path: Path) -> None: + """Multiple summary files should raise RuntimeError.""" + data = make_summary({"ragas:context_recall": (0.9, 100.0)}) + write_summary(tmp_path, data, "run_a_summary.json") + write_summary(tmp_path, data, "run_b_summary.json") + + with pytest.raises(RuntimeError, match="Multiple summary files"): + find_and_load_summary(str(tmp_path)) + + def test_raises_on_missing_directory(self, tmp_path: Path) -> None: + """Non-existent directory should raise FileNotFoundError.""" + with pytest.raises(FileNotFoundError, match="Directory not found"): + find_and_load_summary(str(tmp_path / "nonexistent_subdir")) + + +# --------------------------------------------------------------------------- +# Tests for compute_metric_deltas +# --------------------------------------------------------------------------- + + +class TestComputeMetricDeltas: + """Tests for the compute_metric_deltas function.""" + + def test_no_regression(self) -> None: + """Scores that stay the same or improve should all be PASS.""" + baseline = make_summary( + { + "ragas:context_recall": (0.85, 100.0), + "custom:answer_correctness": (0.90, 95.0), + } + ) + current = make_summary( + { + "ragas:context_recall": (0.88, 100.0), + "custom:answer_correctness": (0.90, 95.0), + } + ) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + assert all(d["status"] == "PASS" for d in deltas) + + def test_critical_regression_is_fail(self) -> None: + """A critical metric dropping beyond the threshold should be FAIL.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary({"ragas:context_recall": (0.80, 100.0)}) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + faith = [d for d in deltas if d["metric"] == "ragas:context_recall"][0] + assert faith["status"] == "FAIL" + assert faith["is_critical"] is True + assert faith["score_delta"] == pytest.approx(-0.10, abs=1e-6) + + def test_noncritical_regression_is_warn(self) -> None: + """A non-critical metric dropping beyond threshold should be WARN, not FAIL.""" + baseline = make_summary({"custom:intent_eval": (0.95, 100.0)}) + current = make_summary({"custom:intent_eval": (0.80, 100.0)}) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + intent = [d for d in deltas if d["metric"] == "custom:intent_eval"][0] + assert intent["status"] == "WARN" + assert intent["is_critical"] is False + + def test_drop_within_threshold_is_pass(self) -> None: + """A small drop within the allowed delta should still be PASS.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary({"ragas:context_recall": (0.88, 100.0)}) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + faith = [d for d in deltas if d["metric"] == "ragas:context_recall"][0] + assert faith["status"] == "PASS" + + def test_mixed_results(self) -> None: + """Multiple metrics with different outcomes.""" + baseline = make_summary( + { + "ragas:context_recall": (0.90, 100.0), + "ragas:context_precision_with_reference": (0.85, 90.0), + "custom:intent_eval": (0.95, 100.0), + "ragas:context_relevance": (0.80, 80.0), + } + ) + current = make_summary( + { + "ragas:context_recall": (0.50, 80.0), + "ragas:context_precision_with_reference": (0.86, 92.0), + "custom:intent_eval": (0.70, 80.0), + "ragas:context_relevance": (0.82, 85.0), + } + ) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + by_metric = {d["metric"]: d["status"] for d in deltas} + + assert by_metric["ragas:context_recall"] == "FAIL" + assert by_metric["ragas:context_precision_with_reference"] == "PASS" + assert by_metric["custom:intent_eval"] == "WARN" + assert by_metric["ragas:context_relevance"] == "PASS" + + def test_critical_metric_only_in_baseline_is_fail(self) -> None: + """A critical metric present in baseline but missing from current is FAIL.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary({}) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + faith = [d for d in deltas if d["metric"] == "ragas:context_recall"][0] + assert faith["current_mean"] is None + assert faith["score_delta"] is None + assert faith["status"] == "FAIL" + + def test_noncritical_metric_only_in_baseline_is_warn(self) -> None: + """A non-critical metric present in baseline but missing from current is WARN.""" + baseline = make_summary({"custom:intent_eval": (0.90, 100.0)}) + current = make_summary({}) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + intent = [d for d in deltas if d["metric"] == "custom:intent_eval"][0] + assert intent["current_mean"] is None + assert intent["score_delta"] is None + assert intent["status"] == "WARN" + + def test_metric_only_in_current(self) -> None: + """A new metric in current but not in baseline gets None deltas and PASS.""" + baseline = make_summary({}) + current = make_summary({"ragas:context_recall": (0.90, 100.0)}) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + faith = [d for d in deltas if d["metric"] == "ragas:context_recall"][0] + assert faith["baseline_mean"] is None + assert faith["score_delta"] is None + assert faith["status"] == "PASS" + + def test_custom_thresholds(self) -> None: + """A drop of 0.05 should FAIL with critical_delta=0.01 but PASS with 0.10.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary({"ragas:context_recall": (0.85, 100.0)}) + + strict = compute_metric_deltas( + baseline, current, critical_delta=0.01, warn_delta=0.01 + ) + assert strict[0]["status"] == "FAIL" + + lenient = compute_metric_deltas( + baseline, current, critical_delta=0.10, warn_delta=0.10 + ) + assert lenient[0]["status"] == "PASS" + + def test_just_within_threshold_is_pass(self) -> None: + """A drop smaller than the threshold should be PASS.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + # Drop of 0.02, threshold is 0.03 — should pass + current = make_summary({"ragas:context_recall": (0.88, 100.0)}) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + assert deltas[0]["status"] == "PASS" + + def test_just_beyond_threshold_is_fail(self) -> None: + """A drop larger than the threshold should be FAIL for critical metrics.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + # Drop of 0.04, threshold is 0.03 — should fail + current = make_summary({"ragas:context_recall": (0.86, 100.0)}) + + deltas = compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + assert deltas[0]["status"] == "FAIL" + + +# --------------------------------------------------------------------------- +# Tests for generate_markdown_summary +# --------------------------------------------------------------------------- + + +class TestGenerateMarkdownSummary: + """Tests for the generate_markdown_summary function.""" + + def _make_deltas( + self, + baseline: dict[str, Any], + current: dict[str, Any], + ) -> list[dict[str, Any]]: + """Shortcut to compute deltas from two summaries.""" + return compute_metric_deltas( + baseline, current, critical_delta=0.03, warn_delta=0.03 + ) + + def test_contains_header_and_table(self) -> None: + """Output should have the title, metadata, and a markdown table.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary({"ragas:context_recall": (0.91, 100.0)}) + deltas = self._make_deltas(baseline, current) + + md = generate_markdown_summary(deltas, baseline, current) + + assert "# Regression Comparison Summary" in md + assert "| Metric |" in md + assert "ragas:context_recall" in md + + def test_shows_regression_on_critical_fail(self) -> None: + """Result line should say REGRESSION when a critical metric fails.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary({"ragas:context_recall": (0.50, 80.0)}) + deltas = self._make_deltas(baseline, current) + + md = generate_markdown_summary(deltas, baseline, current) + + assert "**Result: REGRESSION**" in md + + def test_shows_warning_on_noncritical_drop(self) -> None: + """Result line should say WARNING when only non-critical metrics drop.""" + baseline = make_summary({"custom:intent_eval": (0.95, 100.0)}) + current = make_summary({"custom:intent_eval": (0.70, 80.0)}) + deltas = self._make_deltas(baseline, current) + + md = generate_markdown_summary(deltas, baseline, current) + + assert "**Result: WARNING**" in md + + def test_shows_pass_when_no_regressions(self) -> None: + """Result line should say PASS when nothing regressed.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary({"ragas:context_recall": (0.92, 100.0)}) + deltas = self._make_deltas(baseline, current) + + md = generate_markdown_summary(deltas, baseline, current) + + assert "**Result: PASS**" in md + + def test_includes_evaluation_counts(self) -> None: + """Summary should show evaluation counts from both runs.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary( + {"ragas:context_recall": (0.90, 100.0)}, + timestamp="2026-02-01T00:00:00", + ) + deltas = self._make_deltas(baseline, current) + + md = generate_markdown_summary(deltas, baseline, current) + + assert str(baseline["total_evaluations"]) in md + assert str(current["total_evaluations"]) in md + assert "2026-01-01" in md + assert "2026-02-01" in md + + def test_fail_status_is_bold(self) -> None: + """FAIL entries in the table should be bold for visibility.""" + baseline = make_summary({"ragas:context_recall": (0.90, 100.0)}) + current = make_summary({"ragas:context_recall": (0.50, 80.0)}) + deltas = self._make_deltas(baseline, current) + + md = generate_markdown_summary(deltas, baseline, current) + + assert "**FAIL**" in md