fix(evals): record the config that actually ran and preserve candidate_k when downgrading rerank

Author: codebybilal18

app/evals/run.py

@@ -34,7 +34,7 @@
 from app.evals.graders.deterministic import grade_citations, grade_refusal
 from app.evals.graders.judges import grade_correctness, grade_faithfulness
 from app.evals.graders.retrieval import grade_retrieval
-from app.generation.answer import QueryRequestSpec, answer_once
+from app.generation.answer import QueryRequestSpec, answer_once, resolve_config
 from app.generation.prompts.registry import all_prompt_ids
 from app.observability.cost import UsageMeter
 
@@ -233,7 +233,25 @@ async def run_eval(
     if limit:
         questions = questions[:limit]
 
-    config = get_config(config_name)
+    # Record the config that will actually run, not the one requested.
+    #
+    # resolve_config() substitutes the LLM reranker when a cross-encoder cannot
+    # load. Storing the requested config here would file the run under
+    # "cross-encoder rerank" while an LLM scorer produced every number — the
+    # published table would then attribute results to a component that never
+    # executed, which is the precise failure this project exists to argue
+    # against.
+    requested = get_config(config_name)
+    config = resolve_config(config_name)
+    if config.rerank is not requested.rerank:
+        log.warning(
+            "config %s requested rerank=%s but will run rerank=%s — the run is "
+            "recorded under what actually executes",
+            config_name,
+            requested.rerank.value,
+            config.rerank.value,
+        )
+
     log.info(
         "eval %s (%s) — %s questions %s, space=%s",
         config.name,

app/generation/answer.py

@@ -95,11 +95,15 @@ def resolve_config(name: str | None) -> RetrievalConfig:
         not settings.is_serverless and LocalCrossEncoderReranker.available()
     ):
         why = "serverless" if settings.is_serverless else "cross-encoder unavailable"
-        downgraded = get_config("serverless")
-        downgraded.name = config.name
-        downgraded.top_k = config.top_k
-        downgraded.candidate_k = min(config.candidate_k, downgraded.candidate_k)
-        downgraded.label = f"{config.label} ({why}: LLM reranker)"
+        downgraded = config.model_copy(deep=True)
+        downgraded.rerank = RerankStrategy.LLM
+        if settings.is_serverless:
+            # Only the serverless path narrows the candidate set; it is a
+            # latency budget, not part of what the config means. Applying it to
+            # a local benchmark run would change retrieval depth as well as the
+            # reranker, and the comparison would no longer isolate one variable.
+            downgraded.candidate_k = min(config.candidate_k, get_config("serverless").candidate_k)
+        downgraded.label = f"{config.label} — LLM reranker ({why})"
         log.info("config %s downgraded to the LLM reranker — %s", config.name, why)
         return downgraded
     return config