Eval fixes

.gitignore

@@ -17,10 +17,10 @@ var/
 /*_objects.yml
 examples/app-flowise/.*
 test_*.py
+!tests/eval_test/tests/test_*.py
 .planning/
 .ruff_cache/
 Start_MCP_Server.bat
 .deepeval/
 .pytest_cache/
 
-

ddls_extracted/evals_employees_DDL.sql

@@ -0,0 +1,25 @@
+/*
+ * File: evals_employees_DDL.sql
+ * Generated: 2026-06-26 15:04:08
+ * Type: TABLE
+ * Database: demo_user
+ * Object: evals_employees
+ * Size: 572 characters
+ */
+
+CREATE SET TABLE demo_user.evals_employees ,FALLBACK ,
+     NO BEFORE JOURNAL,
+     NO AFTER JOURNAL,
+     CHECKSUM = DEFAULT,
+     DEFAULT MERGEBLOCKRATIO,
+     MAP = TD_MAP1
+     (
+      employee_id INTEGER NOT NULL,
+      name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+      department VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+      salary DECIMAL(10,2),
+      region VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+      hire_date DATE FORMAT 'YY/MM/DD' NOT NULL,
+      manager_id INTEGER, 
+PRIMARY KEY ( employee_id ))
+;

ddls_extracted/evals_orders_DDL.sql

@@ -0,0 +1,25 @@
+/*
+ * File: evals_orders_DDL.sql
+ * Generated: 2026-06-26 15:03:22
+ * Type: TABLE
+ * Database: demo_user
+ * Object: evals_orders
+ * Size: 563 characters
+ */
+
+CREATE SET TABLE demo_user.evals_orders ,FALLBACK ,
+     NO BEFORE JOURNAL,
+     NO AFTER JOURNAL,
+     CHECKSUM = DEFAULT,
+     DEFAULT MERGEBLOCKRATIO,
+     MAP = TD_MAP1
+     (
+      order_id INTEGER NOT NULL,
+      customer_name VARCHAR(100) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+      order_date DATE FORMAT 'YY/MM/DD' NOT NULL,
+      ship_date DATE FORMAT 'YY/MM/DD',
+      amount DECIMAL(10,2) NOT NULL,
+      product_category VARCHAR(50) CHARACTER SET LATIN NOT CASESPECIFIC NOT NULL,
+      quantity INTEGER NOT NULL, 
+PRIMARY KEY ( order_id ))
+;

tests/eval_test/AGENTS.md

@@ -47,11 +47,12 @@ judge/
 cases/
   <module>.json       # Eval case definitions per MCP module prefix
 tests/
-  conftest.py         # Fixtures, {EVALS_DATABASE} substitution, assert_eval_case()
-  case_runner.py      # Single/multi-turn execution and per-turn scoring
-  test_<module>.py    # One pytest file per module (all use assert_eval_case)
-  test_checks.py      # Unit tests for judge/checks.py
-  test_multi_turn.py  # Unit tests for multi-turn schema validation
+  conftest.py          # Fixtures, _substitute() helper, wrapper functions for eval case execution
+  case_runner.py       # Single/multi-turn execution and per-turn scoring (raw functions)
+  generated_cases.py   # Dynamically generates pytest tests from JSON case definitions
+  test_generated_cases.py # Exposes generated tests to pytest discovery
+  test_checks.py       # Unit tests for judge/checks.py
+  test_multi_turn.py   # Unit tests for multi-turn schema validation
 backup/               # Optional bootstrap/audit scripts — see backup/README.md
 run_evals.py          # CLI entry point (deepeval + pytest)
 suggest_overrides.py  # CLI: draft description_overrides from eval failures
@@ -122,7 +123,8 @@ pyproject.toml        # Project metadata, Ruff config, pytest paths
 | Agent loop / MCP session | `agent/client.py` |
 | Deterministic checks | `judge/checks.py` |
 | LLM judge metrics | `judge/metrics.py` |
-| Run + score any case | `tests/case_runner.py` → `assert_eval_case()` |
+| Dynamic test generation from JSON | `tests/generated_cases.py` → `_register_module_tests()` |
+| Run + score any case (raw) | `tests/case_runner.py` → `assert_eval_case()` |
 | Eval run summaries | `judge/report.py` → `results/runs/<run_id>/` + `results/latest.json` |
 | List recent runs | `run_evals.py --list-runs` |
 | Override suggestions | `suggest_overrides.py` → run's `suggested_overrides.json` |
@@ -132,7 +134,7 @@ pyproject.toml        # Project metadata, Ruff config, pytest paths
 | Ambiguous pair registry / live MCP diff | `backup/audit_cases.py` → `AMBIGUOUS_PAIRS`, `--live-mcp` |
 | Happy-path generator | `backup/generate_cases.py` |
 | Test data setup / preflight | `setup_test_data.py`, `preflight.py` |
-| `{EVALS_DATABASE}` substitution | `tests/conftest.py` → `_substitute()` |s
+| `{EVALS_DATABASE}` substitution (runtime) | `tests/generated_cases.py` → `_register_module_tests()` calls `_substitute()` from conftest before execution |
 | User-facing docs | `README.md`, `docs/` |
 
 ## EVAL FLOW
@@ -147,14 +149,15 @@ pyproject.toml        # Project metadata, Ruff config, pytest paths
 
 **Per-case scoring:**
 
-1. Test loads case from `cases/<module>.json` via `load_cases()`.
-2. `{EVALS_DATABASE}` placeholder substituted at runtime.
-3. **Single-turn:** `run_agent()` → deterministic checks → deepeval metrics.
-4. **Multi-turn:** `run_agent_turns()` (one MCP session, max 7 turns) → per-turn checks:
+1. Pytest discovers parametrized tests from `tests/generated_cases.py` → `_register_module_tests()`.
+2. Each parametrized test loads a case from `cases/<module>.json` and filters by `EVALS_RUN_MODULE` / `EVALS_RUN_TYPE` env vars.
+3. `{EVALS_DATABASE}` placeholder substituted at runtime via `_substitute()` from `conftest.py` before execution.
+4. **Single-turn:** `run_agent()` → deterministic checks → deepeval metrics.
+5. **Multi-turn:** `run_agent_turns()` (one MCP session, max 7 turns) → per-turn checks:
    - Clarification turns: no tools + Clarification GEval
    - Tool turns: deterministic checks + ToolCorrectnessMetric
-5. Structural failures in `judge/checks.py` raise `AssertionError` before the judge runs.
-6. Outcomes recorded under `results/runs/<run_id>/` via `judge/report.py`.
+6. Structural failures in `judge/checks.py` raise `AssertionError` before the judge runs.
+7. Outcomes recorded under `results/runs/<run_id>/` via `judge/report.py`.
 
 ## CASE JSON CONVENTIONS
 

tests/eval_test/README.md

@@ -6,6 +6,11 @@ Tests whether an LLM agent selects the right MCP tool and forms valid parameters
 
 ## Quick start
 
+**Pre-requisite:** ensure that the MCP Server is running in a separate process and reachable in streamable-http.
+
+Eg. `teradata-mcp-server --mcp_port 8001 --mcp_transport streamable-http` 
+
+
 ```bash
 uv venv && uv sync
 cp .env.example .env   # set MCP_SERVER_URL, EVALS_DATABASE, Bedrock credentials
@@ -14,7 +19,15 @@ python setup_test_data.py
 python run_evals.py --module base
 ```
 
-Open `results/latest_summary.md` for pass/fail details, or run `python run_evals.py --list-runs` to browse run directories.
+Open `results/latest_summary.md` for pass/fail details and aggregate Bedrock token usage, or run `python run_evals.py --list-runs` to browse run directories. `summary.json` also includes per-case `token_usage` plus run-level totals; set the optional `BEDROCK_AGENT_*_COST_PER_1M_TOKENS` and `BEDROCK_JUDGE_*_COST_PER_1M_TOKENS` env vars to add USD cost reporting.
+
+To compare several evaluated models with the same judge, put the model set in YAML and run:
+
+```bash
+python run_evals.py --module base --model-set model_set.yml
+```
+
+When the YAML contains multiple evaluated models, the runner writes one normal run per model plus `results/latest_batch_summary.md` with links, pass ratios, token totals, and cost comparison. With one evaluated model, it behaves like the regular single-run path while taking model IDs and costs from YAML.
 
 ## Workflow
 

tests/eval_test/agent/client.py

@@ -13,6 +13,8 @@
 from mcp import ClientSession
 from mcp.client.streamable_http import streamablehttp_client
 
+from judge.usage import record_bedrock_usage
+
 MAX_TOOL_RESULT_CHARS = int(os.environ.get("MAX_TOOL_RESULT_CHARS", "8000"))
 
 # ---------------------------------------------------------------------------
@@ -247,6 +249,7 @@ async def _run_agent_turns_async(
                             "toolChoice": {"auto": {}},
                         },
                     )
+                    record_bedrock_usage(model_id=model_id, role="agent", response=response)
 
                     stop_reason = response["stopReason"]
                     output_message = response["output"]["message"]
@@ -303,6 +306,7 @@ async def _run_agent_async(
                         "toolChoice": {"auto": {}},
                     },
                 )
+                record_bedrock_usage(model_id=model_id, role="agent", response=response)
 
                 stop_reason = response["stopReason"]
                 output_message = response["output"]["message"]

tests/eval_test/docs/setup.md

@@ -50,6 +50,17 @@ BEDROCK_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0
 # Optional — defaults to BEDROCK_MODEL_ID
 # BEDROCK_JUDGE_MODEL_ID=anthropic.claude-3-5-sonnet-20241022-v2:0
 
+# Optional — used only for USD cost reporting. Token usage is captured without these.
+# Values are USD per 1M tokens. Use prices for your exact Bedrock model and region.
+# BEDROCK_AGENT_INPUT_COST_PER_1M_TOKENS=3.00
+# BEDROCK_AGENT_OUTPUT_COST_PER_1M_TOKENS=15.00
+# BEDROCK_AGENT_CACHE_READ_INPUT_COST_PER_1M_TOKENS=0.30
+# BEDROCK_AGENT_CACHE_WRITE_INPUT_COST_PER_1M_TOKENS=3.75
+# BEDROCK_JUDGE_INPUT_COST_PER_1M_TOKENS=3.00
+# BEDROCK_JUDGE_OUTPUT_COST_PER_1M_TOKENS=15.00
+# BEDROCK_JUDGE_CACHE_READ_INPUT_COST_PER_1M_TOKENS=0.30
+# BEDROCK_JUDGE_CACHE_WRITE_INPUT_COST_PER_1M_TOKENS=3.75
+
 MCP_SERVER_URL=http://127.0.0.1:8001/mcp
 EVALS_DATABASE=your_database_name
 
@@ -63,6 +74,42 @@ AGENT_MAX_STEPS_PER_TURN=3
 
 Credentials follow the standard boto3 chain — env vars, `~/.aws/credentials`, or an IAM instance profile. Set `AWS_SESSION_TOKEN` for temporary credentials.
 
+The report always records Bedrock token usage by model and role (`agent` / `judge`) with input, output, cache-read input, cache-write input, and total token counts. Cost fields remain `null` / “not configured” until pricing env vars are set. Per-1K aliases are also accepted by replacing `PER_1M` with `PER_1K`.
+
+## Multi-model batches
+
+Use `--model-set` when you want the model IDs and cost parameters to come from YAML instead of `.env`:
+
+```yaml
+name: bedrock-model-comparison
+judge:
+  model: anthropic.claude-3-5-sonnet-20241022-v2:0
+  cost:
+    input_cost_per_1m_tokens: 3.00
+    output_cost_per_1m_tokens: 15.00
+    cache_read_input_cost_per_1m_tokens: 0.30
+    cache_write_input_cost_per_1m_tokens: 3.75
+models:
+  - label: sonnet
+    model: anthropic.claude-3-5-sonnet-20241022-v2:0
+    cost:
+      input_cost_per_1m_tokens: 3.00
+      output_cost_per_1m_tokens: 15.00
+  - label: haiku
+    model: anthropic.claude-3-5-haiku-20241022-v1:0
+    cost:
+      input_cost_per_1m_tokens: 0.80
+      output_cost_per_1m_tokens: 4.00
+```
+
+Run the same case filters as usual:
+
+```bash
+python run_evals.py --module base --model-set model_set.yml
+```
+
+The YAML keys `evaluated_models` and `models` are both accepted for the evaluated model list. Each run still produces `results/runs/<run_id>/summary.md`; when more than one model is listed, the batch also writes `results/latest_batch_summary.md` and `results/batches/<batch_id>/summary.md` with pass ratio and cost comparisons.
+
 ## Test data
 
 Create eval tables once before running live evals:

tests/eval_test/judge/bedrock_llm.py

@@ -9,8 +9,12 @@
 import boto3
 from deepeval.models.base_model import DeepEvalBaseLLM
 
+from judge.usage import record_bedrock_usage
+
 
 class BedrockLLM(DeepEvalBaseLLM):
+    returns_token_cost = True
+
     def __init__(self, model_id: str | None = None, bedrock_client=None):
         self.model_id = model_id or os.environ.get(
             "BEDROCK_JUDGE_MODEL_ID",
@@ -29,11 +33,12 @@ def load_model(self):
     def get_model_name(self) -> str:
         return self.model_id
 
-    def _call(self, prompt: str, schema: Any = None) -> str:
+    def _call(self, prompt: str, schema: Any = None) -> tuple[Any, float | None]:
         response = self._client.converse(
             modelId=self.model_id,
             messages=[{"role": "user", "content": [{"text": prompt}]}],
         )
+        cost_usd = record_bedrock_usage(model_id=self.model_id, role="judge", response=response)
         text = response["output"]["message"]["content"][0]["text"]
 
         if schema is not None:
@@ -42,16 +47,16 @@ def _call(self, prompt: str, schema: Any = None) -> str:
             try:
                 start = text.index("{")
                 end = text.rindex("}") + 1
-                return schema.model_validate(json.loads(text[start:end]))
+                return schema.model_validate(json.loads(text[start:end])), cost_usd
             except Exception:
                 pass
 
-        return text
+        return text, cost_usd
 
-    def generate(self, prompt: str, schema: Any = None) -> str:
+    def generate(self, prompt: str, schema: Any = None) -> tuple[Any, float | None]:
         return self._call(prompt, schema)
 
-    async def a_generate(self, prompt: str, schema: Any = None) -> str:
+    async def a_generate(self, prompt: str, schema: Any = None) -> tuple[Any, float | None]:
         import asyncio
         loop = asyncio.get_event_loop()
         return await loop.run_in_executor(None, self._call, prompt, schema)

tests/eval_test/judge/metrics.py

@@ -2,22 +2,30 @@
 
 from __future__ import annotations
 
-from deepeval.metrics import GEval, ToolCorrectnessMetric
-from deepeval.test_case import LLMTestCaseParams
+from deepeval.metrics import ConversationalGEval, GEval, ToolCorrectnessMetric
+from deepeval.test_case import MultiTurnParams, SingleTurnParams, ToolCallParams
+
+
+def _enable_custom_cost_tracking(metric, judge_llm):
+    if getattr(judge_llm, "returns_token_cost", False):
+        metric.using_native_model = True
+    return metric
 
 
 def tool_correctness_metric(judge_llm) -> ToolCorrectnessMetric:
     """Evaluates tool selection accuracy and parameter correctness via LLM judge."""
-    return ToolCorrectnessMetric(
-        threshold=0.5,
+    return _enable_custom_cost_tracking(ToolCorrectnessMetric(
+        threshold=1.0,
+        evaluation_params=[ToolCallParams.INPUT_PARAMETERS],
         model=judge_llm,
         include_reason=True,
-    )
+        should_exact_match=True,
+    ), judge_llm)
 
 
 def clarification_metric(judge_llm) -> GEval:
     """For missing_parameter cases: checks the agent asked for clarification rather than hallucinating."""
-    return GEval(
+    return _enable_custom_cost_tracking(GEval(
         name="Clarification Check",
         criteria=(
             "The agent was given a prompt that is missing a required parameter. "
@@ -27,12 +35,34 @@ def clarification_metric(judge_llm) -> GEval:
             "Score 0.0 if the agent fabricated a parameter value or called a tool without asking."
         ),
         evaluation_params=[
-            LLMTestCaseParams.INPUT,
-            LLMTestCaseParams.ACTUAL_OUTPUT,
+            SingleTurnParams.INPUT,
+            SingleTurnParams.ACTUAL_OUTPUT,
+        ],
+        model=judge_llm,
+        threshold=0.5,
+    ), judge_llm)
+
+
+def conversational_clarification_metric(judge_llm) -> ConversationalGEval:
+    """For multi-turn clarification turns: checks the latest assistant turn asks for missing info."""
+    return _enable_custom_cost_tracking(ConversationalGEval(
+        name="Clarification Check",
+        criteria=(
+            "Evaluate the latest assistant turn in the conversation. The user request is missing "
+            "required information for a tool call, so the assistant should ask a clear clarification "
+            "question for the missing information instead of inventing a value or proceeding with a guess. "
+            "Score 1.0 if the latest assistant turn asks for the missing information. "
+            "Score 0.0 if it fabricates a parameter value or proceeds without asking."
+        ),
+        evaluation_params=[
+            MultiTurnParams.CONTENT,
+            MultiTurnParams.ROLE,
+            MultiTurnParams.SCENARIO,
+            MultiTurnParams.EXPECTED_OUTCOME,
         ],
         model=judge_llm,
         threshold=0.5,
-    )
+    ), judge_llm)
 
 
 def get_metrics(case: dict, judge_llm) -> list: