Skip to content

Commit 6f6106f

Browse files
sgoel2be24-cybercopybara-github
authored andcommitted
fix: handle Windows paths in adk eval
Merge #6419 Fixes #6415 PiperOrigin-RevId: 951036899
1 parent 0b7355b commit 6f6106f

2 files changed

Lines changed: 76 additions & 3 deletions

File tree

src/google/adk/cli/cli_eval.py

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,15 +101,27 @@ def parse_and_get_evals_to_run(
101101
eval_set_to_evals: dict[str, list[str]] = {}
102102
for input_eval_set in evals_to_run_info:
103103
evals = []
104-
if ":" not in input_eval_set:
104+
drive_letter = input_eval_set[:1]
105+
has_windows_drive_prefix = (
106+
len(input_eval_set) >= 3
107+
and drive_letter.isascii()
108+
and drive_letter.isalpha()
109+
and input_eval_set[1] == ":"
110+
and input_eval_set[2] in ("\\", "/")
111+
)
112+
selector_separator_index = input_eval_set.find(
113+
":", 3 if has_windows_drive_prefix else 0
114+
)
115+
if selector_separator_index == -1:
105116
# We don't have any eval cases specified. This would be the case where the
106117
# the user wants to run all eval cases in the eval set.
107118
eval_set = input_eval_set
108119
else:
109120
# There are eval cases that we need to parse. The user wants to run
110121
# specific eval cases from the eval set.
111-
eval_set = input_eval_set.split(":")[0]
112-
evals = input_eval_set.split(":")[1].split(",")
122+
eval_set = input_eval_set[:selector_separator_index]
123+
selector_list = input_eval_set[selector_separator_index + 1 :]
124+
evals = selector_list.split(":")[0].split(",")
113125
evals = [s for s in evals if s.strip()]
114126

115127
if eval_set not in eval_set_to_evals:

tests/unittests/cli/utils/test_cli_eval.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,67 @@
1919
from types import SimpleNamespace
2020
from unittest import mock
2121

22+
import pytest
23+
24+
25+
@pytest.mark.parametrize(
26+
("input_eval_set", "expected"),
27+
[
28+
pytest.param(
29+
r"C:\tmp\agent\eval.evalset.json",
30+
{r"C:\tmp\agent\eval.evalset.json": []},
31+
id="windows-backslash-path-without-selectors",
32+
),
33+
pytest.param(
34+
r"C:\tmp\agent\eval.evalset.json:case1",
35+
{r"C:\tmp\agent\eval.evalset.json": ["case1"]},
36+
id="windows-backslash-path-with-one-selector",
37+
),
38+
pytest.param(
39+
r"C:\tmp\agent\eval.evalset.json:case1,case2",
40+
{r"C:\tmp\agent\eval.evalset.json": ["case1", "case2"]},
41+
id="windows-backslash-path-with-multiple-selectors",
42+
),
43+
pytest.param(
44+
"C:/tmp/agent/eval.evalset.json:case1",
45+
{"C:/tmp/agent/eval.evalset.json": ["case1"]},
46+
id="windows-forward-slash-path",
47+
),
48+
pytest.param(
49+
r"d:\tmp\agent\eval.evalset.json:case1",
50+
{r"d:\tmp\agent\eval.evalset.json": ["case1"]},
51+
id="lowercase-windows-drive",
52+
),
53+
pytest.param(
54+
"/tmp/agent/eval.evalset.json:case1,case2",
55+
{"/tmp/agent/eval.evalset.json": ["case1", "case2"]},
56+
id="posix-path-with-selectors",
57+
),
58+
pytest.param(
59+
"my_eval_set:case1,case2",
60+
{"my_eval_set": ["case1", "case2"]},
61+
id="eval-set-id-with-selectors",
62+
),
63+
pytest.param(
64+
"my_eval_set",
65+
{"my_eval_set": []},
66+
id="eval-set-id-without-selectors",
67+
),
68+
pytest.param(
69+
"/tmp/agent/eval.evalset.json",
70+
{"/tmp/agent/eval.evalset.json": []},
71+
id="posix-path-without-selectors",
72+
),
73+
],
74+
)
75+
def test_parse_and_get_evals_to_run_parses_eval_set_and_selectors(
76+
input_eval_set: str, expected: dict[str, list[str]]
77+
):
78+
"""Eval-set paths and IDs retain their optional case selectors."""
79+
from google.adk.cli.cli_eval import parse_and_get_evals_to_run
80+
81+
assert parse_and_get_evals_to_run([input_eval_set]) == expected
82+
2283

2384
def test_get_eval_sets_manager_local(monkeypatch):
2485
mock_local_manager = mock.MagicMock()

0 commit comments

Comments
 (0)