Skip to content

Commit ad7b1e4

Browse files
authored
Merge pull request galaxyproject#22070 from jmchilton/agent_testing
Replace mocked agent tests with static YAML backend for deterministic API/E2E testing
2 parents 65e360b + 4b03059 commit ad7b1e4

19 files changed

Lines changed: 1106 additions & 242 deletions

File tree

client/src/components/DatasetInformation/DatasetError.vue

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ onMounted(async () => {
155155
may not always be accurate.
156156
</span>
157157
</p>
158-
<BCard v-if="'tool_stderr' in jobDetails" class="mb-2">
158+
<BCard v-if="'tool_stderr' in jobDetails" class="mb-2" data-description="galaxy wizard card">
159159
<GalaxyWizard
160160
view="error"
161161
:query="jobDetails.tool_stderr ?? ''"

client/src/components/GalaxyWizard.vue

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -91,46 +91,60 @@ async function sendFeedback(value: "up" | "down") {
9191
</script>
9292

9393
<template>
94-
<div>
95-
<GButton v-if="!queryResponse" class="w-100" variant="info" :disabled="busy" @click="submitQuery">
94+
<div data-description="galaxy wizard">
95+
<GButton
96+
v-if="!queryResponse"
97+
class="w-100"
98+
variant="info"
99+
:disabled="busy"
100+
data-description="galaxy wizard analyze button"
101+
@click="submitQuery">
96102
<span v-if="!busy"> Let our Help Wizard Figure it out! </span>
97103
<LoadingSpan v-else message="Thinking" />
98104
</GButton>
99105
<div :class="props.view == 'wizard' && 'mt-4'">
100-
<div v-if="busy">
106+
<div v-if="busy" data-description="galaxy wizard loading">
101107
<BSkeleton animation="wave" width="85%" />
102108
<BSkeleton animation="wave" width="55%" />
103109
<BSkeleton animation="wave" width="70%" />
104110
</div>
105111
<div v-else>
106112
<!-- eslint-disable-next-line vue/no-v-html -->
107-
<div class="chatResponse" v-html="renderMarkdown(queryResponse)" />
113+
<div
114+
class="chatResponse"
115+
data-description="galaxy wizard response"
116+
v-html="renderMarkdown(queryResponse)" />
108117

109118
<template v-if="errorMessage">
110119
<hr class="error-divider" />
111120
<div class="error-message">{{ errorMessage }}</div>
112121
</template>
113122
</div>
114123

115-
<div v-if="queryResponse && !hasError" class="feedback-buttons mt-2">
124+
<div
125+
v-if="queryResponse && !hasError"
126+
class="feedback-buttons mt-2"
127+
data-description="galaxy wizard feedback">
116128
<hr class="w-100" />
117129
<h4>Was this answer helpful?</h4>
118130
<GButton
119131
color="green"
120132
:disabled="feedback !== null"
121133
:class="{ submitted: feedback === 'up' }"
134+
data-description="galaxy wizard feedback up"
122135
@click="sendFeedback('up')">
123136
<FontAwesomeIcon :icon="faThumbsUp" fixed-width />
124137
</GButton>
125138
<GButton
126139
color="red"
127140
:disabled="feedback !== null"
128141
:class="{ submitted: feedback === 'down' }"
142+
data-description="galaxy wizard feedback down"
129143
@click="sendFeedback('down')">
130144
<FontAwesomeIcon :icon="faThumbsDown" fixed-width />
131145
</GButton>
132146
<i v-if="!feedback">This feedback helps us improve our responses.</i>
133-
<i v-else>Thank you for your feedback!</i>
147+
<i v-else data-description="galaxy wizard feedback ack">Thank you for your feedback!</i>
134148
</div>
135149
</div>
136150
</div>

client/src/utils/navigation/navigation.yml

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1454,6 +1454,68 @@ job_details:
14541454
selector: '//td[@id="galaxy-tool-id"][normalize-space(text()) = "${tool_id}"]'
14551455
tool_exit_code: '#exit-code'
14561456

1457+
chatgxy:
1458+
selectors:
1459+
activity: '#activity-chatgxy'
1460+
_: '.chatgxy-container'
1461+
header: '.chatgxy-header'
1462+
new_chat_button: '.chatgxy-header .btn-outline-primary'
1463+
delete_chat_button: '.chatgxy-header .btn-outline-danger'
1464+
messages: '.chat-messages'
1465+
input: '#chat-input'
1466+
send_button: '.send-button'
1467+
welcome_message: '.system-notice'
1468+
query_cell: '.entry-query .query-text'
1469+
response_cell: '.entry-response'
1470+
response_content: '.entry-response .response-content'
1471+
agent_indicator: '.entry-response .agent-indicator'
1472+
loading: '.loading-entry'
1473+
feedback_up: '.entry-response .feedback-btn[title="Helpful"]'
1474+
feedback_down: '.entry-response .feedback-btn[title="Not helpful"]'
1475+
feedback_ack: '.entry-response .feedback-ack'
1476+
meta_tag: '.entry-response .meta-tag'
1477+
1478+
history_panel:
1479+
selectors:
1480+
_: '.activity-panel[data-description="ChatGXY"]'
1481+
toggle_selection_mode: 'button[title="Select chats to delete"]'
1482+
cancel_selection: 'button[title="Cancel selection"]'
1483+
history_item: '[data-description="sidebar item"]'
1484+
history_checkbox: '[data-description="sidebar item"] .history-checkbox'
1485+
delete_selected_button: '.selection-toolbar .btn-danger'
1486+
select_all_toggle: '.select-all-toggle'
1487+
empty_message: '[data-description="sidebar list empty"]'
1488+
1489+
galaxy_wizard:
1490+
selectors:
1491+
_:
1492+
selector: 'galaxy wizard'
1493+
type: data-description
1494+
analyze_button:
1495+
selector: 'galaxy wizard analyze button'
1496+
type: data-description
1497+
loading:
1498+
selector: 'galaxy wizard loading'
1499+
type: data-description
1500+
response:
1501+
selector: 'galaxy wizard response'
1502+
type: data-description
1503+
feedback_section:
1504+
selector: 'galaxy wizard feedback'
1505+
type: data-description
1506+
feedback_up:
1507+
selector: 'galaxy wizard feedback up'
1508+
type: data-description
1509+
feedback_down:
1510+
selector: 'galaxy wizard feedback down'
1511+
type: data-description
1512+
feedback_ack:
1513+
selector: 'galaxy wizard feedback ack'
1514+
type: data-description
1515+
wizard_card:
1516+
selector: 'galaxy wizard card'
1517+
type: data-description
1518+
14571519
zip_import_wizard:
14581520
selectors:
14591521
_: '.zip-import-wizard'

lib/galaxy/agents/factory.py

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
"""Factory for building the appropriate AgentRegistry from config."""
2+
3+
import logging
4+
from typing import TYPE_CHECKING
5+
6+
from .registry import (
7+
AgentRegistry,
8+
build_default_registry,
9+
)
10+
from .static_backend import StaticAgentRegistry
11+
12+
if TYPE_CHECKING:
13+
from galaxy.config import GalaxyAppConfiguration
14+
15+
log = logging.getLogger(__name__)
16+
17+
18+
def build_registry(config: "GalaxyAppConfiguration") -> AgentRegistry:
19+
"""Build an AgentRegistry based on config.
20+
21+
Uses StaticAgentRegistry when inference_services.static_responses is set,
22+
otherwise builds the default registry with real agents.
23+
"""
24+
inference_config = getattr(config, "inference_services", None) or {}
25+
static_responses = inference_config.get("static_responses") if isinstance(inference_config, dict) else None
26+
if static_responses:
27+
log.info(f"Static agent backend loaded: {static_responses}")
28+
return StaticAgentRegistry(static_responses)
29+
return build_default_registry(config)
Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
"""Static agent backend for deterministic testing without LLM calls.
2+
3+
Provides StaticAgent and StaticAgentRegistry that return canned responses
4+
from YAML rules. Swap at the DI container level — no mocks, no pydantic-ai.
5+
"""
6+
7+
import re
8+
from typing import (
9+
Any,
10+
Optional,
11+
)
12+
13+
import yaml
14+
15+
from .base import (
16+
ActionSuggestion,
17+
AgentResponse,
18+
BaseGalaxyAgent,
19+
GalaxyAgentDependencies,
20+
)
21+
from .registry import AgentRegistry
22+
23+
24+
class StaticAgent(BaseGalaxyAgent):
25+
"""Agent that returns canned responses from YAML rules.
26+
27+
Subclasses BaseGalaxyAgent but skips pydantic-ai Agent creation entirely.
28+
Only process() is meaningful — all other BaseGalaxyAgent methods are stubs.
29+
"""
30+
31+
agent_type = "static" # overridden per-instance
32+
33+
def __init__(
34+
self,
35+
agent_type_str: str,
36+
rules: list[dict[str, Any]],
37+
fallback: dict[str, Any],
38+
defaults: dict[str, Any],
39+
):
40+
# Intentionally skip super().__init__() — no pydantic-ai Agent needed.
41+
self.agent_type = agent_type_str
42+
self._rules = rules
43+
self._fallback = fallback
44+
self._defaults = defaults
45+
46+
def _create_agent(self):
47+
raise NotImplementedError("StaticAgent does not use pydantic-ai")
48+
49+
def get_system_prompt(self) -> str:
50+
return ""
51+
52+
async def process(self, query: str, context: Optional[dict[str, Any]] = None) -> AgentResponse:
53+
for rule in self._rules:
54+
if self._rule_matches(rule.get("match", {}), query, context):
55+
return self._make_response(rule["response"])
56+
return self._make_response(self._fallback)
57+
58+
def _rule_matches(self, match: dict[str, Any], query: str, context: Optional[dict[str, Any]]) -> bool:
59+
if "agent_type" in match and match["agent_type"] != self.agent_type:
60+
return False
61+
if "query" in match and not re.search(match["query"], query):
62+
return False
63+
if "context" in match:
64+
if not context:
65+
return False
66+
for field, pattern in match["context"].items():
67+
if field not in context or not re.search(pattern, str(context[field])):
68+
return False
69+
return True
70+
71+
def _make_response(self, resp: dict[str, Any]) -> AgentResponse:
72+
raw_suggestions = resp.get("suggestions", [])
73+
suggestions = [ActionSuggestion(**s) for s in raw_suggestions]
74+
return AgentResponse(
75+
content=resp.get("content", self._fallback.get("content", "")),
76+
confidence=resp.get("confidence", self._defaults.get("confidence", "medium")),
77+
agent_type=resp.get("agent_type", self.agent_type),
78+
suggestions=suggestions,
79+
metadata={**resp.get("metadata", {}), "static_backend": True},
80+
reasoning=resp.get("reasoning"),
81+
)
82+
83+
84+
class StaticAgentRegistry(AgentRegistry):
85+
"""Registry that returns StaticAgent instances from YAML config.
86+
87+
Subclasses AgentRegistry so it's type-compatible with the DI container.
88+
"""
89+
90+
def __init__(self, config_path: str):
91+
super().__init__()
92+
with open(config_path) as f:
93+
self._config: dict[str, Any] = yaml.safe_load(f) or {}
94+
self._rules: list[dict[str, Any]] = self._config.get("rules", [])
95+
self._fallback: dict[str, Any] = self._config.get("fallback", {})
96+
self._defaults: dict[str, Any] = self._config.get("defaults", {})
97+
98+
# Collect known agent_types from rules
99+
self._known_types: set[str] = set()
100+
for rule in self._rules:
101+
match = rule.get("match", {})
102+
if "agent_type" in match:
103+
self._known_types.add(match["agent_type"])
104+
105+
def get_agent(self, agent_type: str, deps: GalaxyAgentDependencies) -> StaticAgent:
106+
"""Return a StaticAgent that matches rules for this agent_type."""
107+
applicable = [r for r in self._rules if r.get("match", {}).get("agent_type", agent_type) == agent_type]
108+
return StaticAgent(agent_type, applicable, self._fallback, self._defaults)
109+
110+
def is_registered(self, agent_type: str) -> bool:
111+
return agent_type in self._known_types or bool(self._fallback)
112+
113+
def list_agents(self) -> list[str]:
114+
return sorted(self._known_types)
115+
116+
def get_agent_info(self, agent_type: str) -> dict[str, Any]:
117+
return {
118+
"agent_type": agent_type,
119+
"class_name": "StaticAgent",
120+
"module": "galaxy.agents.static_backend",
121+
"metadata": {"static_backend": True},
122+
"description": "Static test agent",
123+
}
124+
125+
def list_agent_info(self) -> list[dict[str, Any]]:
126+
return [self.get_agent_info(t) for t in sorted(self._known_types)]

lib/galaxy/app.py

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,8 @@
2525
jobs,
2626
tools,
2727
)
28-
from galaxy.agents.registry import (
29-
AgentRegistry,
30-
build_default_registry,
31-
)
28+
from galaxy.agents.factory import build_registry as build_agent_registry
29+
from galaxy.agents.registry import AgentRegistry
3230
from galaxy.carbon_emissions import get_carbon_intensity_entry
3331
from galaxy.celery.base_task import (
3432
GalaxyTaskBeforeStart,
@@ -775,7 +773,7 @@ def __init__(self, **kwargs) -> None:
775773
self.queue_worker = self._register_singleton(GalaxyQueueWorker, GalaxyQueueWorker(self))
776774

777775
# AI agent registry and service
778-
agent_registry = build_default_registry(self.config)
776+
agent_registry = build_agent_registry(self.config)
779777
self._register_singleton(AgentRegistry, agent_registry)
780778
self._register_singleton(AgentService, AgentService(self.config, JobQueryManager(self), agent_registry))
781779

lib/galaxy/config/schemas/config_schema.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4091,6 +4091,9 @@ mapping:
40914091
Agents and plugins inherit from 'default' configuration, which itself falls back to global ai_model/ai_api_key settings.
40924092
All agents are enabled by default.
40934093
Example: inference_services: { default: { model: gpt-4o-mini, temperature: 0.7 }, custom_tool: { enabled: false }, jupyterlite: { model: gpt-4o } }
4094+
Set static_responses to a YAML file path to replace all LLM calls with
4095+
deterministic responses for testing:
4096+
inference_services: { static_responses: test/integration/static_agents.yml }
40944097
40954098
enable_tool_recommendations:
40964099
type: bool

lib/galaxy/managers/configuration.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,13 @@
2020
log = logging.getLogger(__name__)
2121

2222

23+
def _get_registry_type(config) -> str:
24+
inference_config = getattr(config, "inference_services", None) or {}
25+
if isinstance(inference_config, dict) and inference_config.get("static_responses"):
26+
return "static"
27+
return "default"
28+
29+
2330
class ConfigurationManager:
2431
"""Interface/service object for interacting with configuration and related data."""
2532

@@ -232,6 +239,7 @@ def _config_is_truthy(item, key, **context):
232239
"llm_api_configured": lambda item, key, **context: bool(
233240
item.ai_api_key or item.ai_api_base_url or getattr(item, "inference_services", None)
234241
),
242+
"llm_registry_type": lambda item, key, **context: _get_registry_type(item),
235243
"install_tool_dependencies": _use_config,
236244
"install_repository_dependencies": _use_config,
237245
"install_resolver_dependencies": _use_config,

0 commit comments

Comments
 (0)