Skip to content

Commit efb01ce

Browse files
abossardCopilot
andcommitted
Add reasoning_effort config + new tools for major speed improvement
Performance: - reasoning_effort='low' as default — reduces gpt-5-nano from 512 reasoning tokens (~7s) to 0-192 tokens (~1-3s) per LLM call - Configurable per agent: low (fast), medium, high (deep), default - Both agent_builder and legacy chat agent use reasoning_effort='low' New tools: - csv_count_tickets: count matching tickets WITHOUT returning data. Lets the LLM check 'how many VPN tickets?' (~50 tokens) before deciding to fetch details (~5000 tokens) - csv_search_tickets_with_details: search + return full details (notes, resolution, description) in ONE call. Eliminates the N × csv_get_ticket drill-down pattern that caused the 'Ticket Knowledgebase Creator' to make 5+ sequential LLM calls Impact on 'Ticket Knowledgebase Creator' agent: Before: search(compact) → get_ticket × N → generate = 5+ LLM calls × ~5s = 25s+ After: search_with_details(query, limit=10) → generate = 2 LLM calls × ~2s = 4s Also fixed: removed stale response_format: json_object from build_llm (was causing strict tool errors). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 2505233 commit efb01ce

8 files changed

Lines changed: 152 additions & 48 deletions

File tree

backend/agent_builder/engine/react_runner.py

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,19 +32,21 @@ def build_llm(
3232
base_url: str = "",
3333
temperature: float = 0.0,
3434
max_tokens: int = 0,
35+
reasoning_effort: str = "low",
3536
) -> Any:
36-
"""Construct a ChatOpenAI instance with JSON output mode."""
37+
"""Construct a ChatOpenAI instance with configurable reasoning effort."""
3738
from langchain_openai import ChatOpenAI
3839

3940
kwargs: dict[str, Any] = {
4041
"model": model,
4142
"api_key": api_key,
4243
"base_url": base_url or None,
4344
"temperature": temperature,
44-
"model_kwargs": {"response_format": {"type": "json_object"}},
4545
}
4646
if max_tokens > 0:
4747
kwargs["max_tokens"] = max_tokens
48+
if reasoning_effort and reasoning_effort != "default":
49+
kwargs["reasoning_effort"] = reasoning_effort
4850
return ChatOpenAI(**kwargs)
4951

5052

backend/agent_builder/models/agent.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ class AgentDefinition(SQLModel, table=True):
4949
default=4096,
5050
description="Max LLM response tokens",
5151
)
52+
reasoning_effort: str = SField(
53+
default="low",
54+
description="Reasoning effort: low (fast), medium, high (deep thinking), default (model default)",
55+
)
5256
output_instructions: str = SField(
5357
default="",
5458
description="Custom output format instructions (empty = default markdown)",
@@ -134,6 +138,7 @@ def to_dict(self) -> dict[str, Any]:
134138
"temperature": self.temperature,
135139
"recursion_limit": self.recursion_limit,
136140
"max_tokens": self.max_tokens,
141+
"reasoning_effort": self.reasoning_effort,
137142
"output_instructions": self.output_instructions,
138143
"output_schema": self.output_schema,
139144
"show_in_menu": self.show_in_menu,
@@ -154,6 +159,7 @@ class AgentDefinitionCreate(BaseModel):
154159
temperature: float = Field(default=0.0, ge=0.0, le=2.0, description="LLM temperature")
155160
recursion_limit: int = Field(default=3, ge=1, le=100, description="Max ReAct iterations")
156161
max_tokens: int = Field(default=4096, ge=0, description="Max response tokens")
162+
reasoning_effort: str = Field(default="low", description="Reasoning effort: low, medium, high, default")
157163
output_instructions: str = Field(default="", description="Custom output format instructions")
158164
output_schema: dict[str, Any] = Field(
159165
default_factory=dict,
@@ -174,6 +180,7 @@ class AgentDefinitionUpdate(BaseModel):
174180
temperature: Optional[float] = Field(default=None, ge=0.0, le=2.0)
175181
recursion_limit: Optional[int] = Field(default=None, ge=1, le=100)
176182
max_tokens: Optional[int] = Field(default=None, ge=0)
183+
reasoning_effort: Optional[str] = Field(default=None)
177184
output_instructions: Optional[str] = Field(default=None)
178185
output_schema: Optional[dict[str, Any]] = Field(default=None)
179186
tool_names: Optional[list[str]] = Field(default=None)

backend/agent_builder/persistence/database.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ def _run_migrations(engine) -> None:
3131
_ensure_column(engine, "workbench_agent_definitions", "output_instructions", "TEXT NOT NULL DEFAULT ''")
3232
_ensure_column(engine, "workbench_agent_definitions", "output_schema", "TEXT NOT NULL DEFAULT '{}'")
3333
_ensure_column(engine, "workbench_agent_definitions", "show_in_menu", "BOOLEAN NOT NULL DEFAULT 0")
34+
_ensure_column(engine, "workbench_agent_definitions", "reasoning_effort", "TEXT NOT NULL DEFAULT 'low'")
3435

3536
def _ensure_column(engine, table_name: str, column_name: str, column_ddl: str) -> None:
3637
with Session(engine) as session:

backend/agent_builder/service.py

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -77,25 +77,18 @@ def llm(self) -> Any:
7777
"OPENAI_API_KEY is required to run agents. "
7878
"Set it via environment variable or pass openai_api_key."
7979
)
80-
self._llm = build_llm(self._model, self._api_key, self._base_url)
80+
self._llm = build_llm(self._model, self._api_key, self._base_url, reasoning_effort="low")
8181
return self._llm
8282

8383
def _resolve_llm_for_agent(self, agent_def: "AgentDefinition") -> Any:
8484
"""Build an LLM instance using per-agent overrides or service defaults."""
85-
agent_model = agent_def.model.strip() if agent_def.model else ""
86-
uses_defaults = (
87-
not agent_model
88-
and agent_def.temperature == 0.0
89-
and agent_def.max_tokens == 4096
90-
)
91-
if uses_defaults:
92-
return self.llm
9385
return build_llm(
94-
model=agent_model or self._model,
86+
model=(agent_def.model.strip() or self._model),
9587
api_key=self._api_key,
9688
base_url=self._base_url,
9789
temperature=agent_def.temperature,
9890
max_tokens=agent_def.max_tokens,
91+
reasoning_effort=agent_def.reasoning_effort or "low",
9992
)
10093

10194
# ------------------------------------------------------------------
@@ -287,6 +280,7 @@ def _build_agent_snapshot(self, agent: AgentDefinition) -> dict[str, Any]:
287280
"temperature": agent.temperature,
288281
"recursion_limit": agent.recursion_limit,
289282
"max_tokens": agent.max_tokens,
283+
"reasoning_effort": agent.reasoning_effort,
290284
"output_instructions": agent.output_instructions,
291285
"output_schema": agent.output_schema,
292286
"tool_names": list(agent.tool_names),
@@ -355,6 +349,7 @@ def create_agent(self, data: AgentDefinitionCreate) -> AgentDefinition:
355349
temperature=data.temperature,
356350
recursion_limit=data.recursion_limit,
357351
max_tokens=data.max_tokens,
352+
reasoning_effort=data.reasoning_effort,
358353
output_instructions=data.output_instructions,
359354
show_in_menu=data.show_in_menu,
360355
)
@@ -402,6 +397,8 @@ def update_agent(
402397
agent.recursion_limit = data.recursion_limit
403398
if data.max_tokens is not None:
404399
agent.max_tokens = data.max_tokens
400+
if data.reasoning_effort is not None:
401+
agent.reasoning_effort = data.reasoning_effort
405402
if data.output_instructions is not None:
406403
agent.output_instructions = data.output_instructions
407404
if data.output_schema is not None:

backend/agents.py

Lines changed: 50 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -199,21 +199,19 @@ def __init__(self):
199199
"Please set OPENAI_API_KEY environment variable."
200200
)
201201

202-
# Initialize ChatOpenAI with JSON output mode
202+
# Initialize ChatOpenAI with low reasoning effort for speed
203203
self.llm = ChatOpenAI(
204204
model=OPENAI_MODEL,
205205
api_key=OPENAI_API_KEY,
206206
base_url=OPENAI_BASE_URL or None,
207207
temperature=0.0,
208-
model_kwargs={"response_format": {"type": "json_object"}},
208+
reasoning_effort="low",
209209
)
210210

211211
# CSV tools only
212212
self.tools = self._build_csv_tools()
213213
self._system_prompt = self._build_system_prompt()
214-
# Pre-bind tools with strict=True for JSON mode compatibility
215-
model_with_tools = self.llm.bind_tools(self.tools, strict=True) if self.tools else self.llm
216-
self._react_agent = create_react_agent(model_with_tools, self.tools)
214+
self._react_agent = create_react_agent(self.llm, self.tools)
217215

218216
def _build_system_prompt(self) -> str:
219217
"""Build a concise system prompt optimized for low-latency tool usage."""
@@ -334,6 +332,40 @@ def _csv_ticket_stats() -> str:
334332
stats = service.get_ticket_stats()
335333
return json.dumps(stats, default=str)
336334

335+
def _csv_count_tickets(query: str = "", status: str | None = None) -> str:
336+
"""Count matching tickets without returning data. Fast check before fetching details."""
337+
tickets = service.list_tickets()
338+
if status:
339+
try:
340+
status_enum = TicketStatus(status.lower())
341+
tickets = [t for t in tickets if t.status == status_enum]
342+
except Exception:
343+
pass
344+
if query.strip():
345+
q = query.strip().lower()
346+
tickets = [t for t in tickets if q in " ".join([
347+
t.summary or "", t.description or "", t.notes or "",
348+
t.requester_name or "", t.assigned_group or "",
349+
]).lower()]
350+
return json.dumps({"count": len(tickets), "query": query})
351+
352+
def _csv_search_with_details(query: str, limit: int = 10) -> str:
353+
"""Search tickets with full details (notes, resolution, description) in one call."""
354+
q = query.lower()
355+
detail_fields = compact_default_fields + ["notes", "resolution", "description", "incident_id"]
356+
matched = []
357+
for t in service.list_tickets():
358+
text = " ".join([
359+
t.summary or "", t.description or "", t.notes or "",
360+
t.requester_name or "", t.assigned_group or "", t.city or "",
361+
]).lower()
362+
if q in text:
363+
dump = t.model_dump()
364+
matched.append({k: v for k, v in dump.items() if k in detail_fields})
365+
if len(matched) >= min(max(limit, 1), 25):
366+
break
367+
return json.dumps(matched, default=str)
368+
337369
return [
338370
StructuredTool.from_function(
339371
func=_csv_list_tickets,
@@ -381,6 +413,19 @@ def _csv_ticket_stats() -> str:
381413
name="csv_ticket_stats",
382414
description="Get aggregated statistics: total, by_status, by_priority, by_group, by_city.",
383415
),
416+
StructuredTool.from_function(
417+
func=_csv_count_tickets,
418+
name="csv_count_tickets",
419+
description="Count matching tickets WITHOUT returning data. Use to check result size before fetching details. Fast and cheap.",
420+
),
421+
StructuredTool.from_function(
422+
func=_csv_search_with_details,
423+
name="csv_search_tickets_with_details",
424+
description=(
425+
"Search tickets AND return full details (notes, resolution, description) in one call. "
426+
"Use for knowledgebase, analysis, or detailed reports. Limit defaults to 10, max 25."
427+
),
428+
),
384429
]
385430

386431
async def run_agent(self, request: AgentRequest) -> AgentResponse:

backend/operations.py

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,58 @@ async def op_csv_search_tickets(query: str, limit: int = 25) -> list[dict]:
277277
return _compact_tickets(matches)
278278

279279

280+
# Fields included in detailed search results (compact + notes/resolution for knowledgebase use)
281+
_DETAIL_TICKET_FIELDS = _COMPACT_TICKET_FIELDS | {"notes", "resolution", "description"}
282+
283+
284+
@operation(
285+
name="csv_count_tickets",
286+
description="Count how many tickets match a search query WITHOUT returning the data. Use this to check result size before fetching details. Fast and cheap.",
287+
http_method="GET",
288+
)
289+
async def op_csv_count_tickets(query: str = "", status: str | None = None) -> dict:
290+
"""Count matching tickets without returning them."""
291+
_ensure_csv_loaded()
292+
parsed_status = _parse_status(status)
293+
tickets = _csv_service.list_tickets(status=parsed_status)
294+
if query.strip():
295+
q = query.strip().lower()
296+
tickets = [t for t in tickets if q in " ".join([
297+
t.incident_id or "", t.summary or "", t.description or "",
298+
t.notes or "", t.requester_name or "", t.assigned_group or "", t.city or "",
299+
]).lower()]
300+
return {"count": len(tickets), "query": query, "status": status}
301+
302+
303+
@operation(
304+
name="csv_search_tickets_with_details",
305+
description="Search tickets AND return full details (notes, resolution, description) in one call. Use this when you need ticket content for analysis, knowledgebase articles, or detailed reports. Limit defaults to 10 to keep responses fast.",
306+
http_method="GET",
307+
)
308+
async def op_csv_search_tickets_with_details(query: str, limit: int = 10) -> list[dict]:
309+
"""Search tickets with full details — avoids needing csv_get_ticket per result."""
310+
_ensure_csv_loaded()
311+
q = query.strip().lower()
312+
if not q:
313+
return []
314+
normalized_limit = min(max(limit, 1), 25)
315+
matches: list[Ticket] = []
316+
for ticket in _csv_service.list_tickets():
317+
haystack = " ".join([
318+
ticket.incident_id or "", ticket.summary or "", ticket.description or "",
319+
ticket.notes or "", ticket.resolution or "", ticket.requester_name or "",
320+
ticket.assigned_group or "", ticket.city or "", ticket.service or "",
321+
]).lower()
322+
if q in haystack:
323+
matches.append(ticket)
324+
if len(matches) >= normalized_limit:
325+
break
326+
return [
327+
{k: v for k, v in t.model_dump(mode="json").items() if k in _DETAIL_TICKET_FIELDS}
328+
for t in matches
329+
]
330+
331+
280332
@operation(
281333
name="csv_ticket_stats",
282334
description="Get aggregated statistics for tickets loaded from CSV",

docs/ticker_reminder/FALL_1_SOFTWARE_UPDATE.md

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717

1818
| Feld | Wert |
1919
|------|------|
20-
| **Incident ID** | INC000016323643 |
21-
| **Zusammenfassung** | CM032993 Windows Update läuft nicht durch |
20+
| **Incident ID** | INC00001632XXXX |
21+
| **Zusammenfassung** | Windows Update läuft nicht durch |
2222
| **Priorität** | Medium |
2323
| **Dringlichkeit** | 3-Medium |
2424
| **Auswirkung** | 3-Moderate/Limited |
@@ -30,29 +30,29 @@
3030

3131
| Feld | Wert |
3232
|------|------|
33-
| **Organisation** | EDI-BAG, Sektion Medizinische Leistungen |
33+
| **Organisation** | [Departement], [Abteilung] |
3434
| **Standort** | Bern |
35-
| **Gerät** | HP EliteBook X360 830 G9 (CM032993) |
35+
| **Gerät** | HP EliteBook X360 830 G9 |
3636
| **Service** | BS_Microsoft SCCM |
3737

3838
### Beschreibung (Original)
3939

40-
> Rückruf unter → +41 58 462 6987
40+
> Rückruf unter → [Telefonnummer]
4141
>
4242
> Grund der Ticketeröffnung → Störung
4343
>
4444
> Thema → Software/Applikation - Standard Software - Microsoft Windows
4545
>
46-
> Hallo Servicedesk, auf dem Notebook CM032993 von von Gunten Elisabeth schlägt das Windows Update fehl. Bitte prüfen.
46+
> Hallo Servicedesk, auf dem Notebook [Client-ID] von [Benutzername] schlägt das Windows Update fehl. Bitte prüfen.
4747
4848
### Verlauf
4949

5050
| Zeitpunkt | Status | Bearbeiter |
5151
|-----------|--------|------------|
52-
| 04.11.2025 14:31 | New | U80747730 |
53-
| 05.11.2025 14:09 | Assigned | U80799778 |
54-
| 05.11.2025 17:32 | In Progress | U80797658 (Klee Jose Juan) |
55-
| 11.11.2025 13:14 | Pending | U80797658 |
52+
| 04.11.2025 14:31 | New | Bearbeiter A |
53+
| 05.11.2025 14:09 | Assigned | Bearbeiter B |
54+
| 05.11.2025 17:32 | In Progress | Bearbeiter C |
55+
| 11.11.2025 13:14 | Pending | Bearbeiter C |
5656

5757
- **Gruppen-Transfers:** 1
5858
- **Gesamt-Transfers:** 3
@@ -98,16 +98,16 @@ Gehen Sie den Fall durch und überlegen Sie sich Antworten zu folgenden Fragen:
9898

9999
| Incident ID | Zusammenfassung |
100100
|-------------|-----------------|
101-
| INC000016300803 | Es werden keine Updates angezeigt |
102-
| INC000016318424 | Update Windows nicht erfolgt |
103-
| INC000016322181 | Windows Update |
104-
| INC000016327819 | Microsoft SCCM: Win Update schlägt fehl |
105-
| INC000016327913 | Fehler bei Software Update-Installation |
106-
| INC000016328288 | Windows 11, version 24H2 x64 2025-10 Update |
107-
| INC000016338364 | Windows 11 Update kann nicht installiert werden |
108-
| INC000016344758 | Update W11 geht nicht |
109-
| INC000016346074 | SBA-O: Update Fehler |
110-
| INC000016325638 | Software wird nicht deinstalliert |
101+
| INC...0803 | Es werden keine Updates angezeigt |
102+
| INC...8424 | Update Windows nicht erfolgt |
103+
| INC...2181 | Windows Update |
104+
| INC...7819 | Microsoft SCCM: Win Update schlägt fehl |
105+
| INC...7913 | Fehler bei Software Update-Installation |
106+
| INC...8288 | Windows 11, version 24H2 x64 2025-10 Update |
107+
| INC...8364 | Windows 11 Update kann nicht installiert werden |
108+
| INC...4758 | Update W11 geht nicht |
109+
| INC...6074 | SBA-O: Update Fehler |
110+
| INC...5638 | Software wird nicht deinstalliert |
111111

112112
---
113113

0 commit comments

Comments
 (0)