|
7 | 7 | to detect that drift by measuring Ghost Consistency Score (CCS): the fraction |
8 | 8 | of vocabulary from the earliest turns still present in the most recent turns. |
9 | 9 |
|
| 10 | +The integration path here stays on the public AgentChat surface: |
| 11 | +collect `TaskResult.messages` from `AssistantAgent.run()` / `run_stream()`, |
| 12 | +append them to an external history list, and score that history after each run. |
| 13 | +
|
10 | 14 | Usage: |
11 | 15 | python main.py |
12 | 16 |
|
13 | 17 | Requires: |
14 | 18 | pip install autogen-agentchat |
15 | 19 |
|
16 | | -Optional (for richer semantic distance): |
17 | | - pip install sentence-transformers |
| 20 | + Optional: |
| 21 | + Use with `AssistantAgent.run()` / `run_stream()` by passing the |
| 22 | + accumulated `TaskResult.messages` history into `observe_result()`. |
18 | 23 | """ |
19 | 24 |
|
20 | 25 | from __future__ import annotations |
21 | 26 |
|
22 | 27 | import re |
23 | | -import sys |
24 | 28 | from collections import Counter |
25 | 29 | from typing import Any, Dict, List, Optional, Sequence |
26 | 30 |
|
@@ -155,40 +159,15 @@ def check(self, messages: Sequence[Any]) -> Dict[str, Any]: |
155 | 159 |
|
156 | 160 | return result |
157 | 161 |
|
158 | | - def patch_agent(self, agent: Any) -> None: |
| 162 | + def observe_result(self, history: List[Any], task_result: Any) -> Dict[str, Any]: |
159 | 163 | """ |
160 | | - Monkey-patch an AssistantAgent so that every reply automatically |
161 | | - runs a consistency check. Prints a warning when drift is detected. |
| 164 | + Append one public AgentChat task result into `history`, then score it. |
| 165 | +
|
| 166 | + This works with the current public `AssistantAgent.run()` / |
| 167 | + `run_stream()` API instead of monkey-patching private internals. |
162 | 168 | """ |
163 | | - original = agent.generate_reply |
164 | | - monitor = self |
165 | | - |
166 | | - def _patched(*args, **kwargs): |
167 | | - reply = original(*args, **kwargs) |
168 | | - msgs = [] |
169 | | - for attr in ("chat_messages", "_oai_messages", "messages"): |
170 | | - candidate = getattr(agent, attr, None) |
171 | | - if candidate is None: |
172 | | - continue |
173 | | - if isinstance(candidate, dict): |
174 | | - for v in candidate.values(): |
175 | | - msgs = list(v) |
176 | | - break |
177 | | - elif isinstance(candidate, list): |
178 | | - msgs = list(candidate) |
179 | | - if msgs: |
180 | | - break |
181 | | - if msgs: |
182 | | - r = monitor.check(msgs) |
183 | | - if r["drift_detected"]: |
184 | | - print( |
185 | | - f"[BehavioralMonitor] \u26a0 drift at turn {r['turn']}: " |
186 | | - f"CCS={r['ccs']:.3f}, ghost={r['ghost_terms']}", |
187 | | - file=sys.stderr, |
188 | | - ) |
189 | | - return reply |
190 | | - |
191 | | - agent.generate_reply = _patched |
| 169 | + history.extend(getattr(task_result, "messages", [])) |
| 170 | + return self.check(history) |
192 | 171 |
|
193 | 172 |
|
194 | 173 | # --------------------------------------------------------------------------- |
|
0 commit comments