Add audit summary to run results and acceptance test suite

Introduce summarize_audit_events() that derives status, event counts,
tool names, destructive call count, and approval state from audit logs.
Include audit_summary in CLI and TUI run result payloads.  Add
acceptance test suite (tests/acceptance/) covering daily CLI read-only
workflow, CLI prompt approval resume, TUI chat/memory/progress flow,
and TUI prompt approval, gated by python3 -m pytest tests/acceptance.

Author: johnteee

README.md

@@ -181,6 +181,9 @@ teaagent agent run gpt "Summarize the tests" --permission-mode read-only
 # Run tests
 pytest
 
+# Run user-facing acceptance workflows
+python3 -m pytest tests/acceptance
+
 # Lint
 ruff check .
 ruff format --check .

docs/acceptance.md

@@ -0,0 +1,19 @@
+# Acceptance Coverage
+
+TeaAgent acceptance tests live under `tests/acceptance/` and verify user-facing workflows rather than isolated primitives.
+
+## Covered
+
+- Daily CLI read-only workflow: `agent preflight`, `agent run`, `agent show`, audit persistence, and run-level audit summary.
+- CLI prompt approval workflow: destructive call pauses, `agent resume` auto-approves the pending call id, writes the file, and reports audit summary.
+- Daily TUI workflow: chat mode, memory injection, progress streaming, agent answer persistence in session history.
+- TUI prompt approval workflow: approval prompt, destructive write, final run payload, and audit summary.
+
+## Next User Stories
+
+- Live provider conformance gated by environment variables.
+- MCP client compatibility flow with auth, session lifecycle, `tools/list`, and `tools/call`.
+- A2A federation flow with remote discovery, capability routing, delegation, failure fallback, and audit correlation.
+- Managed runtime flow that forwards task context/tools and records managed runtime audit events.
+- Long-running worker flow covering `ultrawork start`, heartbeat/status, logs, and stop.
+- Workspace edit flow covering hash reads, patch application, git status, test execution, and final diff summary.

teaagent/cli/_handlers/_agent.py

@@ -10,7 +10,7 @@
 from teaagent.model_routing import route_model
 from teaagent.policy import parse_permission_mode
 from teaagent.preflight import preflight
-from teaagent.run_store import RunStore
+from teaagent.run_store import RunStore, summarize_audit_events
 from teaagent.runner import ApprovalRequest, RunResult
 
 
@@ -152,7 +152,12 @@ def _execute_agent_task(
 
         with suppress(Exception):
             _telemetry_sink.force_flush()
-    payload = run_result_payload(result, routing=routing.to_dict() if routing else None)
+    events = store.show_run(result.run_id)
+    payload = run_result_payload(
+        result,
+        routing=routing.to_dict() if routing else None,
+        audit_summary=summarize_audit_events(events),
+    )
     if resumed_from:
         payload['resumed_from'] = resumed_from
         payload['task'] = task
@@ -165,7 +170,10 @@ def _execute_agent_task(
 
 
 def run_result_payload(
-    result: RunResult, *, routing: Optional[dict[str, Any]]
+    result: RunResult,
+    *,
+    routing: Optional[dict[str, Any]],
+    audit_summary: Optional[dict[str, Any]] = None,
 ) -> dict[str, Any]:
     payload: dict[str, Any] = {
         'run_id': result.run_id,
@@ -175,6 +183,8 @@ def run_result_payload(
         'routing': routing,
         'final_answer': result.final_answer.content if result.final_answer else None,
     }
+    if audit_summary is not None:
+        payload['audit_summary'] = audit_summary
     if 'approval' in result.metadata:
         payload['approval'] = result.metadata['approval']
     return payload

teaagent/run_store.py

@@ -206,3 +206,39 @@ def summarize(self, path: Path) -> Optional[RunSummary]:
 
 def safe_run_id(run_id: str) -> str:
     return ''.join(ch for ch in run_id if ch.isalnum() or ch in {'-', '_'}) or 'run'
+
+
+def summarize_audit_events(events: list[dict[str, Any]]) -> dict[str, Any]:
+    event_counts: dict[str, int] = {}
+    tool_names: list[str] = []
+    destructive_tool_calls = 0
+    approval_required = False
+    status = 'unknown'
+    for event in events:
+        event_type = str(event.get('event_type', ''))
+        event_counts[event_type] = event_counts.get(event_type, 0) + 1
+        payload = event.get('payload', {})
+        if not isinstance(payload, dict):
+            payload = {}
+        if event_type == 'tool_call_started':
+            tool_name = payload.get('tool_name')
+            if isinstance(tool_name, str) and tool_name not in tool_names:
+                tool_names.append(tool_name)
+            annotations = payload.get('annotations', {})
+            if isinstance(annotations, dict) and annotations.get('destructive'):
+                destructive_tool_calls += 1
+        if event_type == 'tool_call_pending_approval':
+            approval_required = True
+        if event_type == 'run_completed':
+            status = 'completed'
+        elif event_type == 'run_paused':
+            status = str(payload.get('status', 'pending_approval'))
+        elif event_type == 'run_failed':
+            status = f'failed:{payload.get("category", "system")}'
+    return {
+        'status': status,
+        'event_counts': event_counts,
+        'tool_names': tool_names,
+        'destructive_tool_calls': destructive_tool_calls,
+        'approval_required': approval_required,
+    }

teaagent/tui/__init__.py

@@ -16,7 +16,7 @@
 from teaagent.memory import MemoryCatalog
 from teaagent.model_routing import route_model
 from teaagent.policy import PermissionMode
-from teaagent.run_store import RunStore
+from teaagent.run_store import RunStore, summarize_audit_events
 from teaagent.runner import ApprovalRequest, RunResult
 from teaagent.session import ChatMessage, ChatSession, SessionStore
 
@@ -256,6 +256,7 @@ def _run_agent_task(
             initial_observations=initial_observations,
         )
         store.logger_for_result(result, audit)
+        audit_summary = summarize_audit_events(store.show_run(result.run_id))
 
         if self.chat:
             chat_session: Optional[ChatSession] = self._current_session()
@@ -275,7 +276,9 @@ def _run_agent_task(
             self.output_fn(result.final_answer.content)
         else:
             payload = self._run_result_payload(
-                result, routing=routing.to_dict() if routing else None
+                result,
+                routing=routing.to_dict() if routing else None,
+                audit_summary=audit_summary,
             )
             if initial_observations:
                 payload['replayed_observations'] = len(initial_observations)
@@ -295,7 +298,11 @@ def _approval_handler(self, request: ApprovalRequest) -> bool:
         return approved
 
     def _run_result_payload(
-        self, result: RunResult, *, routing: Optional[dict]
+        self,
+        result: RunResult,
+        *,
+        routing: Optional[dict],
+        audit_summary: Optional[dict[str, Any]] = None,
     ) -> dict:
         payload = {
             'run_id': result.run_id,
@@ -309,6 +316,8 @@ def _run_result_payload(
         }
         if 'approval' in result.metadata:
             payload['approval'] = result.metadata['approval']
+        if audit_summary is not None:
+            payload['audit_summary'] = audit_summary
         if result.error_message is not None:
             payload['error'] = result.error_message
         return payload

tests/acceptance/test_daily_cli.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import io
+import json
+import tempfile
+import unittest
+from contextlib import redirect_stdout
+from pathlib import Path
+from unittest.mock import patch
+
+from conftest import FakeAdapter
+
+from teaagent.cli import main
+
+
+class DailyCLIAcceptanceTests(unittest.TestCase):
+    def test_daily_cli_read_only_run_preflight_and_audit_summary(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / 'README.md').write_text('hello teaagent', encoding='utf-8')
+            adapter = FakeAdapter(
+                [
+                    '{"type":"tool","tool_name":"workspace_read_file","arguments":{"path":"README.md"},"call_id":"read-1"}',
+                    '{"type":"final","content":"repo summarized"}',
+                ]
+            )
+
+            preflight_out = io.StringIO()
+            with redirect_stdout(preflight_out):
+                preflight_code = main(
+                    [
+                        'agent',
+                        'preflight',
+                        'gpt',
+                        'Summarize README.md for onboarding',
+                        '--root',
+                        tmp,
+                        '--permission-mode',
+                        'read-only',
+                    ]
+                )
+            preflight_payload = json.loads(preflight_out.getvalue())
+
+            run_out = io.StringIO()
+            with (
+                patch('teaagent.cli.create_llm_adapter', return_value=adapter),
+                redirect_stdout(run_out),
+            ):
+                run_code = main(
+                    [
+                        'agent',
+                        'run',
+                        'gpt',
+                        'Summarize README.md for onboarding',
+                        '--root',
+                        tmp,
+                        '--permission-mode',
+                        'read-only',
+                    ]
+                )
+            run_payload = json.loads(run_out.getvalue())
+
+            show_out = io.StringIO()
+            with redirect_stdout(show_out):
+                show_code = main(
+                    ['agent', 'show', run_payload['run_id'], '--root', tmp]
+                )
+            events = json.loads(show_out.getvalue())
+
+            self.assertEqual(preflight_code, 0)
+            self.assertTrue(preflight_payload['ready'])
+            self.assertEqual(preflight_payload['permission_mode'], 'read-only')
+            self.assertEqual(run_code, 0)
+            self.assertEqual(run_payload['status'], 'completed')
+            self.assertEqual(run_payload['final_answer'], 'repo summarized')
+            self.assertEqual(run_payload['audit_summary']['status'], 'completed')
+            self.assertEqual(
+                run_payload['audit_summary']['tool_names'], ['workspace_read_file']
+            )
+            self.assertEqual(run_payload['audit_summary']['approval_required'], False)
+            self.assertEqual(show_code, 0)
+            self.assertIn('run_completed', [event['event_type'] for event in events])
+
+    def test_daily_cli_prompt_approval_resume_is_auditable(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            first_adapter = FakeAdapter(
+                [
+                    '{"type":"tool","tool_name":"workspace_write_file","arguments":{"path":"TODO.md","content":"done"},"call_id":"write-1"}'
+                ]
+            )
+            first_out = io.StringIO()
+            with (
+                patch('teaagent.cli.create_llm_adapter', return_value=first_adapter),
+                redirect_stdout(first_out),
+            ):
+                first_code = main(
+                    ['agent', 'run', 'gpt', 'Create TODO.md', '--root', tmp]
+                )
+            first_payload = json.loads(first_out.getvalue())
+
+            resume_adapter = FakeAdapter(
+                [
+                    '{"type":"tool","tool_name":"workspace_write_file","arguments":{"path":"TODO.md","content":"done"},"call_id":"write-1"}',
+                    '{"type":"final","content":"created todo"}',
+                ]
+            )
+            resume_out = io.StringIO()
+            with (
+                patch('teaagent.cli.create_llm_adapter', return_value=resume_adapter),
+                redirect_stdout(resume_out),
+            ):
+                resume_code = main(
+                    ['agent', 'resume', 'gpt', first_payload['run_id'], '--root', tmp]
+                )
+            resume_payload = json.loads(resume_out.getvalue())
+
+            self.assertEqual(first_code, 1)
+            self.assertEqual(first_payload['status'], 'pending_approval')
+            self.assertTrue(first_payload['audit_summary']['approval_required'])
+            self.assertEqual(first_payload['approval']['call_id'], 'write-1')
+            self.assertEqual(resume_code, 0)
+            self.assertEqual(resume_payload['status'], 'completed')
+            self.assertEqual(resume_payload['resumed_from'], first_payload['run_id'])
+            self.assertEqual(resume_payload['auto_approved_call_id'], 'write-1')
+            self.assertEqual(
+                resume_payload['audit_summary']['destructive_tool_calls'], 1
+            )
+            self.assertEqual(
+                (Path(tmp) / 'TODO.md').read_text(encoding='utf-8'), 'done'
+            )
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/acceptance/test_daily_tui.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+from conftest import FakeAdapter
+
+from teaagent.tui import TeaAgentTUI
+
+
+class DailyTUIAcceptanceTests(unittest.TestCase):
+    def test_daily_tui_chat_memory_progress_and_audit_summary(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / 'note.txt').write_text('hello', encoding='utf-8')
+            output: list[str] = []
+            adapter = FakeAdapter(
+                [
+                    '{"type":"tool","tool_name":"workspace_read_file","arguments":{"path":"note.txt"},"call_id":"read-1"}',
+                    '{"type":"final","content":"note summarized"}',
+                ]
+            )
+            tui = TeaAgentTUI(
+                root=tmp,
+                input_fn=lambda _prompt: 'exit',
+                output_fn=output.append,
+                adapter_factory=lambda _provider, _model: adapter,
+            )
+
+            self.assertTrue(tui.handle_command('chat on'))
+            self.assertTrue(
+                tui.handle_command(
+                    'memory add summarize note.txt Prefer concise summaries'
+                )
+            )
+            self.assertTrue(tui.handle_command('progress on'))
+            self.assertTrue(tui.handle_command('ask summarize note.txt'))
+            self.assertTrue(tui.handle_command('session show'))
+
+            joined = '\n'.join(output)
+            session_payload = json.loads(output[-1])
+
+            self.assertIn('chat: on', joined)
+            self.assertIn('progress: on', joined)
+            self.assertIn('tool: workspace_read_file', joined)
+            self.assertIn('note summarized', output)
+            self.assertEqual(len(session_payload['messages']), 2)
+            self.assertIn(
+                'Prefer concise summaries', adapter.requests[0].messages[0].content
+            )
+
+    def test_daily_tui_prompt_approval_is_auditable(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            output: list[str] = []
+            adapter = FakeAdapter(
+                [
+                    '{"type":"tool","tool_name":"workspace_write_file","arguments":{"path":"TODO.md","content":"done"},"call_id":"write-1"}',
+                    '{"type":"final","content":"created todo"}',
+                ]
+            )
+            tui = TeaAgentTUI(
+                root=tmp,
+                input_fn=lambda _prompt: 'yes',
+                output_fn=output.append,
+                adapter_factory=lambda _provider, _model: adapter,
+            )
+
+            self.assertTrue(tui.handle_command('ask create TODO.md'))
+            approval_payload = json.loads(output[1])
+            result_payload = json.loads(output[-1])
+
+            self.assertEqual(approval_payload['status'], 'approval_required')
+            self.assertEqual(result_payload['status'], 'completed')
+            self.assertTrue(result_payload['audit_summary']['approval_required'])
+            self.assertEqual(
+                result_payload['audit_summary']['destructive_tool_calls'], 1
+            )
+            self.assertEqual(
+                (Path(tmp) / 'TODO.md').read_text(encoding='utf-8'), 'done'
+            )
+
+
+if __name__ == '__main__':
+    unittest.main()