Skip to content

Commit d12c132

Browse files
johnteeejohnteee
committed
Split teaagent/llm_conformance.py into a package with separated types and runner
- Extract CheckResult, ConformanceTier, ModelConformanceReport, ModelConformanceResult, TieredConformanceReport, and TieredConformanceResult dataclasses into _types.py - Rename llm_conformance.py → _runner.py, importing types from the sibling module - Add __init__.py re-exporting all public symbols for backward-compatible imports - Fix .gitignore pattern from */__pycache__ to **/__pycache__ for nested cache directories
1 parent f4dfda3 commit d12c132

5 files changed

Lines changed: 174 additions & 141 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
.DS_Store
2-
*/__pycache__
2+
**/__pycache__
33
.venv/
44
.teaagent/
55
.pytest_cache/

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ All notable changes to TeaAgent are tracked here.
1010
- Split `teaagent/code_mode.py` into a `teaagent/code_mode/` package with focused modules: `_types.py`, `_validation.py`, `_child_process.py`, and `_container.py`.
1111
- Split `teaagent/cli/_handlers.py` into a `teaagent/cli/_handlers/` package and extracted agent-run lifecycle logic into `_agent.py` while preserving command handler imports.
1212
- Continued splitting `teaagent/cli/_handlers/` by moving doctor, memory, model, MCP, misc, and audit handlers into dedicated modules and keeping stable re-exports in `__init__.py`.
13+
- Split `teaagent/llm_conformance.py` into a `teaagent/llm_conformance/` package with `_types.py` and `_runner.py`, preserving existing imports.
1314
- Made `teaagent.cli.main()` accept injectable `_adapter_factory`, `_serve_mcp_http`, `_check_graphqlite`, `_check_llm`, and `_run_model_conformance` keyword arguments, enabling handler extraction without breaking existing tests.
1415
- Split `teaagent/workspace_tools.py` into a `teaagent/workspace_tools/` package with four focused modules: `_config.py`, `_helpers.py`, `_shell.py`, `_files.py`. Backward-compatible public imports preserved via `__init__.py` re-exports.
1516
- Expanded audit string redaction with patterns for JWT tokens, AWS access keys (`AKIA...`), and GitHub personal access tokens (`ghp_...`, `github_pat_...`).

‎teaagent/llm_conformance/__init__.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
from ._runner import (
2+
AdapterFactory,
3+
ConfigurationChecker,
4+
run_model_conformance,
5+
run_tiered_conformance,
6+
)
7+
from ._types import (
8+
CheckResult,
9+
ConformanceTier,
10+
ModelConformanceReport,
11+
ModelConformanceResult,
12+
TieredConformanceReport,
13+
TieredConformanceResult,
14+
)
15+
16+
__all__ = [
17+
'AdapterFactory',
18+
'CheckResult',
19+
'ConfigurationChecker',
20+
'ConformanceTier',
21+
'ModelConformanceReport',
22+
'ModelConformanceResult',
23+
'TieredConformanceReport',
24+
'TieredConformanceResult',
25+
'run_model_conformance',
26+
'run_tiered_conformance',
27+
]

teaagent/llm_conformance.py renamed to teaagent/llm_conformance/_runner.py

Lines changed: 12 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
from __future__ import annotations
22

3-
from dataclasses import dataclass, field
4-
from enum import Enum
53
from typing import Callable, Iterable
64

75
from teaagent.llm import (
@@ -12,135 +10,14 @@
1210
create_llm_adapter,
1311
)
1412

15-
16-
class ConformanceTier(str, Enum):
17-
SMOKE = 'smoke'
18-
CONTRACT = 'contract'
19-
20-
21-
@dataclass(frozen=True)
22-
class CheckResult:
23-
name: str
24-
status: str
25-
detail: str = ''
26-
27-
28-
@dataclass(frozen=True)
29-
class TieredConformanceResult:
30-
provider: str
31-
tier: str
32-
status: str
33-
checks: list[CheckResult] = field(default_factory=list)
34-
model: str | None = None
35-
error: str | None = None
36-
37-
def as_dict(self) -> dict[str, object]:
38-
payload: dict[str, object] = {
39-
'provider': self.provider,
40-
'tier': self.tier,
41-
'status': self.status,
42-
'checks': [
43-
{'name': c.name, 'status': c.status, 'detail': c.detail}
44-
for c in self.checks
45-
],
46-
}
47-
if self.model is not None:
48-
payload['model'] = self.model
49-
if self.error:
50-
payload['error'] = self.error
51-
return payload
52-
53-
54-
@dataclass(frozen=True)
55-
class TieredConformanceReport:
56-
tier: str
57-
results: list[TieredConformanceResult]
58-
59-
@property
60-
def passed(self) -> int:
61-
return sum(1 for r in self.results if r.status == 'passed')
62-
63-
@property
64-
def failed(self) -> int:
65-
return sum(1 for r in self.results if r.status == 'failed')
66-
67-
@property
68-
def skipped(self) -> int:
69-
return sum(1 for r in self.results if r.status == 'skipped')
70-
71-
@property
72-
def ok(self) -> bool:
73-
return self.failed == 0
74-
75-
def as_dict(self) -> dict[str, object]:
76-
return {
77-
'tier': self.tier,
78-
'ok': self.ok,
79-
'passed': self.passed,
80-
'failed': self.failed,
81-
'skipped': self.skipped,
82-
'results': [r.as_dict() for r in self.results],
83-
}
84-
85-
86-
@dataclass(frozen=True)
87-
class ModelConformanceResult:
88-
provider: str
89-
status: str
90-
model: str | None = None
91-
content: str = ''
92-
error: str | None = None
93-
input_tokens: int = 0
94-
output_tokens: int = 0
95-
estimated_cost_cents: float = 0.0
96-
97-
def as_dict(self) -> dict[str, object]:
98-
payload: dict[str, object] = {
99-
'provider': self.provider,
100-
'status': self.status,
101-
}
102-
if self.model is not None:
103-
payload['model'] = self.model
104-
if self.content:
105-
payload['content'] = self.content
106-
if self.error:
107-
payload['error'] = self.error
108-
if self.status == 'passed':
109-
payload['input_tokens'] = self.input_tokens
110-
payload['output_tokens'] = self.output_tokens
111-
payload['estimated_cost_cents'] = self.estimated_cost_cents
112-
return payload
113-
114-
115-
@dataclass(frozen=True)
116-
class ModelConformanceReport:
117-
results: list[ModelConformanceResult]
118-
119-
@property
120-
def passed(self) -> int:
121-
return sum(1 for result in self.results if result.status == 'passed')
122-
123-
@property
124-
def failed(self) -> int:
125-
return sum(1 for result in self.results if result.status == 'failed')
126-
127-
@property
128-
def skipped(self) -> int:
129-
return sum(1 for result in self.results if result.status == 'skipped')
130-
131-
@property
132-
def ok(self) -> bool:
133-
return self.failed == 0
134-
135-
def as_dict(self) -> dict[str, object]:
136-
return {
137-
'ok': self.ok,
138-
'passed': self.passed,
139-
'failed': self.failed,
140-
'skipped': self.skipped,
141-
'results': [result.as_dict() for result in self.results],
142-
}
143-
13+
from ._types import (
14+
CheckResult,
15+
ConformanceTier,
16+
ModelConformanceReport,
17+
ModelConformanceResult,
18+
TieredConformanceReport,
19+
TieredConformanceResult,
20+
)
14421

14522
AdapterFactory = Callable[..., object]
14623
ConfigurationChecker = Callable[[str], tuple[bool, str]]
@@ -266,13 +143,11 @@ def _run_tiered_provider(
266143
provider=provider, tier=tier.value, status='skipped', error=message
267144
)
268145

146+
checks: list[CheckResult] = []
147+
resolved_model: str | None = None
269148
try:
270149
adapter = adapter_factory(provider, model=model)
271-
resolved_model = None
272-
273-
checks: list[CheckResult] = []
274150

275-
# --- smoke: non-empty response ---
276151
smoke_response = adapter.complete( # type: ignore[attr-defined]
277152
LLMRequest(
278153
messages=[LLMMessage(role='user', content='Reply with exactly: ok')],
@@ -299,7 +174,6 @@ def _run_tiered_provider(
299174
)
300175

301176
if tier == ConformanceTier.CONTRACT:
302-
# --- contract: exact content match ---
303177
if smoke_content == 'ok':
304178
checks.append(CheckResult(name='exact_content_match', status='passed'))
305179
else:
@@ -311,7 +185,6 @@ def _run_tiered_provider(
311185
)
312186
)
313187

314-
# --- contract: system prompt adherence ---
315188
sys_response = adapter.complete( # type: ignore[attr-defined]
316189
LLMRequest(
317190
messages=[LLMMessage(role='user', content='Say hello briefly.')],
@@ -333,7 +206,6 @@ def _run_tiered_provider(
333206
)
334207
)
335208

336-
# --- contract: token budget reported ---
337209
if smoke_response.input_tokens > 0 and smoke_response.output_tokens > 0:
338210
checks.append(
339211
CheckResult(name='token_budget_reported', status='passed')
@@ -352,8 +224,8 @@ def _run_tiered_provider(
352224
provider=provider,
353225
tier=tier.value,
354226
status='failed',
355-
checks=checks if 'checks' in dir() else [],
356-
model=resolved_model if 'resolved_model' in dir() else None,
227+
checks=checks,
228+
model=resolved_model,
357229
error=f'{type(exc).__name__}: {exc}',
358230
)
359231

‎teaagent/llm_conformance/_types.py‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
from __future__ import annotations
2+
3+
from dataclasses import dataclass, field
4+
from enum import Enum
5+
6+
7+
class ConformanceTier(str, Enum):
8+
SMOKE = 'smoke'
9+
CONTRACT = 'contract'
10+
11+
12+
@dataclass(frozen=True)
13+
class CheckResult:
14+
name: str
15+
status: str
16+
detail: str = ''
17+
18+
19+
@dataclass(frozen=True)
20+
class TieredConformanceResult:
21+
provider: str
22+
tier: str
23+
status: str
24+
checks: list[CheckResult] = field(default_factory=list)
25+
model: str | None = None
26+
error: str | None = None
27+
28+
def as_dict(self) -> dict[str, object]:
29+
payload: dict[str, object] = {
30+
'provider': self.provider,
31+
'tier': self.tier,
32+
'status': self.status,
33+
'checks': [
34+
{'name': c.name, 'status': c.status, 'detail': c.detail}
35+
for c in self.checks
36+
],
37+
}
38+
if self.model is not None:
39+
payload['model'] = self.model
40+
if self.error:
41+
payload['error'] = self.error
42+
return payload
43+
44+
45+
@dataclass(frozen=True)
46+
class TieredConformanceReport:
47+
tier: str
48+
results: list[TieredConformanceResult]
49+
50+
@property
51+
def passed(self) -> int:
52+
return sum(1 for r in self.results if r.status == 'passed')
53+
54+
@property
55+
def failed(self) -> int:
56+
return sum(1 for r in self.results if r.status == 'failed')
57+
58+
@property
59+
def skipped(self) -> int:
60+
return sum(1 for r in self.results if r.status == 'skipped')
61+
62+
@property
63+
def ok(self) -> bool:
64+
return self.failed == 0
65+
66+
def as_dict(self) -> dict[str, object]:
67+
return {
68+
'tier': self.tier,
69+
'ok': self.ok,
70+
'passed': self.passed,
71+
'failed': self.failed,
72+
'skipped': self.skipped,
73+
'results': [r.as_dict() for r in self.results],
74+
}
75+
76+
77+
@dataclass(frozen=True)
78+
class ModelConformanceResult:
79+
provider: str
80+
status: str
81+
model: str | None = None
82+
content: str = ''
83+
error: str | None = None
84+
input_tokens: int = 0
85+
output_tokens: int = 0
86+
estimated_cost_cents: float = 0.0
87+
88+
def as_dict(self) -> dict[str, object]:
89+
payload: dict[str, object] = {
90+
'provider': self.provider,
91+
'status': self.status,
92+
}
93+
if self.model is not None:
94+
payload['model'] = self.model
95+
if self.content:
96+
payload['content'] = self.content
97+
if self.error:
98+
payload['error'] = self.error
99+
if self.status == 'passed':
100+
payload['input_tokens'] = self.input_tokens
101+
payload['output_tokens'] = self.output_tokens
102+
payload['estimated_cost_cents'] = self.estimated_cost_cents
103+
return payload
104+
105+
106+
@dataclass(frozen=True)
107+
class ModelConformanceReport:
108+
results: list[ModelConformanceResult]
109+
110+
@property
111+
def passed(self) -> int:
112+
return sum(1 for result in self.results if result.status == 'passed')
113+
114+
@property
115+
def failed(self) -> int:
116+
return sum(1 for result in self.results if result.status == 'failed')
117+
118+
@property
119+
def skipped(self) -> int:
120+
return sum(1 for result in self.results if result.status == 'skipped')
121+
122+
@property
123+
def ok(self) -> bool:
124+
return self.failed == 0
125+
126+
def as_dict(self) -> dict[str, object]:
127+
return {
128+
'ok': self.ok,
129+
'passed': self.passed,
130+
'failed': self.failed,
131+
'skipped': self.skipped,
132+
'results': [result.as_dict() for result in self.results],
133+
}

0 commit comments

Comments
 (0)