Skip to content

Commit 3ff7f47

Browse files
committed
Add hosted-provider smoke env gate and acceptance test
1 parent 213770f commit 3ff7f47

4 files changed

Lines changed: 84 additions & 1 deletion

File tree

docs/acceptance.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@ TeaAgent acceptance tests live under `tests/acceptance/` and verify user-facing
1414
- Long-running worker flow: background worker start, list, show, log tail, and stop lifecycle.
1515
- Workspace edit flow: hash-anchored read/edit, git status, test command execution, git diff inspection, and final diff summary.
1616
- Live provider conformance flow: provider checks are skipped unless an explicit environment gate is set, preventing accidental live API calls in CI.
17+
- Hosted-provider smoke flow: `model smoke --live-env-var` skips live adapter calls unless CI explicitly sets the gate to `1`.
1718

1819
## Next User Stories
1920

20-
- Expand acceptance coverage for external hosted-provider runs when CI secrets are intentionally provisioned.
21+
- Define the next product-facing acceptance story beyond safe provider gates, such as release packaging smoke or IDE command flow.

teaagent/cli/_handlers/_model.py

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import argparse
44
import json
5+
import os
56
from typing import Any
67

78
from teaagent.llm import LLMMessage, LLMRequest, available_providers
@@ -14,6 +15,16 @@ def model_providers(_args: argparse.Namespace) -> int:
1415

1516

1617
def model_smoke(args: argparse.Namespace) -> int:
18+
live_env_var = getattr(args, 'live_env_var', None)
19+
if live_env_var is not None and os.environ.get(live_env_var) != '1':
20+
print_json(
21+
{
22+
'provider': args.provider,
23+
'skipped': True,
24+
'reason': f'gated by env {live_env_var}=1',
25+
}
26+
)
27+
return 0
1728
adapter = args._adapter_factory(args.provider, model=args.model) # type: ignore[attr-defined]
1829
response = adapter.complete(
1930
LLMRequest(

teaagent/cli/_model_parsers.py

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ def register(
2727
smoke.add_argument(
2828
'--max-tokens', type=int, default=32, help='Maximum output tokens.'
2929
)
30+
smoke.add_argument(
31+
'--live-env-var',
32+
default=None,
33+
help='Skip smoke call unless this environment variable is set to 1.',
34+
)
3035
smoke.set_defaults(func=handlers['smoke'])
3136

3237
conformance = subs.add_parser(
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
from __future__ import annotations
2+
3+
import io
4+
import json
5+
import os
6+
import unittest
7+
from contextlib import redirect_stdout
8+
from unittest.mock import patch
9+
10+
from conftest import FakeAdapter
11+
12+
from teaagent.cli import main
13+
14+
15+
class ModelSmokeGatingFlowAcceptanceTests(unittest.TestCase):
16+
def test_smoke_skips_without_required_env_and_does_not_call_provider(self) -> None:
17+
def fail_factory(provider: str, model: str | None = None) -> object:
18+
raise AssertionError(f'provider should not be called: {provider}')
19+
20+
output = io.StringIO()
21+
with redirect_stdout(output):
22+
exit_code = main(
23+
[
24+
'model',
25+
'smoke',
26+
'gpt',
27+
'--live-env-var',
28+
'TEAAGENT_ACCEPT_HOSTED_SMOKE',
29+
],
30+
_adapter_factory=fail_factory,
31+
)
32+
payload = json.loads(output.getvalue())
33+
34+
self.assertEqual(exit_code, 0)
35+
self.assertTrue(payload['skipped'])
36+
self.assertEqual(payload['provider'], 'gpt')
37+
self.assertIn('TEAAGENT_ACCEPT_HOSTED_SMOKE', payload['reason'])
38+
39+
def test_smoke_runs_when_required_env_is_set(self) -> None:
40+
adapter = FakeAdapter(['ok'])
41+
output = io.StringIO()
42+
with (
43+
patch.dict(os.environ, {'TEAAGENT_ACCEPT_HOSTED_SMOKE': '1'}),
44+
redirect_stdout(output),
45+
):
46+
exit_code = main(
47+
[
48+
'model',
49+
'smoke',
50+
'gpt',
51+
'--live-env-var',
52+
'TEAAGENT_ACCEPT_HOSTED_SMOKE',
53+
],
54+
_adapter_factory=lambda _provider, model=None: adapter,
55+
)
56+
payload = json.loads(output.getvalue())
57+
58+
self.assertEqual(exit_code, 0)
59+
self.assertEqual(payload['provider'], 'fake')
60+
self.assertEqual(payload['model'], 'fake-model')
61+
self.assertEqual(payload['content'], 'ok')
62+
self.assertNotIn('skipped', payload)
63+
64+
65+
if __name__ == '__main__':
66+
unittest.main()

0 commit comments

Comments
 (0)