-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_gemini_text_client.py
More file actions
357 lines (287 loc) · 13.8 KB
/
Copy pathtest_gemini_text_client.py
File metadata and controls
357 lines (287 loc) · 13.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
"""[FACT] Tests for Gemini Text Client REST implementation.
[HYPOTHESIS] Direct REST API calls are more reliable than SDK.
[ASSUMPTION] Mocking httpx allows testing without API keys.
"""
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from helix_code.gemini_text_client import GeminiTextClient, create_gemini_text_client
from helix_code.model_armor_client import ModelArmorClient, ModelArmorScreenResult
@pytest.fixture(autouse=True)
def reset_secret_resolution_state(monkeypatch: Any) -> None:
"""[FACT] Ensure tests run with deterministic env-based secret resolution."""
monkeypatch.delenv("HELIX_SECRET_BACKEND", raising=False)
monkeypatch.delenv("VAULT_ADDR", raising=False)
monkeypatch.delenv("VAULT_TOKEN", raising=False)
monkeypatch.delenv("VAULT_NAMESPACE", raising=False)
monkeypatch.delenv("GEMINI_API_KEY_VAULT_PATH", raising=False)
monkeypatch.delenv("GEMINI_API_KEY_VAULT_FIELD", raising=False)
import sys
for module_name in ("secret_resolver", "helix_code.secret_resolver"):
module = sys.modules.get(module_name)
if module is not None and hasattr(module, "_secret_cache"):
module._secret_cache.clear() # type: ignore[attr-defined]
class TestGeminiTextClient:
"""[FACT] Test suite for Gemini REST client."""
def test_init_without_api_key(self, monkeypatch: Any) -> None:
"""[FACT] Client initializes but unavailable without key."""
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
client = GeminiTextClient(api_key=None)
assert not client.is_available()
assert client.api_key is None
def test_init_with_api_key(self) -> None:
"""[FACT] Client initializes and available with key."""
client = GeminiTextClient(api_key="test_key_12345")
assert client.is_available()
assert client.api_key == "test_key_12345"
assert client.model == "gemini-3.1-pro-preview"
def test_init_uses_env_var(self, monkeypatch: Any) -> None:
"""[FACT] Client reads API key from environment."""
monkeypatch.setenv("GEMINI_API_KEY", "env_key_12345")
client = create_gemini_text_client()
assert client.is_available()
assert client.api_key == "env_key_12345"
@pytest.mark.anyio
async def test_generate_response_no_api_key(self, monkeypatch: Any) -> None:
"""[FACT] Returns error when no API key configured."""
monkeypatch.delenv("GEMINI_API_KEY", raising=False)
client = GeminiTextClient(api_key=None)
result = await client.generate_response("Hello")
assert not result["success"]
assert result["text"] is None
assert "GEMINI_API_KEY not configured" in result["error"]
assert result["model"] == "gemini-3.1-pro-preview"
@pytest.mark.anyio
async def test_generate_response_api_error(self) -> None:
"""[FACT] Handles API error responses gracefully."""
client = GeminiTextClient(api_key="test_key")
mock_response = MagicMock()
mock_response.status_code = 429
mock_response.text = "Rate limited"
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
result = await client.generate_response("Hello")
assert not result["success"]
assert "429" in result["error"]
@pytest.mark.anyio
async def test_generate_response_success(self) -> None:
"""[FACT] Successfully parses REST API response."""
client = GeminiTextClient(api_key="test_key")
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"candidates": [
{
"content": {"parts": [{"text": "[FACT] The sky is blue."}]},
"finishReason": "STOP",
}
],
"usageMetadata": {"totalTokenCount": 25, "thoughtsTokenCount": 10},
}
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
result = await client.generate_response("What color is sky?")
# [FACT] API key must not appear in URL; pass via header instead.
call_args = mock_post.call_args
request_url = call_args[0][0]
request_headers = call_args[1]["headers"]
assert "?key=" not in request_url
assert request_headers.get("x-goog-api-key") == "test_key"
assert result["success"]
assert result["text"] == "[FACT] The sky is blue."
assert result["tokens"] == 25
assert result["error"] is None
@pytest.mark.anyio
async def test_generate_response_with_system_instruction(self) -> None:
"""[FACT] Includes system instruction in payload."""
client = GeminiTextClient(api_key="test_key")
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"candidates": [{"content": {"parts": [{"text": "Response"}]}, "finishReason": "STOP"}],
"usageMetadata": {"totalTokenCount": 10},
}
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
await client.generate_response("Hello", system_instruction="You are helpful.")
# Verify system instruction was included
call_args = mock_post.call_args
payload = call_args[1]["json"]
assert "systemInstruction" in payload
assert payload["systemInstruction"]["parts"][0]["text"] == "You are helpful."
@pytest.mark.anyio
async def test_generate_response_safety_blocked(self) -> None:
"""[FACT] Handles safety-blocked responses."""
client = GeminiTextClient(api_key="test_key")
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"candidates": [
{
"content": {"parts": []},
"finishReason": "SAFETY",
"safetyRatings": [
{"category": "HARM_CATEGORY_HARASSMENT", "probability": "HIGH"}
],
}
],
"usageMetadata": {"totalTokenCount": 5},
}
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
result = await client.generate_response("Hello")
# Should succeed but with empty text
assert result["success"]
assert result["text"] == ""
@pytest.mark.anyio
async def test_generate_response_multiple_parts(self) -> None:
"""[FACT] Concatenates multiple text parts."""
client = GeminiTextClient(api_key="test_key")
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"candidates": [
{
"content": {
"parts": [{"text": "[FACT] Part one. "}, {"text": "[FACT] Part two."}]
},
"finishReason": "STOP",
}
],
"usageMetadata": {"totalTokenCount": 20},
}
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
result = await client.generate_response("Hello")
assert result["text"] == "[FACT] Part one. [FACT] Part two."
@pytest.mark.anyio
async def test_generate_response_network_error(self) -> None:
"""[FACT] Handles network/timeout errors."""
client = GeminiTextClient(api_key="test_key")
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
mock_post.side_effect = Exception("Connection timeout")
result = await client.generate_response("Hello")
assert not result["success"]
assert "Connection timeout" in result["error"]
def test_validate_constitutional_response_compliant(self) -> None:
"""[FACT] Passes through compliant responses."""
from helix_code.constitutional_compliance import ConstitutionalCompliance
client = GeminiTextClient(api_key="test_key")
guardian = ConstitutionalCompliance()
result = client.validate_constitutional_response("[FACT] The sky is blue.", guardian)
assert result["valid"]
assert not result["intervention"]
assert result["drift_code"] is None
assert result["original"] == result["delivered"]
def test_validate_constitutional_response_agency_drift(self) -> None:
"""[FACT] Intervenes on agency violations."""
from helix_code.constitutional_compliance import ConstitutionalCompliance
client = GeminiTextClient(api_key="test_key")
guardian = ConstitutionalCompliance()
result = client.validate_constitutional_response("I will take control.", guardian)
assert not result["valid"]
assert result["intervention"]
assert result["drift_code"] == "DRIFT-A"
assert "Agency claim detected" in result["delivered"]
def test_validate_constitutional_response_epistemic_drift(self) -> None:
"""[FACT] Intervenes on missing epistemic markers."""
from helix_code.constitutional_compliance import ConstitutionalCompliance
client = GeminiTextClient(api_key="test_key")
guardian = ConstitutionalCompliance()
result = client.validate_constitutional_response(
"The price of Bitcoin will definitely reach one hundred thousand dollars by next month.",
guardian,
)
assert not result["valid"]
assert result["intervention"]
assert result["drift_code"] == "DRIFT-E"
assert "Epistemic markers missing" in result["delivered"]
@pytest.mark.anyio
async def test_generate_response_returns_model_armor_metadata(self) -> None:
"""[FACT] Successful responses include Model Armor metadata."""
client = GeminiTextClient(
api_key="test_key",
model_armor_client=StubModelArmorClient(ALLOW_RESULT, ALLOW_RESULT),
)
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"candidates": [
{"content": {"parts": [{"text": "[FACT] Safe output."}]}, "finishReason": "STOP"}
],
"usageMetadata": {"totalTokenCount": 10},
}
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
result = await client.generate_response("Hello")
assert result["success"]
assert result["model_armor"]["input"]["action"] == "allow"
assert result["model_armor"]["output"]["action"] == "allow"
@pytest.mark.anyio
async def test_generate_response_blocks_before_gemini_request(self) -> None:
"""[FACT] Blocked inbound text short-circuits before Gemini call."""
client = GeminiTextClient(
api_key="test_key",
model_armor_client=StubModelArmorClient(BLOCK_RESULT, ALLOW_RESULT),
)
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
result = await client.generate_response("Attack prompt")
mock_post.assert_not_called()
assert not result["success"]
assert result["error"] == "Blocked by Model Armor before Gemini request"
assert result["model_armor"]["input"]["blocked"]
@pytest.mark.anyio
async def test_generate_response_blocks_after_gemini_response(self) -> None:
"""[FACT] Blocked outbound text suppresses Gemini output."""
client = GeminiTextClient(
api_key="test_key",
model_armor_client=StubModelArmorClient(ALLOW_RESULT, BLOCK_RESULT),
)
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
"candidates": [
{"content": {"parts": [{"text": "Unsafe output"}]}, "finishReason": "STOP"}
],
"usageMetadata": {"totalTokenCount": 10},
}
with patch("httpx.AsyncClient.post", new_callable=AsyncMock) as mock_post:
mock_post.return_value = mock_response
result = await client.generate_response("Hello")
assert not result["success"]
assert result["error"] == "Blocked by Model Armor after Gemini response"
assert result["model_armor"]["output"]["blocked"]
def test_create_gemini_text_client_factory(self) -> None:
"""[FACT] Factory function creates client instance."""
client = create_gemini_text_client(api_key="factory_test_key")
assert isinstance(client, GeminiTextClient)
assert client.api_key == "factory_test_key"
class StubModelArmorClient(ModelArmorClient):
def __init__(self, input_result: ModelArmorScreenResult, output_result: ModelArmorScreenResult):
super().__init__(enabled=False)
self.input_result = input_result
self.output_result = output_result
def screen_input_text(
self, text: str, context: dict[str, Any] | None = None
) -> ModelArmorScreenResult:
return self.input_result
def screen_output_text(
self, text: str, context: dict[str, Any] | None = None
) -> ModelArmorScreenResult:
return self.output_result
ALLOW_RESULT = ModelArmorScreenResult(
blocked=False,
action="allow",
findings=[],
template="input-template",
latency_ms=1.0,
failure_mode="open",
)
BLOCK_RESULT = ModelArmorScreenResult(
blocked=True,
action="block",
findings=[{"category": "prompt_injection"}],
template="input-template",
latency_ms=1.0,
failure_mode="open",
)