forked from PracticalMind/gateframe
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_context.py
More file actions
216 lines (177 loc) · 7.35 KB
/
Copy pathtest_context.py
File metadata and controls
216 lines (177 loc) · 7.35 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
from typing import Any
import pytest
from gateframe.core.context import StepRecord, WorkflowContext
from gateframe.core.contract import ValidationResult
from gateframe.core.failure import FailureMode, FailureResult
from gateframe.core.rule import Rule
class _FailingRule(Rule):
def __init__(self, name: str, mode: FailureMode) -> None:
super().__init__(name)
self._mode = mode
def validate(self, output: Any, **context: Any) -> FailureResult | None:
return FailureResult(
rule_name=self.name,
failure_mode=self._mode,
message=f"{self.name}: rejected.",
)
class _PassingRule(Rule):
def validate(self, output: Any, **context: Any) -> FailureResult | None:
return None
def _make_result(
passed: bool = True,
failures: list[FailureResult] | None = None,
contract_name: str = "test",
) -> ValidationResult:
if failures is None:
failures = []
failure_modes = {f.failure_mode for f in failures}
return ValidationResult(
passed=passed,
contract_name=contract_name,
failures=failures,
rules_applied=1,
rules_failed=len(failures),
has_hard_fail=FailureMode.HARD_FAIL in failure_modes,
has_soft_fail=FailureMode.SOFT_FAIL in failure_modes,
)
def _soft_failure(name: str = "rule_a") -> FailureResult:
return FailureResult(
rule_name=name,
failure_mode=FailureMode.SOFT_FAIL,
message=f"{name}: soft fail.",
)
def _silent_failure(name: str = "rule_b") -> FailureResult:
return FailureResult(
rule_name=name,
failure_mode=FailureMode.SILENT_FAIL,
message=f"{name}: silent fail.",
)
def _hard_failure(name: str = "rule_c") -> FailureResult:
return FailureResult(
rule_name=name,
failure_mode=FailureMode.HARD_FAIL,
message=f"{name}: hard fail.",
)
def _retry_failure(name: str = "rule_d") -> FailureResult:
return FailureResult(
rule_name=name,
failure_mode=FailureMode.RETRY,
message=f"{name}: retry.",
)
class TestWorkflowContext:
def test_initial_confidence_is_one(self) -> None:
ctx = WorkflowContext("wf1")
assert ctx.confidence == 1.0
def test_custom_initial_confidence(self) -> None:
ctx = WorkflowContext("wf1", initial_confidence=0.8)
assert ctx.confidence == 0.8
def test_soft_fail_degrades_confidence(self) -> None:
ctx = WorkflowContext("wf1")
result = _make_result(passed=False, failures=[_soft_failure()])
ctx.update(result)
assert ctx.confidence == pytest.approx(0.85)
def test_multiple_soft_fails_stack(self) -> None:
ctx = WorkflowContext("wf1")
result = _make_result(
passed=False,
failures=[_soft_failure("a"), _soft_failure("b"), _soft_failure("c")],
)
ctx.update(result)
assert ctx.confidence == pytest.approx(0.55)
def test_hard_fail_does_not_degrade(self) -> None:
ctx = WorkflowContext("wf1")
result = _make_result(passed=False, failures=[_hard_failure()])
ctx.update(result)
assert ctx.confidence == 1.0
def test_retry_does_not_degrade(self) -> None:
ctx = WorkflowContext("wf1")
result = _make_result(passed=False, failures=[_retry_failure()])
ctx.update(result)
assert ctx.confidence == 1.0
def test_silent_fail_degrades_confidence(self) -> None:
ctx = WorkflowContext("wf1")
result = _make_result(passed=False, failures=[_silent_failure()])
ctx.update(result)
assert ctx.confidence == pytest.approx(0.9)
def test_cross_step_cumulative_degradation(self) -> None:
ctx = WorkflowContext("wf1")
ctx.update(_make_result(passed=False, failures=[_soft_failure()]))
ctx.update(_make_result(passed=False, failures=[_soft_failure()]))
assert ctx.confidence == pytest.approx(0.70)
def test_threshold_breach_detected(self) -> None:
ctx = WorkflowContext("wf1", escalation_threshold=0.8)
ctx.update(_make_result(passed=False, failures=[_soft_failure(), _soft_failure()]))
assert ctx.threshold_breached is True
def test_threshold_not_breached(self) -> None:
ctx = WorkflowContext("wf1", escalation_threshold=0.5)
ctx.update(_make_result(passed=False, failures=[_soft_failure()]))
assert ctx.threshold_breached is False
def test_confidence_floor_at_zero(self) -> None:
ctx = WorkflowContext("wf1", soft_fail_penalty=0.5)
for _ in range(5):
ctx.update(_make_result(passed=False, failures=[_soft_failure()]))
assert ctx.confidence == 0.0
def test_custom_penalties(self) -> None:
ctx = WorkflowContext("wf1", soft_fail_penalty=0.25, silent_fail_penalty=0.2)
ctx.update(_make_result(passed=False, failures=[_soft_failure()]))
assert ctx.confidence == pytest.approx(0.75)
ctx.update(_make_result(passed=False, failures=[_silent_failure()]))
assert ctx.confidence == pytest.approx(0.55)
def test_history_records_each_step(self) -> None:
ctx = WorkflowContext("wf1")
ctx.update(_make_result(passed=True))
ctx.update(_make_result(passed=False, failures=[_soft_failure()]))
assert ctx.step_count == 2
assert ctx.history[0].passed is True
assert ctx.history[1].passed is False
def test_history_returns_copy(self) -> None:
ctx = WorkflowContext("wf1")
ctx.update(_make_result())
history = ctx.history
history.clear()
assert ctx.step_count == 1
def test_to_dict_structure(self) -> None:
ctx = WorkflowContext("wf1")
ctx.update(_make_result())
data = ctx.to_dict()
assert data["workflow_id"] == "wf1"
assert "confidence" in data
assert "escalation_threshold" in data
assert "threshold_breached" in data
assert "step_count" in data
assert "history" in data
assert len(data["history"]) == 1
def test_passing_step_keeps_confidence(self) -> None:
ctx = WorkflowContext("wf1")
ctx.update(_make_result(passed=True))
assert ctx.confidence == 1.0
def test_reset_restores_initial_confidence_and_clears_history(self) -> None:
ctx = WorkflowContext("wf1", initial_confidence=0.8)
ctx.update(_make_result(passed=False, failures=[_soft_failure()]))
assert ctx.confidence == pytest.approx(0.65)
assert ctx.step_count == 1
ctx.reset()
assert ctx.confidence == pytest.approx(0.8)
assert ctx.step_count == 0
assert ctx.history == []
assert ctx.threshold_breached is False
class TestStepRecord:
def test_to_dict(self) -> None:
record = StepRecord(
step_index=0,
contract_name="test",
passed=False,
confidence_before=1.0,
confidence_after=0.85,
penalty_applied=0.15,
failure_modes=[FailureMode.SOFT_FAIL],
)
data = record.to_dict()
assert data["step_index"] == 0
assert data["contract_name"] == "test"
assert data["passed"] is False
assert data["confidence_before"] == 1.0
assert data["confidence_after"] == 0.85
assert data["penalty_applied"] == 0.15
assert data["failure_modes"] == ["soft_fail"]
assert "timestamp" in data