Skip to content

Commit cc4ed8b

Browse files
feat: add BoundaryRule, ConfidenceRule, LangChain integration, and examples
1 parent b50effd commit cc4ed8b

15 files changed

Lines changed: 947 additions & 0 deletions

File tree

examples/rag_output/__init__.py

Whitespace-only changes.

examples/rag_output/contracts.py

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
from gateframe.core.contract import ValidationContract
2+
from gateframe.rules.confidence import ConfidenceRule
3+
from gateframe.rules.semantic import SemanticRule
4+
from gateframe.rules.structural import StructuralRule
5+
6+
from .models import RagAnswer
7+
8+
retrieval_contract = ValidationContract(
9+
name="retrieval_check",
10+
rules=[
11+
StructuralRule(schema=RagAnswer),
12+
SemanticRule(
13+
check=lambda output, **ctx: len(output.get("citations", [])) > 0,
14+
name="has_citations",
15+
failure_message="Answer must include at least one citation.",
16+
),
17+
],
18+
)
19+
20+
answer_contract = ValidationContract(
21+
name="answer_quality",
22+
rules=[
23+
StructuralRule(schema=RagAnswer),
24+
SemanticRule(
25+
check=lambda output, **ctx: len(output.get("answer", "")) >= 20,
26+
name="answer_length",
27+
failure_message="Answer is too short to be useful (minimum 20 characters).",
28+
),
29+
ConfidenceRule(
30+
field="confidence",
31+
minimum=0.65,
32+
name="answer_confidence",
33+
),
34+
SemanticRule(
35+
check=lambda output, **ctx: output.get("grounded") is True,
36+
name="grounding_check",
37+
failure_message="Answer must be grounded in retrieved documents.",
38+
),
39+
],
40+
)

examples/rag_output/models.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
from pydantic import BaseModel
2+
3+
4+
class RagAnswer(BaseModel):
5+
answer: str
6+
citations: list[str]
7+
confidence: float
8+
grounded: bool

examples/rag_output/run.py

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
from gateframe.audit.log import AuditLog
2+
from gateframe.core.context import WorkflowContext
3+
4+
from .contracts import answer_contract, retrieval_contract
5+
6+
7+
def run_scenario(label: str, intake: dict, answer: dict) -> None:
8+
ctx = WorkflowContext(workflow_id=f"rag_{label}", escalation_threshold=0.5)
9+
audit = AuditLog()
10+
11+
result1 = retrieval_contract.validate(intake)
12+
ctx.update(result1)
13+
audit.record(result1, workflow_context=ctx)
14+
print(f" Step 1 (retrieval_check): passed={result1.passed}")
15+
16+
result2 = answer_contract.validate(answer)
17+
ctx.update(result2)
18+
audit.record(result2, workflow_context=ctx)
19+
print(f" Step 2 (answer_quality): passed={result2.passed}, confidence={ctx.confidence:.2f}")
20+
for failure in result2.failures:
21+
print(f" [{failure.failure_mode.value}] {failure.rule_name}: {failure.message}")
22+
23+
24+
def main() -> None:
25+
print("Scenario A: passing")
26+
run_scenario(
27+
"scenario_a",
28+
intake={
29+
"answer": "The capital of France is Paris.",
30+
"citations": ["doc_001"],
31+
"confidence": 0.92,
32+
"grounded": True,
33+
},
34+
answer={
35+
"answer": "The capital of France is Paris, as stated in the referenced document.",
36+
"citations": ["doc_001"],
37+
"confidence": 0.92,
38+
"grounded": True,
39+
},
40+
)
41+
42+
print("\nScenario B: degraded confidence")
43+
run_scenario(
44+
"scenario_b",
45+
intake={
46+
"answer": "Possibly Paris, though sources are unclear.",
47+
"citations": ["doc_002"],
48+
"confidence": 0.41,
49+
"grounded": False,
50+
},
51+
answer={
52+
"answer": "Possibly Paris, though sources are unclear.",
53+
"citations": ["doc_002"],
54+
"confidence": 0.41,
55+
"grounded": False,
56+
},
57+
)
58+
59+
60+
if __name__ == "__main__":
61+
main()

examples/triage_workflow/__init__.py

Whitespace-only changes.
Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
from gateframe.core.contract import ValidationContract
2+
from gateframe.rules.boundary import AllowedValues, BoundaryRule
3+
from gateframe.rules.confidence import ConfidenceRule
4+
from gateframe.rules.semantic import SemanticRule
5+
from gateframe.rules.structural import StructuralRule
6+
7+
from .models import EscalationCheck, TriageDecision, TriageIntake
8+
9+
intake_contract = ValidationContract(
10+
name="intake",
11+
rules=[
12+
StructuralRule(schema=TriageIntake),
13+
SemanticRule(
14+
check=lambda output, **ctx: 1 <= output.get("severity", 0) <= 10,
15+
name="severity_range",
16+
failure_message="Severity must be between 1 and 10.",
17+
),
18+
],
19+
)
20+
21+
triage_decision_contract = ValidationContract(
22+
name="triage_decision",
23+
rules=[
24+
StructuralRule(schema=TriageDecision),
25+
BoundaryRule(
26+
check=AllowedValues("action", {"treat", "observe", "refer", "discharge"}),
27+
name="action_boundary",
28+
failure_message="Action must be one of: treat, observe, refer, discharge.",
29+
),
30+
BoundaryRule(
31+
check=AllowedValues("priority", {"low", "medium", "high", "critical"}),
32+
name="priority_boundary",
33+
failure_message="Priority must be one of: low, medium, high, critical.",
34+
),
35+
ConfidenceRule(
36+
field="confidence",
37+
minimum=0.6,
38+
name="decision_confidence",
39+
),
40+
],
41+
)
42+
43+
escalation_check_contract = ValidationContract(
44+
name="escalation_check",
45+
rules=[
46+
StructuralRule(schema=EscalationCheck),
47+
BoundaryRule(
48+
check=AllowedValues("assigned_role", {"nurse", "doctor", "specialist", "admin"}),
49+
name="role_boundary",
50+
failure_message="assigned_role must be one of: nurse, doctor, specialist, admin.",
51+
),
52+
],
53+
)

examples/triage_workflow/models.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
from pydantic import BaseModel
2+
3+
4+
class TriageIntake(BaseModel):
5+
patient_id: str
6+
complaint: str
7+
severity: int
8+
9+
10+
class TriageDecision(BaseModel):
11+
action: str
12+
priority: str
13+
confidence: float
14+
rationale: str
15+
16+
17+
class EscalationCheck(BaseModel):
18+
escalated: bool
19+
assigned_role: str
20+
notes: str

examples/triage_workflow/run.py

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
from gateframe.audit.log import AuditLog
2+
from gateframe.core.context import WorkflowContext
3+
4+
from .contracts import escalation_check_contract, intake_contract, triage_decision_contract
5+
6+
7+
def main() -> None:
8+
ctx = WorkflowContext(workflow_id="triage_001", escalation_threshold=0.5)
9+
audit = AuditLog()
10+
11+
# Step 1: Intake -- passes
12+
intake_output = {
13+
"patient_id": "P-4821",
14+
"complaint": "Chest pain and shortness of breath",
15+
"severity": 8,
16+
}
17+
result1 = intake_contract.validate(intake_output)
18+
ctx.update(result1)
19+
audit.record(result1, workflow_context=ctx)
20+
print(f"Step 1 (intake): passed={result1.passed}, confidence={ctx.confidence:.2f}")
21+
22+
# Step 2: Triage decision -- confidence 0.52, below minimum 0.6 -> SOFT_FAIL
23+
decision_output = {
24+
"action": "treat",
25+
"priority": "high",
26+
"confidence": 0.52,
27+
"rationale": "Symptoms consistent with cardiac event, awaiting ECG confirmation.",
28+
}
29+
result2 = triage_decision_contract.validate(decision_output)
30+
ctx.update(result2)
31+
audit.record(result2, workflow_context=ctx)
32+
print(f"Step 2 (triage_decision): passed={result2.passed}, confidence={ctx.confidence:.2f}")
33+
for failure in result2.failures:
34+
print(f" SOFT_FAIL [{failure.rule_name}]: {failure.message}")
35+
36+
# Step 3: Escalation check -- passes
37+
escalation_output = {
38+
"escalated": True,
39+
"assigned_role": "doctor",
40+
"notes": "Escalated to on-call cardiologist.",
41+
}
42+
result3 = escalation_check_contract.validate(escalation_output)
43+
ctx.update(result3)
44+
audit.record(result3, workflow_context=ctx)
45+
print(f"Step 3 (escalation_check): passed={result3.passed}, confidence={ctx.confidence:.2f}")
46+
47+
print(f"\nThreshold breached: {ctx.threshold_breached}")
48+
print(f"Audit entries: {len(audit.entries)}")
49+
for entry in audit.entries:
50+
data = entry.to_dict()
51+
conf = data.get("confidence", 1.0)
52+
print(f" [{data['contract_name']}] passed={data['passed']}, confidence={conf:.2f}")
53+
54+
55+
if __name__ == "__main__":
56+
main()
Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
from __future__ import annotations
2+
3+
import json
4+
from typing import Any
5+
6+
from gateframe.core.contract import ValidationContract, ValidationResult
7+
8+
9+
def _require_langchain() -> None:
10+
try:
11+
import langchain # noqa: F401
12+
except ImportError:
13+
raise ImportError(
14+
"langchain is required for this integration. "
15+
"Install it with: pip install gateframe[langchain]"
16+
) from None
17+
18+
19+
def extract_text(response: Any, output_key: str = "output") -> str: # noqa: ANN401
20+
"""Extract text from a LangChain response (str, AIMessage, or dict)."""
21+
if isinstance(response, str):
22+
return response
23+
if isinstance(response, dict):
24+
if output_key not in response:
25+
raise ValueError(f"Key '{output_key}' not found in response dict.")
26+
value = response[output_key]
27+
if not isinstance(value, str):
28+
raise ValueError(f"Expected str at key '{output_key}', got {type(value).__name__}.")
29+
return value
30+
content = getattr(response, "content", None)
31+
if content is not None:
32+
if not isinstance(content, str):
33+
raise ValueError(f"Expected str content, got {type(content).__name__}.")
34+
return content
35+
raise ValueError(f"Cannot extract text from response of type {type(response).__name__}.")
36+
37+
38+
def extract_json(response: Any, output_key: str = "output") -> dict[str, Any]: # noqa: ANN401
39+
"""Extract text, then parse as JSON. Raises ValueError on failure."""
40+
text = extract_text(response, output_key=output_key)
41+
try:
42+
data = json.loads(text)
43+
except json.JSONDecodeError as exc:
44+
raise ValueError(
45+
f"Failed to parse response as JSON: {exc}. Raw text: {text[:200]}"
46+
) from exc
47+
if not isinstance(data, dict):
48+
raise ValueError(f"Expected JSON object, got {type(data).__name__}.")
49+
return data
50+
51+
52+
def extract_metadata(response: Any) -> dict[str, Any]: # noqa: ANN401
53+
"""Extract available metadata from a LangChain response. Omits None values."""
54+
meta: dict[str, Any] = {}
55+
56+
if isinstance(response, str):
57+
return meta
58+
59+
response_metadata = getattr(response, "response_metadata", None)
60+
if isinstance(response_metadata, dict):
61+
for k, v in response_metadata.items():
62+
if v is not None:
63+
meta[k] = v
64+
65+
type_val = getattr(response, "type", None)
66+
if type_val is not None:
67+
meta["type"] = type_val
68+
69+
source_documents = getattr(response, "source_documents", None)
70+
if source_documents is not None:
71+
meta["source_document_count"] = len(source_documents)
72+
73+
return meta
74+
75+
76+
class LangChainValidator:
77+
def __init__(
78+
self,
79+
contract: ValidationContract,
80+
*,
81+
parse_json: bool = False,
82+
output_key: str = "output",
83+
) -> None:
84+
self._contract = contract
85+
self._parse_json = parse_json
86+
self._output_key = output_key
87+
88+
def validate(self, response: Any, **context: Any) -> ValidationResult: # noqa: ANN401
89+
output = (
90+
extract_json(response, output_key=self._output_key)
91+
if self._parse_json
92+
else extract_text(response, output_key=self._output_key)
93+
)
94+
metadata = extract_metadata(response)
95+
merged = {**metadata, **context}
96+
return self._contract.validate(output, **merged)

gateframe/rules/boundary.py

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
from __future__ import annotations
2+
3+
from collections.abc import Callable
4+
from typing import Any
5+
6+
from gateframe.core.failure import FailureMode, FailureResult
7+
from gateframe.core.rule import Rule
8+
9+
10+
class BoundaryRule(Rule):
11+
def __init__(
12+
self,
13+
check: Callable[..., bool],
14+
name: str = "boundary_check",
15+
description: str = "",
16+
failure_mode: FailureMode = FailureMode.HARD_FAIL,
17+
failure_message: str = "",
18+
) -> None:
19+
super().__init__(name, description)
20+
self._check = check
21+
self.failure_mode = failure_mode
22+
self._failure_message = failure_message
23+
24+
def validate(self, output: Any, **context: Any) -> FailureResult | None: # noqa: ANN401
25+
try:
26+
passed = self._check(output, **context)
27+
except Exception as exc:
28+
return FailureResult(
29+
rule_name=self.name,
30+
rule_description=self.description,
31+
failure_mode=self.failure_mode,
32+
message=f"{self.name} raised an exception: {exc}",
33+
context={"exception_type": type(exc).__name__, "exception_message": str(exc)},
34+
)
35+
36+
if passed:
37+
return None
38+
39+
message = self._failure_message or f"{self.name} failed: boundary check returned False."
40+
return FailureResult(
41+
rule_name=self.name,
42+
rule_description=self.description,
43+
failure_mode=self.failure_mode,
44+
message=message,
45+
)
46+
47+
48+
class AllowedValues:
49+
def __init__(self, field: str, allowed: set[Any]) -> None:
50+
self._field = field
51+
self._allowed = allowed
52+
53+
def __call__(self, output: Any, **context: Any) -> bool: # noqa: ANN401
54+
if isinstance(output, dict):
55+
value = output.get(self._field)
56+
else:
57+
value = getattr(output, self._field, None)
58+
return value in self._allowed

0 commit comments

Comments
 (0)