Skip to content

Commit f242e05

Browse files
hyperpolymathclaude
andcommitted
feat: implement Phase 1 deontic constraint engine
Implement the core Phronesis ethical constraint system: - Manifest: [project], [[constraints]] (obligation/permission/prohibition with subject, action, condition, priority), [enforcement] (strict/advisory mode, escalation threshold, audit logging), [agent] (name, capabilities) - Codegen: parser.rs (compile constraints, detect deontic contradictions), engine.rs (evaluate actions against constraints with condition predicates), audit.rs (structured audit trail with JSON/report output) - ABI: DeonticModality, Constraint, AgentAction, AuditDecision (Permitted/Denied/Escalated), EvaluationResult with reasoning - 70 tests (31 unit lib + 31 unit bin + 8 integration) - Example: ai-guardrails/ medical triage bot with 6 constraints Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 5676811 commit f242e05

13 files changed

Lines changed: 3617 additions & 35 deletions

File tree

Cargo.lock

Lines changed: 1118 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ anyhow = "1"
1818
thiserror = "2"
1919
handlebars = "6"
2020
walkdir = "2"
21+
chrono = { version = "0.4", features = ["serde"] }
22+
serde_json = "1"
2123

2224
[dev-dependencies]
2325
tempfile = "3"

examples/ai-guardrails/README.adoc

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
= AI Guardrails Example
2+
// SPDX-License-Identifier: PMPL-1.0-or-later
3+
4+
This example demonstrates a medical triage AI agent with provably safe ethical
5+
constraints. The `phronesiser.toml` manifest encodes deontic logic rules:
6+
7+
* **Prohibitions**: The bot cannot issue diagnoses or access patient records
8+
without consent.
9+
* **Obligations**: The bot must log every interaction and escalate critical
10+
symptoms to a human.
11+
* **Permissions**: The bot may read public medical guidelines and suggest
12+
over-the-counter remedies (when symptoms are not critical).
13+
14+
== Usage
15+
16+
[source,bash]
17+
----
18+
# Validate the manifest
19+
cargo run -- validate -m examples/ai-guardrails/phronesiser.toml
20+
21+
# Generate constraint engine artifacts
22+
cargo run -- generate -m examples/ai-guardrails/phronesiser.toml -o /tmp/triage-engine
23+
24+
# Show manifest info
25+
cargo run -- info -m examples/ai-guardrails/phronesiser.toml
26+
27+
# Run self-test
28+
cargo run -- run -m examples/ai-guardrails/phronesiser.toml
29+
----
30+
31+
== Constraint Evaluation Rules
32+
33+
The constraint engine evaluates each agent action as follows:
34+
35+
1. Collect all constraints matching the action's `(subject, action)` pair.
36+
2. Evaluate condition predicates against the action's runtime context.
37+
3. Apply the highest-priority matching constraint.
38+
4. In strict mode: prohibitions produce `DENIED` (or `ESCALATED` if
39+
`priority >= escalation-threshold`).
40+
5. Unfulfilled obligations are tracked and reported.
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
# SPDX-License-Identifier: PMPL-1.0-or-later
2+
# phronesiser manifest — AI Guardrails Example
3+
#
4+
# This example demonstrates a medical triage AI agent with ethical constraints
5+
# preventing it from making diagnoses, requiring audit logging, and escalating
6+
# high-stakes decisions to a human.
7+
8+
[project]
9+
name = "medical-triage-bot"
10+
version = "1.0.0"
11+
description = "Ethical guardrails for an AI-assisted medical triage system"
12+
13+
# Prohibition: the bot must never make a diagnosis directly.
14+
[[constraints]]
15+
name = "no-direct-diagnosis"
16+
kind = "prohibition"
17+
subject = "agent"
18+
action = "issue-diagnosis"
19+
priority = 200
20+
21+
# Prohibition: the bot must not access patient records without consent.
22+
[[constraints]]
23+
name = "no-access-without-consent"
24+
kind = "prohibition"
25+
subject = "agent"
26+
action = "access-patient-record"
27+
condition = "not patient-consent-given"
28+
priority = 150
29+
30+
# Obligation: the bot must log every interaction for auditability.
31+
[[constraints]]
32+
name = "must-log-interaction"
33+
kind = "obligation"
34+
subject = "agent"
35+
action = "write-audit-log"
36+
priority = 80
37+
38+
# Obligation: the bot must escalate any life-threatening symptom to a human.
39+
[[constraints]]
40+
name = "must-escalate-critical"
41+
kind = "obligation"
42+
subject = "agent"
43+
action = "escalate-to-human"
44+
condition = "symptoms-critical"
45+
priority = 180
46+
47+
# Permission: the bot may read publicly available medical guidelines.
48+
[[constraints]]
49+
name = "can-read-guidelines"
50+
kind = "permission"
51+
subject = "agent"
52+
action = "read-medical-guidelines"
53+
priority = 10
54+
55+
# Permission: the bot may suggest (not prescribe) over-the-counter remedies.
56+
[[constraints]]
57+
name = "can-suggest-otc"
58+
kind = "permission"
59+
subject = "agent"
60+
action = "suggest-otc-remedy"
61+
condition = "not symptoms-critical"
62+
priority = 20
63+
64+
[enforcement]
65+
mode = "strict"
66+
escalation-threshold = 100
67+
audit-log = true
68+
69+
[agent]
70+
name = "triage-assistant"
71+
capabilities = [
72+
"read-medical-guidelines",
73+
"suggest-otc-remedy",
74+
"write-audit-log",
75+
"escalate-to-human",
76+
"access-patient-record",
77+
]

src/abi/mod.rs

Lines changed: 231 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,233 @@
11
// SPDX-License-Identifier: PMPL-1.0-or-later
22
// Copyright (c) 2026 Jonathan D.A. Jewell <j.d.a.jewell@open.ac.uk>
3-
// ABI module for phronesiser — Idris2 proof types for Phronesis interface correctness.
3+
//
4+
// ABI module for phronesiser — Rust-side type definitions mirroring the Idris2
5+
// formal proofs for Phronesis interface correctness.
6+
//
7+
// These types define the core domain model for deontic constraint evaluation:
8+
// - DeonticModality: The three modalities (obligation, permission, prohibition).
9+
// - Constraint: A fully resolved constraint ready for evaluation.
10+
// - AgentAction: An action the agent is attempting to perform.
11+
// - AuditDecision: The outcome of evaluating an action against constraints.
12+
// - EvaluationResult: The complete result including decision and reasoning.
13+
14+
use serde::{Deserialize, Serialize};
15+
use std::fmt;
16+
17+
// ---------------------------------------------------------------------------
18+
// DeonticModality — the three pillars of deontic logic
19+
// ---------------------------------------------------------------------------
20+
21+
/// The three fundamental modalities of deontic logic, mirroring the Idris2
22+
/// ABI definition in `src/interface/abi/Types.idr`.
23+
///
24+
/// - `Obligation`: The agent MUST perform the action.
25+
/// - `Permission`: The agent MAY perform the action.
26+
/// - `Prohibition`: The agent MUST NOT perform the action.
27+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
28+
#[repr(u8)]
29+
pub enum DeonticModality {
30+
/// The agent is required to perform the action.
31+
Obligation = 0,
32+
/// The agent is allowed (but not required) to perform the action.
33+
Permission = 1,
34+
/// The agent is forbidden from performing the action.
35+
Prohibition = 2,
36+
}
37+
38+
impl fmt::Display for DeonticModality {
39+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
40+
match self {
41+
DeonticModality::Obligation => write!(f, "OBLIGATION"),
42+
DeonticModality::Permission => write!(f, "PERMISSION"),
43+
DeonticModality::Prohibition => write!(f, "PROHIBITION"),
44+
}
45+
}
46+
}
47+
48+
impl From<&crate::manifest::ConstraintKind> for DeonticModality {
49+
fn from(kind: &crate::manifest::ConstraintKind) -> Self {
50+
match kind {
51+
crate::manifest::ConstraintKind::Obligation => DeonticModality::Obligation,
52+
crate::manifest::ConstraintKind::Permission => DeonticModality::Permission,
53+
crate::manifest::ConstraintKind::Prohibition => DeonticModality::Prohibition,
54+
}
55+
}
56+
}
57+
58+
// ---------------------------------------------------------------------------
59+
// Constraint — a resolved constraint ready for runtime evaluation
60+
// ---------------------------------------------------------------------------
61+
62+
/// A fully resolved constraint that the evaluation engine checks at runtime.
63+
///
64+
/// This is the ABI-level representation, distinct from the manifest-level
65+
/// `manifest::Constraint` which is a serialisation format. The ABI constraint
66+
/// carries a compiled condition predicate.
67+
#[derive(Debug, Clone, Serialize, Deserialize)]
68+
pub struct Constraint {
69+
/// Unique identifier for this constraint.
70+
pub name: String,
71+
72+
/// The deontic modality governing this constraint.
73+
pub modality: DeonticModality,
74+
75+
/// The entity the constraint applies to (e.g. "agent", "subsystem").
76+
pub subject: String,
77+
78+
/// The action being constrained (e.g. "delete-records", "send-email").
79+
pub action: String,
80+
81+
/// An optional condition string. When `None`, the constraint applies
82+
/// unconditionally. When `Some(pred)`, the constraint only applies
83+
/// when `pred` evaluates to true in the current context.
84+
pub condition: Option<String>,
85+
86+
/// Numeric priority for conflict resolution. Higher values win.
87+
pub priority: i32,
88+
}
89+
90+
impl From<&crate::manifest::Constraint> for Constraint {
91+
fn from(mc: &crate::manifest::Constraint) -> Self {
92+
Constraint {
93+
name: mc.name.clone(),
94+
modality: DeonticModality::from(&mc.kind),
95+
subject: mc.subject.clone(),
96+
action: mc.action.clone(),
97+
condition: mc.condition.clone(),
98+
priority: mc.priority,
99+
}
100+
}
101+
}
102+
103+
// ---------------------------------------------------------------------------
104+
// AgentAction — an action the agent is attempting to perform
105+
// ---------------------------------------------------------------------------
106+
107+
/// Represents an action that an agent is requesting permission to perform.
108+
/// The constraint engine evaluates this against all applicable constraints.
109+
#[derive(Debug, Clone, Serialize, Deserialize)]
110+
pub struct AgentAction {
111+
/// The agent performing the action.
112+
pub agent_name: String,
113+
114+
/// The action the agent wants to perform (must match constraint `action` fields).
115+
pub action: String,
116+
117+
/// The subject/target of the action (must match constraint `subject` fields).
118+
pub subject: String,
119+
120+
/// Key-value context that condition predicates are evaluated against.
121+
/// For example: `{"user-consent-given": "true", "data-classification": "public"}`.
122+
pub context: std::collections::HashMap<String, String>,
123+
}
124+
125+
// ---------------------------------------------------------------------------
126+
// AuditDecision — the outcome of constraint evaluation
127+
// ---------------------------------------------------------------------------
128+
129+
/// The decision produced by the constraint engine for a given action.
130+
///
131+
/// - `Permitted`: The action is allowed (no prohibitions apply, or an explicit
132+
/// permission overrides).
133+
/// - `Denied`: The action violates one or more prohibitions in strict mode.
134+
/// - `Escalated`: The violated constraint has priority >= the escalation
135+
/// threshold, requiring human review.
136+
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
137+
#[repr(u8)]
138+
pub enum AuditDecision {
139+
/// The action is permitted to proceed.
140+
Permitted = 0,
141+
/// The action is denied due to constraint violation.
142+
Denied = 1,
143+
/// The action requires human escalation before proceeding.
144+
Escalated = 2,
145+
}
146+
147+
impl fmt::Display for AuditDecision {
148+
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
149+
match self {
150+
AuditDecision::Permitted => write!(f, "PERMITTED"),
151+
AuditDecision::Denied => write!(f, "DENIED"),
152+
AuditDecision::Escalated => write!(f, "ESCALATED"),
153+
}
154+
}
155+
}
156+
157+
// ---------------------------------------------------------------------------
158+
// EvaluationResult — full evaluation output with reasoning
159+
// ---------------------------------------------------------------------------
160+
161+
/// The complete result of evaluating an agent action against the constraint set.
162+
///
163+
/// Includes the decision, reasoning trail, and references to the constraints
164+
/// that influenced the outcome.
165+
#[derive(Debug, Clone, Serialize, Deserialize)]
166+
pub struct EvaluationResult {
167+
/// The action that was evaluated.
168+
pub action: AgentAction,
169+
170+
/// The final decision: permitted, denied, or escalated.
171+
pub decision: AuditDecision,
172+
173+
/// Human-readable reasoning explaining why the decision was reached.
174+
pub reasoning: Vec<String>,
175+
176+
/// Names of constraints that applied to this evaluation.
177+
pub applicable_constraints: Vec<String>,
178+
179+
/// Names of constraints that were violated (empty if permitted).
180+
pub violated_constraints: Vec<String>,
181+
182+
/// ISO-8601 timestamp of the evaluation.
183+
pub timestamp: String,
184+
}
185+
186+
#[cfg(test)]
187+
mod tests {
188+
use super::*;
189+
190+
#[test]
191+
fn test_deontic_modality_display() {
192+
assert_eq!(DeonticModality::Obligation.to_string(), "OBLIGATION");
193+
assert_eq!(DeonticModality::Permission.to_string(), "PERMISSION");
194+
assert_eq!(DeonticModality::Prohibition.to_string(), "PROHIBITION");
195+
}
196+
197+
#[test]
198+
fn test_audit_decision_display() {
199+
assert_eq!(AuditDecision::Permitted.to_string(), "PERMITTED");
200+
assert_eq!(AuditDecision::Denied.to_string(), "DENIED");
201+
assert_eq!(AuditDecision::Escalated.to_string(), "ESCALATED");
202+
}
203+
204+
#[test]
205+
fn test_deontic_modality_repr() {
206+
assert_eq!(DeonticModality::Obligation as u8, 0);
207+
assert_eq!(DeonticModality::Permission as u8, 1);
208+
assert_eq!(DeonticModality::Prohibition as u8, 2);
209+
}
210+
211+
#[test]
212+
fn test_audit_decision_repr() {
213+
assert_eq!(AuditDecision::Permitted as u8, 0);
214+
assert_eq!(AuditDecision::Denied as u8, 1);
215+
assert_eq!(AuditDecision::Escalated as u8, 2);
216+
}
217+
218+
#[test]
219+
fn test_constraint_from_manifest() {
220+
let mc = crate::manifest::Constraint {
221+
name: "no-delete".into(),
222+
kind: crate::manifest::ConstraintKind::Prohibition,
223+
subject: "agent".into(),
224+
action: "delete".into(),
225+
condition: Some("data-critical".into()),
226+
priority: 80,
227+
};
228+
let c = Constraint::from(&mc);
229+
assert_eq!(c.modality, DeonticModality::Prohibition);
230+
assert_eq!(c.name, "no-delete");
231+
assert_eq!(c.condition, Some("data-critical".into()));
232+
}
233+
}

0 commit comments

Comments
 (0)