Skip to content

Commit 54e0f4d

Browse files
shadowbipnodeshadowbipnode
committed
Add operational safety scoring
1 parent c52a094 commit 54e0f4d

2 files changed

Lines changed: 61 additions & 0 deletions

File tree

‎src/components/ProfessionalResult.jsx‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,29 @@ const ProfessionalResult = ({ result, compact = false, hidePrimaryArtifact = fal
275275
? "#FBBF24"
276276
: "#FF4D6A";
277277

278+
const remediationSafety = first(result.remediation_safety, result.action_safety);
279+
const evidenceQuality = first(result.evidence_quality, result.evidence_strength);
280+
const rollbackConfidence = first(result.rollback_confidence, result.rollback_trust);
281+
282+
const safetyColor = remediationSafety === "READ_ONLY_SAFE" || remediationSafety === "REVERSIBLE_SAFE"
283+
? "#00D4AA"
284+
: remediationSafety === "PARTIAL_RISK"
285+
? "#FBBF24"
286+
: "#FF4D6A";
287+
288+
const evidenceQualityColor = evidenceQuality === "DIRECT_EVIDENCE"
289+
? "#00D4AA"
290+
: evidenceQuality === "PARTIAL_EVIDENCE"
291+
? "#FBBF24"
292+
: "#FF4D6A";
293+
294+
const rollbackConfidenceColor =
295+
rollbackConfidence === "ROLLBACK_NOT_REQUIRED" || rollbackConfidence === "VERIFIED_ROLLBACK"
296+
? "#00D4AA"
297+
: rollbackConfidence === "PARTIAL_ROLLBACK"
298+
? "#FBBF24"
299+
: "#FF4D6A";
300+
278301
const rollback = commandText(first(result.rollback_commands, result.rollback));
279302
const rollbackReversibility = first(result.rollback_reversibility, result.rollback_safety, result.reversibility);
280303
const rollbackRiskReason = first(result.rollback_risk_reason, result.rollback_risk, result.reversibility_reason);
@@ -359,6 +382,9 @@ const ProfessionalResult = ({ result, compact = false, hidePrimaryArtifact = fal
359382
<div style={{ marginBottom: 12 }}>
360383
{normalizedSeverity && <ResultPill label="RISK" value={normalizedSeverity} color={severityColor} />}
361384
{confidence && <ResultPill label="CONFIDENCE" value={confidence} color={confidenceColor} />}
385+
{remediationSafety && <ResultPill label="SAFETY" value={remediationSafety} color={safetyColor} />}
386+
{evidenceQuality && <ResultPill label="EVIDENCE" value={evidenceQuality} color={evidenceQualityColor} />}
387+
{rollbackConfidence && <ResultPill label="ROLLBACK TRUST" value={rollbackConfidence} color={rollbackConfidenceColor} />}
362388
{requiresSudo !== undefined && <ResultPill label="SUDO" value={requiresSudo ? "YES" : "NO"} color={requiresSudo ? "#FBBF24" : "#00D4AA"} />}
363389
{destructive !== undefined && <ResultPill label="DESTRUCTIVE" value={destructive === true ? "YES" : destructive === false ? "NO" : destructive} color={destructive ? "#FF4D6A" : "#00D4AA"} />}
364390
{isRemoteObservation && <ResultPill label="MODE" value="REMOTE OBSERVATION" color="#38BDF8" />}

‎src/utils/aiProviders.js‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,17 @@ PROFESSIONAL OUTPUT RULES FOR ${toolName}:
171171
- STRONG_VERIFICATION means the verification directly confirms the intended operational outcome.
172172
- WEAK_VERIFICATION means the verification is indirect, partial, or only confirms process/status.
173173
- AMBIGUOUS_VERIFICATION means the verification may pass while the real issue remains unresolved or needs interpretation.
174+
- Always include operational safety scoring fields: remediation_safety, evidence_quality, rollback_confidence.
175+
- remediation_safety must use ONLY: READ_ONLY_SAFE, REVERSIBLE_SAFE, PARTIAL_RISK, HIGH_RISK, DESTRUCTIVE.
176+
- evidence_quality must use ONLY: DIRECT_EVIDENCE, PARTIAL_EVIDENCE, ASSUMPTION_HEAVY.
177+
- rollback_confidence must use ONLY: ROLLBACK_NOT_REQUIRED, VERIFIED_ROLLBACK, PARTIAL_ROLLBACK, UNKNOWN_ROLLBACK.
178+
- remediation_safety measures how safe the proposed action is, not how likely the diagnosis is.
179+
- evidence_quality measures how directly the input supports the diagnosis.
180+
- rollback_confidence measures whether rollback is required, realistic, and validated.
181+
- Use ROLLBACK_NOT_REQUIRED when the recommended workflow is read-only or does not change system state.
182+
- VERIFIED_ROLLBACK must only be used when the previous state, configuration, image version, snapshot, backup, or reversible action is explicitly known or captured.
183+
- UNKNOWN_ROLLBACK must be used when state-changing remediation is suggested but no validated previous state, backup, or rollback target exists.
184+
- PARTIAL_ROLLBACK should be used when rollback is theoretically possible but incomplete, risky, or dependent on unknown prior state.
174185
- Always include rollback guidance. If no rollback is needed, say exactly: "No rollback needed for read-only checks."
175186
- rollback_reversibility must use ONLY: FULLY_REVERSIBLE, PARTIALLY_REVERSIBLE, NON_REVERSIBLE_WITHOUT_BACKUP. Do not invent extended enum values.
176187
- Use arrays for command blocks: fix_commands, verification_commands, rollback_commands. Each array item must be one complete shell command or one comment line starting with #.
@@ -376,6 +387,9 @@ Use this exact JSON schema:
376387
{
377388
"severity": "LOW|MEDIUM|HIGH|CRITICAL",
378389
"confidence": "LOW|MEDIUM|HIGH",
390+
"remediation_safety": "READ_ONLY_SAFE|REVERSIBLE_SAFE|PARTIAL_RISK|HIGH_RISK|DESTRUCTIVE",
391+
"evidence_quality": "DIRECT_EVIDENCE|PARTIAL_EVIDENCE|ASSUMPTION_HEAVY",
392+
"rollback_confidence": "ROLLBACK_NOT_REQUIRED|VERIFIED_ROLLBACK|PARTIAL_ROLLBACK|UNKNOWN_ROLLBACK",
379393
"requires_sudo": true,
380394
"detected_stack": ["nginx", "docker", "systemd", "LND", "Bitcoin Core", "Tor"],
381395
"title": "Brief diagnostic title",
@@ -431,6 +445,9 @@ Respond STRICTLY with this JSON format:
431445
"explanation": "What the command does and why these flags/options are used",
432446
"severity": "LOW|MEDIUM|HIGH|CRITICAL",
433447
"confidence": "LOW|MEDIUM|HIGH",
448+
"remediation_safety": "READ_ONLY_SAFE|REVERSIBLE_SAFE|PARTIAL_RISK|HIGH_RISK|DESTRUCTIVE",
449+
"evidence_quality": "DIRECT_EVIDENCE|PARTIAL_EVIDENCE|ASSUMPTION_HEAVY",
450+
"rollback_confidence": "ROLLBACK_NOT_REQUIRED|VERIFIED_ROLLBACK|PARTIAL_ROLLBACK|UNKNOWN_ROLLBACK",
434451
"requires_sudo": true,
435452
"destructive": false,
436453
"detected_stack": ["systemd", "Docker/Compose", "nginx/reverse-proxy"],
@@ -481,6 +498,9 @@ Response MUST be in this EXACT JSON format:
481498
"summary": "one sentence summary",
482499
"severity": "LOW|MEDIUM|HIGH|CRITICAL",
483500
"confidence": "LOW|MEDIUM|HIGH",
501+
"remediation_safety": "READ_ONLY_SAFE|REVERSIBLE_SAFE|PARTIAL_RISK|HIGH_RISK|DESTRUCTIVE",
502+
"evidence_quality": "DIRECT_EVIDENCE|PARTIAL_EVIDENCE|ASSUMPTION_HEAVY",
503+
"rollback_confidence": "ROLLBACK_NOT_REQUIRED|VERIFIED_ROLLBACK|PARTIAL_ROLLBACK|UNKNOWN_ROLLBACK",
484504
"requires_sudo": true,
485505
"destructive": false,
486506
"detected_stack": ["systemd", "Docker/Compose", "nginx/reverse-proxy"],
@@ -537,6 +557,9 @@ Respond STRICTLY with this JSON format:
537557
"explanation": "brief explanation of key settings and why they were chosen",
538558
"severity": "LOW|MEDIUM|HIGH|CRITICAL",
539559
"confidence": "LOW|MEDIUM|HIGH",
560+
"remediation_safety": "READ_ONLY_SAFE|REVERSIBLE_SAFE|PARTIAL_RISK|HIGH_RISK|DESTRUCTIVE",
561+
"evidence_quality": "DIRECT_EVIDENCE|PARTIAL_EVIDENCE|ASSUMPTION_HEAVY",
562+
"rollback_confidence": "ROLLBACK_NOT_REQUIRED|VERIFIED_ROLLBACK|PARTIAL_ROLLBACK|UNKNOWN_ROLLBACK",
540563
"requires_sudo": true,
541564
"detected_stack": ["systemd", "Docker/Compose", "nginx/reverse-proxy", "Bitcoin Core", "LND/Lightning", "Tor"],
542565
"next_best_action": "first safe action before deployment",
@@ -598,6 +621,9 @@ Respond STRICTLY with this JSON format:
598621
"diagnosis": "most likely cause based on the description",
599622
"severity": "LOW|MEDIUM|HIGH|CRITICAL",
600623
"confidence": "LOW|MEDIUM|HIGH",
624+
"remediation_safety": "READ_ONLY_SAFE|REVERSIBLE_SAFE|PARTIAL_RISK|HIGH_RISK|DESTRUCTIVE",
625+
"evidence_quality": "DIRECT_EVIDENCE|PARTIAL_EVIDENCE|ASSUMPTION_HEAVY",
626+
"rollback_confidence": "ROLLBACK_NOT_REQUIRED|VERIFIED_ROLLBACK|PARTIAL_ROLLBACK|UNKNOWN_ROLLBACK",
601627
"requires_sudo": true,
602628
"detected_stack": ["systemd", "Docker/Compose", "nginx/reverse-proxy", "Bitcoin Core", "LND/Lightning", "Tor"],
603629
"next_best_action": "the single safest first command/action",
@@ -682,6 +708,9 @@ Respond STRICTLY with this JSON format:
682708
"dependencies": "required packages or tools, or null",
683709
"severity": "LOW|MEDIUM|HIGH|CRITICAL",
684710
"confidence": "LOW|MEDIUM|HIGH",
711+
"remediation_safety": "READ_ONLY_SAFE|REVERSIBLE_SAFE|PARTIAL_RISK|HIGH_RISK|DESTRUCTIVE",
712+
"evidence_quality": "DIRECT_EVIDENCE|PARTIAL_EVIDENCE|ASSUMPTION_HEAVY",
713+
"rollback_confidence": "ROLLBACK_NOT_REQUIRED|VERIFIED_ROLLBACK|PARTIAL_ROLLBACK|UNKNOWN_ROLLBACK",
685714
"requires_sudo": true,
686715
"destructive": false,
687716
"detected_stack": ["systemd", "Docker/Compose", "nginx/reverse-proxy", "Bitcoin Core", "LND/Lightning", "Tor"],
@@ -740,6 +769,9 @@ Respond STRICTLY with this JSON format:
740769
"risk_level": "LOW|MEDIUM|HIGH|CRITICAL",
741770
"severity": "LOW|MEDIUM|HIGH|CRITICAL",
742771
"confidence": "LOW|MEDIUM|HIGH",
772+
"remediation_safety": "READ_ONLY_SAFE|REVERSIBLE_SAFE|PARTIAL_RISK|HIGH_RISK|DESTRUCTIVE",
773+
"evidence_quality": "DIRECT_EVIDENCE|PARTIAL_EVIDENCE|ASSUMPTION_HEAVY",
774+
"rollback_confidence": "ROLLBACK_NOT_REQUIRED|VERIFIED_ROLLBACK|PARTIAL_ROLLBACK|UNKNOWN_ROLLBACK",
743775
"requires_sudo": true,
744776
"detected_stack": ["SSH", "nginx/reverse-proxy", "Docker/Compose", "Bitcoin Core", "LND/Lightning", "Tor"],
745777
"next_best_action": "the single safest first hardening action or validation command",
@@ -800,6 +832,9 @@ Respond STRICTLY with this JSON format:
800832
"risk_level": "LOW|MEDIUM|HIGH|CRITICAL",
801833
"severity": "LOW|MEDIUM|HIGH|CRITICAL",
802834
"confidence": "LOW|MEDIUM|HIGH",
835+
"remediation_safety": "READ_ONLY_SAFE|REVERSIBLE_SAFE|PARTIAL_RISK|HIGH_RISK|DESTRUCTIVE",
836+
"evidence_quality": "DIRECT_EVIDENCE|PARTIAL_EVIDENCE|ASSUMPTION_HEAVY",
837+
"rollback_confidence": "ROLLBACK_NOT_REQUIRED|VERIFIED_ROLLBACK|PARTIAL_ROLLBACK|UNKNOWN_ROLLBACK",
803838
"requires_sudo": true,
804839
"detected_stack": ["SSH", "TLS", "nginx/reverse-proxy", "Docker/Compose"],
805840
"next_best_action": "the single safest first action",

0 commit comments

Comments
 (0)