Skip to content

Commit 16d80ad

Browse files
committed
fix(scanner): downgrade clean risk score to degraded on incomplete coverage
The scan summary reported a 0/100 "clean" verdict even when part of the scanner fleet failed, so a user saw an all-clear while 40% of scanners never ran. Coverage was only surfaced as a separate WARNING block. Tie risk-score confidence to scanner coverage: - SecurityScanSummary / ScanSummary carry scanners_run/failed/total. - GetScanSummary computes coverage on the primary pass and downgrades a would-be "clean" verdict to a new "degraded" status when any scanner failed. "dangerous"/"warnings"/"failed" verdicts are left intact. - CLI report annotates the Risk Score line with "(degraded — N of M scanners did not run)". - Swagger + CLI docs updated. Related #MCP-2401
1 parent 61d78da commit 16d80ad

9 files changed

Lines changed: 173 additions & 23 deletions

File tree

cmd/mcpproxy/security_cmd.go

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1742,19 +1742,28 @@ func printReportTable(serverName string, report map[string]interface{}, failedSc
17421742
scannedAt := getMapString(report, "scanned_at")
17431743
jobID := getMapString(report, "job_id")
17441744

1745+
// F-10 / MCP-2401: scanner coverage. Read counts up front so the risk-score
1746+
// line can flag low confidence when part of the fleet never ran.
1747+
scannersRun := int(getMapFloat(report, "scanners_run"))
1748+
scannersFailed := int(getMapFloat(report, "scanners_failed"))
1749+
scannersTotal := int(getMapFloat(report, "scanners_total"))
1750+
17451751
fmt.Printf("Security Report: %s\n", serverName)
17461752
if jobID != "" {
17471753
fmt.Printf("Scan ID: %s\n", jobID)
17481754
}
1749-
fmt.Printf("Risk Score: %s/100\n", riskScore)
1755+
riskLine := fmt.Sprintf("Risk Score: %s/100", riskScore)
1756+
if scannersFailed > 0 && scannersTotal > 0 {
1757+
// MCP-2401: the score was computed from an incomplete scan. Flag the
1758+
// degraded confidence on the number itself, not only in the warning
1759+
// block below, so a glance at "0/100" isn't read as an all-clear.
1760+
riskLine += fmt.Sprintf(" (degraded — %d of %d scanners did not run)", scannersFailed, scannersTotal)
1761+
}
1762+
fmt.Println(riskLine)
17501763
if scannedAt != "" {
17511764
fmt.Printf("Scanned: %s\n", formatTimestamp(scannedAt))
17521765
}
17531766

1754-
// F-10: scanner coverage line. Pull counts from the aggregated report.
1755-
scannersRun := int(getMapFloat(report, "scanners_run"))
1756-
scannersFailed := int(getMapFloat(report, "scanners_failed"))
1757-
scannersTotal := int(getMapFloat(report, "scanners_total"))
17581767
if scannersTotal > 0 {
17591768
line := fmt.Sprintf("Scanners: %d run, %d failed", scannersRun, scannersFailed)
17601769
if scannersFailed > 0 && len(failedScanners) > 0 {

cmd/mcpproxy/security_cmd_test.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -329,3 +329,50 @@ func TestPrintReportTableRendersFailedScannerReasons(t *testing.T) {
329329
}
330330
}
331331
}
332+
333+
// TestPrintReportTableFlagsDegradedRiskScore verifies MCP-2401: when coverage
334+
// is incomplete the risk-score line itself carries a degraded marker, so a low
335+
// "X/100" number is not read as a trustworthy all-clear.
336+
func TestPrintReportTableFlagsDegradedRiskScore(t *testing.T) {
337+
report := map[string]interface{}{
338+
"job_id": "scan-foo-1",
339+
"risk_score": float64(0),
340+
"scanned_at": "2026-04-26T17:39:05Z",
341+
"scanners_run": float64(3),
342+
"scanners_failed": float64(2),
343+
"scanners_total": float64(5),
344+
"findings": []interface{}{},
345+
}
346+
out := captureStdout(t, func() {
347+
_ = printReportTable("foo", report, nil)
348+
})
349+
if !strings.Contains(out, "Risk Score:") || !strings.Contains(out, "degraded") {
350+
t.Errorf("expected degraded marker on the Risk Score line; got:\n%s", out)
351+
}
352+
if !strings.Contains(out, "2 of 5") {
353+
t.Errorf("expected coverage detail '2 of 5' on the Risk Score line; got:\n%s", out)
354+
}
355+
}
356+
357+
// TestPrintReportTableFullCoverageNoDegradedMarker ensures a complete scan does
358+
// not get a degraded marker on the risk-score line.
359+
func TestPrintReportTableFullCoverageNoDegradedMarker(t *testing.T) {
360+
report := map[string]interface{}{
361+
"job_id": "scan-foo-2",
362+
"risk_score": float64(0),
363+
"scanned_at": "2026-04-26T17:39:05Z",
364+
"scanners_run": float64(5),
365+
"scanners_failed": float64(0),
366+
"scanners_total": float64(5),
367+
"findings": []interface{}{},
368+
}
369+
out := captureStdout(t, func() {
370+
_ = printReportTable("foo", report, nil)
371+
})
372+
// The "Risk Score:" line must not be annotated as degraded.
373+
for _, line := range strings.Split(out, "\n") {
374+
if strings.HasPrefix(line, "Risk Score:") && strings.Contains(line, "degraded") {
375+
t.Errorf("did not expect degraded marker for full coverage; got line: %q", line)
376+
}
377+
}
378+
}

docs/cli/security-commands.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ Scan completed for "everything".
293293
294294
Security Report: everything
295295
Scan ID: scan-everything-1775804891180898000
296-
Risk Score: 0/100
296+
Risk Score: 0/100 (degraded — 1 of 7 scanners did not run)
297297
Scanned: 2026-04-10 10:08:19
298298
Scanners: 6 run, 1 failed (ramparts) of 7
299299
@@ -464,7 +464,7 @@ Use `-o json`, `-o yaml`, or `-o sarif` for machine-readable output.
464464
```
465465
Security Report: everything
466466
Scan ID: scan-everything-1775804891180898000
467-
Risk Score: 0/100
467+
Risk Score: 0/100 (degraded — 1 of 7 scanners did not run)
468468
Scanned: 2026-04-10 10:08:19
469469
Scanners: 6 run, 1 failed (ramparts) of 7
470470

internal/contracts/types.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,14 @@ type DiagnosticFixStep struct {
9797
type SecurityScanSummary struct {
9898
LastScanAt *time.Time `json:"last_scan_at,omitempty"`
9999
RiskScore int `json:"risk_score"` // 0-100
100-
Status string `json:"status"` // "clean", "warnings", "dangerous", "failed", "not_scanned", "scanning"
100+
Status string `json:"status"` // "clean", "degraded", "warnings", "dangerous", "failed", "not_scanned", "scanning"
101101
FindingCounts *FindingCounts `json:"finding_counts,omitempty"`
102+
// Scanner coverage for the primary scan pass. "degraded" status means
103+
// ScannersFailed > 0, so the risk score reflects an incomplete scan and a
104+
// low score should not be read as a trustworthy all-clear (MCP-2401).
105+
ScannersRun int `json:"scanners_run"`
106+
ScannersFailed int `json:"scanners_failed"`
107+
ScannersTotal int `json:"scanners_total"`
102108
}
103109

104110
// FindingCounts groups findings by user-facing threat category.

internal/security/scanner/service.go

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1546,16 +1546,21 @@ func (s *Service) GetScanSummary(ctx context.Context, serverName string) *ScanSu
15461546
return summary
15471547
}
15481548

1549-
// Check scanner statuses on primary job
1549+
// Compute scanner coverage for the primary (security) scan pass. This drives
1550+
// the "degraded" verdict below: a clean/low risk score is not trustworthy
1551+
// when some scanners never ran (MCP-2401). If no scanner completed at all the
1552+
// scan is a flat failure, not merely degraded.
15501553
if len(primaryJob.ScannerStatuses) > 0 {
1551-
allFailed := true
15521554
for _, ss := range primaryJob.ScannerStatuses {
1553-
if ss.Status == ScanJobStatusCompleted {
1554-
allFailed = false
1555-
break
1555+
summary.ScannersTotal++
1556+
switch ss.Status {
1557+
case ScanJobStatusCompleted:
1558+
summary.ScannersRun++
1559+
case ScanJobStatusFailed:
1560+
summary.ScannersFailed++
15561561
}
15571562
}
1558-
if allFailed {
1563+
if summary.ScannersRun == 0 {
15591564
summary.Status = "failed"
15601565
s.cacheScanSummary(serverName, summary)
15611566
return summary
@@ -1600,6 +1605,7 @@ func (s *Service) GetScanSummary(ctx context.Context, serverName string) *ScanSu
16001605
summary.Status = "failed"
16011606
}
16021607
}
1608+
summary.degradeIfIncompleteCoverage()
16031609
s.cacheScanSummary(serverName, summary)
16041610
return summary
16051611
}
@@ -1632,17 +1638,37 @@ func (s *Service) GetScanSummary(ctx context.Context, serverName string) *ScanSu
16321638
summary.Status = "clean" // Only informational findings
16331639
}
16341640

1641+
// Incomplete coverage downgrades a would-be "clean" verdict (MCP-2401).
1642+
summary.degradeIfIncompleteCoverage()
1643+
16351644
// Cache for fast subsequent reads
16361645
s.cacheScanSummary(serverName, summary)
16371646
return summary
16381647
}
16391648

1649+
// degradeIfIncompleteCoverage downgrades a "clean" verdict to "degraded" when
1650+
// at least one scanner failed, so a low/zero risk score is not read as a
1651+
// trustworthy all-clear while a chunk of the scanner fleet never ran. Findings-
1652+
// driven verdicts ("dangerous"/"warnings") are left intact — they already
1653+
// signal risk; coverage can only have hidden more, never less (MCP-2401).
1654+
func (sum *ScanSummary) degradeIfIncompleteCoverage() {
1655+
if sum.Status == "clean" && sum.ScannersFailed > 0 {
1656+
sum.Status = "degraded"
1657+
}
1658+
}
1659+
16401660
// ScanSummary is a compact representation of scan status for the server list.
16411661
type ScanSummary struct {
16421662
LastScanAt *time.Time `json:"last_scan_at,omitempty"`
16431663
RiskScore int `json:"risk_score"`
1644-
Status string `json:"status"` // clean, warnings, dangerous, failed, not_scanned, scanning
1664+
Status string `json:"status"` // clean, degraded, warnings, dangerous, failed, not_scanned, scanning
16451665
FindingCounts *FindingCounts `json:"finding_counts,omitempty"`
1666+
// Scanner coverage for the primary (security) scan pass. When ScannersFailed
1667+
// > 0 the risk score is computed from incomplete data, so a "clean"/low score
1668+
// is not trustworthy — Status is reported as "degraded" instead (MCP-2401).
1669+
ScannersRun int `json:"scanners_run"`
1670+
ScannersFailed int `json:"scanners_failed"`
1671+
ScannersTotal int `json:"scanners_total"`
16461672
}
16471673

16481674
// FindingCounts groups findings by user-facing threat level.

internal/security/scanner/service_test.go

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1049,9 +1049,53 @@ func TestServiceGetScanSummaryPartialSuccess(t *testing.T) {
10491049
if summary == nil {
10501050
t.Fatal("expected non-nil summary")
10511051
}
1052-
// At least one scanner succeeded, so status should be "clean" (no findings)
1053-
if summary.Status != "clean" {
1054-
t.Errorf("expected status 'clean', got %q", summary.Status)
1052+
// MCP-2401: a scanner failed, so coverage is incomplete. A "0/100 clean"
1053+
// verdict here is misleading — the score's confidence is degraded because
1054+
// 1 of 2 scanners never ran. Status must reflect that, not plain "clean".
1055+
if summary.Status != "degraded" {
1056+
t.Errorf("expected status 'degraded' for incomplete coverage, got %q", summary.Status)
1057+
}
1058+
if summary.ScannersTotal != 2 || summary.ScannersRun != 1 || summary.ScannersFailed != 1 {
1059+
t.Errorf("expected coverage 1 run / 1 failed / 2 total, got %d run / %d failed / %d total",
1060+
summary.ScannersRun, summary.ScannersFailed, summary.ScannersTotal)
1061+
}
1062+
}
1063+
1064+
// TestServiceGetScanSummaryDegradedWithInfoFindings covers MCP-2401 when the
1065+
// surviving scanners produced only informational findings but coverage is
1066+
// incomplete — the verdict must degrade rather than read "clean".
1067+
func TestServiceGetScanSummaryDegradedWithInfoFindings(t *testing.T) {
1068+
svc, store, _ := newTestService(t)
1069+
1070+
now := time.Now()
1071+
_ = store.SaveScanJob(&ScanJob{
1072+
ID: "j-degraded-info",
1073+
ServerName: "server-a",
1074+
Status: ScanJobStatusCompleted,
1075+
Scanners: []string{"s1", "s2"},
1076+
StartedAt: now,
1077+
ScannerStatuses: []ScannerJobStatus{
1078+
{ScannerID: "s1", Status: ScanJobStatusCompleted, FindingsCount: 1},
1079+
{ScannerID: "s2", Status: ScanJobStatusFailed, Error: "image not found"},
1080+
},
1081+
})
1082+
_ = store.SaveScanReport(&ScanReport{
1083+
ID: "r1", JobID: "j-degraded-info", ServerName: "server-a", ScannerID: "s1",
1084+
Findings: []ScanFinding{
1085+
{RuleID: "info-1", ThreatLevel: ThreatLevelInfo, Severity: "info", Title: "informational"},
1086+
},
1087+
ScannedAt: now,
1088+
})
1089+
1090+
summary := svc.GetScanSummary(context.Background(), "server-a")
1091+
if summary == nil {
1092+
t.Fatal("expected non-nil summary")
1093+
}
1094+
if summary.Status != "degraded" {
1095+
t.Errorf("expected status 'degraded' for incomplete coverage with info-only findings, got %q", summary.Status)
1096+
}
1097+
if summary.ScannersFailed != 1 {
1098+
t.Errorf("expected 1 failed scanner, got %d", summary.ScannersFailed)
10551099
}
10561100
}
10571101

@@ -1081,6 +1125,11 @@ func TestServiceGetScanSummaryClean(t *testing.T) {
10811125
if summary.Status != "clean" {
10821126
t.Errorf("expected status 'clean', got %q", summary.Status)
10831127
}
1128+
// MCP-2401: full coverage (no failed scanners) keeps the verdict "clean".
1129+
if summary.ScannersTotal != 1 || summary.ScannersRun != 1 || summary.ScannersFailed != 0 {
1130+
t.Errorf("expected coverage 1 run / 0 failed / 1 total, got %d run / %d failed / %d total",
1131+
summary.ScannersRun, summary.ScannersFailed, summary.ScannersTotal)
1132+
}
10841133
}
10851134

10861135
func TestServiceGetScanSummaryEmptyScan(t *testing.T) {

internal/server/server.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2694,9 +2694,12 @@ func (a *scanSummaryEnricherAdapter) GetSecurityScanSummary(ctx context.Context,
26942694
return nil
26952695
}
26962696
out := &contracts.SecurityScanSummary{
2697-
LastScanAt: summary.LastScanAt,
2698-
RiskScore: summary.RiskScore,
2699-
Status: summary.Status,
2697+
LastScanAt: summary.LastScanAt,
2698+
RiskScore: summary.RiskScore,
2699+
Status: summary.Status,
2700+
ScannersRun: summary.ScannersRun,
2701+
ScannersFailed: summary.ScannersFailed,
2702+
ScannersTotal: summary.ScannersTotal,
27002703
}
27012704
if summary.FindingCounts != nil {
27022705
out.FindingCounts = &contracts.FindingCounts{

oas/docs.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

oas/swagger.yaml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1771,8 +1771,18 @@ components:
17711771
risk_score:
17721772
description: 0-100
17731773
type: integer
1774+
scanners_failed:
1775+
type: integer
1776+
scanners_run:
1777+
description: |-
1778+
Scanner coverage for the primary scan pass. "degraded" status means
1779+
ScannersFailed > 0, so the risk score reflects an incomplete scan and a
1780+
low score should not be read as a trustworthy all-clear (MCP-2401).
1781+
type: integer
1782+
scanners_total:
1783+
type: integer
17741784
status:
1775-
description: '"clean", "warnings", "dangerous", "failed", "not_scanned",
1785+
description: '"clean", "degraded", "warnings", "dangerous", "failed", "not_scanned",
17761786
"scanning"'
17771787
type: string
17781788
type: object

0 commit comments

Comments
 (0)