Skip to content

Commit ccfa3dc

Browse files
committed
fix(scanner): downgrade clean risk score to degraded on incomplete coverage
The scan summary reported a 0/100 "clean" verdict even when part of the scanner fleet failed, so a user saw an all-clear while 40% of scanners never ran. Coverage was only surfaced as a separate WARNING block. Tie risk-score confidence to scanner coverage: - SecurityScanSummary / ScanSummary carry scanners_run/failed/total. - GetScanSummary computes coverage on the primary pass and downgrades a would-be "clean" verdict to a new "degraded" status when any scanner failed. "dangerous"/"warnings"/"failed" verdicts are left intact. - CLI report annotates the Risk Score line with "(degraded — N of M scanners did not run)". - Swagger + CLI docs updated. Related #MCP-2401
1 parent be485ad commit ccfa3dc

9 files changed

Lines changed: 173 additions & 23 deletions

File tree

cmd/mcpproxy/security_cmd.go

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1823,19 +1823,28 @@ func printReportTable(serverName string, report map[string]interface{}, failedSc
18231823
scannedAt := getMapString(report, "scanned_at")
18241824
jobID := getMapString(report, "job_id")
18251825

1826+
// F-10 / MCP-2401: scanner coverage. Read counts up front so the risk-score
1827+
// line can flag low confidence when part of the fleet never ran.
1828+
scannersRun := int(getMapFloat(report, "scanners_run"))
1829+
scannersFailed := int(getMapFloat(report, "scanners_failed"))
1830+
scannersTotal := int(getMapFloat(report, "scanners_total"))
1831+
18261832
fmt.Printf("Security Report: %s\n", serverName)
18271833
if jobID != "" {
18281834
fmt.Printf("Scan ID: %s\n", jobID)
18291835
}
1830-
fmt.Printf("Risk Score: %s/100\n", riskScore)
1836+
riskLine := fmt.Sprintf("Risk Score: %s/100", riskScore)
1837+
if scannersFailed > 0 && scannersTotal > 0 {
1838+
// MCP-2401: the score was computed from an incomplete scan. Flag the
1839+
// degraded confidence on the number itself, not only in the warning
1840+
// block below, so a glance at "0/100" isn't read as an all-clear.
1841+
riskLine += fmt.Sprintf(" (degraded — %d of %d scanners did not run)", scannersFailed, scannersTotal)
1842+
}
1843+
fmt.Println(riskLine)
18311844
if scannedAt != "" {
18321845
fmt.Printf("Scanned: %s\n", formatTimestamp(scannedAt))
18331846
}
18341847

1835-
// F-10: scanner coverage line. Pull counts from the aggregated report.
1836-
scannersRun := int(getMapFloat(report, "scanners_run"))
1837-
scannersFailed := int(getMapFloat(report, "scanners_failed"))
1838-
scannersTotal := int(getMapFloat(report, "scanners_total"))
18391848
if scannersTotal > 0 {
18401849
line := fmt.Sprintf("Scanners: %d run, %d failed", scannersRun, scannersFailed)
18411850
if scannersFailed > 0 && len(failedScanners) > 0 {

cmd/mcpproxy/security_cmd_test.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,3 +427,50 @@ func TestPrintReportTableRendersScannerTimings(t *testing.T) {
427427
}
428428
}
429429
}
430+
431+
// TestPrintReportTableFlagsDegradedRiskScore verifies MCP-2401: when coverage
432+
// is incomplete the risk-score line itself carries a degraded marker, so a low
433+
// "X/100" number is not read as a trustworthy all-clear.
434+
func TestPrintReportTableFlagsDegradedRiskScore(t *testing.T) {
435+
report := map[string]interface{}{
436+
"job_id": "scan-foo-1",
437+
"risk_score": float64(0),
438+
"scanned_at": "2026-04-26T17:39:05Z",
439+
"scanners_run": float64(3),
440+
"scanners_failed": float64(2),
441+
"scanners_total": float64(5),
442+
"findings": []interface{}{},
443+
}
444+
out := captureStdout(t, func() {
445+
_ = printReportTable("foo", report, nil)
446+
})
447+
if !strings.Contains(out, "Risk Score:") || !strings.Contains(out, "degraded") {
448+
t.Errorf("expected degraded marker on the Risk Score line; got:\n%s", out)
449+
}
450+
if !strings.Contains(out, "2 of 5") {
451+
t.Errorf("expected coverage detail '2 of 5' on the Risk Score line; got:\n%s", out)
452+
}
453+
}
454+
455+
// TestPrintReportTableFullCoverageNoDegradedMarker ensures a complete scan does
456+
// not get a degraded marker on the risk-score line.
457+
func TestPrintReportTableFullCoverageNoDegradedMarker(t *testing.T) {
458+
report := map[string]interface{}{
459+
"job_id": "scan-foo-2",
460+
"risk_score": float64(0),
461+
"scanned_at": "2026-04-26T17:39:05Z",
462+
"scanners_run": float64(5),
463+
"scanners_failed": float64(0),
464+
"scanners_total": float64(5),
465+
"findings": []interface{}{},
466+
}
467+
out := captureStdout(t, func() {
468+
_ = printReportTable("foo", report, nil)
469+
})
470+
// The "Risk Score:" line must not be annotated as degraded.
471+
for _, line := range strings.Split(out, "\n") {
472+
if strings.HasPrefix(line, "Risk Score:") && strings.Contains(line, "degraded") {
473+
t.Errorf("did not expect degraded marker for full coverage; got line: %q", line)
474+
}
475+
}
476+
}

docs/cli/security-commands.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ Scan completed for "everything".
293293
294294
Security Report: everything
295295
Scan ID: scan-everything-1775804891180898000
296-
Risk Score: 0/100
296+
Risk Score: 0/100 (degraded — 1 of 7 scanners did not run)
297297
Scanned: 2026-04-10 10:08:19
298298
Scanners: 6 run, 1 failed (ramparts) of 7
299299
@@ -472,7 +472,7 @@ Use `-o json`, `-o yaml`, or `-o sarif` for machine-readable output.
472472
```
473473
Security Report: everything
474474
Scan ID: scan-everything-1775804891180898000
475-
Risk Score: 0/100
475+
Risk Score: 0/100 (degraded — 1 of 7 scanners did not run)
476476
Scanned: 2026-04-10 10:08:19
477477
Scanners: 6 run, 1 failed (ramparts) of 7
478478

internal/contracts/types.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,14 @@ type DiagnosticFixStep struct {
9797
type SecurityScanSummary struct {
9898
LastScanAt *time.Time `json:"last_scan_at,omitempty"`
9999
RiskScore int `json:"risk_score"` // 0-100
100-
Status string `json:"status"` // "clean", "warnings", "dangerous", "failed", "not_scanned", "scanning"
100+
Status string `json:"status"` // "clean", "degraded", "warnings", "dangerous", "failed", "not_scanned", "scanning"
101101
FindingCounts *FindingCounts `json:"finding_counts,omitempty"`
102+
// Scanner coverage for the primary scan pass. "degraded" status means
103+
// ScannersFailed > 0, so the risk score reflects an incomplete scan and a
104+
// low score should not be read as a trustworthy all-clear (MCP-2401).
105+
ScannersRun int `json:"scanners_run"`
106+
ScannersFailed int `json:"scanners_failed"`
107+
ScannersTotal int `json:"scanners_total"`
102108
}
103109

104110
// FindingCounts groups findings by user-facing threat category.

internal/security/scanner/service.go

Lines changed: 33 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1599,16 +1599,21 @@ func (s *Service) GetScanSummary(ctx context.Context, serverName string) *ScanSu
15991599
return summary
16001600
}
16011601

1602-
// Check scanner statuses on primary job
1602+
// Compute scanner coverage for the primary (security) scan pass. This drives
1603+
// the "degraded" verdict below: a clean/low risk score is not trustworthy
1604+
// when some scanners never ran (MCP-2401). If no scanner completed at all the
1605+
// scan is a flat failure, not merely degraded.
16031606
if len(primaryJob.ScannerStatuses) > 0 {
1604-
allFailed := true
16051607
for _, ss := range primaryJob.ScannerStatuses {
1606-
if ss.Status == ScanJobStatusCompleted {
1607-
allFailed = false
1608-
break
1608+
summary.ScannersTotal++
1609+
switch ss.Status {
1610+
case ScanJobStatusCompleted:
1611+
summary.ScannersRun++
1612+
case ScanJobStatusFailed:
1613+
summary.ScannersFailed++
16091614
}
16101615
}
1611-
if allFailed {
1616+
if summary.ScannersRun == 0 {
16121617
summary.Status = "failed"
16131618
s.cacheScanSummary(serverName, summary)
16141619
return summary
@@ -1653,6 +1658,7 @@ func (s *Service) GetScanSummary(ctx context.Context, serverName string) *ScanSu
16531658
summary.Status = "failed"
16541659
}
16551660
}
1661+
summary.degradeIfIncompleteCoverage()
16561662
s.cacheScanSummary(serverName, summary)
16571663
return summary
16581664
}
@@ -1685,17 +1691,37 @@ func (s *Service) GetScanSummary(ctx context.Context, serverName string) *ScanSu
16851691
summary.Status = "clean" // Only informational findings
16861692
}
16871693

1694+
// Incomplete coverage downgrades a would-be "clean" verdict (MCP-2401).
1695+
summary.degradeIfIncompleteCoverage()
1696+
16881697
// Cache for fast subsequent reads
16891698
s.cacheScanSummary(serverName, summary)
16901699
return summary
16911700
}
16921701

1702+
// degradeIfIncompleteCoverage downgrades a "clean" verdict to "degraded" when
1703+
// at least one scanner failed, so a low/zero risk score is not read as a
1704+
// trustworthy all-clear while a chunk of the scanner fleet never ran. Findings-
1705+
// driven verdicts ("dangerous"/"warnings") are left intact — they already
1706+
// signal risk; coverage can only have hidden more, never less (MCP-2401).
1707+
func (sum *ScanSummary) degradeIfIncompleteCoverage() {
1708+
if sum.Status == "clean" && sum.ScannersFailed > 0 {
1709+
sum.Status = "degraded"
1710+
}
1711+
}
1712+
16931713
// ScanSummary is a compact representation of scan status for the server list.
16941714
type ScanSummary struct {
16951715
LastScanAt *time.Time `json:"last_scan_at,omitempty"`
16961716
RiskScore int `json:"risk_score"`
1697-
Status string `json:"status"` // clean, warnings, dangerous, failed, not_scanned, scanning
1717+
Status string `json:"status"` // clean, degraded, warnings, dangerous, failed, not_scanned, scanning
16981718
FindingCounts *FindingCounts `json:"finding_counts,omitempty"`
1719+
// Scanner coverage for the primary (security) scan pass. When ScannersFailed
1720+
// > 0 the risk score is computed from incomplete data, so a "clean"/low score
1721+
// is not trustworthy — Status is reported as "degraded" instead (MCP-2401).
1722+
ScannersRun int `json:"scanners_run"`
1723+
ScannersFailed int `json:"scanners_failed"`
1724+
ScannersTotal int `json:"scanners_total"`
16991725
}
17001726

17011727
// FindingCounts groups findings by user-facing threat level.

internal/security/scanner/service_test.go

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1176,9 +1176,53 @@ func TestServiceGetScanSummaryPartialSuccess(t *testing.T) {
11761176
if summary == nil {
11771177
t.Fatal("expected non-nil summary")
11781178
}
1179-
// At least one scanner succeeded, so status should be "clean" (no findings)
1180-
if summary.Status != "clean" {
1181-
t.Errorf("expected status 'clean', got %q", summary.Status)
1179+
// MCP-2401: a scanner failed, so coverage is incomplete. A "0/100 clean"
1180+
// verdict here is misleading — the score's confidence is degraded because
1181+
// 1 of 2 scanners never ran. Status must reflect that, not plain "clean".
1182+
if summary.Status != "degraded" {
1183+
t.Errorf("expected status 'degraded' for incomplete coverage, got %q", summary.Status)
1184+
}
1185+
if summary.ScannersTotal != 2 || summary.ScannersRun != 1 || summary.ScannersFailed != 1 {
1186+
t.Errorf("expected coverage 1 run / 1 failed / 2 total, got %d run / %d failed / %d total",
1187+
summary.ScannersRun, summary.ScannersFailed, summary.ScannersTotal)
1188+
}
1189+
}
1190+
1191+
// TestServiceGetScanSummaryDegradedWithInfoFindings covers MCP-2401 when the
1192+
// surviving scanners produced only informational findings but coverage is
1193+
// incomplete — the verdict must degrade rather than read "clean".
1194+
func TestServiceGetScanSummaryDegradedWithInfoFindings(t *testing.T) {
1195+
svc, store, _ := newTestService(t)
1196+
1197+
now := time.Now()
1198+
_ = store.SaveScanJob(&ScanJob{
1199+
ID: "j-degraded-info",
1200+
ServerName: "server-a",
1201+
Status: ScanJobStatusCompleted,
1202+
Scanners: []string{"s1", "s2"},
1203+
StartedAt: now,
1204+
ScannerStatuses: []ScannerJobStatus{
1205+
{ScannerID: "s1", Status: ScanJobStatusCompleted, FindingsCount: 1},
1206+
{ScannerID: "s2", Status: ScanJobStatusFailed, Error: "image not found"},
1207+
},
1208+
})
1209+
_ = store.SaveScanReport(&ScanReport{
1210+
ID: "r1", JobID: "j-degraded-info", ServerName: "server-a", ScannerID: "s1",
1211+
Findings: []ScanFinding{
1212+
{RuleID: "info-1", ThreatLevel: ThreatLevelInfo, Severity: "info", Title: "informational"},
1213+
},
1214+
ScannedAt: now,
1215+
})
1216+
1217+
summary := svc.GetScanSummary(context.Background(), "server-a")
1218+
if summary == nil {
1219+
t.Fatal("expected non-nil summary")
1220+
}
1221+
if summary.Status != "degraded" {
1222+
t.Errorf("expected status 'degraded' for incomplete coverage with info-only findings, got %q", summary.Status)
1223+
}
1224+
if summary.ScannersFailed != 1 {
1225+
t.Errorf("expected 1 failed scanner, got %d", summary.ScannersFailed)
11821226
}
11831227
}
11841228

@@ -1208,6 +1252,11 @@ func TestServiceGetScanSummaryClean(t *testing.T) {
12081252
if summary.Status != "clean" {
12091253
t.Errorf("expected status 'clean', got %q", summary.Status)
12101254
}
1255+
// MCP-2401: full coverage (no failed scanners) keeps the verdict "clean".
1256+
if summary.ScannersTotal != 1 || summary.ScannersRun != 1 || summary.ScannersFailed != 0 {
1257+
t.Errorf("expected coverage 1 run / 0 failed / 1 total, got %d run / %d failed / %d total",
1258+
summary.ScannersRun, summary.ScannersFailed, summary.ScannersTotal)
1259+
}
12111260
}
12121261

12131262
func TestServiceGetScanSummaryEmptyScan(t *testing.T) {

internal/server/server.go

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2694,9 +2694,12 @@ func (a *scanSummaryEnricherAdapter) GetSecurityScanSummary(ctx context.Context,
26942694
return nil
26952695
}
26962696
out := &contracts.SecurityScanSummary{
2697-
LastScanAt: summary.LastScanAt,
2698-
RiskScore: summary.RiskScore,
2699-
Status: summary.Status,
2697+
LastScanAt: summary.LastScanAt,
2698+
RiskScore: summary.RiskScore,
2699+
Status: summary.Status,
2700+
ScannersRun: summary.ScannersRun,
2701+
ScannersFailed: summary.ScannersFailed,
2702+
ScannersTotal: summary.ScannersTotal,
27002703
}
27012704
if summary.FindingCounts != nil {
27022705
out.FindingCounts = &contracts.FindingCounts{

oas/docs.go

Lines changed: 1 addition & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

oas/swagger.yaml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1771,8 +1771,18 @@ components:
17711771
risk_score:
17721772
description: 0-100
17731773
type: integer
1774+
scanners_failed:
1775+
type: integer
1776+
scanners_run:
1777+
description: |-
1778+
Scanner coverage for the primary scan pass. "degraded" status means
1779+
ScannersFailed > 0, so the risk score reflects an incomplete scan and a
1780+
low score should not be read as a trustworthy all-clear (MCP-2401).
1781+
type: integer
1782+
scanners_total:
1783+
type: integer
17741784
status:
1775-
description: '"clean", "warnings", "dangerous", "failed", "not_scanned",
1785+
description: '"clean", "degraded", "warnings", "dangerous", "failed", "not_scanned",
17761786
"scanning"'
17771787
type: string
17781788
type: object

0 commit comments

Comments
 (0)