Merge pull request #97 from stacknil/codex/v1rc-evidence-confidence-labels

stacknil · web-flow · commit da7aa1bbd3f0 · 2026-06-30T10:27:28.000+08:00
feat(sbom): update evidence confidence labels
diff --git a/scripts/validate-reviewer-routes.py b/scripts/validate-reviewer-routes.py
@@ -208,8 +208,8 @@
         "local_manifest_only",
         "sbom_present",
         "policy_matched",
-        "enrichment_mocked",
-        "enrichment_live",
+        "enrichment_recorded",
+        "provenance_recorded",
         "not a package safety verdict",
         "not a CVE result",
     ),
diff --git a/tools/sbom-diff-and-risk/docs/report-schema.md b/tools/sbom-diff-and-risk/docs/report-schema.md
@@ -130,8 +130,8 @@ not a CVE result. The same value appears at top level, in `summary`, and in
 | `local_manifest_only` | The report was produced from local manifest-style inputs without SBOM input, policy matches, or enrichment evidence. |
 | `sbom_present` | At least one input is an SBOM format such as CycloneDX JSON or SPDX JSON. |
 | `policy_matched` | Local policy evaluation produced at least one blocking, warning, or suppressed policy match. |
-| `enrichment_mocked` | Enrichment-shaped evidence is present without recorded live network access, or the report explicitly marks constructed snapshot evidence as mocked. |
-| `enrichment_live` | Opt-in enrichment recorded live network access for PyPI provenance or OpenSSF Scorecard evidence. |
+| `enrichment_recorded` | Optional non-provenance enrichment evidence, such as OpenSSF Scorecard evidence, was recorded for the report. |
+| `provenance_recorded` | PyPI provenance evidence or provenance-enrichment metadata was recorded for the report. |
 
 `summary.policy` appears only when a policy is applied. Absence of
 `summary.policy` means policy was not used, not that policy evaluation failed.
diff --git a/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md b/tools/sbom-diff-and-risk/docs/summary-json-ci-cookbook.md
@@ -81,7 +81,7 @@ if ($newPackageCount -gt $maxNewPackages) {
 - `summary.enrichment` appears only when PyPI or Scorecard enrichment is used.
 - `summary.evidence_confidence` is always present and can be
   `local_manifest_only`, `sbom_present`, `policy_matched`,
-  `enrichment_mocked`, or `enrichment_live`.
+  `enrichment_recorded`, or `provenance_recorded`.
 - `unchanged` is absent because unchanged components are not modeled.
 - Absence of `summary.policy` or `summary.enrichment` means the feature was
   not used, not that it failed.
diff --git a/tools/sbom-diff-and-risk/examples/sample-provenance-report.json b/tools/sbom-diff-and-risk/examples/sample-provenance-report.json
@@ -12,7 +12,7 @@
       "suspicious_source": 0,
       "not_evaluated": 0
     },
-    "evidence_confidence": "enrichment_mocked",
+    "evidence_confidence": "provenance_recorded",
     "policy": {
       "status": "fail",
       "blocking": 2,
@@ -33,7 +33,7 @@
       }
     }
   },
-  "evidence_confidence": "enrichment_mocked",
+  "evidence_confidence": "provenance_recorded",
   "components": {
     "added": [
       {
@@ -746,7 +746,7 @@
       },
       "exit_code": 1
     },
-    "evidence_confidence": "enrichment_mocked",
+    "evidence_confidence": "provenance_recorded",
     "enrichment": {
       "mode": "opt_in_pypi",
       "pypi_enabled": true,
diff --git a/tools/sbom-diff-and-risk/examples/sample-provenance-report.md b/tools/sbom-diff-and-risk/examples/sample-provenance-report.md
@@ -6,7 +6,7 @@
 - Added: 2
 - Removed: 0
 - Version changes: 1
-- Evidence confidence: enrichment_mocked
+- Evidence confidence: provenance_recorded
 
 ## Risk buckets
 - new_package: 2
diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.json
@@ -12,7 +12,7 @@
       "suspicious_source": 0,
       "not_evaluated": 0
     },
-    "evidence_confidence": "enrichment_mocked",
+    "evidence_confidence": "enrichment_recorded",
     "policy": {
       "status": "warn",
       "blocking": 0,
@@ -32,7 +32,7 @@
       }
     }
   },
-  "evidence_confidence": "enrichment_mocked",
+  "evidence_confidence": "enrichment_recorded",
   "components": {
     "added": [
       {
@@ -575,7 +575,7 @@
       },
       "exit_code": 0
     },
-    "evidence_confidence": "enrichment_mocked",
+    "evidence_confidence": "enrichment_recorded",
     "enrichment": {
       "mode": "opt_in_scorecard",
       "pypi_enabled": false,
diff --git a/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md b/tools/sbom-diff-and-risk/examples/sample-scorecard-report.md
@@ -6,7 +6,7 @@
 - Added: 2
 - Removed: 0
 - Version changes: 1
-- Evidence confidence: enrichment_mocked
+- Evidence confidence: enrichment_recorded
 
 ## Risk buckets
 - new_package: 2
diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/evidence_confidence.py
@@ -9,10 +9,11 @@ def evidence_confidence_for_report(report: CompareReport) -> EvidenceConfidence:
     if report.metadata.evidence_confidence is not None:
         return report.metadata.evidence_confidence
 
-    if _has_enrichment_evidence(report):
-        if _has_live_enrichment(report):
-            return EvidenceConfidence.ENRICHMENT_LIVE
-        return EvidenceConfidence.ENRICHMENT_MOCKED
+    if _has_provenance_record(report):
+        return EvidenceConfidence.PROVENANCE_RECORDED
+
+    if _has_enrichment_record(report):
+        return EvidenceConfidence.ENRICHMENT_RECORDED
 
     if _has_policy_match(report):
         return EvidenceConfidence.POLICY_MATCHED
@@ -27,21 +28,20 @@ def evidence_confidence_value(report: CompareReport) -> str:
     return evidence_confidence_for_report(report).value
 
 
-def _has_enrichment_evidence(report: CompareReport) -> bool:
+def _has_provenance_record(report: CompareReport) -> bool:
     metadata = report.metadata.enrichment
-    if metadata.pypi_enabled or metadata.scorecard_enabled:
+    if metadata.pypi_enabled:
         return True
 
-    return any(component.provenance is not None or component.scorecard is not None for component in _all_components(report))
+    return any(component.provenance is not None for component in _all_components(report))
 
 
-def _has_live_enrichment(report: CompareReport) -> bool:
+def _has_enrichment_record(report: CompareReport) -> bool:
     metadata = report.metadata.enrichment
-    return (
-        metadata.network_access_performed
-        or metadata.pypi_network_access_performed
-        or metadata.scorecard_network_access_performed
-    )
+    if metadata.scorecard_enabled:
+        return True
+
+    return any(component.scorecard is not None for component in _all_components(report))
 
 
 def _has_policy_match(report: CompareReport) -> bool:
diff --git a/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py b/tools/sbom-diff-and-risk/src/sbom_diff_risk/models.py
@@ -22,8 +22,8 @@ class EvidenceConfidence(StrEnum):
     LOCAL_MANIFEST_ONLY = "local_manifest_only"
     SBOM_PRESENT = "sbom_present"
     POLICY_MATCHED = "policy_matched"
-    ENRICHMENT_MOCKED = "enrichment_mocked"
-    ENRICHMENT_LIVE = "enrichment_live"
+    ENRICHMENT_RECORDED = "enrichment_recorded"
+    PROVENANCE_RECORDED = "provenance_recorded"
 
 
 class ProvenanceStatus(StrEnum):
diff --git a/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py b/tools/sbom-diff-and-risk/tests/test_cli_no_enrichment_regression.py
@@ -131,7 +131,7 @@ def __init__(self, *args, **kwargs) -> None:  # noqa: ANN002, ANN003
     assert payload["metadata"]["enrichment"]["mode"] == "opt_in_pypi"
     assert payload["metadata"]["enrichment"]["pypi_enabled"] is True
     assert payload["metadata"]["enrichment"]["pypi_timeout_seconds"] == 2.5
-    assert payload["evidence_confidence"] == "enrichment_mocked"
+    assert payload["evidence_confidence"] == "provenance_recorded"
     assert payload["notes"][1] == "PyPI provenance enrichment was requested explicitly."
     assert payload["trust_signal_notes"] == []
 
@@ -200,5 +200,5 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata:
     assert payload["metadata"]["enrichment"]["mode"] == "opt_in_scorecard"
     assert payload["metadata"]["enrichment"]["scorecard_enabled"] is True
     assert payload["metadata"]["enrichment"]["scorecard_timeout_seconds"] == 4.25
-    assert payload["evidence_confidence"] == "enrichment_mocked"
+    assert payload["evidence_confidence"] == "enrichment_recorded"
     assert payload["notes"][1] == "OpenSSF Scorecard enrichment was requested explicitly."
diff --git a/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py b/tools/sbom-diff-and-risk/tests/test_cli_summary_json.py
@@ -161,7 +161,7 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata:
             },
         },
     }
-    assert payload["evidence_confidence"] == "enrichment_mocked"
+    assert payload["evidence_confidence"] == "provenance_recorded"
     assert "policy" not in payload
 
 
@@ -224,7 +224,7 @@ def build_report_metadata(self) -> ReportEnrichmentMetadata:
             },
         },
     }
-    assert payload["evidence_confidence"] == "enrichment_mocked"
+    assert payload["evidence_confidence"] == "enrichment_recorded"
     assert "policy" not in payload
 
 
diff --git a/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py b/tools/sbom-diff-and-risk/tests/test_evidence_confidence.py
@@ -13,6 +13,16 @@
 from sbom_diff_risk.policy_models import PolicyEvaluation, PolicyLevel, PolicyViolation
 
 
+def test_evidence_confidence_values_are_release_facing_labels() -> None:
+    assert {item.value for item in EvidenceConfidence} == {
+        "local_manifest_only",
+        "sbom_present",
+        "policy_matched",
+        "enrichment_recorded",
+        "provenance_recorded",
+    }
+
+
 def test_evidence_confidence_defaults_to_local_manifest_only() -> None:
     report = _minimal_report(before_format="requirements-txt", after_format="requirements-txt")
 
@@ -43,7 +53,7 @@ def test_evidence_confidence_marks_policy_matched() -> None:
     assert evidence_confidence_for_report(report) is EvidenceConfidence.POLICY_MATCHED
 
 
-def test_evidence_confidence_marks_mocked_enrichment_without_network() -> None:
+def test_evidence_confidence_marks_provenance_recorded_when_pypi_enrichment_is_used() -> None:
     report = _minimal_report(
         before_format="requirements-txt",
         after_format="requirements-txt",
@@ -55,29 +65,29 @@ def test_evidence_confidence_marks_mocked_enrichment_without_network() -> None:
         ),
     )
 
-    assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_MOCKED
+    assert evidence_confidence_for_report(report) is EvidenceConfidence.PROVENANCE_RECORDED
 
 
-def test_evidence_confidence_marks_live_enrichment_when_network_access_was_performed() -> None:
+def test_evidence_confidence_marks_enrichment_recorded_when_scorecard_enrichment_is_used() -> None:
     report = _minimal_report(
         before_format="requirements-txt",
         after_format="requirements-txt",
         enrichment=ReportEnrichmentMetadata(
-            mode="opt_in_pypi",
-            pypi_enabled=True,
-            pypi_network_access_performed=True,
+            mode="opt_in_scorecard",
+            scorecard_enabled=True,
+            scorecard_network_access_performed=True,
             network_access_performed=True,
         ),
     )
 
-    assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_LIVE
+    assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_RECORDED
 
 
-def test_evidence_confidence_allows_explicit_mock_override_for_constructed_snapshots() -> None:
+def test_evidence_confidence_allows_explicit_recorded_override_for_constructed_snapshots() -> None:
     report = _minimal_report(
         before_format="requirements-txt",
         after_format="requirements-txt",
-        evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED,
+        evidence_confidence=EvidenceConfidence.PROVENANCE_RECORDED,
         enrichment=ReportEnrichmentMetadata(
             mode="opt_in_pypi",
             pypi_enabled=True,
@@ -86,7 +96,7 @@ def test_evidence_confidence_allows_explicit_mock_override_for_constructed_snaps
         ),
     )
 
-    assert evidence_confidence_for_report(report) is EvidenceConfidence.ENRICHMENT_MOCKED
+    assert evidence_confidence_for_report(report) is EvidenceConfidence.PROVENANCE_RECORDED
 
 
 def _minimal_report(
diff --git a/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py b/tools/sbom-diff-and-risk/tests/test_provenance_reporting.py
@@ -59,7 +59,7 @@ def test_provenance_report_json_includes_provenance_policy_summary() -> None:
         "warning": 1,
         "suppressed": 0,
     }
-    assert payload["summary"]["evidence_confidence"] == "enrichment_mocked"
+    assert payload["summary"]["evidence_confidence"] == "provenance_recorded"
     assert payload["summary"]["enrichment"] == {
         "status": "used",
         "mode": "opt_in_pypi",
@@ -405,7 +405,7 @@ def _build_sample_provenance_report() -> tuple[CompareReport, Path, Path]:
             strict=False,
             stub=False,
             policy_evaluation=policy_evaluation,
-            evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED,
+            evidence_confidence=EvidenceConfidence.PROVENANCE_RECORDED,
             enrichment=ReportEnrichmentMetadata(
                 mode="opt_in_pypi",
                 pypi_enabled=True,
diff --git a/tools/sbom-diff-and-risk/tests/test_reports.py b/tools/sbom-diff-and-risk/tests/test_reports.py
@@ -324,7 +324,7 @@ def test_reports_include_provenance_policy_details_for_v2_policy() -> None:
             strict=False,
             stub=False,
             policy_evaluation=policy_evaluation,
-            evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED,
+            evidence_confidence=EvidenceConfidence.PROVENANCE_RECORDED,
         ),
         notes=["PyPI provenance enrichment was requested explicitly."],
     )
diff --git a/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py b/tools/sbom-diff-and-risk/tests/test_scorecard_reporting.py
@@ -48,7 +48,7 @@ def test_scorecard_report_json_summary_includes_enrichment_status() -> None:
         "warning": 1,
         "suppressed": 0,
     }
-    assert payload["summary"]["evidence_confidence"] == "enrichment_mocked"
+    assert payload["summary"]["evidence_confidence"] == "enrichment_recorded"
     assert payload["summary"]["enrichment"] == {
         "status": "used",
         "mode": "opt_in_scorecard",
@@ -238,7 +238,7 @@ def _build_sample_scorecard_report() -> tuple[CompareReport, Path, Path]:
             strict=False,
             stub=False,
             policy_evaluation=policy_evaluation,
-            evidence_confidence=EvidenceConfidence.ENRICHMENT_MOCKED,
+            evidence_confidence=EvidenceConfidence.ENRICHMENT_RECORDED,
             enrichment=ReportEnrichmentMetadata(
                 mode="opt_in_scorecard",
                 scorecard_enabled=True,
