Implement milestone 7 findings normalisation and risk enrichment

Author: Ibraheem Olasupo

.github/workflows/findings.yml

@@ -0,0 +1,48 @@
+name: Findings Normalisation
+
+on:
+  pull_request:
+    branches: ["main"]
+  push:
+    branches: ["main"]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: findings-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  findings:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      # actions/checkout v4.2.2
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+
+      # actions/setup-python v5.6.0
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: "3.11"
+          cache: "pip"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip==25.1.1
+          python -m pip install -e ".[dev]"
+
+      - name: Source evidence verification
+        run: make verify-threat-model-evidence verify-api-security-evidence verify-infrastructure-evidence verify-appsec-evidence verify-dynamic-evidence
+
+      - name: Findings tests
+        run: pytest tests/security/test_findings_pipeline.py -q
+
+      - name: Findings normalisation and validation
+        run: make findings-normalise findings-deduplicate findings-enrich findings-validate
+
+      - name: Deterministic findings evidence
+        run: make findings-evidence verify-findings-evidence
+
+      - name: Findings reports
+        run: make findings-report

CONTRIBUTING.md

@@ -10,6 +10,7 @@ Run:
 make quality
 make appsec-fast
 make dynamic-fast
+make findings-full
 ```
 
 Use deterministic synthetic data only. Do not add real patient, NHS, genomic, credential, or cloud account data.
@@ -23,3 +24,4 @@ Use deterministic synthetic data only. Do not add real patient, NHS, genomic, cr
 - Run scanner targets after changing dependencies, Dockerfile, Terraform, authentication or request-handling code.
 - Keep dynamic-security scans pointed only at localhost, loopback or approved local Docker targets.
 - Do not add broad scanner suppressions. Use `security/config/suppressions.yaml` with an owner, expiry and exact path or rule scope.
+- Keep findings IDs deterministic and avoid absolute local paths, raw JWTs, private keys or real data in canonical findings outputs.

Makefile

@@ -1,4 +1,4 @@
-.PHONY: setup install format format-check lint type-check test test-coverage auth-test api-security-test terraform-fmt terraform-fmt-check terraform-init terraform-validate terraform-test infrastructure-test infrastructure-evidence verify-infrastructure-evidence infrastructure-report security-tools secrets-scan sast sast-semgrep sast-bandit semgrep-test sca dependency-audit sbom verify-sbom iac-scan checkov-scan container-build-security container-scan appsec-fast appsec-full appsec-evidence verify-appsec-evidence appsec-report dynamic-tools dynamic-server-start dynamic-server-wait dynamic-server-stop schemathesis-test api-schema-security-test zap-baseline zap-api-scan auth-boundary-test authorisation-boundary-test object-access-test input-mutation-test security-header-test cors-test resource-consumption-test audit-dynamic-test dast dynamic-evidence verify-dynamic-evidence dynamic-report dynamic-fast dynamic-full pre-commit-install pre-commit-run quality run docker-build docker-run threat-model-validate threat-model-evidence verify-threat-model-evidence threat-model-report api-security-evidence verify-api-security-evidence api-security-report dev-token-researcher dev-token-approver dev-token-auditor clean
+.PHONY: setup install format format-check lint type-check test test-coverage auth-test api-security-test terraform-fmt terraform-fmt-check terraform-init terraform-validate terraform-test infrastructure-test infrastructure-evidence verify-infrastructure-evidence infrastructure-report security-tools secrets-scan sast sast-semgrep sast-bandit semgrep-test sca dependency-audit sbom verify-sbom iac-scan checkov-scan container-build-security container-scan appsec-fast appsec-full appsec-evidence verify-appsec-evidence appsec-report dynamic-tools dynamic-server-start dynamic-server-wait dynamic-server-stop schemathesis-test api-schema-security-test zap-baseline zap-api-scan auth-boundary-test authorisation-boundary-test object-access-test input-mutation-test security-header-test cors-test resource-consumption-test audit-dynamic-test dast dynamic-evidence verify-dynamic-evidence dynamic-report dynamic-fast dynamic-full findings-normalise findings-deduplicate findings-enrich findings-validate findings-evidence verify-findings-evidence findings-report findings-full pre-commit-install pre-commit-run quality run docker-build docker-run threat-model-validate threat-model-evidence verify-threat-model-evidence threat-model-report api-security-evidence verify-api-security-evidence api-security-report dev-token-researcher dev-token-approver dev-token-auditor clean
 
 PYTHON ?= python3
 VENV ?= .venv
@@ -183,6 +183,30 @@ dynamic-fast:
 dynamic-full:
 	PYTHONPATH=src $(PYTHON) scripts/dynamic_security_tools.py full
 
+findings-normalise:
+	PYTHONPATH=src $(PYTHON) -m genomic_research_access_api.security.findings normalise
+
+findings-deduplicate:
+	PYTHONPATH=src $(PYTHON) -m genomic_research_access_api.security.findings deduplicate
+
+findings-enrich:
+	PYTHONPATH=src $(PYTHON) -m genomic_research_access_api.security.findings enrich
+
+findings-validate:
+	PYTHONPATH=src $(PYTHON) -m genomic_research_access_api.security.findings validate
+
+findings-evidence:
+	PYTHONPATH=src $(PYTHON) -m genomic_research_access_api.security.findings evidence --timestamp 2026-01-01T00:00:00Z --as-of-date 2026-01-01
+
+verify-findings-evidence:
+	PYTHONPATH=src $(PYTHON) -m genomic_research_access_api.security.findings verify
+
+findings-report:
+	PYTHONPATH=src $(PYTHON) -m genomic_research_access_api.security.findings report
+
+findings-full:
+	PYTHONPATH=src $(PYTHON) -m genomic_research_access_api.security.findings full --timestamp 2026-01-01T00:00:00Z --as-of-date 2026-01-01
+
 pre-commit-install:
 	$(PYTHON) -m pip install pre-commit==3.8.0
 	$(VENV)/bin/pre-commit install

README.md

@@ -28,6 +28,8 @@ Milestone 5 delivered the core AppSec pipeline: pinned scanner configuration, Se
 
 Milestone 6 adds local-only dynamic API security validation: dynamic pytest boundary tests, pinned Schemathesis OpenAPI testing, pinned OWASP ZAP baseline scanning, bounded resource-consumption checks, deterministic dynamic evidence and dynamic-security reports. Dynamic scans are restricted to localhost, loopback and approved local Docker targets.
 
+Milestone 7 adds canonical findings normalisation and risk enrichment across threat-model, AppSec, infrastructure and dynamic-security outputs. It preserves native evidence and governed suppressions, assigns deterministic owners and SLAs, and does not implement release gates or lifecycle workflow.
+
 ## Milestone 1 Scope
 
 Implemented:
@@ -178,6 +180,7 @@ Open `http://127.0.0.1:8000/docs` for FastAPI's local OpenAPI UI.
 - `make infrastructure-test`: run local infrastructure policy tests without AWS credentials.
 - `make verify-infrastructure-evidence`: verify deterministic infrastructure evidence.
 - `make security-tools`: print pinned AppSec scanner inventory.
+- `make findings-full`: normalise existing security evidence into canonical findings, validate deterministic evidence and generate findings reports.
 - `make secrets-scan`: run Gitleaks via local binary or pinned container.
 - `make semgrep-test`: run Semgrep custom rule tests.
 - `make sast`: run Semgrep and Bandit.

SECURITY.md

@@ -6,14 +6,15 @@ If you identify a vulnerability or unsafe pattern in this demonstration code, op
 
 Do not submit real patient data, NHS data, genomic data, credentials, cloud account identifiers, or sensitive operational details in reports.
 
-Current implemented security controls include local JWT authentication, role-based and object-level authorisation, threat-model validation, non-deployed Terraform reference controls, a Milestone 5 AppSec pipeline, and Milestone 6 local dynamic API security validation.
+Current implemented security controls include local JWT authentication, role-based and object-level authorisation, threat-model validation, non-deployed Terraform reference controls, a Milestone 5 AppSec pipeline, Milestone 6 local dynamic API security validation, and Milestone 7 canonical findings normalisation.
 
 Run local security checks with:
 
 ```bash
 make quality
 make appsec-fast
 make dynamic-fast
+make findings-full
 ```
 
-Docker-backed Gitleaks, Trivy, Schemathesis and ZAP scans require a running Docker daemon when native binaries or scanner containers are used. Dynamic scan targets must remain local-only. Do not treat local scanner evidence as a production vulnerability-management programme, penetration test or release approval gate.
+Docker-backed Gitleaks, Trivy, Schemathesis and ZAP scans require a running Docker daemon when native binaries or scanner containers are used. Dynamic scan targets must remain local-only. Do not treat local scanner or findings evidence as a production vulnerability-management programme, penetration test or release approval gate.

config/findings/asset-context.yaml

@@ -0,0 +1,19 @@
+{
+  "schema_version": "1.0",
+  "assets": [
+    {"asset_id": "AST-APP-FASTAPI", "name": "FastAPI application", "type": "application", "criticality": "high", "data_sensitivity": "high", "internet_exposure": "planned", "environment": "locally_validated", "owner": "Application Engineering", "squad": "Application Engineering", "repository_path": "src/genomic_research_access_api", "threat_ids": ["THR-API-001", "THR-API-002"], "security_requirement_ids": ["SR-API-001", "SR-API-003"]},
+    {"asset_id": "AST-AUTHN", "name": "authentication subsystem", "type": "service", "criticality": "critical", "data_sensitivity": "high", "internet_exposure": "planned", "environment": "locally_validated", "owner": "Product Security", "squad": "Identity and Application Security", "repository_path": "src/genomic_research_access_api/security/authentication", "threat_ids": ["THR-AUTH-001", "THR-AUTH-002"], "security_requirement_ids": ["SR-AUTH-001"]},
+    {"asset_id": "AST-AUTHZ", "name": "authorisation subsystem", "type": "service", "criticality": "critical", "data_sensitivity": "high", "internet_exposure": "planned", "environment": "locally_validated", "owner": "Product Security", "squad": "Identity and Application Security", "repository_path": "src/genomic_research_access_api/security/authorisation.py", "threat_ids": ["THR-AUTHZ-001", "THR-AUTHZ-002"], "security_requirement_ids": ["SR-AUTHZ-001", "SR-AUTHZ-002"]},
+    {"asset_id": "AST-WORKFLOW-ACCESS", "name": "access-request workflow", "type": "workflow", "criticality": "high", "data_sensitivity": "high", "internet_exposure": "planned", "environment": "locally_validated", "owner": "Application Engineering", "squad": "Application Engineering", "repository_path": "src/genomic_research_access_api/services/access_requests.py", "threat_ids": ["THR-AUTHZ-003", "THR-AUTHZ-004"], "security_requirement_ids": ["SR-AUTHZ-004", "SR-AUTHZ-005"]},
+    {"asset_id": "AST-AUDIT", "name": "audit subsystem", "type": "service", "criticality": "high", "data_sensitivity": "medium", "internet_exposure": "not_deployed", "environment": "locally_validated", "owner": "Product Security", "squad": "Application Engineering", "repository_path": "src/genomic_research_access_api/audit", "threat_ids": ["THR-AUDIT-001", "THR-AUDIT-002"], "security_requirement_ids": ["SR-AUDIT-001"]},
+    {"asset_id": "AST-CONTAINER", "name": "Docker image", "type": "container_image", "criticality": "high", "data_sensitivity": "medium", "internet_exposure": "not_deployed", "environment": "not_deployed", "owner": "Platform Engineering", "squad": "Platform Engineering", "repository_path": "Dockerfile", "threat_ids": ["THR-SUPPLY-001"], "security_requirement_ids": ["SR-CONTAINER-001"]},
+    {"asset_id": "AST-DEPS-PYTHON", "name": "Python dependency set", "type": "dependency_set", "criticality": "high", "data_sensitivity": "medium", "internet_exposure": "not_deployed", "environment": "configured", "owner": "Application Engineering", "squad": "Application Engineering", "repository_path": "pyproject.toml", "threat_ids": ["THR-SUPPLY-001"], "security_requirement_ids": ["SR-SUPPLY-001"]},
+    {"asset_id": "AST-TERRAFORM", "name": "Terraform codebase", "type": "infrastructure_code", "criticality": "high", "data_sensitivity": "high", "internet_exposure": "planned", "environment": "not_deployed", "owner": "Cloud Security", "squad": "Cloud Platform Security", "repository_path": "infrastructure", "threat_ids": ["THR-CLOUD-001"], "security_requirement_ids": ["SR-INFRA-001"]},
+    {"asset_id": "AST-ECS-TASK", "name": "ECS task definition", "type": "compute", "criticality": "high", "data_sensitivity": "high", "internet_exposure": "planned", "environment": "not_deployed", "owner": "Cloud Security", "squad": "Cloud Platform Security", "repository_path": "infrastructure/modules/compute", "threat_ids": ["THR-CLOUD-001"], "security_requirement_ids": ["SR-INFRA-001"]},
+    {"asset_id": "AST-IAM", "name": "IAM roles", "type": "identity", "criticality": "critical", "data_sensitivity": "high", "internet_exposure": "not_deployed", "environment": "not_deployed", "owner": "Cloud Security", "squad": "Cloud Platform Security", "repository_path": "infrastructure/modules/iam", "threat_ids": ["THR-CI-001"], "security_requirement_ids": ["SR-CI-002"]},
+    {"asset_id": "AST-DYNAMODB", "name": "DynamoDB configuration", "type": "datastore", "criticality": "critical", "data_sensitivity": "high", "internet_exposure": "not_deployed", "environment": "not_deployed", "owner": "Cloud Security", "squad": "Cloud Platform Security", "repository_path": "infrastructure/modules/datastore", "threat_ids": ["THR-DATA-001"], "security_requirement_ids": ["SR-DATA-001"]},
+    {"asset_id": "AST-CLOUDTRAIL", "name": "CloudTrail configuration", "type": "audit_logging", "criticality": "high", "data_sensitivity": "medium", "internet_exposure": "not_deployed", "environment": "not_deployed", "owner": "Cloud Security", "squad": "Cloud Platform Security", "repository_path": "infrastructure/modules/audit", "threat_ids": ["THR-AUDIT-001"], "security_requirement_ids": ["SR-AUDIT-001"]},
+    {"asset_id": "AST-CI", "name": "CI workflows", "type": "ci_cd", "criticality": "high", "data_sensitivity": "medium", "internet_exposure": "configured", "environment": "configured", "owner": "DevSecOps Engineering", "squad": "DevSecOps Engineering", "repository_path": ".github/workflows", "threat_ids": ["THR-SUPPLY-002"], "security_requirement_ids": ["SR-CI-001"]},
+    {"asset_id": "AST-OPENAPI", "name": "OpenAPI contract", "type": "api_contract", "criticality": "high", "data_sensitivity": "medium", "internet_exposure": "planned", "environment": "locally_validated", "owner": "Application Engineering", "squad": "Application Engineering", "repository_path": "outputs/security/dynamic/raw/openapi.json", "threat_ids": ["THR-API-001"], "security_requirement_ids": ["SR-API-001"]}
+  ]
+}

config/findings/deduplication.yaml

@@ -0,0 +1,10 @@
+{
+  "schema_version": "1.0",
+  "exact_keys": ["cve", "cwe", "source_rule_resource", "endpoint_condition", "threat_requirement_control"],
+  "cross_tool_examples": [
+    {"tools": ["pip-audit", "trivy"], "key": "cve+package_name"},
+    {"tools": ["semgrep", "bandit"], "key": "cwe+file+line"},
+    {"tools": ["zap", "dynamic-pytest"], "key": "security_requirement_ids+asset_id"}
+  ],
+  "fuzzy_matching": false
+}

config/findings/ownership.yaml

@@ -0,0 +1,24 @@
+{
+  "schema_version": "1.0",
+  "allowed_owners": ["Application Engineering", "Platform Engineering", "Product Security", "Cloud Security", "DevSecOps Engineering", "Risk Owner", "unowned"],
+  "rules": [
+    {"match": "src/genomic_research_access_api/security/authentication/**", "squad": "Identity and Application Security", "technical_owner": "Product Security", "risk_owner": "Risk Owner", "remediation_owner": "Application Engineering"},
+    {"match": "src/genomic_research_access_api/security/authorisation.py", "squad": "Identity and Application Security", "technical_owner": "Product Security", "risk_owner": "Risk Owner", "remediation_owner": "Application Engineering"},
+    {"match": "src/genomic_research_access_api/**", "squad": "Application Engineering", "technical_owner": "Application Engineering", "risk_owner": "Risk Owner", "remediation_owner": "Application Engineering"},
+    {"match": "infrastructure/modules/iam/**", "squad": "Cloud Platform Security", "technical_owner": "Cloud Security", "risk_owner": "Risk Owner", "remediation_owner": "Cloud Security"},
+    {"match": "infrastructure/**", "squad": "Cloud Platform Security", "technical_owner": "Cloud Security", "risk_owner": "Risk Owner", "remediation_owner": "Cloud Security"},
+    {"match": "Dockerfile", "squad": "Platform Engineering", "technical_owner": "Platform Engineering", "risk_owner": "Risk Owner", "remediation_owner": "Platform Engineering"},
+    {"match": ".github/workflows/**", "squad": "DevSecOps Engineering", "technical_owner": "DevSecOps Engineering", "risk_owner": "Risk Owner", "remediation_owner": "DevSecOps Engineering"},
+    {"match": "pyproject.toml", "squad": "Application Engineering", "technical_owner": "Application Engineering", "risk_owner": "Risk Owner", "remediation_owner": "Application Engineering"}
+  ],
+  "source_tool_defaults": {
+    "checkov": {"squad": "Cloud Platform Security", "technical_owner": "Cloud Security", "risk_owner": "Risk Owner", "remediation_owner": "Cloud Security"},
+    "dynamic-pytest": {"squad": "Application Engineering", "technical_owner": "Application Engineering", "risk_owner": "Risk Owner", "remediation_owner": "Application Engineering"},
+    "gitleaks": {"squad": "Application Engineering", "technical_owner": "Application Engineering", "risk_owner": "Risk Owner", "remediation_owner": "Application Engineering"},
+    "pip-audit": {"squad": "Application Engineering", "technical_owner": "Application Engineering", "risk_owner": "Risk Owner", "remediation_owner": "Application Engineering"},
+    "zap": {"squad": "Application Engineering", "technical_owner": "Application Engineering", "risk_owner": "Risk Owner", "remediation_owner": "Application Engineering"},
+    "trivy": {"squad": "Platform Engineering", "technical_owner": "Platform Engineering", "risk_owner": "Risk Owner", "remediation_owner": "Platform Engineering"},
+    "threat-model": {"squad": "Product Security", "technical_owner": "Product Security", "risk_owner": "Risk Owner", "remediation_owner": "Product Security"}
+  },
+  "unowned": {"squad": "unowned", "technical_owner": "unowned", "risk_owner": "Risk Owner", "remediation_owner": "unowned"}
+}

config/findings/remediation-sla.yaml

@@ -0,0 +1,9 @@
+{
+  "schema_version": "1.0",
+  "portfolio_notice": "Demonstration values for this repository; not Genomics England policy.",
+  "sla_days": {"critical": 3, "high": 14, "medium": 30, "low": 60, "informational": 90, "unknown": 90},
+  "context_adjustments": [
+    {"name": "internet-facing critical secret", "finding_type": "Secret", "severity": "critical", "internet_exposure": "internet_facing", "sla_days": 0},
+    {"name": "suppression review governs follow-up", "suppression_status": "active", "use_suppression_expiry": true}
+  ]
+}

config/findings/risk-scoring.yaml

@@ -0,0 +1,22 @@
+{
+  "schema_version": "1.0",
+  "weights": {
+    "technical_severity": 30,
+    "exploitability": 20,
+    "internet_exposure": 15,
+    "asset_criticality": 15,
+    "data_sensitivity": 10,
+    "privilege_required": 5,
+    "age_or_recurrence": 5
+  },
+  "factor_scores": {
+    "technical_severity": {"critical": 1.0, "high": 0.8, "medium": 0.55, "low": 0.3, "informational": 0.1, "unknown": 0.2},
+    "exploitability": {"known": 1.0, "probable": 0.75, "possible": 0.5, "unlikely": 0.2, "unknown": 0.35},
+    "internet_exposure": {"internet_facing": 1.0, "planned": 0.65, "configured": 0.45, "locally_validated": 0.25, "not_deployed": 0.15, "unknown": 0.3},
+    "asset_criticality": {"critical": 1.0, "high": 0.8, "medium": 0.55, "low": 0.3, "unknown": 0.35},
+    "data_sensitivity": {"high": 1.0, "medium": 0.65, "low": 0.3, "unknown": 0.4},
+    "privilege_required": {"none": 1.0, "low": 0.7, "high": 0.35, "unknown": 0.5},
+    "age_or_recurrence": {"recurring": 1.0, "first_seen": 0.4, "unknown": 0.3}
+  },
+  "priority_thresholds": {"P1": 80, "P2": 60, "P3": 40, "P4": 20, "P5": 0}
+}