Introduce aliases for scope uuids, allow group in template filter 'pick'

Also finally remove deprecated version field in docker-compose.yml
and upgrade Dockerfile from Python 3.10 to 3.12 (would've used 3.14,
like my current Alpine, but then some Python packages wouldn't build)

Signed-off-by: Matthias Büchse <matthias.buechse@alasca.cloud>

Author: mbuechse

compliance-monitor/Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM python:3.10
+FROM python:3.12
 RUN useradd -g users  -u 1001 -m -s /bin/bash runuser
 USER 1001
 WORKDIR /code

compliance-monitor/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   reverse-proxy:
     image: traefik:v2.11
@@ -30,10 +28,10 @@ services:
       - SCM_DB_HOST=postgres
       - SCM_DB_PORT=5432
       - SCM_DB_PASSWORD_FILE=/run/secrets/db_password
-      # pass the following two from the shell or put them into .env
+      # pass the following from the shell or put them into .env
       - SCM_HC_USER
       - SCM_HC_PASSWORD
-      - SCM_BASE_URL=https://compliance.sovereignit.cloud/
+      - SCM_BASE_URL
     volumes:
       - ../Tests:/Tests
       - ./bootstrap.yaml:/code/bootstrap.yaml

compliance-monitor/monitor.py

@@ -28,7 +28,7 @@
 from fastapi import Depends, FastAPI, HTTPException, Request, Response, status
 from fastapi.responses import RedirectResponse
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
-from jinja2 import Environment
+from jinja2 import Environment, pass_context
 from markdown import markdown
 from passlib.context import CryptContext
 import psycopg2
@@ -42,7 +42,7 @@
     db_get_keys, db_insert_report, db_get_recent_results2, db_patch_approval2, db_get_report,
     db_ensure_schema, db_get_apikeys, db_update_apikey, db_filter_apikeys, db_clear_delegates,
     db_find_subjects, db_insert_result2, db_get_relevant_results2, db_add_delegate, db_get_group,
-    db_filter_accounts,
+    db_filter_accounts, db_get_groups,
 )
 
 
@@ -94,6 +94,10 @@ def __init__(self):
 # to achieve this!
 SEP = "-----END SSH SIGNATURE-----\n&"
 ASTERISK_LOOKUP = {'effective': '', 'draft': '*', 'warn': '†', 'deprecated': '††'}
+SCOPE_ALIASES = {
+    'scs-compatible-iaas': '50393e6f-2ae1-4c5c-a62c-3b75f2abef3f',
+    'scs-compatible-kaas': '1fffebe6-fd4b-44d3-a36c-fc58b4bb0180',
+}
 
 
 class ViewType(Enum):
@@ -654,6 +658,17 @@ def _resolve_group(cur, subject, prefix=GROUP_PREFIX):
     return None, [subject]
 
 
+def _resolve_group_locally(groups, subject, prefix=GROUP_PREFIX):
+    group = subject.removeprefix(prefix)
+    if subject != group:
+        return group, list(groups[group])
+    return None, [subject]
+
+
+def _resolve_scope(scopeuuid):
+    return SCOPE_ALIASES.get(scopeuuid, scopeuuid)
+
+
 @app.get("/{view_type}/detail/{subject}/{scopeuuid}")
 async def get_detail(
     request: Request,
@@ -662,6 +677,7 @@ async def get_detail(
     subject: str,
     scopeuuid: str,
 ):
+    scopeuuid = _resolve_scope(scopeuuid)
     with conn.cursor() as cur:
         group, subjects = _resolve_group(cur, subject)
         rows2 = []
@@ -686,6 +702,7 @@ async def get_detail_full(
     subject: str,
     scopeuuid: str,
 ):
+    scopeuuid = _resolve_scope(scopeuuid)
     with conn.cursor() as cur:
         group, subjects = _resolve_group(cur, subject)
         rows2 = []
@@ -709,11 +726,12 @@ async def get_table(
     view_type: ViewType,
 ):
     with conn.cursor() as cur:
+        groups = db_get_groups(cur)
         rows2 = db_get_relevant_results2(cur, approved_only=True)
     results2 = convert_result_rows_to_dict2(rows2, get_scopes(), grace_period_days=GRACE_PERIOD_DAYS)
     return render_view(
         VIEW_TABLE, view_type, results=results2, base_url=settings.base_url, detail_page='detail',
-        title="SCS compliance overview",
+        title="SCS compliance overview", groups=groups,
     )
 
 
@@ -724,11 +742,12 @@ async def get_table_full(
     view_type: ViewType,
 ):
     with conn.cursor() as cur:
+        groups = db_get_groups(cur)
         rows2 = db_get_relevant_results2(cur, approved_only=False)
     results2 = convert_result_rows_to_dict2(rows2, get_scopes(), include_drafts=True)
     return render_view(
         VIEW_TABLE, view_type, results=results2, base_url=settings.base_url, detail_page='detail_full',
-        title="SCS compliance overview (incl. unverified results)", unverified=True,
+        title="SCS compliance overview (incl. unverified results)", unverified=True, groups=groups,
     )
 
 
@@ -739,6 +758,7 @@ async def get_scope(
     view_type: ViewType,
     scopeuuid: str,
 ):
+    scopeuuid = _resolve_scope(scopeuuid)
     spec = get_scopes()[scopeuuid]
     versions = spec['versions']
     # sort by name, and all drafts after all non-drafts
@@ -817,14 +837,18 @@ async def get_healthz(request: Request):
     return Response()  # empty response with status 200
 
 
-def pick_filter(results, scope, *subjects):
+@pass_context
+def pick_filter(ctx, results, scopeuuid, *subjects):
     """Jinja filter to pick scope results from `results` for given `subject` and `scope`"""
+    scopeuuid = _resolve_scope(scopeuuid)
     # simple case (backwards compatible): precisely one subject
     if len(subjects) == 1:
-        return results.get(subjects[0], {}).get(scope, {})
+        group, subjects = _resolve_group_locally(ctx['groups'], subjects[0])
+        if not group:
+            return results.get(subjects[0], {}).get(scopeuuid, {})
     # generalized case: multiple subjects
     # in this case, drop None
-    rs = [results.get(subject, {}).get(scope, {}) for subject in subjects]
+    rs = [results.get(subject, {}).get(scopeuuid, {}) for subject in subjects]
     return [r for r in rs if r is not None]
 
 

compliance-monitor/sql.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+
 from psycopg2 import sql
 from psycopg2.extensions import cursor, connection
 
@@ -288,6 +290,14 @@ def db_get_group(cur: cursor, group):
     return [row[0] for row in cur.fetchall()]
 
 
+def db_get_groups(cur: cursor):
+    cur.execute('''SELECT subject, "group" FROM account WHERE "group" is not null and "group" != '';''')
+    groups = defaultdict(list)
+    for row in cur.fetchall():
+        groups[row[1]].append(row[0])
+    return groups
+
+
 def db_update_apikey(cur: cursor, accountid, apikey_hash):
     sanitized = dict(accountid=accountid, apikey_hash=apikey_hash)
     cur.execute('''

compliance-monitor/templates/overview.md.j2

@@ -11,7 +11,7 @@ Version numbers are suffixed by a symbol depending on state: * for _draft_, †
 
 This is a list of IaaS clouds that we test on a nightly basis against the certificate scope [SCS-compatible IaaS](https://docs.scs.community/standards/scs-compatible-iaas/).
 
-{% set iaas = '50393e6f-2ae1-4c5c-a62c-3b75f2abef3f' -%}
+{% set iaas = 'scs-compatible-iaas' -%}
 | Name  | Description  | Operator  | SCS-compatible IaaS  | HealthMon  |
 |-------|--------------|-----------|----------------------|:----------:|
 | [scs2](https://docs.scs.community/community/cloud-resources/plusserver-gx-scs) | Dev/Test/Demo environment (2nd gen) provided for SCS & GAIA-X context | plusserver GmbH | 
@@ -24,13 +24,13 @@ This is a list of IaaS clouds that we test on a nightly basis against the certif
 {#- #} [{{ results | pick(iaas, 'cc-rrze') | summary }}]({{ detail_url('cc-rrze', iaas) }}) {# -#}
 | (soon) |
 | [CNDS](https://cnds.io/) | Public cloud for customers (2 regions) | artcodix GmbH |
-{#- #} [{{ results | pick(iaas, 'artcodix', 'artcodix-ro') | summary }}]({{ detail_url('group-artcodix', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'group-artcodix') | summary }}]({{ detail_url('group-artcodix', iaas) }}) {# -#}
 | [HM](https://ohm.muc.cloud.cnds.io/) |
 | [Cloud&Heat IaaS](https://www.cloudandheat.com/en/products/cloud-services/infrastructure-as-a-service/) | Public cloud for customers (1 SCS region) | Cloud&Heat Technologies GmbH |
 {#- #} [{{ results | pick(iaas, 'cah-dd8a') | summary }}]({{ detail_url('cah-dd8a', iaas) }}) {# -#}
 | n/a |
 | [pluscloud open](https://www.plusserver.com/en/products/pluscloud-open) | Public cloud for customers (4 regions)  | plusserver GmbH | {# #}
-{#- #}[{{ results | pick(iaas, 'pco-prod1', 'pco-prod2', 'pco-prod3', 'pco-prod4') | summary }}]({{ detail_url('group-pco-prod', iaas) }}) {# -#}
+{#- #}[{{ results | pick(iaas, 'group-pco-prod') | summary }}]({{ detail_url('group-pco-prod', iaas) }}) {# -#}
 | [HM1](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-pco) [HM2](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod2) [HM3](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod3) [HM4](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod4) |
 | [REGIO.cloud](https://regio.digital) | Public cloud for customers | OSISM GmbH |
 {#- #} [{{ results | pick(iaas, 'regio-a') | summary }}]({{ detail_url('regio-a', iaas) }}) {# -#}
@@ -39,7 +39,7 @@ This is a list of IaaS clouds that we test on a nightly basis against the certif
 {#- #} [{{ results | pick(iaas, 'scaleup-occ2') | summary }}]({{ detail_url('scaleup-occ2', iaas) }}) {# -#}
 | [HM](https://health.occ2.scaleup.sovereignit.cloud) |
 | [syseleven](https://www.syseleven.de/en/products-services/openstack-cloud/) | Public OpenStack Cloud (2 SCS regions) | SysEleven GmbH | {# #}
-{#- #} [{{ results | pick(iaas, 'syseleven-dus2', 'syseleven-ham1') | summary }}]({{ detail_url('group-syseleven', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'group-syseleven') | summary }}]({{ detail_url('group-syseleven', iaas) }}) {# -#}
 | (soon) |
 | [Wavestack](https://www.noris.de/wavestack-cloud/) | Public cloud for customers | noris network AG/Wavecon GmbH |
 {#- #} [{{ results | pick(iaas, 'wavestack') | summary }}]({{ detail_url('wavestack', iaas) }}) {# -#}
@@ -52,15 +52,15 @@ This is a list of IaaS clouds that we test on a nightly basis against the certif
 
 This is a list of KaaS clouds that we test on a nightly basis against the certificate scope [SCS-compatible KaaS](https://docs.scs.community/standards/scs-compatible-kaas/).
 
-{% set kaas = '1fffebe6-fd4b-44d3-a36c-fc58b4bb0180' -%}
+{% set kaas = 'scs-compatible-kaas' -%}
 | Name  | Description  | Operator  | SCS-compatible KaaS  |
 |-------|--------------|-----------|----------------------|
 | [noris Sovereign Cloud (nSC)](https://www.noris.de/en/it-services/cloud-services/cloud-solutions-for-enterprise-companies/noris-sovereign-cloud/) | Public KaaS cloud based on Gardener (1 SCS configuration) | noris network AG | {# #}
-{#- #} [{{ results | pick(kaas, 'noris-1.33', 'noris-1.34') | summary }}]({{ detail_url('group-noris', kaas) }}) {# -#}
+{#- #} [{{ results | pick(kaas, 'group-noris') | summary }}]({{ detail_url('group-noris', kaas) }}) {# -#}
 |
 | [ScaleUp Open Cloud](https://www.scaleuptech.com/cloud-hosting/managed-kubernetes/) | Public KaaS cloud based on Gardener (1 SCS configuration) | ScaleUp Technologies GmbH & Co. KG | {# #}
-{#- #} [{{ results | pick(kaas, 'scaleup-1.33', 'scaleup-1.34') | summary }}]({{ detail_url('group-scaleup', kaas) }}) {# -#}
+{#- #} [{{ results | pick(kaas, 'group-scaleup') | summary }}]({{ detail_url('group-scaleup', kaas) }}) {# -#}
 |
 | [Syself Autopilot](https://syself.com/platform) | KaaS offering based on ClusterStacks | Syself GmbH | {# #}
-{#- #} [{{ results | pick(kaas, 'syself-1.33', 'syself-1.34') | summary }}]({{ detail_url('group-syself', kaas) }}) {# -#}
+{#- #} [{{ results | pick(kaas, 'group-syself') | summary }}]({{ detail_url('group-syself', kaas) }}) {# -#}
 |