Introduce aliases for scope uuids, allow group in template filter 'pick' (#1187)

Using the aliases instead of the uuids makes URLs more telling and
reduces confusion for users. Being able to reference groups in the
'pick' filter eliminates the need for enumerating the groups redundantly,
which was quite error prone.

Also finally remove deprecated version field in docker-compose.yml
and upgrade Dockerfile from Python 3.10 to 3.12 (would've used 3.14,
like my current Alpine, but then some Python packages wouldn't build)

Signed-off-by: Matthias Büchse <matthias.buechse@alasca.cloud>

Author: mbuechse

compliance-monitor/Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1
-FROM python:3.10
+FROM python:3.12
 RUN useradd -g users  -u 1001 -m -s /bin/bash runuser
 USER 1001
 WORKDIR /code

compliance-monitor/docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   reverse-proxy:
     image: traefik:v2.11
@@ -30,10 +28,10 @@ services:
       - SCM_DB_HOST=postgres
       - SCM_DB_PORT=5432
       - SCM_DB_PASSWORD_FILE=/run/secrets/db_password
-      # pass the following two from the shell or put them into .env
+      # pass the following from the shell or put them into .env
       - SCM_HC_USER
       - SCM_HC_PASSWORD
-      - SCM_BASE_URL=https://compliance.sovereignit.cloud/
+      - SCM_BASE_URL
     volumes:
       - ../Tests:/Tests
       - ./bootstrap.yaml:/code/bootstrap.yaml

compliance-monitor/monitor.py

@@ -28,7 +28,7 @@
 from fastapi import Depends, FastAPI, HTTPException, Request, Response, status
 from fastapi.responses import RedirectResponse
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
-from jinja2 import Environment
+from jinja2 import Environment, pass_context
 from markdown import markdown
 from passlib.context import CryptContext
 import psycopg2
@@ -42,7 +42,7 @@
     db_get_keys, db_insert_report, db_get_recent_results2, db_patch_approval2, db_get_report,
     db_ensure_schema, db_get_apikeys, db_update_apikey, db_filter_apikeys, db_clear_delegates,
     db_find_subjects, db_insert_result2, db_get_relevant_results2, db_add_delegate, db_get_group,
-    db_filter_accounts,
+    db_filter_accounts, db_get_groups,
 )
 
 
@@ -94,6 +94,10 @@ def __init__(self):
 # to achieve this!
 SEP = "-----END SSH SIGNATURE-----\n&"
 ASTERISK_LOOKUP = {'effective': '', 'draft': '*', 'warn': '†', 'deprecated': '††'}
+SCOPE_ALIASES = {
+    'scs-compatible-iaas': '50393e6f-2ae1-4c5c-a62c-3b75f2abef3f',
+    'scs-compatible-kaas': '1fffebe6-fd4b-44d3-a36c-fc58b4bb0180',
+}
 
 
 class ViewType(Enum):
@@ -651,6 +655,17 @@ def _resolve_group(cur, subject, prefix=GROUP_PREFIX):
     return None, [subject]
 
 
+def _resolve_group_locally(groups, subject, prefix=GROUP_PREFIX):
+    group = subject.removeprefix(prefix)
+    if subject != group:
+        return group, list(groups[group])
+    return None, [subject]
+
+
+def _resolve_scope(scopeuuid):
+    return SCOPE_ALIASES.get(scopeuuid, scopeuuid)
+
+
 @app.get("/{view_type}/detail/{subject}/{scopeuuid}")
 async def get_detail(
     request: Request,
@@ -659,6 +674,7 @@ async def get_detail(
     subject: str,
     scopeuuid: str,
 ):
+    scopeuuid = _resolve_scope(scopeuuid)
     with conn.cursor() as cur:
         group, subjects = _resolve_group(cur, subject)
         rows2 = []
@@ -683,6 +699,7 @@ async def get_detail_full(
     subject: str,
     scopeuuid: str,
 ):
+    scopeuuid = _resolve_scope(scopeuuid)
     with conn.cursor() as cur:
         group, subjects = _resolve_group(cur, subject)
         rows2 = []
@@ -706,11 +723,12 @@ async def get_table(
     view_type: ViewType,
 ):
     with conn.cursor() as cur:
+        groups = db_get_groups(cur)
         rows2 = db_get_relevant_results2(cur, approved_only=True)
     results2 = convert_result_rows_to_dict2(rows2, get_scopes(), grace_period_days=GRACE_PERIOD_DAYS)
     return render_view(
         VIEW_TABLE, view_type, results=results2, base_url=settings.base_url, detail_page='detail',
-        title="SCS compliance overview",
+        title="SCS compliance overview", groups=groups,
     )
 
 
@@ -721,11 +739,12 @@ async def get_table_full(
     view_type: ViewType,
 ):
     with conn.cursor() as cur:
+        groups = db_get_groups(cur)
         rows2 = db_get_relevant_results2(cur, approved_only=False)
     results2 = convert_result_rows_to_dict2(rows2, get_scopes(), include_drafts=True)
     return render_view(
         VIEW_TABLE, view_type, results=results2, base_url=settings.base_url, detail_page='detail_full',
-        title="SCS compliance overview (incl. unverified results)", unverified=True,
+        title="SCS compliance overview (incl. unverified results)", unverified=True, groups=groups,
     )
 
 
@@ -736,6 +755,7 @@ async def get_scope(
     view_type: ViewType,
     scopeuuid: str,
 ):
+    scopeuuid = _resolve_scope(scopeuuid)
     spec = get_scopes()[scopeuuid]
     versions = spec['versions']
     # sort by name, and all drafts after all non-drafts
@@ -814,14 +834,18 @@ async def get_healthz(request: Request):
     return Response()  # empty response with status 200
 
 
-def pick_filter(results, scope, *subjects):
+@pass_context
+def pick_filter(ctx, results, scopeuuid, *subjects):
     """Jinja filter to pick scope results from `results` for given `subject` and `scope`"""
+    scopeuuid = _resolve_scope(scopeuuid)
     # simple case (backwards compatible): precisely one subject
     if len(subjects) == 1:
-        return results.get(subjects[0], {}).get(scope, {})
+        group, subjects = _resolve_group_locally(ctx['groups'], subjects[0])
+        if not group:
+            return results.get(subjects[0], {}).get(scopeuuid, {})
     # generalized case: multiple subjects
     # in this case, drop None
-    rs = [results.get(subject, {}).get(scope, {}) for subject in subjects]
+    rs = [results.get(subject, {}).get(scopeuuid, {}) for subject in subjects]
     return [r for r in rs if r is not None]
 
 

compliance-monitor/sql.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+
 from psycopg2 import sql
 from psycopg2.extensions import cursor, connection
 
@@ -289,6 +291,14 @@ def db_get_group(cur: cursor, group):
     return [row[0] for row in cur.fetchall()]
 
 
+def db_get_groups(cur: cursor):
+    cur.execute('''SELECT subject, "group" FROM account WHERE "group" is not null and "group" != '';''')
+    groups = defaultdict(list)
+    for row in cur.fetchall():
+        groups[row[1]].append(row[0])
+    return groups
+
+
 def db_update_apikey(cur: cursor, accountid, apikey_hash):
     sanitized = dict(accountid=accountid, apikey_hash=apikey_hash)
     cur.execute('''

compliance-monitor/templates/overview.md.j2

@@ -11,7 +11,7 @@ Version numbers are suffixed by a symbol depending on state: * for _draft_, †
 
 This is a list of IaaS clouds that we test on a nightly basis against the certificate scope [SCS-compatible IaaS](https://docs.scs.community/standards/scs-compatible-iaas/).
 
-{% set iaas = '50393e6f-2ae1-4c5c-a62c-3b75f2abef3f' -%}
+{% set iaas = 'scs-compatible-iaas' -%}
 | Name  | Description  | Operator  | SCS-compatible IaaS  | HealthMon  |
 |-------|--------------|-----------|----------------------|:----------:|
 | [scs2](https://docs.scs.community/community/cloud-resources/plusserver-gx-scs) | Dev/Test/Demo environment (2nd gen) provided for SCS & GAIA-X context | plusserver GmbH | 
@@ -24,13 +24,13 @@ This is a list of IaaS clouds that we test on a nightly basis against the certif
 {#- #} [{{ results | pick(iaas, 'cc-rrze') | summary }}]({{ detail_url('cc-rrze', iaas) }}) {# -#}
 | (soon) |
 | [CNDS](https://cnds.io/) | Public cloud for customers (2 regions) | artcodix GmbH |
-{#- #} [{{ results | pick(iaas, 'artcodix', 'artcodix-ro') | summary }}]({{ detail_url('group-artcodix', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'group-artcodix') | summary }}]({{ detail_url('group-artcodix', iaas) }}) {# -#}
 | [HM](https://ohm.muc.cloud.cnds.io/) |
 | [Cloud&Heat IaaS](https://www.cloudandheat.com/en/products/cloud-services/infrastructure-as-a-service/) | Public cloud for customers (1 SCS region) | Cloud&Heat Technologies GmbH |
 {#- #} [{{ results | pick(iaas, 'cah-dd8a') | summary }}]({{ detail_url('cah-dd8a', iaas) }}) {# -#}
 | n/a |
 | [pluscloud open](https://www.plusserver.com/en/products/pluscloud-open) | Public cloud for customers (4 regions)  | plusserver GmbH | {# #}
-{#- #}[{{ results | pick(iaas, 'pco-prod1', 'pco-prod2', 'pco-prod3', 'pco-prod4') | summary }}]({{ detail_url('group-pco-prod', iaas) }}) {# -#}
+{#- #}[{{ results | pick(iaas, 'group-pco-prod') | summary }}]({{ detail_url('group-pco-prod', iaas) }}) {# -#}
 | [HM1](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-pco) [HM2](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod2) [HM3](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod3) [HM4](https://health.prod1.plusserver.sovereignit.cloud:3000/d/9ltTEmlnk/openstack-health-monitor2?orgId=1&var-mycloud=plus-prod4) |
 | [REGIO.cloud](https://regio.digital) | Public cloud for customers | OSISM GmbH |
 {#- #} [{{ results | pick(iaas, 'regio-a') | summary }}]({{ detail_url('regio-a', iaas) }}) {# -#}
@@ -39,7 +39,7 @@ This is a list of IaaS clouds that we test on a nightly basis against the certif
 {#- #} [{{ results | pick(iaas, 'scaleup-occ2') | summary }}]({{ detail_url('scaleup-occ2', iaas) }}) {# -#}
 | [HM](https://health.occ2.scaleup.sovereignit.cloud) |
 | [syseleven](https://www.syseleven.de/en/products-services/openstack-cloud/) | Public OpenStack Cloud (2 SCS regions) | SysEleven GmbH | {# #}
-{#- #} [{{ results | pick(iaas, 'syseleven-dus2', 'syseleven-ham1') | summary }}]({{ detail_url('group-syseleven', iaas) }}) {# -#}
+{#- #} [{{ results | pick(iaas, 'group-syseleven') | summary }}]({{ detail_url('group-syseleven', iaas) }}) {# -#}
 | (soon) |
 | [Wavestack](https://www.noris.de/wavestack-cloud/) | Public cloud for customers | noris network AG/Wavecon GmbH |
 {#- #} [{{ results | pick(iaas, 'wavestack') | summary }}]({{ detail_url('wavestack', iaas) }}) {# -#}
@@ -52,12 +52,12 @@ This is a list of IaaS clouds that we test on a nightly basis against the certif
 
 This is a list of KaaS clouds that we test on a nightly basis against the certificate scope [SCS-compatible KaaS](https://docs.scs.community/standards/scs-compatible-kaas/).
 
-{% set kaas = '1fffebe6-fd4b-44d3-a36c-fc58b4bb0180' -%}
+{% set kaas = 'scs-compatible-kaas' -%}
 | Name  | Description  | Operator  | SCS-compatible KaaS  |
 |-------|--------------|-----------|----------------------|
 | [noris Sovereign Cloud (nSC)](https://www.noris.de/en/it-services/cloud-services/cloud-solutions-for-enterprise-companies/noris-sovereign-cloud/) | Public KaaS cloud based on Gardener (1 SCS configuration) | noris network AG | {# #}
-{#- #} [{{ results | pick(kaas, 'noris-1.33', 'noris-1.34') | summary }}]({{ detail_url('group-noris', kaas) }}) {# -#}
+{#- #} [{{ results | pick(kaas, 'group-noris') | summary }}]({{ detail_url('group-noris', kaas) }}) {# -#}
 |
 | [Syself Autopilot](https://syself.com/platform) | KaaS offering based on ClusterStacks | Syself GmbH | {# #}
-{#- #} [{{ results | pick(kaas, 'syself-1.33', 'syself-1.34') | summary }}]({{ detail_url('group-syself', kaas) }}) {# -#}
+{#- #} [{{ results | pick(kaas, 'group-syself') | summary }}]({{ detail_url('group-syself', kaas) }}) {# -#}
 |