Add compliance history view per subject/scope

Adds a new /page/history/{subject}/{scopeuuid} endpoint (and markdown/
fragment variants) that lists all past compliance reports for a given
subject and scope, newest first, with per-report PASS/FAIL/ABORT counts
and links to each individual report.

Also adds a "compliance history" link to the detail pages, a
history_url() helper in render_view(), and fixes a pre-existing
TypeError crash on fresh databases in db_upgrade_schema() when the
schema version is None.

Signed-off-by: Thomas Güttler <thomas.guettler@syself.com>

Author: guettli

compliance-monitor/monitor.py

@@ -42,7 +42,7 @@
     db_get_keys, db_insert_report, db_get_recent_results2, db_patch_approval2, db_get_report,
     db_ensure_schema, db_get_apikeys, db_update_apikey, db_filter_apikeys, db_clear_delegates,
     db_find_subjects, db_insert_result2, db_get_relevant_results2, db_add_delegate, db_get_group,
-    db_filter_accounts,
+    db_filter_accounts, db_get_report_history,
 )
 
 
@@ -84,6 +84,7 @@ def __init__(self):
 ROLES = {'read_any': 1, 'append_any': 2, 'admin': 4, 'approve': 8}
 # number of days that expired results will be considered in lieu of more recent, but unapproved ones
 GRACE_PERIOD_DAYS = 7
+HISTORY_LIMIT = 500
 # separator between signature and report data; use something like
 #     ssh-keygen \
 #       -Y sign -f ~/.ssh/id_ed25519 -n report myreport.yaml
@@ -122,7 +123,12 @@ class ViewType(Enum):
     ViewType.fragment: 'scope.md',
     ViewType.page: 'overview.html',
 }
-REQUIRED_TEMPLATES = tuple(set(fn for view in (VIEW_REPORT, VIEW_DETAIL, VIEW_TABLE, VIEW_SCOPE) for fn in view.values()))
+VIEW_HISTORY = {
+    ViewType.markdown: 'history.md',
+    ViewType.fragment: 'history.md',
+    ViewType.page: 'overview.html',
+}
+REQUIRED_TEMPLATES = tuple(set(fn for view in (VIEW_REPORT, VIEW_DETAIL, VIEW_TABLE, VIEW_SCOPE, VIEW_HISTORY) for fn in view.values()))
 
 
 # do I hate these globals, but I don't see another way with these frameworks
@@ -580,8 +586,9 @@ def render_view(view, view_type, detail_page='detail', base_url='/', title=None,
         stage1 = view[ViewType.fragment]
     def scope_url(uuid): return f"{base_url}page/scope/{uuid}"  # noqa: E306,E704
     def detail_url(subject, scope): return f"{base_url}page/{detail_page}/{subject}/{scope}"  # noqa: E306,E704
+    def history_url(subject, scope): return f"{base_url}page/history/{subject}/{scope}"  # noqa: E306,E704
     def report_url(report, *args, **kwargs): return _build_report_url(base_url, report, *args, **kwargs)  # noqa: E306,E704
-    fragment = templates_map[stage1].render(base_url=base_url, detail_url=detail_url, report_url=report_url, scope_url=scope_url, **kwargs)
+    fragment = templates_map[stage1].render(base_url=base_url, detail_url=detail_url, history_url=history_url, report_url=report_url, scope_url=scope_url, **kwargs)
     if view_type != ViewType.markdown and stage1.endswith('.md'):
         fragment = markdown(fragment, extensions=['extra'])
     if stage1 != stage2:
@@ -702,6 +709,24 @@ async def get_detail_full(
     )
 
 
+@app.get("/{view_type}/history/{subject}/{scopeuuid}")
+async def get_history(
+    request: Request,
+    conn: Annotated[connection, Depends(get_conn)],
+    view_type: ViewType,
+    subject: str,
+    scopeuuid: str,
+):
+    scopes = get_scopes()
+    scope_name = scopes[scopeuuid]['name'] if scopeuuid in scopes else scopeuuid
+    with conn.cursor() as cur:
+        history = db_get_report_history(cur, subject, scopeuuid, limit=HISTORY_LIMIT)
+    return render_view(
+        VIEW_HISTORY, view_type, history=history, subject=subject, scope_name=scope_name,
+        history_limit=HISTORY_LIMIT, base_url=settings.base_url, title=f'📜 Compliance history for {subject}',
+    )
+
+
 @app.get("/{view_type}/table")
 async def get_table(
     request: Request,

compliance-monitor/sql.py

@@ -194,7 +194,7 @@ def db_upgrade_schema(conn: connection, cur: cursor):
     # that way just in case we want to use another database at some point
     while True:
         current = db_get_schema_version(cur)
-        if current >= SCHEMA_VERSIONS[-1]:  # bail if version is too new (but hope it's compatible)
+        if current is not None and current >= SCHEMA_VERSIONS[-1]:  # bail if version is too new (but hope it's compatible)
             break
         if current is None:
             # this is an empty db, but it also used to be the case with v1
@@ -419,6 +419,25 @@ def db_get_recent_results2(cur: cursor, approved, limit, skip, max_age_days=None
     return [{col: val for col, val in zip(columns, row)} for row in cur.fetchall()]
 
 
+def db_get_report_history(cur: cursor, subject, scopeuuid, limit=500):
+    """list all past reports for a subject/scope, newest first, with pass/fail/abort counts"""
+    cur.execute('''
+    SELECT report.reportuuid, report.checked_at,
+        COUNT(*) FILTER (WHERE result2.result = 1) AS pass_count,
+        COUNT(*) FILTER (WHERE result2.result = -1) AS fail_count,
+        COUNT(*) FILTER (WHERE result2.result = 0) AS abort_count
+    FROM report
+    JOIN result2 ON result2.reportid = report.reportid
+    WHERE report.subject = %(subject)s
+      AND result2.scopeuuid = %(scopeuuid)s
+    GROUP BY report.reportuuid, report.checked_at
+    ORDER BY report.checked_at DESC
+    LIMIT %(limit)s;
+    ''', {"subject": subject, "scopeuuid": scopeuuid, "limit": limit})
+    columns = ('reportuuid', 'checked_at', 'pass_count', 'fail_count', 'abort_count')
+    return [{col: val for col, val in zip(columns, row)} for row in cur.fetchall()]
+
+
 def db_patch_approval2(cur: cursor, record):
     cur.execute('''
     UPDATE result2

compliance-monitor/templates/details.md.j2

@@ -22,6 +22,7 @@ Jump to
 ## {{ subject }}: {{ scope_result.name }}
 
 - [spec overview]({{ scope_url(scopeuuid) }})
+- [📜 compliance history]({{ history_url(subject, scopeuuid) }})
 
 {% if not scope_result.relevant -%}
 

compliance-monitor/templates/history.md.j2

@@ -0,0 +1,14 @@
+## 📜 Compliance history: {{ subject }} — {{ scope_name }}
+
+{% if history %}
+| Date | PASS | FAIL | ABORT | Report |
+|---|---|---|---|---|
+{% for entry in history -%}
+| {{ entry.checked_at | short_isodate }} | {{ entry.pass_count }} | {{ entry.fail_count }} | {{ entry.abort_count }} | [view]({{ report_url(entry.reportuuid) }}) |
+{% endfor %}
+{% if history | length == history_limit %}
+_Showing the most recent {{ history_limit }} reports. Older entries are not displayed._
+{% endif %}
+{% else %}
+No history available.
+{% endif %}

compliance-monitor/templates/report.md.j2

@@ -3,6 +3,7 @@
 - uuid: [{{ report.run.uuid }}]({{ report_url(report.run.uuid, download=True) }})
 - subject: {{ report.subject }}
 - scope: [{{ report.spec.name }}]({{ scope_url(report.spec.uuid) }})
+- [📜 compliance history]({{ history_url(report.subject, report.spec.uuid) }})
 - checked at: {{ report.checked_at }}
 - variable assignment: {% set comma = joiner(", ") %}{% for key, value in report.run.assignment.items() -%}{{comma()}}`{{ key }}`=`{{ value }}`{% endfor %}