[fm] GC sweep for FM creation-marker tables (#10762)

The "creation marker" tables, `rendezvous_alert_created` and
`rendezvous_support_bundle_created`, contain a row for each alert and
support bundle created by FM respectively, written by
`SitrepGuardedInsert`. These prevent FM rendezvous from resurrecting a
resource the user has deleted, so it must live for as long as some
sitrep that requests the resource might still be executed. Once no
current or future sitrep can request the resource, its marker is safe to
delete. This PR adds garbage collection for these creation marker
tables.

This comprises:
- A generic page query (`RendezvousMarkerGcPage`), parameterized by
`FmRendezvousResource` (see PR #10721), that emits the same two-CTE SQL
for both tables, varying only the marker table, resource-id column, and
request table.
- Datastore wrappers driving the sweep for both resources.
- Wiring into the `fm_rendezvous` background task, with stats surfaced
through the internal API and omdb.
- Datastore tests against the dummy resource (deletion predicate, cursor
advance, expectorated SQL), plus GC assertions in the existing
background-task tests covering both real resources.

Fixes #10248.

Author: mergeconflict

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -3759,6 +3759,8 @@ fn print_task_fm_rendezvous(details: &serde_json::Value) {
         alerts,
         support_bundles,
         ereport_marking: marking,
+        alert_marker_gc,
+        support_bundle_marker_gc,
     } = match serde_json::from_value::<FmRendezvousStatus>(details.clone()) {
         Err(error) => {
             eprintln!(
@@ -3936,6 +3938,36 @@ fn print_task_fm_rendezvous(details: &serde_json::Value) {
             }
         },
     );
+    print_op(
+        "garbage collecting alert creation markers",
+        &alert_marker_gc,
+        print_marker_gc_details,
+    );
+    print_op(
+        "garbage collecting support bundle creation markers",
+        &support_bundle_marker_gc,
+        print_marker_gc_details,
+    );
+}
+
+fn print_marker_gc_details(status: &fm_rendezvous::MarkerGcStatus) {
+    let fm_rendezvous::MarkerGcStatus { rows_deleted, batches, errors } =
+        status;
+    const ROWS_DELETED: &str = "rows deleted:";
+    const BATCHES: &str = "batches:";
+    const ERRORS: &str = "errors:";
+    const WIDTH: usize = const_max_len(&[ROWS_DELETED, BATCHES, ERRORS]) + 1;
+    const NUM_WIDTH: usize = 4;
+    println!("      {ROWS_DELETED:<WIDTH$}{rows_deleted:>NUM_WIDTH$}");
+    println!("      {BATCHES:<WIDTH$}{batches:>NUM_WIDTH$}");
+    println!(
+        "{}   {ERRORS:<WIDTH$}{:>NUM_WIDTH$}",
+        warn_if_nonzero(errors.len()),
+        errors.len()
+    );
+    for error in errors {
+        println!("        > {error}");
+    }
 }
 
 fn print_task_trust_quorum_manager(details: &serde_json::Value) {

dev-tools/omdb/tests/successes.out

@@ -750,6 +750,16 @@ task: "fm_rendezvous"
       batch size:                            1000
       batches:                                  0
       errors:                                   0
+    garbage collecting alert creation markers:
+      started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+      rows deleted:    0
+      batches:         1
+      errors:          0
+    garbage collecting support bundle creation markers:
+      started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+      rows deleted:    0
+      batches:         1
+      errors:          0
 
 task: "fm_sitrep_gc"
   configured period: every <REDACTED_DURATION>s
@@ -1450,6 +1460,16 @@ task: "fm_rendezvous"
       batch size:                            1000
       batches:                                  0
       errors:                                   0
+    garbage collecting alert creation markers:
+      started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+      rows deleted:    0
+      batches:         1
+      errors:          0
+    garbage collecting support bundle creation markers:
+      started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+      rows deleted:    0
+      batches:         1
+      errors:          0
 
 task: "fm_sitrep_gc"
   configured period: every <REDACTED_DURATION>s