feat(api): W10+W11 — Prometheus metrics + per-prover temporal trend

GET /api/v1/proof_attempts/metrics — Prometheus text format:
  verisim_cert_proven_total      (gauge)
  verisim_cert_pending_total     (gauge)
  verisim_cert_sanctified_total  (gauge)
  verisim_cert_revoked_total     (gauge — from W9 revocation view)
Scrape interval-able. Current snapshot: proven=8, pending=13,
sanctified=2, revoked=0.

GET /api/v1/proof_attempts/trend?class=X&prover=Y&bucket_hours=N&limit=M
  — per-prover success-rate time series. Uses ClickHouse's
toStartOfInterval() to bucket by N hours (default 24, max 168 = 1 week).
Returns [{bucket_start, success_count, total_attempts, success_rate}].

Together these enable drift dashboards (trend per prover) and the
PanLL StrategyDrift panel's timeline view.

Verified live:
  safety/z3 daily → 74/75 success on 2026-04-05 bucket (98.7%)
  metrics scrape → 4 gauges, valid Prometheus exposition format

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: hyperpolymath

rust-core/verisim-api/src/lib.rs

@@ -849,6 +849,14 @@ pub fn build_router(state: AppState) -> Router {
             "/api/v1/proof_attempts/coverage",
             get(proof_attempts::coverage_handler),
         )
+        .route(
+            "/api/v1/proof_attempts/trend",
+            get(proof_attempts::trend_handler),
+        )
+        .route(
+            "/api/v1/proof_attempts/metrics",
+            get(proof_attempts::cert_metrics_handler),
+        )
         // Authentication middleware layer
         .layer(axum_middleware::from_fn_with_state(
             auth_state,

rust-core/verisim-api/src/proof_attempts.rs

@@ -309,6 +309,34 @@ pub struct CoverageResponse {
     pub coverage: Vec<CoverageRow>,
 }
 
+/// Query parameters for `GET /api/v1/proof_attempts/trend`.
+#[derive(Debug, Deserialize)]
+pub struct TrendQuery {
+    #[serde(rename = "class")]
+    pub obligation_class: String,
+    pub prover: String,
+    /// Bucket size in hours (default 24, max 168 = 1 week).
+    pub bucket_hours: Option<u32>,
+    /// Max buckets to return (default 90, max 500).
+    pub limit: Option<u32>,
+}
+
+#[derive(Debug, Serialize)]
+pub struct TrendPoint {
+    pub bucket_start: String,
+    pub success_count: u64,
+    pub total_attempts: u64,
+    pub success_rate: f64,
+}
+
+#[derive(Debug, Serialize)]
+pub struct TrendResponse {
+    pub obligation_class: String,
+    pub prover: String,
+    pub bucket_hours: u32,
+    pub points: Vec<TrendPoint>,
+}
+
 // ---------------------------------------------------------------------------
 // V4: Certificate types
 // ---------------------------------------------------------------------------
@@ -637,6 +665,119 @@ pub async fn list_attempts_handler(
     }))
 }
 
+/// `GET /api/v1/proof_attempts/trend?class=X&prover=Y&bucket_hours=24`
+///
+/// W11: per-prover success-rate trend over time (hourly or daily buckets).
+/// Returns a time series so UIs can chart prover drift.
+#[instrument]
+pub async fn trend_handler(
+    Query(params): Query<TrendQuery>,
+) -> Result<Json<TrendResponse>, HandlerError> {
+    require_non_empty(&params.obligation_class, "class")?;
+    require_non_empty(&params.prover, "prover")?;
+    let bucket_hours = params.bucket_hours.unwrap_or(24).min(168);
+    let limit = params.limit.unwrap_or(90).min(500);
+
+    let sql = format!(
+        "SELECT \
+           toString(toStartOfInterval(started_at, INTERVAL {} HOUR)) AS bucket_start, \
+           countIf(outcome = 'success') AS success_count, \
+           count() AS total_attempts \
+         FROM verisim.proof_attempts \
+         WHERE obligation_class = '{}' AND prover_used = '{}' \
+         GROUP BY bucket_start \
+         ORDER BY bucket_start DESC \
+         LIMIT {} \
+         FORMAT JSONEachRow",
+        bucket_hours,
+        escape_sql_str(&params.obligation_class),
+        escape_sql_str(&params.prover),
+        limit
+    );
+
+    let body = send_clickhouse_read(&sql).await?;
+    let points: Vec<TrendPoint> = body
+        .lines()
+        .filter(|l| !l.trim().is_empty())
+        .filter_map(|line| {
+            let v: serde_json::Value = serde_json::from_str(line).ok()?;
+            let n = u64_from_json(&v["total_attempts"])?;
+            let s = u64_from_json(&v["success_count"])?;
+            Some(TrendPoint {
+                bucket_start: v["bucket_start"]
+                    .as_str()?
+                    .replacen(' ', "T", 1),
+                success_count: s,
+                total_attempts: n,
+                success_rate: if n > 0 { (s as f64) / (n as f64) } else { 0.0 },
+            })
+        })
+        .collect();
+
+    Ok(Json(TrendResponse {
+        obligation_class: params.obligation_class,
+        prover: params.prover,
+        bucket_hours,
+        points,
+    }))
+}
+
+/// `GET /api/v1/proof_attempts/metrics` — Prometheus-format cert events.
+///
+/// W10: exposes cert counters in Prometheus text format. Call from a
+/// Prometheus scrape target every N seconds to time-series the
+/// mint/revoke cycle.
+#[instrument]
+pub async fn cert_metrics_handler() -> Result<String, HandlerError> {
+    let mut output = String::new();
+
+    // Count proven certs
+    let proven_count_sql = "SELECT count() FROM verisim.mv_proven_certificates \
+                            WHERE status = 'proven' FORMAT TSV";
+    let proven_body = send_clickhouse_read(proven_count_sql)
+        .await
+        .unwrap_or_default();
+    let proven_n: u64 = proven_body.trim().parse().unwrap_or(0);
+
+    // Count pending certs
+    let pending_count_sql = "SELECT count() FROM verisim.mv_proven_certificates \
+                             WHERE status = 'pending' FORMAT TSV";
+    let pending_body = send_clickhouse_read(pending_count_sql)
+        .await
+        .unwrap_or_default();
+    let pending_n: u64 = pending_body.trim().parse().unwrap_or(0);
+
+    // Count sanctified classes
+    let sanctified_count_sql = "SELECT count() FROM verisim.mv_sanctify_certificates \
+                                WHERE status = 'sanctified' FORMAT TSV";
+    let sanctified_body = send_clickhouse_read(sanctified_count_sql)
+        .await
+        .unwrap_or_default();
+    let sanctified_n: u64 = sanctified_body.trim().parse().unwrap_or(0);
+
+    // Count revocations
+    let revoked_count_sql = "SELECT count() FROM verisim.mv_cert_revocations FORMAT TSV";
+    let revoked_body = send_clickhouse_read(revoked_count_sql)
+        .await
+        .unwrap_or_default();
+    let revoked_n: u64 = revoked_body.trim().parse().unwrap_or(0);
+
+    output.push_str("# HELP verisim_cert_proven_total Total PROVEN certificates\n");
+    output.push_str("# TYPE verisim_cert_proven_total gauge\n");
+    output.push_str(&format!("verisim_cert_proven_total {}\n", proven_n));
+    output.push_str("# HELP verisim_cert_pending_total Total pending certificates\n");
+    output.push_str("# TYPE verisim_cert_pending_total gauge\n");
+    output.push_str(&format!("verisim_cert_pending_total {}\n", pending_n));
+    output.push_str("# HELP verisim_cert_sanctified_total Total SANCTIFIED classes\n");
+    output.push_str("# TYPE verisim_cert_sanctified_total gauge\n");
+    output.push_str(&format!("verisim_cert_sanctified_total {}\n", sanctified_n));
+    output.push_str("# HELP verisim_cert_revoked_total Certs proven all-time but pending in 30-day window\n");
+    output.push_str("# TYPE verisim_cert_revoked_total gauge\n");
+    output.push_str(&format!("verisim_cert_revoked_total {}\n", revoked_n));
+
+    Ok(output)
+}
+
 /// `GET /api/v1/proof_attempts/coverage`
 ///
 /// Returns cross-repo coverage stats per obligation_class: how many distinct