fix: more robust metrics reporting in IRM monitor (#23038)

Fixes an issue where stuck requests would update the gauge after it was
already updated by subsequent requests that succeeded quickly.
This forces the gauge to always be updated in sequence, or the result is
just dropped

Also added some logging so we can see what's happening

Author: spypsy

.github/workflows/deploy-irm.yml

@@ -69,7 +69,7 @@ jobs:
       CLUSTER_NAME: ${{ inputs.cluster }}
       GKE_CLUSTER_CONTEXT: "gke_testnet-440309_us-west1-a_${{ inputs.cluster }}"
       REGION: us-west1-a
-      INFURA_SECRET_NAME: infura-${{ inputs.l1_network }}-url
+      ETHEREUM_HOSTS_SECRET_NAME: irm-ethereum-hosts-${{ inputs.l1_network }}
 
     runs-on: ubuntu-latest
     steps:
@@ -150,4 +150,4 @@ jobs:
           echo "L1 network: ${{ inputs.l1_network }}"
           echo "Image tag: ${IMAGE_TAG}"
 
-          ./spartan/metrics/irm-monitor/scripts/update-monitoring.sh $NAMESPACE $MONITORING_NAMESPACE ${{ inputs.network }} $INFURA_SECRET_NAME
+          ./spartan/metrics/irm-monitor/scripts/update-monitoring.sh $NAMESPACE $MONITORING_NAMESPACE ${{ inputs.network }} $ETHEREUM_HOSTS_SECRET_NAME

spartan/metrics/irm-monitor/VERSION

@@ -1 +1 @@
-2.0.0
+2.1.1

spartan/metrics/irm-monitor/index.ts

@@ -2,33 +2,50 @@ import express, { Request, Response } from "express";
 import { createPublicClient, http } from "viem";
 import client from "prom-client";
 
-const { ROLLUP_CONTRACT_ADDRESS, ETHEREUM_HOST, NETWORK } = process.env;
+const { ROLLUP_CONTRACT_ADDRESS, ETHEREUM_HOSTS, NETWORK } = process.env;
 
 //////////////////////////////
 // IMPORTANT: Bump VERSION file when making changes
 //////////////////////////////
 
-if (!ROLLUP_CONTRACT_ADDRESS || !ETHEREUM_HOST || !NETWORK) {
+const ethereumRpcUrls = (ETHEREUM_HOSTS ?? "")
+  .split(",")
+  .map((u: string) => u.trim())
+  .filter(Boolean);
+
+if (!ROLLUP_CONTRACT_ADDRESS || ethereumRpcUrls.length === 0 || !NETWORK) {
   console.error(
-    "ROLLUP_CONTRACT_ADDRESS, ETHEREUM_HOST and NETWORK are required. Provided: ",
+    "ROLLUP_CONTRACT_ADDRESS, ETHEREUM_HOSTS and NETWORK are required. Provided: ",
     ROLLUP_CONTRACT_ADDRESS,
-    ETHEREUM_HOST,
-    NETWORK
+    ETHEREUM_HOSTS,
+    NETWORK,
   );
   throw new Error(
-    "ROLLUP_CONTRACT_ADDRESS, ETHEREUM_HOST and NETWORK are required"
+    "ROLLUP_CONTRACT_ADDRESS, ETHEREUM_HOSTS and NETWORK are required",
   );
 }
 
 if (!ROLLUP_CONTRACT_ADDRESS.startsWith("0x")) {
   throw new Error("ROLLUP_CONTRACT_ADDRESS must start with 0x");
 }
 
-const transport = http(ETHEREUM_HOST);
+const RPC_TIMEOUT_MS = 12_000;
 
-const publicClient = createPublicClient({
-  transport,
-});
+const publicClientsByRpcUrl = new Map<
+  string,
+  ReturnType<typeof createPublicClient>
+>();
+
+function getPublicClient(rpcUrl: string) {
+  let c = publicClientsByRpcUrl.get(rpcUrl);
+  if (!c) {
+    c = createPublicClient({
+      transport: http(rpcUrl, { timeout: RPC_TIMEOUT_MS }),
+    });
+    publicClientsByRpcUrl.set(rpcUrl, c);
+  }
+  return c;
+}
 
 const ROLLUP_ABI = [
   {
@@ -74,23 +91,95 @@ const pendingCheckpointNumberGauge = new client.Gauge({
   labelNames: ["network"],
 });
 
-async function updateCheckpointNumbers(): Promise<void> {
-  try {
-    const provenCheckpointNumber = await publicClient.readContract({
+const POLL_INTERVAL_MS = 36_000;
+
+let lastStartedUpdateId = 0;
+
+async function readCheckpointsFromRpc(
+  rpcUrl: string,
+  blockNumber: bigint,
+): Promise<{ proven: number; pending: number }> {
+  const publicClient = getPublicClient(rpcUrl);
+  const [provenCheckpointNumber, pendingCheckpointNumber] = await Promise.all([
+    publicClient.readContract({
       address: ROLLUP_CONTRACT_ADDRESS as `0x${string}`,
       abi: ROLLUP_ABI,
       functionName: "getProvenCheckpointNumber",
-    });
-    provenCheckpointNumberGauge.set(Number(provenCheckpointNumber));
-
-    const pendingCheckpointNumber = await publicClient.readContract({
+      blockNumber,
+    }),
+    publicClient.readContract({
       address: ROLLUP_CONTRACT_ADDRESS as `0x${string}`,
       abi: ROLLUP_ABI,
       functionName: "getPendingCheckpointNumber",
+      blockNumber,
+    }),
+  ]);
+  return {
+    proven: Number(provenCheckpointNumber),
+    pending: Number(pendingCheckpointNumber),
+  };
+}
+
+async function updateCheckpointNumbers(): Promise<void> {
+  const thisUpdateId = ++lastStartedUpdateId;
+  const startedAt = Date.now();
+  try {
+    const blockNumber = await getPublicClient(
+      ethereumRpcUrls[0]!,
+    ).getBlockNumber();
+    const settled = await Promise.allSettled(
+      ethereumRpcUrls.map((url) => readCheckpointsFromRpc(url, blockNumber)),
+    );
+
+    if (thisUpdateId !== lastStartedUpdateId) {
+      console.log("skipped stale checkpoint read", {
+        updateId: thisUpdateId,
+        latestUpdateId: lastStartedUpdateId,
+        elapsedMs: Date.now() - startedAt,
+      });
+      return;
+    }
+
+    const successes: { proven: number; pending: number }[] = [];
+    const failures: { rpcUrl: string; reason: unknown }[] = [];
+    for (let i = 0; i < settled.length; i++) {
+      const r = settled[i]!;
+      const rpcUrl = ethereumRpcUrls[i]!;
+      if (r.status === "fulfilled") {
+        successes.push(r.value);
+      } else {
+        failures.push({ rpcUrl, reason: r.reason });
+      }
+    }
+
+    if (successes.length === 0) {
+      console.error(
+        `checkpoint update failed: all ${ethereumRpcUrls.length} RPC host(s) failed (updateId=${thisUpdateId})`,
+        failures,
+      );
+      return;
+    }
+
+    const proven = Math.max(...successes.map((s) => s.proven));
+    const pending = Math.max(...successes.map((s) => s.pending));
+    provenCheckpointNumberGauge.set(proven);
+    pendingCheckpointNumberGauge.set(pending);
+    console.log("checkpoints updated", {
+      updateId: thisUpdateId,
+      proven,
+      pending,
+      rpcHostsOk: successes.length,
+      rpcHostsFailed: failures.length,
+      elapsedMs: Date.now() - startedAt,
     });
-    pendingCheckpointNumberGauge.set(Number(pendingCheckpointNumber));
+    if (failures.length > 0) {
+      console.warn(
+        `checkpoint read: ${failures.length} RPC host(s) failed; using max across ${successes.length} successful response(s)`,
+        failures.map((f) => ({ rpcUrl: f.rpcUrl, reason: f.reason })),
+      );
+    }
   } catch (error) {
-    console.error("Error updating checkpoint numbers:", error);
+    console.error(`checkpoint update failed (updateId=${thisUpdateId})`, error);
   }
 }
 
@@ -102,10 +191,16 @@ app.get("/metrics", async (_req: Request, res: Response) => {
 
 const port = process.env.PORT ? Number(process.env.PORT) : 8080;
 app.listen(port, () => {
-  console.log(`Metrics server listening on port ${port}`);
+  console.log("metrics server listening", {
+    port,
+    network: NETWORK,
+    rollup: ROLLUP_CONTRACT_ADDRESS,
+    ethereumRpcUrls,
+    pollIntervalMs: POLL_INTERVAL_MS,
+  });
 });
 
-setInterval(updateCheckpointNumbers, 36000);
+setInterval(updateCheckpointNumbers, POLL_INTERVAL_MS);
 updateCheckpointNumbers();
 
 // Expose default process metrics, including process_start_time_seconds

spartan/metrics/irm-monitor/kubernetes/monitoring-deployment.yaml

@@ -23,11 +23,11 @@ spec:
               containerPort: 8080
               protocol: TCP
           env:
-            - name: ETHEREUM_HOST
+            - name: ETHEREUM_HOSTS
               valueFrom:
                 secretKeyRef:
                   name: irm-monitor-secrets
-                  key: infura-sepolia-url
+                  key: ethereum-hosts
             - name: ROLLUP_CONTRACT_ADDRESS
               value: "${ROLLUP_CONTRACT_ADDRESS}"
             - name: NETWORK

spartan/metrics/irm-monitor/kubernetes/monitoring-secret.yaml

@@ -4,5 +4,5 @@ metadata:
   name: irm-monitor-secrets
 type: Opaque
 data:
-  infura-sepolia-url: ""
+  ethereum-hosts: ""
   grafana-cloud-password: ""

spartan/metrics/irm-monitor/scripts/update-monitoring.sh

@@ -8,7 +8,7 @@ set -e
 NAMESPACE=${1:-"testnet"}
 MONITORING_NAMESPACE=${2:-"$NAMESPACE-irm"}
 NETWORK=${3:-"$NAMESPACE"}
-INFURA_URL_SECRET=${4:-"infura-sepolia-url"}
+ETHEREUM_HOSTS_SECRET=${4:-"irm-ethereum-hosts-sepolia"}
 
 # Deployment name includes the monitoring namespace prefix
 export DEPLOYMENT_NAME="${MONITORING_NAMESPACE}-monitor"
@@ -130,7 +130,7 @@ fi
 
 # Fetch GCP secrets
 echo "Fetching GCP secrets..."
-INFURA_URL=$(gcloud secrets versions access latest --secret=$INFURA_URL_SECRET)
+ETHEREUM_HOSTS=$(gcloud secrets versions access latest --secret=$ETHEREUM_HOSTS_SECRET)
 GRAFANA_PASSWORD=$(gcloud secrets versions access latest --secret=grafana-cloud-password)
 
 # Ensure monitoring namespace exists
@@ -139,7 +139,7 @@ kubectl get ns "$MONITORING_NAMESPACE" >/dev/null 2>&1 || kubectl create ns "$MO
 # Create/update secrets
 echo "Applying monitoring secrets..."
 kubectl -n "$MONITORING_NAMESPACE" create secret generic irm-monitor-secrets \
-  --from-literal=infura-sepolia-url="$INFURA_URL" \
+  --from-literal=ethereum-hosts="$ETHEREUM_HOSTS" \
   --from-literal=grafana-cloud-password="$GRAFANA_PASSWORD" \
   --dry-run=client -o yaml | kubectl apply -f -