apollo_dashboard,apollo_deployments: add top-level intervalSec to Alerts; use it in alert builder

Move the evaluation interval from a per-alert field to a top-level field
on the Alerts struct. The interval is group-scoped in Grafana, so a single
value for all alerts is the correct model.

Also restructure alert_builder.py to upload/dump per rule-group instead of
per individual alert, using the top-level intervalSec as the group interval.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: Itay-Tsabary-Starkware

crates/apollo_dashboard/resources/dev_grafana_alerts.json

@@ -1,4 +1,5 @@
 {
+  "intervalSec": 30,
   "alerts": [
     {
       "name": "batcher_storage_open_read_transactions",

crates/apollo_dashboard/src/alert_definitions.rs

@@ -692,5 +692,5 @@ pub fn get_apollo_alerts() -> Alerts {
     alerts.push(get_state_sync_lag());
     alerts.append(&mut get_state_sync_stuck_vec());
 
-    Alerts::new(alerts)
+    Alerts::new(alerts, EVALUATION_INTERVAL_SEC_DEFAULT)
 }

crates/apollo_dashboard/src/alerts.rs

@@ -16,11 +16,14 @@ pub(crate) const SECS_IN_MIN: u64 = 60;
 /// Alerts to be configured in the dashboard.
 #[derive(Debug, Serialize)]
 pub struct Alerts {
+    // The interval in seconds between evaluations of all alerts in the group.
+    #[serde(rename = "intervalSec")]
+    interval_sec: u64,
     alerts: Vec<Alert>,
 }
 
 impl Alerts {
-    pub(crate) fn new(alerts: Vec<Alert>) -> Self {
+    pub(crate) fn new(alerts: Vec<Alert>, interval_sec: u64) -> Self {
         // Validate that there are no duplicate alert names.
         alerts
             .iter()
@@ -39,7 +42,7 @@ impl Alerts {
                 panic!("Duplicate placeholder name found across alerts: {duplicate}")
             });
 
-        Self { alerts }
+        Self { interval_sec, alerts }
     }
 }
 

deployments/monitoring/src/builders/alert_builder.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+import collections
 import json
 import os
 import sys
@@ -20,12 +21,6 @@
 )
 from common.logger import get_logger
 from grafana_client import GrafanaApi
-from grafana_client.client import (
-    GrafanaBadInputError,
-    GrafanaClientError,
-    GrafanaException,
-    GrafanaServerError,
-)
 from tenacity import before_sleep_log, retry, stop_after_attempt, wait_fixed
 
 # Global logger (initialized in alert_builder function)
@@ -69,7 +64,6 @@ def create_alert_rule(
     title: str,
     folder_uid: str,
     rule_group: str,
-    interval_sec: int,
     _for: str,
     expr: str,
     conditions: list[dict[str, any]],
@@ -82,7 +76,6 @@ def create_alert_rule(
     alert_rule["title"] = title
     alert_rule["folderUID"] = folder_uid
     alert_rule["ruleGroup"] = rule_group
-    alert_rule["intervalSec"] = interval_sec
     alert_rule["for"] = _for
     alert_rule["labels"] = labels
     alert_rule["data"] = [
@@ -100,6 +93,10 @@ def create_alert_rule(
     return alert_rule
 
 
+def create_rule_group(name: str, interval_sec: int, rules: list[dict[str, any]]) -> dict[str, any]:
+    return {"name": name, "interval": interval_sec, "rules": rules}
+
+
 def get_all_folders(client: GrafanaApi) -> list[dict[str, any]]:
     logger.debug("Getting all folders")
     return client.folder.get_all_folders()
@@ -128,25 +125,14 @@ def create_folder_return_uid(client: GrafanaApi, title: str) -> str:
         return folder["uid"]
 
 
-def dump_alert(output_dir: str, alert: dict[str, any]) -> None:
-    alert_full_path = f"{output_dir}/{alert['name']}.json".lower().replace(" ", "_")
+def dump_rule_group(output_dir: str, rule_group: dict[str, any]) -> None:
+    group_full_path = f"{output_dir}/{rule_group['name']}.json".lower().replace(" ", "_")
     os.makedirs(output_dir, exist_ok=True)
-    ordered = {"title": alert["title"], **{k: v for k, v in alert.items() if k != "title"}}
-    with open(alert_full_path, "w") as f:
-        json.dump(ordered, f, indent=2)
-    # Format with professional colors: Alert (white bold), name (cyan), saved to (white bold), path (dim cyan)
+    with open(group_full_path, "w") as f:
+        json.dump(rule_group, f, indent=2)
     logger.info(
-        f'[bold white]Alert[/bold white] "[blue]{alert["name"]}[/blue]" [bold white]saved to[/bold white] [dim white]{alert_full_path}[/dim white]'
-    )
-
-
-def get_alert_rule_group(client: GrafanaApi, folder_uid: str, group_uid: str) -> str:
-    logger.debug(f'Getting alert rule group "{group_uid}"')
-    rule_group = client.alertingprovisioning.get_rule_group(
-        folder_uid=folder_uid, group_uid=group_uid
+        f'[bold white]Rule group[/bold white] "[blue]{rule_group["name"]}[/blue]" [bold white]saved to[/bold white] [dim white]{group_full_path}[/dim white]'
     )
-    logger.debug(f"Got alert group: {rule_group}")
-    return rule_group
 
 
 @retry(
@@ -296,7 +282,10 @@ def alert_builder(args: argparse.Namespace):
         # Exit cleanly without traceback
         sys.exit(1)
 
-    alerts = []
+    interval_sec = dev_alerts["intervalSec"]
+
+    # group_name -> list of alert rules (preserving insertion order within each group)
+    groups: dict[str, list[dict[str, any]]] = collections.defaultdict(list)
 
     for dev_alert in dev_alerts["alerts"]:
         # Apply config overrides to replace placeholders
@@ -315,13 +304,14 @@ def alert_builder(args: argparse.Namespace):
             )
         else:
             expr = remove_expr_placeholder(expr=dev_alert["expr"])
-        alerts.append(
+
+        group_name = dev_alert["ruleGroup"]
+        groups[group_name].append(
             create_alert_rule(
                 name=dev_alert["name"],
                 title=dev_alert["title"],
                 folder_uid=folder_uid,
-                interval_sec=dev_alert["intervalSec"],
-                rule_group=dev_alert["ruleGroup"],
+                rule_group=group_name,
                 _for=dev_alert["for"],
                 expr=expr,
                 conditions=dev_alert["conditions"],
@@ -333,66 +323,31 @@ def alert_builder(args: argparse.Namespace):
             )
         )
 
-    alerts.sort(key=lambda a: a["title"])
+    rule_groups = [
+        create_rule_group(
+            name=group_name,
+            interval_sec=interval_sec,
+            rules=sorted(rules, key=lambda a: a["title"]),
+        )
+        for group_name, rules in sorted(groups.items())
+    ]
 
-    for alert in alerts:
+    for rule_group in rule_groups:
         if args.debug:
-            logger.debug(json.dumps(alert))
+            logger.debug(json.dumps(rule_group))
         if not args.dry_run:
-            alert_created_or_exists = False
             try:
-                client.alertingprovisioning.create_alertrule(
-                    alertrule=alert,
-                    disable_provenance=True,
-                )
-                logger.info(f'Alert "{alert["name"]}" uploaded to Grafana successfully')
-                alert_created_or_exists = True
-
-            except GrafanaBadInputError as e:
-                if "alerting.alert-rule.conflict" in e.message:
-                    logger.info(f'Alert "{alert["name"]}" already exists. Skipping creation.')
-                    alert_created_or_exists = True
-                else:
-                    # Handle other bad input errors
-                    logger.error(
-                        f'Failed to create alert "{alert["name"]}". Bad input: {e.message}'
-                    )
-            except GrafanaClientError as e:
-                # Handle other client-side errors (e.g., invalid request)
-                logger.error(f'Failed to create alert "{alert["name"]}". Client error: {e.message}')
-            except GrafanaServerError as e:
-                # Handle server-side errors (5xx errors)
-                logger.error(f'Failed to create alert "{alert["name"]}". Server error: {e.message}')
-            except GrafanaException as e:
-                # Catch any other Grafana-related exceptions
-                logger.error(
-                    f'Failed to create alert "{alert["name"]}". Grafana error: {e.message}'
+                update_alert_rule_group(
+                    client=client,
+                    folder_uid=folder_uid,
+                    group_uid=rule_group["name"],
+                    alertrule_group=rule_group,
                 )
             except Exception as e:
-                # Catch any other exceptions (non-Grafana-related)
-                logger.error(f'Failed to create alert "{alert["name"]}". Unexpected error: {e}')
-
-            # Only update rule group interval if alert was successfully created or already exists
-            if alert_created_or_exists:
-                try:
-                    group_uid = alert["ruleGroup"]
-                    rule_group = get_alert_rule_group(
-                        client=client, folder_uid=folder_uid, group_uid=group_uid
-                    )
-                    if rule_group["interval"] != alert["intervalSec"]:
-                        rule_group["interval"] = alert["intervalSec"]
-                        update_alert_rule_group(
-                            client=client,
-                            folder_uid=folder_uid,
-                            group_uid=group_uid,
-                            alertrule_group=rule_group,
-                        )
-                        logger.info(f'Alert rule group "{group_uid}" updated successfully')
-                except Exception as e:
-                    logger.error(f'Failed to update alert rule group "{alert["ruleGroup"]}". {e}')
+                logger.error(f'Failed to update rule group "{rule_group["name"]}". {e}')
 
         if args.out_dir:
             output_dir = f"{args.out_dir}/alerts"
-            dump_alert(output_dir=output_dir, alert=alert)
+            dump_rule_group(output_dir=output_dir, rule_group=rule_group)
 
     logger.info("Done building grafana alerts")

deployments/sequencer/src/constructs/grafana.py

@@ -105,10 +105,7 @@ def __init__(
         super().__init__(scope, id, cluster, namespace)
 
         self.grafana_alert_group = grafana_alert_rule_group
-        self.grafana_alert_files = self.grafana_alert_group.get_alert_files()
-        self.hash_value = generate_random_hash(from_string=f"{self.cluster}-{self.namespace}")
-        self.custom_name = f"{self.namespace}-arg-{self.hash_value}"
-        self._get_shared_grafana_alert_rule_group()
+        self._create_alert_rule_group_crds()
 
     def _exec_err_state_enum_selector(self, exec_err_state: str) -> Optional[str]:
         """Convert string to ExecErrState enum."""
@@ -167,33 +164,30 @@ def _get_shared_grafana_alert_rule_group_rules(self, rule: Dict[str, Any]):
             ],
         )
 
-    def _get_shared_grafana_alert_rule_group_spec(self):
-        """Build the spec for the alert rule group."""
-        loaded_alert_rules = [
-            self.grafana_alert_group.load(str(alert_file))
-            for alert_file in self.grafana_alert_files
-        ]
-        # Keep rule order deterministic so generated YAML has stable PR diffs.
-        loaded_alert_rules.sort(key=lambda rule: rule["title"].lower())
-        rules = [
-            self._get_shared_grafana_alert_rule_group_rules(alert_rule)
-            for alert_rule in loaded_alert_rules
-        ]
-
-        return SharedGrafanaAlertRuleGroupSpec(
-            name=self.custom_name,
-            instance_selector=SharedGrafanaAlertRuleGroupSpecInstanceSelector(),
-            interval="1m",
-            editable=False,
-            folder_ref=self.cluster,
-            rules=rules,
-        )
-
-    def _get_shared_grafana_alert_rule_group(self):
-        """Create the SharedGrafanaAlertRuleGroup resource."""
-        return SharedGrafanaAlertRuleGroup(
-            self,
-            self.node.id,
-            metadata=self._get_api_object_metadata(name=self.custom_name),
-            spec=self._get_shared_grafana_alert_rule_group_spec(),
-        )
+    def _create_alert_rule_group_crds(self):
+        """Create one SharedGrafanaAlertRuleGroup CRD per rule group."""
+        for group_file in sorted(self.grafana_alert_group.get_alert_files()):
+            group_data = self.grafana_alert_group.load(str(group_file))
+            group_name = group_data["name"]
+            interval_sec = group_data["interval"]
+            # Keep rule order deterministic so generated YAML has stable PR diffs.
+            rules_data = sorted(group_data["rules"], key=lambda rule: rule["title"].lower())
+
+            k8s_group_name = group_name.replace("_", "-")
+            custom_name = f"{self.namespace}-arg-{k8s_group_name}"
+
+            rules = [self._get_shared_grafana_alert_rule_group_rules(rule) for rule in rules_data]
+            spec = SharedGrafanaAlertRuleGroupSpec(
+                name=custom_name,
+                instance_selector=SharedGrafanaAlertRuleGroupSpecInstanceSelector(),
+                interval=f"{interval_sec}s",
+                editable=False,
+                folder_ref=self.cluster,
+                rules=rules,
+            )
+            SharedGrafanaAlertRuleGroup(
+                self,
+                custom_name,
+                metadata=self._get_api_object_metadata(name=custom_name),
+                spec=spec,
+            )