Merge branch 'master' into rob-846_holmes_docs_allow_direct_loki_tempo_connection

nherment · web-flow · commit dcf98f45f448 · 2025-05-13T07:09:01.000+02:00
diff --git a/.gitignore b/.gitignore
@@ -21,3 +21,5 @@ skaffold.dev.yaml
 tests/last_used_tag.txt
 pytest.ini
 gen-config-test
+
+**/.claude/settings.local.json
diff --git a/docs/configuration/alertmanager-integration/_pull_integration.rst b/docs/configuration/alertmanager-integration/_pull_integration.rst
@@ -24,6 +24,12 @@ If Robusta fails to auto-detect the Prometheus and Alertmanager urls - and you s
         # prometheus_auth: ...
         # alertmanager_auth: ...
 
+        # If using a multi-tenant prometheus or alertmanager, pass the org id to all queries
+        # prometheus_additional_headers:
+        #   X-Scope-OrgID: <org id>
+        # alertmanager_additional_headers:
+        #   X-Scope-OrgID: <org id>
+
 .. code-annotations::
     1. Example: http://alertmanager-Helm_release_name-kube-prometheus-alertmanager.default.svc.cluster.local:9093.
     2. Example: http://Helm_Release_Name-kube-prometheus-prometheus.default.svc.cluster.local:9090
diff --git a/docs/configuration/alertmanager-integration/_testing_integration.rst b/docs/configuration/alertmanager-integration/_testing_integration.rst
@@ -3,12 +3,43 @@ Verify it Works
 
 Send a dummy alert to AlertManager:
 
-.. code-block:: bash
+.. tab-set::
 
-    robusta demo-alert
+    .. tab-item:: Robusta CLI
+
+        If you have the Robusta CLI installed, you can send a test alert using the following command:
+
+        .. code-block:: bash
+
+            robusta demo-alert
+
+    .. tab-item:: Robusta UI
+
+        In the Robusta UI, go to the "Clusters" tab, choose the right cluster and click "Simulate Alert".
+        
+        .. image:: /images/robusta-ui-simulate-alert-1.png
+           :alt: Choose the cluster
+           :width: 900
+           :align: center
+
+        Then 
+
+        1. Check **Send alert with no resource**.
+        2. Provide a name for the alert in the **Alert name (identifier)** field (e.g., "Testing Prod AlertManager").
+        3. Select **Alert Manager** under the "Send alert to" section.
+        4. Click the **Simulate Alert** button to send the test alert.
+
+        .. image:: /images/robusta-ui-simulate-alert-2.png
+           :alt: Send Test Alert
+           :width: 600
+           :align: center
 
 If everything is setup properly, this alert will reach Robusta. It will show up in the Robusta UI, Slack, and other configured sinks.
 
+.. note::
+
+    It might take a few minutes for the alert to arrive due to AlertManager's `group_wait` and `group_interval` settings. More info `here <https://prometheus.io/docs/alerting/latest/configuration/#:~:text=How%20long%20to%20wait%20before%20sending%20a%20notification%20about%20new%20alerts%20that%0A%23%20are%20added%20to%20a%20group%20of%20alerts%20for%20which%20an%20initial%20notification%20has%0A%23%20already%20been%20sent>`_.
+
 .. details:: I configured AlertManager, but I'm not receiving alerts?
     :class: warning
 
diff --git a/docs/configuration/alertmanager-integration/coralogix_managed_prometheus.rst b/docs/configuration/alertmanager-integration/coralogix_managed_prometheus.rst
@@ -82,6 +82,12 @@ Metrics querying lets Robusta pull metrics from Coralogix Managed Prometheus.
       # grafana_api_key: <YOUR GRAFANA EDITOR API KEY> # (1)
       # alertmanager_flavor: grafana
 
+        # If using a multi-tenant prometheus or alertmanager, pass the org id to all queries
+        # prometheus_additional_headers:
+        #   X-Scope-OrgID: <org id>
+        # alertmanager_additional_headers:
+        #   X-Scope-OrgID: <org id>
+
 .. code-annotations::
     1. This is necessary for Robusta to create silences when using Grafana Alerts, because of minor API differences in the AlertManager embedded in Grafana.
 
diff --git a/docs/configuration/alertmanager-integration/grafana-alert-manager.rst b/docs/configuration/alertmanager-integration/grafana-alert-manager.rst
@@ -83,6 +83,12 @@ Modify and add the following config to ``generated_values.yaml`` and :ref:`updat
         # prometheus_additional_labels:
         #   cluster: 'CLUSTER_NAME_HERE'
 
+        # If using a multi-tenant prometheus or alertmanager, pass the org id to all queries
+        # prometheus_additional_headers:
+        #   X-Scope-OrgID: <org id>
+        # alertmanager_additional_headers:
+        #   X-Scope-OrgID: <org id>
+        
 .. code-annotations::
     1. This is necessary for Robusta to create silences when using Grafana Alerts, because of minor API differences in the AlertManager embedded in Grafana.
 
diff --git a/docs/configuration/alertmanager-integration/victoria-metrics.rst b/docs/configuration/alertmanager-integration/victoria-metrics.rst
@@ -8,26 +8,48 @@ You will need to configure two integrations: one to send alerts to Robusta and a
 Send Alerts to Robusta
 ============================
 
-To configure, edit AlertManager's configuration:
+Add the following to your Victoria Metrics Alertmanager configuration (e.g., Helm values file or VMAlertmanagerConfig CRD):
+
+.. code-block:: yaml
+
+    receivers:
+      - name: 'robusta'
+      webhook_configs:
+          - url: 'http://<ROBUSTA-HELM-RELEASE-NAME>-runner.<NAMESPACE>.svc.cluster.local/api/alerts'
+          send_resolved: true # (3)
+
+    route: # (1)
+      routes:
+      - receiver: 'robusta'
+          group_by: [ '...' ]
+          group_wait: 1s
+          group_interval: 1s
+          matchers:
+          - severity =~ ".*"
+          repeat_interval: 4h
+          continue: true # (2)
+
+.. code-annotations::
+    1. Put Robusta's route as the first route, to guarantee it receives alerts. If you can't do so, you must guarantee all previous routes has ``continue: true`` set.
+    2. Keep sending alerts to receivers defined after Robusta.
+    3. Important, so Robusta knows when alerts are resolved.
 
-.. include:: ./_alertmanager-config.rst
 
 .. include:: ./_testing_integration.rst
 
-Configure Metric Querying
+Configure Metrics Querying
 ====================================
 
-Metrics querying lets Robusta pull metrics and create silences.
+Robusta can query metrics and create silences using Victoria Metrics. If both are in the same Kubernetes cluster, Robusta can auto-detect the Victoria Metrics service. To verify, go to the "Apps" tab in Robusta, select an application, and check for usage graphs.
 
-Add the following to ``generated_values.yaml`` and :ref:`update Robusta <Simple Upgrade>`.
+If auto-detection fails you must add the ``prometheus_url`` parameter and :ref:`update Robusta <Simple Upgrade>`.
 
 .. code-block:: yaml
 
     globalConfig: # this line should already exist
         # add the lines below
-        alertmanager_url: "http://ALERT_MANAGER_SERVICE_NAME.NAMESPACE.svc.cluster.local:9093"
-        grafana_url: ""
-        prometheus_url: "http://VICTORIA_METRICS_SERVICE_NAME.NAMESPACE.svc.cluster.local:8429"
+        alertmanager_url: "http://<VM_ALERT_MANAGER_SERVICE_NAME>.<NAMESPACE>.svc.cluster.local:9093" # Example:"http://vmalertmanager-victoria-metrics-vm.default.svc.cluster.local:9093/"        
+        prometheus_url: "http://VM_Metrics_SERVICE_NAME.NAMESPACE.svc.cluster.local:8429" # Example:"http://vmsingle-vmks-victoria-metrics-k8s-stack.default.svc.cluster.local:8429"
         # Add any labels that are relevant to the specific cluster (optional)
         # prometheus_additional_labels:
         #   cluster: 'CLUSTER_NAME_HERE'
@@ -39,6 +61,12 @@ Add the following to ``generated_values.yaml`` and :ref:`update Robusta <Simple
         # grafana_api_key: <YOUR GRAFANA EDITOR API KEY> # (1)
         # alertmanager_flavor: grafana
 
+        # If using a multi-tenant prometheus or alertmanager, pass the org id to all queries
+        # prometheus_additional_headers:
+        #   X-Scope-OrgID: <org id>
+        # alertmanager_additional_headers:
+        #   X-Scope-OrgID: <org id>
+
 .. code-annotations::
     1. This is necessary for Robusta to create silences when using Grafana Alerts, because of minor API differences in the AlertManager embedded in Grafana.
 
diff --git a/docs/configuration/holmesgpt/toolsets/aws.rst b/docs/configuration/holmesgpt/toolsets/aws.rst
@@ -74,21 +74,7 @@ Configuration
 
   .. md-tab-item:: Robusta Helm Chart
 
-    .. code-block:: yaml
-
-        holmes:
-            additionalEnvVars:
-                - name: AWS_ACCESS_KEY_ID
-                  value: AKIXDDDSDSdSA
-                - name: AWS_SECRET_ACCESS_KEY
-                  value: =wJalrXUtnFEMI/KNG/bPxRfiCYEXAMPLEKEY
-                - name: AWS_DEFAULT_REGION
-                  value: us-west-2
-            toolsets:
-                aws/rds:
-                    enabled: true
-
-    .. include:: ./_toolset_configuration.inc.rst
+    This builtin toolset is currently only available in HolmesGPT CLI.
 
   .. md-tab-item:: Holmes CLI
 
diff --git a/docs/images/robusta-ui-simulate-alert-1.png b/docs/images/robusta-ui-simulate-alert-1.png
diff --git a/docs/images/robusta-ui-simulate-alert-2.png b/docs/images/robusta-ui-simulate-alert-2.png
diff --git a/docs/playbook-reference/index.rst b/docs/playbook-reference/index.rst
@@ -12,4 +12,4 @@
    automatic-remediation-examples/index
    prometheus-examples/index
    kubernetes-examples/index
-   logs-triggers/index
+   Log Based Alerting <logs-triggers/index>
diff --git a/docs/playbook-reference/logs-triggers/index.rst b/docs/playbook-reference/logs-triggers/index.rst
@@ -1,5 +1,7 @@
-Automate Responses to Application Logs
-==========================================
+:hide-toc:
+
+Log-based alerting and auto-remediation with fluentbit, prometheus, and robusta
+==================================================================================
 
 This tutorial walks you through building an automation that detects specific patterns in Kubernetes pod logs and responds automatically.
 
diff --git a/helm/robusta/Chart.lock b/helm/robusta/Chart.lock
@@ -4,6 +4,6 @@ dependencies:
   version: 55.7.0
 - name: holmes
   repository: https://robusta-charts.storage.googleapis.com
-  version: 0.10.9
-digest: sha256:5510176739e62f2765876c1dc397518230ae4e4d2103a9630a6546ae65716c37
-generated: "2025-04-14T11:54:09.485155+03:00"
+  version: 0.11.1
+digest: sha256:0b622c65ad0b2d18c52c4c4d8f3c19f6176ce89b1ae7c98411211ee2d50d4b46
+generated: "2025-05-12T09:51:33.315282+03:00"
diff --git a/helm/robusta/Chart.yaml b/helm/robusta/Chart.yaml
@@ -15,6 +15,6 @@ dependencies:
     condition: enablePrometheusStack
     repository: "https://prometheus-community.github.io/helm-charts"
   - name: holmes
-    version: 0.10.9
+    version: 0.11.1
     condition: enableHolmesGPT
     repository: "https://robusta-charts.storage.googleapis.com"
diff --git a/helm/robusta/charts/holmes-0.10.9.tgz b/helm/robusta/charts/holmes-0.10.9.tgz
diff --git a/helm/robusta/charts/holmes-0.11.1.tgz b/helm/robusta/charts/holmes-0.11.1.tgz
diff --git a/playbooks/robusta_playbooks/krr.py b/playbooks/robusta_playbooks/krr.py
@@ -178,11 +178,14 @@ class KRRSecret(BaseModel):
     secret_value: str
     command_flag: str
 
-    def __init__(self, env_var_name: str, secret_key: str, secret_value: str, command_flag: str):
-        secret_b64_str = base64.b64encode(bytes(secret_value, "utf-8")).decode("utf-8")
-        super().__init__(
-            env_var_name=env_var_name, secret_key=secret_key, secret_value=secret_b64_str, command_flag=command_flag
-        )
+    @validator('secret_value', pre=True, always=True, allow_reuse=True)
+    def encode_secret_value(cls, v: str) -> str:
+        if isinstance(v, str):
+            return base64.b64encode(bytes(v, "utf-8")).decode("utf-8")
+        raise ValueError("secret_value must be a string")
+
+class KRRSecretKeyValuePair(KRRSecret):
+    arg_key: str
 
 
 def _generate_krr_job_secret(scan_id: str, krr_secrets: Optional[List[KRRSecret]]) -> Optional[JobSecret]:
@@ -210,7 +213,14 @@ def _generate_krr_env_vars(
 def _generate_additional_env_args(krr_secrets: Optional[List[KRRSecret]]) -> str:
     if not krr_secrets:
         return ""
-    return " ".join(f"{secret.command_flag} '$({secret.env_var_name})'" for secret in krr_secrets)
+    additional_args = []
+    for secret in krr_secrets:
+        if isinstance(secret, KRRSecretKeyValuePair):
+            additional_args.append(f"{secret.command_flag} '{secret.arg_key}:$({secret.env_var_name})'")
+        else:
+            additional_args.append(f"{secret.command_flag} '$({secret.env_var_name})'")
+
+    return " ".join(additional_args)
 
 
 def _generate_cmd_line_args(prom_config: PrometheusConfig) -> str:
@@ -240,6 +250,22 @@ def _generate_prometheus_secrets(prom_config: PrometheusConfig) -> List[KRRSecre
                 command_flag="--prometheus-auth-header",
             )
         )
+    
+    if prom_config.headers:
+        for header_name, header_value in prom_config.headers.items():
+                
+            env_var_name = f"PROMETHEUS_HEADER_{header_name.upper().replace('-', '_')}"
+            secret_key = f"prometheus-header-{header_name.lower()}"
+            
+            krr_secrets.append(
+                KRRSecretKeyValuePair(
+                    env_var_name=env_var_name,
+                    arg_key=header_name,
+                    secret_key=secret_key,
+                    secret_value=header_value,
+                    command_flag="--prometheus-headers",
+                )
+            )
 
     if isinstance(prom_config, AWSPrometheusConfig):
         krr_secrets.extend(
diff --git a/src/robusta/core/model/base_params.py b/src/robusta/core/model/base_params.py
@@ -310,6 +310,7 @@ class PrometheusParams(ActionParams):
     :var prometheus_url: Prometheus url. If omitted, we will try to find a prometheus instance in the same cluster
     :var prometheus_auth: Prometheus auth header to be used in Authorization header. If omitted, we will not add any auth header
     :var prometheus_url_query_string: Additional query string parameters to be appended to the Prometheus connection URL
+    :var prometheus_additional_headers: additional HTTP headers (if defined) to add to every prometheus query
     :var prometheus_additional_labels: A dictionary of additional labels needed for multi-cluster prometheus
     :var add_additional_labels: adds the additional labels (if defined) to the query
 
@@ -325,6 +326,7 @@ class PrometheusParams(ActionParams):
     prometheus_url: Optional[str] = None
     prometheus_auth: Optional[SecretStr] = None
     prometheus_url_query_string: Optional[str] = None
+    prometheus_additional_headers: Optional[Dict[str, str]] = None
     prometheus_additional_labels: Optional[Dict[str, str]] = None
     add_additional_labels: bool = True
     prometheus_graphs_overrides: Optional[List[OverrideGraph]] = None
diff --git a/src/robusta/integrations/prometheus/utils.py b/src/robusta/integrations/prometheus/utils.py
@@ -48,10 +48,13 @@ def generate_prometheus_config(prometheus_params: PrometheusParams) -> Prometheu
         "url": url,
         "disable_ssl": not PROMETHEUS_SSL_ENABLED,
         "additional_labels": prometheus_params.prometheus_additional_labels,
-        "prometheus_url_query_string": prometheus_params.prometheus_url_query_string,
+        "prometheus_url_query_string": prometheus_params.prometheus_url_query_string
     }
     if prometheus_params.prometheus_auth:
         baseconfig["prometheus_auth"] = prometheus_params.prometheus_auth.get_secret_value()
+        
+    if prometheus_params.prometheus_additional_headers:
+        baseconfig["headers"] = prometheus_params.prometheus_additional_headers
 
     # aws config
     if AWS_ACCESS_KEY:
diff --git a/src/robusta/integrations/slack/sender.py b/src/robusta/integrations/slack/sender.py
@@ -2,6 +2,7 @@
 import logging
 import ssl
 import tempfile
+import re
 from datetime import datetime, timedelta
 from itertools import chain
 from typing import Any, Dict, List, Optional, Set
@@ -52,6 +53,7 @@
 ACTION_LINK = "link"
 SlackBlock = Dict[str, Any]
 MAX_BLOCK_CHARS = 3000
+MENTION_PATTERN = re.compile(r"<[^>]+>")
 
 
 class SlackSender:
@@ -407,10 +409,25 @@ def __create_holmes_callback(self, finding: Finding) -> CallbackBlock:
             }
         )
 
+    @staticmethod
+    def extract_mentions(title) -> (str, str):
+        mentions = MENTION_PATTERN.findall(title)
+
+        mention = ""
+        if mentions:
+            mention = " " + " ".join(mentions)
+            title = MENTION_PATTERN.sub("", title).strip()
+
+        return title, mention
+
+
     def __create_finding_header(
         self, finding: Finding, status: FindingStatus, platform_enabled: bool, include_investigate_link: bool
     ) -> MarkdownBlock:
         title = finding.title.removeprefix("[RESOLVED] ")
+
+        title, mention = self.extract_mentions(title)
+
         sev = finding.severity
         if finding.source == FindingSource.PROMETHEUS:
             status_name: str = (
@@ -426,7 +443,7 @@ def __create_finding_header(
             title = f"<{finding.get_investigate_uri(self.account_id, self.cluster_name)}|*{title}*>"
         return MarkdownBlock(
             f"""{status_name} {sev.to_emoji()} *{sev.name.capitalize()}*
-{title}"""
+{title}{mention}"""
         )
 
     def __create_links(
diff --git a/src/robusta/runner/telemetry_service.py b/src/robusta/runner/telemetry_service.py
@@ -34,9 +34,9 @@ def __init__(self, endpoint: str, periodic_time_sec: float, registry: Registry):
             try:
                 sentry_sdk.init(
                     SENTRY_DSN,
-                    enable_tracing=True,
-                    traces_sample_rate=0.01,
-                    profiles_sample_rate=0.01, 
+                    enable_tracing=False,
+                    traces_sample_rate=0,
+                    profiles_sample_rate=0, 
                     release=RUNNER_VERSION,
                     integrations=[
                         ThreadingIntegration(propagate_scope=True),
diff --git a/src/robusta/utils/silence_utils.py b/src/robusta/utils/silence_utils.py
@@ -56,11 +56,13 @@ def to_list(self) -> List[str]:
 class AlertManagerParams(ActionParams):
     """
     :var alertmanager_url: Alternative Alert Manager url to send requests.
+    :var alertmanager_additional_headers: additional HTTP headers (if defined) to add to every alertmanager request
     """
 
     alertmanager_flavor: str = None  # type: ignore
     alertmanager_url: Optional[str]
     alertmanager_auth: Optional[SecretStr] = None
+    alertmanager_additional_headers: Optional[Dict[str, str]] = None
     grafana_api_key: str = None  # type: ignore
 
     @validator("alertmanager_url", allow_reuse=True)
@@ -143,6 +145,9 @@ def gen_alertmanager_headers(params: AlertManagerParams) -> Dict:
 
     elif params.alertmanager_auth:
         headers.update({"Authorization": params.alertmanager_auth.get_secret_value()})
+        
+    if params.alertmanager_additional_headers:
+        headers.update(params.alertmanager_additional_headers)
 
     return headers
 
diff --git a/tests/slack/test_slack_mentions.py b/tests/slack/test_slack_mentions.py
