From b7b1f7022e3e398b17480b132335055e3dd74616 Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Thu, 30 Oct 2025 13:37:10 +0200 Subject: [PATCH 1/4] added supabase conectivity log and instructions on failure --- src/robusta/core/exceptions.py | 9 ++++++ .../core/sinks/robusta/dal/supabase_dal.py | 29 +++++++++++++++---- src/robusta/model/config.py | 5 +++- 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/src/robusta/core/exceptions.py b/src/robusta/core/exceptions.py index 94d54e114..2bae7763b 100644 --- a/src/robusta/core/exceptions.py +++ b/src/robusta/core/exceptions.py @@ -14,3 +14,12 @@ class NoAlertManagerUrlFound(Exception): """Exception, when AlertManager url is incorrect""" pass + + +class SupabaseDnsException(Exception): + """Exception for Supabase DNS/connectivity issues (host resolution / firewall). + + Raised when the runner cannot resolve or connect to the configured Supabase URL. + """ + + pass diff --git a/src/robusta/core/sinks/robusta/dal/supabase_dal.py b/src/robusta/core/sinks/robusta/dal/supabase_dal.py index 5a32fe09f..84d4f5fc1 100644 --- a/src/robusta/core/sinks/robusta/dal/supabase_dal.py +++ b/src/robusta/core/sinks/robusta/dal/supabase_dal.py @@ -6,7 +6,6 @@ import threading from typing import Any, Dict, List, Optional, Tuple from uuid import uuid4 - from cachetools import TTLCache import requests from postgrest._sync.request_builder import SyncQueryRequestBuilder @@ -18,6 +17,7 @@ from supabase.lib.client_options import ClientOptions from robusta.core.model.cluster_status import ClusterStatus +from robusta.core.exceptions import SupabaseDnsException from robusta.core.model.env_vars import SUPABASE_TIMEOUT_SECONDS from robusta.core.model.helm_release import HelmRelease from robusta.core.model.jobs import JobInfo @@ -541,10 +541,29 @@ def publish_helm_releases(self, helm_releases: List[HelmRelease]): def sign_in(self) -> str: logging.info("Supabase dal login") - res = self.client.auth.sign_in_with_password({"email": self.email, "password": self.password}) - self.client.auth.set_session(res.session.access_token, res.session.refresh_token) - self.client.postgrest.auth(res.session.access_token) - return res.user.id + try: + res = self.client.auth.sign_in_with_password({"email": self.email, "password": self.password}) + self.client.auth.set_session(res.session.access_token, res.session.refresh_token) + self.client.postgrest.auth(res.session.access_token) + return res.user.id + except Exception as e: + # Check if this is a DNS-related error + error_msg = str(e).lower() + if any(dns_indicator in error_msg for dns_indicator in [ + "temporary failure in name resolution", + "name resolution", + "dns", + "name or service not known", + "nodename nor servname provided" + ]): + message = ( + f"\n{e}\nCannot connect to Robusta SaaS <{self.url}>. " + f"\nThis is often due to DNS issues or Firewall policies. " + f"\nPlease run the following command in your cluster and verify it does not print 'Could not resolve host': " + f"\ncurl -I {self.url}\n" + ) + raise SupabaseDnsException(message) from e + raise def to_db_cluster_status(self, data: ClusterStatus) -> Dict[str, Any]: db_cluster_status = data.dict() diff --git a/src/robusta/model/config.py b/src/robusta/model/config.py index 8172b8069..78961b1f3 100644 --- a/src/robusta/model/config.py +++ b/src/robusta/model/config.py @@ -11,6 +11,7 @@ from robusta.core.pubsub.events_pubsub import EventsPubSub from robusta.core.sinks.robusta.robusta_sink import RobustaSink from robusta.core.sinks.robusta.robusta_sink_params import RobustaSinkConfigWrapper, RobustaSinkParams +from robusta.core.exceptions import SupabaseDnsException from robusta.core.sinks.sink_base import SinkBase from robusta.core.sinks.sink_config import SinkConfigBase from robusta.core.sinks.sink_factory import SinkFactory @@ -91,7 +92,9 @@ def construct_new_sinks( except Exception as e: has_sink_errors = True - logging.error(f"Failed to initialize sink {sink_name}: {e}", exc_info=True) + # Don't show trace for Robusta SaaS DNS-related exception to avoid hiding the error in noisy logs + exec_info = not isinstance(e, SupabaseDnsException) + logging.error(f"Failed to initialize sink {sink_name}: {e}", exc_info=exec_info) if not continue_on_sink_errors: raise # Skip this sink if continue_on_sink_errors is True From 90bffb98966dd19fa4eb87de3328a505115967d1 Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Thu, 30 Oct 2025 13:47:39 +0200 Subject: [PATCH 2/4] update error message --- src/robusta/core/sinks/robusta/dal/supabase_dal.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/robusta/core/sinks/robusta/dal/supabase_dal.py b/src/robusta/core/sinks/robusta/dal/supabase_dal.py index 84d4f5fc1..2c0adf049 100644 --- a/src/robusta/core/sinks/robusta/dal/supabase_dal.py +++ b/src/robusta/core/sinks/robusta/dal/supabase_dal.py @@ -557,7 +557,7 @@ def sign_in(self) -> str: "nodename nor servname provided" ]): message = ( - f"\n{e}\nCannot connect to Robusta SaaS <{self.url}>. " + f"\n{e.__class__.__name__}: {e}\nCannot connect to Robusta SaaS <{self.url}>. " f"\nThis is often due to DNS issues or Firewall policies. " f"\nPlease run the following command in your cluster and verify it does not print 'Could not resolve host': " f"\ncurl -I {self.url}\n" From c2c155e8b4c2c02d44cea6a62cd3c4d247535907 Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Thu, 30 Oct 2025 14:25:05 +0200 Subject: [PATCH 3/4] swallowing stack trace and changing message --- src/robusta/core/exceptions.py | 12 ++++++++++-- src/robusta/core/sinks/robusta/dal/supabase_dal.py | 9 ++------- src/robusta/runner/config_loader.py | 13 ++++++++----- 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/robusta/core/exceptions.py b/src/robusta/core/exceptions.py index 2bae7763b..058a33921 100644 --- a/src/robusta/core/exceptions.py +++ b/src/robusta/core/exceptions.py @@ -21,5 +21,13 @@ class SupabaseDnsException(Exception): Raised when the runner cannot resolve or connect to the configured Supabase URL. """ - - pass + def __init__(self, original_exception: Exception, url: str): + message = ( + f"\n{original_exception.__class__.__name__}: {original_exception}\n" + f"Error connecting to {url}\n" + f"This is often due to DNS issues or firewall policies - to troubleshoot run in your cluster:\n" + f"curl -I {url}" + ) + super().__init__(message) + self.original_exception = original_exception + self.url = url diff --git a/src/robusta/core/sinks/robusta/dal/supabase_dal.py b/src/robusta/core/sinks/robusta/dal/supabase_dal.py index 2c0adf049..07679105b 100644 --- a/src/robusta/core/sinks/robusta/dal/supabase_dal.py +++ b/src/robusta/core/sinks/robusta/dal/supabase_dal.py @@ -556,13 +556,8 @@ def sign_in(self) -> str: "name or service not known", "nodename nor servname provided" ]): - message = ( - f"\n{e.__class__.__name__}: {e}\nCannot connect to Robusta SaaS <{self.url}>. " - f"\nThis is often due to DNS issues or Firewall policies. " - f"\nPlease run the following command in your cluster and verify it does not print 'Could not resolve host': " - f"\ncurl -I {self.url}\n" - ) - raise SupabaseDnsException(message) from e + dns_exc = SupabaseDnsException(e, self.url) + raise dns_exc from e raise def to_db_cluster_status(self, data: ClusterStatus) -> Dict[str, Any]: diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index 5046dc470..1fa218386 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -43,6 +43,7 @@ from robusta.model.playbook_definition import PlaybookDefinition from robusta.utils.cluster_provider_discovery import cluster_provider from robusta.utils.file_system_watcher import FileSystemWatcher +from robusta.core.exceptions import SupabaseDnsException class ConfigLoader: @@ -271,11 +272,13 @@ def __reload_playbook_packages(self, change_name): str(runner_config.global_config.get("cluster_name", "no_cluster")).encode("utf-8") ).hexdigest() - except Exception: - logging.error( - "Error (re)loading playbooks/related resources, exiting.", - exc_info=True, - ) + except Exception as e: + print_exception = not isinstance(e, SupabaseDnsException) # Don't show trace for Robusta SaaS DNS-related exception to avoid hiding the error in noisy logs + + logging.error( + "Error (re)loading playbooks/related resources, exiting.", + exc_info=print_exception, + ) # Kill the whole process group (which means this process and all of its descendant # processes). The rest of the runner shutdown happens in robusta.runner.process_setup. os.killpg(os.getpgid(0), signal.SIGTERM) From 2310e7065168f8daa9e57faef6d4fa65d9551776 Mon Sep 17 00:00:00 2001 From: "avi@robusta.dev" Date: Thu, 30 Oct 2025 14:26:37 +0200 Subject: [PATCH 4/4] fix indentation --- src/robusta/runner/config_loader.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/robusta/runner/config_loader.py b/src/robusta/runner/config_loader.py index 1fa218386..cbf72b8c1 100644 --- a/src/robusta/runner/config_loader.py +++ b/src/robusta/runner/config_loader.py @@ -272,13 +272,13 @@ def __reload_playbook_packages(self, change_name): str(runner_config.global_config.get("cluster_name", "no_cluster")).encode("utf-8") ).hexdigest() - except Exception as e: - print_exception = not isinstance(e, SupabaseDnsException) # Don't show trace for Robusta SaaS DNS-related exception to avoid hiding the error in noisy logs - - logging.error( - "Error (re)loading playbooks/related resources, exiting.", - exc_info=print_exception, - ) + except Exception as e: + print_exception = not isinstance(e, SupabaseDnsException) # Don't show trace for Robusta SaaS DNS-related exception to avoid hiding the error in noisy logs + + logging.error( + "Error (re)loading playbooks/related resources, exiting.", + exc_info=print_exception, + ) # Kill the whole process group (which means this process and all of its descendant # processes). The rest of the runner shutdown happens in robusta.runner.process_setup. os.killpg(os.getpgid(0), signal.SIGTERM)