Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions src/robusta/core/exceptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,20 @@ class NoAlertManagerUrlFound(Exception):
"""Exception, when AlertManager url is incorrect"""

pass


class SupabaseDnsException(Exception):
"""Exception for Supabase DNS/connectivity issues (host resolution / firewall).

Raised when the runner cannot resolve or connect to the configured Supabase URL.
"""
def __init__(self, original_exception: Exception, url: str):
message = (
f"\n{original_exception.__class__.__name__}: {original_exception}\n"
f"Error connecting to {url}\n"
f"This is often due to DNS issues or firewall policies - to troubleshoot run in your cluster:\n"
f"curl -I {url}"
)
super().__init__(message)
self.original_exception = original_exception
self.url = url
24 changes: 19 additions & 5 deletions src/robusta/core/sinks/robusta/dal/supabase_dal.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import threading
from typing import Any, Dict, List, Optional, Tuple
from uuid import uuid4

from cachetools import TTLCache
import requests
from postgrest._sync.request_builder import SyncQueryRequestBuilder
Expand All @@ -18,6 +17,7 @@
from supabase.lib.client_options import ClientOptions

from robusta.core.model.cluster_status import ClusterStatus
from robusta.core.exceptions import SupabaseDnsException
from robusta.core.model.env_vars import SUPABASE_TIMEOUT_SECONDS
from robusta.core.model.helm_release import HelmRelease
from robusta.core.model.jobs import JobInfo
Expand Down Expand Up @@ -555,10 +555,24 @@ def publish_helm_releases(self, helm_releases: List[HelmRelease]):

def sign_in(self) -> str:
logging.info("Supabase dal login")
res = self.client.auth.sign_in_with_password({"email": self.email, "password": self.password})
self.client.auth.set_session(res.session.access_token, res.session.refresh_token)
self.client.postgrest.auth(res.session.access_token)
return res.user.id
try:
res = self.client.auth.sign_in_with_password({"email": self.email, "password": self.password})
self.client.auth.set_session(res.session.access_token, res.session.refresh_token)
self.client.postgrest.auth(res.session.access_token)
return res.user.id
except Exception as e:
# Check if this is a DNS-related error
error_msg = str(e).lower()
if any(dns_indicator in error_msg for dns_indicator in [
"temporary failure in name resolution",
"name resolution",
"dns",
"name or service not known",
"nodename nor servname provided"
]):
dns_exc = SupabaseDnsException(e, self.url)
raise dns_exc from e
raise

def to_db_cluster_status(self, data: ClusterStatus) -> Dict[str, Any]:
db_cluster_status = data.dict()
Expand Down
5 changes: 4 additions & 1 deletion src/robusta/model/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from robusta.core.pubsub.events_pubsub import EventsPubSub
from robusta.core.sinks.robusta.robusta_sink import RobustaSink
from robusta.core.sinks.robusta.robusta_sink_params import RobustaSinkConfigWrapper, RobustaSinkParams
from robusta.core.exceptions import SupabaseDnsException
from robusta.core.sinks.sink_base import SinkBase
from robusta.core.sinks.sink_config import SinkConfigBase
from robusta.core.sinks.sink_factory import SinkFactory
Expand Down Expand Up @@ -91,7 +92,9 @@ def construct_new_sinks(

except Exception as e:
has_sink_errors = True
logging.error(f"Failed to initialize sink {sink_name}: {e}", exc_info=True)
# Don't show trace for Robusta SaaS DNS-related exception to avoid hiding the error in noisy logs
exec_info = not isinstance(e, SupabaseDnsException)
logging.error(f"Failed to initialize sink {sink_name}: {e}", exc_info=exec_info)
if not continue_on_sink_errors:
raise
# Skip this sink if continue_on_sink_errors is True
Expand Down
7 changes: 5 additions & 2 deletions src/robusta/runner/config_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
from robusta.model.playbook_definition import PlaybookDefinition
from robusta.utils.cluster_provider_discovery import cluster_provider
from robusta.utils.file_system_watcher import FileSystemWatcher
from robusta.core.exceptions import SupabaseDnsException


class ConfigLoader:
Expand Down Expand Up @@ -271,10 +272,12 @@ def __reload_playbook_packages(self, change_name):
str(runner_config.global_config.get("cluster_name", "no_cluster")).encode("utf-8")
).hexdigest()

except Exception:
except Exception as e:
print_exception = not isinstance(e, SupabaseDnsException) # Don't show trace for Robusta SaaS DNS-related exception to avoid hiding the error in noisy logs

logging.error(
"Error (re)loading playbooks/related resources, exiting.",
exc_info=True,
exc_info=print_exception,
)
# Kill the whole process group (which means this process and all of its descendant
# processes). The rest of the runner shutdown happens in robusta.runner.process_setup.
Expand Down
Loading