feat: added panic hook

Author: raphael-goetz

crates/taurus/src/app/mod.rs

@@ -19,13 +19,14 @@ use tucana::shared::module_status::StatusVariant;
 use crate::client::runtime_execution::TaurusRuntimeExecutionService;
 use crate::client::runtime_status::TaurusRuntimeStatusService;
 use crate::config::Config;
-use crate::telemetry::{self, TelemetrySettings};
+use crate::telemetry::{self, TelemetrySettings, errors};
 
 pub async fn run() {
     load_env_file();
 
     let config = Config::new();
     let telemetry = init_telemetry(&config);
+    install_panic_logging();
     let engine = ExecutionEngine::new();
     let client = connect_nats(&config).await;
 
@@ -54,6 +55,12 @@ pub async fn run() {
             && !err.is_cancelled()
         {
             log::warn!("Runtime status heartbeat task ended unexpectedly: {}", err);
+            errors::record(
+                "task",
+                "runtime_status_heartbeat.task",
+                &err,
+                "mode=dynamic",
+            );
         }
     }
     update_stopped_status(runtime_status_service.as_ref()).await;
@@ -76,6 +83,31 @@ fn init_telemetry(config: &Config) -> telemetry::Telemetry {
     .unwrap_or_else(|error| panic!("failed to initialize telemetry: {error}"))
 }
 
+fn install_panic_logging() {
+    std::panic::set_hook(Box::new(move |panic_info| {
+        let message = if let Some(message) = panic_info.payload().downcast_ref::<&str>() {
+            *message
+        } else if let Some(message) = panic_info.payload().downcast_ref::<String>() {
+            message.as_str()
+        } else {
+            "<non-string panic payload>"
+        };
+
+        let location = panic_info
+            .location()
+            .map(|location| {
+                format!(
+                    "{}:{}:{}",
+                    location.file(),
+                    location.line(),
+                    location.column()
+                )
+            })
+            .unwrap_or_else(|| "unknown".into());
+        errors::panic(message, &location);
+    }));
+}
+
 fn environment_label(environment: &Environment) -> &'static str {
     match environment {
         Environment::Development => "development",
@@ -98,6 +130,7 @@ async fn connect_nats(config: &Config) -> async_nats::Client {
             client
         }
         Err(err) => {
+            errors::record("transport", "nats.connect", &err, "component=nats");
             panic!("Failed to connect to NATS server: {}", err);
         }
     }
@@ -113,6 +146,12 @@ fn spawn_health_task(config: &Config) -> Option<JoinHandle<()>> {
         Ok(address) => address,
         Err(err) => {
             log::error!("Failed to parse gRPC address: {:?}", err);
+            errors::record(
+                "configuration",
+                "health.address.parse",
+                &err,
+                "service=health",
+            );
             return None;
         }
     };
@@ -125,6 +164,7 @@ fn spawn_health_task(config: &Config) -> Option<JoinHandle<()>> {
             .await
         {
             log::error!("Health server error: {:?}", err);
+            errors::record("server", "health.serve", &err, "service=health");
         } else {
             log::info!("Health server stopped gracefully");
         }
@@ -223,6 +263,12 @@ fn read_module_status_identifiers(definition_path: &str) -> Vec<String> {
                 "Failed to read module definitions for runtime status: {:?}",
                 err
             );
+            errors::record_message(
+                "configuration",
+                "definitions.read_modules",
+                format!("{err:?}"),
+                format!("path={definition_path}"),
+            );
             Vec::new()
         }
     }

crates/taurus/src/app/worker.rs

@@ -12,7 +12,7 @@ use tucana::shared::execution_result;
 use tucana::shared::{ExecutionFlow, ExecutionResult, NodeExecutionResult, Value};
 
 use crate::client::runtime_execution::TaurusRuntimeExecutionService;
-use crate::telemetry::metrics;
+use crate::telemetry::{errors, metrics};
 
 pub fn spawn_worker(
     client: async_nats::Client,
@@ -33,6 +33,12 @@ pub fn spawn_worker(
             }
             Err(err) => {
                 log::error!("Failed to subscribe to 'execution.*': {:?}", err);
+                errors::record(
+                    "transport",
+                    "nats.subscribe",
+                    &err,
+                    "subject=execution.* queue=taurus",
+                );
                 return;
             }
         };
@@ -93,6 +99,16 @@ async fn process_execution_message(
                 err,
                 &message.payload
             );
+            errors::record(
+                "serialization",
+                "execution_flow.decode",
+                &err,
+                format!(
+                    "subject={} payload_bytes={}",
+                    message.subject,
+                    message.payload.len()
+                ),
+            );
             if let Some(execution_service) = runtime_execution_service.as_mut() {
                 execution_service
                     .update_runtime_execution(build_decode_error_result(requested_execution_id))

crates/taurus/src/client/runtime_execution.rs

@@ -11,6 +11,8 @@ use tucana::{
     },
 };
 
+use crate::telemetry::errors;
+
 pub struct TaurusRuntimeExecutionService {
     client: ExecutionServiceClient<Channel>,
     aquila_token: String,
@@ -60,6 +62,12 @@ impl TaurusRuntimeExecutionService {
             }
             Err(err) => {
                 log::error!("Failed to update RuntimeExecution: {:?}", err);
+                errors::record(
+                    "transport",
+                    "aquila.execution.update",
+                    &err,
+                    "service=runtime_execution",
+                );
             }
         }
     }

crates/taurus/src/client/runtime_status.rs

@@ -11,6 +11,8 @@ use tucana::{
     shared::{ModuleStatus, module_status::StatusVariant},
 };
 
+use crate::telemetry::errors;
+
 pub struct TaurusRuntimeStatusService {
     channel: Channel,
     identifiers: Vec<String>,
@@ -59,6 +61,12 @@ impl TaurusRuntimeStatusService {
                 }
                 Err(err) => {
                     log::error!("Failed to update RuntimeStatus: {:?}", err);
+                    errors::record(
+                        "transport",
+                        "aquila.runtime_status.update",
+                        &err,
+                        "service=runtime_status",
+                    );
                 }
             }
         }
@@ -71,6 +79,7 @@ fn now_unix_seconds() -> i64 {
         Ok(time) => time.as_secs() as i64,
         Err(err) => {
             log::error!("cannot get current system time: {:?}", err);
+            errors::record("system", "time.now", &err, "clock=system");
             0
         }
     }

crates/taurus/src/telemetry/mod.rs

@@ -1,3 +1,3 @@
 pub mod metrics;
 
-pub use code0_flow::flow_telemetry::{OpenTelemetry, Telemetry, TelemetrySettings};
+pub use code0_flow::flow_telemetry::{OpenTelemetry, Telemetry, TelemetrySettings, errors};