diff --git a/CHANGELOG.md b/CHANGELOG.md index 58c9fbd29..7f5540efa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ incremented upon a breaking change and the patch version will be incremented for **Changed** - Allow initialization for Vanilla Solana projects with IDLs ([435](https://github.com/Ackee-Blockchain/trident/pull/435)) +- improve invariant handling and exit-code behavior in fuzzing ([457](https://github.com/Ackee-Blockchain/trident/pull/457)) ## [0.12.0] - 2025-11-27 diff --git a/crates/cli/src/command/fuzz.rs b/crates/cli/src/command/fuzz.rs index fd16e0eb5..3ca95a95f 100644 --- a/crates/cli/src/command/fuzz.rs +++ b/crates/cli/src/command/fuzz.rs @@ -63,7 +63,7 @@ pub(crate) enum FuzzCommand { long = "exit-code", required = false, value_name = "MODE", - help = "Exit with non-zero code on failures. Modes: 'all' (any failure), 'invariants' (only fuzz test assertions), 'panics' (only program panics)." + help = "Exit with non-zero code on failures. Modes: 'all' (any failure), 'invariants' (only custom invariant/assert failures)." )] exit_code: Option, #[arg( diff --git a/crates/client/src/commander/mod.rs b/crates/client/src/commander/mod.rs index cb9415143..6d32f91c8 100644 --- a/crates/client/src/commander/mod.rs +++ b/crates/client/src/commander/mod.rs @@ -27,8 +27,8 @@ pub enum Error { BuildProgramsFailed, #[error("fuzzing failed")] FuzzingFailed, - #[error("Fuzzing found failing invariants or unhandled panics")] - FuzzingFailedInvariantOrPanic, + #[error("Fuzzing failed due to exit-code policy (invariants/all)")] + FuzzingFailedPolicy, #[error("Coverage error: {0}")] Coverage(#[from] crate::coverage::CoverageError), #[error("Cannot find the trident-tests directory in the current workspace")] @@ -120,26 +120,31 @@ impl Commander { } /// Manages a child process in an async context, specifically for monitoring fuzzing tasks. - /// Waits for the process to exit or a Ctrl+C signal. Prints an error message if the process - /// exits with an error, and sleeps briefly on Ctrl+C. Throws `Error::FuzzingFailed` on errors. + /// Waits for the process to exit or a Ctrl+C signal. + /// + /// Exit-code semantics: + /// - `0`: success + /// - `99`: policy failure from fuzz runner (only treated as error when policy is enabled) + /// - other non-zero: runtime failure (always treated as error) /// /// # Arguments /// * `child` - A mutable reference to a `Child` process. + /// * `policy_enabled` - True when `--exit-code` policy is active. /// /// # Errors - /// * Throws `Error::FuzzingFailed` if waiting on the child process fails. + /// * Throws `Error::FuzzingFailed` or `Error::FuzzingFailedPolicy` on failure. #[throws] - async fn handle_child(child: &mut Child, with_exit_code: bool) { + async fn handle_child(child: &mut Child, policy_enabled: bool) { tokio::select! { res = child.wait() => match res { Ok(status) => match status.code() { Some(code) => { - match (code, with_exit_code) { - (0, _) => {} // fuzzing did not find any failing invariants or panics and we dont care about exit code - (99, true) => throw!(Error::FuzzingFailedInvariantOrPanic), // fuzzing found failing invariants or panics and we care about exit code - (99, false) => {} // fuzzing found failing invariants or panics and we dont care about exit code - (_, _) => throw!(Error::FuzzingFailed), // fuzzing failed for some other reason so we care about exit code + match (code, policy_enabled) { + (0, _) => {} + (99, true) => throw!(Error::FuzzingFailedPolicy), + (99, false) => {} + (_, _) => throw!(Error::FuzzingFailed), } } None => throw!(Error::FuzzingFailed), diff --git a/crates/client/src/exit_code.rs b/crates/client/src/exit_code.rs index 586b61417..6f854a35b 100644 --- a/crates/client/src/exit_code.rs +++ b/crates/client/src/exit_code.rs @@ -3,18 +3,15 @@ use std::str::FromStr; /// Specifies which type of failures should cause a non-zero exit code. /// -/// - `All`: Exit non-zero on any failure (program panics or invariant failures) -/// - `Invariants`: Exit non-zero only on invariant/assert failures in fuzz tests -/// - `Panics`: Exit non-zero only on program panics (program failed to complete) +/// - `All`: Exit non-zero on any policy failure (program panics or custom invariant failures) +/// - `Invariants`: Exit non-zero only on custom invariant failures in fuzz tests #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] pub enum ExitCodeMode { - /// Exit non-zero on any failure (default behavior when exit code is enabled) + /// Exit non-zero on any policy failure (default behavior when exit code is enabled) #[default] All, - /// Exit non-zero only on invariant/assert failures in fuzz tests + /// Exit non-zero only on custom invariant failures in fuzz tests Invariants, - /// Exit non-zero only on program panics (program failed to complete) - Panics, } impl ExitCodeMode { @@ -23,7 +20,6 @@ impl ExitCodeMode { match self { ExitCodeMode::All => "all", ExitCodeMode::Invariants => "invariants", - ExitCodeMode::Panics => "panics", } } @@ -31,11 +27,6 @@ impl ExitCodeMode { pub fn triggers_on_invariants(&self) -> bool { matches!(self, ExitCodeMode::All | ExitCodeMode::Invariants) } - - /// Check if this mode should trigger exit code for program panics - pub fn triggers_on_panics(&self) -> bool { - matches!(self, ExitCodeMode::All | ExitCodeMode::Panics) - } } impl fmt::Display for ExitCodeMode { @@ -51,9 +42,8 @@ impl FromStr for ExitCodeMode { match s.to_lowercase().as_str() { "all" => Ok(ExitCodeMode::All), "invariants" => Ok(ExitCodeMode::Invariants), - "panics" => Ok(ExitCodeMode::Panics), _ => Err(format!( - "Invalid exit code mode '{}'. Valid values are: all, invariants, panics", + "Invalid exit code mode '{}'. Valid values are: all, invariants", s )), } @@ -71,10 +61,6 @@ mod tests { ExitCodeMode::from_str("invariants").unwrap(), ExitCodeMode::Invariants ); - assert_eq!( - ExitCodeMode::from_str("panics").unwrap(), - ExitCodeMode::Panics - ); assert_eq!(ExitCodeMode::from_str("ALL").unwrap(), ExitCodeMode::All); assert!(ExitCodeMode::from_str("invalid").is_err()); } @@ -82,12 +68,6 @@ mod tests { #[test] fn test_triggers() { assert!(ExitCodeMode::All.triggers_on_invariants()); - assert!(ExitCodeMode::All.triggers_on_panics()); - assert!(ExitCodeMode::Invariants.triggers_on_invariants()); - assert!(!ExitCodeMode::Invariants.triggers_on_panics()); - - assert!(!ExitCodeMode::Panics.triggers_on_invariants()); - assert!(ExitCodeMode::Panics.triggers_on_panics()); } } diff --git a/crates/client/src/utils.rs b/crates/client/src/utils.rs index bbedd3451..fa07fc71a 100644 --- a/crates/client/src/utils.rs +++ b/crates/client/src/utils.rs @@ -538,6 +538,9 @@ mod tests { .await .unwrap(); + // Keep TempDir alive across post-await checks to avoid eager drop in async state machine. + assert!(temp_dir.path().exists()); + // Verify no backup was created let backup_path = settings_path.with_extension("json.backup"); assert!(!backup_path.exists()); @@ -574,6 +577,9 @@ mod tests { .await .unwrap(); + // Keep TempDir alive across post-await checks to avoid eager drop in async state machine. + assert!(temp_dir.path().exists()); + // Verify no backup was created let backup_path = settings_path.with_extension("json.backup"); assert!(!backup_path.exists()); diff --git a/crates/fuzz/src/invariant.rs b/crates/fuzz/src/invariant.rs new file mode 100644 index 000000000..9ce36ea01 --- /dev/null +++ b/crates/fuzz/src/invariant.rs @@ -0,0 +1,38 @@ +/// Marker type for intentional invariant violations. +/// Used to distinguish user-defined invariant failures from unexpected panics. +#[derive(Debug)] +pub struct InvariantViolation(pub String); + +/// Checks a condition and panics with `InvariantViolation` if false. +/// +/// Use this macro to mark assertions as intentional invariant checks. +/// When an invariant fails, it will be counted and collected separately +/// from unexpected panics (bugs in fuzz test code). +/// +/// # Examples +/// +/// ```ignore +/// // Simple condition check +/// invariant!(balance_after == balance_before - amount); +/// invariant!(account.is_initialized); +/// invariant!(owner != Pubkey::default()); +/// +/// // With custom message +/// invariant!(balance > 0, "Balance must be positive"); +/// invariant!(a == b, "Expected {} but got {}", a, b); +/// ``` +#[macro_export] +macro_rules! invariant { + ($cond:expr) => { + if !$cond { + std::panic::panic_any($crate::invariant::InvariantViolation( + format!("invariant violation: {}", stringify!($cond)) + )); + } + }; + ($cond:expr, $($msg:tt)*) => { + if !$cond { + std::panic::panic_any($crate::invariant::InvariantViolation(format!($($msg)*))); + } + }; +} diff --git a/crates/fuzz/src/lib.rs b/crates/fuzz/src/lib.rs index 7b1fe454a..ba388168f 100644 --- a/crates/fuzz/src/lib.rs +++ b/crates/fuzz/src/lib.rs @@ -1,5 +1,6 @@ pub mod address_storage; pub mod error; +pub mod invariant; pub mod trident; pub mod trident_rng; @@ -67,6 +68,10 @@ pub mod fuzzing { /// Error pub use super::error::*; + /// Invariant checking + pub use super::invariant; + pub use super::invariant::InvariantViolation; + /// Account discriminator trait pub use super::AccountDiscriminator; diff --git a/crates/fuzz/src/trident/flow_executor.rs b/crates/fuzz/src/trident/flow_executor.rs index c9eff9050..9b800d205 100644 --- a/crates/fuzz/src/trident/flow_executor.rs +++ b/crates/fuzz/src/trident/flow_executor.rs @@ -1,7 +1,9 @@ +use std::io::IsTerminal; use std::panic::catch_unwind; use std::panic::AssertUnwindSafe; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; +use std::sync::mpsc; use std::sync::Arc; use std::thread; use std::time::Instant; @@ -40,12 +42,10 @@ mod config { /// Specifies which type of failures should cause a non-zero exit code. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ExitCodeMode { - /// Exit non-zero on any failure (program panics or invariant failures) + /// Exit non-zero on any policy failure (program panics or custom invariant failures) All, - /// Exit non-zero only on invariant/assert failures in fuzz tests + /// Exit non-zero only on custom invariant failures in fuzz tests Invariants, - /// Exit non-zero only on program panics (program failed to complete) - Panics, } impl ExitCodeMode { @@ -55,23 +55,145 @@ impl ExitCodeMode { match s.to_lowercase().as_str() { "all" => ExitCodeMode::All, "invariants" => ExitCodeMode::Invariants, - "panics" => ExitCodeMode::Panics, - _ => ExitCodeMode::All, // Default to All for backwards compatibility + _ => ExitCodeMode::All, // Invalid values fall back to All } }) } +} + +/// Events sent from worker threads to the UI/controller thread in parallel fuzzing. +enum WorkerEvent { + ProgressDelta(u64), + InvariantFailure(String), + ProgramPanicsDelta(u64), +} + +/// Final process exit outcomes for a fuzzing run. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum FuzzRunExit { + Success, + PolicyFailure, + RuntimeFailure, +} + +impl FuzzRunExit { + fn code(self) -> i32 { + match self { + FuzzRunExit::Success => 0, + FuzzRunExit::PolicyFailure => 99, + FuzzRunExit::RuntimeFailure => 1, + } + } +} + +/// Inputs required to decide the final process outcome for policy-controlled failures. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct ExitDecisionInput { + exit_code_mode: Option, + invariant_failed: bool, + program_panicked: bool, +} + +fn determine_exit_outcome(input: ExitDecisionInput) -> FuzzRunExit { + let Some(mode) = input.exit_code_mode else { + // No explicit exit-code policy: invariants/program panics do not fail the run. + // Unexpected fuzz-test panics (e.g. unwrap on None) are handled separately + // as runtime failures. + return FuzzRunExit::Success; + }; + + let should_fail = match mode { + ExitCodeMode::All => input.invariant_failed || input.program_panicked, + ExitCodeMode::Invariants => input.invariant_failed, + }; + + if should_fail { + FuzzRunExit::PolicyFailure + } else { + FuzzRunExit::Success + } +} + +fn colors_enabled() -> bool { + std::io::stderr().is_terminal() && std::env::var_os("NO_COLOR").is_none() +} + +fn paint_red(text: &str) -> String { + if colors_enabled() { + format!("\x1b[31m{}\x1b[0m", text) + } else { + text.to_string() + } +} + +fn paint_bold_yellow(text: &str) -> String { + if colors_enabled() { + format!("\x1b[1;33m{}\x1b[0m", text) + } else { + text.to_string() + } +} - /// Check if this mode should trigger exit code for invariant failures - pub fn triggers_on_invariants(&self) -> bool { - matches!(self, ExitCodeMode::All | ExitCodeMode::Invariants) +fn paint_cyan(text: &str) -> String { + if colors_enabled() { + format!("\x1b[36m{}\x1b[0m", text) + } else { + text.to_string() } +} - /// Check if this mode should trigger exit code for program panics - pub fn triggers_on_panics(&self) -> bool { - matches!(self, ExitCodeMode::All | ExitCodeMode::Panics) +fn paint_magenta(text: &str) -> String { + if colors_enabled() { + format!("\x1b[35m{}\x1b[0m", text) + } else { + text.to_string() } } +fn format_invariant_line(text: &str) -> String { + const PREFIX: &str = "Assertion failed at "; + const SEED_PREFIX: &str = " (seed: "; + + if !colors_enabled() { + return text.to_string(); + } + + // Expected format from handle_panic: + // "Assertion failed at : (seed: )" + if let Some(location_start) = text.strip_prefix(PREFIX) { + if let Some(seed_idx) = location_start.rfind(SEED_PREFIX) { + let before_seed = &location_start[..seed_idx]; + let seed_with_suffix = &location_start[seed_idx + SEED_PREFIX.len()..]; + if let Some(seed) = seed_with_suffix.strip_suffix(')') { + if let Some(separator_idx) = before_seed.find(": ") { + let location = &before_seed[..separator_idx]; + let message = &before_seed[separator_idx + 2..]; + return format!( + "{} {}{}: {}{}{}{}", + paint_red("!"), + PREFIX, + paint_cyan(location), + message, + SEED_PREFIX, + paint_magenta(seed), + ")" + ); + } + } + } + } + + // Fallback to accent-only if line doesn't match expected format. + format!("{} {}", paint_red("!"), text) +} + +/// Final aggregated runtime summary produced by the UI/controller thread. +struct ParallelRunSummary { + invariant_failures: u64, + program_panics: u64, + panic_messages: Vec, +} + /// Trait for executing fuzzing flows in the Trident framework /// /// This trait defines the interface for fuzzing executors that can run @@ -182,8 +304,11 @@ pub trait FlowExecutor: Send + 'static + Sized { } /// Extracts the panic message from a panic payload. - /// Panics can have either &str or String payloads, so we handle both cases. + /// Handles InvariantViolation, &str, and String payloads. fn extract_panic_message(panic_err: &Box) -> String { + if let Some(inv) = panic_err.downcast_ref::() { + return inv.0.clone(); + } panic_err .downcast_ref::<&str>() .map(|s| s.to_string()) @@ -215,38 +340,6 @@ pub trait FlowExecutor: Send + 'static + Sized { ) } - /// Determines the exit code based on panic status, program panics, and exit code mode. - /// - /// - `exit_code_mode`: The mode specifying which failures trigger non-zero exit - /// - `invariant_failed`: True if fuzz test assertions/invariants failed (caught panics) - /// - `fuzzing_data`: Contains program panic information (transaction_panicked > 0) - /// - /// Returns 99 if the specified failure type occurred, 0 otherwise. - fn determine_exit_code( - exit_code_mode: Option, - invariant_failed: bool, - fuzzing_data: &TridentFuzzingData, - ) -> i32 { - let Some(mode) = exit_code_mode else { - // No exit code mode specified - use metrics exit code (program panics only) - return fuzzing_data.get_exit_code(); - }; - - let program_panicked = fuzzing_data.get_exit_code() != 0; - - let should_fail = match mode { - ExitCodeMode::All => invariant_failed || program_panicked, - ExitCodeMode::Invariants => invariant_failed, - ExitCodeMode::Panics => program_panicked, - }; - - if should_fail { - 99 - } else { - 0 - } - } - /// Gets the master seed from environment variable or generates a random one. /// The master seed is used to initialize all fuzzer instances for reproducible runs. fn get_or_generate_master_seed() -> [u8; config::SEED_SIZE] { @@ -302,6 +395,8 @@ pub trait FlowExecutor: Send + 'static + Sized { let is_debug_mode = std::env::var(config::ENV_FUZZ_DEBUG).is_ok(); let exit_code_mode = ExitCodeMode::from_env(); let mut invariant_failed = false; // Tracks fuzz test assertion/invariant failures + let mut invariant_failure_count: u64 = 0; + let mut panic_messages: Vec = Vec::new(); // Configure debug seed if in debug mode if is_debug_mode { @@ -338,16 +433,26 @@ pub trait FlowExecutor: Send + 'static + Sized { let _ = fuzzer.execute_flows(flow_calls_per_iteration); })); - // Handle any panics that occurred (invariant/assertion failures in fuzz tests) + // Handle panics - only catch InvariantViolation, re-throw others if let Err(panic_err) = panic_result { - invariant_failed = true; - let panic_msg = Self::handle_panic(&panic_err, &mut fuzzer, None); - - // Display panic message via progress bar or stderr - if let Some(ref pb) = pb { - pb.println(panic_msg); + if panic_err + .downcast_ref::() + .is_some() + { + // Intentional invariant failure - count it, continue fuzzing + invariant_failed = true; + invariant_failure_count += 1; + let panic_msg = Self::handle_panic(&panic_err, &mut fuzzer, None); + + // In debug mode print immediately, otherwise collect for end + if is_debug_mode { + eprintln!("{}", format_invariant_line(&panic_msg)); + } else { + panic_messages.push(panic_msg); + } } else { - eprintln!("{}", panic_msg); + // Unexpected panic (bug in fuzz test) - re-throw it + std::panic::resume_unwind(panic_err); } } @@ -358,10 +463,20 @@ pub trait FlowExecutor: Send + 'static + Sized { // Handle coverage profiling if enabled Self::handle_coverage_if_enabled(&mut fuzzer, i + 1); - // Update progress bar + // Update progress bar with live stats if let Some(ref pb) = pb { pb.inc(flow_calls_per_iteration); - pb.set_message(format!("Iteration {}/{} completed", i + 1, iterations)); + let program_panics = fuzzer + .trident_mut() + .get_fuzzing_data() + .get_program_panic_count(); + pb.set_message(format!( + "Iteration {}/{} | Invariant failures: {} | Program panics: {}", + i + 1, + iterations, + invariant_failure_count, + program_panics + )); } } @@ -370,15 +485,33 @@ pub trait FlowExecutor: Send + 'static + Sized { pb.finish_with_message("Fuzzing completed!"); } + // Print collected invariant failure messages + if !panic_messages.is_empty() { + eprintln!( + "\n{}", + paint_bold_yellow(&format!( + "--- Invariant Failures ({}) ---", + panic_messages.len() + )) + ); + for msg in &panic_messages { + eprintln!("{}", format_invariant_line(msg)); + } + } + // Generate metrics if enabled let fuzzing_data = fuzzer.trident_mut().get_fuzzing_data(); Self::output_metrics_if_enabled(&fuzzing_data); - // Exit with appropriate code if exit code mode is enabled - if exit_code_mode.is_some() { - let exit_code = - Self::determine_exit_code(exit_code_mode, invariant_failed, &fuzzing_data); - std::process::exit(exit_code); + let outcome = determine_exit_outcome(ExitDecisionInput { + exit_code_mode, + invariant_failed, + program_panicked: fuzzing_data.get_program_panic_count() > 0, + }); + + // In single-thread mode we only force-exit when policy asks for a non-zero code. + if outcome == FuzzRunExit::PolicyFailure { + std::process::exit(outcome.code()); } } @@ -391,9 +524,10 @@ pub trait FlowExecutor: Send + 'static + Sized { master_seed: [u8; 32], ) { let iterations_per_thread = iterations / num_threads as u64; + let remainder_iterations = iterations % num_threads as u64; let total_flow_calls = iterations * flow_calls_per_iteration; let exit_code_mode = ExitCodeMode::from_env(); - let invariant_failed = Arc::new(AtomicBool::new(false)); // Tracks fuzz test assertion failures + let (event_tx, event_rx) = mpsc::channel::(); // Setup shared progress bar let main_pb = indicatif::ProgressBar::new(total_flow_calls); @@ -405,131 +539,167 @@ pub trait FlowExecutor: Send + 'static + Sized { .progress_chars("#>-"), ); main_pb.set_message(format!( - "Fuzzing with {} threads - {} iterations with {} flow calls each", - num_threads, iterations, flow_calls_per_iteration + "Fuzzing with {} threads | Invariant failures: 0 | Program panics: 0", + num_threads )); + // Single UI/controller owner: consumes worker events, updates progress bar, + // and builds global live counters + invariant messages. + let ui_handle = thread::spawn(move || -> ParallelRunSummary { + let mut invariant_failures = 0u64; + let mut program_panics = 0u64; + let mut panic_messages = Vec::new(); + + while let Ok(event) = event_rx.recv() { + match event { + WorkerEvent::ProgressDelta(delta) => { + main_pb.inc(delta); + } + WorkerEvent::InvariantFailure(message) => { + invariant_failures += 1; + panic_messages.push(message); + } + WorkerEvent::ProgramPanicsDelta(delta) => { + program_panics += delta; + } + } + + main_pb.set_message(format!( + "Invariant failures: {} | Program panics: {}", + invariant_failures, program_panics + )); + } + + main_pb.finish_with_message("Parallel fuzzing completed!"); + ParallelRunSummary { + invariant_failures, + program_panics, + panic_messages, + } + }); + // Spawn worker threads let mut handles = Vec::new(); for thread_id in 0..num_threads { - let thread_iterations = iterations_per_thread; + let thread_iterations = iterations_per_thread + + if (thread_id as u64) < remainder_iterations { + 1 + } else { + 0 + }; if thread_iterations == 0 { continue; // Skip threads with no work } - let main_pb_clone = main_pb.clone(); - let invariant_failed_clone = invariant_failed.clone(); - let handle = thread::spawn(move || -> TridentFuzzingData { - Self::run_thread_workload( - master_seed, - thread_id, - thread_iterations, - flow_calls_per_iteration, - main_pb_clone, - invariant_failed_clone, - ) + let event_tx_clone = event_tx.clone(); + let handle = thread::spawn(move || -> Result { + let panic_result = catch_unwind(AssertUnwindSafe(|| { + run_thread_workload_impl::( + master_seed, + thread_id, + thread_iterations, + flow_calls_per_iteration, + event_tx_clone, + ) + })); + + match panic_result { + Ok(thread_metrics) => Ok(thread_metrics), + Err(panic_err) => { + let location = PANIC_LOCATION + .with(|cell| cell.take().unwrap_or_else(|| "unknown".to_string())); + let message = Self::extract_panic_message(&panic_err); + Err(format!("{} at {}", message, location)) + } + } }); handles.push(handle); } + // Drop original sender so channel closes when all workers are done. + drop(event_tx); // Collect results from all threads let mut fuzzing_data = TridentFuzzingData::with_master_seed(master_seed); + let mut worker_thread_failed = false; + let mut worker_thread_panic_messages: Vec = Vec::new(); for handle in handles { match handle.join() { - Ok(thread_metrics) => { + Ok(Ok(thread_metrics)) => { fuzzing_data._merge(thread_metrics); } + Ok(Err(worker_panic_msg)) => { + worker_thread_panic_messages.push(worker_panic_msg); + worker_thread_failed = true; + } Err(err) => { - // This should rarely happen since we catch panics inside threads - // Only occurs if the thread itself crashes (not user code) - eprintln!("Warning: Thread failed to join (not a fuzz test panic)"); - if let Some(s) = err.downcast_ref::<&str>() { - eprintln!(" Message: {}", s); + // Worker thread crashed unexpectedly (not a handled invariant failure). + // Buffer messages and print them only after progress bar finalization. + let message = if let Some(s) = err.downcast_ref::<&str>() { + s.to_string() } else if let Some(s) = err.downcast_ref::() { - eprintln!(" Message: {}", s); - } - // Continue processing other threads + s.clone() + } else { + "unknown panic payload".to_string() + }; + worker_thread_panic_messages.push(message); + worker_thread_failed = true; } } } - main_pb.finish_with_message("Parallel fuzzing completed!"); - - // Determine and set exit code - let exit_code = Self::determine_exit_code( - exit_code_mode, - invariant_failed.load(Ordering::Relaxed), - &fuzzing_data, - ); - - Self::output_metrics_if_enabled(&fuzzing_data); - println!("MASTER SEED used: {:?}", &hex::encode(master_seed)); - - std::process::exit(exit_code); - } - - /// Runs the fuzzing workload for a single thread. - /// This is extracted to reduce complexity in fuzz_parallel. - fn run_thread_workload( - master_seed: [u8; 32], - thread_id: usize, - thread_iterations: u64, - flow_calls_per_iteration: u64, - progress_bar: indicatif::ProgressBar, - invariant_failed: Arc, - ) -> TridentFuzzingData { - let mut fuzzer = Self::new(); - fuzzer - .trident_mut() - .set_master_seed_and_thread_id(master_seed, thread_id); - - // Track progress updates to avoid excessive bar updates - let mut last_update = Instant::now(); - let mut local_counter = 0u64; - - // Execute iterations for this thread - for i in 0..thread_iterations { - // Catch panics from user code (assertions, invariants, etc.) - let panic_result = catch_unwind(AssertUnwindSafe(|| { - let _ = fuzzer.execute_flows(flow_calls_per_iteration); - })); - - // Handle any panics that occurred (invariant/assertion failures in fuzz tests) - if let Err(panic_err) = panic_result { - let panic_msg = - Self::handle_panic(&panic_err, &mut fuzzer, Some(&invariant_failed)); - progress_bar.println(panic_msg); - } - - // Prepare for next iteration - fuzzer.trident_mut().next_iteration(); - fuzzer.reset_fuzz_accounts(); + // Get final aggregated runtime summary from the controller thread. + let run_summary = ui_handle.join().unwrap_or_else(|_| ParallelRunSummary { + invariant_failures: 0, + program_panics: 0, + panic_messages: Vec::new(), + }); - // Handle coverage profiling (only thread 0 to avoid duplicate work) - if thread_id == 0 { - Self::handle_coverage_if_enabled(&mut fuzzer, i + 1); + // Print collected invariant failure messages + if !run_summary.panic_messages.is_empty() { + eprintln!( + "\n{}", + paint_bold_yellow(&format!( + "--- Invariant Failures ({}) ---", + run_summary.panic_messages.len() + )) + ); + for msg in &run_summary.panic_messages { + eprintln!("{}", format_invariant_line(msg)); } + } - // Batch progress updates for performance - local_counter += flow_calls_per_iteration; - let should_update = local_counter >= config::PROGRESS_UPDATE_INTERVAL - || last_update.elapsed() >= config::PROGRESS_UPDATE_DURATION - || i == thread_iterations - 1; // Always update on last iteration - - if should_update { - progress_bar.inc(local_counter); - local_counter = 0; - last_update = Instant::now(); + if !worker_thread_panic_messages.is_empty() { + eprintln!( + "\n--- Worker Thread Crashes ({}) ---", + worker_thread_panic_messages.len() + ); + for message in &worker_thread_panic_messages { + eprintln!("Warning: Thread failed to join (not a fuzz test panic)"); + eprintln!(" Message: {}", message); } } - // Ensure any remaining progress is reported - if local_counter > 0 { - progress_bar.inc(local_counter); + if worker_thread_failed { + eprintln!("Fuzzing aborted: one or more worker threads crashed unexpectedly."); + std::process::exit(FuzzRunExit::RuntimeFailure.code()); } - fuzzer.trident_mut().get_fuzzing_data() + Self::output_metrics_if_enabled(&fuzzing_data); + // Sanity output for debugging possible metric/reporting drift. + debug_assert_eq!( + run_summary.program_panics, + fuzzing_data.get_program_panic_count(), + "Runtime panic counter diverged from merged metrics" + ); + println!("MASTER SEED used: {:?}", &hex::encode(master_seed)); + + let outcome = determine_exit_outcome(ExitDecisionInput { + exit_code_mode, + invariant_failed: run_summary.invariant_failures > 0, + program_panicked: run_summary.program_panics > 0, + }); + std::process::exit(outcome.code()); } /// Handles LLVM coverage collection if coverage profiling is enabled. @@ -570,3 +740,163 @@ pub trait FlowExecutor: Send + 'static + Sized { }); } } + +fn send_worker_event(event_tx: &mpsc::Sender, event: WorkerEvent) -> bool { + event_tx.send(event).is_ok() +} + +#[allow(clippy::too_many_arguments)] +/// Runs the fuzzing workload for a single thread. +/// This is extracted to reduce complexity in fuzz_parallel. +fn run_thread_workload_impl( + master_seed: [u8; 32], + thread_id: usize, + thread_iterations: u64, + flow_calls_per_iteration: u64, + event_tx: mpsc::Sender, +) -> TridentFuzzingData { + let mut fuzzer = E::new(); + fuzzer + .trident_mut() + .set_master_seed_and_thread_id(master_seed, thread_id); + + // Track progress updates to avoid excessive bar updates + let mut last_update = Instant::now(); + let mut local_counter = 0u64; + let mut local_observed_program_panics = 0u64; + + // Execute iterations for this thread + for i in 0..thread_iterations { + // Catch panics from user code (assertions, invariants, etc.) + let panic_result = catch_unwind(AssertUnwindSafe(|| { + let _ = fuzzer.execute_flows(flow_calls_per_iteration); + })); + + // Handle panics - only catch InvariantViolation, re-throw others + if let Err(panic_err) = panic_result { + if panic_err + .downcast_ref::() + .is_some() + { + // Intentional invariant failure - count it, continue fuzzing + let panic_msg = E::handle_panic(&panic_err, &mut fuzzer, None); + if !send_worker_event(&event_tx, WorkerEvent::InvariantFailure(panic_msg)) { + return fuzzer.trident_mut().get_fuzzing_data(); + } + } else { + // Unexpected panic (bug in fuzz test) - re-throw it + std::panic::resume_unwind(panic_err); + } + } + + // Prepare for next iteration + fuzzer.trident_mut().next_iteration(); + fuzzer.reset_fuzz_accounts(); + + // Handle coverage profiling (only thread 0 to avoid duplicate work) + if thread_id == 0 { + E::handle_coverage_if_enabled(&mut fuzzer, i + 1); + } + + // Batch progress updates for performance + local_counter += flow_calls_per_iteration; + let should_update = local_counter >= config::PROGRESS_UPDATE_INTERVAL + || last_update.elapsed() >= config::PROGRESS_UPDATE_DURATION + || i == thread_iterations - 1; // Always update on last iteration + + if should_update { + if !send_worker_event(&event_tx, WorkerEvent::ProgressDelta(local_counter)) { + return fuzzer.trident_mut().get_fuzzing_data(); + } + let thread_prog_panics = fuzzer + .trident_mut() + .get_fuzzing_data() + .get_program_panic_count(); + let new_panics = thread_prog_panics.saturating_sub(local_observed_program_panics); + if new_panics > 0 { + if !send_worker_event(&event_tx, WorkerEvent::ProgramPanicsDelta(new_panics)) { + return fuzzer.trident_mut().get_fuzzing_data(); + } + local_observed_program_panics = thread_prog_panics; + } + local_counter = 0; + last_update = Instant::now(); + } + } + + // Ensure any remaining progress is reported + if local_counter > 0 && !send_worker_event(&event_tx, WorkerEvent::ProgressDelta(local_counter)) + { + return fuzzer.trident_mut().get_fuzzing_data(); + } + + // Flush any panics that happened since the last batched UI update. + let final_thread_prog_panics = fuzzer + .trident_mut() + .get_fuzzing_data() + .get_program_panic_count(); + let final_new_panics = final_thread_prog_panics.saturating_sub(local_observed_program_panics); + if final_new_panics > 0 + && !send_worker_event(&event_tx, WorkerEvent::ProgramPanicsDelta(final_new_panics)) + { + return fuzzer.trident_mut().get_fuzzing_data(); + } + + fuzzer.trident_mut().get_fuzzing_data() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn exit_outcome_no_mode_ignores_policy_failures() { + let outcome = determine_exit_outcome(ExitDecisionInput { + exit_code_mode: None, + invariant_failed: true, + program_panicked: true, + }); + assert_eq!(outcome, FuzzRunExit::Success); + } + + #[test] + fn exit_outcome_invariants_mode_only_fails_on_invariants() { + let invariant_fail = determine_exit_outcome(ExitDecisionInput { + exit_code_mode: Some(ExitCodeMode::Invariants), + invariant_failed: true, + program_panicked: false, + }); + assert_eq!(invariant_fail, FuzzRunExit::PolicyFailure); + + let program_panic_only = determine_exit_outcome(ExitDecisionInput { + exit_code_mode: Some(ExitCodeMode::Invariants), + invariant_failed: false, + program_panicked: true, + }); + assert_eq!(program_panic_only, FuzzRunExit::Success); + } + + #[test] + fn exit_outcome_all_mode_fails_on_either_source() { + let invariant_fail = determine_exit_outcome(ExitDecisionInput { + exit_code_mode: Some(ExitCodeMode::All), + invariant_failed: true, + program_panicked: false, + }); + assert_eq!(invariant_fail, FuzzRunExit::PolicyFailure); + + let program_panic = determine_exit_outcome(ExitDecisionInput { + exit_code_mode: Some(ExitCodeMode::All), + invariant_failed: false, + program_panicked: true, + }); + assert_eq!(program_panic, FuzzRunExit::PolicyFailure); + } + + #[test] + fn exit_codes_are_stable() { + assert_eq!(FuzzRunExit::Success.code(), 0); + assert_eq!(FuzzRunExit::PolicyFailure.code(), 99); + assert_eq!(FuzzRunExit::RuntimeFailure.code(), 1); + } +} diff --git a/crates/metrics/src/lib.rs b/crates/metrics/src/lib.rs index dc7a2caf1..750cc15cf 100644 --- a/crates/metrics/src/lib.rs +++ b/crates/metrics/src/lib.rs @@ -254,4 +254,8 @@ impl TridentFuzzingData { pub fn get_exit_code(&self) -> i32 { self.metrics.get_exit_code() } + + pub fn get_program_panic_count(&self) -> u64 { + self.metrics.get_program_panic_count() + } } diff --git a/crates/metrics/src/transactions/transaction_stats.rs b/crates/metrics/src/transactions/transaction_stats.rs index 9fc7faff1..42077c60c 100644 --- a/crates/metrics/src/transactions/transaction_stats.rs +++ b/crates/metrics/src/transactions/transaction_stats.rs @@ -253,4 +253,11 @@ impl FuzzingStatistics { } 0 } + + pub(crate) fn get_program_panic_count(&self) -> u64 { + self.transactions + .values() + .map(|stats| stats.transaction_panicked) + .sum() + } } diff --git a/documentation/docs/basics/commands.md b/documentation/docs/basics/commands.md index f545452d0..b03a42d16 100644 --- a/documentation/docs/basics/commands.md +++ b/documentation/docs/basics/commands.md @@ -118,7 +118,9 @@ Runs the specified Fuzz Target using Trident's Manually Guided Fuzzing (e.g., fu - `all` - Exit non-zero on any failure (program panics or invariant failures) - `invariants` - Exit non-zero only on fuzz test assertion/invariant failures - - `panics` - Exit non-zero only on program panics (program failed to complete) + +Without `--exit-code`, Trident does not fail the process on invariant failures or program panics. +Unexpected fuzz-test panics (for example `unwrap()` on `None`) are always treated as runtime errors and fail the run. **Examples:** @@ -131,9 +133,6 @@ trident fuzz run fuzz_0 --exit-code all # Exit non-zero only on invariant failures (useful in CI to catch assertion bugs) trident fuzz run fuzz_0 -e invariants - -# Exit non-zero only on program panics (useful to detect unhandled errors) -trident fuzz run fuzz_0 -e panics ``` --- diff --git a/documentation/docs/trident-advanced/invariants-assertions/index.md b/documentation/docs/trident-advanced/invariants-assertions/index.md index 1fd72cfdd..5187fbfa5 100644 --- a/documentation/docs/trident-advanced/invariants-assertions/index.md +++ b/documentation/docs/trident-advanced/invariants-assertions/index.md @@ -1,6 +1,26 @@ # Invariants and Assertions -Invariants are conditions that must always hold true for your program to be considered correct. In Trident, you validate program behavior by capturing account states before and after transactions, then checking the expected changes with custom invariant methods. +Invariants are conditions that must always hold true for your program to be considered correct. In Trident, you validate program behavior by capturing account states before and after transactions, then checking the expected changes using the `invariant!` macro. + +## The `invariant!` Macro + +Use the `invariant!` macro to define intentional invariant checks. When an invariant fails: + +- It is counted and collected separately from unexpected panics +- Fuzzing continues to find more issues +- All failures are reported at the end with their seeds for reproduction + +```rust +// Simple condition +invariant!(balance_after == balance_before - amount); + +// With custom message +invariant!(balance > 0, "Balance must be positive"); +invariant!(a == b, "Expected {} but got {}", a, b); +``` + +!!! note "Invariants vs Regular Panics" + Regular panics (like `unwrap()` on `None` or index out of bounds) are treated as bugs in your fuzz test and will crash immediately. Only `invariant!` failures are collected and allow fuzzing to continue. ## How Invariants Work @@ -9,7 +29,7 @@ The validation pattern in Trident follows these steps: 1. **Capture state before transaction** 2. **Execute the transaction** 3. **Capture state after transaction** -4. **Validate changes with invariant methods** +4. **Validate changes with `invariant!`** ## Basic Example @@ -36,12 +56,6 @@ impl FuzzTest { .expect("Account not found"); self.transfer_invariant(balance_before, balance_after, 100); - } else { - // Handle expected failures - assert!( - result.is_custom_error_with_code(6001_u32), - "Expected insufficient funds error" - ); } } @@ -51,10 +65,11 @@ impl FuzzTest { after: UserAccount, amount: u64, ) { - assert_eq!( - after.balance, + invariant!( + after.balance == before.balance - amount, + "Balance should decrease by transfer amount: expected {}, got {}", before.balance - amount, - "Balance should decrease by transfer amount" + after.balance ); } } @@ -66,15 +81,16 @@ impl FuzzTest { - **Validate State Changes**: Ensure account modifications are correct - **Test Edge Cases**: Verify behavior under various conditions - **Prevent Regressions**: Catch bugs introduced by code changes +- **Continue Fuzzing**: Find multiple issues in a single run ## Writing Invariant Methods Invariant methods should: - Take before/after states as parameters -- Use descriptive assertion messages +- Use descriptive messages with `invariant!` - Focus on one specific behavior -- Handle both success and failure cases +- Include relevant values in error messages ```rust fn token_mint_invariant( @@ -83,12 +99,31 @@ fn token_mint_invariant( mint_after: MintAccount, minted_amount: u64, ) { - assert_eq!( - mint_after.supply, - mint_before.supply + minted_amount, - "Token supply should increase by minted amount" + invariant!( + mint_after.supply == mint_before.supply + minted_amount, + "Token supply should increase by minted amount: {} + {} != {}", + mint_before.supply, + minted_amount, + mint_after.supply ); } ``` +## Exit Code Modes + +When running fuzz tests in CI/CD, use `--exit-code` to control which failures cause non-zero exit: + +```bash +# Exit non-zero on any failure +trident fuzz run fuzz_0 --exit-code all + +# Exit non-zero only on invariant failures +trident fuzz run fuzz_0 --exit-code invariants +``` + +Notes: + +- Without `--exit-code`, invariant failures and program panics are reported but do not force a non-zero process exit. +- Unexpected fuzz-test panics (for example `unwrap()` on `None`) are always treated as runtime errors and fail the run. + For more complex examples and patterns, see the [Trident Examples](../../trident-examples/trident-examples.md) page. diff --git a/examples/hello_world/trident-tests/fuzz_0/test_fuzz.rs b/examples/hello_world/trident-tests/fuzz_0/test_fuzz.rs index c083061fc..8a051e92e 100644 --- a/examples/hello_world/trident-tests/fuzz_0/test_fuzz.rs +++ b/examples/hello_world/trident-tests/fuzz_0/test_fuzz.rs @@ -85,8 +85,18 @@ impl FuzzTest { .trident .get_account_with_type::(&hello_world, None); if let Some(hello_world_account) = hello_world_account { - assert!(hello_world_account.input == input); - assert!(hello_world_account.timestamp == res.get_transaction_timestamp()); + invariant!( + hello_world_account.input == input, + "Input mismatch: expected {}, got {}", + input, + hello_world_account.input + ); + invariant!( + hello_world_account.timestamp == res.get_transaction_timestamp(), + "Timestamp mismatch: expected {}, got {}", + res.get_transaction_timestamp(), + hello_world_account.timestamp + ); } let returned_value = res.get_return_data().unwrap(); diff --git a/examples/trident-benchmark/maze_1/trident-tests/fuzz_0/test_fuzz.rs b/examples/trident-benchmark/maze_1/trident-tests/fuzz_0/test_fuzz.rs index 1e31023c3..afaf570a5 100644 --- a/examples/trident-benchmark/maze_1/trident-tests/fuzz_0/test_fuzz.rs +++ b/examples/trident-benchmark/maze_1/trident-tests/fuzz_0/test_fuzz.rs @@ -125,8 +125,18 @@ impl FuzzTest { .get_account_with_type::(&state, None) .expect("State not found"); - assert!(state_after.x == state_before.x); - assert!(state_after.y == state_before.y + 1); + invariant!( + state_after.x == state_before.x, + "X should not change: expected {}, got {}", + state_before.x, + state_after.x + ); + invariant!( + state_after.y == state_before.y + 1, + "Y should increase by 1: expected {}, got {}", + state_before.y + 1, + state_after.y + ); } }