[smoke-test] Decrease retries from 16 to 4

.config/nextest.toml

@@ -28,6 +28,9 @@ status-level = "skip"
 failure-output = "immediate-final"
 fail-fast = true
 # different than ci
-test-threads = 4
-retries = 3
+test-threads = 6
+# Reduced from 3 to 1: combined with SWARM_BUILD_NUM_RETRIES=1, gives 2×2=4 total
+# attempts per test (was 4×4=16). Fewer retries reduce resource contention when
+# multiple tests compete for ports on the same CI instance.
+retries = 1
 slow-timeout = { period = "1800s", terminate-after = 1 }

scripts/dev_setup.sh

@@ -598,9 +598,17 @@ function install_z3 {
     echo "but this install will go to ${INSTALL_DIR}/z3."
     echo "you may want to remove the shared instance to avoid version confusion"
   fi
-  if command -v "${INSTALL_DIR}z3" &>/dev/null && [[ "$("${INSTALL_DIR}z3" --version || true)" =~ .*${Z3_VERSION}.* ]]; then
-    echo "Z3 ${Z3_VERSION} already installed"
-    return
+  # Verify the existing z3 can actually run and reports the right version.
+  # A broken binary (wrong glibc, corrupt download) will fail this check and be reinstalled.
+  if command -v "${INSTALL_DIR}z3" &>/dev/null; then
+    Z3_OUTPUT="$("${INSTALL_DIR}z3" --version 2>/dev/null || true)"
+    if [[ "$Z3_OUTPUT" =~ .*${Z3_VERSION}.* ]]; then
+      echo "Z3 ${Z3_VERSION} already installed and working"
+      return
+    else
+      echo "Z3 binary exists but version check failed (output: '${Z3_OUTPUT}'). Reinstalling."
+      rm -f "${INSTALL_DIR}z3"
+    fi
   fi
   if [[ "$(uname)" == "Linux" ]]; then
     Z3_PKG="z3-$Z3_VERSION-x64-glibc-2.31"
@@ -625,6 +633,11 @@ function install_z3 {
     chmod +x "${INSTALL_DIR}z3"
   )
   rm -rf "$TMPFILE"
+  # Verify the installed binary actually works
+  if ! "${INSTALL_DIR}z3" --version 2>/dev/null | grep -q "${Z3_VERSION}"; then
+    echo "WARNING: z3 installation may be broken. '${INSTALL_DIR}z3 --version' failed."
+    echo "Move Prover tests will likely fail. Check glibc compatibility."
+  fi
 }
 
 function install_cvc5 {

testsuite/smoke-test/src/consensus_observer.rs

@@ -23,13 +23,6 @@ async fn test_consensus_observer_fast_sync_epoch_changes() {
     state_sync_utils::test_fullnode_fast_sync(true, true).await;
 }
 
-#[tokio::test]
-#[ignore] // Ignore this test because it is a subset of test_consensus_observer_fast_sync_epoch_changes
-async fn test_consensus_observer_fast_sync_no_epoch_changes() {
-    // Test fast syncing with consensus observer and without epoch changes
-    state_sync_utils::test_fullnode_fast_sync(false, true).await;
-}
-
 #[tokio::test]
 async fn test_consensus_observer_fullnode_restart() {
     // Create a swarm of 1 validator with consensus observer enabled

testsuite/smoke-test/src/smoke_test_environment.rs

@@ -16,7 +16,10 @@ use rand::rngs::OsRng;
 use std::{env, num::NonZeroUsize, path::PathBuf, sync::Arc};
 use tokio::task::JoinHandle;
 
-const SWARM_BUILD_NUM_RETRIES: u8 = 3;
+// Reduced from 3 to 1: with nextest retries = 1, total attempts per test = 2 × 2 = 4.
+// Previously 3 internal × 3 nextest = 16 total attempts, which amplified resource contention
+// when multiple tests ran in parallel on constrained CI instances.
+const SWARM_BUILD_NUM_RETRIES: u8 = 1;
 
 #[derive(Clone)]
 pub struct SwarmBuilder {

testsuite/smoke-test/src/state_sync.rs

@@ -170,42 +170,6 @@ async fn test_fullnode_execution_sync_epoch_changes() {
     state_sync_utils::test_fullnode_sync(vfn_peer_id, &mut swarm, true, false).await;
 }
 
-#[tokio::test]
-#[ignore] // Ignore this test because it is a subset of test_fullnode_output_sync_epoch_changes
-async fn test_fullnode_output_sync_no_epoch_changes() {
-    // Create a validator swarm of 1 validator node
-    let mut swarm = new_local_swarm_with_aptos(1).await;
-
-    // Create a fullnode config that uses transaction outputs to sync
-    let mut vfn_config = NodeConfig::get_default_vfn_config();
-    vfn_config
-        .state_sync
-        .state_sync_driver
-        .continuous_syncing_mode = ContinuousSyncingMode::ApplyTransactionOutputs;
-
-    // Create the fullnode and test its ability to sync
-    let vfn_peer_id = state_sync_utils::create_fullnode(vfn_config, &mut swarm).await;
-    state_sync_utils::test_fullnode_sync(vfn_peer_id, &mut swarm, false, false).await;
-}
-
-#[tokio::test]
-#[ignore] // Ignore this test because it is a subset of test_fullnode_execution_sync_epoch_changes
-async fn test_fullnode_execution_sync_no_epoch_changes() {
-    // Create a validator swarm of 1 validator node
-    let mut swarm = new_local_swarm_with_aptos(1).await;
-
-    // Create a fullnode config that uses transactions to sync
-    let mut vfn_config = NodeConfig::get_default_vfn_config();
-    vfn_config
-        .state_sync
-        .state_sync_driver
-        .continuous_syncing_mode = ContinuousSyncingMode::ExecuteTransactions;
-
-    // Create the fullnode and test its ability to sync
-    let vfn_peer_id = state_sync_utils::create_fullnode(vfn_config, &mut swarm).await;
-    state_sync_utils::test_fullnode_sync(vfn_peer_id, &mut swarm, false, false).await;
-}
-
 #[tokio::test]
 async fn test_single_validator_reboot() {
     // Create a swarm of 1 validator
@@ -268,26 +232,12 @@ async fn test_validator_sync_and_participate_epoch_changes() {
     test_validator_sync_and_participate(false, true).await;
 }
 
-#[tokio::test]
-#[ignore] // Ignore this test because it is a subset of test_validator_sync_and_participate_epoch_changes
-async fn test_validator_sync_and_participate_no_epoch_changes() {
-    // Test the default syncing method without epoch changes
-    test_validator_sync_and_participate(false, false).await;
-}
-
 #[tokio::test]
 async fn test_validator_fast_sync_and_participate_epoch_changes() {
     // Test fast syncing with epoch changes
     test_validator_sync_and_participate(true, true).await;
 }
 
-#[tokio::test]
-#[ignore] // Ignore this test because it is a subset of test_validator_fast_sync_and_participate_epoch_changes
-async fn test_validator_fast_sync_and_participate_no_epoch_changes() {
-    // Test fast syncing without epoch changes
-    test_validator_sync_and_participate(true, false).await;
-}
-
 #[ignore] // Ignore this test because it takes a long time. But, it works so it shouldn't be removed.
 #[tokio::test]
 async fn test_validator_output_sync_small_network_limit() {
@@ -396,13 +346,6 @@ async fn test_validator_fast_sync_exponential_backoff_epoch_changes() {
     test_validator_sync_exponential_backoff(true).await;
 }
 
-#[tokio::test]
-#[ignore] // Ignore this test because it is a subset of test_validator_fast_sync_exponential_backoff_epoch_changes
-async fn test_validator_fast_sync_exponential_backoff_no_epoch_changes() {
-    // Test fast syncing without exponential backoff and no epoch changes
-    test_validator_sync_exponential_backoff(false).await;
-}
-
 #[tokio::test]
 async fn test_validator_execution_sync_epoch_changes() {
     // Create a swarm of 4 validators using transaction syncing