Relax timeout to 180s

Author: Fridge003

deep_gemm/include/deep_gemm/comm/barrier.cuh

@@ -50,15 +50,15 @@ CUTLASS_DEVICE void nvlink_barrier(const layout::Workspace& workspace,
             ptx::red_add_rel_sys(sym_buffer.map(signal_ptr, thread_idx), signal_sign ? -1 : 1);
         sync_scope();
 
-        // Update status and wait arrival (with 30s timeout, at 2 GHz)
-        constexpr int64_t kNumTimeoutCycles = 30ll * 2000000000ll;
+        // Update status and wait arrival (with 180s timeout, at 2 GHz)
+        constexpr int64_t kNumTimeoutCycles = 180ll * 2000000000ll;
         if (thread_idx == 0) {
             ptx::red_add(counter_ptr, 1);
             const int target = signal_sign ? 0 : static_cast<int>(kNumRanks);
             const auto start_clock = clock64();
             while (ptx::ld_acq_sys(signal_ptr) != target) {
                 if (clock64() - start_clock >= kNumTimeoutCycles) {
-                    printf("DeepGEMM NVLink barrier timeout (30s): rank=%d, counter=%d, signal=%d, target=%d, phase=%d, sign=%d, tag=%d\n",
+                    printf("DeepGEMM NVLink barrier timeout (180s): rank=%d, counter=%d, signal=%d, target=%d, phase=%d, sign=%d, tag=%d\n",
                            sym_buffer.rank_idx, *counter_ptr, ptx::ld_acq_sys(signal_ptr), target, signal_phase, signal_sign, kTag);
                     DG_DEVICE_ASSERT(false and "NVLink barrier timeout");
                 }