Windows: Implement mutex to serialize initialization across multiple VeraCrypt instances

When multiple VeraCrypt.exe instances were launched simultaneously, race conditions
could occur during the WM_INITDIALOG processing phase, potentially causing application
crashes or hang. This was because the initialization logic handles critical operations
like mounting/unmounting volumes and processing favorite volumes that modify global
system state.

This commit:
- Adds a named local session mutex (MainInitMutex) that serializes the WM_INITDIALOG handler
- Implements proper acquisition and release of the mutex during initialization
- Ensures proper cleanup of mutex resources on application exit

Author: idrassi

src/Mount/Mount.c

@@ -187,6 +187,10 @@ static DWORD				LastKnownLogicalDrives;
 
 static volatile LONG FavoriteMountOnGoing = 0;
 
+
+const wchar_t* MainInitMutexName = L"Local\\VeraCryptMainInit_02B831C5_401D_4A0D_8CC5_98D2C4CEB5F2";
+static HANDLE MainInitMutex = NULL;		/* Mutex for main dialog WM_INITDIALOG */
+static BOOL MainInitMutexAcquired = FALSE;	/* TRUE if the main window mutex has been acquired */
 static HANDLE TaskBarIconMutex = NULL;
 static BOOL MainWindowHidden = FALSE;
 static int pwdChangeDlgMode	= PCDM_CHANGE_PASSWORD;
@@ -202,6 +206,29 @@ static HMODULE hWtsLib = NULL;
 static WTSREGISTERSESSIONNOTIFICATION   fnWtsRegisterSessionNotification = NULL;
 static WTSUNREGISTERSESSIONNOTIFICATION fnWtsUnRegisterSessionNotification = NULL;
 
+void AcquireMainInitMutex ()
+{
+	if (MainInitMutex && !MainInitMutexAcquired)
+	{
+		DWORD dwWaitResult;
+		dwWaitResult = WaitForSingleObject (MainInitMutex, INFINITE);
+		if (dwWaitResult == WAIT_OBJECT_0 || dwWaitResult == WAIT_ABANDONED)
+		{
+			// Mutex acquired successfully
+			MainInitMutexAcquired = TRUE;
+		}
+	}
+}
+
+void ReleaseMainInitMutex ()
+{
+	if (MainInitMutex && MainInitMutexAcquired)
+	{
+		ReleaseMutex (MainInitMutex);
+		MainInitMutexAcquired = FALSE;
+	}
+}
+
 // Used to opt-in to receive notification about power events. 
 // This is mandatory to support Windows 10 Modern Standby and Windows 8.1 Connected Standby power model.
 // https://docs.microsoft.com/en-us/windows-hardware/design/device-experiences/prepare-software-for-modern-standby
@@ -426,6 +453,13 @@ static void localcleanup (void)
 	}
 
 	RandStop (TRUE);
+
+	if (MainInitMutex != NULL)
+	{
+		ReleaseMainInitMutex ();
+		CloseHandle (MainInitMutex);
+		MainInitMutex = NULL;
+	}
 }
 
 #ifndef BS_SPLITBUTTON
@@ -7093,6 +7127,9 @@ BOOL CALLBACK MainDialogProc (HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lPa
 			bUseSecureDesktop = FALSE;
 			bUseLegacyMaxPasswordLength = FALSE;
 
+			// lock the init mutex
+			AcquireMainInitMutex ();
+
 			ResetWrongPwdRetryCount ();
 
 			ExtractCommandLine (hwndDlg, (wchar_t *) lParam);
@@ -7149,7 +7186,8 @@ BOOL CALLBACK MainDialogProc (HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lPa
 			if (ComServerMode)
 			{
 				InitDialog (hwndDlg);
-
+				// unlock mutex since we are starting the COM server
+				ReleaseMainInitMutex ();
 				if (!ComServerMain ())
 				{
 					handleWin32Error (hwndDlg, SRC_POS);
@@ -7532,6 +7570,8 @@ BOOL CALLBACK MainDialogProc (HWND hwndDlg, UINT uMsg, WPARAM wParam, LPARAM lPa
 				RegisterWtsAndPowerNotification(hwndDlg);
 			DoPostInstallTasks (hwndDlg);
 			ResetCurrentDirectory ();
+			// unlock the init mutex
+			ReleaseMainInitMutex ();
 		}
 		return 0;
 
@@ -10201,6 +10241,26 @@ int WINAPI wWinMain (HINSTANCE hInstance, HINSTANCE hPrevInstance, wchar_t *lpsz
 		AbortProcess ("NODRIVER");
 	}
 
+	/* Initialize Main mutex */
+	SECURITY_ATTRIBUTES sa;
+	SECURITY_DESCRIPTOR sd;
+	sa.nLength = sizeof(SECURITY_ATTRIBUTES);
+	sa.lpSecurityDescriptor = &sd;
+	sa.bInheritHandle = FALSE;
+
+	// Initialize a security descriptor with a NULL DACL (everyone full access)
+	if (InitializeSecurityDescriptor(&sd, SECURITY_DESCRIPTOR_REVISION) &&
+		SetSecurityDescriptorDacl(&sd, TRUE, NULL, FALSE))
+	{
+		// Use the security attributes when creating the mutex
+		MainInitMutex = CreateMutexW(&sa, FALSE, MainInitMutexName);
+	}
+	else
+	{
+		// If security descriptor initialization fails, fall back to default security attributes
+		MainInitMutex = CreateMutexW(NULL, FALSE, MainInitMutexName);
+	}
+
 	/* Create the main dialog box */
 	DialogBoxParamW (hInstance, MAKEINTRESOURCEW (IDD_MOUNT_DLG), NULL, (DLGPROC) MainDialogProc,
 			(LPARAM) lpszCommandLine);