fix(AltTabFullscreen): force borderless-windowed at swap-chain creation

The previous approach subclassed the engine WndProc and called SetFullscreenState(FALSE) on focus loss, but never restored fullscreen on focus gain. After alt-tab back, the swap chain was windowed while the engine believed it was still in exclusive mode, causing a permanent black-screen lockup on the render thread.

Switch to the same approach used by SSE Engine Fixes / Display Tweaks: at D3D11CreateDeviceAndSwapChain, force Windowed=TRUE and clear ALLOW_MODE_SWITCH whenever the engine requested exclusive. Keep MakeWindowAssociation(NO_WINDOW_CHANGES | NO_ALT_ENTER) so DXGI cannot promote us back into exclusive on Alt+Enter. Drop the WndProc subclass and all g_swapChain/g_hwnd/g_origWndProc state.

Author: northaxosky

.Build/F4SE/Plugins/Addictol.toml

@@ -165,7 +165,7 @@ bPipBoyCursorConstraints = true
 # Fixes a bug where the muzzle-flash light keeps illuminating the scene after the flash ends.
 bMuzzleFlashLight = true
 
-# Fixes the exclusive-fullscreen Alt-Tab hang by dropping fullscreen state on focus loss and blocking DXGI's auto Alt+Enter handler.
+# Fixes the exclusive-fullscreen Alt-Tab hang by forcing the swap chain to borderless-windowed at creation and blocking DXGI's auto Alt+Enter handler.
 bAltTabFullscreen = true
 
 # Fixes a CTD when scrapping or wiring after a settlement mod has been removed, by cleaning up orphan power-grid entries left behind by deleted references.

Addictol/Source/Modules/AdModuleAltTabFullscreen.cpp

@@ -16,31 +16,6 @@ namespace Addictol
 		ID3D11Device**, D3D_FEATURE_LEVEL*, ID3D11DeviceContext**);
 
 	static TD3D11CreateDeviceAndSwapChain OriginalCreate = nullptr;
-	static IDXGISwapChain* g_swapChain = nullptr;
-	static HWND g_hwnd = nullptr;
-	static WNDPROC g_origWndProc = nullptr;
-
-	static LRESULT CALLBACK Hook_WndProc(HWND a_hwnd, UINT a_msg, WPARAM a_wp, LPARAM a_lp) noexcept
-	{
-		if (g_swapChain)
-		{
-			const bool focusLost =
-				a_msg == WM_KILLFOCUS ||
-				(a_msg == WM_ACTIVATEAPP && a_wp == FALSE) ||
-				(a_msg == WM_ACTIVATE && LOWORD(a_wp) == WA_INACTIVE);
-
-			if (focusLost)
-			{
-				BOOL isFullscreen = FALSE;
-				if (SUCCEEDED(g_swapChain->GetFullscreenState(&isFullscreen, nullptr)) && isFullscreen)
-					g_swapChain->SetFullscreenState(FALSE, nullptr);
-			}
-		}
-
-		if (g_origWndProc)
-			return CallWindowProcA(g_origWndProc, a_hwnd, a_msg, a_wp, a_lp);
-		return DefWindowProcA(a_hwnd, a_msg, a_wp, a_lp);
-	}
 
 	static HRESULT WINAPI Hook_D3D11Create(
 		IDXGIAdapter*            a_adapter,
@@ -56,36 +31,39 @@ namespace Addictol
 		D3D_FEATURE_LEVEL*       a_outFeatureLevel,
 		ID3D11DeviceContext**    a_outContext) noexcept
 	{
+		// Force borderless-windowed when the engine requested exclusive fullscreen.
+		// The vanilla alt-tab hang is rooted in exclusive-mode ResizeBuffers re-entry;
+		// never entering exclusive eliminates the bug class entirely.
+		DXGI_SWAP_CHAIN_DESC patchedDesc{};
+		const DXGI_SWAP_CHAIN_DESC* descToUse = a_desc;
+		if (a_desc && !a_desc->Windowed)
+		{
+			patchedDesc = *a_desc;
+			patchedDesc.Windowed = TRUE;
+			patchedDesc.Flags &= ~DXGI_SWAP_CHAIN_FLAG_ALLOW_MODE_SWITCH;
+			descToUse = &patchedDesc;
+			REX::INFO("Alt-Tab Fullscreen: forcing exclusive swap chain to windowed ({}x{}, hwnd {})"sv,
+				a_desc->BufferDesc.Width, a_desc->BufferDesc.Height,
+				reinterpret_cast<void*>(a_desc->OutputWindow));
+		}
+
 		const auto hr = OriginalCreate
 			? OriginalCreate(a_adapter, a_driverType, a_software, a_flags, a_featureLevels,
-				a_numFeatureLevels, a_sdkVersion, a_desc, a_outSwapChain, a_outDevice,
+				a_numFeatureLevels, a_sdkVersion, descToUse, a_outSwapChain, a_outDevice,
 				a_outFeatureLevel, a_outContext)
 			: E_FAIL;
 
-		if (FAILED(hr) || !a_outSwapChain || !*a_outSwapChain || !a_desc)
+		if (FAILED(hr) || !a_outSwapChain || !*a_outSwapChain || !descToUse)
 			return hr;
 
-		// AddRef so WndProc dispatched during engine teardown won't dereference a freed swap chain.
-		g_swapChain = *a_outSwapChain;
-		g_swapChain->AddRef();
-		g_hwnd = a_desc->OutputWindow;
-
+		// Block DXGI's auto Alt+Enter; without this, DXGI can transition us back into exclusive.
 		IDXGIFactory* factory = nullptr;
-		if (SUCCEEDED(g_swapChain->GetParent(__uuidof(IDXGIFactory), reinterpret_cast<void**>(&factory))) && factory)
+		if (SUCCEEDED((*a_outSwapChain)->GetParent(__uuidof(IDXGIFactory), reinterpret_cast<void**>(&factory))) && factory)
 		{
-			factory->MakeWindowAssociation(g_hwnd, DXGI_MWA_NO_WINDOW_CHANGES | DXGI_MWA_NO_ALT_ENTER);
+			factory->MakeWindowAssociation(descToUse->OutputWindow, DXGI_MWA_NO_WINDOW_CHANGES | DXGI_MWA_NO_ALT_ENTER);
 			factory->Release();
 		}
 
-		if (g_hwnd && !g_origWndProc)
-		{
-			if (auto* prior = reinterpret_cast<WNDPROC>(GetWindowLongPtrA(g_hwnd, GWLP_WNDPROC)))
-			{
-				g_origWndProc = prior;
-				SetWindowLongPtrA(g_hwnd, GWLP_WNDPROC, reinterpret_cast<LONG_PTR>(&Hook_WndProc));
-			}
-		}
-
 		return hr;
 	}