Improve VM crash dumping

cmake/DaemonGame.cmake

@@ -119,6 +119,9 @@ function(buildGameModule module_slug)
 		set_target_properties(${module_target} PROPERTIES PREFIX "")
 		set(GAMEMODULE_DEFINITIONS "${GAMEMODULE_DEFINITIONS};BUILD_VM_IN_PROCESS")
 	else()
+		if (module_slug STREQUAL "native-exe")
+			set(GAMEMODULE_DEFINITIONS "${GAMEMODULE_DEFINITIONS};BUILD_VM_NATIVE_EXE")
+		endif()
 		add_executable("${module_target}" ${module_target_args})
 	endif()
 

src/common/System.cpp

@@ -312,22 +312,38 @@ static const char *WindowsExceptionString(DWORD code)
 		return "Unknown exception";
 	}
 }
+
+#ifdef _MSC_VER
+// The standard library implements calling std::terminate with an exception filter
+static LPTOP_LEVEL_EXCEPTION_FILTER originalExceptionFilter;
+#endif
+
  ALIGN_STACK_FOR_MINGW static LONG WINAPI CrashHandler(PEXCEPTION_POINTERS ExceptionInfo)
 {
 	// Reset handler so that any future errors cause a crash
 	SetUnhandledExceptionFilter(nullptr);
 
-	// TODO: backtrace
+#ifdef _MSC_VER
+	constexpr DWORD CPP_EXCEPTION = 0xe06d7363;
+	if (ExceptionInfo->ExceptionRecord->ExceptionCode == CPP_EXCEPTION && originalExceptionFilter) {
+		return originalExceptionFilter(ExceptionInfo);
+	}
+#endif
 
 	Sys::Error("Crashed with exception 0x%lx: %s", ExceptionInfo->ExceptionRecord->ExceptionCode, WindowsExceptionString(ExceptionInfo->ExceptionRecord->ExceptionCode));
 }
 void SetupCrashHandler()
 {
+#ifdef _MSC_VER
+	originalExceptionFilter =
+#endif
 	SetUnhandledExceptionFilter(CrashHandler);
 }
 #elif defined(__native_client__)
 static void CrashHandler(const void* data, size_t n)
 {
+    // Note: this only works on the main thread. Otherwise we hit
+    // Sys::Error("SendMsg from non-main VM thread");
     VM::CrashDump(static_cast<const uint8_t*>(data), n);
     Sys::Error("Crashed with NaCl exception");
 }

src/shared/CommonProxies.cpp

@@ -437,8 +437,17 @@ namespace VM {
 
     void InitializeProxies(int milliseconds) {
         baseTime = Sys::SteadyClock::now() - std::chrono::milliseconds(milliseconds);
-        Cmd::InitializeProxy();
         Cvar::InitializeProxy();
+
+#ifdef BUILD_VM_NATIVE_EXE
+        // Set up the crash handler now that we have cvars to know if we want to use it.
+        // The handler wouldn't have worked before StaticInitMsg anyway since it needs IPC to log anything.
+        if (useNativeExeCrashHandler.Get()) {
+            Sys::SetupCrashHandler();
+        }
+#endif
+
+        Cmd::InitializeProxy();
     }
 
     static void HandleMiscSyscall(int minor, Util::Reader& reader, IPC::Channel& channel) {

src/shared/VMMain.cpp

@@ -37,6 +37,13 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 IPC::Channel VM::rootChannel;
 
+#ifdef BUILD_VM_NATIVE_EXE
+// When using native exe, turn this off if you want to use core dumps (*nix) or attach a debugger
+// upon crashing (Windows). Disabling it is like -nocrashhandler for Daemon.
+Cvar::Cvar<bool> VM::useNativeExeCrashHandler(
+	VM_STRING_PREFIX "nativeExeCrashHandler", "catch and log crashes/fatal exceptions in native exe VM", Cvar::NONE, true);
+#endif
+
 #ifdef BUILD_VM_IN_PROCESS
 // Special exception type used to cleanly exit a thread for in-process VMs
 // Using an anonymous namespace so the compiler knows that the exception is
@@ -73,7 +80,7 @@ static void CommonInit(Sys::OSHandle rootSocket)
 	}
 }
 
-static void LogFatalError(Str::StringRef message)
+static void SendErrorMsg(Str::StringRef message)
 {
 	// Only try sending an ErrorMsg once
 	static std::atomic_flag errorEntered;
@@ -89,19 +96,35 @@ static void LogFatalError(Str::StringRef message)
 	}
 }
 
+#ifdef __native_client__
+// HACK: when we get a fatal exception in the terminate handler and call abort() to trigger
+// a crash dump (as there doesn't seem to be an API for requesting a minidump directly),
+// the error message is passed through this variable.
+static char realErrorMessage[256];
+#endif
+
 void Sys::Error(Str::StringRef message)
 {
 	if (!OnMainThread()) {
-		// On a non-main thread we can't rely on IPC, so we may not be able to communicate the
-		// error message. So try to trigger a crash dump instead (exiting with abort() triggers
-		// one but exiting with _exit() doesn't). This will give something to work with when
-		// debugging. (For the main thread case a message is usually enough to diagnose the problem
-		// so we don't generate a crash dump; those consume disk space after all.)
+		// On a non-main thread we can't use IPC, so we can't communicate the error message. Just exit.
 		// Also note that throwing ExitException would only work as intended on the main thread.
+#ifdef __native_client__
+		// Don't abort, to avoid "nacl_loader.exe has stopped working" popup on Windows
+		_exit(222);
+#else
+		// Trigger a core dump if enabled. Would give us something to work with since the
+		// error message can't be shown.
 		std::abort();
+#endif
 	}
 
-	LogFatalError(message);
+#ifdef __native_client__
+	if (realErrorMessage[0]) {
+		message = realErrorMessage;
+	}
+#endif
+
+	SendErrorMsg(message);
 
 #ifdef BUILD_VM_IN_PROCESS
 	// Then engine will close the root socket when it wants us to exit, which
@@ -143,13 +166,28 @@ extern "C" DLLEXPORT ALIGN_STACK_FOR_MINGW void vmMain(Sys::OSHandle rootSocket)
 // traditional Unix core dump (for native exe), a debugger, etc.
 NORETURN static void TerminateHandler()
 {
+#ifdef __native_client__
+	// Using a lambda triggers -Wformat-security...
+#	define DispatchError(...) snprintf(realErrorMessage, sizeof(realErrorMessage), __VA_ARGS__)
+#elif defined(BUILD_VM_NATIVE_EXE)
+	auto DispatchError = [](const char* msg, const auto&... fmtArgs) {
+		if (VM::useNativeExeCrashHandler.Get()) {
+			Sys::Error(msg, fmtArgs...);
+		} else {
+			Log::Warn(XSTRING(VM_NAME) " VM terminating:");
+			Log::Warn(msg, fmtArgs...);
+			// fall through to abort() for core dump etc.
+		}
+	};
+#endif
+
 	if (Sys::OnMainThread()) {
 		try {
 			throw; // A terminate handler is only called if there is an active exception
 		} catch (std::exception& err) {
-			LogFatalError(Str::Format("Unhandled exception (%s): %s", typeid(err).name(), err.what()));
+			DispatchError("Unhandled exception (%s): %s", typeid(err).name(), err.what());
 		} catch (...) {
-			LogFatalError("Unhandled exception of unknown type");
+			DispatchError("Unhandled exception of unknown type");
 		}
 	}
 	std::abort();
@@ -177,7 +215,9 @@ int main(int argc, char** argv)
 	std::set_terminate(TerminateHandler);
 	// Set up crash handling for this process. This will allow crashes to be
 	// sent back to the engine and reported to the user.
+#ifdef __native_client__
 	Sys::SetupCrashHandler();
+#endif
 
 	try {
 		CommonInit(rootSocket);

src/shared/VMMain.h

@@ -38,6 +38,10 @@ namespace VM {
 	// Root channel used to communicate with the engine
 	extern IPC::Channel rootChannel;
 
+#ifdef BUILD_VM_NATIVE_EXE
+	extern Cvar::Cvar<bool> useNativeExeCrashHandler;
+#endif
+
 	// Functions each specific gamelogic should implement
 	void VMInit();
 	void VMHandleSyscall(uint32_t id, Util::Reader reader);