From f56d1c6bd2530f955820766866d766e27e36b1ab Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 9 Jan 2026 15:55:29 -0800 Subject: [PATCH 1/5] FEXCore/Allocator: Make pmr::default_resource a late initialized static object Just a little bit of late initialization to this pmr object using placement new. Removes an `atexit` registration that contributes to crashing on exit. --- FEXCore/Source/Utils/Allocator.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/FEXCore/Source/Utils/Allocator.cpp b/FEXCore/Source/Utils/Allocator.cpp index bbf9e09c16..bdd1290e1d 100644 --- a/FEXCore/Source/Utils/Allocator.cpp +++ b/FEXCore/Source/Utils/Allocator.cpp @@ -20,15 +20,27 @@ #include #include #include +#include + #ifndef _WIN32 #include #include #endif namespace fextl::pmr { -static fextl::pmr::default_resource FEXDefaultResource; +static std::once_flag default_resource_initialized {}; +static fextl::pmr::default_resource* FEXDefaultResource {}; +alignas(alignof(fextl::pmr::default_resource)) static char FEXDefaultResourcePlacement[sizeof(fextl::pmr::default_resource)]; + std::pmr::memory_resource* get_default_resource() { - return &FEXDefaultResource; + // This dance is necessary to avoid an atexit allocator call. + if (FEXDefaultResource) { + return FEXDefaultResource; + } + + std::call_once(default_resource_initialized, + []() { FEXDefaultResource = new (FEXDefaultResourcePlacement) fextl::pmr::default_resource {}; }); + return FEXDefaultResource; } } // namespace fextl::pmr From 677de7b345ed5cf345f82fdbe6065e60fbc12378 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 9 Jan 2026 15:59:10 -0800 Subject: [PATCH 2/5] FEXCore/Allocator: Remove unique_ptr behaviour from Alloc64 Even though we leak `Alloc64` on shutdown, that isn't good enough to avoid the `atexit` handler deallocating memory. So we need to use a raw pointer and leak it. Removes an `atexit` registration that contributes to crashing on exit. --- FEXCore/Source/Utils/Allocator.cpp | 4 +- .../Source/Utils/Allocator/64BitAllocator.cpp | 38 +++++++------------ .../Source/Utils/Allocator/HostAllocator.h | 9 +++-- 3 files changed, 21 insertions(+), 30 deletions(-) diff --git a/FEXCore/Source/Utils/Allocator.cpp b/FEXCore/Source/Utils/Allocator.cpp index bdd1290e1d..269f87c862 100644 --- a/FEXCore/Source/Utils/Allocator.cpp +++ b/FEXCore/Source/Utils/Allocator.cpp @@ -55,7 +55,7 @@ using GLIBC_MALLOC_Hook = void* (*)(size_t, const void* caller); using GLIBC_REALLOC_Hook = void* (*)(void*, size_t, const void* caller); using GLIBC_FREE_Hook = void (*)(void*, const void* caller); -fextl::unique_ptr Alloc64 {}; +Alloc::HostAllocator* Alloc64 {}; void* FEX_mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) { void* Result = Alloc64->Mmap(addr, length, prot, flags, fd, offset); @@ -111,7 +111,7 @@ void ClearHooks() { FEXCore::Allocator::mmap = ::mmap; FEXCore::Allocator::munmap = ::munmap; - Alloc::OSAllocator::ReleaseAllocatorWorkaround(std::move(Alloc64)); + Alloc::OSAllocator::ReleaseAllocatorWorkaround(Alloc64); } #pragma GCC diagnostic pop diff --git a/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp b/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp index 74515ef895..926b7c7b19 100644 --- a/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp +++ b/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp @@ -572,32 +572,23 @@ OSAllocator_64Bit::~OSAllocator_64Bit() { } } -fextl::unique_ptr Create64BitAllocator() { - return fextl::make_unique(); -} - -template -struct alloc_delete : public std::default_delete { - void operator()(T* ptr) const { - if (ptr) { - const auto size = sizeof(T); - const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE); +Alloc::HostAllocator* Create64BitAllocator() { + const auto size = sizeof(OSAllocator_64Bit); + const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE); - std::destroy_at(ptr); - ::munmap(ptr, MinPage); - } + auto ptr = ::mmap(nullptr, MinPage, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (ptr == MAP_FAILED) { + ERROR_AND_DIE_FMT("Couldn't allocate memory region"); } - template - requires (std::is_base_of_v) - operator fextl::default_delete() { - return fextl::default_delete(); - } -}; + FEXCore::Allocator::VirtualName("FEXMem_Misc", reinterpret_cast(ptr), MinPage); + + return ::new (ptr) OSAllocator_64Bit(); +} template requires (!std::is_array_v) -fextl::unique_ptr make_alloc_unique(FEXCore::Allocator::MemoryRegion& Base, Args&&... args) { +T* make_alloc(FEXCore::Allocator::MemoryRegion& Base, Args&&... args) { const auto size = sizeof(T); const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE); if (Base.Size < size || MinPage != FEXCore::Utils::FEX_PAGE_SIZE) { @@ -616,11 +607,10 @@ fextl::unique_ptr make_alloc_unique(FEXCore::Allocator::MemoryRegion& Base, A Base.Size -= MinPage; Base.Ptr = reinterpret_cast(reinterpret_cast(Base.Ptr) + MinPage); - auto Result = ::new (ptr) T(std::forward(args)...); - return fextl::unique_ptr>(Result); + return ::new (ptr) T(std::forward(args)...); } -fextl::unique_ptr Create64BitAllocatorWithRegions(fextl::vector& Regions) { +Alloc::HostAllocator* Create64BitAllocatorWithRegions(fextl::vector& Regions) { // This is a bit tricky as we can't allocate memory safely except from the Regions provided. Otherwise we might overwrite memory pages we // don't own. Scan the memory regions and find the smallest one. FEXCore::Allocator::MemoryRegion& Smallest = Regions[0]; @@ -630,7 +620,7 @@ fextl::unique_ptr Create64BitAllocatorWithRegions(fextl::v } } - return make_alloc_unique(Smallest, Regions); + return make_alloc(Smallest, Regions); } } // namespace Alloc::OSAllocator diff --git a/FEXCore/Source/Utils/Allocator/HostAllocator.h b/FEXCore/Source/Utils/Allocator/HostAllocator.h index e425474897..f324ae0e11 100644 --- a/FEXCore/Source/Utils/Allocator/HostAllocator.h +++ b/FEXCore/Source/Utils/Allocator/HostAllocator.h @@ -51,14 +51,15 @@ class GlobalAllocator { } // namespace Alloc namespace Alloc::OSAllocator { -fextl::unique_ptr Create64BitAllocator(); -fextl::unique_ptr Create64BitAllocatorWithRegions(fextl::vector& Regions); -static inline void ReleaseAllocatorWorkaround(fextl::unique_ptr Allocator) { +Alloc::HostAllocator* Create64BitAllocator(); +Alloc::HostAllocator* Create64BitAllocatorWithRegions(fextl::vector& Regions); +static inline void ReleaseAllocatorWorkaround(Alloc::HostAllocator* Allocator) { // XXX: This is currently a leak. // We can't work around this yet until static initializers that allocate memory are completely removed from our codebase // The allocator is also intrusively allocated, so the unique_ptr tries to double free the HostAllocator object. // Luckily we only remove this on process shutdown, so the kernel will do the cleanup for us - Allocator.release(); + // + // delete Allocator; } } // namespace Alloc::OSAllocator From 3facc22d642335830198637b8270570d14ba79ee Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Fri, 9 Jan 2026 16:11:57 -0800 Subject: [PATCH 3/5] FEXCore/Config: Late initialize two objects These need to be late initialized because they allocate memory and register an `atexit` handler to deallocate. Removes an `atexit` registration that contributes to crashing on exit. --- FEXCore/Source/Interface/Config/Config.cpp | 38 +++++++++++-------- Source/Common/Config.cpp | 2 +- Source/Tools/FEXGetConfig/Main.cpp | 2 +- .../TestHarnessRunner/TestHarnessRunner.cpp | 2 +- 4 files changed, 25 insertions(+), 19 deletions(-) diff --git a/FEXCore/Source/Interface/Config/Config.cpp b/FEXCore/Source/Interface/Config/Config.cpp index be169803f4..c9ea2bcefe 100644 --- a/FEXCore/Source/Interface/Config/Config.cpp +++ b/FEXCore/Source/Interface/Config/Config.cpp @@ -49,22 +49,24 @@ enum Paths { PATH_CONFIG_TELEMETRY_FOLDER, PATH_LAST, }; -static std::array Paths; +using PathsType = std::array; +static PathsType* Paths {}; +alignas(alignof(PathsType)) static char PathsPlacement[sizeof(PathsType)]; void SetDataDirectory(const std::string_view Path, bool Global) { - Paths[PATH_DATA_DIR_LOCAL + Global] = Path; + (*Paths)[PATH_DATA_DIR_LOCAL + Global] = Path; } void SetConfigDirectory(const std::string_view Path, bool Global) { - Paths[PATH_CONFIG_DIR_LOCAL + Global] = Path; + (*Paths)[PATH_CONFIG_DIR_LOCAL + Global] = Path; } void SetConfigFileLocation(const std::string_view Path, bool Global) { - Paths[PATH_CONFIG_FILE_LOCAL + Global] = Path; + (*Paths)[PATH_CONFIG_FILE_LOCAL + Global] = Path; } const fextl::string& GetTelemetryDirectory() { - auto& Path = Paths[PATH_CONFIG_TELEMETRY_FOLDER]; + auto& Path = (*Paths)[PATH_CONFIG_TELEMETRY_FOLDER]; if (Path.empty()) { FEX_CONFIG_OPT(TelemetryDirectory, TELEMETRYDIRECTORY); if (!TelemetryDirectory().empty()) { @@ -79,15 +81,15 @@ const fextl::string& GetTelemetryDirectory() { } const fextl::string& GetDataDirectory(bool Global) { - return Paths[PATH_DATA_DIR_LOCAL + Global]; + return (*Paths)[PATH_DATA_DIR_LOCAL + Global]; } const fextl::string& GetConfigDirectory(bool Global) { - return Paths[PATH_CONFIG_DIR_LOCAL + Global]; + return (*Paths)[PATH_CONFIG_DIR_LOCAL + Global]; } const fextl::string& GetConfigFileLocation(bool Global) { - return Paths[PATH_CONFIG_FILE_LOCAL + Global]; + return (*Paths)[PATH_CONFIG_FILE_LOCAL + Global]; } fextl::string GetApplicationConfig(const std::string_view Program, bool Global) { @@ -110,7 +112,9 @@ fextl::string GetApplicationConfig(const std::string_view Program, bool Global) return fextl::fmt::format("{}{}.json", ConfigFile, Program); } -static fextl::map> ConfigLayers; +using ConfigLayerType = fextl::map>; +static ConfigLayerType* ConfigLayers; +alignas(alignof(ConfigLayerType)) static char ConfigLayersPlacement[sizeof(ConfigLayerType)]; class MetaLayer; static FEXCore::Config::MetaLayer* Meta {}; @@ -172,8 +176,8 @@ void MetaLayer::Load() { OptionMap.clear(); for (auto CurrentLayer = LoadOrder.begin(); CurrentLayer != LoadOrder.end(); ++CurrentLayer) { - auto it = ConfigLayers.find(*CurrentLayer); - if (it != ConfigLayers.end() && *CurrentLayer != Type) { + auto it = ConfigLayers->find(*CurrentLayer); + if (it != ConfigLayers->end() && *CurrentLayer != Type) { // Merge this layer's options to this layer MergeConfigMap(it->second->GetOptionMap()); } @@ -234,19 +238,21 @@ void MetaLayer::MergeConfigMap(const LayerOptions& Options) { } void Initialize() { + Paths = new (PathsPlacement) PathsType {}; + ConfigLayers = new (ConfigLayersPlacement) ConfigLayerType {}; AddLayer(fextl::make_unique(FEXCore::Config::LayerType::LAYER_TOP)); - Meta = dynamic_cast(ConfigLayers.begin()->second.get()); + Meta = dynamic_cast(ConfigLayers->begin()->second.get()); } void Shutdown() { - ConfigLayers.clear(); + ConfigLayers->clear(); Meta = nullptr; } void Load() { for (auto CurrentLayer = LoadOrder.begin(); CurrentLayer != LoadOrder.end(); ++CurrentLayer) { - auto it = ConfigLayers.find(*CurrentLayer); - if (it != ConfigLayers.end()) { + auto it = ConfigLayers->find(*CurrentLayer); + if (it != ConfigLayers->end()) { it->second->Load(); } } @@ -412,7 +418,7 @@ void ReloadMetaLayer() { } void AddLayer(fextl::unique_ptr _Layer) { - ConfigLayers.emplace(_Layer->GetLayerType(), std::move(_Layer)); + ConfigLayers->emplace(_Layer->GetLayerType(), std::move(_Layer)); } bool Exists(ConfigOption Option) { diff --git a/Source/Common/Config.cpp b/Source/Common/Config.cpp index 62fa61f94b..45dd260f66 100644 --- a/Source/Common/Config.cpp +++ b/Source/Common/Config.cpp @@ -456,8 +456,8 @@ ApplicationNames GetApplicationNames(const fextl::vector& Args, b void LoadConfig(fextl::string ProgramName, char** const envp, const PortableInformation& PortableInfo) { const bool IsPortable = PortableInfo.IsPortable; - FEX::Config::InitializeConfigs(PortableInfo); FEXCore::Config::Initialize(); + FEX::Config::InitializeConfigs(PortableInfo); if (!IsPortable) { FEXCore::Config::AddLayer(CreateGlobalMainLayer()); } diff --git a/Source/Tools/FEXGetConfig/Main.cpp b/Source/Tools/FEXGetConfig/Main.cpp index d7510aa99e..d0a7ffd37a 100644 --- a/Source/Tools/FEXGetConfig/Main.cpp +++ b/Source/Tools/FEXGetConfig/Main.cpp @@ -95,8 +95,8 @@ TSOEmulationFacts GetTSOEmulationFacts() { } // namespace int main(int argc, char** argv, char** envp) { - FEX::Config::InitializeConfigs(FEX::Config::PortableInformation {}); FEXCore::Config::Initialize(); + FEX::Config::InitializeConfigs(FEX::Config::PortableInformation {}); FEXCore::Config::AddLayer(FEX::Config::CreateGlobalMainLayer()); FEXCore::Config::AddLayer(FEX::Config::CreateMainLayer()); // No FEX arguments passed through command line diff --git a/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp b/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp index 4a0f851155..45d48c066c 100644 --- a/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp +++ b/Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp @@ -196,8 +196,8 @@ int main(int argc, char** argv, char** const envp) { LogMan::Throw::InstallHandler(AssertHandler); LogMan::Msg::InstallHandler(MsgHandler); - FEX::Config::InitializeConfigs(FEX::Config::PortableInformation {}); FEXCore::Config::Initialize(); + FEX::Config::InitializeConfigs(FEX::Config::PortableInformation {}); FEXCore::Config::AddLayer(FEX::Config::CreateEnvironmentLayer(envp)); FEXCore::Config::Load(); From 3b0656ce354900fb5610148741f4ca23571e1c95 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Mon, 26 Jan 2026 12:44:07 -0800 Subject: [PATCH 4/5] Github: Enforce expected amount of cxa_atexit registrations There is only one registration that I've not been able to track down why it happens, but it is something to do with std::ostream. This just ensures in CI that we don't ever register anymore atexit handlers so we don't regress on crashing on `exit`. --- .github/workflows/ccpp.yml | 4 ++++ Scripts/CountInstancesOfAtExit.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100755 Scripts/CountInstancesOfAtExit.py diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index f293353765..9896286c24 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -148,6 +148,10 @@ jobs: with: target: struct_verifier + - name: Ensure expected atexit registrations + shell: bash + run: $GITHUB_WORKSPACE/Scripts/CountInstancesOfAtExit.py ${{runner.workspace}}/build/Bin/FEX "__cxa_atexit@plt>$" 1 ; echo $? + - name: Remove old SHM regions if: ${{ always() }} run: cmake --build build --target remove_old_shm_regions diff --git a/Scripts/CountInstancesOfAtExit.py b/Scripts/CountInstancesOfAtExit.py new file mode 100755 index 0000000000..d357a18ac8 --- /dev/null +++ b/Scripts/CountInstancesOfAtExit.py @@ -0,0 +1,21 @@ +#!/usr/bin/python3 +import sys +import subprocess + +def main(): + # Args: + if (len(sys.argv) < 4): + sys.exit() + + result = subprocess.run(['sh', '-c', "llvm-objdump -D {} | grep \'{}\' | wc -l".format(sys.argv[1], sys.argv[2])], stdout=subprocess.PIPE) + Count = int(result.stdout.decode('ascii')) + + if (Count != int(sys.argv[3])): + sys.exit(-1) + + return 0 + +if __name__ == "__main__": + # execute only if run as a script + sys.exit(main()) + From 0a8d63cecd1a1c9912d9341f7e36df0220a21985 Mon Sep 17 00:00:00 2001 From: Ryan Houdek Date: Wed, 14 Jan 2026 12:36:25 -0800 Subject: [PATCH 5/5] Bash scripting is the worst. --- .github/workflows/ccpp.yml | 4 +++- Scripts/CountInstancesOfAtExit.py | 21 --------------------- 2 files changed, 3 insertions(+), 22 deletions(-) delete mode 100755 Scripts/CountInstancesOfAtExit.py diff --git a/.github/workflows/ccpp.yml b/.github/workflows/ccpp.yml index 9896286c24..7e1a709eb1 100644 --- a/.github/workflows/ccpp.yml +++ b/.github/workflows/ccpp.yml @@ -150,7 +150,9 @@ jobs: - name: Ensure expected atexit registrations shell: bash - run: $GITHUB_WORKSPACE/Scripts/CountInstancesOfAtExit.py ${{runner.workspace}}/build/Bin/FEX "__cxa_atexit@plt>$" 1 ; echo $? + run: | + COUNT=$(llvm-objdump -D build/Bin/FEX | grep '__cxa_atexit@plt>\\$' | wc -l) + [ "$COUNT" -eq 1 ] || { echo "Expected 1 atexit handlers, found $COUNT"; exit 1; } - name: Remove old SHM regions if: ${{ always() }} diff --git a/Scripts/CountInstancesOfAtExit.py b/Scripts/CountInstancesOfAtExit.py deleted file mode 100755 index d357a18ac8..0000000000 --- a/Scripts/CountInstancesOfAtExit.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/python3 -import sys -import subprocess - -def main(): - # Args: - if (len(sys.argv) < 4): - sys.exit() - - result = subprocess.run(['sh', '-c', "llvm-objdump -D {} | grep \'{}\' | wc -l".format(sys.argv[1], sys.argv[2])], stdout=subprocess.PIPE) - Count = int(result.stdout.decode('ascii')) - - if (Count != int(sys.argv[3])): - sys.exit(-1) - - return 0 - -if __name__ == "__main__": - # execute only if run as a script - sys.exit(main()) -