Github: Enforce expected amount of cxa_atexit registrations

.github/workflows/ccpp.yml

@@ -148,6 +148,12 @@ jobs:
       with:
         target: struct_verifier
 
+    - name: Ensure expected atexit registrations
+      shell: bash
+      run: |
+        COUNT=$(llvm-objdump -D build/Bin/FEX | grep '__cxa_atexit@plt>\\$' | wc -l)
+        [ "$COUNT" -eq 1 ] || { echo "Expected 1 atexit handlers, found $COUNT"; exit 1; }
+
     - name: Remove old SHM regions
       if: ${{ always() }}
       run: cmake --build build --target remove_old_shm_regions

FEXCore/Source/Interface/Config/Config.cpp

@@ -49,22 +49,24 @@ enum Paths {
   PATH_CONFIG_TELEMETRY_FOLDER,
   PATH_LAST,
 };
-static std::array<fextl::string, Paths::PATH_LAST> Paths;
+using PathsType = std::array<fextl::string, Paths::PATH_LAST>;
+static PathsType* Paths {};
+alignas(alignof(PathsType)) static char PathsPlacement[sizeof(PathsType)];
 
 void SetDataDirectory(const std::string_view Path, bool Global) {
-  Paths[PATH_DATA_DIR_LOCAL + Global] = Path;
+  (*Paths)[PATH_DATA_DIR_LOCAL + Global] = Path;
 }
 
 void SetConfigDirectory(const std::string_view Path, bool Global) {
-  Paths[PATH_CONFIG_DIR_LOCAL + Global] = Path;
+  (*Paths)[PATH_CONFIG_DIR_LOCAL + Global] = Path;
 }
 
 void SetConfigFileLocation(const std::string_view Path, bool Global) {
-  Paths[PATH_CONFIG_FILE_LOCAL + Global] = Path;
+  (*Paths)[PATH_CONFIG_FILE_LOCAL + Global] = Path;
 }
 
 const fextl::string& GetTelemetryDirectory() {
-  auto& Path = Paths[PATH_CONFIG_TELEMETRY_FOLDER];
+  auto& Path = (*Paths)[PATH_CONFIG_TELEMETRY_FOLDER];
   if (Path.empty()) {
     FEX_CONFIG_OPT(TelemetryDirectory, TELEMETRYDIRECTORY);
     if (!TelemetryDirectory().empty()) {
@@ -79,15 +81,15 @@ const fextl::string& GetTelemetryDirectory() {
 }
 
 const fextl::string& GetDataDirectory(bool Global) {
-  return Paths[PATH_DATA_DIR_LOCAL + Global];
+  return (*Paths)[PATH_DATA_DIR_LOCAL + Global];
 }
 
 const fextl::string& GetConfigDirectory(bool Global) {
-  return Paths[PATH_CONFIG_DIR_LOCAL + Global];
+  return (*Paths)[PATH_CONFIG_DIR_LOCAL + Global];
 }
 
 const fextl::string& GetConfigFileLocation(bool Global) {
-  return Paths[PATH_CONFIG_FILE_LOCAL + Global];
+  return (*Paths)[PATH_CONFIG_FILE_LOCAL + Global];
 }
 
 fextl::string GetApplicationConfig(const std::string_view Program, bool Global) {
@@ -110,7 +112,9 @@ fextl::string GetApplicationConfig(const std::string_view Program, bool Global)
   return fextl::fmt::format("{}{}.json", ConfigFile, Program);
 }
 
-static fextl::map<FEXCore::Config::LayerType, fextl::unique_ptr<FEXCore::Config::Layer>> ConfigLayers;
+using ConfigLayerType = fextl::map<FEXCore::Config::LayerType, fextl::unique_ptr<FEXCore::Config::Layer>>;
+static ConfigLayerType* ConfigLayers;
+alignas(alignof(ConfigLayerType)) static char ConfigLayersPlacement[sizeof(ConfigLayerType)];
 class MetaLayer;
 static FEXCore::Config::MetaLayer* Meta {};
 
@@ -172,8 +176,8 @@ void MetaLayer::Load() {
   OptionMap.clear();
 
   for (auto CurrentLayer = LoadOrder.begin(); CurrentLayer != LoadOrder.end(); ++CurrentLayer) {
-    auto it = ConfigLayers.find(*CurrentLayer);
-    if (it != ConfigLayers.end() && *CurrentLayer != Type) {
+    auto it = ConfigLayers->find(*CurrentLayer);
+    if (it != ConfigLayers->end() && *CurrentLayer != Type) {
       // Merge this layer's options to this layer
       MergeConfigMap(it->second->GetOptionMap());
     }
@@ -234,19 +238,21 @@ void MetaLayer::MergeConfigMap(const LayerOptions& Options) {
 }
 
 void Initialize() {
+  Paths = new (PathsPlacement) PathsType {};
+  ConfigLayers = new (ConfigLayersPlacement) ConfigLayerType {};
   AddLayer(fextl::make_unique<MetaLayer>(FEXCore::Config::LayerType::LAYER_TOP));
-  Meta = dynamic_cast<MetaLayer*>(ConfigLayers.begin()->second.get());
+  Meta = dynamic_cast<MetaLayer*>(ConfigLayers->begin()->second.get());
 }
 
 void Shutdown() {
-  ConfigLayers.clear();
+  ConfigLayers->clear();
   Meta = nullptr;
 }
 
 void Load() {
   for (auto CurrentLayer = LoadOrder.begin(); CurrentLayer != LoadOrder.end(); ++CurrentLayer) {
-    auto it = ConfigLayers.find(*CurrentLayer);
-    if (it != ConfigLayers.end()) {
+    auto it = ConfigLayers->find(*CurrentLayer);
+    if (it != ConfigLayers->end()) {
       it->second->Load();
     }
   }
@@ -412,7 +418,7 @@ void ReloadMetaLayer() {
 }
 
 void AddLayer(fextl::unique_ptr<FEXCore::Config::Layer> _Layer) {
-  ConfigLayers.emplace(_Layer->GetLayerType(), std::move(_Layer));
+  ConfigLayers->emplace(_Layer->GetLayerType(), std::move(_Layer));
 }
 
 bool Exists(ConfigOption Option) {

FEXCore/Source/Utils/Allocator.cpp

@@ -20,15 +20,27 @@
 #include <cstdint>
 #include <cstdio>
 #include <fcntl.h>
+#include <mutex>
+
 #ifndef _WIN32
 #include <sys/mman.h>
 #include <sys/user.h>
 #endif
 
 namespace fextl::pmr {
-static fextl::pmr::default_resource FEXDefaultResource;
+static std::once_flag default_resource_initialized {};
+static fextl::pmr::default_resource* FEXDefaultResource {};
+alignas(alignof(fextl::pmr::default_resource)) static char FEXDefaultResourcePlacement[sizeof(fextl::pmr::default_resource)];
+
 std::pmr::memory_resource* get_default_resource() {
-  return &FEXDefaultResource;
+  // This dance is necessary to avoid an atexit allocator call.
+  if (FEXDefaultResource) {
+    return FEXDefaultResource;
+  }
+
+  std::call_once(default_resource_initialized,
+                 []() { FEXDefaultResource = new (FEXDefaultResourcePlacement) fextl::pmr::default_resource {}; });
+  return FEXDefaultResource;
 }
 } // namespace fextl::pmr
 
@@ -43,7 +55,7 @@ using GLIBC_MALLOC_Hook = void* (*)(size_t, const void* caller);
 using GLIBC_REALLOC_Hook = void* (*)(void*, size_t, const void* caller);
 using GLIBC_FREE_Hook = void (*)(void*, const void* caller);
 
-fextl::unique_ptr<Alloc::HostAllocator> Alloc64 {};
+Alloc::HostAllocator* Alloc64 {};
 
 void* FEX_mmap(void* addr, size_t length, int prot, int flags, int fd, off_t offset) {
   void* Result = Alloc64->Mmap(addr, length, prot, flags, fd, offset);
@@ -99,7 +111,7 @@ void ClearHooks() {
   FEXCore::Allocator::mmap = ::mmap;
   FEXCore::Allocator::munmap = ::munmap;
 
-  Alloc::OSAllocator::ReleaseAllocatorWorkaround(std::move(Alloc64));
+  Alloc::OSAllocator::ReleaseAllocatorWorkaround(Alloc64);
 }
 #pragma GCC diagnostic pop
 

FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -572,32 +572,23 @@ OSAllocator_64Bit::~OSAllocator_64Bit() {
   }
 }
 
-fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocator() {
-  return fextl::make_unique<OSAllocator_64Bit>();
-}
-
-template<class T>
-struct alloc_delete : public std::default_delete<T> {
-  void operator()(T* ptr) const {
-    if (ptr) {
-      const auto size = sizeof(T);
-      const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE);
+Alloc::HostAllocator* Create64BitAllocator() {
+  const auto size = sizeof(OSAllocator_64Bit);
+  const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE);
 
-      std::destroy_at(ptr);
-      ::munmap(ptr, MinPage);
-    }
+  auto ptr = ::mmap(nullptr, MinPage, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
+  if (ptr == MAP_FAILED) {
+    ERROR_AND_DIE_FMT("Couldn't allocate memory region");
   }
 
-  template<typename U>
-  requires (std::is_base_of_v<U, T>)
-  operator fextl::default_delete<U>() {
-    return fextl::default_delete<U>();
-  }
-};
+  FEXCore::Allocator::VirtualName("FEXMem_Misc", reinterpret_cast<void*>(ptr), MinPage);
+
+  return ::new (ptr) OSAllocator_64Bit();
+}
 
 template<class T, class... Args>
 requires (!std::is_array_v<T>)
-fextl::unique_ptr<T> make_alloc_unique(FEXCore::Allocator::MemoryRegion& Base, Args&&... args) {
+T* make_alloc(FEXCore::Allocator::MemoryRegion& Base, Args&&... args) {
   const auto size = sizeof(T);
   const auto MinPage = FEXCore::AlignUp(size, FEXCore::Utils::FEX_PAGE_SIZE);
   if (Base.Size < size || MinPage != FEXCore::Utils::FEX_PAGE_SIZE) {
@@ -616,11 +607,10 @@ fextl::unique_ptr<T> make_alloc_unique(FEXCore::Allocator::MemoryRegion& Base, A
   Base.Size -= MinPage;
   Base.Ptr = reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(Base.Ptr) + MinPage);
 
-  auto Result = ::new (ptr) T(std::forward<Args>(args)...);
-  return fextl::unique_ptr<T, alloc_delete<T>>(Result);
+  return ::new (ptr) T(std::forward<Args>(args)...);
 }
 
-fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocatorWithRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions) {
+Alloc::HostAllocator* Create64BitAllocatorWithRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions) {
   // This is a bit tricky as we can't allocate memory safely except from the Regions provided. Otherwise we might overwrite memory pages we
   // don't own. Scan the memory regions and find the smallest one.
   FEXCore::Allocator::MemoryRegion& Smallest = Regions[0];
@@ -630,7 +620,7 @@ fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocatorWithRegions(fextl::v
     }
   }
 
-  return make_alloc_unique<OSAllocator_64Bit>(Smallest, Regions);
+  return make_alloc<OSAllocator_64Bit>(Smallest, Regions);
 }
 
 } // namespace Alloc::OSAllocator

FEXCore/Source/Utils/Allocator/HostAllocator.h

@@ -51,14 +51,15 @@ class GlobalAllocator {
 } // namespace Alloc
 
 namespace Alloc::OSAllocator {
-fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocator();
-fextl::unique_ptr<Alloc::HostAllocator> Create64BitAllocatorWithRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions);
-static inline void ReleaseAllocatorWorkaround(fextl::unique_ptr<Alloc::HostAllocator> Allocator) {
+Alloc::HostAllocator* Create64BitAllocator();
+Alloc::HostAllocator* Create64BitAllocatorWithRegions(fextl::vector<FEXCore::Allocator::MemoryRegion>& Regions);
+static inline void ReleaseAllocatorWorkaround(Alloc::HostAllocator* Allocator) {
   // XXX: This is currently a leak.
   // We can't work around this yet until static initializers that allocate memory are completely removed from our codebase
   // The allocator is also intrusively allocated, so the unique_ptr tries to double free the HostAllocator object.
   // Luckily we only remove this on process shutdown, so the kernel will do the cleanup for us
-  Allocator.release();
+  //
+  // delete Allocator;
 }
 
 } // namespace Alloc::OSAllocator

Source/Common/Config.cpp

@@ -456,8 +456,8 @@ ApplicationNames GetApplicationNames(const fextl::vector<fextl::string>& Args, b
 
 void LoadConfig(fextl::string ProgramName, char** const envp, const PortableInformation& PortableInfo) {
   const bool IsPortable = PortableInfo.IsPortable;
-  FEX::Config::InitializeConfigs(PortableInfo);
   FEXCore::Config::Initialize();
+  FEX::Config::InitializeConfigs(PortableInfo);
   if (!IsPortable) {
     FEXCore::Config::AddLayer(CreateGlobalMainLayer());
   }

Source/Tools/FEXGetConfig/Main.cpp

@@ -95,8 +95,8 @@ TSOEmulationFacts GetTSOEmulationFacts() {
 } // namespace
 
 int main(int argc, char** argv, char** envp) {
-  FEX::Config::InitializeConfigs(FEX::Config::PortableInformation {});
   FEXCore::Config::Initialize();
+  FEX::Config::InitializeConfigs(FEX::Config::PortableInformation {});
   FEXCore::Config::AddLayer(FEX::Config::CreateGlobalMainLayer());
   FEXCore::Config::AddLayer(FEX::Config::CreateMainLayer());
   // No FEX arguments passed through command line

Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp

@@ -196,8 +196,8 @@ int main(int argc, char** argv, char** const envp) {
   LogMan::Throw::InstallHandler(AssertHandler);
   LogMan::Msg::InstallHandler(MsgHandler);
 
-  FEX::Config::InitializeConfigs(FEX::Config::PortableInformation {});
   FEXCore::Config::Initialize();
+  FEX::Config::InitializeConfigs(FEX::Config::PortableInformation {});
   FEXCore::Config::AddLayer(FEX::Config::CreateEnvironmentLayer(envp));
   FEXCore::Config::Load();