Cleanup threads when they exit

External/FEXCore/Source/Interface/Context/Context.cpp

@@ -119,20 +119,12 @@ namespace FEXCore::Context {
     CTX->StopThread(Thread);
   }
 
+  void DestroyThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
+    CTX->DestroyThread(Thread);
+  }
+
   void DeleteForkedThreads(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
-    // This function is called after fork
-    // We need to cleanup some of the thread data that is dead
-    for (auto &DeadThread : CTX->Threads) {
-      if (DeadThread == Thread) {
-        continue;
-      }
-
-      // Setting running to false ensures that when they are shutdown we won't send signals to kill them
-      DeadThread->RunningEvents.Running = false;
-    }
-
-    // We now only have one thread
-    CTX->IdleWaitRefCount = 1;
+    CTX->DeleteForkedThreads(Thread);
   }
 
   void SetSignalDelegator(FEXCore::Context::Context *CTX, FEXCore::SignalDelegator *SignalDelegation) {

External/FEXCore/Source/Interface/Context/Context.h

@@ -197,6 +197,9 @@ namespace FEXCore::Context {
     void CopyMemoryMapping(FEXCore::Core::InternalThreadState *ParentThread, FEXCore::Core::InternalThreadState *ChildThread);
     void RunThread(FEXCore::Core::InternalThreadState *Thread);
 
+    void DestroyThread(FEXCore::Core::InternalThreadState *Thread);
+    void DeleteForkedThreads(FEXCore::Core::InternalThreadState *ExceptForThread);
+
     std::vector<FEXCore::Core::InternalThreadState*> *const GetThreads() { return &Threads; }
 
     void AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename);

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -25,6 +25,7 @@
 #include <fstream>
 #include <unistd.h>
 #include <filesystem>
+#include <algorithm>
 
 #include "Interface/Core/GdbServer.h"
 
@@ -621,6 +622,47 @@ namespace FEXCore::Context {
     return Thread;
   }
 
+  void Context::DestroyThread(FEXCore::Core::InternalThreadState *Thread) {
+    // remove new thread object
+    {
+      std::lock_guard<std::mutex> lk(ThreadCreationMutex);
+
+      auto It = std::find(Threads.begin(), Threads.end(), Thread);
+      LogMan::Throw::A(It != Threads.end(), "Thread wasn't in Threads");
+
+      Threads.erase(It);
+    }
+
+    if (std::this_thread::get_id() == Thread->ExecutionThread.get_id()) {
+      // To be able to delete a thread from itself, we need to detached the std::thread object
+      Thread->ExecutionThread.detach();
+    }
+    delete Thread;
+  }
+
+  void Context::DeleteForkedThreads(FEXCore::Core::InternalThreadState *ExceptForThread) {
+    // This function is called after fork
+    // We need to cleanup some of the thread data that is dead
+    for (auto &DeadThread : Threads) {
+      if (DeadThread == ExceptForThread) {
+        continue;
+      }
+
+      // Setting running to false ensures that when they are shutdown we won't send signals to kill them
+      DeadThread->RunningEvents.Running = false;
+
+      // Clean up thread state
+      delete DeadThread;
+    }
+
+    // Remove all threads but the current thread from Threads
+    Threads.clear();
+    Threads.push_back(ExceptForThread);
+
+    // We now only have one thread
+    IdleWaitRefCount = 1;
+  }
+
   void Context::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr, uint64_t Start, uint64_t Length) {
     Thread->LookupCache->AddBlockMapping(Address, Ptr, Start, Length);
   }
@@ -1181,6 +1223,11 @@ namespace FEXCore::Context {
     IdleWaitCV.notify_all();
 
     SignalDelegation->UninstallTLSState(Thread);
+
+    // If the parent thread is waiting to join, then we can't destroy our thread object
+    if (!Thread->DestroyedByParent && Thread != Thread->CTX->ParentThread) {
+      Thread->CTX->DestroyThread(Thread);
+    }
   }
 
   void FlushCodeRange(FEXCore::Core::InternalThreadState *Thread, uint64_t Start, uint64_t Length) {

External/FEXCore/include/FEXCore/Core/Context.h

@@ -218,6 +218,7 @@ namespace FEXCore::Context {
   void InitializeThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
   void RunThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
   void StopThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
+  void DestroyThread(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
   void DeleteForkedThreads(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread);
   void SetSignalDelegator(FEXCore::Context::Context *CTX, FEXCore::SignalDelegator *SignalDelegation);
   void SetSyscallHandler(FEXCore::Context::Context *CTX, FEXCore::HLE::SyscallHandler *Handler);

External/FEXCore/include/FEXCore/Debug/InternalThreadState.h

@@ -102,8 +102,9 @@ namespace FEXCore::Core {
     uint32_t CompileBlockReentrantRefCount{};
     std::shared_ptr<FEXCore::CompileService> CompileService;
     bool IsCompileService{false};
+    bool DestroyedByParent{false};  // Should the parent destroy this thread, or it destory itself
 
-    FEXCore::Core::CpuStateFrame BaseFrameState{};
+    alignas(16) FEXCore::Core::CpuStateFrame BaseFrameState{};
 
   };
   static_assert(std::is_standard_layout<InternalThreadState>::value, "This needs to be standard layout");

Source/Tests/LinuxSyscalls/x32/Thread.cpp

@@ -111,12 +111,19 @@ namespace FEX::HLE::x32 {
         // Return the new threads TID
         uint64_t Result = NewThread->ThreadManager.GetTID();
 
+        if (flags & CLONE_VFORK) {
+          NewThread->DestroyedByParent = true;
+        }
+
         // Actually start the thread
         FEXCore::Context::RunThread(Thread->CTX, NewThread);
 
         if (flags & CLONE_VFORK) {
           // If VFORK is set then the calling process is suspended until the thread exits with execve or exit
           NewThread->ExecutionThread.join();
+
+          // Normally a thread cleans itself up on exit. But because we need to join, we are now responsible 
+          FEXCore::Context::DestroyThread(Thread->CTX, NewThread);
         }
         SYSCALL_ERRNO();
       }

Source/Tests/LinuxSyscalls/x64/Thread.cpp

@@ -111,12 +111,19 @@ namespace FEX::HLE::x64 {
         uint64_t Result = NewThread->ThreadManager.GetTID();
         LogMan::Msg::D("Child [%d] starting at: 0x%lx. Parent was at 0x%lx", Result, NewThread->CurrentFrame->State.rip, Thread->CurrentFrame->State.rip);
 
+        if (flags & CLONE_VFORK) {
+          NewThread->DestroyedByParent = true;
+        }
+
         // Actually start the thread
         FEXCore::Context::RunThread(Thread->CTX, NewThread);
 
         if (flags & CLONE_VFORK) {
           // If VFORK is set then the calling process is suspended until the thread exits with execve or exit
           NewThread->ExecutionThread.join();
+
+          // Normally a thread cleans itself up on exit. But because we need to join, we are now responsible
+          FEXCore::Context::DestroyThread(Thread->CTX, NewThread);
         }
         SYSCALL_ERRNO();
       }