Merge pull request #2739 from Sonicadvance1/fork_mutexes

Linux: Fixes hangs due to mutexes locked while fork happens.

External/FEXCore/Source/Interface/Context/Context.h

@@ -152,7 +152,9 @@ namespace FEXCore::Context {
        * @param Thread The internal FEX thread state object
        */
       void DestroyThread(FEXCore::Core::InternalThreadState *Thread) override;
-      void CleanupAfterFork(FEXCore::Core::InternalThreadState *Thread) override;
+
+      void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) override;
+      void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) override;
       void SetSignalDelegator(FEXCore::SignalDelegator *SignalDelegation) override;
       void SetSyscallHandler(FEXCore::HLE::SyscallHandler *Handler) override;
 
@@ -252,7 +254,7 @@ namespace FEXCore::Context {
     Event PauseWait;
     bool Running{};
 
-    std::shared_mutex CodeInvalidationMutex;
+    FEXCore::ForkableSharedMutex CodeInvalidationMutex;
 
     FEXCore::CPUIDEmu CPUID;
     FEXCore::HLE::SyscallHandler *SyscallHandler{};
@@ -289,7 +291,7 @@ namespace FEXCore::Context {
     template<auto Fn>
     static uint64_t ThreadExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
       auto Thread = Frame->Thread;
-      ScopedDeferredSignalWithSharedLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+      ScopedDeferredSignalWithForkableSharedLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
 
       return Fn(Frame, record);
     }
@@ -300,8 +302,7 @@ namespace FEXCore::Context {
       auto Thread = Frame->Thread;
 
       LogMan::Throw::AFmt(Thread->ThreadManager.GetTID() == FHU::Syscalls::gettid(), "Must be called from owning thread {}, not {}", Thread->ThreadManager.GetTID(), FHU::Syscalls::gettid());
-
-      ScopedDeferredSignalWithUniqueLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
+      ScopedDeferredSignalWithForkableUniqueLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
 
       ThreadRemoveCodeEntry(Thread, GuestRIP);
     }

External/FEXCore/Source/Interface/Core/Core.cpp

@@ -8,6 +8,7 @@ $end_info$
 */
 
 #include <cstdint>
+#include "FEXCore/Utils/DeferredSignalMutex.h"
 #include "Interface/Context/Context.h"
 #include "Interface/Core/LookupCache.h"
 #include "Interface/Core/Core.h"
@@ -24,6 +25,7 @@ $end_info$
 #include "Interface/IR/Passes/RegisterAllocationPass.h"
 #include "Interface/IR/Passes.h"
 #include "Interface/IR/PassManager.h"
+#include "Utils/Allocator.h"
 #include "Utils/Allocator/HostAllocator.h"
 
 #include <FEXCore/Config/Config.h>
@@ -663,7 +665,17 @@ namespace FEXCore::Context {
     delete Thread;
   }
 
-  void ContextImpl::CleanupAfterFork(FEXCore::Core::InternalThreadState *LiveThread) {
+  void ContextImpl::UnlockAfterFork(FEXCore::Core::InternalThreadState *LiveThread, bool Child) {
+    Allocator::UnlockAfterFork(LiveThread, Child);
+
+    if (Child) {
+      CodeInvalidationMutex.StealAndDropActiveLocks();
+    }
+    else {
+      CodeInvalidationMutex.unlock();
+      return;
+    }
+
     // This function is called after fork
     // We need to cleanup some of the thread data that is dead
     for (auto &DeadThread : Threads) {
@@ -702,6 +714,11 @@ namespace FEXCore::Context {
     FEXCore::Threads::Thread::CleanupAfterFork();
   }
 
+  void ContextImpl::LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) {
+    CodeInvalidationMutex.lock();
+    Allocator::LockBeforeFork(Thread);
+  }
+
   void ContextImpl::AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr) {
     Thread->LookupCache->AddBlockMapping(Address, Ptr);
   }
@@ -1048,7 +1065,7 @@ namespace FEXCore::Context {
     auto Thread = Frame->Thread;
 
     // Invalidate might take a unique lock on this, to guarantee that during invalidation no code gets compiled
-    std::shared_lock lk(CodeInvalidationMutex);
+    ScopedDeferredSignalWithForkableSharedLock lk(CodeInvalidationMutex, Thread);
 
     // Is the code in the cache?
     // The backends only check L1 and L2, not L3
@@ -1228,7 +1245,7 @@ namespace FEXCore::Context {
     // Potential deferred since Thread might not be valid.
     // Thread object isn't valid very early in frontend's initialization.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
-    ScopedPotentialDeferredSignalWithUniqueLock CodeInvalidationLock(CodeInvalidationMutex, Thread);
+    ScopedPotentialDeferredSignalWithForkableUniqueLock lk(CodeInvalidationMutex, Thread);
 
     InvalidateGuestCodeRangeInternal(this, Start, Length);
   }
@@ -1237,7 +1254,7 @@ namespace FEXCore::Context {
     // Potential deferred since Thread might not be valid.
     // Thread object isn't valid very early in frontend's initialization.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
-    ScopedPotentialDeferredSignalWithUniqueLock CodeInvalidationLock(CodeInvalidationMutex, Thread);
+    ScopedPotentialDeferredSignalWithForkableUniqueLock lk(CodeInvalidationMutex, Thread);
 
     InvalidateGuestCodeRangeInternal(this, Start, Length);
     CallAfter(Start, Length);
@@ -1265,7 +1282,7 @@ namespace FEXCore::Context {
   }
 
   void ContextImpl::ThreadAddBlockLink(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestDestination, uintptr_t HostLink, const std::function<void()> &delinker) {
-    std::shared_lock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex);
+    ScopedDeferredSignalWithForkableSharedLock lk(static_cast<ContextImpl*>(Thread->CTX)->CodeInvalidationMutex, Thread);
 
     Thread->LookupCache->AddBlockLink(GuestDestination, HostLink, delinker);
   }

External/FEXCore/Source/Utils/Allocator.cpp

@@ -340,5 +340,17 @@ namespace FEXCore::Allocator {
       ::munmap(Region.Ptr, Region.Size);
     }
   }
+
+  void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) {
+    if (Alloc64) {
+      Alloc64->LockBeforeFork(Thread);
+    }
+  }
+
+  void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) {
+    if (Alloc64) {
+      Alloc64->UnlockAfterFork(Thread, Child);
+    }
+  }
 }
 #endif

External/FEXCore/Source/Utils/Allocator.h

@@ -0,0 +1,10 @@
+#pragma once
+
+namespace FEXCore::Core {
+struct InternalThreadState;
+}
+
+namespace FEXCore::Allocator {
+  void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread);
+  void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child);
+}

External/FEXCore/Source/Utils/Allocator/64BitAllocator.cpp

@@ -49,6 +49,19 @@ namespace Alloc::OSAllocator {
       void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) override;
       int Munmap(void *addr, size_t length) override;
 
+      void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) override {
+        AllocationMutex.lock();
+      }
+
+      void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) override {
+        if (Child) {
+          AllocationMutex.StealAndDropActiveLocks();
+        }
+        else {
+          AllocationMutex.unlock();
+        }
+      }
+
     private:
       // Upper bound is the maximum virtual address space of the host processor
       uintptr_t UPPER_BOUND = (1ULL << 57);
@@ -139,7 +152,7 @@ namespace Alloc::OSAllocator {
       LiveRegionListType *LiveRegions{};
 
       Alloc::ForwardOnlyIntrusiveArenaAllocator *ObjectAlloc{};
-      std::mutex AllocationMutex{};
+      FEXCore::ForkableUniqueMutex AllocationMutex;
       void DetermineVASize();
 
       LiveVMARegion *MakeRegionActive(ReservedRegionListType::iterator ReservedIterator, uint64_t UsedSize) {
@@ -258,7 +271,7 @@ void *OSAllocator_64Bit::Mmap(void *addr, size_t length, int prot, int flags, in
   size_t NumberOfPages = length / FHU::FEX_PAGE_SIZE;
 
   // This needs a mutex to be thread safe
-  FEXCore::ScopedPotentialDeferredSignalWithMutex lk(AllocationMutex, TLSThread);
+  FEXCore::ScopedPotentialDeferredSignalWithForkableMutex lk(AllocationMutex, TLSThread);
 
   uint64_t AllocatedOffset{};
   LiveVMARegion *LiveRegion{};
@@ -446,7 +459,7 @@ int OSAllocator_64Bit::Munmap(void *addr, size_t length) {
   }
 
   // This needs a mutex to be thread safe
-  FEXCore::ScopedPotentialDeferredSignalWithMutex lk(AllocationMutex, TLSThread);
+  FEXCore::ScopedPotentialDeferredSignalWithForkableMutex lk(AllocationMutex, TLSThread);
 
   length = FEXCore::AlignUp(length, FHU::FEX_PAGE_SIZE);
 
@@ -571,7 +584,7 @@ OSAllocator_64Bit::OSAllocator_64Bit() {
 
 OSAllocator_64Bit::~OSAllocator_64Bit() {
   // This needs a mutex to be thread safe
-  FEXCore::ScopedPotentialDeferredSignalWithMutex lk(AllocationMutex, TLSThread);
+  FEXCore::ScopedPotentialDeferredSignalWithForkableMutex lk(AllocationMutex, TLSThread);
 
   // Walk the pages and deallocate
   // First walk the live regions

External/FEXCore/Source/Utils/Allocator/HostAllocator.h

@@ -22,6 +22,9 @@ namespace Alloc {
 
       virtual void *Mmap(void *addr, size_t length, int prot, int flags, int fd, off_t offset) { return nullptr; }
       virtual int Munmap(void *addr, size_t length) { return -1; }
+
+      virtual void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) {}
+      virtual void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) {}
   };
 
   class GlobalAllocator {

External/FEXCore/include/FEXCore/Core/Context.h

@@ -243,7 +243,8 @@ namespace FEXCore::Context {
       FEX_DEFAULT_VISIBILITY virtual void RunThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void StopThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void DestroyThread(FEXCore::Core::InternalThreadState *Thread) = 0;
-      FEX_DEFAULT_VISIBILITY virtual void CleanupAfterFork(FEXCore::Core::InternalThreadState *Thread) = 0;
+      FEX_DEFAULT_VISIBILITY virtual void LockBeforeFork(FEXCore::Core::InternalThreadState *Thread) {}
+      FEX_DEFAULT_VISIBILITY virtual void UnlockAfterFork(FEXCore::Core::InternalThreadState *Thread, bool Child) {}
       FEX_DEFAULT_VISIBILITY virtual void SetSignalDelegator(FEXCore::SignalDelegator *SignalDelegation) = 0;
       FEX_DEFAULT_VISIBILITY virtual void SetSyscallHandler(FEXCore::HLE::SyscallHandler *Handler) = 0;
 

External/FEXCore/include/FEXCore/Utils/DeferredSignalMutex.h

@@ -11,6 +11,91 @@
 #include <unistd.h>
 
 namespace FEXCore {
+#ifndef _WIN32
+  // Replacement for std::mutexes to deal with unlocking issues in the face of Linux fork() semantics.
+  //
+  // A fork() only clones the parent's calling thread. Other threads are silently dropped, which permanently leaves any mutexes owned by them locked.
+  // To address this issue, ForkableUniqueMutex and ForkableSharedMutex provide a way to forcefully remove any dangling locks and reset the mutexes to their default state.
+  class ForkableUniqueMutex final {
+    public:
+      ForkableUniqueMutex()
+        : Mutex (PTHREAD_MUTEX_INITIALIZER) {
+      }
+
+      // Move-only type
+      ForkableUniqueMutex(const ForkableUniqueMutex&) = delete;
+      ForkableUniqueMutex& operator=(const ForkableUniqueMutex&) = delete;
+      ForkableUniqueMutex(ForkableUniqueMutex &&rhs) = default;
+      ForkableUniqueMutex& operator=(ForkableUniqueMutex &&) = default;
+
+      void lock() {
+        [[maybe_unused]] const auto Result = pthread_mutex_lock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to lock with {}", __func__, Result);
+      }
+      void unlock() {
+        [[maybe_unused]] const auto Result = pthread_mutex_unlock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to unlock with {}", __func__, Result);
+      }
+      // Initialize the internal pthread object to its default initializer state.
+      // Should only ever be used in the child process when a Linux fork() has occured.
+      void StealAndDropActiveLocks() {
+        Mutex = PTHREAD_MUTEX_INITIALIZER;
+      }
+    private:
+      pthread_mutex_t Mutex;
+  };
+
+  class ForkableSharedMutex final {
+    public:
+      ForkableSharedMutex()
+        : Mutex (PTHREAD_RWLOCK_INITIALIZER) {
+      }
+
+      // Move-only type
+      ForkableSharedMutex(const ForkableSharedMutex&) = delete;
+      ForkableSharedMutex& operator=(const ForkableSharedMutex&) = delete;
+      ForkableSharedMutex(ForkableSharedMutex &&rhs) = default;
+      ForkableSharedMutex& operator=(ForkableSharedMutex &&) = default;
+
+      void lock() {
+        [[maybe_unused]] const auto Result = pthread_rwlock_wrlock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to lock with {}", __func__, Result);
+      }
+      void unlock() {
+        [[maybe_unused]] const auto Result = pthread_rwlock_unlock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to unlock with {}", __func__, Result);
+      }
+      void lock_shared() {
+        [[maybe_unused]] const auto Result = pthread_rwlock_rdlock(&Mutex);
+        LOGMAN_THROW_A_FMT(Result == 0, "{} failed to lock with {}", __func__, Result);
+      }
+
+      void unlock_shared() {
+        unlock();
+      }
+
+      bool try_lock() {
+        const auto Result = pthread_rwlock_trywrlock(&Mutex);
+        return Result == 0;
+      }
+
+      bool try_lock_shared() {
+        const auto Result = pthread_rwlock_tryrdlock(&Mutex);
+        return Result == 0;
+      }
+      // Initialize the internal pthread object to its default initializer state.
+      // Should only ever be used in the child process when a Linux fork() has occured.
+      void StealAndDropActiveLocks() {
+        Mutex = PTHREAD_RWLOCK_INITIALIZER;
+      }
+    private:
+      pthread_rwlock_t Mutex;
+  };
+#else
+  // Windows doesn't support forking, so these can be standard mutexes.
+  using ForkableUniqueMutex = std::mutex;
+  using ForkableSharedMutex = std::shared_mutex;
+#endif
 
   template<typename MutexType, void (MutexType::*lock_fn)(), void (MutexType::*unlock_fn)()>
   class ScopedDeferredSignalWithMutexBase final {
@@ -66,6 +151,20 @@ namespace FEXCore {
   using ScopedDeferredSignalWithSharedLock = ScopedDeferredSignalWithMutexBase<std::shared_mutex, &std::shared_mutex::lock_shared, &std::shared_mutex::unlock_shared>;
   using ScopedDeferredSignalWithUniqueLock = ScopedDeferredSignalWithMutexBase<std::shared_mutex, &std::shared_mutex::lock, &std::shared_mutex::unlock>;
 
+  // Forkable variant
+  using ScopedDeferredSignalWithForkableMutex      = ScopedDeferredSignalWithMutexBase<
+    FEXCore::ForkableUniqueMutex,
+    &FEXCore::ForkableUniqueMutex::lock,
+    &FEXCore::ForkableUniqueMutex::unlock>;
+  using ScopedDeferredSignalWithForkableSharedLock = ScopedDeferredSignalWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock_shared,
+    &FEXCore::ForkableSharedMutex::unlock_shared>;
+  using ScopedDeferredSignalWithForkableUniqueLock = ScopedDeferredSignalWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock,
+    &FEXCore::ForkableSharedMutex::unlock>;
+
   template<typename MutexType, void (MutexType::*lock_fn)(), void (MutexType::*unlock_fn)()>
   class ScopedPotentialDeferredSignalWithMutexBase final {
     public:
@@ -131,4 +230,18 @@ namespace FEXCore {
   using ScopedPotentialDeferredSignalWithMutex      = ScopedPotentialDeferredSignalWithMutexBase<std::mutex, &std::mutex::lock, &std::mutex::unlock>;
   using ScopedPotentialDeferredSignalWithSharedLock = ScopedPotentialDeferredSignalWithMutexBase<std::shared_mutex, &std::shared_mutex::lock_shared, &std::shared_mutex::unlock_shared>;
   using ScopedPotentialDeferredSignalWithUniqueLock = ScopedPotentialDeferredSignalWithMutexBase<std::shared_mutex, &std::shared_mutex::lock, &std::shared_mutex::unlock>;
+
+  // Forkable variant
+  using ScopedPotentialDeferredSignalWithForkableMutex      = ScopedPotentialDeferredSignalWithMutexBase<
+    FEXCore::ForkableUniqueMutex,
+    &FEXCore::ForkableUniqueMutex::lock,
+    &FEXCore::ForkableUniqueMutex::unlock>;
+  using ScopedPotentialDeferredSignalWithForkableSharedLock = ScopedPotentialDeferredSignalWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock_shared,
+    &FEXCore::ForkableSharedMutex::unlock_shared>;
+  using ScopedPotentialDeferredSignalWithForkableUniqueLock = ScopedPotentialDeferredSignalWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock,
+    &FEXCore::ForkableSharedMutex::unlock>;
 }

FEXHeaderUtils/FEXHeaderUtils/ScopedSignalMask.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/DeferredSignalMutex.h>
 
 #include <atomic>
 #include <cstdint>
@@ -106,4 +106,17 @@ namespace FHU {
   using ScopedSignalMaskWithMutex = ScopedSignalMaskWithMutexBase<std::mutex, &std::mutex::lock, &std::mutex::unlock>;
   using ScopedSignalMaskWithSharedLock = ScopedSignalMaskWithMutexBase<std::shared_mutex, &std::shared_mutex::lock_shared, &std::shared_mutex::unlock_shared>;
   using ScopedSignalMaskWithUniqueLock = ScopedSignalMaskWithMutexBase<std::shared_mutex, &std::shared_mutex::lock, &std::shared_mutex::unlock>;
+
+  using ScopedSignalMaskWithForkableMutex = ScopedSignalMaskWithMutexBase<
+    FEXCore::ForkableUniqueMutex,
+    &FEXCore::ForkableUniqueMutex::lock,
+    &FEXCore::ForkableUniqueMutex::unlock>;
+  using ScopedSignalMaskWithForkableSharedLock = ScopedSignalMaskWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock_shared,
+    &FEXCore::ForkableSharedMutex::unlock_shared>;
+  using ScopedSignalMaskWithForkableUniqueLock = ScopedSignalMaskWithMutexBase<
+    FEXCore::ForkableSharedMutex,
+    &FEXCore::ForkableSharedMutex::lock,
+    &FEXCore::ForkableSharedMutex::unlock>;
 }

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls.cpp

@@ -580,12 +580,33 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args
     if (!AnyFlagsSet(flags, CLONE_THREAD)) {
       // Has an unsupported flag
       // Fall to a handler that can handle this case
+      auto Thread = Frame->Thread;
+
+      args->SignalMask = ~0ULL;
+      ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &args->SignalMask, &args->SignalMask, sizeof(args->SignalMask));
+      Thread->CTX->LockBeforeFork(Frame->Thread);
+
+      FEX::HLE::_SyscallHandler->LockBeforeFork();
+
+      uint64_t Result{};
       if (args->Type == TYPE_CLONE2) {
-        return Clone2Handler(Frame, args);
+        Result = Clone2Handler(Frame, args);
       }
       else {
-        return Clone3Handler(Frame, args);
+        Result = Clone3Handler(Frame, args);
       }
+
+      if (Result != 0) {
+        // Parent
+        // Unlock the mutexes on both sides of the fork
+        FEX::HLE::_SyscallHandler->UnlockAfterFork(false);
+
+        // Clear all the other threads that are being tracked
+        Thread->CTX->UnlockAfterFork(Frame->Thread, false);
+
+        ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &args->SignalMask, nullptr, sizeof(args->SignalMask));
+      }
+      return Result;
     }
     else {
       LogMan::Msg::IFmt("Unsupported flag with CLONE_THREAD. This breaks TLS, falling down classic thread path");
@@ -634,6 +655,7 @@ uint64_t CloneHandler(FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args
       // Normally a thread cleans itself up on exit. But because we need to join, we are now responsible
       Thread->CTX->DestroyThread(NewThread);
     }
+
     SYSCALL_ERRNO();
   }
 };
@@ -808,17 +830,16 @@ uint64_t UnimplementedSyscallSafe(FEXCore::Core::CpuStateFrame *Frame, uint64_t
 }
 
 void SyscallHandler::LockBeforeFork() {
-  // XXX shared_mutex has issues with locking and forks
-  // VMATracking.Mutex.lock();
-
-  // Add other mutexes here
+  VMATracking.Mutex.lock();
 }
 
-void SyscallHandler::UnlockAfterFork() {
-  // Add other mutexes here
-
-  // XXX shared_mutex has issues with locking and forks
-  // VMATracking.Mutex.unlock();
+void SyscallHandler::UnlockAfterFork(bool Child) {
+  if (Child) {
+    VMATracking.Mutex.StealAndDropActiveLocks();
+  }
+  else {
+    VMATracking.Mutex.unlock();
+  }
 }
 
 static bool isHEX(char c) {

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls.h

@@ -15,6 +15,7 @@ $end_info$
 #include <FEXCore/HLE/SourcecodeResolver.h>
 #include <FEXCore/IR/IR.h>
 #include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/DeferredSignalMutex.h>
 #include <FEXCore/fextl/fmt.h>
 #include <FEXCore/fextl/map.h>
 #include <FEXCore/fextl/memory.h>
@@ -220,7 +221,7 @@ public:
 
   ///// FORK tracking /////
   void LockBeforeFork();
-  void UnlockAfterFork();
+  void UnlockAfterFork(bool Child);
 
   SourcecodeResolver *GetSourcecodeResolver() override { return this; }
 
@@ -327,7 +328,7 @@ private:
   struct VMATracking {
     using VMAEntry = SyscallHandler::VMAEntry;
     // Held while reading/writing this struct
-    std::shared_mutex Mutex;
+    FEXCore::ForkableSharedMutex Mutex;
 
     // Memory ranges indexed by page aligned starting address
     fextl::map<uint64_t, VMAEntry> VMAs;
@@ -430,6 +431,7 @@ enum TypeOfClone {
 
 struct clone3_args {
   TypeOfClone Type;
+  uint64_t SignalMask;
   kernel_clone3_args args;
 };
 

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp

@@ -140,7 +140,13 @@ namespace FEX::HLE {
       // If we don't have CLONE_THREAD then we are effectively a fork
       // Clear all the other threads that are being tracked
       // Frame->Thread is /ONLY/ safe to access when CLONE_THREAD flag is not set
-      CTX->CleanupAfterFork(Frame->Thread);
+      // Unlock the mutexes on both sides of the fork
+      FEX::HLE::_SyscallHandler->UnlockAfterFork(true);
+
+      // Clear all the other threads that are being tracked
+      Thread->CTX->UnlockAfterFork(Frame->Thread, true);
+
+      ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &CloneArgs->SignalMask, nullptr, sizeof(CloneArgs->SignalMask));
 
       Thread->CurrentFrame->State.gregs[FEXCore::X86State::REG_RAX] = 0;
       Thread->CurrentFrame->State.gregs[FEXCore::X86State::REG_RBX] = 0;
@@ -187,6 +193,11 @@ namespace FEX::HLE {
 
   uint64_t ForkGuest(FEXCore::Core::InternalThreadState *Thread, FEXCore::Core::CpuStateFrame *Frame, uint32_t flags, void *stack, size_t StackSize, pid_t *parent_tid, pid_t *child_tid, void *tls) {
     // Just before we fork, we lock all syscall mutexes so that both processes will end up with a locked mutex
+
+    uint64_t Mask{~0ULL};
+    ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &Mask, &Mask, sizeof(Mask));
+    Thread->CTX->LockBeforeFork(Frame->Thread);
+
     FEX::HLE::_SyscallHandler->LockBeforeFork();
 
     const bool IsVFork = flags & CLONE_VFORK;
@@ -215,11 +226,17 @@ namespace FEX::HLE {
     else {
       Result = fork();
     }
+    const bool IsChild = Result == 0;
 
-    // Unlock the mutexes on both sides of the fork
-    FEX::HLE::_SyscallHandler->UnlockAfterFork();
+    if (IsChild) {
+      // Unlock the mutexes on both sides of the fork
+      FEX::HLE::_SyscallHandler->UnlockAfterFork(IsChild);
+
+      // Clear all the other threads that are being tracked
+      Thread->CTX->UnlockAfterFork(Frame->Thread, IsChild);
+
+      ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &Mask, nullptr, sizeof(Mask));
 
-    if (Result == 0) {
       // Child
       // update the internal TID
       Thread->ThreadManager.TID = FHU::Syscalls::gettid();
@@ -227,9 +244,6 @@ namespace FEX::HLE {
       FEX::HLE::_SyscallHandler->FM.UpdatePID(Thread->ThreadManager.PID);
       Thread->ThreadManager.clear_child_tid = nullptr;
 
-      // Clear all the other threads that are being tracked
-      Thread->CTX->CleanupAfterFork(Frame->Thread);
-
       // only a  single thread running so no need to remove anything from the thread array
 
       // Handle child setup now
@@ -275,6 +289,14 @@ namespace FEX::HLE {
         }
       }
 
+      // Unlock the mutexes on both sides of the fork
+      FEX::HLE::_SyscallHandler->UnlockAfterFork(IsChild);
+
+      // Clear all the other threads that are being tracked
+      Thread->CTX->UnlockAfterFork(Frame->Thread, IsChild);
+
+      ::syscall(SYS_rt_sigprocmask, SIG_SETMASK, &Mask, nullptr, sizeof(Mask));
+
       // VFork needs the parent to wait for the child to exit.
       if (IsVFork) {
         // Wait for the read end of the pipe to close.

Source/Tools/FEXLoader/LinuxSyscalls/SyscallsSMCTracking.cpp

@@ -54,7 +54,7 @@ bool SyscallHandler::HandleSegfault(FEXCore::Core::InternalThreadState *Thread,
 
   {
     // Can't use the deferred signal lock in the SIGSEGV handler.
-    FHU::ScopedSignalMaskWithSharedLock lk(_SyscallHandler->VMATracking.Mutex);
+    FHU::ScopedSignalMaskWithForkableSharedLock lk(_SyscallHandler->VMATracking.Mutex);
 
     auto VMATracking = &_SyscallHandler->VMATracking;
 
@@ -111,7 +111,7 @@ void SyscallHandler::MarkGuestExecutableRange(FEXCore::Core::InternalThreadState
       return;
     }
 
-    FEXCore::ScopedDeferredSignalWithSharedLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableSharedLock lk(VMATracking.Mutex, Thread);
 
     // Find the first mapping at or after the range ends, or ::end().
     // Top points to the address after the end of the range
@@ -166,7 +166,7 @@ void SyscallHandler::MarkGuestExecutableRange(FEXCore::Core::InternalThreadState
 
 // Used for AOT
 FEXCore::HLE::AOTIRCacheEntryLookupResult SyscallHandler::LookupAOTIRCacheEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestAddr) {
-  FEXCore::ScopedDeferredSignalWithSharedLock lk(VMATracking.Mutex, Thread);
+  FEXCore::ScopedDeferredSignalWithForkableSharedLock lk(VMATracking.Mutex, Thread);
 
   // Get the first mapping after GuestAddr, or end
   // GuestAddr is inclusive
@@ -194,7 +194,7 @@ void SyscallHandler::TrackMmap(FEXCore::Core::InternalThreadState *Thread, uintp
     // NOTE: Frontend calls this with a nullptr Thread during initialization, but
     //       providing this code with a valid Thread object earlier would allow
     //       us to be more optimal by using ScopedDeferredSignalWithUniqueLock instead
-    FEXCore::ScopedPotentialDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedPotentialDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     static uint64_t AnonSharedId = 1;
 
@@ -233,6 +233,7 @@ void SyscallHandler::TrackMmap(FEXCore::Core::InternalThreadState *Thread, uintp
   }
 
   if (SMCChecks != FEXCore::Config::CONFIG_SMC_NONE) {
+    // VMATracking.Mutex can't be held while executing this, otherwise it hangs if the JIT is in the process of looking up code in the AOT JIT.
     CTX->InvalidateGuestCodeRange(Thread, (uintptr_t)Base, Size);
   }
 }
@@ -244,7 +245,7 @@ void SyscallHandler::TrackMunmap(FEXCore::Core::InternalThreadState *Thread, uin
     // Frontend calls this with nullptr Thread during initialization.
     // This is why `ScopedPotentialDeferredSignalWithUniqueLock` is used here.
     // To be more optimal the frontend should provide this code with a valid Thread object earlier.
-    FEXCore::ScopedPotentialDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedPotentialDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     VMATracking.ClearUnsafe(CTX, Base, Size);
   }
@@ -258,7 +259,7 @@ void SyscallHandler::TrackMprotect(FEXCore::Core::InternalThreadState *Thread, u
   Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE);
 
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     VMATracking.ChangeUnsafe(Base, Size, VMAProt::fromProt(Prot));
   }
@@ -273,7 +274,7 @@ void SyscallHandler::TrackMremap(FEXCore::Core::InternalThreadState *Thread, uin
   NewSize = FEXCore::AlignUp(NewSize, FHU::FEX_PAGE_SIZE);
 
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     const auto OldVMA = VMATracking.LookupVMAUnsafe(OldAddress);
 
@@ -331,7 +332,7 @@ void SyscallHandler::TrackShmat(FEXCore::Core::InternalThreadState *Thread, int
   uint64_t Length = stat.shm_segsz;
 
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     // TODO
     MRID mrid{SpecialDev::SHM, static_cast<uint64_t>(shmid)};
@@ -353,7 +354,7 @@ void SyscallHandler::TrackShmat(FEXCore::Core::InternalThreadState *Thread, int
 void SyscallHandler::TrackShmdt(FEXCore::Core::InternalThreadState *Thread, uintptr_t Base) {
   uintptr_t Length = 0;
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
 
     Length = VMATracking.ClearShmUnsafe(CTX, Base);
   }
@@ -367,8 +368,7 @@ void SyscallHandler::TrackShmdt(FEXCore::Core::InternalThreadState *Thread, uint
 void SyscallHandler::TrackMadvise(FEXCore::Core::InternalThreadState *Thread, uintptr_t Base, uintptr_t Size, int advice) {
   Size = FEXCore::AlignUp(Size, FHU::FEX_PAGE_SIZE);
   {
-    FEXCore::ScopedDeferredSignalWithUniqueLock lk(VMATracking.Mutex, Thread);
-
+    FEXCore::ScopedDeferredSignalWithForkableUniqueLock lk(VMATracking.Mutex, Thread);
     // TODO
   }
 }