FEXCore: Moves OS thread creation to the frontend

Fairly lightweight since it is almost 1:1 transplanting the code from
FEXCore in to the SyscallHandler's thread creation code.

Minor changes:
- ExecutionThreadHandler gets freed before executing the thread
   - Saves 16-bytes of memory per thread
- Start all threads paused by default
   - Since I moved the code to the frontend, I noticed we needed to do
     some post thread-creation setup.
   - Without the pause we were racing code execution with TLS setup and
     a few other things.

FEXCore/Source/Interface/Context/Context.h

@@ -117,7 +117,8 @@ namespace FEXCore::Context {
        *    - CTX->RunUntilExit(Thread);
        *  OS thread Creation:
        *    - Thread = CreateThread(0, 0, NewState, PPID);
-       *    - InitializeThread(Thread);
+       *    - Thread->ExecutionThread = FEXCore::Threads::Thread::Create(ThreadHandler, Arg);
+       *    - ThreadHandler calls `CTX->ExecutionThread(Thread)`
        *  OS fork (New thread created with a clone of thread state):
        *    - clone{2, 3}
        *    - Thread = CreateThread(0, 0, CopyOfThreadState, PPID);
@@ -132,14 +133,6 @@ namespace FEXCore::Context {
 
       // Public for threading
       void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) override;
-      /**
-       * @brief Initializes the OS thread object and prepares to start executing on that new OS thread
-       *
-       * @param Thread The internal FEX thread state object
-       *
-       * The OS thread will wait until RunThread is executed
-       */
-      void InitializeThread(FEXCore::Core::InternalThreadState *Thread) override;
       /**
        * @brief Starts the OS thread object to start executing guest code
        *

FEXCore/Source/Interface/Core/Core.cpp

@@ -511,28 +511,6 @@ namespace FEXCore::Context {
     Dispatcher->ExecuteDispatch(Thread->CurrentFrame);
   }
 
-  struct ExecutionThreadHandler {
-    ContextImpl *This;
-    FEXCore::Core::InternalThreadState *Thread;
-  };
-
-  static void *ThreadHandler(void* Data) {
-    ExecutionThreadHandler *Handler = reinterpret_cast<ExecutionThreadHandler*>(Data);
-    Handler->This->ExecutionThread(Handler->Thread);
-    FEXCore::Allocator::free(Handler);
-    return nullptr;
-  }
-
-  void ContextImpl::InitializeThread(FEXCore::Core::InternalThreadState *Thread) {
-    // This will create the execution thread but it won't actually start executing
-    ExecutionThreadHandler *Arg = reinterpret_cast<ExecutionThreadHandler*>(FEXCore::Allocator::malloc(sizeof(ExecutionThreadHandler)));
-    Arg->This = this;
-    Arg->Thread = Thread;
-    Thread->ExecutionThread = FEXCore::Threads::Thread::Create(ThreadHandler, Arg);
-
-    // Wait for the thread to have started
-    Thread->ThreadWaiting.Wait();
-  }
 
   void ContextImpl::InitializeThreadTLSData(FEXCore::Core::InternalThreadState *Thread) {
     // Let's do some initial bookkeeping here
@@ -550,6 +528,9 @@ namespace FEXCore::Context {
     if (ThunkHandler) {
       ThunkHandler->RegisterTLSState(Thread);
     }
+#ifndef _WIN32
+    Alloc::OSAllocator::RegisterTLSData(Thread);
+#endif
   }
 
   void ContextImpl::RunThread(FEXCore::Core::InternalThreadState *Thread) {
@@ -1131,10 +1112,6 @@ namespace FEXCore::Context {
     Thread->ExitReason = FEXCore::Context::ExitReason::EXIT_WAITING;
 
     InitializeThreadTLSData(Thread);
-#ifndef _WIN32
-    Alloc::OSAllocator::RegisterTLSData(Thread);
-#endif
-
     ++IdleWaitRefCount;
 
     // Now notify the thread that we are initialized

FEXCore/include/FEXCore/Core/Context.h

@@ -256,7 +256,6 @@ namespace FEXCore::Context {
       FEX_DEFAULT_VISIBILITY virtual FEXCore::Core::InternalThreadState* CreateThread(uint64_t InitialRIP, uint64_t StackPointer, FEXCore::Core::CPUState *NewThreadState = nullptr, uint64_t ParentTID = 0) = 0;
 
       FEX_DEFAULT_VISIBILITY virtual void ExecutionThread(FEXCore::Core::InternalThreadState *Thread) = 0;
-      FEX_DEFAULT_VISIBILITY virtual void InitializeThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void RunThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void StopThread(FEXCore::Core::InternalThreadState *Thread) = 0;
       FEX_DEFAULT_VISIBILITY virtual void DestroyThread(FEXCore::Core::InternalThreadState *Thread) = 0;

Source/Tools/FEXLoader/LinuxSyscalls/Syscalls/Thread.cpp

@@ -40,6 +40,21 @@ $end_info$
 ARG_TO_STR(idtype_t, "%u")
 
 namespace FEX::HLE {
+
+  struct ExecutionThreadHandler {
+    FEXCore::Context::Context *CTX;
+    FEXCore::Core::InternalThreadState *Thread;
+  };
+
+  static void *ThreadHandler(void* Data) {
+    ExecutionThreadHandler *Handler = reinterpret_cast<ExecutionThreadHandler*>(Data);
+    auto CTX = Handler->CTX;
+    auto Thread = Handler->Thread;
+    FEXCore::Allocator::free(Handler);
+    CTX->ExecutionThread(Thread);
+    return nullptr;
+  }
+
   FEXCore::Core::InternalThreadState *CreateNewThread(FEXCore::Context:: Context *CTX, FEXCore::Core::CpuStateFrame *Frame, FEX::HLE::clone3_args *args) {
     uint64_t flags = args->args.flags;
     FEXCore::Core::CPUState NewThreadState{};
@@ -59,18 +74,6 @@ namespace FEX::HLE {
     }
 
     auto NewThread = CTX->CreateThread(0, 0, &NewThreadState, args->args.parent_tid);
-    bool NeedsXIDCheck = FEX::HLE::_SyscallHandler->NeedXIDCheck();
-    NewThread->StartPaused = NeedsXIDCheck;
-    CTX->InitializeThread(NewThread);
-
-    if (NeedsXIDCheck) {
-      // The first time an application creates a thread, GLIBC installs their SETXID signal handler.
-      // FEX needs to capture all signals and defer them to the guest.
-      // Once FEX creates its first guest thread, overwrite the GLIBC SETXID handler *again* to ensure
-      // FEX maintains control of the signal handler on this signal.
-      FEX::HLE::_SyscallHandler->GetSignalDelegator()->CheckXIDHandler();
-      FEX::HLE::_SyscallHandler->DisableXIDCheck();
-    }
 
     if (FEX::HLE::_SyscallHandler->Is64BitMode()) {
       if (flags & CLONE_SETTLS) {
@@ -86,6 +89,27 @@ namespace FEX::HLE {
       x32::AdjustRipForNewThread(NewThread->CurrentFrame);
     }
 
+    // We need to do some post-thread creation setup.
+    NewThread->StartPaused = true;
+
+    // Initialize a new thread for execution.
+    ExecutionThreadHandler *Arg = reinterpret_cast<ExecutionThreadHandler*>(FEXCore::Allocator::malloc(sizeof(ExecutionThreadHandler)));
+    Arg->CTX = CTX;
+    Arg->Thread = NewThread;
+    NewThread->ExecutionThread = FEXCore::Threads::Thread::Create(ThreadHandler, Arg);
+
+    // Wait for the thread to have started.
+    NewThread->ThreadWaiting.Wait();
+
+    if (FEX::HLE::_SyscallHandler->NeedXIDCheck()) {
+      // The first time an application creates a thread, GLIBC installs their SETXID signal handler.
+      // FEX needs to capture all signals and defer them to the guest.
+      // Once FEX creates its first guest thread, overwrite the GLIBC SETXID handler *again* to ensure
+      // FEX maintains control of the signal handler on this signal.
+      FEX::HLE::_SyscallHandler->GetSignalDelegator()->CheckXIDHandler();
+      FEX::HLE::_SyscallHandler->DisableXIDCheck();
+    }
+
     // Return the new threads TID
     uint64_t Result = NewThread->ThreadManager.GetTID();