Merge pull request #6106 from Sonicadvance1/310

Minor fixes

.github/workflows/mingw_build.yml

@@ -47,7 +47,8 @@ jobs:
       run: |
         cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake \
         -DMINGW_TRIPLE=$MINGW_TRIPLE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_TESTING=False \
-        -DCMAKE_INSTALL_PREFIX="$PWD"/build/install
+        -DCMAKE_INSTALL_PREFIX="$PWD"/build/install \
+        -DBUILD_FEX_MINGW_TESTS=True -DLLVM_MINGW="$HOME/llvm-mingw/build/"
 
     - name: Build
       run: cmake --build build

.github/workflows/pr-code-format.yml

@@ -61,3 +61,19 @@ jobs:
             --start-rev "$START_REV" \
             --end-rev "$END_REV" \
             --changed-files "$CHANGED_FILES"
+
+      - name: Check long CMake lines
+        if: always()
+        run: |
+          # check lines > 140 chars
+          # unit tests are left alone for now
+          # FEXInterpreter is also left alone until its long echo lines can be removed
+          echo "$CHANGED_FILES" | tr ',' '\n' | grep -ve "unittests\|External\|FEXInterpreter" \
+            | grep -e "CMakeLists.txt\|\.cmake" | xargs -L1 wc -L | while read -r line; do
+            if [ "$(echo "$line" | cut -d' ' -f1)" -gt 140 ]; then
+              echo "$line"
+            fi
+          done | tee lines.txt
+
+          # if any lines were emitted, fail
+          [ ! -s lines.txt ]

.neocmake.toml

@@ -0,0 +1 @@
+line_max_words = 140

CMakeLists.txt

@@ -5,40 +5,69 @@ include(CheckIncludeFiles)
 check_include_files("gdb/jit-reader.h" HAVE_GDB_JIT_READER_H)
 
 option(BUILD_FEX_LINUX_TESTS "Build FEXLinuxTests (requires x86 compiler)" FALSE)
+option(BUILD_FEX_MINGW_TESTS "Build FEXMingwTests (requires llvm-mingw compiler)" FALSE)
 option(BUILD_THUNKS "Build thunks" FALSE)
-option(BUILD_FEXCONFIG "Build FEXConfig" TRUE)
 option(ENABLE_CLANG_THUNKS "Build thunks with clang" TRUE)
+
+option(BUILD_FEXCONFIG "Build FEXConfig" TRUE)
 option(ENABLE_IWYU "Enable the Include What You Use sanitizer" FALSE)
 option(ENABLE_LTO "Enable LTO with compilation" TRUE)
-option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
+option(ENABLE_COVERAGE "Enable Code Coverage" FALSE)
 set(USE_LINKER "" CACHE STRING "Path to a custom linker program")
+
+option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
 option(ENABLE_UBSAN "Enable the Clang Undefined Behavior Sanitizer" FALSE)
 option(ENABLE_ASAN "Enable the Clang Address Sanitizer" FALSE)
 option(ENABLE_TSAN "Enable the Clang Thread Sanitizer" FALSE)
-option(ENABLE_COVERAGE "Enable Code Coverage" FALSE)
+
 option(ENABLE_ASSERTIONS "Enable debug assertions" FALSE)
 option(ENABLE_GDB_SYMBOLS "Enable GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H})
+
 option(ENABLE_STRICT_WERROR "Enable stricter -Werror" FALSE)
 option(ENABLE_WERROR "Enable -Werror" FALSE)
+
 option(ENABLE_FEX_ALLOCATOR "Enable allocator for FEX" TRUE)
 option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enable jemalloc glibc allocator" TRUE)
+
 option(ENABLE_OFFLINE_TELEMETRY "Enable FEX offline telemetry" TRUE)
 option(ENABLE_COMPILE_TIME_TRACE "Enable time trace compile option" FALSE)
+
 option(ENABLE_LIBCXX "Use LLVM's libc++ instead of the GNU libstdc++" FALSE)
 option(ENABLE_CCACHE "Enable ccache for build caching" TRUE)
+
 option(ENABLE_VIXL_SIMULATOR "Use the VIXL simulator for emulation (only useful for CI testing)" FALSE)
 option(ENABLE_VIXL_DISASSEMBLER "Enable debug disassembler output with VIXL" FALSE)
 option(ENABLE_ZYDIS "Enable x86/x86-64 guest disassembler output with Zydis" FALSE)
+
 option(USE_LEGACY_BINFMTMISC "Use legacy method of setting up binfmt_misc" FALSE)
+
 option(ENABLE_FEXCORE_PROFILER "Enable FEXCore's timeline profiling capabilities" FALSE)
 set(FEXCORE_PROFILER_BACKEND "gpuvis" CACHE STRING "Set which backend to use for FEXCore's profiler")
 set_property(CACHE FEXCORE_PROFILER_BACKEND PROPERTY STRINGS gpuvis tracy)
+
 option(ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT "Enables glibc memory allocation hooking with fault for CI testing")
+
 option(USE_PDB_DEBUGINFO "Build debug info in PDB format" FALSE)
 option(BUILD_STEAM_SUPPORT "Enable Steam integration" FALSE)
 
-set(X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
-set(X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
+set(X86_32_TOOLCHAIN_FILE
+  "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake"
+  CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
+
+set(X86_64_TOOLCHAIN_FILE
+  "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake"
+  CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
+
+if (BUILD_FEX_MINGW_TESTS)
+  set(LLVM_MINGW "" CACHE FILEPATH "Path to llvm_mingw")
+  set(X86_32_MINGW_TOOLCHAIN_FILE
+    "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_mingw_x86_32.cmake"
+    CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting mingw_i686")
+  set(X86_64_MINGW_TOOLCHAIN_FILE
+    "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_mingw_x86_64.cmake"
+    CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting mingw_x86_64")
+endif()
+
 set(X86_DEV_ROOTFS "/" CACHE FILEPATH "Path to the sysroot used for cross-compiling for i686 and x86_64")
 set(DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
 set(HOSTLIBS_DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
@@ -81,7 +110,8 @@ else ()
   message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
   set(CLANG_MINIMUM_VERSION 13.0)
   if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
-    message(FATAL_ERROR "Clang version too old for FEX. Need at least ${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}")
+    message(FATAL_ERROR "Clang version too old for FEX. Need at least "
+      "${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}")
   endif()
 endif()
 
@@ -137,7 +167,8 @@ if (ENABLE_FEXCORE_PROFILER)
     # Tracy can gather call stack samples in regular intervals, but this
     # isn't useful for us since it would usually sample opaque JIT code
     add_compile_definitions(TRACY_NO_SAMPLING=1)
-    # This pulls in libbacktrace which allocators in global constructors (before FEX can set up its allocator hooks)
+    # This pulls in libbacktrace which allocators in global constructors
+    # (before FEX can set up its allocator hooks)
     add_compile_definitions(TRACY_NO_CALLSTACK=1)
     if (MINGW)
       message(FATAL_ERROR "Tracy profiler not supported on MinGW")
@@ -174,8 +205,7 @@ set(OVERRIDE_HASH "detect" CACHE STRING "Override the FEX git hash")
 
 get_property(IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 if (NOT IS_MULTI_CONFIG AND NOT CMAKE_BUILD_TYPE)
-    set(CMAKE_BUILD_TYPE Release
-        CACHE STRING "Choose the type of build." FORCE)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
     message(STATUS "No build type set, defaulting to a Release build")
 endif()
 
@@ -224,11 +254,17 @@ check_cxx_source_compiles(
   HAS_CLANG_PRESERVE_ALL)
 unset(CMAKE_REQUIRED_FLAGS)
 if (HAS_CLANG_PRESERVE_ALL)
-  if (MINGW)
+  message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
+  set(CLANG_MINIMUM_VERSION 22.1)
+  if (MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
     message(STATUS "Ignoring broken clang::preserve_all support")
     set(HAS_CLANG_PRESERVE_ALL FALSE)
   else()
     message(STATUS "Has clang::preserve_all")
+    if (MINGW)
+      message(STATUS "Disabling asynchronous-unwind-tables due to bug with preserve_all and mingw")
+      add_compile_options(-fno-asynchronous-unwind-tables)
+    endif()
   endif()
 endif()
 
@@ -246,7 +282,7 @@ check_cxx_source_compiles(
   return program_invocation_name == nullptr;
   }"
   HAS_PROGRAM_INVOCATION_NAME)
-add_compile_definitions("HAS_PROGRAM_INVOCATION_NAME=${HAS_PROGRAM_INVOCATION_NAME}")
+add_compile_definitions( "HAS_PROGRAM_INVOCATION_NAME=${HAS_PROGRAM_INVOCATION_NAME}")
 
 if (ENABLE_VIXL_SIMULATOR)
   # We can run the simulator on both x86-64 or AArch64 hosts
@@ -278,7 +314,8 @@ if (ENABLE_COMPILE_TIME_TRACE)
   link_libraries(-ftime-trace)
 endif()
 
-set(PTHREAD_LIB pthread)
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
 
 if (USE_LINKER)
   message(STATUS "Overriding linker to: ${USE_LINKER}")
@@ -287,8 +324,8 @@ endif()
 
 if (ENABLE_LIBCXX)
   message(WARNING "This is an unsupported configuration and should only be used for testing")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi")
+  string(APPEND CMAKE_CXX_FLAGS " -std=c++11 -stdlib=libc++")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -stdlib=libc++ -lc++abi")
 endif()
 
 if (NOT ENABLE_OFFLINE_TELEMETRY)
@@ -298,15 +335,16 @@ endif()
 
 if (ENABLE_UBSAN)
   # See https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
-  # and related discussion for the use of -fno-sanitize=alignment -fno-sanitize=function
-  # with UBSAN.
-  # alignment: we don't follow a strict alignment policy, for example IR uses packed structs
-  # that are regularly access unaligned.
-  # function: syscalls cast function pointers to void (*)(unsigned long...), causing warnings
-  # related to this access.
+  # and related discussion for the use of -fno-sanitize=alignment and -fno-sanitize=function with UBSAN.
+  # alignment: we don't follow a strict alignment policy, for example IR uses packed structs that are regularly access unaligned.
+  # function: syscalls cast function pointers to void (*)(unsigned long...), causing warnings related to this access.
   add_compile_definitions(ENABLE_UBSAN=1)
-  add_compile_options(-fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=function -fno-sanitize-recover=undefined)
-  link_libraries(-fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=function -fno-sanitize-recover=undefined)
+  add_compile_options(-fno-omit-frame-pointer -fsanitize=undefined
+    -fno-sanitize=alignment -fno-sanitize=function
+    -fno-sanitize-recover=undefined)
+  link_libraries(-fno-omit-frame-pointer -fsanitize=undefined
+    -fno-sanitize=alignment -fno-sanitize=function
+    -fno-sanitize-recover=undefined)
 endif()
 
 if (ENABLE_ASAN)
@@ -328,7 +366,8 @@ endif()
 if (ENABLE_JEMALLOC_GLIBC_ALLOC)
   # The glibc jemalloc subproject which hooks the glibc allocator.
   # Required for thunks to work.
-  # All host native libraries will use this allocator, while *most* other FEX internal allocations will use the other jemalloc allocator.
+  # All host native libraries will use this allocator, while *most*
+  # other FEX internal allocations will use the other jemalloc allocator.
   add_subdirectory(External/jemalloc_glibc/)
 elseif (NOT MINGW)
   message(STATUS
@@ -354,11 +393,11 @@ if (USE_PDB_DEBUGINFO)
   add_link_options(-g -Wl,--pdb=)
 endif()
 
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
-set(CMAKE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_LINKER_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
+string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " -fno-omit-frame-pointer")
+string(APPEND CMAKE_LINKER_FLAGS_RELWITHDEBINFO " -fno-omit-frame-pointer")
 
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fomit-frame-pointer")
-set(CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-pointer")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fomit-frame-pointer")
+string(APPEND CMAKE_LINKER_FLAGS_RELEASE " -fomit-frame-pointer")
 
 ## Modules ##
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/Data/CMake/)
@@ -424,7 +463,8 @@ if (BUILD_TESTING)
     add_subdirectory(External/Catch2/)
 
     # Pull in catch_discover_tests definition
-    list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/External/Catch2/contrib/")
+    list(APPEND CMAKE_MODULE_PATH
+      "${CMAKE_CURRENT_SOURCE_DIR}/External/Catch2/contrib/")
   endif()
 
   include(Catch)
@@ -451,7 +491,8 @@ endif()
 
 if (NOT range-v3_FOUND)
   add_subdirectory(External/range-v3/)
-  target_compile_definitions(range-v3 INTERFACE RANGES_DISABLE_DEPRECATED_WARNINGS)
+  target_compile_definitions(range-v3 INTERFACE
+    RANGES_DISABLE_DEPRECATED_WARNINGS)
 endif()
 
 add_subdirectory(External/tiny-json/)
@@ -492,11 +533,13 @@ if(ENABLE_WERROR OR ENABLE_STRICT_WERROR)
 endif()
 
 set(FEX_TUNE_COMPILE_FLAGS)
+
 if (NOT TUNE_ARCH STREQUAL "generic")
   set(TUNE_ARCH_STRING "${TUNE_ARCH}")
   if(ARCHITECTURE_arm64)
     set(TUNE_ARCH_STRING "${TUNE_ARCH}+crc")
   endif()
+
   check_cxx_compiler_flag("-march=${TUNE_ARCH_STRING}" COMPILER_SUPPORTS_ARCH_TYPE)
   if(COMPILER_SUPPORTS_ARCH_TYPE)
     list(APPEND FEX_TUNE_COMPILE_FLAGS "-march=${TUNE_ARCH_STRING}")
@@ -513,19 +556,23 @@ if (TUNE_CPU STREQUAL "native")
   if(ARCHITECTURE_arm64)
     if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 999999.0)
       # Clang 12.0 fixed the -mcpu=native bug with mixed big.little implementers
-      # Clang can not currently check for native Apple M1 type in hypervisor. Currently disabled
+      # Clang can not currently check for native Apple M1 type in hypervisor.
+      # Currently disabled
       check_cxx_compiler_flag("-mcpu=native" COMPILER_SUPPORTS_CPU_TYPE)
       if(COMPILER_SUPPORTS_CPU_TYPE)
         list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=native")
       endif()
     else()
-      execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py" "/proc/cpuinfo" "${CMAKE_CXX_COMPILER_VERSION}"
+      execute_process(COMMAND python3
+        "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py"
+        "/proc/cpuinfo" "${CMAKE_CXX_COMPILER_VERSION}"
         OUTPUT_VARIABLE AARCH64_CPU)
 
       string(STRIP ${AARCH64_CPU} AARCH64_CPU)
 
       execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/NeedDisabledSVE.py"
         RESULT_VARIABLE NEEDS_SVE_DISABLED)
+
       if (NEEDS_SVE_DISABLED)
         message(STATUS "Platform has bugged SVE. Disabling")
         set(AARCH64_CPU "cortex-a78")
@@ -547,7 +594,8 @@ elseif (NOT TUNE_CPU STREQUAL "none")
   if(COMPILER_SUPPORTS_CPU_TYPE)
     list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=${TUNE_CPU}")
   else()
-    message(FATAL_ERROR "Trying to compile cpu type '${TUNE_CPU}' but the compiler doesn't support this")
+    message(FATAL_ERROR "Trying to compile cpu type '${TUNE_CPU}' "
+      "but the compiler doesn't support this")
   endif()
 endif()
 
@@ -610,19 +658,38 @@ add_compile_options(-Wall)
 if (BUILD_TESTING)
   message(STATUS "Unit tests are enabled")
 
-  set(TEST_JOB_COUNT "" CACHE STRING "Override number of parallel jobs to use while running tests")
+  set(TEST_JOB_COUNT "" CACHE STRING
+    "Override number of parallel jobs to use while running tests")
   if (TEST_JOB_COUNT)
     message(STATUS "Running tests with ${TEST_JOB_COUNT} jobs")
   elseif(CMAKE_VERSION VERSION_LESS "3.29")
-    execute_process(COMMAND "nproc" OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE TEST_JOB_COUNT)
+    execute_process(COMMAND "nproc"
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      OUTPUT_VARIABLE TEST_JOB_COUNT)
   endif()
+
   set(TEST_JOB_FLAG "-j${TEST_JOB_COUNT}")
 
-  # Runs the whole test suite, with large pages enabled to reduce the cost of fork() in ctest.
+  # When running "ninja tests", defined below, it is useful to intentionally
+  # oversubscribe the CPU cores since many of the tests spend their time idle.
+  if ("$CACHE{TEST_JOB_COUNT}" STREQUAL "")
+    execute_process(COMMAND "nproc"
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      OUTPUT_VARIABLE TESTS_TARGET_JOB_COUNT)
+
+    math(EXPR TESTS_TARGET_JOB_COUNT "${TESTS_TARGET_JOB_COUNT} * 3 / 2")
+    set(TESTS_TARGET_JOB_FLAG "-j${TESTS_TARGET_JOB_COUNT}")
+  else()
+    set(TESTS_TARGET_JOB_FLAG ${TEST_JOB_FLAG})
+  endif()
+
+  # Runs the whole test suite, with large pages enabled to
+  # reduce the cost of fork() in ctest.
   add_custom_target(tests
     WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
     USES_TERMINAL
-    COMMAND ${CMAKE_COMMAND} -E env GLIBC_TUNABLES=glibc.malloc.hugetlb=1 ctest "--progress" "--timeout" "302" ${TEST_JOB_FLAG})
+    COMMAND ${CMAKE_COMMAND} -E env GLIBC_TUNABLES=glibc.malloc.hugetlb=1
+      ctest "--progress" "--timeout" "302" ${TESTS_TARGET_JOB_FLAG})
 endif()
 
 add_subdirectory(External/SoftFloat-3e/)
@@ -642,15 +709,15 @@ if (NOT BUILD_STEAM_SUPPORT)
   add_subdirectory(Data/AppConfig/)
 endif()
 
-# Install the ThunksDB file
-file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.json)
-
-# Any application configuration json file gets installed
-foreach(CONFIG_SRC ${CONFIG_SOURCES})
-  install(FILES ${CONFIG_SRC}
-    DESTINATION ${DATA_DIRECTORY}/
-    COMPONENT Runtime)
-endforeach()
+if (NOT MINGW)
+  # Install the ThunksDB file
+  file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.json)
+  foreach(CONFIG_SRC ${CONFIG_SOURCES})
+    install(FILES ${CONFIG_SRC}
+      DESTINATION ${DATA_DIRECTORY}/
+      COMPONENT Runtime)
+  endforeach()
+endif()
 
 if (BUILD_TESTING)
   add_subdirectory(unittests/)

CodeEmitter/CodeEmitter/ASIMDOps.inl

@@ -768,6 +768,16 @@ public:
       ASIMD2RegMisc(0, size, 0b11101, rd, rn);
     }
   }
+  ///< size is the destination size.
+  ///< source size is the next size up.
+  void bfcvtn(QRegister rd, QRegister rn) {
+    ASIMD2RegMisc(0, SubRegSize::i32Bit, 0b10110, rd.D(), rn.D());
+  }
+  ///< size is the destination size.
+  ///< source size is the next size up.
+  void bfcvtn2(QRegister rd, QRegister rn) {
+    ASIMD2RegMisc(0, SubRegSize::i32Bit, 0b10110, rd, rn);
+  }
 
   template<IsQOrDRegister T>
   void rev32(SubRegSize size, T rd, T rn) {

Data/CMake/LinkerGC.cmake

@@ -3,7 +3,7 @@
 # This applies some common linker options that reduce code size and linking time in Release mode. Namely:
 # --gc-sections: Linktime garbage collection, discards unused sections from the final output
 # --strip-all  : Similar to running `strip`, discards the symbol table from the final output
-# --as-needed  : Only includes libraries that are actually needed in the final output.
+# --as-needed  : Only includes libraries that are actually needed in the final output
 
 macro(LinkerGC target)
   if (CMAKE_BUILD_TYPE MATCHES "RELEASE")

Data/CMake/toolchain_mingw.cmake

@@ -6,14 +6,17 @@ set(CMAKE_CXX_COMPILER ${MINGW_TRIPLE}-clang++)
 set(CMAKE_DLLTOOL ${MINGW_TRIPLE}-dlltool)
 set(CMAKE_AR ${MINGW_TRIPLE}-ar)
 
-# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
-# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
+# Compile everything as static to avoid requiring the MinGW runtime libraries,
+# force page aligned sections so that debug symbols work correctly,
+# and disable loop alignment to workaround an LLVM bug
 # (https://github.com/llvm/llvm-project/issues/47432)
 set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
 set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+
 set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
 set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
 set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+
 set(CMAKE_SYSTEM_NAME Windows)
 set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
 

Data/CMake/toolchain_mingw_x86_32.cmake

@@ -0,0 +1,30 @@
+set(CMAKE_SYSTEM_PROCESSOR i686)
+
+set(LLVM_MINGW "" CACHE FILEPATH "Path to LLVM_MINGW")
+if (LLVM_MINGW)
+  set(LLVM_MINGW_BIN "${LLVM_MINGW}/bin/")
+else()
+  set(LLVM_MINGW_BIN "")
+endif()
+
+set(MINGW_TRIPLE "i686-w64-mingw32")
+
+set(CMAKE_RC_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-windres)
+set(CMAKE_C_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang)
+set(CMAKE_CXX_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang++)
+set(CMAKE_DLLTOOL ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-dlltool)
+set(CMAKE_AR ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-ar)
+
+# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
+# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
+# (https://github.com/llvm/llvm-project/issues/47432)
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_SYSTEM_NAME Windows)
+set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+

Data/CMake/toolchain_mingw_x86_64.cmake

@@ -0,0 +1,29 @@
+set(CMAKE_SYSTEM_PROCESSOR x86_64)
+
+set(LLVM_MINGW "" CACHE FILEPATH "Path to LLVM_MINGW")
+if (LLVM_MINGW)
+  set(LLVM_MINGW_BIN "${LLVM_MINGW}/bin/")
+else()
+  set(LLVM_MINGW_BIN "")
+endif()
+
+set(MINGW_TRIPLE "x86_64-w64-mingw32")
+
+set(CMAKE_RC_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-windres)
+set(CMAKE_C_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang)
+set(CMAKE_CXX_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang++)
+set(CMAKE_DLLTOOL ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-dlltool)
+set(CMAKE_AR ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-ar)
+
+# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
+# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
+# (https://github.com/llvm/llvm-project/issues/47432)
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_SYSTEM_NAME Windows)
+set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

Data/binfmts/CMakeLists.txt

@@ -15,7 +15,8 @@ if (NOT USE_LEGACY_BINFMTMISC)
   configure_file(FEX-x86.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf)
   configure_file(FEX-x86_64.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf)
 
-  install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf
+  install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf
+    ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf
     DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/binfmt.d/
     COMPONENT Runtime)
 else()

External/code-format-helper/requirements_formatting.txt

@@ -4,40 +4,40 @@
 #
 #    pip-compile --generate-hashes --output-file=requirements_formatting.txt --strip-extras requirements_formatting.txt.in
 #
-black==26.3.1 \
-    --hash=sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c \
-    --hash=sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7 \
-    --hash=sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff \
-    --hash=sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b \
-    --hash=sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07 \
-    --hash=sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78 \
-    --hash=sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f \
-    --hash=sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5 \
-    --hash=sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b \
-    --hash=sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e \
-    --hash=sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a \
-    --hash=sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac \
-    --hash=sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a \
-    --hash=sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54 \
-    --hash=sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2 \
-    --hash=sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f \
-    --hash=sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1 \
-    --hash=sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5 \
-    --hash=sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2 \
-    --hash=sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f \
-    --hash=sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1 \
-    --hash=sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c \
-    --hash=sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839 \
-    --hash=sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983 \
-    --hash=sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb \
-    --hash=sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56 \
-    --hash=sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568
+black==26.10.1 \
+    --hash=sha256:03c0ddd93bb392e71209903a691767eb366fe1a76deb9509ccbaae9e1f14bb52 \
+    --hash=sha256:0ce08b367307b0fd91c9dd1d4084e62b05b3055475f951f0f34a46b6e2393b64 \
+    --hash=sha256:182f6c32be38074b16d378498c498b32cb51928178ee611485344972c35ec9c6 \
+    --hash=sha256:1935b32f5326028019856e18cb42b4da63db23765dc84464cec723e0de478a9b \
+    --hash=sha256:19fa8f5beb5e77c54c9c7e21d00cc93ed6c8b6228ee385616906d6befe081143 \
+    --hash=sha256:2520037aa62f8a1454d0811b8f5c88b444445b03a4bfba480d8d220893b64c34 \
+    --hash=sha256:28842f9a8207cc1df6eb983a35a14c5a0dfcd603d214fe82d84bef552afd2e3a \
+    --hash=sha256:289282aa2e09d3162312a3be1788ff21b08e9ea9cc4a81e656024728b32428fb \
+    --hash=sha256:2ffbc023a12d0c729408823b8f10514490bd0baa301d0d4e21a7240249f9507f \
+    --hash=sha256:3414a0c52901964dceabd98c7c56beac0f964115a116ecedcce7247359b14017 \
+    --hash=sha256:4d9a90516db1d99c25dbb20cc0998e0e01531dd903466c7744e56d66f864220a \
+    --hash=sha256:51d5e417e700fe6ec0b0ecdc408c6f6cb5def80328f31f724993d82c6486b746 \
+    --hash=sha256:5cd88fd7b444ca51f3fc883b6f6657ea53a258b0b2eef6d9f2dfcfa17ce0e27b \
+    --hash=sha256:5f9f83beae62437e060dafd53d7f1fc327e3d3494f74d72ee5c2b73eb90fc4e7 \
+    --hash=sha256:70ccbd175b7f6be29d2b727ee7ca6b4c54053df59da653a6df80b175d20a94fa \
+    --hash=sha256:7bdade400bfe24d78a7762896acc2f9a8e1a17fb0fd0536bf6b7c7097cf3eec7 \
+    --hash=sha256:8375962579d537364cc0efa19b1474481915d3a793f9fc0774901814c5e5b5f4 \
+    --hash=sha256:978113a40223a6aaefc17364176a809a320e6b288683841427fff04c6d7b4130 \
+    --hash=sha256:9a0219b29cd70e49f920acb7081e6ce5025c719008447c521d0200dcad93206a \
+    --hash=sha256:b5347d760f0c02bb00dd249384cab71c3bf828b4f68d5b401eb116e0390f147d \
+    --hash=sha256:b6272cfd7e1e8e271f5b0e0207259fe2834687e5cb9b5f620b34a44db9754993 \
+    --hash=sha256:d42dd2fac7c342ae67e64ee99c9532e20b2a84e92c79ed3317fa2ef54c801d93 \
+    --hash=sha256:d5bd3518d8e97138fef295230b1e9804076d69fa4e3594071494a8c68abe6266 \
+    --hash=sha256:d8b3a9074a680b3c5749633714e9ae3992a1e5a23343a97ad61cd9b119b444d2 \
+    --hash=sha256:f6dba8138cdc99061ef07b958ac082d2aa057b6961d1936f9717c350f02bab5f \
+    --hash=sha256:fe85fc4019bee59bc495c0f2a8ee76c5cd02c7015508d94a967ba2376f39a52c \
+    --hash=sha256:ff57f63029aa1353fa8b1b0c8971fd88a6c92dc766608d2eee33ad2deb23270e
     # via
     #   -r requirements_formatting.txt.in
     #   darker
-certifi==2025.7.14 \
-    --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \
-    --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995
+certifi==2026.10.19 \
+    --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae \
+    --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b
     # via
     #   -r requirements_formatting.txt.in
     #   requests
@@ -403,9 +403,9 @@ typing-extensions==4.14.1 \
     --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \
     --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76
     # via pygithub
-urllib3==2.7.0 \
-    --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \
-    --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897
+urllib3==2.8.0 \
+    --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \
+    --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63
     # via
     #   -r requirements_formatting.txt.in
     #   pygithub

External/code-format-helper/requirements_formatting.txt.in

@@ -1,10 +1,10 @@
-black>=26.3.1
+black>=26.10.1
 darker==2.1.1
 PyGithub==2.6.1
 cryptography>=50.0.0
-urllib3>=2.7.0
+urllib3>=2.8.0
 requests>=2.33.0
 idna>=3.15
-certifi>=2024.7.4
+certifi>=2026.10.19
 PyNaCl>=1.6.2
 PyJWT>=2.15.1

FEXCore/CMakeLists.txt

@@ -33,9 +33,11 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/git_version.h.in
 include_directories(${CMAKE_BINARY_DIR}/generated)
 
 # Disable strict aliasing for all build modes
-# See discussion in https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
+# See discussion in
+# https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
 # for background context.
-add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fno-strict-aliasing> $<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>)
+add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fno-strict-aliasing>
+  $<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>)
 
 add_subdirectory(Source/)
 

FEXCore/Source/CMakeLists.txt

@@ -104,9 +104,13 @@ if (ENABLE_ZYDIS)
 endif()
 
 if (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)
-  list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
+  list(APPEND DEFINES
+    "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all))"
+    "-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
 else()
-  list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=;-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0")
+  list(APPEND DEFINES
+    "-DFEXCORE_PRESERVE_ALL_ATTR="
+    "-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0")
 endif()
 
 set(LIBS fmt::fmt xxHash::xxhash FEXHeaderUtils CodeEmitter cephes_128bit)
@@ -187,8 +191,8 @@ add_custom_command(
   OUTPUT "${OUTPUT_MAN_NAME}"
   DEPENDS "${INPUT_CONFIG_NAME}"
   DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py"
-  COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py" "${INPUT_CONFIG_NAME}" "${OUTPUT_CONFIG_NAME}" "${OUTPUT_MAN_NAME}"
-  "${OUTPUT_CONFIG_OPTION_NAME}")
+  COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py"
+    "${INPUT_CONFIG_NAME}" "${OUTPUT_CONFIG_NAME}" "${OUTPUT_MAN_NAME}" "${OUTPUT_CONFIG_OPTION_NAME}")
 
 add_custom_command(
   OUTPUT "${OUTPUT_MAN_NAME_COMPRESS}"
@@ -284,8 +288,8 @@ function(AddLibrary Name Type)
   add_library(${Name} ${Type} $<TARGET_OBJECTS:${PROJECT_NAME}_object>)
   set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
 
-  # During generation of the import library (dll.a), MinGW needs some extra symbols from libraries
-  # such as fmt, which are propagated by FEXCore_Base. Wonderful.
+  # During generation of the import library (dll.a), MinGW needs some extra
+  # symbols from libraries such as fmt, which are propagated by FEXCore_Base
   if (MINGW)
     target_link_libraries(${Name} PRIVATE FEXCore_Base)
   endif()
@@ -311,7 +315,8 @@ if (ENABLE_FEX_ALLOCATOR)
   target_include_directories(JemallocLibs PRIVATE "${PROJECT_SOURCE_DIR}/include/")
 endif()
 if (ENABLE_JEMALLOC_GLIBC_ALLOC)
-  set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1)
+  set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp
+    PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1)
   target_link_libraries(JemallocLibs INTERFACE FEX_jemalloc_glibc)
 endif()
 

FEXCore/Source/Interface/Config/Config.json.in

@@ -42,6 +42,19 @@
           "Enable expensive validation when loading code caches"
         ]
       },
+      "JITFeatures": {
+        "Type": "strenum",
+        "Default": "FEXCore::Config::JITFeatures::OFF",
+        "AffectsCodeGen": "true",
+        "Enums": {
+          "ENABLEX87FIP": "enablex87fip"
+        },
+        "Desc": [
+          "Allows controlling JIT features in the unified location",
+          "\toff: Default JIT features.",
+          "\tenablex87fip: Turns on the x87 FIP emulation feature"
+        ]
+      },
       "HostFeatures": {
         "Type": "strenum",
         "Default": "FEXCore::Config::HostFeatures::OFF",
@@ -95,7 +108,11 @@
           "ENABLEI8MM": "enablei8mm",
           "DISABLEI8MM": "disablei8mm",
           "ENABLEDOTPROD": "enabledotprod",
-          "DISABLEDOTPROD": "disabledotprod"
+          "DISABLEDOTPROD": "disabledotprod",
+          "ENABLEBF16": "enablebf16",
+          "DISABLEBF16": "disablebf16",
+          "ENABLETPIDRRO": "enabletpidrro",
+          "DISABLETPIDRRO": "disabletpidrro"
         },
         "Desc": [
           "Allows controlling of the CPU features in the JIT.",
@@ -122,7 +139,9 @@
           "\t{enable,disable}sse4a: Will force enable or disable SSE4a even if the host doesn't support it",
           "\t{enable,disable}mops: Will force enable or disable FEAT_MOPS even if the host doesn't support it",
           "\t{enable,disable}i8mm: Will force enable or disable i8mm even if the host doesn't support it",
-          "\t{enable,disable}dotprod: Will force enable or disable dotprod even if the host doesn't support it"
+          "\t{enable,disable}dotprod: Will force enable or disable dotprod even if the host doesn't support it",
+          "\t{enable,disable}bf16: Will force enable or disable bf16 even if the host doesn't support it",
+          "\t{enable,disable}tpidrro: Will force enable or disable tpidrro for CPU index even if the host doesn't support it"
         ]
       },
       "SmallTSCScale": {
@@ -354,6 +373,15 @@
           "Can potentially introduce more stutters, more likely the higher the number.",
           "Don't have this number larger than the increase count!"
         ]
+      },
+      "UndefinedFlagAcidTest": {
+        "Type": "bool",
+        "Default": "false",
+        "AffectsCodeGen": "true",
+        "Desc": [
+          "Ensures that all undefined flags generate consistent results",
+          "This is a useful test to determine if undefined flags are causing problems"
+        ]
       }
     },
     "Debug": {

FEXCore/Source/Interface/Context/Context.h

@@ -371,6 +371,7 @@ public:
     FEX_CONFIG_OPT(SoftwareRNG, SOFTWARERNG);
     FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
     FEX_CONFIG_OPT(MonoHacks, MONOHACKS);
+    FEX_CONFIG_OPT(JITFeatures, JITFEATURES);
   } Config;
 
   bool SoftwareRNGEnabled() const {

FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp

@@ -665,11 +665,11 @@ void Arm64Emitter::FillSpecialRegs(ARMEmitter::Register TmpReg, ARMEmitter::Regi
     // We don't bother spilling these in SpillStaticRegs,
     // since all that matters is we restore them on a fill.
     // It's not a concern if they get trounced by something else.
-    if (EmitterCTX->HostFeatures.SupportsSVE256) {
+    if (EmitterCTX->HostFeatures.SupportsSVE256()) {
       ptrue(ARMEmitter::SubRegSize::i8Bit, PRED_TMP_32B, ARMEmitter::PredicatePattern::SVE_VL32);
     }
 
-    if (EmitterCTX->HostFeatures.SupportsSVE128) {
+    if (EmitterCTX->HostFeatures.SupportsSVE128()) {
       ptrue(ARMEmitter::SubRegSize::i8Bit, PRED_TMP_16B, ARMEmitter::PredicatePattern::SVE_VL16);
     }
 
@@ -739,7 +739,7 @@ void Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, SpillStaticRegOp
   }
 
   if (Options.FPRs) {
-    if (EmitterCTX->HostFeatures.SupportsAVX && EmitterCTX->HostFeatures.SupportsSVE256) {
+    if (EmitterCTX->HostFeatures.SupportsAVX() && EmitterCTX->HostFeatures.SupportsSVE256()) {
       for (size_t i = 0; i < StaticFPRegisters.size(); i++) {
         const auto Reg = StaticFPRegisters[i];
 
@@ -826,7 +826,7 @@ void Arm64Emitter::FillStaticRegs(FillStaticRegOptions Options) {
   FillSpecialRegs(TmpReg, TmpReg2, {.SetFIZ = true, .SetPredRegs = Options.FPRs});
 
   if (Options.FPRs) {
-    if (EmitterCTX->HostFeatures.SupportsAVX && EmitterCTX->HostFeatures.SupportsSVE256) {
+    if (EmitterCTX->HostFeatures.SupportsAVX() && EmitterCTX->HostFeatures.SupportsSVE256()) {
       for (size_t i = 0; i < StaticFPRegisters.size(); i++) {
         const auto Reg = StaticFPRegisters[i];
         if (((1U << Reg.Idx()) & Options.FPRFillMask) != 0) {
@@ -990,7 +990,7 @@ void Arm64Emitter::PopGeneralRegisters(std::span<const ARMEmitter::Register> Reg
 }
 
 size_t Arm64Emitter::PushDynamicRegs(ARMEmitter::Register TmpReg) {
-  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
+  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
   const auto GPRSize = GeneralRegistersNotPreserved.size() * Core::CPUState::GPR_REG_SIZE;
   const auto FPRRegSize = CanUseSVE256 ? 32 : 16;
   const auto FPRSize = GeneralFPRegisters.size() * FPRRegSize;
@@ -1013,7 +1013,7 @@ size_t Arm64Emitter::PushDynamicRegs(ARMEmitter::Register TmpReg) {
 }
 
 void Arm64Emitter::PopDynamicRegs() {
-  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
+  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
 
   // Pop vectors first
   PopVectorRegisters(CanUseSVE256, GeneralFPRegisters);
@@ -1023,7 +1023,7 @@ void Arm64Emitter::PopDynamicRegs() {
 }
 
 size_t Arm64Emitter::SpillForPreserveAllABICall(ARMEmitter::Register TmpReg, bool FPRs) {
-  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
+  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
   const auto FPRRegSize = CanUseSVE256 ? 32 : 16;
 
   std::span<const ARMEmitter::Register> DynamicGPRs {};
@@ -1078,7 +1078,7 @@ size_t Arm64Emitter::SpillForPreserveAllABICall(ARMEmitter::Register TmpReg, boo
 }
 
 void Arm64Emitter::FillForPreserveAllABICall(bool FPRs) {
-  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
+  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
 
   std::span<const ARMEmitter::Register> DynamicGPRs {};
   std::span<const ARMEmitter::VRegister> DynamicFPRs {};
@@ -1129,4 +1129,9 @@ void Arm64Emitter::Align16B() {
   }
 }
 
+ARMEmitter::Register Arm64Emitter::GetX86GPRMapping(FEXCore::X86State::X86Reg Reg) {
+  LOGMAN_THROW_A_FMT(Reg <= FEXCore::X86State::X86Reg::REG_R15, "Incorrect register");
+  return StaticRegisters[Reg];
+}
+
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h

@@ -235,6 +235,8 @@ protected:
 
   void Align16B();
 
+  ARMEmitter::Register GetX86GPRMapping(FEXCore::X86State::X86Reg Reg);
+
 #ifdef VIXL_SIMULATOR
   // Generates a vixl simulator runtime call.
   //

FEXCore/Source/Interface/Core/CPUBackend.h

@@ -115,7 +115,7 @@ namespace CPU {
      * @return Information about the compiled code block.
      */
     [[nodiscard]]
-    virtual CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
+    virtual CompiledCode EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                      FEXCore::Core::DebugData* DebugData, bool CheckTF) = 0;
 
     virtual CompiledCode LoadCachedCode(std::span<const uint8_t> HostBytes) {

FEXCore/Source/Interface/Core/CPUID.cpp

@@ -89,6 +89,18 @@ namespace ProductNames {
   static const char ARM_Blizzard_M2Pro[] = "Apple Blizzard (M2 Pro)";
   static const char ARM_Avalanche_M2Max[] = "Apple Avalanche (M2 Max)";
   static const char ARM_Blizzard_M2Max[] = "Apple Blizzard (M2 Max)";
+  static const char ARM_Sawtooth_M3[] = "Apple Sawtooth (M3)";
+  static const char ARM_Everest_M3[] = "Apple Everest (M3)";
+  static const char ARM_Sawtooth_M3Pro[] = "Apple Sawtooth (M3 Pro)";
+  static const char ARM_Everest_M3Pro[] = "Apple Everest (M3 Pro)";
+  static const char ARM_Sawtooth_M3Max[] = "Apple Sawtooth (M3 Max)";
+  static const char ARM_Everest_M3Max[] = "Apple Everest (M3 Max)";
+  static const char ARM_DonanP_M4[] = "Apple DonanP (M4)";
+  static const char ARM_DonanE_M4[] = "Apple DonanE (M4)";
+  static const char ARM_TahitiP_A18[] = "Apple TahitiP (A18)";
+  static const char ARM_TahitiE_A18[] = "Apple TahitiE (A18)";
+  static const char ARM_SotraP_M5[] = "Apple SotraP (M5)";
+  static const char ARM_SotraE_M5[] = "Apple SotraE (M5)";
   static const char ARM_AppleSilicon[] = "Apple Silicon";
 
   static const char ARM_ORYON_1[] = "Oryon-1";
@@ -187,11 +199,17 @@ void CPUIDEmu::SetupHostHybridFlag() {
   // CPU priority order
   // This is mostly arbitrary but will sort by some sort of CPU priority by performance
   // Relative list so things they will commonly end up in big.little configurations sort of relate
-  static constexpr std::array<CPUMIDR, 68> CPUMIDRs = {{
+  static constexpr std::array<CPUMIDR, 80> CPUMIDRs = {{
     // Typically big CPU cores
     {0x51, 0x002, 1, ProductNames::ARM_ORYON_3}, // Qualcomm Oryon-3
     {0x51, 0x001, 1, ProductNames::ARM_ORYON_1}, // Qualcomm Oryon-1
 
+    {0x61, 0x065, 1, ProductNames::ARM_SotraP_M5},       // Apple SotraP (M5)
+    {0x61, 0x061, 1, ProductNames::ARM_TahitiP_A18},     // Apple TahitiP (A18)
+    {0x61, 0x053, 1, ProductNames::ARM_DonanP_M4},       // Apple DonanP (M4)
+    {0x61, 0x048, 1, ProductNames::ARM_Sawtooth_M3Max},  // Apple Sawtooth (M3 Max)
+    {0x61, 0x044, 1, ProductNames::ARM_Sawtooth_M3Pro},  // Apple Sawtooth (M3 Pro)
+    {0x61, 0x042, 1, ProductNames::ARM_Sawtooth_M3},     // Apple Sawtooth (M3)
     {0x61, 0x039, 1, ProductNames::ARM_Avalanche_M2Max}, // Apple Avalanche (M2 Max)
     {0x61, 0x035, 1, ProductNames::ARM_Avalanche_M2Pro}, // Apple Avalanche (M2 Pro)
     {0x61, 0x033, 1, ProductNames::ARM_Avalanche_M2},    // Apple Avalanche (M2)
@@ -248,6 +266,12 @@ void CPUIDEmu::SetupHostHybridFlag() {
     {0x41, 0xd07, 1, ProductNames::ARM_A57}, // A57
 
     // Typically Little CPU cores
+    {0x61, 0x064, 1, ProductNames::ARM_SotraE_M5},      // Apple SotraE (M5)
+    {0x61, 0x060, 1, ProductNames::ARM_TahitiE_A18},    // Apple TahitiE (A18)
+    {0x61, 0x052, 1, ProductNames::ARM_DonanE_M4},      // Apple DonanE (M4)
+    {0x61, 0x049, 0, ProductNames::ARM_Everest_M3Max},  // Apple Everest (M3 Max)
+    {0x61, 0x045, 0, ProductNames::ARM_Everest_M3Pro},  // Apple Everest (M3 Pro)
+    {0x61, 0x043, 0, ProductNames::ARM_Everest_M3},     // Apple Everest (M3)
     {0x61, 0x038, 0, ProductNames::ARM_Blizzard_M2Max}, // Apple Blizzard (M2 Max)
     {0x61, 0x034, 0, ProductNames::ARM_Blizzard_M2Pro}, // Apple Blizzard (M2 Pro)
     {0x61, 0x032, 0, ProductNames::ARM_Blizzard_M2},    // Apple Blizzard (M2)
@@ -419,7 +443,7 @@ void CPUIDEmu::SetupHostHybridFlag() {}
 
 
 void CPUIDEmu::SetupFeatures() {
-  if (CTX->HostFeatures.SupportsAVX) {
+  if (CTX->HostFeatures.SupportsAVX()) {
     XCR0 |= XCR0_AVX;
   }
 
@@ -673,6 +697,9 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
     const uint32_t SupportsWFXT = CTX->HostFeatures.SupportsWFXT;
 
     const uint32_t SupportsRAND = CTX->HostFeatures.SupportsRAND || CTX->SoftwareRNGEnabled();
+    // When the X87 FIP feature is enabled, we update the state pointers on every instruction.
+    // We don't even update them on exception so this isn't quite true the other way anyway.
+    const uint32_t X87DataPtrExcept = (CTX->Config.JITFeatures & FEXCore::Config::JITFeatures::ENABLEX87FIP) == 0;
 
     // Number of subfunctions
     // TODO: For now, subfunction 1 only exposes AVX-VNNI so we make it conditional
@@ -684,7 +711,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (SupportsAVX() << 3) |           // BMI1
               (0 << 4) |                       // Intel Hardware Lock Elison
               (SupportsAVX() << 5) |           // AVX2 support
-              (1 << 6) |                       // FPU data pointer updated only on exception
+              (X87DataPtrExcept << 6) |        // FPU data pointer updated only on exception
               (1 << 7) |                       // SMEP support
               (SupportsAVX() << 8) |           // BMI2
               (SupportsEnhancedREPMOVS << 9) | // Enhanced REP MOVSB/STOSB
@@ -758,7 +785,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 11) |                  // rtm-always-abort
               (0 << 12) |                  // Reserved
               (0 << 13) |                  // TSX Force Abort (TSX will force abort if attempted)
-              (0 << 14) |                  // SERIALIZE instruction
+              (1 << 14) |                  // SERIALIZE instruction
               ((Hybrid ? 1U : 0U) << 15) | // Hybrid
               (0 << 16) |                  // TSXLDTRK (TSX Suspend load address tracking) - Allows untracked memory loads inside TSX region
               (0 << 17) |                  // Reserved
@@ -777,6 +804,8 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 30) |                  // Arch capabilities - MSR module specific
               (0 << 31);                   // SSBD - Speculative Store Bypass Disable
   } else if (Leaf == 1) {
+    const uint32_t SupportsAVXConvert = SupportsAVX() && CTX->HostFeatures.SupportsBF16;
+
     Res.eax = (0U << 0) |              // SHA512
               (0U << 1) |              // SM3
               (0U << 2) |              // SM4
@@ -825,31 +854,31 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0U << 5);  // MSR_IMM
 
     // Bits 25-31 also reserved.
-    Res.edx = (0U << 0) |  // Reserved
-              (0U << 1) |  // Reserved
-              (0U << 2) |  // Reserved
-              (0U << 3) |  // Reserved
-              (0U << 4) |  // AVX_VNNI_INT8
-              (0U << 5) |  // AVX_NE_CONVERT
-              (0U << 6) |  // Reserved
-              (0U << 7) |  // Reserved
-              (0U << 8) |  // AMX_COMPLEX
-              (0U << 9) |  // Reserved
-              (0U << 10) | // AVX_VNNI_INT16
-              (0U << 11) | // Reserved
-              (0U << 12) | // Reserved
-              (0U << 13) | // UTMR (User-timer events)
-              (0U << 14) | // PREFETCHI
-              (0U << 15) | // USER_MSR
-              (0U << 16) | // Reserved
-              (0U << 17) | // UIRET_UIF
-              (0U << 18) | // CET_SSS
-              (0U << 19) | // AVX10
-              (0U << 20) | // Reserved
-              (0U << 21) | // APX_F
-              (0U << 22) | // SEC-TEE_ATTESTATION
-              (0U << 23) | // MWAIT
-              (0U << 24);  // SLSM (Static LSM)
+    Res.edx = (0U << 0) |                 // Reserved
+              (0U << 1) |                 // Reserved
+              (0U << 2) |                 // Reserved
+              (0U << 3) |                 // Reserved
+              (0U << 4) |                 // AVX_VNNI_INT8
+              (SupportsAVXConvert << 5) | // AVX_NE_CONVERT
+              (0U << 6) |                 // Reserved
+              (0U << 7) |                 // Reserved
+              (0U << 8) |                 // AMX_COMPLEX
+              (0U << 9) |                 // Reserved
+              (0U << 10) |                // AVX_VNNI_INT16
+              (0U << 11) |                // Reserved
+              (0U << 12) |                // Reserved
+              (0U << 13) |                // UTMR (User-timer events)
+              (0U << 14) |                // PREFETCHI
+              (0U << 15) |                // USER_MSR
+              (0U << 16) |                // Reserved
+              (0U << 17) |                // UIRET_UIF
+              (0U << 18) |                // CET_SSS
+              (0U << 19) |                // AVX10
+              (0U << 20) |                // Reserved
+              (0U << 21) |                // APX_F
+              (0U << 22) |                // SEC-TEE_ATTESTATION
+              (0U << 23) |                // MWAIT
+              (0U << 24);                 // SLSM (Static LSM)
   } else if (Leaf == 2) {
     // All bits are reserved except for EDX
     Res.eax = 0;
@@ -1133,14 +1162,14 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf) con
 
 FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h(uint32_t Leaf, uint32_t CPU) const {
   FEXCore::CPUID::FunctionResults Res {};
-  auto& Data = PerCPUData[CPU];
+  auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
   memcpy(&Res, Data.ProductName, std::min(strlen(Data.ProductName), sizeof(FEXCore::CPUID::FunctionResults)));
   return Res;
 }
 
 FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf, uint32_t CPU) const {
   FEXCore::CPUID::FunctionResults Res {};
-  auto& Data = PerCPUData[CPU];
+  auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
   const auto RemainingStringSize = std::max<ssize_t>(0, strlen(Data.ProductName) - 16);
   memcpy(&Res, Data.ProductName + 16, std::min<size_t>(RemainingStringSize, sizeof(FEXCore::CPUID::FunctionResults)));
   return Res;
@@ -1148,7 +1177,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf, uin
 
 FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf, uint32_t CPU) const {
   FEXCore::CPUID::FunctionResults Res {};
-  auto& Data = PerCPUData[CPU];
+  auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
   const auto RemainingStringSize = std::max<ssize_t>(0, strlen(Data.ProductName) - 32);
   memcpy(&Res, Data.ProductName + 32, std::min<size_t>(RemainingStringSize, sizeof(FEXCore::CPUID::FunctionResults)));
   return Res;

FEXCore/Source/Interface/Core/CPUID.h

@@ -119,6 +119,7 @@ private:
   bool Hybrid {};
   uint32_t Cores {};
   FEX_CONFIG_OPT(HideHypervisorBit, HIDEHYPERVISORBIT);
+  FEX_CONFIG_OPT(JITFeatures, JITFEATURES);
 
   // XFEATURE_ENABLED_MASK
   // Mask that configures what features are enabled on the CPU.

FEXCore/Source/Interface/Core/Core.cpp

@@ -264,8 +264,8 @@ uint32_t ContextImpl::ReconstructCompactedEFLAGS(FEXCore::Core::InternalThreadSt
 void ContextImpl::ReconstructXMMRegisters(const FEXCore::Core::InternalThreadState* Thread, __uint128_t* XMM_Low, __uint128_t* YMM_High) {
   const size_t MaximumRegisters = Config.Is64BitMode ? FEXCore::Core::CPUState::NUM_XMMS : 8;
 
-  if (YMM_High != nullptr && HostFeatures.SupportsAVX) {
-    const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256;
+  if (YMM_High != nullptr && HostFeatures.SupportsAVX()) {
+    const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256();
 
     if (SupportsConvergedRegisters) {
       ///< Output wants to de-interleave
@@ -288,8 +288,8 @@ void ContextImpl::ReconstructXMMRegisters(const FEXCore::Core::InternalThreadSta
 
 void ContextImpl::SetXMMRegistersFromState(FEXCore::Core::InternalThreadState* Thread, const __uint128_t* XMM_Low, const __uint128_t* YMM_High) {
   const size_t MaximumRegisters = Config.Is64BitMode ? FEXCore::Core::CPUState::NUM_XMMS : 8;
-  if (YMM_High != nullptr && HostFeatures.SupportsAVX) {
-    const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256;
+  if (YMM_High != nullptr && HostFeatures.SupportsAVX()) {
+    const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256();
 
     if (SupportsConvergedRegisters) {
       ///< Output wants to de-interleave
@@ -715,13 +715,19 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
               LogMan::Msg::EFmt("Invalid or Unknown instruction: {} 0x{:x}", TableInfo->Name ?: "UND", Block.Entry - GuestRIP);
             }
 
-            if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::INVALID_INST ||
-                Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BAD_RELOCATION) {
+            if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_0 ||
+                Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_1 ||
+                Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_2 ||
+                Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_3) {
+              Thread->OpDispatcher->BreakpointOp(DecodedInfo, FEXCore::ToUnderlying(Block.BlockStatus) -
+                                                                FEXCore::ToUnderlying(Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_0));
+            } else if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::INVALID_INST ||
+                       Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BAD_RELOCATION) {
               Thread->OpDispatcher->InvalidOp(DecodedInfo);
             } else if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::UNIMPLEMENTED_INST) {
               Thread->OpDispatcher->UnimplementedOp(DecodedInfo);
             } else {
-              Thread->OpDispatcher->NoExecOp(DecodedInfo);
+              Thread->OpDispatcher->NoExecOrPartialOp(DecodedInfo);
             }
           }
 
@@ -832,7 +838,7 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
   // If the trap flag is set we generate single instruction blocks that each check to generate a single step exception.
   bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];
 
-  auto CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), TFSet);
+  auto CompiledCode = Thread->CPUBackend->EmitJITCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), TFSet);
 
   // Release the IR
   Thread->OpDispatcher->DelayedDisownBuffer();

FEXCore/Source/Interface/Core/DiskCache.cpp

@@ -409,17 +409,24 @@ namespace DiskCache {
       uint64_t HostFeaturesHash;
     } MachineBucketData = {FormatVersion, HostFeatureHash.HostFeaturesHash};
 
-    fextl::vector<uint8_t> BucketBytes(sizeof(MachineBucketData) + (sizeof(uint8_t) * 2) + SerializedConfig.size());
+    uint8_t ProcessFeaturesHash {};
+    memcpy(&ProcessFeaturesHash, &HostFeatureHash.ProcessFeaturesHash, sizeof(ProcessFeaturesHash));
+    LogMan::Msg::IFmt("Disk Cache features:");
+    LogMan::Msg::IFmt("HostFeatures Hash: 0x{:016x}", HostFeatureHash.HostFeaturesHash);
+    LogMan::Msg::IFmt("ProcessFeaturesHash: 0x{:02x}", ProcessFeaturesHash);
+
+    fextl::vector<uint8_t> BucketBytes(sizeof(MachineBucketData) + sizeof(HostFeatureHash.ProcessFeaturesHash) + SerializedConfig.size());
     memcpy(BucketBytes.data(), &MachineBucketData, sizeof(MachineBucketData));
 
     // 64-bit mode and HostType is in the ProcessBucket hash only instead of the MachineBucketHash
     // These effect code-gen, but they aren't part of the MachineBucketHash, as it comes from Process state.
-    BucketBytes[sizeof(MachineBucketData)] = CTX->Config.Is64BitMode;
-    BucketBytes[sizeof(MachineBucketData) + 1] = FEXCore::ToUnderlying(HostFeatureHash.HostType);
-    memcpy(BucketBytes.data() + sizeof(MachineBucketData) + 2, SerializedConfig.data(), SerializedConfig.size());
+    memcpy(&BucketBytes[sizeof(MachineBucketData)], &HostFeatureHash.ProcessFeaturesHash, sizeof(HostFeatureHash.ProcessFeaturesHash));
+    memcpy(BucketBytes.data() + sizeof(MachineBucketData) + sizeof(HostFeatureHash.ProcessFeaturesHash), SerializedConfig.data(),
+           SerializedConfig.size());
 
     uint64_t MachineBucketHash = XXH3_64bits(BucketBytes.data(), sizeof(MachineBucketData));
-    uint64_t ProcessBucketHash = XXH3_64bits(BucketBytes.data() + sizeof(MachineBucketData), 2 + SerializedConfig.size());
+    uint64_t ProcessBucketHash =
+      XXH3_64bits(BucketBytes.data() + sizeof(MachineBucketData), sizeof(HostFeatureHash.ProcessFeaturesHash) + SerializedConfig.size());
     BucketHash.high64 = MachineBucketHash;
     BucketHash.low64 = ProcessBucketHash;
 
@@ -1026,9 +1033,8 @@ namespace DiskCache {
     const size_t TotalSize = GuestCodeOffset + GuestCode.size();
 
     // we'll copy everything into here and pass it to the Writer, then return to caller quickly
-    fextl::vector<uint8_t> Blob;
-    Blob.resize(TotalSize);
-    uint8_t* BlobData = Blob.data();
+    auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(TotalSize);
+    uint8_t* BlobData = BlobRef->data();
 
     BlobFixedHeader Header {
       .GuestSize = (uint32_t)GuestCode.size(),
@@ -1182,7 +1188,6 @@ namespace DiskCache {
       MakeLookupKey(Thread, GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));
 
     // blob done, publish to index as in-memory for now, flush to disk below
-    auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(std::move(Blob));
     struct IndexEntry NewEntry {nullptr, 0, (uint32_t)TotalSize, Header.GuestSize, Header.GuestHash, BlobRef};
     NewEntry.GuestExtents = ExactGuestCodeExtents;
     {

FEXCore/Source/Interface/Core/DiskCache.h

@@ -249,7 +249,7 @@ namespace DiskCache {
   // The current version of the diskcache.
   // This must be changed any time codegen changes occur!
   // Be aware of the impact of changing this frequently!
-  static constexpr uint16_t FormatVersion = 30;
+  static constexpr uint16_t FormatVersion = 43;
 
   static constexpr uint32_t LOOKUP_KEY_MAX_BUCKET_DEPTH = 500;
 

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -417,6 +417,7 @@ void Dispatcher::EmitDispatcher() {
     brk(0);
   }
 
+  ARMEmitter::BackwardLabel GuestSIGSEGVLabelPostSpill;
   {
     // Guest Overflow handler
     // Needs to be distinct from the SignalHandlerReturnAddress
@@ -424,6 +425,8 @@ void Dispatcher::EmitDispatcher() {
 
     SpillStaticRegs(TMP1);
 
+    (void)Bind(&GuestSIGSEGVLabelPostSpill);
+
     // hlt/udf = SIGILL
     // brk = SIGTRAP
     // ??? = SIGSEGV
@@ -439,6 +442,26 @@ void Dispatcher::EmitDispatcher() {
     }
   }
 
+  {
+    // Faultless synchronous signal.
+    // Some games abuse SIGILL/SIGTRAP/SIGSEGV heavily and the fault overhead is visible.
+    // Instead of double faulting, clean up state and jump to the frontend to handle this.
+    // `FEXCore::Core::CpuStateFrame::SynchronousFaultData` needs to be setup at this point.
+    GuestSignal_SynchronousFault = GetCursorAddress<uint64_t>();
+
+    SpillStaticRegs(TMP1);
+
+    ldr(ARMEmitter::XReg::x0, &l_CTX);
+    mov(ARMEmitter::XReg::x1, STATE);
+    ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.SynchronousExceptionHandler));
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
+    } else {
+      br(ARMEmitter::Reg::r2);
+    }
+  }
+
   {
     ThreadPauseHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
     SpillStaticRegs(TMP1);
@@ -514,6 +537,28 @@ void Dispatcher::EmitDispatcher() {
     (void)b(&LoopTop);
   }
 
+  auto EmitL1Lookup = [&](int rip_offset = 0) {
+    // L1 lookup after syscall. Mirrors the implementation in `ExitFunction`.
+    // Most syscall instructions just return to the following RIP, so it is highly likely to be in the L1 in tight loops.
+    ldr(TMP3, STATE_PTR(CpuStateFrame, State.rip));
+    ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.L1Pointer));
+
+    if (rip_offset) {
+      add(ARMEmitter::Size::i64Bit, TMP3, TMP3, rip_offset);
+    }
+
+    // Calculate (tmp1 + ((ripreg & L1_ENTRIES_MASK) << 4)) for the address
+    // L1Mask is pre-shifted.
+    and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, TMP3, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));
+    add(TMP1, TMP1, TMP2);
+
+    ldp<ARMEmitter::IndexType::OFFSET>(TMP2, TMP1, TMP1, 0);
+
+    sub(TMP1, TMP1, TMP3);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);
+    br(TMP2);
+  };
+
   {
     // All dynamic and static registers are spilled coming in to this handler.
     // It's also the end of block and RIP might have changed, so we jump directly to the top of the loop.
@@ -546,8 +591,90 @@ void Dispatcher::EmitDispatcher() {
     // We can safely claim we are no longer in a syscall
     str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
 
-    // Now go back to the regular dispatcher loop
-    (void)b(&LoopTop);
+    EmitL1Lookup();
+  }
+
+  {
+    // All dynamic and static registers are spilled coming in to this handler.
+    // It's also the end of block and RIP might have changed, so we jump directly to the top of the loop.
+    ThreadDispatchCPUIDHandler = GetCursorAddress<uint64_t>();
+
+    // CPUID behaves like an ISB.
+    isb();
+
+    auto RAXReg = GetX86GPRMapping(FEXCore::X86State::REG_RAX);
+    auto RBXReg = GetX86GPRMapping(FEXCore::X86State::REG_RBX);
+    auto RCXReg = GetX86GPRMapping(FEXCore::X86State::REG_RCX);
+    auto RDXReg = GetX86GPRMapping(FEXCore::X86State::REG_RDX);
+
+    ARMEmitter::ForwardLabel SetupCPUIDFault {};
+    ldr(TMP2, STATE, offsetof(FEXCore::Core::CPUState, CoreFlags));
+    (void)tbz(TMP2, FEXCore::ilog2(FEXCore::ToUnderlying(FEXCore::Core::CoreFlagsEnum::CPUID_ENABLED)), &SetupCPUIDFault);
+
+    // TMP1 passes the instruction size. Usually only 2-bytes but prefixes can make it 15.
+    ldr(TMP2, STATE_PTR(CpuStateFrame, State.rip));
+    add(TMP2, TMP2, TMP1);
+    str(TMP2, STATE_PTR(CpuStateFrame, State.rip));
+
+    mov(ARMEmitter::XReg::x1, RAXReg.X());
+    mov(ARMEmitter::XReg::x2, RCXReg.X());
+
+    // x0 = CPUID Handler
+    // x1 = CPUID Function (RAX)
+    // x2 = CPUID Leaf (RCX)
+    ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));
+    ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDFunction));
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
+    } else {
+      blr(ARMEmitter::Reg::r3);
+    }
+
+    // Results are in x0, x1
+    auto Res0 = ARMEmitter::Reg::r0;
+    auto Res1 = ARMEmitter::Reg::r1;
+
+    if (!TMP_ABIARGS) {
+      // SRA Overlaps, need to move.
+      mov(ARMEmitter::Size::i64Bit, TMP1, Res0);
+      mov(ARMEmitter::Size::i64Bit, TMP2, Res1);
+      Res0 = TMP1;
+      Res1 = TMP2;
+    }
+
+    FillStaticRegs();
+
+    // Results need to be moved to 32-bit EAX, ECX, EBX, EDX.
+    mov(ARMEmitter::Size::i32Bit, RAXReg, Res0);
+    mov(ARMEmitter::Size::i32Bit, RCXReg, Res1);
+    ubfx(ARMEmitter::Size::i64Bit, RBXReg, Res0, 32, 32);
+    ubfx(ARMEmitter::Size::i64Bit, RDXReg, Res1, 32, 32);
+
+    EmitL1Lookup();
+
+    // CPUID fault handling
+    (void)Bind(&SetupCPUIDFault);
+
+    constexpr Core::CpuStateFrame::SynchronousFaultDataStruct State = {
+      .FaultToTopAndGeneratedException = 1,
+      .Signal = Core::FAULT_SIGSEGV,
+      .TrapNo = X86State::X86_TRAPNO_GP,
+      .si_code = 0x80,
+      .err_code = 0,
+      .FaultAddress = 0, // Comes from register.
+    };
+
+    uint64_t Constant {};
+    memcpy(&Constant, &State, sizeof(uint64_t));
+
+    LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
+
+    str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
+    ldr(TMP2, STATE_PTR(CpuStateFrame, State.rip));
+    str(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
+
+    (void)b(&GuestSIGSEGVLabelPostSpill);
   }
 
   {
@@ -572,44 +699,6 @@ void Dispatcher::EmitDispatcher() {
     (void)b(&LoopTop);
   }
 
-  auto EmitLongALUOpHandler = [&](auto R, auto Offset) {
-    auto Address = GetCursorAddress<uint64_t>();
-
-    PushDynamicRegs(TMP4);
-    SpillStaticRegs(TMP4);
-
-    if (!TMP_ABIARGS) {
-      mov(ARMEmitter::XReg::x0, TMP1);
-      mov(ARMEmitter::XReg::x1, TMP2);
-      mov(ARMEmitter::XReg::x2, TMP3);
-    }
-
-    ldr(ARMEmitter::XReg::x3, R, Offset);
-    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-      GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
-    } else {
-      blr(ARMEmitter::Reg::r3);
-    }
-
-    // Result is now in x0, x1
-    if (!TMP_ABIARGS) {
-      mov(TMP1, ARMEmitter::XReg::x0);
-      mov(TMP2, ARMEmitter::XReg::x1);
-    }
-
-    FillStaticRegs();
-
-    // Fix the stack and any values that were stepped on
-    PopDynamicRegs();
-
-    // Go back to our code block
-    ret();
-    return Address;
-  };
-
-  LUDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LUDIV));
-  LDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LDIV));
-
   EmitF64Sin();
   EmitF64Cos();
   EmitF64Tan();
@@ -2686,14 +2775,14 @@ void Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread)
     Ptrs.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddressSpillSRA;
     Ptrs.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddressSpillSRA;
     Ptrs.ThreadDispatchSyscallHandler = ThreadDispatchSyscallHandler;
+    Ptrs.ThreadDispatchCPUIDHandler = ThreadDispatchCPUIDHandler;
     Ptrs.ThreadDispatchRemoveCodeEntry = ThreadDispatchRemoveCodeEntry;
     Ptrs.GuestSignal_SIGILL = GuestSignal_SIGILL;
     Ptrs.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
     Ptrs.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
+    Ptrs.GuestSignal_SynchronousFault = GuestSignal_SynchronousFault;
     Ptrs.SignalReturnHandler = SignalHandlerReturnAddress;
     Ptrs.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
-    Ptrs.LUDIVHandler = LUDIVHandlerAddress;
-    Ptrs.LDIVHandler = LDIVHandlerAddress;
     Ptrs.F64SinHandler = F64SinHandlerAddress;
     Ptrs.F64CosHandler = F64CosHandlerAddress;
     Ptrs.F64TanHandler = F64TanHandlerAddress;

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h

@@ -82,6 +82,7 @@ private:
   uint64_t ThreadPauseHandlerAddress {};
   uint64_t ThreadPauseHandlerAddressSpillSRA {};
   uint64_t ThreadDispatchSyscallHandler {};
+  uint64_t ThreadDispatchCPUIDHandler {};
   uint64_t ThreadDispatchRemoveCodeEntry {};
   uint64_t ExitFunctionLinkerAddress {};
   uint64_t SignalHandlerReturnAddress {};
@@ -89,6 +90,7 @@ private:
   uint64_t GuestSignal_SIGILL {};
   uint64_t GuestSignal_SIGTRAP {};
   uint64_t GuestSignal_SIGSEGV {};
+  uint64_t GuestSignal_SynchronousFault {};
 
   uint64_t PauseReturnInstruction {};
   std::array<uint64_t, FallbackABI::FABI_UNKNOWN> ABIPointers {};
@@ -97,10 +99,6 @@ private:
   uint64_t Start {};
   uint64_t End {};
 
-  // Long division helpers
-  uint64_t LUDIVHandlerAddress {};
-  uint64_t LDIVHandlerAddress {};
-
   // F64 reduced-precision shared handlers
   uint64_t F64SinHandlerAddress {};
   uint64_t F64CosHandlerAddress {};

FEXCore/Source/Interface/Core/Frontend.cpp

@@ -79,39 +79,56 @@ Decoder::Decoder(FEXCore::Core::InternalThreadState* Thread)
     X87Table = &FEXCore::X86Tables::X87F80Ops;
   }
 
-  if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {
+  if (CTX->HostFeatures.SupportsAVX() && CTX->HostFeatures.SupportsSVE256()) {
     VEXTable = &FEXCore::X86Tables::VEXTableOps;
     VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps;
-  } else if (CTX->HostFeatures.SupportsAVX) {
+  } else if (CTX->HostFeatures.SupportsAVX()) {
     VEXTable = &FEXCore::X86Tables::VEXTableOps_AVX128;
     VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps_AVX128;
   }
 }
 
-bool Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {
+int32_t Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {
   // Check for wraparound
   if (Address + Size < Address) {
-    return false;
+    // Say that any bytes that wrapped around are unreadable
+    return -(Address + Size);
   }
 
-  while (Address < ExecutableRangeBase || Address + Size > ExecutableRangeEnd) {
+  if (Thread->CurrentFrame->State.Dr7) [[unlikely]] {
+    // If Dr7 has anything enabled then check it.
+    for (size_t i = 0; i < Core::CPUState::NUM_DEBUG_ADDRESSES; ++i) {
+      if (Core::CPUState::IsInstructionBPEnabled(i, Thread->CurrentFrame->State.Dr7)) {
+        size_t DrSize = Core::CPUState::BreakpointSize(i, Thread->CurrentFrame->State.Dr7);
+        if (Address >= Thread->CurrentFrame->State.Dr[i] && (Address + Size - 1) < (Thread->CurrentFrame->State.Dr[i] + DrSize)) {
+          HitBreakpoint = i;
+          return false;
+        }
+      }
+    }
+  }
+
+  uint64_t RemainingSize = Size;
+  while (Address < ExecutableRangeBase || Address + RemainingSize > ExecutableRangeEnd) {
     auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, Address);
     ExecutableRangeBase = RangeInfo.Base;
     ExecutableRangeEnd = RangeInfo.Base + RangeInfo.Size;
     ExecutableRangeWritable = RangeInfo.Writable;
 
     if (RangeInfo.Size == 0) {
-      return false;
+      // Return the number of bytes that couldn't be decoded.
+      return -(Size - RemainingSize);
     }
 
     uint64_t RangeRemainingSize = ExecutableRangeEnd - Address;
-    if (Size > RangeRemainingSize) {
-      Size -= RangeRemainingSize;
+    if (RemainingSize > RangeRemainingSize) {
+      RemainingSize -= RangeRemainingSize;
       Address += RangeRemainingSize;
     }
   }
 
-  return true;
+  // Say all bytes are readable.
+  return Size;
 }
 
 uint8_t Decoder::ReadByte() {
@@ -129,7 +146,7 @@ uint8_t Decoder::ReadByte() {
 
 std::optional<uint8_t> Decoder::PeekByte(uint8_t Offset) {
   uint64_t ByteAddress = reinterpret_cast<uint64_t>(InstStream.InstStream + InstructionSize + Offset);
-  if (CheckRangeExecutable(ByteAddress, 1)) {
+  if (CheckRangeExecutable(ByteAddress, 1) == 1) {
     return InstStream.AdjustedInstStream[InstructionSize + Offset];
   } else {
     return std::nullopt;
@@ -143,7 +160,8 @@ std::pair<uint64_t, bool> Decoder::ReadData(uint8_t Size) {
   uint64_t Address = reinterpret_cast<uint64_t>(InstStream.InstStream + InstructionSize);
   LastFieldReadOffset = (uint8_t)InstructionSize;
   LastFieldReadSize = Size;
-  if (CheckRangeExecutable(Address, Size)) {
+  auto ReadableSize = CheckRangeExecutable(Address, Size);
+  if (ReadableSize == Size) {
     std::memcpy(&Res, &InstStream.AdjustedInstStream[InstructionSize], Size);
   } else {
     HitNonExecutableRange = true;
@@ -151,13 +169,14 @@ std::pair<uint64_t, bool> Decoder::ReadData(uint8_t Size) {
     Res = 0;
   }
 
-
+  // Skip the number of decoded bytes regardless to get instruction size.
+  // Even partially decoded instruction.
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
-  for (size_t i = 0; i < Size; ++i) {
+  for (size_t i = 0; i < std::abs(ReadableSize); ++i) {
     ReadByte();
   }
 #else
-  SkipBytes(Size);
+  SkipBytes(std::abs(ReadableSize));
 #endif
 
   if (Relocations) {
@@ -942,23 +961,24 @@ Decoder::DecodedBlockStatus Decoder::DecodeInstructionImpl(uint64_t PC) {
           // If any of these three prefixes are used then it falls down the subtable
           // Additionally: If you hit repeat of differnt prefixes then only the LAST one before this one works for subtable selection
 
-          bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;
-          bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
+          const bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;
+          const bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
+          const bool NoPrefixOrder = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_PREFIX_ORDER) != 0;
 
           DecodeREXIfValid(-2);
           if (NoOverlay) { // This section of the table ignores prefix extention
             return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);
-          } else if (LastEscapePrefix == 0xF3) { // REP
+          } else if (LastEscapePrefix == 0xF3 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_REP_PREFIX)) { // REP
             // Remove prefix so it doesn't effect calculations.
             // This is only an escape prefix rather tan modifier now
             DecodeInst->Flags &= ~DecodeFlags::FLAG_REP_PREFIX;
             return NormalOpHeader(&FEXCore::X86Tables::RepModOps[EscapeOp], EscapeOp);
-          } else if (LastEscapePrefix == 0xF2) { // REPNE
+          } else if (LastEscapePrefix == 0xF2 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_REPNE_PREFIX)) { // REPNE
             // Remove prefix so it doesn't effect calculations.
             // This is only an escape prefix rather tan modifier now
             DecodeInst->Flags &= ~DecodeFlags::FLAG_REPNE_PREFIX;
             return NormalOpHeader(&FEXCore::X86Tables::RepNEModOps[EscapeOp], EscapeOp);
-          } else if (LastEscapePrefix == 0x66 && !NoOverlay66) { // Operand Size
+          } else if ((LastEscapePrefix == 0x66 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_OPERAND_SIZE)) && !NoOverlay66) { // Operand Size
             // Remove prefix so it doesn't effect calculations.
             // This is only an escape prefix rather tan modifier now
             DecodeInst->Flags &= ~DecodeFlags::FLAG_OPERAND_SIZE;
@@ -1078,21 +1098,27 @@ Decoder::DecodedBlockStatus Decoder::DecodeInstruction(uint64_t PC) {
   // Will be set if DecodeInstructionImpl tries to read non-executable memory
   HitNonExecutableRange = false;
   HitBadRelocation = false;
+  HitBreakpoint = ~0U;
   auto ErrorDuringDecoding = DecodeInstructionImpl(PC);
 
   if (ErrorDuringDecoding != DecodedBlockStatus::SUCCESS || HitNonExecutableRange || HitBadRelocation) [[unlikely]] {
     // Put an invalid instruction in the stream so the core can raise SIGILL if hit
     // Error while decoding instruction. We don't know the table or instruction size
-    const auto InstSize = DecodeInst->InstSize;
     DecodeInst->TableInfo = nullptr;
-    DecodeInst->InstSize = 0;
+
+    if (HitBreakpoint != ~0U) {
+      return static_cast<Decoder::DecodedBlockStatus>(FEXCore::ToUnderlying(DecodedBlockStatus::BREAKPOINT_0) + HitBreakpoint);
+    }
 
     // A decode error can be caused by substituting zero for an inaccessible
     // instruction byte, so the instruction fetch fault takes priority.
     if (HitNonExecutableRange) {
-      return InstSize ? DecodedBlockStatus::PARTIAL_DECODE_INST : DecodedBlockStatus::NOEXEC_INST;
+      return DecodeInst->InstSize ? DecodedBlockStatus::PARTIAL_DECODE_INST : DecodedBlockStatus::NOEXEC_INST;
     }
 
+    // Keep InstSize around for PARTIAL_DECODE_INST so we can get the offset where it partial decoded.
+    DecodeInst->InstSize = 0;
+
     if (HitBadRelocation) {
       return DecodedBlockStatus::BAD_RELOCATION;
     }
@@ -1352,8 +1378,8 @@ const Decoder::DecodeStream Decoder::AdjustAddrForSpecialRegion(const uint8_t* _
   constexpr uint64_t VSyscall_Base = 0xFFFF'FFFF'FF60'0000ULL;
   constexpr uint64_t VSyscall_End = VSyscall_Base + 0x1000;
 
-  if (BlockInfo.Is64BitMode && CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux && RIP >= VSyscall_Base &&
-      RIP < VSyscall_End) {
+  if (BlockInfo.Is64BitMode && CTX->HostFeatures.ProcessFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux &&
+      RIP >= VSyscall_Base && RIP < VSyscall_End) {
     // VSyscall
     // This doesn't exist on AArch64 and on x86_64 hosts this is emulated with faults to a region mapped with --xp permissions
     // Offset     0: vgettimeofday
@@ -1650,6 +1676,10 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
           EraseBlock = true;
         } else {
           LogMan::Msg::EFmt("{} instruction in entry block: {:X}",
+                            BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_0       ? "Breakpoint" :
+                            BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_1       ? "Breakpoint" :
+                            BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_2       ? "Breakpoint" :
+                            BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_3       ? "Breakpoint" :
                             BlockIt->BlockStatus == DecodedBlockStatus::INVALID_INST       ? "Invalid" :
                             BlockIt->BlockStatus == DecodedBlockStatus::NOEXEC_INST        ? "NoExec" :
                             BlockIt->BlockStatus == DecodedBlockStatus::BAD_RELOCATION     ? "BadRelocation" :

FEXCore/Source/Interface/Core/Frontend.h

@@ -30,6 +30,10 @@ public:
     PARTIAL_DECODE_INST,
     BAD_RELOCATION,
     UNIMPLEMENTED_INST,
+    BREAKPOINT_0,
+    BREAKPOINT_1,
+    BREAKPOINT_2,
+    BREAKPOINT_3,
   };
 
   enum class DataMaskType : uint8_t { NONE, MOV, BRANCH, DISP, NOP };
@@ -115,7 +119,10 @@ private:
   void DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block);
   void PruneInlinedBranchDataMasks();
 
-  bool CheckRangeExecutable(uint64_t Address, uint64_t Size);
+  // Check for the number of bytes that are able to be decoded as executable
+  // Size: Returns Size if able to decode all bytes.
+  // Negative: Number of bytes that failed to decode.
+  int32_t CheckRangeExecutable(uint64_t Address, uint64_t Size);
 
   uint8_t ReadByte();
   std::optional<uint8_t> PeekByte(uint8_t Offset);
@@ -153,6 +160,7 @@ private:
   bool ExecutableRangeWritable {};
   bool HitNonExecutableRange {};
   bool HitBadRelocation {};
+  uint32_t HitBreakpoint {~0U};
 
   struct DecodeStream {
     // Original instruction stream RIP location.

FEXCore/Source/Interface/Core/JIT/ALUOps.cpp

@@ -865,6 +865,159 @@ DEF_OP(PExt) {
   }
 }
 
+// Divide a uint128_t by a uint64_t, producing quotient and remainder.
+//
+// Adapted from divllu() in libdivide: https://github.com/ridiculousfish/libdivide/blob/master/doc/divlu.c
+// divllu() specifically is licensed as public domain, despite the zlib license that covers the rest of libdivide.
+// divllu() is itself an adaptation of Knuth's classic Algorithm D for long division, with some optimizations.
+// Where possible we have followed divllu() to the letter, including comments, but some changes were needed
+// due to our limited set of temp registers. Some comments below represent the original C code from divllu()
+// which is intended to match the assembly we emit.
+//
+// Additional reading:
+// https://ridiculousfish.com/blog/posts/labor-of-division-episode-iv.html
+// https://ridiculousfish.com/blog/posts/labor-of-division-episode-v.html
+void Arm64JITCore::EmitDivllu(ARMEmitter::Register Quotient, ARMEmitter::Register Remainder, ARMEmitter::Register Upper,
+                              ARMEmitter::Register Lower, ARMEmitter::Register Divisor) {
+  const auto EmitSize = ARMEmitter::Size::i64Bit;
+  ARMEmitter::ForwardLabel UpperReduced {};
+  ARMEmitter::ForwardLabel HighDigitDone {};
+  ARMEmitter::ForwardLabel LowDigitDone {};
+
+  // We work in base 2**32.
+  // A uint32 holds a single digit. A uint64 holds two digits.
+  // Our numerator is conceptually [num3, num2, num1, num0].
+  // Our denominator is [den1, den0].
+
+  // The normalization shift factor.
+  const auto Shift = TMP1;
+
+  // The divisor, and the high and low words of the dividend.
+  const auto Den = TMP2;
+  const auto NumHi = TMP3;
+  const auto NumLo = TMP4;
+
+  // A partial remainder.
+  const auto Rem = TMP3;
+
+  // Check for overflow and divide by 0.
+  // TODO: IF FEX handled #DE, we could simplify this I think.
+  if (NumHi != Upper) {
+    mov(EmitSize, NumHi, Upper);
+  }
+  cmp(EmitSize, Upper, Divisor);
+  (void)b(ARMEmitter::Condition::CC_LO, &UpperReduced);
+  udiv(EmitSize, Quotient, Upper, Divisor);
+  msub(EmitSize, NumHi, Quotient, Divisor, Upper);
+  (void)Bind(&UpperReduced);
+
+  // Determine the normalization factor. We multiply den by this, so that its leading digit is at
+  // least half b. In binary this means just shifting left by the number of leading zeros, so that
+  // there's a 1 in the MSB.
+  // We also shift numer by the same amount. This cannot overflow because numhi < den.
+  // The expression (-shift & 63) is the same as (64 - shift), except it avoids the UB of shifting
+  // by 64. The funny bitwise 'and' ensures that numlo does not get shifted into numhi if shift is 0.
+
+  // shift = divlu_count_leading_zeros64(den);
+  clz(EmitSize, Shift, Divisor);
+  // den <<= shift;
+  lslv(EmitSize, Den, Divisor, Shift);
+  // numhi <<= shift;
+  lslv(EmitSize, NumHi, NumHi, Shift);
+  // numhi |= (numlo >> (-shift & 63)) & (-(int64_t)shift >> 63);
+  mvn(EmitSize, Quotient, Shift);
+  lsr(EmitSize, Remainder, Lower, 1);
+  lsrv(EmitSize, Remainder, Remainder, Quotient);
+  orr(EmitSize, NumHi, NumHi, Remainder);
+  // numlo <<= shift;
+  lslv(EmitSize, NumLo, Lower, Shift);
+
+  // We wish to compute q1 = [n3 n2 n1] / [d1 d0].
+  // Estimate q1 as [n3 n2] / [d1], and then correct it.
+  // Note while qhat may be 2 digits, q1 is always 1 digit.
+  {
+    const auto Den1 = TMP1;
+    const auto Den0 = TMP1;
+
+    // The estimated quotient, and estimated remainder.
+    const auto QHat = Quotient;
+    const auto RHat = Remainder;
+
+    // Variables used to correct the estimated quotient.
+    const auto C1 = TMP1;
+    const auto C2 = Remainder;
+
+    // qhat = numhi / den1;
+    lsr(EmitSize, Den1, Den, 32);
+    udiv(EmitSize, QHat, NumHi, Den1);
+    // rhat = numhi % den1;
+    msub(EmitSize, RHat, QHat, Den1, NumHi);
+    // c1 = qhat * den0;
+    mov(ARMEmitter::Size::i32Bit, Den0, Den);
+    mul(EmitSize, C1, QHat, Den0);
+    // c2 = rhat * b + num1;
+    extr(EmitSize, C2, RHat, NumLo, 32);
+    // if (c1 > c2)
+    //     qhat -= (c1 - c2 > den) ? 2 : 1;
+    // C1 holds c1 - c2 after the subs.
+    subs(EmitSize, C1, C1, C2);
+    (void)b(ARMEmitter::Condition::CC_LS, &HighDigitDone);
+    sub(EmitSize, QHat, QHat, 1);
+    cmp(EmitSize, C1, Den);
+    (void)b(ARMEmitter::Condition::CC_LS, &HighDigitDone);
+    sub(EmitSize, QHat, QHat, 1);
+    (void)Bind(&HighDigitDone);
+    // q1 = (uint32_t)qhat;
+    // This move is a nop for us.
+  }
+
+  // Compute the true (partial) remainder.
+  // rem = numhi * b + num1 - q1 * den;
+  extr(EmitSize, Rem, NumHi, NumLo, 32);
+  msub(EmitSize, Rem, Quotient, Den, Rem);
+
+  // We wish to compute q0 = [rem1 rem0 n0] / [d1 d0].
+  // Estimate q0 as [rem1 rem0] / [d1] and correct it.
+  {
+    const auto Den1 = TMP1;
+    const auto Den0 = TMP3;
+
+    // The estimated quotient, and its corresponding remainder
+    const auto QHat = Remainder;
+    const auto RHat = TMP1;
+
+    // Variables used to correct the estimated quotient.
+    const auto C1 = TMP3;
+    const auto C2 = NumLo;
+
+    // qhat = rem / den1;
+    lsr(EmitSize, Den1, Den, 32);
+    udiv(EmitSize, QHat, Rem, Den1);
+    // rhat = rem % den1;
+    msub(EmitSize, RHat, QHat, Den1, Rem);
+    // c1 = qhat * den0;
+    mov(ARMEmitter::Size::i32Bit, Den0, Den);
+    mul(EmitSize, C1, QHat, Den0);
+    // c2 = rhat * b + num0;
+    bfi(EmitSize, C2, RHat, 32, 32);
+    // if (c1 > c2)
+    //     qhat -= (c1 - c2 > den) ? 2 : 1;
+    // C1 holds c1 - c2 after the subs.
+    subs(EmitSize, C1, C1, C2);
+    (void)b(ARMEmitter::Condition::CC_LS, &LowDigitDone);
+    sub(EmitSize, QHat, QHat, 1);
+    cmp(EmitSize, C1, Den);
+    (void)b(ARMEmitter::Condition::CC_LS, &LowDigitDone);
+    sub(EmitSize, QHat, QHat, 1);
+    (void)Bind(&LowDigitDone);
+    // q0 = (uint32_t)qhat;
+    // This move is a nop for us.
+  }
+
+  // q = ((uint64_t)q1 << 32) | q0;
+  orr(EmitSize, Quotient, Remainder, Quotient, ARMEmitter::ShiftType::LSL, 32);
+}
+
 DEF_OP(Div) {
   auto Op = IROp->C<IR::IROp_Div>();
   const auto OpSize = IROp->Size;
@@ -930,22 +1083,48 @@ DEF_OP(Div) {
 
     // If the sign bit matches then the result is zero
     (void)cbz(EmitSize, TMP1, &Only64Bit);
+    // The x86 instruction raises #DE when the divisor is zero which we don't do.
+    // TODO: #DE
+    (void)cbz(EmitSize, Divisor, &Only64Bit);
 
     // Long divide
+    // EmitDivllu() is the unsigned case, so we need to "cast" to unsigned first,
+    // and then apply the correct sign back to the results afterwards.
     {
-      mov(EmitSize, TMP1, Upper);
-      mov(EmitSize, TMP2, Lower);
-      mov(EmitSize, TMP3, Divisor);
+      const auto Sign = TMP1;
+      const auto AbsUpper = TMP3;
+      const auto AbsLower = TMP4;
+      const auto AbsDivisor = TMP2;
 
-      ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LDIVHandler));
+      mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
+      str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
 
-      str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
-      blr(TMP4);
-      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
+      // Extract sign bit of the dividend.
+      asr(EmitSize, Sign, Upper, 63);
 
-      // Move results to the destination registers
-      mov(EmitSize, Quotient, TMP1);
-      mov(EmitSize, Remainder, TMP2);
+      // Compute absolute value of the 128 bit dividend.
+      eor(EmitSize, AbsLower, Lower, Sign);
+      eor(EmitSize, AbsUpper, Upper, Sign);
+      subs(EmitSize, AbsLower, AbsLower, Sign);
+      sbc(EmitSize, AbsUpper, AbsUpper, Sign);
+
+      // Absolute value of the divisor.
+      cmp(EmitSize, Divisor, 0);
+      cneg(EmitSize, AbsDivisor, Divisor, ARMEmitter::Condition::CC_LT);
+
+      // Now do the unsigned divide.
+      EmitDivllu(Quotient, Remainder, AbsUpper, AbsLower, AbsDivisor);
+
+      // The quotient is negative when the dividend and the divisor have different signs.
+      eor(EmitSize, TMP1, Upper, Divisor);
+      cmp(EmitSize, TMP1, 0);
+      cneg(EmitSize, Quotient, Quotient, ARMEmitter::Condition::CC_LT);
+
+      ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
+      msr(ARMEmitter::SystemRegister::NZCV, TMP1);
+
+      // Compute the remainder.
+      msub(EmitSize, Remainder, Quotient, Divisor, Lower);
 
       // Skip 64-bit path
       (void)b(&LongDIVRet);
@@ -1013,22 +1192,22 @@ DEF_OP(UDiv) {
     // Check the upper bits for zero
     // If the upper bits are zero then we can do a 64-bit divide
     (void)cbz(EmitSize, Upper, &Only64Bit);
+    // The x86 instruction raises #DE when the divisor is zero which we don't do.
+    // TODO: #DE
+    (void)cbz(EmitSize, Divisor, &Only64Bit);
 
     // Long divide
     {
-      mov(EmitSize, TMP1, Upper);
-      mov(EmitSize, TMP2, Lower);
-      mov(EmitSize, TMP3, Divisor);
+      mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
+      str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
 
-      ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LUDIVHandler));
+      EmitDivllu(Quotient, Remainder, Upper, Lower, Divisor);
 
-      str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
-      blr(TMP4);
-      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
+      ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
+      msr(ARMEmitter::SystemRegister::NZCV, TMP1);
 
-      // Move results to the destination registers
-      mov(EmitSize, Quotient, TMP1);
-      mov(EmitSize, Remainder, TMP2);
+      // Compute the remainder.
+      msub(EmitSize, Remainder, Quotient, Divisor, Lower);
 
       // Skip 64-bit path
       (void)b(&LongDIVRet);

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -395,45 +395,12 @@ DEF_OP(ThreadRemoveCodeEntry) {
 DEF_OP(CPUID) {
   auto Op = IROp->C<IR::IROp_CPUID>();
 
-  isb();
-  mov(ARMEmitter::Size::i64Bit, TMP2, GetReg(Op->Function));
-  mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Leaf));
+  SpillStaticRegs(TMP1);
 
-  PushDynamicRegs(TMP4);
-  SpillStaticRegs(TMP4);
-
-  // x0 = CPUID Handler
-  // x1 = CPUID Function
-  // x2 = CPUID Leaf
-  ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));
-  ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDFunction));
-
-  if (!TMP_ABIARGS) {
-    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, TMP2);
-    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, TMP3);
-  }
-
-  if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-    GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
-  } else {
-    blr(ARMEmitter::Reg::r3);
-  }
-
-  if (!TMP_ABIARGS) {
-    mov(ARMEmitter::Size::i64Bit, TMP1, ARMEmitter::Reg::r0);
-    mov(ARMEmitter::Size::i64Bit, TMP2, ARMEmitter::Reg::r1);
-  }
-
-  FillStaticRegs();
-
-  PopDynamicRegs();
-
-  // Results are in x0, x1
-  // Results want to be 4xi32 scalars
-  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX), TMP1);
-  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutECX), TMP2);
-  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEBX), TMP1, 32, 32);
-  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX), TMP2, 32, 32);
+  LoadConstant(ARMEmitter::Size::i32Bit, TMP1, Op->InstSize);
+  // Jump to the syscall dispatch handler. We won't return after this.
+  ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadDispatchCPUIDHandler));
+  br(TMP2);
 }
 
 DEF_OP(XGetBV) {

FEXCore/Source/Interface/Core/JIT/ConversionOps.cpp

@@ -405,6 +405,37 @@ DEF_OP(VFCVTN2) {
   }
 }
 
+DEF_OP(VBFCVTN) {
+  LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsBF16, "Need BF16 for VBFCVTN");
+  const auto Op = IROp->C<IR::IROp_VBFCVTN>();
+
+  const auto Dst = GetVReg(Node);
+  const auto Vector = GetVReg(Op->Vector);
+
+  bfcvtn(Dst.Q(), Vector.Q());
+}
+
+DEF_OP(VBFCVTN2) {
+  LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsBF16, "Need BF16 for VBFCVTN");
+  const auto Op = IROp->C<IR::IROp_VBFCVTN2>();
+
+  const auto Dst = GetVReg(Node);
+  const auto VectorLower = GetVReg(Op->VectorLower);
+  const auto VectorUpper = GetVReg(Op->VectorUpper);
+
+  auto Lower = VectorLower;
+  if (Dst != VectorLower) {
+    mov(VTMP1.Q(), VectorLower.Q());
+    Lower = VTMP1;
+  }
+
+  bfcvtn2(Lower.Q(), VectorUpper.Q());
+
+  if (Dst != VectorLower) {
+    mov(Dst.Q(), Lower.Q());
+  }
+}
+
 DEF_OP(Vector_FToI) {
   const auto Op = IROp->C<IR::IROp_Vector_FToI>();
   const auto OpSize = IROp->Size;

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -46,34 +46,9 @@ $end_info$
 #endif
 
 namespace {
-struct DivRem {
-  uint64_t Quotient;
-  uint64_t Remainder;
-};
-
-static struct DivRem LUDIV(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {
-  __uint128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
-
-  return {
-    .Quotient = (uint64_t)(Source / Divisor),
-    .Remainder = (uint64_t)(Source % Divisor),
-  };
-}
-
-static struct DivRem
-LDIV(uint64_t SrcHigh, uint64_t SrcLow, int64_t Divisor) {
-  __int128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
-
-  return {
-    .Quotient = (uint64_t)(Source / Divisor),
-    .Remainder = (uint64_t)(Source % Divisor),
-  };
-}
-
 #ifndef _WIN32
 
-static std::optional<uint64_t>
-RDRANDFallback(uint64_t Reseed) {
+static std::optional<uint64_t> RDRANDFallback(uint64_t Reseed) {
   uint64_t Value {};
   FHU::Syscalls::getrandom(&Value, sizeof(Value), 0);
   return Value;
@@ -87,8 +62,7 @@ RDRANDFallback(uint64_t Reseed) {
 
 // Reference implementation by Sebastiano Vigna, public domain (CC0)
 // https://prng.di.unimi.it/splitmix64.c
-static std::atomic<uint64_t>
-  RNGState {static_cast<uint64_t>(__builtin_readcyclecounter())};
+static std::atomic<uint64_t> RNGState {static_cast<uint64_t>(__builtin_readcyclecounter())};
 
 static std::optional<uint64_t> RDRANDFallback(uint64_t Reseed) {
   const uint64_t State = RNGState.load(std::memory_order_relaxed) + 0x9E3779B97F4A7C15ULL;
@@ -651,9 +625,9 @@ void Arm64JITCore::Op_NoOp(const IR::IROp_Header* IROp, IR::Ref Node) {}
 Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread)
   : CPUBackend(*ctx, Thread)
   , Arm64Emitter(ctx)
-  , HostSupportsSVE128 {ctx->HostFeatures.SupportsSVE128 != 0}
-  , HostSupportsSVE256 {ctx->HostFeatures.SupportsSVE256 != 0}
-  , HostSupportsAVX256 {ctx->HostFeatures.SupportsAVX && ctx->HostFeatures.SupportsSVE256}
+  , HostSupportsSVE128 {ctx->HostFeatures.SupportsSVE128()}
+  , HostSupportsSVE256 {ctx->HostFeatures.SupportsSVE256()}
+  , HostSupportsAVX256 {ctx->HostFeatures.SupportsAVX() && ctx->HostFeatures.SupportsSVE256()}
   , HostSupportsRPRES {ctx->HostFeatures.SupportsRPRES != 0}
   , HostSupportsAFP {ctx->HostFeatures.SupportsAFP != 0}
   , CTX {ctx}
@@ -697,8 +671,6 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
       Ptrs.SyscallHandlerFunc = PMF.GetVTableEntry(CTX->SyscallHandler);
     }
     Ptrs.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Arm64JITCore::ExitFunctionLink);
-    Ptrs.LUDIV = reinterpret_cast<uint64_t>(LUDIV);
-    Ptrs.LDIV = reinterpret_cast<uint64_t>(LDIV);
     Ptrs.RDRANDFallback = reinterpret_cast<uint64_t>(RDRANDFallback);
   }
 
@@ -782,13 +754,17 @@ void Arm64JITCore::EmitTFCheck() {
     .TrapNo = X86State::X86_TRAPNO_DB,
     .si_code = 2,
     .err_code = 0,
+    .FaultAddress = 0,
   };
 
   uint64_t Constant {};
-  memcpy(&Constant, &State, sizeof(State));
+  memcpy(&Constant, &State, sizeof(uint64_t));
 
-  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
+  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
+  // Can't use stp, offset is too large.
   str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
+  str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
+
   ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
   br(TMP1);
 
@@ -876,9 +852,9 @@ CodeBuffer::CodeBufferAllocation Arm64JITCore::AllocateCodeBufferInSharedCache(s
   return AllocatedInfo;
 }
 
-CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
+CPUBackend::CompiledCode Arm64JITCore::EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                                    FEXCore::Core::DebugData* DebugData, bool CheckTF) {
-  FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
+  FEXCORE_PROFILE_SCOPED("Arm64::EmitJITCode");
 
   const auto PrevNumAllocations = Relocations.size();
 

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -51,7 +51,7 @@ public:
   ~Arm64JITCore() override;
 
   [[nodiscard]]
-  CPUBackend::CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
+  CPUBackend::CompiledCode EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                        FEXCore::Core::DebugData* DebugData, bool CheckTF) override;
 
   [[nodiscard]]
@@ -634,6 +634,9 @@ private:
 
   void EmitTFCheck();
 
+  void EmitDivllu(ARMEmitter::Register Quotient, ARMEmitter::Register Remainder, ARMEmitter::Register Upper, ARMEmitter::Register Lower,
+                  ARMEmitter::Register Divisor);
+
   void EmitSuspendInterruptCheck();
 
   void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF);

FEXCore/Source/Interface/Core/JIT/MiscOps.cpp

@@ -68,31 +68,42 @@ DEF_OP(Break) {
     .TrapNo = Op->Reason.TrapNumber,
     .si_code = Op->Reason.si_code,
     .err_code = Op->Reason.ErrorRegister,
+    .FaultAddress = 0, // Comes from SSA argument.
   };
 
   uint64_t Constant {};
-  memcpy(&Constant, &State, sizeof(State));
+  memcpy(&Constant, &State, sizeof(uint64_t));
 
-  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
+  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
+
+  auto FaultGPR = GetZeroableReg(Op->FaultAddress);
+
+  // Can't use stp, offset is too large.
   str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
+  str(FaultGPR.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
 
-  switch (Op->Reason.Signal) {
-  case Core::FAULT_SIGILL:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
+  if (CTX->HostFeatures.ProcessFeatures.SupportsFaultFreeSynchronousFaults) {
+    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SynchronousFault));
     br(TMP1);
-    break;
-  case Core::FAULT_SIGTRAP:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
-    br(TMP1);
-    break;
-  case Core::FAULT_SIGSEGV:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
-    br(TMP1);
-    break;
-  default:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
-    br(TMP1);
-    break;
+  } else {
+    switch (Op->Reason.Signal) {
+    case Core::FAULT_SIGILL:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
+      br(TMP1);
+      break;
+    case Core::FAULT_SIGTRAP:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
+      br(TMP1);
+      break;
+    case Core::FAULT_SIGSEGV:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
+      br(TMP1);
+      break;
+    default:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
+      br(TMP1);
+      break;
+    }
   }
 }
 

FEXCore/Source/Interface/Core/JIT/VectorOps.cpp

@@ -930,6 +930,9 @@ DEF_OP(VectorImm) {
 
   const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
+  if (Op->Do64BitExpansion) {
+    LOGMAN_THROW_A_FMT(!Is256Bit, "64-bit expansion unsupported on 256-bit operations.");
+  }
   const auto ElementSize = Op->Header.ElementSize;
   const auto SubRegSize = ConvertSubRegSize8(IROp);
 
@@ -945,12 +948,22 @@ DEF_OP(VectorImm) {
       dup_imm(SubRegSize, Dst.Z(), static_cast<int8_t>(Op->Immediate));
     }
   } else {
-    if (ElementSize == IR::OpSize::i64Bit) {
+    if (!Op->Do64BitExpansion && ElementSize == IR::OpSize::i64Bit) {
       // movi with 64bit element size doesn't do what we want here
       LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Op->Immediate) << Op->ShiftAmount);
       dup(SubRegSize, Dst.Q(), TMP1.R());
     } else {
-      movi(SubRegSize, Dst.Q(), Op->Immediate, Op->ShiftAmount);
+      uint64_t Imm = Op->Immediate;
+      if (Op->Do64BitExpansion) {
+        // Need to manually expand the immediate for this case.
+        Imm = 0;
+        for (size_t i = 0; i < 8; ++i) {
+          if ((Op->Immediate >> i) & 1) {
+            Imm |= (0xFFULL << (i * 8));
+          }
+        }
+      }
+      movi(SubRegSize, Dst.Q(), Imm, Op->ShiftAmount);
     }
   }
 }

FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp

@@ -184,30 +184,35 @@ void OpDispatchBuilder::SecondaryALUOp(OpcodeArgs) {
   switch (Op->OP) {
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 0):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 0):
+  case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x82), 0):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 0):
     IROp = FEXCore::IR::IROps::OP_ADD;
     AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHADD;
     break;
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 1):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 1):
+  case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x82), 1):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 1):
     IROp = FEXCore::IR::IROps::OP_OR;
     AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHOR;
     break;
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 4):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 4):
+  case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x82), 4):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 4):
     IROp = FEXCore::IR::IROps::OP_ANDWITHFLAGS;
     AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHAND;
     break;
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 5):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 5):
+  case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x82), 5):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 5):
     IROp = FEXCore::IR::IROps::OP_SUB;
     AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHSUB;
     break;
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 6):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 6):
+  case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x82), 6):
   case OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 6):
     IROp = FEXCore::IR::IROps::OP_XOR;
     AtomicIROp = FEXCore::IR::IROps::OP_ATOMICFETCHXOR;
@@ -1004,6 +1009,7 @@ void OpDispatchBuilder::TESTOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX) {
   }
 
   InvalidateAF();
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::ARPLOp(OpcodeArgs) {
@@ -1364,20 +1370,19 @@ void OpDispatchBuilder::MOVOffsetOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::CPUIDOp(OpcodeArgs) {
-  Ref Src = LoadGPRRegister(X86State::REG_RAX);
-  Ref Leaf = LoadGPRRegister(X86State::REG_RCX);
+  BlockSetRIP = true;
 
-  Ref RAX = _AllocateGPR(false);
-  Ref RBX = _AllocateGPR(false);
-  Ref RCX = _AllocateGPR(false);
-  Ref RDX = _AllocateGPR(false);
+  // Calculate flags early.
+  CalculateDeferredFlags();
 
-  _CPUID(Src, Leaf, RAX, RBX, RCX, RDX);
+  // Similar to Syscall. Store the RIP to the start of the instruction.
+  // CPUID operation will modify the RIP itself.
+  const auto GPRSize = GetGPROpSize();
+  auto NewRIP = GetRelocatedPC(Op, -Op->InstSize);
+  _StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
 
-  StoreGPRRegister(X86State::REG_RAX, RAX);
-  StoreGPRRegister(X86State::REG_RBX, RBX);
-  StoreGPRRegister(X86State::REG_RCX, RCX);
-  StoreGPRRegister(X86State::REG_RDX, RDX);
+  FlushRegisterCache();
+  _CPUID(Op->InstSize);
 }
 
 uint32_t OpDispatchBuilder::GetConstantShift(X86Tables::DecodedOp Op, bool Is1Bit) {
@@ -1410,6 +1415,7 @@ void OpDispatchBuilder::SHLOp(OpcodeArgs) {
 
   Ref Result = _Lshl(Size == OpSize::i64Bit ? OpSize::i64Bit : OpSize::i32Bit, Dest, Src);
   HandleShift(Op, Result, Dest, ShiftType::LSL, Src);
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::SHLImmediateOp(OpcodeArgs, bool SHL1Bit) {
@@ -1423,6 +1429,7 @@ void OpDispatchBuilder::SHLImmediateOp(OpcodeArgs, bool SHL1Bit) {
   CalculateFlags_ShiftLeftImmediate(OpSizeFromSrc(Op), Result, Dest, Shift);
   CalculateDeferredFlags();
   StoreResultGPR(Op, Result);
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::SHROp(OpcodeArgs) {
@@ -1432,6 +1439,8 @@ void OpDispatchBuilder::SHROp(OpcodeArgs) {
 
   auto ALUOp = _Lshr(std::max(OpSize::i32Bit, Size), Dest, Src);
   HandleShift(Op, ALUOp, Dest, ShiftType::LSR, Src);
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::SHRImmediateOp(OpcodeArgs, bool SHR1Bit) {
@@ -1444,6 +1453,8 @@ void OpDispatchBuilder::SHRImmediateOp(OpcodeArgs, bool SHR1Bit) {
   CalculateFlags_ShiftRightImmediate(OpSizeFromSrc(Op), ALUOp, Dest, Shift);
   CalculateDeferredFlags();
   StoreResultGPR(Op, ALUOp);
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::SHLDOp(OpcodeArgs) {
@@ -1487,6 +1498,8 @@ void OpDispatchBuilder::SHLDOp(OpcodeArgs) {
   Res = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, Shift, Constant(0), Dest, Res);
 
   HandleShift(Op, Res, Dest, ShiftType::LSL, Shift);
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::SHLDImmediateOp(OpcodeArgs) {
@@ -1518,6 +1531,8 @@ void OpDispatchBuilder::SHLDImmediateOp(OpcodeArgs) {
     // Ensure Zext still occurs
     StoreResultGPR(Op, Dest);
   }
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::SHRDOp(OpcodeArgs) {
@@ -1552,6 +1567,8 @@ void OpDispatchBuilder::SHRDOp(OpcodeArgs) {
   Res = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, Shift, Constant(0), Dest, Res);
 
   HandleShift(Op, Res, Dest, ShiftType::LSR, Shift);
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::SHRDImmediateOp(OpcodeArgs) {
@@ -1582,6 +1599,8 @@ void OpDispatchBuilder::SHRDImmediateOp(OpcodeArgs) {
     // Ensure Zext still occurs
     StoreResultGPR(Op, Dest);
   }
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::ASHROp(OpcodeArgs, bool Immediate, bool SHR1Bit) {
@@ -1611,6 +1630,8 @@ void OpDispatchBuilder::ASHROp(OpcodeArgs, bool Immediate, bool SHR1Bit) {
 
     HandleShift(Op, Result, Dest, ShiftType::ASR, Src);
   }
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
 }
 
 void OpDispatchBuilder::RotateOp(OpcodeArgs, bool Left, bool IsImmediate, bool Is1Bit) {
@@ -1681,6 +1702,8 @@ void OpDispatchBuilder::ANDNBMIOp(OpcodeArgs) {
 
   StoreResultGPR(Op, Dest);
   CalculateFlags_Logical(OpSizeFromSrc(Op), Dest);
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::BEXTRBMIOp(OpcodeArgs) {
@@ -1724,6 +1747,8 @@ void OpDispatchBuilder::BEXTRBMIOp(OpcodeArgs) {
   // AF are undefined.
   SetNZ_ZeroCV(GetOpSize(Dest), Dest);
   InvalidatePF_AF();
+
+  AcidTestFlags(X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::BLSIBMIOp(OpcodeArgs) {
@@ -1745,6 +1770,8 @@ void OpDispatchBuilder::BLSIBMIOp(OpcodeArgs) {
   SetNZ_ZeroCV(Size, Result);
   InvalidatePF_AF();
   SetCFInverted(GetRFLAG(X86State::RFLAG_ZF_RAW_LOC));
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::BLSMSKBMIOp(OpcodeArgs) {
@@ -1765,6 +1792,8 @@ void OpDispatchBuilder::BLSMSKBMIOp(OpcodeArgs) {
   // and O) while setting S.
   SetNZ_ZeroCV(Size, Result);
   SetCFInverted(CFInv);
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::BLSRBMIOp(OpcodeArgs) {
@@ -1782,6 +1811,8 @@ void OpDispatchBuilder::BLSRBMIOp(OpcodeArgs) {
   SetNZ_ZeroCV(Size, Result);
   SetCFInverted(CFInv);
   InvalidatePF_AF();
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 // Handles SARX, SHLX, and SHRX
@@ -1841,6 +1872,8 @@ void OpDispatchBuilder::BZHI(OpcodeArgs) {
   InvalidatePF_AF();
   SetNZ_ZeroCV(Size, Result);
   SetCFInverted(CFInv);
+
+  AcidTestFlags(X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::RORX(OpcodeArgs) {
@@ -2565,6 +2598,9 @@ void OpDispatchBuilder::BTOp(OpcodeArgs, uint32_t SrcIndex, BTAction Action) {
     // to insert into CF directly. We could optimize perhaps.
     SetCFDirect(Value, 0, true);
   }
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_ZF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC,
+                X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::IMUL1SrcOp(OpcodeArgs) {
@@ -2604,6 +2640,8 @@ void OpDispatchBuilder::IMUL1SrcOp(OpcodeArgs) {
 
   StoreResultGPR(Op, Dest);
   CalculateFlags_MUL(Size, Dest, ResultHigh);
+
+  AcidTestFlags(X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_ZF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::IMUL2SrcOp(OpcodeArgs) {
@@ -2643,6 +2681,8 @@ void OpDispatchBuilder::IMUL2SrcOp(OpcodeArgs) {
 
   StoreResultGPR(Op, Dest);
   CalculateFlags_MUL(Size, Dest, ResultHigh);
+
+  AcidTestFlags(X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_ZF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::IMULOp(OpcodeArgs) {
@@ -2698,6 +2738,8 @@ void OpDispatchBuilder::IMULOp(OpcodeArgs) {
   }
 
   CalculateFlags_MUL(Size, Result, ResultHigh);
+
+  AcidTestFlags(X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_ZF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::MULOp(OpcodeArgs) {
@@ -2868,6 +2910,8 @@ void OpDispatchBuilder::DAAOp(OpcodeArgs) {
   SetCFInverted(CFInv);
   CalculatePF(AL);
   SetAFAndFixup(AF);
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC);
 }
 
 void OpDispatchBuilder::DASOp(OpcodeArgs) {
@@ -2894,6 +2938,8 @@ void OpDispatchBuilder::DASOp(OpcodeArgs) {
   SetCFDirect(NewCF);
   CalculatePF(AL);
   SetAFAndFixup(AF);
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC);
 }
 
 void OpDispatchBuilder::AAAOp(OpcodeArgs) {
@@ -2911,6 +2957,7 @@ void OpDispatchBuilder::AAAOp(OpcodeArgs) {
   // AL = AL & 0x0F
   A = _And(OpSize::i32Bit, A, Constant(0xFF0F));
   StoreGPRRegister(X86State::REG_RAX, A, OpSize::i16Bit);
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_ZF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::AASOp(OpcodeArgs) {
@@ -2928,6 +2975,8 @@ void OpDispatchBuilder::AASOp(OpcodeArgs) {
   // AL = AL & 0x0F
   A = _And(OpSize::i32Bit, A, Constant(0xFF0F));
   StoreGPRRegister(X86State::REG_RAX, A, OpSize::i16Bit);
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_ZF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::AAMOp(OpcodeArgs) {
@@ -2942,6 +2991,8 @@ void OpDispatchBuilder::AAMOp(OpcodeArgs) {
   SetNZ_ZeroCV(OpSize::i8Bit, Res);
   CalculatePF(Res);
   InvalidateAF();
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_CF_RAW_LOC);
 }
 
 void OpDispatchBuilder::AADOp(OpcodeArgs) {
@@ -2955,6 +3006,8 @@ void OpDispatchBuilder::AADOp(OpcodeArgs) {
   SetNZ_ZeroCV(OpSize::i8Bit, Result);
   CalculatePF(Result);
   InvalidateAF();
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_CF_RAW_LOC);
 }
 
 void OpDispatchBuilder::XLATOp(OpcodeArgs) {
@@ -3102,7 +3155,38 @@ void OpDispatchBuilder::SMSWOp(OpcodeArgs) {
   }
 
   if (IsMemDst) {
-    // Memory destinatino always writes only 16-bits.
+    // Memory destination always writes only 16-bits.
+    DstSize = OpSize::i16Bit;
+  }
+
+  StoreResultGPR_WithOpSize(Op, Op->Dest, Const, DstSize);
+}
+
+void OpDispatchBuilder::SLDTOp(OpcodeArgs) {
+  const bool IsMemDst = DestIsMem(Op);
+
+  IR::OpSize DstSize {OpSize::iInvalid};
+  Ref Const = _Constant(0);
+
+  const auto OpAddr = X86Tables::DecodeFlags::GetOpAddr(Op->Flags, 0);
+  if (Is64BitMode) {
+    DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST  ? OpSize::i16Bit :
+              OpAddr == X86Tables::DecodeFlags::FLAG_WIDENING_SIZE_LAST ? OpSize::i64Bit :
+                                                                          OpSize::i32Bit;
+
+    if (!IsMemDst && DstSize == OpSize::i32Bit) {
+      // Special-case version of `sldt ebx`. This instruction does an insert in to the lower 32-bits on 64-bit hosts.
+      // Override and insert.
+      auto Dest = LoadSourceGPR_WithOpSize(Op, Op->Dest, GetGPROpSize(), Op->Flags);
+      Const = _Bfi(OpSize::i64Bit, 32, 0, Dest, Const);
+      DstSize = OpSize::i64Bit;
+    }
+  } else {
+    DstSize = OpAddr == X86Tables::DecodeFlags::FLAG_OPERAND_SIZE_LAST ? OpSize::i16Bit : OpSize::i32Bit;
+  }
+
+  if (IsMemDst) {
+    // Memory destination always writes only 16-bits.
     DstSize = OpSize::i16Bit;
   }
 
@@ -3754,6 +3838,10 @@ void OpDispatchBuilder::DIVOp(OpcodeArgs) {
     StoreGPRRegister(X86State::REG_RAX, Quotient, Size);
     StoreGPRRegister(X86State::REG_RDX, Remainder, Size);
   }
+
+  // All flags are undefined.
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_ZF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC,
+                X86State::RFLAG_PF_RAW_LOC, X86State::RFLAG_CF_RAW_LOC);
 }
 
 void OpDispatchBuilder::IDIVOp(OpcodeArgs) {
@@ -3797,6 +3885,10 @@ void OpDispatchBuilder::IDIVOp(OpcodeArgs) {
     StoreGPRRegister(X86State::REG_RAX, Quotient, Size);
     StoreGPRRegister(X86State::REG_RDX, Remainder, Size);
   }
+
+  // All flags are undefined.
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_ZF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC,
+                X86State::RFLAG_PF_RAW_LOC, X86State::RFLAG_CF_RAW_LOC);
 }
 
 void OpDispatchBuilder::BSFOp(OpcodeArgs) {
@@ -3819,6 +3911,9 @@ void OpDispatchBuilder::BSFOp(OpcodeArgs) {
   // applications might rely on that in the wild.
   auto SelectOp = NZCVSelect(GPRSize, CondClass::EQ, Dest, Result);
   StoreResultGPR_WithOpSize(Op, Op->Dest, SelectOp, DstSize);
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC,
+                X86State::RFLAG_CF_RAW_LOC);
 }
 
 void OpDispatchBuilder::BSROp(OpcodeArgs) {
@@ -3837,6 +3932,9 @@ void OpDispatchBuilder::BSROp(OpcodeArgs) {
   // If Src was zero then the destination doesn't get modified
   auto SelectOp = NZCVSelect(GPRSize, CondClass::EQ, Dest, Result);
   StoreResultGPR_WithOpSize(Op, Op->Dest, SelectOp, DstSize);
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC,
+                X86State::RFLAG_CF_RAW_LOC);
 }
 
 void OpDispatchBuilder::CMPXCHGOp(OpcodeArgs) {
@@ -4653,11 +4751,11 @@ OpDispatchBuilder::OpDispatchBuilder(FEXCore::Context::ContextImpl* ctx, FEXCore
   : IREmitter {ctx->OpDispatcherAllocator, ctx->HostFeatures.SupportsTSOImm9 != 0}
   , CTX {ctx}
   , Thread {Thread} {
-  if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {
+  if (CTX->HostFeatures.SupportsAVX() && CTX->HostFeatures.SupportsSVE256()) {
     SaveAVXStateFunc = &OpDispatchBuilder::SaveAVXState;
     RestoreAVXStateFunc = &OpDispatchBuilder::RestoreAVXState;
     DefaultAVXStateFunc = &OpDispatchBuilder::DefaultAVXState;
-  } else if (CTX->HostFeatures.SupportsAVX) {
+  } else if (CTX->HostFeatures.SupportsAVX()) {
     SaveAVXStateFunc = &OpDispatchBuilder::AVX128_SaveAVXState;
     RestoreAVXStateFunc = &OpDispatchBuilder::AVX128_RestoreAVXState;
     DefaultAVXStateFunc = &OpDispatchBuilder::AVX128_DefaultAVXState;
@@ -4782,12 +4880,14 @@ void OpDispatchBuilder::ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::I
   case FEXCore::IR::IROps::OP_AND:
   case FEXCore::IR::IROps::OP_OR: {
     CalculateFlags_Logical(Size, Result);
+    AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
     break;
   }
   case FEXCore::IR::IROps::OP_ANDWITHFLAGS: {
     HandleNZ00Write();
     CalculatePF(Result);
     InvalidateAF();
+    AcidTestFlags(X86State::RFLAG_AF_RAW_LOC);
     break;
   }
   default: break;
@@ -4814,7 +4914,7 @@ void OpDispatchBuilder::INTOp(OpcodeArgs) {
   case 0xCD: { // INT imm8
     uint8_t Literal = Op->Src[0].Literal();
 
-    if (CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux) {
+    if (CTX->HostFeatures.ProcessFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux) {
       constexpr uint8_t SYSCALL_LITERAL = 0x80;
       if (Literal == SYSCALL_LITERAL) {
         if (Is64BitMode) [[unlikely]] {
@@ -4826,8 +4926,8 @@ void OpDispatchBuilder::INTOp(OpcodeArgs) {
         SyscallOp(Op, false);
         return;
       }
-    } else if (CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Wow64 ||
-               CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec) {
+    } else if (CTX->HostFeatures.ProcessFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Wow64 ||
+               CTX->HostFeatures.ProcessFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec) {
       constexpr uint8_t SYSCALL_LITERAL = 0x2E;
       if (Literal == SYSCALL_LITERAL) {
         // Can be used for both 64-bit and 32-bit syscalls on windows
@@ -4835,7 +4935,7 @@ void OpDispatchBuilder::INTOp(OpcodeArgs) {
         return;
       }
 
-      if (CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec) {
+      if (CTX->HostFeatures.ProcessFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec) {
         // This is used when QueryPerformanceCounter is called on recent Windows versions, it causes CNTVCT to be written into RAX.
         constexpr uint8_t GET_CNTVCT_LITERAL = 0x81;
         if (Literal == GET_CNTVCT_LITERAL) {
@@ -4938,6 +5038,8 @@ void OpDispatchBuilder::TZCNT(OpcodeArgs) {
   StoreResultGPR(Op, Src);
 
   CalculateFlags_ZCNT(OpSizeFromSrc(Op), Src);
+
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::LZCNT(OpcodeArgs) {
@@ -4947,6 +5049,7 @@ void OpDispatchBuilder::LZCNT(OpcodeArgs) {
   auto Res = _CountLeadingZeroes(OpSizeFromSrc(Op), Src);
   StoreResultGPR(Op, Res);
   CalculateFlags_ZCNT(OpSizeFromSrc(Op), Res);
+  AcidTestFlags(X86State::RFLAG_OF_RAW_LOC, X86State::RFLAG_SF_RAW_LOC, X86State::RFLAG_AF_RAW_LOC, X86State::RFLAG_PF_RAW_LOC);
 }
 
 void OpDispatchBuilder::MOVBEOp(OpcodeArgs) {
@@ -5025,6 +5128,11 @@ void OpDispatchBuilder::StoreFenceOrCLFlush(OpcodeArgs) {
   }
 }
 
+void OpDispatchBuilder::SerializeOp(OpcodeArgs) {
+  _Fence(FenceType::Inst);
+  _Fence(FenceType::Store);
+}
+
 void OpDispatchBuilder::UMonitorOrCLRSSBSY(OpcodeArgs) {
   if (DestIsMem(Op) || !CTX->HostFeatures.SupportsWFXT) {
     // CLRSSBSY
@@ -5073,7 +5181,7 @@ void OpDispatchBuilder::RDTSCPOp(OpcodeArgs) {
   //  - Explicitly use an MFENCE before this instruction if you want this behaviour
   // This instruction is not an execution fence, so subsequent instructions can execute after this
   //  - Explicitly use an LFENCE after RDTSCP if you want to block this behaviour
-  if (CTX->HostFeatures.HostType != FEXCore::HostFeatures::HostTypeEnum::Linux && !CTX->HostFeatures.SupportsCPUIndexInTPIDRRO) {
+  if (CTX->HostFeatures.ProcessFeatures.HostType != FEXCore::HostFeatures::HostTypeEnum::Linux && !CTX->HostFeatures.SupportsCPUIndexInTPIDRRO) {
     // RDTSCP is unsupported on Win32 platforms if TPIDRRO isn't supported.
     UnimplementedOp(Op);
     return;
@@ -5087,7 +5195,7 @@ void OpDispatchBuilder::RDTSCPOp(OpcodeArgs) {
 }
 
 void OpDispatchBuilder::RDPIDOp(OpcodeArgs) {
-  if (CTX->HostFeatures.HostType != FEXCore::HostFeatures::HostTypeEnum::Linux && !CTX->HostFeatures.SupportsCPUIndexInTPIDRRO) {
+  if (CTX->HostFeatures.ProcessFeatures.HostType != FEXCore::HostFeatures::HostTypeEnum::Linux && !CTX->HostFeatures.SupportsCPUIndexInTPIDRRO) {
     // RDTSCP is unsupported on Win32 platforms if TPIDRRO isn't supported.
     UnimplementedOp(Op);
     return;
@@ -5143,14 +5251,13 @@ void OpDispatchBuilder::RDRANDOp(OpcodeArgs, bool Reseed) {
   }
 }
 
-void OpDispatchBuilder::BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition) {
+void OpDispatchBuilder::BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition, Ref RelocatedPC) {
   const auto GPRSize = GetGPROpSize();
 
   // We don't actually support this instruction
   // Multiblock may hit it though
   _StoreContextGPR(GPRSize, GetRelocatedPC(Op, -Op->InstSize), offsetof(FEXCore::Core::CPUState, rip));
-  Break(BreakDefinition);
-
+  Break(BreakDefinition, RelocatedPC);
 
   if (Multiblock) {
     auto NextBlock = CreateNewCodeBlockAfter(GetCurrentBlock());
@@ -5161,6 +5268,25 @@ void OpDispatchBuilder::BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDe
   }
 }
 
+void OpDispatchBuilder::BreakpointOp(OpcodeArgs, uint32_t BP) {
+  const auto GPRSize = GetGPROpSize();
+  auto NewRIP = GetRelocatedPC(Op, -Op->InstSize);
+  _StoreContextGPR(GPRSize, NewRIP, offsetof(FEXCore::Core::CPUState, rip));
+
+  uint32_t DR6Value {1U << BP};
+  _StoreContextGPR(GPRSize, _Constant(DR6Value), offsetof(FEXCore::Core::CPUState, Dr6));
+
+  IR::BreakDefinition Reason;
+  Reason.ErrorRegister = 0;
+  Reason.Signal = Core::FAULT_SIGTRAP;
+  Reason.TrapNumber = X86State::X86_TRAPNO_DB;
+  Reason.si_code = 0x80;
+
+  Break(Reason);
+
+  BlockSetRIP = true;
+}
+
 void OpDispatchBuilder::UnimplementedOp(OpcodeArgs) {
   BreakOp(Op, FEXCore::IR::BreakDefinition {
                 .ErrorRegister = 0,
@@ -5188,13 +5314,21 @@ void OpDispatchBuilder::InvalidOp(OpcodeArgs) {
               });
 }
 
-void OpDispatchBuilder::NoExecOp(OpcodeArgs) {
-  BreakOp(Op, FEXCore::IR::BreakDefinition {
-                .ErrorRegister = X86State::X86_PF_PROT | X86State::X86_PF_USER | X86State::X86_PF_INSTR,
-                .Signal = Core::FAULT_SIGSEGV,
-                .TrapNumber = X86State::X86_TRAPNO_PF,
-                .si_code = 2, // SEGV_ACCERR
-              });
+void OpDispatchBuilder::NoExecOrPartialOp(OpcodeArgs) {
+  // Partial decoded or fully no-exec instruction.
+  // Get the RIP inside the instruction where it failed to decode.
+  // - Full no-exec will be the first byte.
+  // - Partial will be the first byte that failed to decode.
+  auto RelocatedPC = GetRelocatedPC(Op, 0);
+
+  BreakOp(Op,
+          FEXCore::IR::BreakDefinition {
+            .ErrorRegister = X86State::X86_PF_PROT | X86State::X86_PF_USER | X86State::X86_PF_INSTR,
+            .Signal = Core::FAULT_SIGSEGV,
+            .TrapNumber = X86State::X86_TRAPNO_PF,
+            .si_code = 2, // SEGV_ACCERR
+          },
+          RelocatedPC);
 }
 
 #undef OpcodeArgs

FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -207,9 +207,12 @@ public:
     FlushRegisterCache();
     return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock, PatchSiteAddress, PatchSiteSize);
   }
-  IRPair<IROp_Break> Break(BreakDefinition Reason) {
+  IRPair<IROp_Break> Break(BreakDefinition Reason, Ref RelocatedPC = nullptr) {
     FlushRegisterCache();
-    return _Break(Reason);
+    if (!RelocatedPC) {
+      RelocatedPC = Constant(0);
+    }
+    return _Break(Reason, RelocatedPC);
   }
   IRPair<IROp_Thunk> Thunk(Ref ArgPtr, SHA256Sum ThunkNameHash) {
     FlushRegisterCache();
@@ -217,6 +220,8 @@ public:
   }
 
   bool FinishOp(uint64_t NextRIP, bool LastOp) {
+    ++RegCache.Index;
+
     // If we are switching to a new block and this current block has yet to set a RIP
     // Then we need to insert an unconditional jump from the current block to the one we are going to
     // This happens most frequently when an instruction jumps backwards to another location
@@ -488,6 +493,7 @@ public:
   void SGDTOp(OpcodeArgs);
   void SIDTOp(OpcodeArgs);
   void SMSWOp(OpcodeArgs);
+  void SLDTOp(OpcodeArgs);
 
   enum class VectorOpType {
     MMX,
@@ -755,6 +761,7 @@ public:
   void X87LDENV(OpcodeArgs);
   void X87ModifySTP(OpcodeArgs, bool Inc);
   void X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2);
+  void X87NOP(OpcodeArgs);
 
   enum class FCOMIFlags {
     FLAGS_X87,
@@ -828,6 +835,7 @@ public:
   void LoadFenceOrXRSTOR(OpcodeArgs);
   void MemFenceOrXSAVEOPT(OpcodeArgs);
   void StoreFenceOrCLFlush(OpcodeArgs);
+  void SerializeOp(OpcodeArgs);
   void UMonitorOrCLRSSBSY(OpcodeArgs);
   void UMWaitOp(OpcodeArgs);
   void CLZeroOp(OpcodeArgs);
@@ -894,7 +902,8 @@ public:
   void Extrq(OpcodeArgs);
   void Insertq(OpcodeArgs);
 
-  void BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition);
+  void BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition, Ref RelocatedPC = nullptr);
+  void BreakpointOp(OpcodeArgs, uint32_t BP);
   void UnimplementedOp(OpcodeArgs);
   void PermissionRestrictedOp(OpcodeArgs);
 
@@ -1092,6 +1101,11 @@ public:
 
   void AVX128_VCVTPH2PS(OpcodeArgs);
   void AVX128_VCVTPS2PH(OpcodeArgs);
+  void AVX128_VBCSTNEBF162PS(OpcodeArgs);
+  void AVX128_VBCSTNESH2PS(OpcodeArgs);
+  void AVX128_VCVTNEEBF162PS(OpcodeArgs, bool Even);
+  void AVX128_VCVTNEEPH2PS(OpcodeArgs, bool Even);
+  void AVX128_VCVTNEPS2BF16(OpcodeArgs);
 
   // End of AVX 128-bit implementation
 
@@ -1126,10 +1140,16 @@ public:
   void AVXVectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
   void AVXVectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);
 
+  void AVXVBCSTNEBF162PS(OpcodeArgs);
+  void AVXVBCSTNESH2PS(OpcodeArgs);
+  void AVXVCVTNEEBF162PS(OpcodeArgs, bool Even);
+  void AVXVCVTNEEPH2PS(OpcodeArgs, bool Even);
+  void AVXVCVTNEPS2BF16(OpcodeArgs);
+
   // End of AVX 256-bit implementation
 
   void InvalidOp(OpcodeArgs);
-  void NoExecOp(OpcodeArgs);
+  void NoExecOrPartialOp(OpcodeArgs);
 
   void SetPackedRFLAG(bool Lower8, Ref Src);
   Ref GetPackedRFLAG(uint32_t FlagsMask = ~0U);
@@ -1180,10 +1200,20 @@ public:
     if (!SRAOnly) {
       RectifyCarryInvert(CFInvertedABI);
 
-      // Also we can't trust the LastDef accross block boundaries, so clear that as well.
+      // Also we can't trust the LastDef or PrevDef arrays accross block boundaries, so clear them as well.
       for (auto& Def : RegCache.LastDef) {
         Def = nullptr;
       }
+      for (auto& Def : RegCache.PrevDef) {
+        Def = nullptr;
+      }
+      for (auto& DefIndex : RegCache.LastDefIndex) {
+        DefIndex = 0;
+      }
+      for (auto& DefIndex : RegCache.PrevDefIndex) {
+        DefIndex = 0;
+      }
+      RegCache.Index = 0;
     }
 
     if (!MMXOnly) {
@@ -1300,7 +1330,7 @@ protected:
       /* On ASIMD platforms, the emulation happens to preserve NZCV, unlike the
        * more optimal SVE implementation that clobbers.
        */
-      if (!CTX->HostFeatures.SupportsSVE128 && !CTX->HostFeatures.SupportsSVE256) {
+      if (!CTX->HostFeatures.SupportsSVE()) {
         return;
       }
 
@@ -1321,6 +1351,7 @@ protected:
 
 private:
   FEX_CONFIG_OPT(ReducedPrecisionMode, X87REDUCEDPRECISION);
+  FEX_CONFIG_OPT(UndefinedFlagAcidTest, UNDEFINEDFLAGACIDTEST);
 
   struct JumpTargetInfo {
     Ref BlockEntry;
@@ -1503,7 +1534,7 @@ private:
   void SaveAVXState(Ref MemBase);
 
   void XRstorOpImpl(OpcodeArgs);
-  void RestoreX87State(Ref MemBase);
+  void RestoreX87State(OpcodeArgs, Ref MemBase);
   void RestoreSSEState(Ref MemBase);
   void RestoreMXCSRState(Ref MXCSR);
   void RestoreAVXState(Ref MemBase);
@@ -1646,7 +1677,7 @@ private:
   }
 
   constexpr OpSize GetGuestVectorLength() const {
-    return (CTX->HostFeatures.SupportsSVE256 && CTX->HostFeatures.SupportsAVX) ? OpSize::i256Bit : OpSize::i128Bit;
+    return (CTX->HostFeatures.SupportsSVE256() && CTX->HostFeatures.SupportsAVX()) ? OpSize::i256Bit : OpSize::i128Bit;
   }
 
   [[nodiscard]]
@@ -1904,6 +1935,55 @@ private:
     InvalidateReg(Core::CPUState::AF_AS_GREG);
   }
 
+  // Sets the desired flags to zero when acid test is enabled.
+  // Not meant to be fast, just meant to produce consistent results.
+  void AcidTestFlags(unsigned BitOffset = 0, unsigned BitOffset2 = 0, unsigned BitOffset3 = 0, unsigned BitOffset4 = 0,
+                     unsigned BitOffset5 = 0, unsigned BitOffset6 = 0) {
+    if (!UndefinedFlagAcidTest) [[likely]] {
+      return;
+    }
+
+    unsigned BitFlags {};
+    if (BitOffset) {
+      BitFlags |= 1U << BitOffset;
+    }
+    if (BitOffset2) {
+      BitFlags |= 1U << BitOffset2;
+    }
+    if (BitOffset3) {
+      BitFlags |= 1U << BitOffset3;
+    }
+    if (BitOffset4) {
+      BitFlags |= 1U << BitOffset4;
+    }
+    if (BitOffset5) {
+      BitFlags |= 1U << BitOffset5;
+    }
+    if (BitOffset6) {
+      BitFlags |= 1U << BitOffset6;
+    }
+
+    auto Flag = Constant(0);
+    if (BitFlags & (1U << X86State::RFLAG_SF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_SF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_ZF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_ZF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_CF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_CF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_OF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_OF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_AF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_AF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_AF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_AF_RAW_LOC);
+    }
+  }
+
   [[nodiscard]]
   static CondClass CondForNZCVBit(unsigned BitOffset, bool Invert) {
     switch (BitOffset) {
@@ -1971,8 +2051,16 @@ private:
     Ref Value[64];
 
     // Keep track of where each register was last written to within the current block,
-    // for use in peephole optimizations.
+    // for use in peephole optimizations. LastDefIndex is a parallel structure,
+    // which tracks the index of each Op in LastDef.
     Ref LastDef[64];
+    uint32_t LastDefIndex[64];
+    // Similar to LastDef, but tracks the write before LastDef. Think of it as LastDef^2.
+    Ref PrevDef[64];
+    uint32_t PrevDefIndex[64];
+
+    // The index of the current instruction in the current block.
+    uint32_t Index;
   } RegCache {};
 
   void InvalidateReg(uint8_t Index) {
@@ -2095,7 +2183,10 @@ private:
     RegCache.Value[Index] = Value;
     RegCache.Cached |= Bit;
     RegCache.Written |= Bit;
+    RegCache.PrevDef[Index] = RegCache.LastDef[Index];
+    RegCache.PrevDefIndex[Index] = RegCache.LastDefIndex[Index];
     RegCache.LastDef[Index] = Value;
+    RegCache.LastDefIndex[Index] = RegCache.Index;
   }
 
   void InvalidateHighAVXRegisters() {
@@ -2207,8 +2298,7 @@ private:
     Ref V_inv = GetRFLAG(FEXCore::X86State::RFLAG_OF_RAW_LOC, true);
     SetRFLAG<FEXCore::X86State::RFLAG_PF_RAW_LOC>(V_inv);
 
-    // Intel: OF, SF, and AF set to zero
-    // AMD:   no mention of OF, SF and AF but actual hardware seems to always zero
+    // Both Intel and AMD manual state that OF, SF, and AF are set to zero.
     //
     // Zero AF. Note that the comparison sets the raw PF to 0/1 above, so
     // PF[4] is 0 so the XOR with PF will have no effect, so setting the AF
@@ -2439,6 +2529,12 @@ private:
   void SetX87FTW(Ref FTW);
   Ref GetX87FTW_Helper();
   void SetX87Top(Ref Value);
+  void StoreX87FIP(FEXCore::X86Tables::DecodedOp Op) {
+    if (CTX->Config.JITFeatures & FEXCore::Config::JITFeatures::ENABLEX87FIP) {
+      // Only store FIP if the feature is enabled.
+      _StoreContextGPR(OpSize::i64Bit, GetRelocatedPC(Op, -Op->InstSize), offsetof(FEXCore::Core::CPUState, FIP));
+    }
+  }
 
   void ChgStateX87_MMX() override {
     LOGMAN_THROW_A_FMT(MMXState == MMXState_X87, "Expected state to be x87");

FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp

@@ -2067,7 +2067,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
     BaseAddr = Invalid();
   }
 
-  if (CTX->HostFeatures.SupportsSVE128) {
+  if (CTX->HostFeatures.SupportsSVE128()) {
     if (ElementLoadSize == OpSize::i64Bit && AddrElementSize == OpSize::i32Bit) {
       // In the case that FEX is loading double the amount of data than the number of address bits then we can optimize this case.
       // For 256-bits of data we need to sign extend all four 32-bit address elements to be 64-bit.
@@ -2170,7 +2170,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs,
 
   bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (BaseAddr == Invalid() && VSIB.Scale != 1);
 
-  if (CTX->HostFeatures.SupportsSVE128 && NeedsSVEScale) {
+  if (CTX->HostFeatures.SupportsSVE128() && NeedsSVEScale) {
     // SVE gather instructions don't support scaling their vector elements by anything other than 1 or the address element size.
     // Pre-scale 64-bit addresses in the case that scale doesn't match in-order to hit SVE code paths more frequently.
     // Only hit this path if the host supports SVE. Otherwise it's a degradation for the ASIMD codepath.
@@ -2236,7 +2236,7 @@ void OpDispatchBuilder::AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize) {
   bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (VSIB.BaseAddr == Invalid() && VSIB.Scale != 1);
 
   const bool NeedsExplicitSVEPath =
-    CTX->HostFeatures.SupportsSVE128 && AddrElementSize == OpSize::i32Bit && ElementLoadSize == OpSize::i32Bit && NeedsSVEScale;
+    CTX->HostFeatures.SupportsSVE128() && AddrElementSize == OpSize::i32Bit && ElementLoadSize == OpSize::i32Bit && NeedsSVEScale;
 
   RefPair Result {};
   if (NeedsExplicitSVEPath) {
@@ -2355,4 +2355,124 @@ void OpDispatchBuilder::AVX128_VCVTPS2PH(OpcodeArgs) {
   }
 }
 
+void OpDispatchBuilder::AVX128_VBCSTNEBF162PS(OpcodeArgs) {
+  const auto DstSize = GetDstSize(Op);
+  const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
+
+  auto Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags);
+
+  // bfloat16 is converted to fp32 and broadcast through the vector.
+  // bfloat16->float32 is left-shift by 16.
+  Src1 = _VShlI(OpSize::i128Bit, OpSize::i32Bit, Src1, 16);
+  Src1 = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src1, 0);
+
+  RefPair Result {};
+  Result.Low = Src1;
+
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Result.Low;
+  }
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VBCSTNESH2PS(OpcodeArgs) {
+  const auto DstSize = GetDstSize(Op);
+  const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
+
+  auto Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags);
+
+  // float16 is converted to fp32 and broadcast through the vector.
+  Src1 = _Float_FToF(OpSize::i32Bit, OpSize::i16Bit, Src1);
+  Src1 = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src1, 0);
+
+  RefPair Result {};
+  Result.Low = Src1;
+
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Result.Low;
+  }
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VCVTNEEBF162PS(OpcodeArgs, bool Even) {
+  const auto DstSize = GetDstSize(Op);
+  const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
+
+  auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);
+
+  RefPair Result {};
+
+  auto Convert = [&](auto Src, bool Even) -> Ref {
+    // Even or odd elements of each 128-bit register is converted from bfloat16 to float32.
+    // For even, just left shift the elements.
+    if (Even) {
+      return _VShlI(OpSize::i128Bit, OpSize::i32Bit, Src, 16);
+    }
+
+    // For Odds, we just need to mask off the lower 16-bits of each 32-bit element.
+    // Easiest way to do this is with a right then left shift.
+    auto Tmp = _VUShrI(OpSize::i128Bit, OpSize::i32Bit, Src, 16);
+    return _VShlI(OpSize::i128Bit, OpSize::i32Bit, Tmp, 16);
+  };
+
+  Result.Low = Convert(Src.Low, Even);
+
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Convert(Src.High, Even);
+  }
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VCVTNEEPH2PS(OpcodeArgs, bool Even) {
+  const auto DstSize = GetDstSize(Op);
+  const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
+
+  auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);
+
+  RefPair Result {};
+
+  auto Convert = [&](auto Src, bool Even) -> Ref {
+    // Transpose even or odd elements and then fcvtl.
+    if (Even) {
+      Src = _VUnZip(OpSize::i128Bit, OpSize::i16Bit, Src, Src);
+    } else {
+      Src = _VUnZip2(OpSize::i128Bit, OpSize::i16Bit, Src, Src);
+    }
+    return _Vector_FToF(OpSize::i128Bit, OpSize::i32Bit, Src, OpSize::i16Bit);
+  };
+
+  Result.Low = Convert(Src.Low, Even);
+
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Convert(Src.High, Even);
+  }
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VCVTNEPS2BF16(OpcodeArgs) {
+  const auto SrcSize = OpSizeFromSrc(Op);
+  const auto Is128BitSrc = SrcSize == OpSize::i128Bit;
+  auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128BitSrc);
+
+  // Convert 4 or 8 32-bits floats in to bfloat16.
+  // This instruction only operates with rounding mode set to `Rounds to Nearest (even)`.
+
+  RefPair Result {};
+  Result.Low = _VBFCVTN(OpSize::i128Bit, Src.Low);
+  if (!Is128BitSrc) {
+    Result.Low = _VBFCVTN2(OpSize::i128Bit, Result.Low, Src.High);
+  }
+
+  Result.High = LoadZeroVector(OpSize::i128Bit);
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp

@@ -27,7 +27,7 @@ void OpDispatchBuilder::SHA1NEXTEOp(OpcodeArgs) {
   Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
 
   Ref Result {};
-  if (CTX->HostFeatures.SupportsSVE128) {
+  if (CTX->HostFeatures.SupportsSVE128()) {
     auto ZeroVec = LoadZeroVector(OpSize::i128Bit);
     auto Tmp = _VInsElement(OpSize::i128Bit, OpSize::i32Bit, 3, 3, ZeroVec, Dest);
     auto Xar = _VXar(OpSize::i128Bit, OpSize::i32Bit, ZeroVec, Tmp, 2);
@@ -158,11 +158,6 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
     UnimplementedOp(Op);
     return;
   }
-  Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);
-  Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
-  // Hardcoded to XMM0
-  auto XMM0 = LoadXMMRegister(0);
-
   auto shuffle_abcd = [this](Ref Src1, Ref Src2) -> Ref {
     // Generates a suitable SHA256 `abcd` configuration from x86 format.
     auto Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
@@ -175,14 +170,99 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
     return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
   };
 
+  // Peephole: Attempt to identify this pattern:
+  //   sha256rnds2 xmm1, xmm2
+  //   pshufd xmm0, xmm0, 0xe (or psrldq xmm0, 8)
+  //   sha256rnds2 xmm2, xmm1
+  // This is an idiom used by various crypto libraries (OpenSSL etc).
+  // Since the ARM64 SHA256 instructions compute all four rounds,
+  // we can elide the second SHA256RNDS2 and the key shuffle.
+  // Since this matching is done on IR nodes, this is somewhat fragile,
+  // since changes to the translation of the constitiuent instructions
+  // would break the pattern.
+  //
+  // NB: xmm1 and xmm2 are used here as examples, but note that
+  // xmm0 is hardcoded as the key register in SHA256RNDS2.
+
+  if (Op->Dest.IsGPR() && Op->Src[0].IsGPR()) {
+    // xmm1 in our example above.
+    const auto X = Op->Src[0].Data.GPR.GPR - X86State::REG_XMM_0;
+    // xmm2 in our example above.
+    const auto Y = Op->Dest.Data.GPR.GPR - X86State::REG_XMM_0;
+
+    const int KeyIndex = FPR0Index;
+    const int XIndex = FPR0Index + X;
+    const int YIndex = FPR0Index + Y;
+
+    // It is helpful to remember that the current Op
+    // is the "second" SHA256RNDS2 in the example, so we are looking
+    // backwards to find the rest of the pattern.
+    const Ref XLastWritten = RegCache.LastDef[XIndex];
+    const auto XLastWrittenIndex = RegCache.LastDefIndex[XIndex];
+    const Ref KeyLastWritten = RegCache.LastDef[KeyIndex];
+    // xmm0 and X must have been written to in in this block.
+    if (!XLastWritten || !KeyLastWritten) {
+      goto no_peephole;
+    }
+
+    // True if no instruction wrote Y after the first SHA256RNDS2.
+    const bool YUnchanged = !RegCache.LastDef[YIndex] || RegCache.LastDefIndex[YIndex] < XLastWrittenIndex;
+    // True if the key shift is after the first SHA256RNDS2, and no instruction wrote XMM0 afterwards.
+    const bool KeyUnchanged = RegCache.LastDefIndex[KeyIndex] > XLastWrittenIndex &&
+                              (!RegCache.PrevDef[KeyIndex] || RegCache.PrevDefIndex[KeyIndex] < XLastWrittenIndex);
+    if (!YUnchanged || !KeyUnchanged) {
+      goto no_peephole;
+    }
+
+    // Now make sure that the op that wrote the key is what we expect.
+    // This should match the translation of a `pshufd xmm0, xmm0, 0xe` or `psrldq xmm0, 8` instruction.
+    const auto KeyShift = GetOpHeader(WrapNode(KeyLastWritten));
+    const bool IsVExtr = KeyShift->Op == OP_VEXTR && KeyShift->Size == OpSize::i128Bit;
+    if (!IsVExtr) {
+      goto no_peephole;
+    }
+
+    // Compute shift in bytes, since pshufd and psrldq use different element sizes.
+    const auto ShiftInBytes = KeyShift->C<IR::IROp_VExtr>()->Index * IR::OpSizeToSize(KeyShift->ElementSize);
+    const bool IsKeyShift = ShiftInBytes == 8 && IsValueLoadRegister(KeyShift->C<IR::IROp_VExtr>()->VectorUpper, RegClass::FPR, 0);
+    if (!IsKeyShift) {
+      goto no_peephole;
+    }
+
+    // Now check that the rest of the OPs match the pattern.
+    const Ref FirstZip = GetArgIfOp(XLastWritten, OP_VREV64, 0);
+    // A and B are the SHA256H and SHA256H2 output of the first SHA256RNDS2 instruction
+    const Ref A = GetArgIfOp(FirstZip, OP_VZIP2, 1);
+    const Ref B = GetArgIfOp(FirstZip, OP_VZIP2, 0);
+    // Figure out based on A what the source of the first SHA256RNDS2 was.
+    const Ref FirstSrc = GetArgIfOp(GetArgIfOp(GetArgIfOp(A, OP_VSHA256H, 0), OP_VREV64, 0), OP_VZIP2, 0);
+    // True if X holds the result of the first SHA256RNDS2, and its source register was Y.
+    const bool IsFirstResult = FirstSrc && IsValueLoadRegister(WrapNode(FirstSrc), RegClass::FPR, Y);
+    if (!IsFirstResult) {
+      goto no_peephole;
+    }
+
+    // The pattern matched, so we can elide the second SHA256RNDS2,
+    // and instead use the result of the first pair of SHA256H and SHA256H2 instructions
+    // to populate the final result.
+    StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, shuffle_efgh(A, B));
+    return;
+  }
+
+no_peephole:
+  Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);
+  Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+  // Hardcoded to XMM0
+  auto XMM0 = LoadXMMRegister(0);
+
   auto ABCD = shuffle_abcd(Dest, Src);
   auto EFGH = shuffle_efgh(Dest, Src);
 
-  // x86 uses only the bottom 64-bits of the key, so duplicate to match ARM64 semantics.
-  auto Key = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, XMM0, 0);
-
-  auto A = _VSha256H(ABCD, EFGH, Key);
-  auto B = _VSha256H2(EFGH, ABCD, Key);
+  // x86 uses only the lower 64 bits of the key for two rounds.
+  // ARM64 does four rounds, so in order to support the peephole optimization,
+  // we use the full key always.
+  auto A = _VSha256H(ABCD, EFGH, XMM0);
+  auto B = _VSha256H2(EFGH, ABCD, XMM0);
   auto Result = shuffle_abcd(A, B);
 
   StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result);

FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryGroupTables.h

@@ -10,6 +10,11 @@ constexpr uint16_t PF_66 = 2;
 constexpr uint16_t PF_F2 = 3;
 constexpr DispatchTableEntry OpDispatch_SecondaryGroupTables[] = {
   // GROUP 6
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_NONE, 0), 1, &OpDispatchBuilder::SLDTOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F3, 0), 1, &OpDispatchBuilder::SLDTOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_66, 0), 1, &OpDispatchBuilder::SLDTOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F2, 0), 1, &OpDispatchBuilder::SLDTOp},
+
   {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_NONE, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F3, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_66, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
@@ -36,6 +41,8 @@ constexpr DispatchTableEntry OpDispatch_SecondaryGroupTables[] = {
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 4), 1, &OpDispatchBuilder::SMSWOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F2, 4), 1, &OpDispatchBuilder::SMSWOp},
 
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 5), 1, &OpDispatchBuilder::SerializeOp},
+
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F3, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},

FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryModRMTables.h

@@ -10,12 +10,12 @@ constexpr DispatchTableEntry OpDispatch_SecondaryModRMTables[] = {
 
   // REG /2
   {((1 << 3) | 0), 1, &OpDispatchBuilder::XGetBVOp},
+  {((1 << 3) | 1), 1, &OpDispatchBuilder::PermissionRestrictedOp},
 
   // REG /3
   {((2 << 3) | 7), 1, &OpDispatchBuilder::PermissionRestrictedOp},
 
   // REG /7
-  {((3 << 3) | 0), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {((3 << 3) | 1), 1, &OpDispatchBuilder::RDTSCPOp},
   {((3 << 3) | 4), 1, &OpDispatchBuilder::CLZeroOp},
 };

FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp

@@ -3025,7 +3025,7 @@ void OpDispatchBuilder::XSaveOpImpl(OpcodeArgs) {
     StoreIfFlagSet(1, [this, Op] { SaveSSEState(XSaveBase(Op)); });
   }
   // AVX
-  if (CTX->HostFeatures.SupportsAVX) {
+  if (CTX->HostFeatures.SupportsAVX()) {
     StoreIfFlagSet(2, [this, Op] { std::invoke(SaveAVXStateFunc, this, XSaveBase(Op)); });
   }
 
@@ -3072,6 +3072,12 @@ void OpDispatchBuilder::SaveX87State(OpcodeArgs, Ref MemBase) {
 
   { _StoreMemGPR(OpSize::i16Bit, ReconstructFSW_Helper(), MemBase, Constant(2), OpSize::i16Bit, MemOffsetType::SXTX, 1); }
 
+  {
+    auto FIPSize = Op->Flags & X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
+    auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
+    _StoreMemGPR(FIPSize, FIP, MemBase, Constant(8), FIPSize, MemOffsetType::SXTX, 1);
+  }
+
   {
     // Abridged FTW
     auto FTW = _LoadContextGPR(OpSize::i8Bit, offsetof(FEXCore::Core::CPUState, AbridgedFTW));
@@ -3173,7 +3179,7 @@ Ref OpDispatchBuilder::GetMXCSR() {
 void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {
   Ref Mem = MakeSegmentAddress(Op, Op->Src[0]);
 
-  RestoreX87State(Mem);
+  RestoreX87State(Op, Mem);
   RestoreSSEState(Mem);
 
   Ref MXCSR = _LoadMemGPR(OpSize::i32Bit, Mem, Constant(24), OpSize::i32Bit, MemOffsetType::SXTX, 1);
@@ -3219,14 +3225,14 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
 
   // x87
   {
-    RestoreIfFlagSetOrDefault(0, [this, Op] { RestoreX87State(XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });
+    RestoreIfFlagSetOrDefault(0, [this, Op] { RestoreX87State(Op, XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });
   }
   // SSE
   {
     RestoreIfFlagSetOrDefault(1, [this, Op] { RestoreSSEState(XSaveBase(Op)); }, [this] { DefaultSSEState(); });
   }
   // AVX
-  if (CTX->HostFeatures.SupportsAVX) {
+  if (CTX->HostFeatures.SupportsAVX()) {
     RestoreIfFlagSetOrDefault(
       2, [this, Op] { std::invoke(RestoreAVXStateFunc, this, XSaveBase(Op)); }, [this] { std::invoke(DefaultAVXStateFunc, this); });
   }
@@ -3244,7 +3250,7 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
   }
 }
 
-void OpDispatchBuilder::RestoreX87State(Ref MemBase) {
+void OpDispatchBuilder::RestoreX87State(OpcodeArgs, Ref MemBase) {
   _StackForceSlow();
 
   auto NewFCW = _LoadMemGPR(OpSize::i16Bit, MemBase, OpSize::i16Bit);
@@ -3256,6 +3262,12 @@ void OpDispatchBuilder::RestoreX87State(Ref MemBase) {
     Top = ReconstructX87StateFromFSW_Helper(NewFSW);
   }
 
+  {
+    auto FIPSize = Op->Flags & X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
+    auto FIP = _LoadMemGPR(FIPSize, MemBase, Constant(8), FIPSize, MemOffsetType::SXTX, 1);
+    _StoreContextGPR(OpSize::i64Bit, FIP, offsetof(FEXCore::Core::CPUState, FIP));
+  }
+
   {
     // Abridged FTW
     auto NewFTW = _LoadMemGPR(OpSize::i8Bit, MemBase, Constant(4), OpSize::i8Bit, MemOffsetType::SXTX, 1);
@@ -6056,4 +6068,86 @@ void OpDispatchBuilder::Insertq(OpcodeArgs) {
   StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result);
 }
 
+void OpDispatchBuilder::AVXVBCSTNEBF162PS(OpcodeArgs) {
+  const auto DstSize = OpSizeFromDst(Op);
+
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  // bfloat16 is converted to fp32 and broadcast through the vector.
+  // bfloat16->float32 is left-shift by 16.
+  Src = _VShlI(DstSize, OpSize::i32Bit, Src, 16);
+  Src = _VDupElement(DstSize, OpSize::i32Bit, Src, 0);
+
+  StoreResultFPR(Op, Src);
+}
+
+void OpDispatchBuilder::AVXVBCSTNESH2PS(OpcodeArgs) {
+  const auto DstSize = OpSizeFromDst(Op);
+
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  // float16 is converted to fp32 and broadcast through the vector.
+  Src = _Float_FToF(OpSize::i32Bit, OpSize::i16Bit, Src);
+  Src = _VDupElement(DstSize, OpSize::i32Bit, Src, 0);
+
+  StoreResultFPR(Op, Src);
+}
+
+void OpDispatchBuilder::AVXVCVTNEEPH2PS(OpcodeArgs, bool Even) {
+  const auto DstSize = OpSizeFromDst(Op);
+
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  // float16 is converted to fp32 and broadcast through the vector.
+  if (Even) {
+    Src = _VUnZip(DstSize, OpSize::i16Bit, Src, Src);
+  } else {
+    Src = _VUnZip2(DstSize, OpSize::i16Bit, Src, Src);
+  }
+  Src = _Vector_FToF(DstSize, OpSize::i32Bit, Src, OpSize::i16Bit);
+
+  StoreResultFPR(Op, Src);
+}
+
+void OpDispatchBuilder::AVXVCVTNEEBF162PS(OpcodeArgs, bool Even) {
+  const auto DstSize = OpSizeFromDst(Op);
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  auto Convert = [&](auto Src, bool Even) -> Ref {
+    // Even or odd elements of each 128-bit register is converted from bfloat16 to float32.
+    // For even, just left shift the elements.
+    if (Even) {
+      return _VShlI(DstSize, OpSize::i32Bit, Src, 16);
+    }
+
+    // For Odds, we just need to mask off the lower 16-bits of each 32-bit element.
+    // Can't do the same movi+vbsl instruction as ASIMD. Just use shifts instead.
+    auto Tmp = _VUShrI(DstSize, OpSize::i32Bit, Src, 16);
+    return _VShlI(DstSize, OpSize::i32Bit, Tmp, 16);
+  };
+  StoreResultFPR(Op, Convert(Src, Even));
+}
+
+void OpDispatchBuilder::AVXVCVTNEPS2BF16(OpcodeArgs) {
+  const auto DstSize = OpSizeFromDst(Op);
+  const auto Is128Bit = DstSize == OpSize::i128Bit;
+
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  // Convert 4 or 8 32-bits floats in to bfloat16.
+  // This instruction only operates with rounding mode set to `Rounds to Nearest (even)`.
+  Ref Result {};
+  if (Is128Bit) {
+    // Just do the operation in the lower half of the register.
+    Result = _VBFCVTN(OpSize::i128Bit, Src);
+  } else {
+    // Split the operation in to two and recombine.
+    Ref UpperLane = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, Src, 1);
+    auto ResultLow = _VBFCVTN(OpSize::i128Bit, Src);
+    Result = _VBFCVTN2(OpSize::i128Bit, ResultLow, UpperLane);
+  }
+
+  StoreResultFPR(Op, Result);
+}
+
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/X87.cpp

@@ -67,11 +67,13 @@ void OpDispatchBuilder::FLD(OpcodeArgs, IR::OpSize Width) {
     ConvertedData = _F80CVTTo(Data, Width);
   }
   _PushStack(ConvertedData, Data, Width);
+  StoreX87FIP(Op);
 }
 
 // Float LoaD operation with memory operand
 void OpDispatchBuilder::FLDFromStack(OpcodeArgs) {
   _CopyPushStack(Op->OP & 7);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FBLD(OpcodeArgs) {
@@ -79,18 +81,21 @@ void OpDispatchBuilder::FBLD(OpcodeArgs) {
   Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::f80Bit, Op->Flags);
   Ref ConvertedData = _F80BCDLoad(Data);
   _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FBSTP(OpcodeArgs) {
   Ref converted = _F80BCDStore(_ReadStackValue(0));
   StoreResultFPR_WithOpSize(Op, Op->Dest, converted, OpSize::f80Bit, OpSize::i8Bit);
   _PopStackDestroy();
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FLD_Const(OpcodeArgs, NamedVectorConstant K) {
   // Update TOP
   Ref Data = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, K);
   _PushStack(Data, Data, OpSize::f80Bit);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FILD(OpcodeArgs) {
@@ -122,6 +127,7 @@ void OpDispatchBuilder::FILD(OpcodeArgs) {
 
   Ref ConvertedData = _VLoadTwoGPRs(shifted, upper);
   _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
@@ -135,6 +141,7 @@ void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FSTToStack(OpcodeArgs) {
@@ -144,6 +151,7 @@ void OpDispatchBuilder::FSTToStack(OpcodeArgs) {
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 // Store integer to memory (possibly with truncation)
@@ -183,6 +191,7 @@ void OpDispatchBuilder::FIST(OpcodeArgs, bool Truncate) {
   if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -197,6 +206,7 @@ void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -213,6 +223,7 @@ void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
 
   // top of stack is at offset zero
   _F80AddValue(0, Arg);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -227,6 +238,7 @@ void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -247,6 +259,7 @@ void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -264,6 +277,7 @@ void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -288,6 +302,7 @@ void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -305,6 +320,7 @@ void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -329,6 +345,7 @@ void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 Ref OpDispatchBuilder::GetX87FTW_Helper() {
@@ -398,7 +415,8 @@ void OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {
 
   {
     // Instruction Offset
-    _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
+    auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
+    _StoreMemGPR(Size, FIP, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
   }
 
   {
@@ -496,7 +514,8 @@ void OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {
 
   {
     // Instruction Offset
-    _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
+    auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
+    _StoreMemGPR(Size, FIP, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
   }
 
   {
@@ -566,6 +585,12 @@ void OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {
     SetX87FTW(_LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MemOffsetType::SXTX, 1));
   }
 
+  {
+    // FIP
+    auto FIP = _LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
+    _StoreContextGPR(OpSize::i64Bit, FIP, offsetof(FEXCore::Core::CPUState, FIP));
+  }
+
   auto SevenConst = Constant(7);
   auto low = Constant(~0ULL);
   auto high = Constant(0xFFFF);
@@ -619,6 +644,7 @@ void OpDispatchBuilder::FXCH(OpcodeArgs) {
     _F80StackXchange(Offset);
   }
   SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FYL2X(OpcodeArgs, bool IsFYL2XP1) {
@@ -627,6 +653,7 @@ void OpDispatchBuilder::X87FYL2X(OpcodeArgs, bool IsFYL2XP1) {
   } else {
     _F80FYL2XStack();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::FCOMIFlags WhichFlags, bool PopTwice) {
@@ -691,6 +718,7 @@ void OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDisp
   } else if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FTST(OpcodeArgs) {
@@ -710,6 +738,7 @@ void OpDispatchBuilder::FTST(OpcodeArgs) {
   // Set Invalid Operation flag when unordered (NaN comparison)
   // The x87 exception flags are sticky. Preserve earlier result
   SetRFLAG<FEXCore::X86State::X87FLAG_IE_LOC>(_Or(OpSize::i32Bit, GetRFLAG(FEXCore::X86State::X87FLAG_IE_LOC), HostFlag_Unordered));
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2) {
@@ -717,6 +746,7 @@ void OpDispatchBuilder::X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool Ze
   if (ZeroC2) {
     SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(Constant(0));
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87ModifySTP(OpcodeArgs, bool Inc) {
@@ -728,6 +758,7 @@ void OpDispatchBuilder::X87ModifySTP(OpcodeArgs, bool Inc) {
 
   // C1 set to 0
   SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
+  StoreX87FIP(Op);
 }
 
 // Operations dealing with loading and storing environment pieces
@@ -809,15 +840,20 @@ void OpDispatchBuilder::FNINIT(OpcodeArgs) {
   SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(Zero);
   SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(Zero);
   SetRFLAG<FEXCore::X86State::X87FLAG_IE_LOC>(Zero);
+
+  // fninit resets the FIP.
+  _StoreContextGPR(OpSize::i64Bit, _Constant(0), offsetof(FEXCore::Core::CPUState, FIP));
 }
 
 void OpDispatchBuilder::X87FFREE(OpcodeArgs) {
   _InvalidateStack(Op->OP & 7);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87EMMS(OpcodeArgs) {
   // Tags all get set to 0b11
   _InvalidateStack(0xff);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
@@ -856,6 +892,7 @@ void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
 
   Ref VecCond = _VDupFromGPR(OpSize::i128Bit, OpSize::i64Bit, SelectCC0All1(CC));
   _F80VBSLStack(OpSize::i128Bit, VecCond, Op->OP & 7, 0);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FXAM(OpcodeArgs) {
@@ -960,6 +997,7 @@ void OpDispatchBuilder::X87FXAM(OpcodeArgs) {
   SetRFLAG<FEXCore::X86State::X87FLAG_C0_LOC>(C0);
   SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(C2);
   SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(C3);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FXTRACT(OpcodeArgs) {
@@ -970,6 +1008,11 @@ void OpDispatchBuilder::X87FXTRACT(OpcodeArgs) {
   auto Sig = _F80XTRACT_SIG(Top);
   _PushStack(Exp, Invalid(), OpSize::iInvalid);
   _PushStack(Sig, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
+}
+
+void OpDispatchBuilder::X87NOP(OpcodeArgs) {
+  StoreX87FIP(Op);
 }
 
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/X87F64.cpp

@@ -68,6 +68,7 @@ void OpDispatchBuilder::FLDF64(OpcodeArgs, IR::OpSize Width) {
     ConvertedData = _F80CVT(OpSize::i64Bit, Data);
   }
   _PushStack(ConvertedData, Data, Width);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
@@ -76,6 +77,7 @@ void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
   Ref ConvertedData = _F80BCDLoad(Data);
   ConvertedData = _F80CVT(OpSize::i64Bit, ConvertedData);
   _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
@@ -83,11 +85,13 @@ void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
   converted = _F80BCDStore(converted);
   StoreResultFPR_WithOpSize(Op, Op->Dest, converted, OpSize::f80Bit, OpSize::i8Bit);
   _PopStackDestroy();
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FLDF64_Const(OpcodeArgs, uint64_t Num) {
   auto Data = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(Num));
   _PushStack(Data, Data, OpSize::i64Bit);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FILDF64(OpcodeArgs) {
@@ -100,6 +104,7 @@ void OpDispatchBuilder::FILDF64(OpcodeArgs) {
   }
   auto ConvertedData = _Float_FromGPR_S(OpSize::i64Bit, ReadWidth == OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, Data);
   _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FISTF64(OpcodeArgs, bool Truncate) {
@@ -128,6 +133,7 @@ void OpDispatchBuilder::FISTF64(OpcodeArgs, bool Truncate) {
   if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -142,6 +148,7 @@ void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -165,6 +172,7 @@ void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
 
   // top of stack is at offset zero
   _F80AddValue(0, arg);
+  StoreX87FIP(Op);
 }
 
 // FIXME: following is very similar to FADDF64
@@ -180,6 +188,7 @@ void OpDispatchBuilder::FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -207,6 +216,7 @@ void OpDispatchBuilder::FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -231,6 +241,7 @@ void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -264,6 +275,7 @@ void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -288,6 +300,8 @@ void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
+
     return;
   }
 
@@ -321,6 +335,7 @@ void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FTSTF64(OpcodeArgs) {
@@ -330,6 +345,7 @@ void OpDispatchBuilder::FTSTF64(OpcodeArgs) {
   // Now we do our comparison.
   _F80StackTest(0);
   ConvertNZCVToX87();
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FCOMIF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::FCOMIFlags WhichFlags, bool PopTwice) {
@@ -376,6 +392,7 @@ void OpDispatchBuilder::FCOMIF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpD
   } else if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {
@@ -429,5 +446,6 @@ void OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {
   _PopStackDestroy();
   _PushStack(Exp, Invalid(), OpSize::iInvalid);
   _PushStack(Sig, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/X86Tables/SecondaryGroupTables.cpp

@@ -61,7 +61,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     // Pulls from other MODRM table
 
     // GROUP 6
-    {OPD(TYPE_GROUP_6, PF_NONE, 0), 1, X86InstInfo{"SLDT",  TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
+    {OPD(TYPE_GROUP_6, PF_NONE, 0), 1, X86InstInfo{"SLDT",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
     {OPD(TYPE_GROUP_6, PF_NONE, 1), 1, X86InstInfo{"STR",   TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST,  0}},
     {OPD(TYPE_GROUP_6, PF_NONE, 2), 1, X86InstInfo{"LLDT",  TYPE_PRIV, FLAGS_NONE,       0}},
     {OPD(TYPE_GROUP_6, PF_NONE, 3), 1, X86InstInfo{"LTR",   TYPE_INST, FLAGS_NONE,       0}},
@@ -70,7 +70,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     {OPD(TYPE_GROUP_6, PF_NONE, 6), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,    0}},
     {OPD(TYPE_GROUP_6, PF_NONE, 7), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,    0}},
 
-    {OPD(TYPE_GROUP_6, PF_F3, 0), 1, X86InstInfo{"SLDT",    TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
+    {OPD(TYPE_GROUP_6, PF_F3, 0), 1, X86InstInfo{"SLDT",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
     {OPD(TYPE_GROUP_6, PF_F3, 1), 1, X86InstInfo{"STR",     TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST,  0}},
     {OPD(TYPE_GROUP_6, PF_F3, 2), 1, X86InstInfo{"LLDT",    TYPE_PRIV, FLAGS_NONE,       0}},
     {OPD(TYPE_GROUP_6, PF_F3, 3), 1, X86InstInfo{"LTR",     TYPE_INST, FLAGS_NONE,       0}},
@@ -79,7 +79,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     {OPD(TYPE_GROUP_6, PF_F3, 6), 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,    0}},
     {OPD(TYPE_GROUP_6, PF_F3, 7), 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,    0}},
 
-    {OPD(TYPE_GROUP_6, PF_66, 0), 1, X86InstInfo{"SLDT",    TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
+    {OPD(TYPE_GROUP_6, PF_66, 0), 1, X86InstInfo{"SLDT",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
     {OPD(TYPE_GROUP_6, PF_66, 1), 1, X86InstInfo{"STR",     TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST,  0}},
     {OPD(TYPE_GROUP_6, PF_66, 2), 1, X86InstInfo{"LLDT",    TYPE_PRIV, FLAGS_NONE,       0}},
     {OPD(TYPE_GROUP_6, PF_66, 3), 1, X86InstInfo{"LTR",     TYPE_INST, FLAGS_NONE,       0}},
@@ -88,7 +88,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     {OPD(TYPE_GROUP_6, PF_66, 6), 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,    0}},
     {OPD(TYPE_GROUP_6, PF_66, 7), 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,    0}},
 
-    {OPD(TYPE_GROUP_6, PF_F2, 0), 1, X86InstInfo{"SLDT",    TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
+    {OPD(TYPE_GROUP_6, PF_F2, 0), 1, X86InstInfo{"SLDT",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
     {OPD(TYPE_GROUP_6, PF_F2, 1), 1, X86InstInfo{"STR",     TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST,  0}},
     {OPD(TYPE_GROUP_6, PF_F2, 2), 1, X86InstInfo{"LLDT",    TYPE_PRIV, FLAGS_NONE,       0}},
     {OPD(TYPE_GROUP_6, PF_F2, 3), 1, X86InstInfo{"LTR",     TYPE_INST, FLAGS_NONE,       0}},
@@ -103,7 +103,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     {OPD(TYPE_GROUP_7, PF_NONE, 2), 1, X86InstInfo{"",     TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
     {OPD(TYPE_GROUP_7, PF_NONE, 3), 1, X86InstInfo{"",     TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
     {OPD(TYPE_GROUP_7, PF_NONE, 4), 1, X86InstInfo{"SMSW", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,          0}},
-    {OPD(TYPE_GROUP_7, PF_NONE, 5), 1, X86InstInfo{"",     TYPE_INVALID, FLAGS_NONE,            0}},
+    {OPD(TYPE_GROUP_7, PF_NONE, 5), 1, X86InstInfo{"SERIALIZE", TYPE_INST, FLAGS_NONE,            0}},
     {OPD(TYPE_GROUP_7, PF_NONE, 6), 1, X86InstInfo{"LMSW", TYPE_INST, FLAGS_MODRM,          0}},
     {OPD(TYPE_GROUP_7, PF_NONE, 7), 1, X86InstInfo{"INVLPG", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
 

FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp

@@ -12,6 +12,19 @@ $end_info$
 
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
+enum ModRM_LUT {
+  ENTRY_3_0,
+  ENTRY_MAX,
+};
+
+constexpr std::array<X86InstInfo[2], ENTRY_MAX> ModRM_ArchSelect_LUT = {{
+  // ENTRY_3_0
+  {
+    {"SWAPGS",   TYPE_INST,    FLAGS_NONE, 0, { .OpDispatch = &IR::OpDispatchBuilder::UnimplementedOp }},
+    {"SWAPGS",   TYPE_INST,    FLAGS_NONE, 0, { .OpDispatch = &IR::OpDispatchBuilder::PermissionRestrictedOp }},
+  },
+}};
+
 constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableOps = []() consteval {
   std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> Table{};
   constexpr U8U8InfoStruct SecondaryModRMExtensionOpTable[] = {
@@ -27,7 +40,7 @@ constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableO
 
     // REG /2
     {((1 << 3) | 0), 1, X86InstInfo{"XGETBV",   TYPE_INST,    FLAGS_NONE, 0}},
-    {((1 << 3) | 1), 1, X86InstInfo{"XSETBV",   TYPE_PRIV,    FLAGS_NONE, 0}},
+    {((1 << 3) | 1), 1, X86InstInfo{"XSETBV",   TYPE_INST,    FLAGS_NONE, 0}},
     {((1 << 3) | 2), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},
     {((1 << 3) | 3), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},
     {((1 << 3) | 4), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},
@@ -46,7 +59,7 @@ constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableO
     {((2 << 3) | 7), 1, X86InstInfo{"INVLPGA",  TYPE_INST,    FLAGS_NONE, 0}},
 
     // REG /7
-    {((3 << 3) | 0), 1, X86InstInfo{"SWAPGS",   TYPE_INST,    FLAGS_NONE, 0}},
+    {((3 << 3) | 0), 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = ModRM_ArchSelect_LUT[ENTRY_3_0] }}},
     {((3 << 3) | 1), 1, X86InstInfo{"RDTSCP",   TYPE_INST,    FLAGS_NONE, 0}},
     {((3 << 3) | 2), 1, X86InstInfo{"MONITORX", TYPE_PRIV,    FLAGS_NONE, 0}},
     {((3 << 3) | 3), 1, X86InstInfo{"MWAITX",   TYPE_PRIV,    FLAGS_NONE, 0}},

FEXCore/Source/Interface/Core/X86Tables/SecondaryTables.cpp

@@ -204,7 +204,7 @@ constexpr std::array<X86InstInfo, MAX_SECOND_TABLE_SIZE> SecondBaseOps = []() co
 
     {0xA0, 1, X86InstInfo{"",        TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A0] }}},
     {0xA1, 1, X86InstInfo{"",        TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A1] }}},
-    {0xA2, 1, X86InstInfo{"CPUID",   TYPE_INST,     FLAGS_NO_OVERLAY,                                              0}},
+    {0xA2, 1, X86InstInfo{"CPUID",   TYPE_INST,     FLAGS_NO_OVERLAY | FLAGS_BLOCK_END,                                              0}},
     {0xA3, 1, X86InstInfo{"BT",      TYPE_INST,     FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY | FLAGS_SUPPORTS_LOCK, 0}},
     {0xA4, 1, X86InstInfo{"SHLD",    TYPE_INST,     FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY,                                              1}},
     {0xA5, 1, X86InstInfo{"SHLD",    TYPE_INST,     FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY,                           0}},
@@ -226,12 +226,12 @@ constexpr std::array<X86InstInfo, MAX_SECOND_TABLE_SIZE> SecondBaseOps = []() co
     {0xB5, 1, X86InstInfo{"LGS",     TYPE_INVALID, FLAGS_NO_OVERLAY,                                                                                0}},
     {0xB6, 1, X86InstInfo{"MOVZX",   TYPE_INST, GenFlagsSrcSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY,                                        0}},
     {0xB7, 1, X86InstInfo{"MOVZX",   TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY,                                       0}},
-    {0xB8, 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,                                                                                      0}},
+    {0xB8, 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NO_PREFIX_ORDER,                                                                           0}},
     {0xB9, 1, X86InstInfo{"",        TYPE_GROUP_10, FLAGS_NO_OVERLAY,                                                                               0}},
     {0xBA, 1, X86InstInfo{"",        TYPE_GROUP_8, FLAGS_NO_OVERLAY,                                                                                0}},
     {0xBB, 1, X86InstInfo{"BTC",     TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY | FLAGS_SUPPORTS_LOCK,   0}},
-    {0xBC, 1, X86InstInfo{"BSF",     TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66,                                                                   0}},
-    {0xBD, 1, X86InstInfo{"BSR",     TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66,                                                                   0}},
+    {0xBC, 1, X86InstInfo{"BSF",     TYPE_INST, FLAGS_MODRM | FLAGS_NO_PREFIX_ORDER | FLAGS_NO_OVERLAY66,                                           0}},
+    {0xBD, 1, X86InstInfo{"BSR",     TYPE_INST, FLAGS_MODRM | FLAGS_NO_PREFIX_ORDER | FLAGS_NO_OVERLAY66,                                           0}},
     {0xBE, 1, X86InstInfo{"MOVSX",   TYPE_INST, GenFlagsSrcSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY,                                        0}},
     {0xBF, 1, X86InstInfo{"MOVSX",   TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY,                                       0}},
 

FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp

@@ -321,6 +321,7 @@ namespace AVX128 {
     {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i64Bit>},
     {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i128Bit>},
 
+    {OPD(2, 0b10, 0x72), 1, &OpDispatchBuilder::AVX128_VCVTNEPS2BF16},
     {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i8Bit>},
     {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i16Bit>},
 
@@ -365,6 +366,13 @@ namespace AVX128 {
     {OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 1, 3>},  // VFMADDSUB
     {OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD
 
+    {OPD(2, 0b00, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEPH2PS, false>},
+    {OPD(2, 0b01, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEPH2PS, true>},
+    {OPD(2, 0b10, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, true>},
+    {OPD(2, 0b11, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, false>},
+    {OPD(2, 0b01, 0xB1), 1, &IR::OpDispatchBuilder::AVX128_VBCSTNESH2PS},
+    {OPD(2, 0b10, 0xB1), 1, &IR::OpDispatchBuilder::AVX128_VBCSTNEBF162PS},
+
     {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 3, 1>},  // VFMADDSUB
     {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD
 
@@ -771,6 +779,7 @@ namespace AVX256 {
     {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i64Bit>},
     {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i128Bit>},
 
+    {OPD(2, 0b10, 0x72), 1, &OpDispatchBuilder::AVXVCVTNEPS2BF16},
     {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i8Bit>},
     {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i16Bit>},
 
@@ -815,6 +824,13 @@ namespace AVX256 {
     {OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 1, 3>},  // VFMADDSUB
     {OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD
 
+    {OPD(2, 0b00, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEPH2PS, false>},
+    {OPD(2, 0b01, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEPH2PS, true>},
+    {OPD(2, 0b10, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEBF162PS, true>},
+    {OPD(2, 0b11, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEBF162PS, false>},
+    {OPD(2, 0b01, 0xB1), 1, &IR::OpDispatchBuilder::AVXVBCSTNESH2PS},
+    {OPD(2, 0b10, 0xB1), 1, &IR::OpDispatchBuilder::AVXVBCSTNEBF162PS},
+
     {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 3, 1>},  // VFMADDSUB
     {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD
 
@@ -1226,6 +1242,7 @@ auto BaseTableLambda = [](const auto RuntimeTable) consteval {
     {OPD(2, 0b01, 0x59), 1, X86InstInfo{"VPBROADCASTQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0x5A), 1, X86InstInfo{"VBROADCASTI128", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_SF_MOD_MEM_ONLY | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
 
+    {OPD(2, 0b10, 0x72), 1, X86InstInfo{"VCVTNEPS2BF16", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0x78), 1, X86InstInfo{"VPBROADCASTB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0x79), 1, X86InstInfo{"VPBROADCASTW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
 
@@ -1270,6 +1287,13 @@ auto BaseTableLambda = [](const auto RuntimeTable) consteval {
     {OPD(2, 0b01, 0xA6), 1, X86InstInfo{"VFMADDSUB213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0xA7), 1, X86InstInfo{"VFMSUBADD213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
 
+    {OPD(2, 0b00, 0xB0), 1, X86InstInfo{"VCVTNEOPH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b01, 0xB0), 1, X86InstInfo{"VCVTNEEPH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b10, 0xB0), 1, X86InstInfo{"VCVTNEEBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b11, 0xB0), 1, X86InstInfo{"VCVTNEOBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b01, 0xB1), 1, X86InstInfo{"VBCSTNESH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b10, 0xB1), 1, X86InstInfo{"VBCSTNEBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+
     {OPD(2, 0b01, 0xB6), 1, X86InstInfo{"VFMADDSUB231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0xB7), 1, X86InstInfo{"VFMSUBADD231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
 

FEXCore/Source/Interface/Core/X86Tables/X86Tables.h

@@ -407,7 +407,9 @@ namespace InstFlags {
   constexpr InstFlagType FLAGS_CALL = (1ULL << 30);
   constexpr InstFlagType FLAGS_SUPPORTS_LOCK = (1ULL << 31);
   constexpr InstFlagType FLAGS_LITERAL_PATCHABLE = (1ULL << 32);
-  // Flags [57..33]: Undefined
+  constexpr InstFlagType FLAGS_NO_PREFIX_ORDER = (1ULL << 33);
+
+  // Flags [57..34]: Undefined
   // Flags [60..58]: Dst size
   constexpr InstFlagType FLAGS_SIZE_DST_OFF = 58;
   // Flags [63..61]: Src size

FEXCore/Source/Interface/Core/X86Tables/X87Tables.cpp

@@ -69,7 +69,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F64OpTable = {{
 
   {OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},
   {OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},
-  {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP
+  {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::X87NOP}, // FNOP
   // D1 = Invalid
   // D8 = Invalid
   {OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},
@@ -341,7 +341,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F80OpTable = {{
 
   {OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},
   {OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},
-  {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP
+  {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::X87NOP}, // FNOP
   // D1 = Invalid
   // D8 = Invalid
   {OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},

FEXCore/Source/Interface/IR/IR.json

@@ -321,7 +321,7 @@
         "DestSize": "Size",
         "RAOverride": "2"
       },
-      "Break BreakDefinition:$Reason": {
+      "Break BreakDefinition:$Reason, GPR:$FaultAddress": {
         "HasSideEffects": true
       },
       "CallbackReturn": {
@@ -337,9 +337,8 @@
         "HasSideEffects": true
       },
 
-      "GPR:$EAX, GPR:$EBX, GPR:$ECX, GPR:$EDX = CPUID GPR:$Function, GPR:$Leaf": {
+      "CPUID i8:$InstSize": {
         "Desc": ["Calls in to the CPUID handler function to return emulated CPUID"],
-        "DestSize": "OpSize::i32Bit",
         "HasSideEffects": true
       },
       "GPR:$EAX, GPR:$EDX = XGetBV GPR:$Function": {
@@ -530,6 +529,7 @@
       },
 
       "SSA = LoadMem RegisterClass:$Class, OpSize:#Size, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale": {
+        "HasSideEffects": true,
         "Inline": ["", "Mem"],
         "DestSize": "Size"
       },
@@ -575,6 +575,7 @@
       },
 
       "SSA = LoadMemTSO RegisterClass:$Class, OpSize:#Size, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale": {
+        "HasSideEffects": true,
         "Desc": ["Does a x86 TSO compatible load from memory. Offset must be Invalid()."
                 ],
         "Inline": ["", "Memtso"],
@@ -590,6 +591,7 @@
       },
 
       "FPR = VLoadVectorMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Mask, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale": {
+        "HasSideEffects": true,
         "Desc": ["Does a masked load similar to VPMASKMOV/VMASKMOV where the upper bit of each element",
                  "determines whether or not that element will be loaded from memory"],
         "ImplicitFlagClobber": true,
@@ -605,6 +607,7 @@
         "ElementSize": "ElementSize"
       },
       "FPR = VLoadVectorGatherMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$Mask, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, OpSize:$VectorIndexElementSize, u8:$OffsetScale, u8:$DataElementOffsetStart, u8:$IndexElementOffsetStart, OpSize:$AddrSize, i1:$MaskAllOnes{false}": {
+        "HasSideEffects": true,
         "Desc": [
           "Does a masked load similar to VPGATHERD* where the upper bit of each element",
           "determines whether or not that element will be loaded from memory.",
@@ -620,6 +623,7 @@
         ]
       },
       "FPR = VLoadVectorGatherMaskedQPS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$MaskReg, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, u8:$OffsetScale, OpSize:$AddrSize, i1:$MaskAllOnes{false}": {
+        "HasSideEffects": true,
         "Desc": [
           "Does a masked load similar to VPGATHERQPS where the upper bit of each element",
           "determines whether or not that element will be loaded from memory.",
@@ -637,6 +641,7 @@
         ]
       },
       "FPR = VLoadVectorElement OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$DstSrc, u8:$Index, GPR:$Addr": {
+        "HasSideEffects": true,
         "Desc": ["Does a memory load to a single element of a vector.",
                  "Leaves the rest of the vector's data intact.",
                  "Matches arm64 ld1 semantics"],
@@ -776,6 +781,7 @@
         ]
       },
       "FPR = VLoadNonTemporal OpSize:#RegisterSize, GPR:$Addr, i8:$Offset": {
+        "HasSideEffects": true,
         "Desc": ["Does a non-temporal memory load of a vector.",
                  "Matches arm64 SVE ldnt1b semantics.",
                  "Specifically weak-memory model ordered to match x86 non-temporal stores."
@@ -1863,7 +1869,7 @@
         "DestSize": "RegisterSize"
       },
 
-      "FPR = VectorImm OpSize:#RegisterSize, OpSize:#ElementSize, u8:$Immediate, u8:$ShiftAmount{0}": {
+      "FPR = VectorImm OpSize:#RegisterSize, OpSize:#ElementSize, u8:$Immediate, u8:$ShiftAmount{0}, i1:$Do64BitExpansion{false}": {
         "Desc": ["Generates a vector with each element containg the immediate zexted"
                 ],
         "DestSize": "RegisterSize",
@@ -2761,6 +2767,28 @@
           "RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
         ]
       },
+      "FPR = VBFCVTN OpSize:#RegisterSize, FPR:$Vector": {
+        "Desc": [
+          "Converts fp32 to bf16."
+        ],
+        "DestSize": "RegisterSize",
+        "ElementSize": "FEXCore::IR::OpSize::i16Bit",
+        "EmitValidation": [
+          "RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
+        ]
+      },
+      "FPR = VBFCVTN2 OpSize:#RegisterSize, FPR:$VectorLower, FPR:$VectorUpper": {
+        "TiedSource": 0,
+        "Desc": [
+          "Converts fp32 to bf16 and inserting in to the high bits.",
+          "Bottom half is untouched"
+        ],
+        "DestSize": "RegisterSize",
+        "ElementSize": "FEXCore::IR::OpSize::i16Bit",
+        "EmitValidation": [
+          "RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
+        ]
+      },
       "FPR = Vector_FToI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, RoundType:$Round": {
         "Desc": ["Vector op: Rounds float to integral",
                  "Rounding mode determined by argument"

FEXCore/Source/Interface/IR/IREmitter.cpp

@@ -21,6 +21,7 @@ static bool IsFragmentExit(FEXCore::IR::IROps Op) {
   switch (Op) {
   case OP_THREADREMOVECODEENTRY:
   case OP_SYSCALL:
+  case OP_CPUID:
   case OP_EXITFUNCTION:
   case OP_BREAK: return true;
   default: return false;

FEXCore/Source/Interface/IR/IREmitter.h

@@ -350,6 +350,18 @@ public:
     return IROp->Op == OP_VCMPEQ && IROp->Args[0].ID() == IROp->Args[1].ID();
   }
 
+  // Returns true if the value is a load of register Reg of class Class.
+  bool IsValueLoadRegister(OrderedNodeWrapper ssa, RegClass Class, uint32_t Reg) const {
+    auto RealNode = ssa.GetNode(DualListData.ListBegin());
+    const auto* IROp = RealNode->Op(DualListData.DataBegin());
+    if (IROp->Op != OP_LOADREGISTER) {
+      return false;
+    }
+
+    auto Op = IROp->C<IR::IROp_LoadRegister>();
+    return Op->Class == Class && Op->Reg == Reg;
+  }
+
   FEXCore::IR::IROp_Header* GetOpHeader(OrderedNodeWrapper ssa) {
     Ref RealNode = ssa.GetNode(DualListData.ListBegin());
     return RealNode->Op(DualListData.DataBegin());
@@ -363,6 +375,15 @@ public:
     return node->Wrapped(DualListData.ListBegin());
   }
 
+  // Returns argument Index of Node if the op of Node is Op. Else returns nullptr.
+  Ref GetArgIfOp(Ref Node, IROps Op, unsigned Index) {
+    if (!Node || GetOpHeader(WrapNode(Node))->Op != Op) {
+      return nullptr;
+    }
+
+    return UnwrapNode(GetOpHeader(WrapNode(Node))->Args[Index]);
+  }
+
   NodeIterator GetIterator(OrderedNodeWrapper wrapper) {
     return NodeIterator(DualListData.ListBegin(), DualListData.DataBegin(), wrapper);
   }

FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp

@@ -527,27 +527,6 @@ bool ConstrainedRAPass::TryPostRAMerge(Ref LastNode, Ref CodeNode, IROp_Header*
       IREmit->RemovePostRA(CodeNode);
       return false;
     }
-  } else if (IROp->Op == OP_CPUID && PhysicalRegister(IROp->Args[0]) == PhysicalRegister(LastNode) && LastOp->Op == OP_CONSTANT) {
-    // Try to constant fold. As a limitation of merging only 2 instructions, we
-    // can only handle constant functions, not constant leafs. This could be
-    // lifted if we generalized at a (significant) complexity cost.
-    uint64_t ConstantFunction = LastOp->C<IROp_Constant>()->Constant;
-    auto Op = IROp->CW<IR::IROp_CPUID>();
-
-    const auto SupportsConstant = CPUID->DoesFunctionReportConstantData(ConstantFunction);
-    if (SupportsConstant.SupportsConstantFunction == CPUIDEmu::SupportsConstant::CONSTANT &&
-        SupportsConstant.NeedsLeaf != CPUIDEmu::NeedsLeafConstant::NEEDSLEAFCONSTANT) {
-      const auto Result = CPUID->RunFunction(ConstantFunction, 0 /* leaf */);
-
-      IREmit->SetWriteCursorBefore(CodeNode);
-      IREmit->_Fence(IR::FenceType::Inst);
-      IREmit->_Constant(Result.eax).Node->Reg = PhysicalRegister(Op->OutEAX).Raw;
-      IREmit->_Constant(Result.ebx).Node->Reg = PhysicalRegister(Op->OutEBX).Raw;
-      IREmit->_Constant(Result.ecx).Node->Reg = PhysicalRegister(Op->OutECX).Raw;
-      IREmit->_Constant(Result.edx).Node->Reg = PhysicalRegister(Op->OutEDX).Raw;
-      IREmit->RemovePostRA(CodeNode);
-      return false;
-    }
   }
 
   // Merge moves that are immediately consumed.

FEXCore/Source/Utils/Allocator.cpp

@@ -23,7 +23,9 @@
 #include <fcntl.h>
 #ifndef _WIN32
 #include <sys/mman.h>
+#include <sys/resource.h>
 #include <sys/user.h>
+#include <unistd.h>
 #endif
 
 namespace fextl::pmr {
@@ -214,6 +216,52 @@ fextl::vector<MemoryRegion> CollectMemoryGaps(uintptr_t Begin, uintptr_t End, in
   FEX_UNREACHABLE;
 }
 
+// Tracks the region below the main thread stack, which is reserved for stack growth,
+// as well as the guard region directly below the stack window.
+static uintptr_t MainThreadStackReserveBegin {};
+static uintptr_t MainThreadStackWindowBegin {};
+static uintptr_t MainThreadStackWindowEnd {};
+static size_t MainThreadStackGuardSize {};
+static uintptr_t MainThreadStackGuardBegin {};
+
+// Currently, guest signal handlers leak space on the main stack
+// if they exit via longjmp instead of sigreturn. Because of this memory leak
+// we must allow the main thread stack to grow without bound. This technically
+// violates RLIMIT_STACK, but until we fix the leak, we don't have much choice.
+// TODO: Once the leak is fixed, remove this function entirely.
+bool TryGrowMainThreadStack(uintptr_t FaultAddress) {
+  // Only grow the stack when the fault is in the guard region.
+  if (FaultAddress < MainThreadStackGuardBegin || FaultAddress >= MainThreadStackWindowBegin) {
+    return false;
+  }
+
+  // Compute the required size of the stack region.
+  // Double the current stack, up to 1GB, then just add 1GB at a time.
+  constexpr size_t OneGB = 1ULL << 30;
+  const size_t RequiredSize = MainThreadStackWindowEnd - FaultAddress;
+  size_t NewSize = MainThreadStackWindowEnd - MainThreadStackWindowBegin;
+
+  while (NewSize < RequiredSize) {
+    if (NewSize < OneGB) {
+      NewSize = std::min(NewSize * 2, OneGB);
+    } else {
+      NewSize += OneGB;
+    }
+  }
+
+  // Don't grow past the bottom of the reserved region,
+  // which is where the first dynamically loaded libraries live.
+  const uintptr_t NewWindowBegin = std::max(MainThreadStackWindowEnd - NewSize, MainThreadStackReserveBegin);
+  if (mprotect(reinterpret_cast<void*>(NewWindowBegin), MainThreadStackWindowBegin - NewWindowBegin, PROT_READ | PROT_WRITE) != 0) {
+    return false;
+  }
+
+  // Move the window and the guard region below it down.
+  MainThreadStackWindowBegin = NewWindowBegin;
+  MainThreadStackGuardBegin = std::max(NewWindowBegin - MainThreadStackGuardSize, MainThreadStackReserveBegin);
+  return true;
+}
+
 fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
   const uintptr_t StackLocation_u64 = reinterpret_cast<uintptr_t>(alloca(0));
 
@@ -231,7 +279,7 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
   // If the memory bounds include the stack, blocking all memory regions will
   // limit the stack size to the current value. To allow some stack growth,
   // we don't block the memory gap directly below the stack memory but
-  // instead map it as readable+writable.
+  // instead reserve it for the stack, see TryGrowMainThreadStack.
   {
     auto StackRegionIt = std::find_if(Regions.begin(), Regions.end(), [StackLocation_u64](auto& Region) {
       return reinterpret_cast<uintptr_t>(Region.Ptr) + Region.Size > StackLocation_u64;
@@ -243,16 +291,40 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
 
     if (IsStackMapping && StackRegionIt != Regions.begin() &&
         reinterpret_cast<uintptr_t>(std::prev(StackRegionIt)->Ptr) + std::prev(StackRegionIt)->Size <= End) {
-      // Allocate the region under the stack as READ | WRITE so the stack can still grow
       --StackRegionIt;
 
-      auto Alloc =
-        ::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);
+      // Get the page size, so we can make sure to align on page boundaries.
+      const auto PageSize = sysconf(_SC_PAGESIZE);
+      const size_t HostPageSize = PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE;
+
+      // Allocate RLIMIT_STACK worth of memory for the main thread stack, or 64MB if no limit is set.
+      struct rlimit StackLimit {};
+      size_t Size = 1ULL << 26;
+      if (getrlimit(RLIMIT_STACK, &StackLimit) == 0 && StackLimit.rlim_cur != RLIM_INFINITY) {
+        Size = std::max<size_t>(FEXCore::AlignUp(StackLimit.rlim_cur, HostPageSize), HostPageSize);
+      }
+      Size = std::min(Size, StackRegionIt->Size);
+
+      // Reserve all of the memory between the top of the stack, and the start of the next region.
+      // We must map it with PROT_NONE, otherwise the MM will assume these pages are "in use"
+      // and charge this usage towards VmData. In practice this is multiple TBs of VA space.
+      auto Alloc = ::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);
 
       LogMan::Throw::AFmt(Alloc != MAP_FAILED, "StealMemoryRegion:Stack: mmap({}, {:x}) failed: {}", fmt::ptr(StackRegionIt->Ptr),
                           StackRegionIt->Size, errno);
       LogMan::Throw::AFmt(Alloc == StackRegionIt->Ptr, "mmap returned {} instead of {}", Alloc, fmt::ptr(StackRegionIt->Ptr));
 
+      // Now determine the bounds for the stack region, and its guard region, which is used by TryGrowMainThreadStack().
+      MainThreadStackReserveBegin = reinterpret_cast<uintptr_t>(StackRegionIt->Ptr);
+      MainThreadStackWindowEnd = MainThreadStackReserveBegin + StackRegionIt->Size;
+      MainThreadStackWindowBegin = MainThreadStackWindowEnd - Size;
+      MainThreadStackGuardSize = 256 * HostPageSize;
+      MainThreadStackGuardBegin = std::max(MainThreadStackWindowBegin - MainThreadStackGuardSize, MainThreadStackReserveBegin);
+
+      // Protect the stack window.
+      const auto Result = mprotect(reinterpret_cast<void*>(MainThreadStackWindowBegin), Size, PROT_READ | PROT_WRITE);
+      LogMan::Throw::AFmt(Result == 0, "StealMemoryRegion:Stack: mprotect({:x}, {:x}) failed: {}", MainThreadStackWindowBegin, Size, errno);
+
       Regions.erase(StackRegionIt);
     }
   }

FEXCore/Source/Utils/AllocatorHooks.cpp

@@ -27,9 +27,13 @@ using vma_name_hook_type = void (*)(const char* name, const void* address, size_
 #ifdef ENABLE_FEX_ALLOCATOR
 typedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size);
 typedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size);
+typedef void (*rp_mcommit_hook_type)(void* address, size_t size);
+typedef void (*rp_mdecommit_hook_type)(void* address, size_t size);
 typedef void (*vma_name_hook_type)(const char* name, const void* address, size_t size);
 extern "C" rp_mmap_hook_type rp_mmap_hook;
 extern "C" rp_munmap_hook_type rp_munmap_hook;
+extern "C" rp_mcommit_hook_type rp_mcommit_hook;
+extern "C" rp_mdecommit_hook_type rp_mdecommit_hook;
 extern "C" vma_name_hook_type rp_name_hook;
 
 #ifndef _WIN32
@@ -112,8 +116,11 @@ static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t*
     alignment = 0;
   }
 
+  // Protect with PROT_NONE so that the whole heap is not charged against VmData.
+  // In the memory commit callback, FEX_rp_memory_commit, we reprotect with
+  // PROT_READ | PROT_WRITE to commit memory on demand.
   size_t map_size = AlignUp(size + alignment, global_config.page_size);
-  auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  auto ptr = fex_mmap_hook(0, map_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 
   if (ptr == MAP_FAILED) {
     ptr = nullptr;
@@ -149,7 +156,9 @@ static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t*
 }
 
 static void FEX_rp_memory_commit(void* address, size_t size) {
-  // NOP-implementation.
+  if (mprotect(address, size, PROT_READ | PROT_WRITE)) {
+    fprintf(stderr, "Failed to commit VMA region.");
+  }
 }
 
 static void FEX_rp_memory_decommit(void* address, size_t size) {
@@ -160,6 +169,11 @@ static void FEX_rp_memory_decommit(void* address, size_t size) {
   if (madvise(address, size, MADV_DONTNEED)) {
     fprintf(stderr, "Failed to decommit VMA region.");
   }
+
+  // Drop the protections back to PROT_NONE so the pages are no longer charged towards VmData.
+  if (mprotect(address, size, PROT_NONE)) {
+    fprintf(stderr, "Failed to decommit VMA region.");
+  }
 }
 
 static void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) {
@@ -190,6 +204,8 @@ void InitializeAllocator(size_t PageSize) {
   rpmalloc_initialize_config(&global_interface, &global_config);
   rp_mmap_hook = FEX_rp_mmap;
   rp_munmap_hook = FEX_rp_memory_unmap;
+  rp_mcommit_hook = FEX_rp_memory_commit;
+  rp_mdecommit_hook = FEX_rp_memory_decommit;
   rp_name_hook = LocalVirtualName;
 }
 #else

FEXCore/include/FEXCore/Core/CoreState.h

@@ -71,6 +71,12 @@ static_assert(std::is_standard_layout_v<NonAtomicRefCounter<uint64_t>>, "Needs t
 static_assert(std::is_trivially_copyable_v<NonAtomicRefCounter<uint64_t>>, "needs to be trivially copyable");
 static_assert(sizeof(NonAtomicRefCounter<uint64_t>) == sizeof(uint64_t), "Needs to be correct size");
 
+// Flags about the core state that the JIT can query.
+enum class CoreFlagsEnum : uint64_t {
+  CPUID_ENABLED = (1ULL << 0),
+};
+FEX_DEF_NUM_OPS(CoreFlagsEnum)
+
 struct alignas(64) CPUState {
   // Allows more efficient handling of the register
   // file in the event AVX is not supported.
@@ -107,7 +113,9 @@ struct alignas(64) CPUState {
   uint64_t L1Pointer {};
   uint64_t L1Mask {};
   uint64_t callret_sp {};
-  uint64_t _pad1 {};
+
+  // CPUID is default enabled.
+  CoreFlagsEnum CoreFlags {CoreFlagsEnum::CPUID_ENABLED};
 
   // Cacheline: 1,2,3,4
   // The high 128-bits of AVX registers when not being emulated by SVE256.
@@ -188,10 +196,41 @@ struct alignas(64) CPUState {
     GDT->Limit1 = Limit >> 16;
   }
 
+  static bool IsInstructionBPEnabled(unsigned BPIndex, uint64_t Dr7) {
+    LOGMAN_THROW_A_FMT(BPIndex < NUM_DEBUG_ADDRESSES, "Too large of an index");
+
+    // If the GE/LE flags and Transaction type is instruction then it is enabled.
+    uint64_t GELEEnabledMask = 0b11ULL << (2 * BPIndex);
+    uint64_t TransactionTypeMask = 0b11ULL << (16 + (4 * BPIndex));
+    return (Dr7 & GELEEnabledMask) && (Dr7 & TransactionTypeMask) == 0;
+  };
+
+  static size_t BreakpointSize(unsigned BPIndex, uint64_t Dr7) {
+    LOGMAN_THROW_A_FMT(BPIndex < NUM_DEBUG_ADDRESSES, "Too large of an index");
+
+    switch ((Dr7 >> (18 + (4 * BPIndex))) & 0b11) {
+    case 0: return 1;
+    case 1: return 2;
+    case 2: return 8;
+    case 3: return 4;
+    default: FEX_UNREACHABLE;
+    }
+  }
+
   uint16_t FCW {0x37F};
   uint8_t AbridgedFTW {};
 
   uint8_t _pad2[5];
+  uint64_t FIP {};
+
+  // Debug registers 0-3 are addresses.
+  static constexpr size_t NUM_DEBUG_ADDRESSES = 4;
+  uint64_t Dr[NUM_DEBUG_ADDRESSES] {};
+  // Debug register 6 is a status return register.
+  uint64_t Dr6 {};
+  // Debug register 7 is the control register.
+  uint64_t Dr7 {};
+
   // PF/AF are statically mapped as-if they were r16/r17 (which do not exist in
   // x86 otherwise). This allows a straightforward mapping for SRA.
   static constexpr uint8_t PF_AS_GREG = 16;
@@ -247,7 +286,6 @@ struct alignas(64) CPUState {
   // Follows encoding of the TI bit in segment selector encoding.
   constexpr static uint32_t SEGMENT_ARRAY_INDEX_GDT = 0;
   constexpr static uint32_t SEGMENT_ARRAY_INDEX_LDT = 1;
-  Core::CPUState::gdt_segment private_gdt[32] {};
 };
 static_assert(std::is_trivially_copyable_v<CPUState>, "Needs to be trivial");
 static_assert(std::is_standard_layout_v<CPUState>, "This needs to be standard layout");
@@ -348,13 +386,17 @@ struct JITPointers {
   uint64_t SyscallHandlerFunc {};
   uint64_t ExitFunctionLink {};
   uint64_t MonoBackpatcherWrite {};
-  uint64_t LUDIV {};
-  uint64_t LDIV {};
   uint64_t RDRANDFallback {};
   uint64_t ThunkCallbackRet {};
 
+  /**
+   * @name Frontend handler pointers.
+   * @{ */
   // Handles returning/calling ARM64EC code from the JIT, expects the target PC in TMP3
   uint64_t ExitFunctionEC {};
+  // Handles a synchronous exception without faulting to jump in to the handler. Frontend specific.
+  uint64_t SynchronousExceptionHandler {};
+  /**  @} */
 
   FallbackABIInfo FallbackHandlerPointers[FallbackHandlerIndex::OPINDEX_MAX];
   uint64_t NamedVectorConstantPointers[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX];
@@ -372,15 +414,15 @@ struct JITPointers {
   uint64_t ThreadStopHandlerSpillSRA {};
   uint64_t ThreadPauseHandlerSpillSRA {};
   uint64_t ThreadDispatchSyscallHandler {};
+  uint64_t ThreadDispatchCPUIDHandler {};
   uint64_t ThreadDispatchRemoveCodeEntry {};
   uint64_t GuestSignal_SIGILL {};
   uint64_t GuestSignal_SIGTRAP {};
   uint64_t GuestSignal_SIGSEGV {};
+  uint64_t GuestSignal_SynchronousFault {};
   uint64_t SignalReturnHandler {};
   uint64_t SignalReturnHandlerRT {};
   uint64_t L2Pointer {};
-  uint64_t LUDIVHandler {};
-  uint64_t LDIVHandler {};
   uint64_t F64SinHandler {};
   uint64_t F64CosHandler {};
   uint64_t F64TanHandler {};
@@ -417,17 +459,19 @@ struct CpuStateFrame {
    */
   uint64_t InSyscallInfo {};
 
-  uint32_t SignalHandlerRefCounter {};
-
-  struct alignas(8) SynchronousFaultDataStruct {
+  struct SynchronousFaultDataStruct {
     bool FaultToTopAndGeneratedException {};
     uint8_t Signal;
     uint8_t TrapNo;
     uint8_t si_code;
     uint16_t err_code;
     uint16_t _pad : 16;
+    // Fault address can be different from the FaultRIP so need to distinguish.
+    uint64_t FaultAddress;
   } SynchronousFaultData;
 
+  uint32_t SignalHandlerRefCounter {};
+
   InternalThreadState* Thread;
 
 #ifdef _WIN32
@@ -444,7 +488,7 @@ static_assert(offsetof(CpuStateFrame, Pointers) + sizeof(CpuStateFrame::Pointers
                                                                                             "than architecture maximum 32768");
 
 static_assert(std::is_standard_layout<CpuStateFrame>::value, "This needs to be standard layout");
-static_assert(sizeof(CpuStateFrame::SynchronousFaultData) == 8, "This needs to be 8 bytes");
+static_assert(sizeof(CpuStateFrame::SynchronousFaultData) == 16, "This needs to be 16 bytes");
 static_assert(alignof(CpuStateFrame::SynchronousFaultDataStruct) == 8, "This needs to be 8 bytes");
 static_assert(offsetof(CpuStateFrame, SynchronousFaultData) % 8 == 0, "This needs to be aligned");
 } // namespace FEXCore::Core

FEXCore/include/FEXCore/Core/HostFeatures.h

@@ -14,7 +14,7 @@ namespace FEXCore {
  */
 struct HostFeatures {
   // Changes code generation slightly.
-  enum class HostTypeEnum : uint32_t {
+  enum class HostTypeEnum : uint8_t {
     Unknown,
     Linux,
     Wow64,
@@ -24,7 +24,22 @@ struct HostFeatures {
   // Whether or not the host supports any kind of SVE implementation.
   [[nodiscard]]
   bool SupportsSVE() const {
-    return SupportsSVE128 || SupportsSVE256;
+    return ProcessFeatures.SupportsSVE128 || ProcessFeatures.SupportsSVE256;
+  }
+
+  [[nodiscard]]
+  bool SupportsSVE128() const {
+    return ProcessFeatures.SupportsSVE128;
+  }
+
+  [[nodiscard]]
+  bool SupportsSVE256() const {
+    return ProcessFeatures.SupportsSVE256;
+  }
+
+  [[nodiscard]]
+  bool SupportsAVX() const {
+    return ProcessFeatures.SupportsAVX;
   }
 
   [[nodiscard]]
@@ -32,63 +47,85 @@ struct HostFeatures {
     return 4 << DCacheLineLog2;
   }
 
+  /**
+   * @brief This structure contains all the data required to determine hashing at the "process" level.
+   * Unlike the `HostFeaturesHash` below, this data changes codegen depending on if the process is Linux, Arm64EC, or Wow64.
+   */
+  struct ProcessFeaturesHash {
+    HostTypeEnum HostType                      : 2 {};
+    uint8_t SupportsFaultFreeSynchronousFaults : 1 {};
+    // This only determines the starting mode a process started in.
+    // The process can technically long jump to-from the other modes.
+    uint8_t Is64BitMode : 1 {};
+
+    // These are technically HostFeatures, but they differ between Linux, wow64, arm64.
+    uint8_t SupportsAVX    : 1 {};
+    uint8_t SupportsSVE128 : 1 {};
+    uint8_t SupportsSVE256 : 1 {};
+
+    uint8_t pad : 1 {};
+  };
+  static_assert(sizeof(ProcessFeaturesHash) == 1);
+
   struct CacheHash {
+    // This hash is used to determine the "bucket" a machine goes in to.
+    // This should match between Linux, Arm64EC, and Wow64.
     uint64_t HostFeaturesHash;
-    HostTypeEnum HostType;
+
+    // This hash is used to determine features that are different between process types.
+    ProcessFeaturesHash ProcessFeaturesHash;
   };
 
   [[nodiscard]]
   CacheHash HashForCaching() const {
-    static_assert(offsetof(HostFeatures, HostType) == 8);
+    static_assert(offsetof(HostFeatures, ProcessFeatures) == 8);
 
     CacheHash Result {};
     memcpy(&Result.HostFeaturesHash, this, sizeof(uint64_t));
-    Result.HostType = HostType;
+    Result.ProcessFeaturesHash = ProcessFeatures;
 
     return Result;
   }
 
   // Affects codegen and is basically machine description.
-  uint32_t DCacheLineLog2              : 4 {};
-  uint32_t SupportsCacheMaintenanceOps : 1 {};
-  uint32_t SupportsAES                 : 1 {};
-  uint32_t SupportsCRC                 : 1 {};
-  uint32_t SupportsCLZERO              : 1 {};
-  uint32_t SupportsAtomics             : 1 {};
-  uint32_t SupportsRCPC                : 1 {};
-  uint32_t SupportsTSOImm9             : 1 {};
-  uint32_t SupportsRAND                : 1 {};
-  uint32_t SupportsAVX                 : 1 {};
-  uint32_t SupportsSVE128              : 1 {};
-  uint32_t SupportsSVE256              : 1 {};
-  uint32_t SupportsSHA                 : 1 {};
-  uint32_t SupportsPMULL_128Bit        : 1 {};
-  uint32_t SupportsCSSC                : 1 {};
-  uint32_t SupportsFCMA                : 1 {};
-  uint32_t SupportsFlagM               : 1 {};
-  uint32_t SupportsFlagM2              : 1 {};
-  uint32_t SupportsRPRES               : 1 {};
-  uint32_t SupportsPreserveAllABI      : 1 {};
-  uint32_t SupportsAES256              : 1 {};
-  uint32_t SupportsSVEBitPerm          : 1 {};
-  uint32_t SupportsCPUIndexInTPIDRRO   : 1 {};
-  uint32_t SupportsFRINTTS             : 1 {};
-  uint32_t SupportsECV                 : 1 {};
-  uint32_t SupportsWFXT                : 1 {};
-  uint32_t Supports3DNow               : 1 {};
-  uint32_t SupportsSSE4a               : 1 {};
-  uint32_t SupportsMOPS                : 1 {};
-  uint32_t SupportsI8MM                : 1 {};
-  uint32_t SupportsDotProd             : 1 {};
-  uint32_t PreferZVAForVZero           : 1 {};
-  uint32_t SupportsAFP                 : 1 {};
-  uint32_t SupportsFloatExceptions     : 1 {};
+  uint32_t DCacheLineLog2              : 4 {}; // [3:0]
+  uint32_t SupportsCacheMaintenanceOps : 1 {}; // [4]
+  uint32_t SupportsAES                 : 1 {}; // [5]
+  uint32_t SupportsCRC                 : 1 {}; // [6]
+  uint32_t SupportsCLZERO              : 1 {}; // [7]
+  uint32_t SupportsAtomics             : 1 {}; // [8]
+  uint32_t SupportsRCPC                : 1 {}; // [9]
+  uint32_t SupportsTSOImm9             : 1 {}; // [10]
+  uint32_t SupportsRAND                : 1 {}; // [11]
+  uint32_t SupportsSHA                 : 1 {}; // [12]
+  uint32_t SupportsPMULL_128Bit        : 1 {}; // [13]
+  uint32_t SupportsCSSC                : 1 {}; // [14]
+  uint32_t SupportsFCMA                : 1 {}; // [15]
+  uint32_t SupportsFlagM               : 1 {}; // [16]
+  uint32_t SupportsFlagM2              : 1 {}; // [17]
+  uint32_t SupportsRPRES               : 1 {}; // [18]
+  uint32_t SupportsPreserveAllABI      : 1 {}; // [19]
+  uint32_t SupportsAES256              : 1 {}; // [20]
+  uint32_t SupportsSVEBitPerm          : 1 {}; // [21]
+  uint32_t SupportsCPUIndexInTPIDRRO   : 1 {}; // [22]
+  uint32_t SupportsFRINTTS             : 1 {}; // [23]
+  uint32_t SupportsECV                 : 1 {}; // [24]
+  uint32_t SupportsWFXT                : 1 {}; // [25]
+  uint32_t Supports3DNow               : 1 {}; // [26]
+  uint32_t SupportsSSE4a               : 1 {}; // [27]
+  uint32_t SupportsMOPS                : 1 {}; // [28]
+  uint32_t SupportsI8MM                : 1 {}; // [29]
+  uint32_t SupportsDotProd             : 1 {}; // [30]
+  uint32_t PreferZVAForVZero           : 1 {}; // [31]
+  uint32_t SupportsAFP                 : 1 {}; // [32]
+  uint32_t SupportsFloatExceptions     : 1 {}; // [33]
+  uint32_t SupportsBF16                : 1 {}; // [34]
   // Flag if this is InstCountCI
-  uint32_t IsInstCountCI : 1 {};
-  uint32_t pad           : 26 {};
+  uint32_t IsInstCountCI : 1 {};  // [38]
+  uint32_t pad           : 28 {}; // [63:35]
 
   // This affects codegen, but it isn't machine state
-  HostTypeEnum HostType {};
+  ProcessFeaturesHash ProcessFeatures {};
 
   // MIDR information
   // Also used for determining number of CPU cores for CPUID

FEXCore/include/FEXCore/Utils/Allocator.h

@@ -79,5 +79,6 @@ FEX_DEFAULT_VISIBILITY fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists(s
 #ifndef _WIN32
 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread);
 FEX_DEFAULT_VISIBILITY void UninstallTLSData(FEXCore::Core::InternalThreadState* Thread);
+FEX_DEFAULT_VISIBILITY bool TryGrowMainThreadStack(uintptr_t FaultAddress);
 #endif
 } // namespace FEXCore::Allocator

FEXCore/include/FEXCore/Utils/PrctlUtils.h

@@ -53,4 +53,8 @@
 #define PR_GET_MDWE 66
 #endif
 
+#ifndef PR_GET_TPIDRRO_CPU
+#define PR_GET_TPIDRRO_CPU 0x7F1DBB0
+#endif
+
 #endif // ifndef _WIN32

FEXCore/unittests/APITests/CMakeLists.txt

@@ -4,9 +4,11 @@ set(LIBS fmt::fmt vixl::vixl Catch2::Catch2WithMain FEXCore_Base JemallocLibs)
 foreach(TEST ${TESTS})
   get_filename_component(TEST_NAME ${TEST} NAME_WLE)
   add_executable(FEXCore_Tests_${TEST_NAME} ${TEST})
+
   target_link_libraries(FEXCore_Tests_${TEST_NAME} PRIVATE ${LIBS})
   target_include_directories(FEXCore_Tests_${TEST_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../Source/")
   target_compile_options(FEXCore_Tests_${TEST_NAME} PRIVATE ${FEX_TUNE_COMPILE_FLAGS})
+
   set_target_properties(FEXCore_Tests_${TEST_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/FEXCore_Tests")
   catch_discover_tests(FEXCore_Tests_${TEST_NAME} TEST_SUFFIX ".${TEST_NAME}.FEXCore_Tests")
 endforeach()

FEXCore/unittests/Emitter/ASIMD_Tests.cpp

@@ -644,6 +644,9 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD two-register m
   TEST_SINGLE(frecpe(SubRegSize::i32Bit, DReg::d30, DReg::d29), "frecpe v30.2s, v29.2s");
   // TEST_SINGLE(frecpe(SubRegSize::i64Bit, DReg::d30, DReg::d29), "frecpe v30.1d, v29.1d");
 
+  TEST_SINGLE(bfcvtn(QReg::q30, QReg::q29), "bfcvtn v30.4h, v29.4s");
+  TEST_SINGLE(bfcvtn2(QReg::q30, QReg::q29), "bfcvtn2 v30.8h, v29.4s");
+
   TEST_SINGLE(rev32(SubRegSize::i8Bit, QReg::q30, QReg::q29), "rev32 v30.16b, v29.16b");
   TEST_SINGLE(rev32(SubRegSize::i16Bit, QReg::q30, QReg::q29), "rev32 v30.8h, v29.8h");
   // TEST_SINGLE(rev32(SubRegSize::i32Bit, QReg::q30, QReg::q29), "rev32 v30.4s, v29.4s");

Scripts/InstructionCountParser.py

@@ -62,6 +62,8 @@ class HostFeatures(Flag) :
     FEATURE_MOPS   = (1 << 17)
     FEATURE_I8MM   = (1 << 18)
     FEATURE_DOTPROD = (1 << 19)
+    FEATURE_BF16    = (1 << 20)
+    FEATURE_TPIDRRO = (1 << 21)
 
 HostFeaturesLookup = {
     "SVE128"  : HostFeatures.FEATURE_SVE128,
@@ -84,6 +86,8 @@ HostFeaturesLookup = {
     "MOPS"    : HostFeatures.FEATURE_MOPS,
     "I8MM"    : HostFeatures.FEATURE_I8MM,
     "DOTPROD" : HostFeatures.FEATURE_DOTPROD,
+    "BF16" : HostFeatures.FEATURE_BF16,
+    "TPIDRRO" : HostFeatures.FEATURE_TPIDRRO,
 }
 
 def GetHostFeatures(data):

Scripts/guest_test_runner.py

@@ -1,11 +1,13 @@
 #!/usr/bin/python3
-import json
 import os
 import sys
-import subprocess
+# NB: This script is run with the "-S" flag, and therefore it can only use the Python standard library.
 
-def DoesFEXSupportAVX(mode):
+def DoesFEXSupportAVX():
     # Check if FEX indicates support for AVX
+    
+    # Import this here lazily to save on startup time
+    import subprocess
     fex_interpreter_path = os.path.dirname(sys.argv[7]) + "/FEX"
 
     args = list()
@@ -30,6 +32,8 @@ def TestRequiresAVXSupport():
     try:
         with open(json_path) as json_file:
             try:
+                # Import this here lazily to save on startup time
+                import json
                 json_data = json.load(json_file)
                 if not isinstance(json_data, dict):
                     raise TypeError('JSON data must be a dict')
@@ -91,7 +95,7 @@ fexecutable = sys.argv[7]
 StartingFEXArgsOffset = 8
 
 # If the test requires AVX and FEX doesn't support it, just pass the test and move on
-if TestRequiresAVXSupport() and not DoesFEXSupportAVX(mode):
+if mode == "guest" and TestRequiresAVXSupport() and not DoesFEXSupportAVX():
     sys.exit(0)
 
 # Open test expected information files and load in to dictionaries.
@@ -130,9 +134,9 @@ if ResultCode == 0:
     for Try in range(TryCount):
         # Run the test and wait for it to end to get the result
         print(RunnerArgs)
-        Process = subprocess.Popen(RunnerArgs)
-        Process.wait()
-        ResultCode = Process.returncode
+        Process = os.posix_spawnp(RunnerArgs[0], RunnerArgs, os.environ)
+        _, Status = os.waitpid(Process, 0)
+        ResultCode = os.waitstatus_to_exitcode(Status)
 
         # Break if the expected output is the result code
         if (expected_output[test_name] == ResultCode):

Scripts/json_config_parse.py

@@ -62,32 +62,38 @@ class Mode(Flag) :
     MODE_64   = 1
 
 class HostFeatures(Flag) :
-    FEATURE_ANY      = 0
-    FEATURE_3DNOW    = (1 << 0)
-    FEATURE_SSE4A    = (1 << 1)
-    FEATURE_AVX      = (1 << 2)
-    FEATURE_RAND     = (1 << 3)
-    FEATURE_SHA      = (1 << 4)
-    FEATURE_CLZERO   = (1 << 5)
-    FEATURE_BMI1     = (1 << 6)
-    FEATURE_BMI2     = (1 << 7)
-    FEATURE_CLWB     = (1 << 8)
-    FEATURE_LINUX    = (1 << 9)
-    FEATURE_AES256   = (1 << 10)
-    FEATURE_AFP      = (1 << 11)
-    FEATURE_SSSE3    = (1 << 12)
-    FEATURE_SSE4_1   = (1 << 13)
-    FEATURE_SSE4_2   = (1 << 14)
-    FEATURE_AES      = (1 << 15)
-    FEATURE_PCLMUL   = (1 << 16)
-    FEATURE_MOVBE    = (1 << 17)
-    FEATURE_ADX      = (1 << 18)
-    FEATURE_XSAVE    = (1 << 19)
-    FEATURE_RDPID    = (1 << 20)
-    FEATURE_CLFLOPT  = (1 << 21)
-    FEATURE_FSGSBASE = (1 << 22)
-    FEATURE_EMMI     = (1 << 23)
-    FEATURE_AVX_VNNI = (1 << 24)
+    FEATURE_ANY       = 0
+    FEATURE_3DNOW     = (1 << 0)
+    FEATURE_SSE4A     = (1 << 1)
+    FEATURE_AVX       = (1 << 2)
+    FEATURE_RAND      = (1 << 3)
+    FEATURE_SHA       = (1 << 4)
+    FEATURE_CLZERO    = (1 << 5)
+    FEATURE_BMI1      = (1 << 6)
+    FEATURE_BMI2      = (1 << 7)
+    FEATURE_CLWB      = (1 << 8)
+    FEATURE_LINUX     = (1 << 9)
+    FEATURE_AES256    = (1 << 10)
+    FEATURE_AFP       = (1 << 11)
+    FEATURE_SSSE3     = (1 << 12)
+    FEATURE_SSE4_1    = (1 << 13)
+    FEATURE_SSE4_2    = (1 << 14)
+    FEATURE_AES       = (1 << 15)
+    FEATURE_PCLMUL    = (1 << 16)
+    FEATURE_MOVBE     = (1 << 17)
+    FEATURE_ADX       = (1 << 18)
+    FEATURE_XSAVE     = (1 << 19)
+    FEATURE_RDPID     = (1 << 20)
+    FEATURE_CLFLOPT   = (1 << 21)
+    FEATURE_FSGSBASE  = (1 << 22)
+    FEATURE_EMMI      = (1 << 23)
+    FEATURE_AVX_VNNI  = (1 << 24)
+    FEATURE_SERIALIZE = (1 << 25)
+    FEATURE_AVX_NE_CONVERT = (1 << 26)
+    FEATURE_AVX_VNNI_INT8 = (1 << 27)
+    FEATURE_AVX_VNNI_INT16 = (1 << 28)
+    FEATURE_AVX_IFMA = (1 << 29)
+    FEATURE_SHA512 = (1 << 30)
 
 RegStringLookup = {
     "NONE":  Regs.REG_NONE,
@@ -150,31 +156,37 @@ ModeStringLookup = {
 }
 
 HostFeaturesLookup = {
-    "3DNOW"    : HostFeatures.FEATURE_3DNOW,
-    "SSE4A"    : HostFeatures.FEATURE_SSE4A,
-    "AVX"      : HostFeatures.FEATURE_AVX,
-    "RAND"     : HostFeatures.FEATURE_RAND,
-    "SHA"      : HostFeatures.FEATURE_SHA,
-    "CLZERO"   : HostFeatures.FEATURE_CLZERO,
-    "BMI1"     : HostFeatures.FEATURE_BMI1,
-    "BMI2"     : HostFeatures.FEATURE_BMI2,
-    "CLWB"     : HostFeatures.FEATURE_CLWB,
-    "LINUX"    : HostFeatures.FEATURE_LINUX,
-    "AES256"   : HostFeatures.FEATURE_AES256,
-    "AFP"      : HostFeatures.FEATURE_AFP,
-    "SSSE3"    : HostFeatures.FEATURE_SSSE3,
-    "SSE4.1"   : HostFeatures.FEATURE_SSE4_1,
-    "SSE4.2"   : HostFeatures.FEATURE_SSE4_2,
-    "AES"      : HostFeatures.FEATURE_AES,
-    "PCLMUL"   : HostFeatures.FEATURE_PCLMUL,
-    "MOVBE"    : HostFeatures.FEATURE_MOVBE,
-    "ADX"      : HostFeatures.FEATURE_ADX,
-    "XSAVE"    : HostFeatures.FEATURE_XSAVE,
-    "RDPID"    : HostFeatures.FEATURE_RDPID,
-    "CLFLOPT"  : HostFeatures.FEATURE_CLFLOPT,
-    "FSGSBASE" : HostFeatures.FEATURE_FSGSBASE,
-    "EMMI"     : HostFeatures.FEATURE_EMMI,
-    "AVX_VNNI" : HostFeatures.FEATURE_AVX_VNNI,
+    "3DNOW"     : HostFeatures.FEATURE_3DNOW,
+    "SSE4A"     : HostFeatures.FEATURE_SSE4A,
+    "AVX"       : HostFeatures.FEATURE_AVX,
+    "RAND"      : HostFeatures.FEATURE_RAND,
+    "SHA"       : HostFeatures.FEATURE_SHA,
+    "CLZERO"    : HostFeatures.FEATURE_CLZERO,
+    "BMI1"      : HostFeatures.FEATURE_BMI1,
+    "BMI2"      : HostFeatures.FEATURE_BMI2,
+    "CLWB"      : HostFeatures.FEATURE_CLWB,
+    "LINUX"     : HostFeatures.FEATURE_LINUX,
+    "AES256"    : HostFeatures.FEATURE_AES256,
+    "AFP"       : HostFeatures.FEATURE_AFP,
+    "SSSE3"     : HostFeatures.FEATURE_SSSE3,
+    "SSE4.1"    : HostFeatures.FEATURE_SSE4_1,
+    "SSE4.2"    : HostFeatures.FEATURE_SSE4_2,
+    "AES"       : HostFeatures.FEATURE_AES,
+    "PCLMUL"    : HostFeatures.FEATURE_PCLMUL,
+    "MOVBE"     : HostFeatures.FEATURE_MOVBE,
+    "ADX"       : HostFeatures.FEATURE_ADX,
+    "XSAVE"     : HostFeatures.FEATURE_XSAVE,
+    "RDPID"     : HostFeatures.FEATURE_RDPID,
+    "CLFLOPT"   : HostFeatures.FEATURE_CLFLOPT,
+    "FSGSBASE"  : HostFeatures.FEATURE_FSGSBASE,
+    "EMMI"      : HostFeatures.FEATURE_EMMI,
+    "AVX_VNNI"  : HostFeatures.FEATURE_AVX_VNNI,
+    "SERIALIZE" : HostFeatures.FEATURE_SERIALIZE,
+    "AVX_NE_CONVERT" : HostFeatures.FEATURE_AVX_NE_CONVERT,
+    "AVX_VNNI_INT8" : HostFeatures.FEATURE_AVX_VNNI_INT8,
+    "AVX_VNNI_INT16" : HostFeatures.FEATURE_AVX_VNNI_INT16,
+    "AVX_IFMA" : HostFeatures.FEATURE_AVX_IFMA,
+    "SHA512" : HostFeatures.FEATURE_SHA512,
 }
 
 def parse_hexstring(s):

Scripts/testharness_runner.py

@@ -1,8 +1,9 @@
 #!/usr/bin/python3
+import os
 import sys
-import subprocess
 from os import path
 from shutil import which
+# NB: This script is run with the "-S" flag, and therefore it can only use the Python standard library.
 
 # Args: <Known Failures file> <Known Failures Type File> <DisabledTestsFile> <DisabledTestsTypeFile> <DisabledTestsRunnerFile> <TestName> <FullTestName> <Test Harness Executable> <Args>...
 
@@ -59,9 +60,9 @@ if (disabled_tests.get(current_test)):
     sys.exit(125)
 
 # Run the test and wait for it to end to get the result
-Process = subprocess.Popen(RunnerArgs)
-Process.wait()
-ResultCode = Process.returncode
+Process = os.posix_spawnp(RunnerArgs[0], RunnerArgs, os.environ)
+_, Status = os.waitpid(Process, 0)
+ResultCode = os.waitstatus_to_exitcode(Status)
 
 # Check for known failures - try full test name first, then partial test name
 is_known_failure = known_failures.get(full_test_name) or known_failures.get(current_test)

Source/Common/FEXServerClient.cpp

@@ -235,6 +235,11 @@ int ConnectToServer(ConnectionOption ConnectionOption) {
   return -1;
 }
 
+void UnlinkServerSocketPath() {
+  ::unlink(GetServerSocketPath(false).c_str());
+  ::unlink(GetServerSocketPath(true).c_str());
+}
+
 bool SetupClient(std::string_view InterpreterPath) {
   ServerFD = FEXServerClient::ConnectToAndStartServer(InterpreterPath);
   if (ServerFD == -1) {

Source/Common/FEXServerClient.h

@@ -65,6 +65,7 @@ fextl::string GetTempFolder();
 fextl::string GetServerMountFolder();
 fextl::string GetServerSocketName();
 fextl::string GetServerSocketPath(bool ForceTmp);
+void UnlinkServerSocketPath();
 int GetServerFD();
 
 bool SetupClient(std::string_view InterpreterPath);

Source/Common/HostFeatures.cpp

@@ -8,6 +8,7 @@
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/HostFeatures.h>
 #include <FEXCore/Utils/FileLoading.h>
+#include <FEXCore/Utils/PrctlUtils.h>
 #include <FEXCore/Utils/StringUtils.h>
 
 #include <range/v3/view/split.hpp>
@@ -353,8 +354,8 @@ static void OverrideFeatures(FEXCore::HostFeatures* Features, uint64_t ForceSVEW
   const bool Enable##name = (HostFeatures() & FEXCore::Config::HostFeatures::ENABLE##enum_name) != 0;   \
   LogMan::Throw::AFmt(!(Disable##name && Enable##name), "Disabling and Enabling CPU feature (" #name ") is mutually exclusive");
 
-  ENABLE_DISABLE_OPTION(SupportsAVX, AVX, AVX);
-  ENABLE_DISABLE_OPTION(SupportsSVE128, SVE, SVE);
+  ENABLE_DISABLE_OPTION(ProcessFeatures.SupportsAVX, AVX, AVX);
+  ENABLE_DISABLE_OPTION(ProcessFeatures.SupportsSVE128, SVE, SVE);
   ENABLE_DISABLE_OPTION(SupportsAFP, AFP, AFP);
   ENABLE_DISABLE_OPTION(SupportsRCPC, LRCPC, LRCPC);
   ENABLE_DISABLE_OPTION(SupportsTSOImm9, LRCPC2, LRCPC2);
@@ -376,6 +377,8 @@ static void OverrideFeatures(FEXCore::HostFeatures* Features, uint64_t ForceSVEW
   ENABLE_DISABLE_OPTION(SupportsMOPS, MOPS, MOPS);
   ENABLE_DISABLE_OPTION(SupportsI8MM, I8MM, I8MM);
   ENABLE_DISABLE_OPTION(SupportsDotProd, DOTPROD, DOTPROD);
+  ENABLE_DISABLE_OPTION(SupportsBF16, BF16, BF16);
+  ENABLE_DISABLE_OPTION(SupportsCPUIndexInTPIDRRO, TPIDRRO, TPIDRRO);
   GET_SINGLE_OPTION(Crypto, CRYPTO);
 
 #undef ENABLE_DISABLE_OPTION
@@ -396,7 +399,7 @@ static void OverrideFeatures(FEXCore::HostFeatures* Features, uint64_t ForceSVEW
   }
 
   ///< Only force enable SVE256 if SVE is already enabled and ForceSVEWidth is set to >= 256.
-  Features->SupportsSVE256 = ForceSVEWidth && ForceSVEWidth >= 256;
+  Features->ProcessFeatures.SupportsSVE256 = ForceSVEWidth && ForceSVEWidth >= 256;
 }
 
 static void HandleErrata(FEXCore::HostFeatures* HostFeatures, uint64_t MIDR) {
@@ -508,18 +511,19 @@ void FetchHostFeatures(FEX::CPUFeatures& Features, FEXCore::HostFeatures& HostFe
   HostFeatures.SupportsWFXT = Features.Supports(CPUFeatures::Feature::WFxt);
   HostFeatures.SupportsI8MM = Features.Supports(CPUFeatures::Feature::I8MM);
   HostFeatures.SupportsDotProd = Features.Supports(CPUFeatures::Feature::DotProd);
+  HostFeatures.SupportsBF16 = Features.Supports(CPUFeatures::Feature::BF16);
 
 #ifdef VIXL_SIMULATOR
   // Hardcode enable SVE with 256-bit wide registers.
-  HostFeatures.SupportsSVE128 = ForceSVEWidth() ? ForceSVEWidth() >= 128 : true;
-  HostFeatures.SupportsSVE256 = ForceSVEWidth() ? ForceSVEWidth() >= 256 : true;
+  HostFeatures.ProcessFeatures.SupportsSVE128 = ForceSVEWidth() ? ForceSVEWidth() >= 128 : true;
+  HostFeatures.ProcessFeatures.SupportsSVE256 = ForceSVEWidth() ? ForceSVEWidth() >= 256 : true;
   HostFeatures.SupportsMOPS = true;
 
   // Simulator has a hardcoded ZVA size of 64-bytes.
   HostFeatures.SupportsCLZERO = true;
   HostFeatures.SupportsAES = true;
   HostFeatures.SupportsCRC = true;
-  HostFeatures.SupportsAVX = true;
+  HostFeatures.ProcessFeatures.SupportsAVX = true;
   HostFeatures.SupportsSHA = true;
   HostFeatures.SupportsPMULL_128Bit = true;
   HostFeatures.SupportsAES256 = true;
@@ -528,8 +532,8 @@ void FetchHostFeatures(FEX::CPUFeatures& Features, FEXCore::HostFeatures& HostFe
   HostFeatures.SupportsRPRES = false;
   HostFeatures.SupportsAFP = false;
 #else
-  HostFeatures.SupportsSVE128 = Features.Supports(CPUFeatures::Feature::SVE2);
-  HostFeatures.SupportsSVE256 = Features.Supports(CPUFeatures::Feature::SVE2) && Features.GetSVEVectorLengthInBits() >= 256;
+  HostFeatures.ProcessFeatures.SupportsSVE128 = Features.Supports(CPUFeatures::Feature::SVE2);
+  HostFeatures.ProcessFeatures.SupportsSVE256 = Features.Supports(CPUFeatures::Feature::SVE2) && Features.GetSVEVectorLengthInBits() >= 256;
   HostFeatures.SupportsMOPS = Features.Supports(CPUFeatures::Feature::MOPS);
 
   // Check if we can support cacheline clears
@@ -542,8 +546,8 @@ void FetchHostFeatures(FEX::CPUFeatures& Features, FEXCore::HostFeatures& HostFe
 #endif
 
   HostFeatures.Supports3DNow = true;
-  HostFeatures.SupportsAVX = true;
-  HostFeatures.SupportsAES256 = HostFeatures.SupportsAVX && HostFeatures.SupportsAES;
+  HostFeatures.ProcessFeatures.SupportsAVX = true;
+  HostFeatures.SupportsAES256 = HostFeatures.SupportsAVX() && HostFeatures.SupportsAES;
   HostFeatures.SupportsPreserveAllABI = FEX_HAS_PRESERVE_ALL_ATTR;
   HostFeatures.PreferZVAForVZero = false;
 
@@ -585,7 +589,7 @@ void FetchHostFeatures(FEX::CPUFeatures& Features, FEXCore::HostFeatures& HostFe
   HostFeatures.SupportsRAND = Feature.Feat_rand;
   HostFeatures.SupportsRCPC = true;
   HostFeatures.SupportsTSOImm9 = true;
-  HostFeatures.SupportsAVX = Feature.Feat_avx;
+  HostFeatures.ProcessFeatures.SupportsAVX = Feature.Feat_avx;
   HostFeatures.SupportsSHA = Feature.Feat_sha;
   HostFeatures.SupportsPMULL_128Bit = Feature.Feat_pclmulqdq;
   HostFeatures.SupportsAES256 = Feature.Feat_aes;
@@ -599,7 +603,16 @@ void FetchHostFeatures(FEX::CPUFeatures& Features, FEXCore::HostFeatures& HostFe
   OverrideFeatures(&HostFeatures, ForceSVEWidth());
 }
 
-FEXCore::HostFeatures FetchHostFeatures() {
+#ifndef _WIN32
+namespace TPIDRRO {
+  bool SupportsTPIDRRO_CPU() {
+    // 1 means supports, anything else is unsupported.
+    return prctl(PR_GET_TPIDRRO_CPU, 0, 0, 0, 0) == 1;
+  }
+} // namespace TPIDRRO
+#endif
+
+FEXCore::HostFeatures FetchHostFeatures(bool Is64BitMode) {
   FEX_CONFIG_OPT(CPUFeatureRegisters, CPUFEATUREREGISTERS);
 
   CPUFeatures Features {};
@@ -627,13 +640,21 @@ FEXCore::HostFeatures FetchHostFeatures() {
   __asm volatile("mrs %[midr], midr_el1" : [midr] "=r"(MIDR));
 #endif
 
-  FEXCore::HostFeatures HostFeatures = {};
+  FEXCore::HostFeatures HostFeatures = {
+#ifndef _WIN32
+    // Wine/Windows side sets this elsewhere.
+    .SupportsCPUIndexInTPIDRRO = TPIDRRO::SupportsTPIDRRO_CPU(),
+#endif
+    .ProcessFeatures =
+      {
+        .HostType = FEXCore::HostFeatures::HostTypeEnum::Linux,
+        // Fault free syncronous faults unsupported on Linux for now.
+        .SupportsFaultFreeSynchronousFaults = false,
+        .Is64BitMode = Is64BitMode,
+      },
+    .ProcessPID = static_cast<uint32_t>(::getpid())};
   FillMIDRInformationViaLinux(&HostFeatures);
   FetchHostFeatures(Features, HostFeatures, true, CTR, MIDR);
-
-  HostFeatures.SupportsCPUIndexInTPIDRRO = false;
-  HostFeatures.HostType = FEXCore::HostFeatures::HostTypeEnum::Linux;
-  HostFeatures.ProcessPID = ::getpid();
   return HostFeatures;
 }
 } // namespace FEX

Source/Common/HostFeatures.h

@@ -692,6 +692,6 @@ void FillMIDRInformationViaLinux(FEXCore::HostFeatures* Features);
 
 void FetchHostFeatures(FEX::CPUFeatures& Features, FEXCore::HostFeatures& HostFeatures, bool SupportsCacheMaintenanceOps, uint64_t CTR,
                        uint64_t MIDR);
-FEXCore::HostFeatures FetchHostFeatures();
+FEXCore::HostFeatures FetchHostFeatures(bool Is64BitMode);
 FEX::CPUFeatures GetCPUFeaturesFromIDRegisters();
 } // namespace FEX

Source/Common/X86Features.h

@@ -1,15 +1,60 @@
 #pragma once
 #include <cstdint>
 
-#ifdef ARCHITECTURE_x86_64
+#include <FEXCore/Core/Context.h>
+
+#if !defined(VIXL_SIMULATOR) && defined(ARCHITECTURE_x86_64)
 #include <cpuid.h>
+#endif
 
 namespace FEX::X86 {
 class Features final {
 public:
+#if !defined(VIXL_SIMULATOR) && defined(ARCHITECTURE_x86_64)
   Features() {
-    cpuid_data data {};
-    data = cpuid(0);
+    GetFeatures();
+  }
+#else
+  Features(FEXCore::Context::Context* CTX)
+    : CTX {CTX} {
+    GetFeatures();
+  }
+#endif
+
+  // Features.
+  bool Feat_3dnow {};
+  bool Feat_sse4a {};
+  bool Feat_bmi1 {};
+  bool Feat_bmi2 {};
+  bool Feat_clwb {};
+  bool Feat_aes {};
+  bool Feat_crc {};
+  bool Feat_rand {};
+  bool Feat_sha {};
+  bool Feat_pclmulqdq {};
+  bool Feat_vaes {};
+  bool Feat_clzero {};
+  bool Feat_avx {};
+  bool Feat_ssse3 {};
+  bool Feat_sse4_1 {};
+  bool Feat_sse4_2 {};
+  bool Feat_movbe {};
+  bool Feat_adx {};
+  bool Feat_xsave {};
+  bool Feat_rdpid {};
+  bool Feat_clflopt {};
+  bool Feat_fsgsbase {};
+  bool Feat_avx_vnni {};
+  bool Feat_serialize {};
+  bool Feat_avx_ne_convert {};
+  bool Feat_avx_vnni_int8 {};
+  bool Feat_avx_vnni_int16 {};
+  bool Feat_avx_ifma {};
+  bool Feat_sha512 {};
+
+private:
+  void GetFeatures() {
+    auto data = cpuid(0);
 
     if (data.eax >= 1) {
       auto data_1 = cpuid(0x1);
@@ -40,10 +85,16 @@ public:
       Feat_vaes = data_7.ecx & (1U << 9);
       Feat_pclmulqdq = Feat_pclmulqdq && (data_7.ecx & (1U << 10));
       Feat_rdpid = data_7.ecx & (1U << 22);
+      Feat_serialize = data_7.edx & (1U << 14);
 
       if (data_7.eax >= 1) {
         auto data_7_1 = cpuid(0x7, 0x1);
+        Feat_sha512 = Feat_avx && (data_7_1.eax & (1U << 0));
         Feat_avx_vnni = Feat_avx && (data_7_1.eax & (1U << 4));
+        Feat_avx_ifma = Feat_avx && (data_7_1.eax & (1U << 23));
+        Feat_avx_vnni_int8 = Feat_avx && (data_7_1.edx & (1U << 4));
+        Feat_avx_ne_convert = Feat_avx && (data_7_1.edx & (1U << 5));
+        Feat_avx_vnni_int16 = Feat_avx && (data_7_1.edx & (1U << 10));
       }
     }
 
@@ -61,41 +112,17 @@ public:
     }
   }
 
-  // Features.
-  bool Feat_3dnow {};
-  bool Feat_sse4a {};
-  bool Feat_bmi1 {};
-  bool Feat_bmi2 {};
-  bool Feat_clwb {};
-  bool Feat_aes {};
-  bool Feat_crc {};
-  bool Feat_rand {};
-  bool Feat_sha {};
-  bool Feat_pclmulqdq {};
-  bool Feat_vaes {};
-  bool Feat_clzero {};
-  bool Feat_avx {};
-  bool Feat_ssse3 {};
-  bool Feat_sse4_1 {};
-  bool Feat_sse4_2 {};
-  bool Feat_movbe {};
-  bool Feat_adx {};
-  bool Feat_xsave {};
-  bool Feat_rdpid {};
-  bool Feat_clflopt {};
-  bool Feat_fsgsbase {};
-  bool Feat_avx_vnni {};
-
-private:
-  struct cpuid_data {
-    uint32_t eax, ebx, ecx, edx;
-  };
-
-  cpuid_data cpuid(uint32_t Function, uint32_t Leaf = 0) {
-    cpuid_data data;
+#if !defined(VIXL_SIMULATOR) && defined(ARCHITECTURE_x86_64)
+  FEXCore::CPUID::FunctionResults cpuid(uint32_t Function, uint32_t Leaf = 0) {
+    FEXCore::CPUID::FunctionResults data;
     __cpuid_count(Function, Leaf, data.eax, data.ebx, data.ecx, data.edx);
     return data;
   }
+#else
+  FEXCore::CPUID::FunctionResults cpuid(uint32_t Function, uint32_t Leaf = 0) {
+    return CTX->RunCPUIDFunction(Function, Leaf);
+  }
+  FEXCore::Context::Context* CTX {};
+#endif
 };
 } // namespace FEX::X86
-#endif

Source/Steam/ServerManager.cpp

@@ -39,6 +39,25 @@ PipesType get_pipe() {
   return pipes;
 }
 
+void WaitForFDEnd(int fd) {
+  while (true) {
+    char buf[4096];
+    auto read_len = ::read(STDIN_FILENO, buf, sizeof(buf));
+    if (read_len < 0) {
+      if (errno == EINTR || errno == EAGAIN) {
+        // Interrupted, try again.
+        continue;
+      } else {
+        // Error on read.
+        break;
+      }
+    } else if (read_len == 0) {
+      // EOF
+      break;
+    }
+  }
+}
+
 int main(int argc, const char** argv, char** const envp) {
   LogMan::Throw::InstallHandler(AssertHandler);
   LogMan::Msg::InstallHandler(MsgHandler);
@@ -84,28 +103,19 @@ int main(int argc, const char** argv, char** const envp) {
   close(pipes.read_pipe);
   pipes.read_pipe = -1;
 
-  // Now that the server is started and watching our pipe, we can close the returned FD, as it'll stay open as long as the pipe is open.
+  // Do a blocking read, discarding any written data and wait for EOF.
+  WaitForFDEnd(STDIN_FILENO);
+
+  // Terminate the write pipe to say we're shutting down.
+  close(pipes.write_pipe);
+  pipes.write_pipe = -1;
+
+  // Wait for the server to clean-up.
+  WaitForFDEnd(ServerFD);
   close(ServerFD);
   ServerFD = -1;
 
-  // Do a blocking read, discarding any written data and wait for EOF.
-  while (true) {
-    char buf[4096];
-    auto read_len = ::read(STDIN_FILENO, buf, sizeof(buf));
-    if (read_len < 0) {
-      if (errno == EINTR || errno == EAGAIN) {
-        // Interrupted, try again.
-        continue;
-      } else {
-        // Error on read.
-        break;
-      }
-    } else if (read_len == 0) {
-      // EOF
-      break;
-    }
-  }
-
-  // Terminating will clean-up.
+  // Unlink the FEXServer socket files so they don't become stale.
+  FEXServerClient::UnlinkServerSocketPath();
   return 0;
 }

Source/Tools/CodeSizeValidation/CMakeLists.txt

@@ -3,4 +3,4 @@ list(APPEND LIBS FEXCore Common CommonTools JemallocLibs)
 add_executable(CodeSizeValidation Main.cpp)
 target_include_directories(CodeSizeValidation PRIVATE ${CMAKE_BINARY_DIR}/generated)
 
-target_link_libraries(CodeSizeValidation PRIVATE ${LIBS} ${PTHREAD_LIB})
+target_link_libraries(CodeSizeValidation PRIVATE ${LIBS} Threads::Threads)

Source/Tools/CodeSizeValidation/Main.cpp

@@ -556,6 +556,8 @@ int main(int argc, char** argv, char** const envp) {
     FEATURE_MOPS = (1U << 17),
     FEATURE_I8MM = (1U << 18),
     FEATURE_DOTPROD = (1U << 19),
+    FEATURE_BF16 = (1U << 20),
+    FEATURE_TPIDRRO = (1U << 21),
   };
 
   uint64_t SVEWidth = 0;
@@ -615,6 +617,12 @@ int main(int argc, char** argv, char** const envp) {
   if (TestHeaderData->EnabledHostFeatures & FEATURE_DOTPROD) {
     HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::ENABLEDOTPROD);
   }
+  if (TestHeaderData->EnabledHostFeatures & FEATURE_BF16) {
+    HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::ENABLEBF16);
+  }
+  if (TestHeaderData->EnabledHostFeatures & FEATURE_TPIDRRO) {
+    HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::ENABLETPIDRRO);
+  }
 
   if (TestHeaderData->EnabledHostFeatures & FEATURE_TSO) {
     FEXCore::Config::Set(FEXCore::Config::ConfigOption::CONFIG_TSOENABLED, "1");
@@ -679,6 +687,12 @@ int main(int argc, char** argv, char** const envp) {
   if (TestHeaderData->DisabledHostFeatures & FEATURE_DOTPROD) {
     HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::DISABLEDOTPROD);
   }
+  if (TestHeaderData->DisabledHostFeatures & FEATURE_BF16) {
+    HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::DISABLEBF16);
+  }
+  if (TestHeaderData->DisabledHostFeatures & FEATURE_TPIDRRO) {
+    HostFeatureControl |= static_cast<uint64_t>(FEXCore::Config::HostFeatures::DISABLETPIDRRO);
+  }
 
   if (TestHeaderData->DisabledHostFeatures & FEATURE_TSO) {
     FEXCore::Config::Set(FEXCore::Config::ConfigOption::CONFIG_TSOENABLED, "0");
@@ -695,7 +709,7 @@ int main(int argc, char** argv, char** const envp) {
   // Create FEXCore context.
   fextl::unique_ptr<FEXCore::Context::Context> CTX;
   {
-    auto HostFeatures = FEX::FetchHostFeatures();
+    auto HostFeatures = FEX::FetchHostFeatures(TestHeaderData->Bitness == 64);
     HostFeatures.IsInstCountCI = true;
     CTX = FEXCore::Context::Context::CreateNewContext(HostFeatures);
   }

Source/Tools/CommonTools/HarnessHelpers.h

@@ -322,6 +322,12 @@ public:
     FEATURE_FSGSBASE = (1 << 22),
     FEATURE_EMMI = (1 << 23),
     FEATURE_AVX_VNNI = (1 << 24),
+    FEATURE_SERIALIZE = (1 << 25),
+    FEATURE_AVX_NE_CONVERT = (1 << 26),
+    FEATURE_AVX_VNNI_INT8 = (1 << 27),
+    FEATURE_AVX_VNNI_INT16 = (1 << 28),
+    FEATURE_AVX_IFMA = (1 << 29),
+    FEATURE_SHA512 = (1 << 30),
   };
 
   bool Requires3DNow() const {
@@ -399,6 +405,24 @@ public:
   bool RequiresAVXVNNI() const {
     return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_AVX_VNNI;
   }
+  bool RequiresSerialize() const {
+    return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_SERIALIZE;
+  }
+  bool RequiresAVXNEConvert() const {
+    return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_AVX_NE_CONVERT;
+  }
+  bool RequiresAVXVNNIINT8() const {
+    return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_AVX_VNNI_INT8;
+  }
+  bool RequiresAVXVNNIINT16() const {
+    return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_AVX_VNNI_INT16;
+  }
+  bool RequiresAVXIFMA() const {
+    return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_AVX_IFMA;
+  }
+  bool RequiresSHA512() const {
+    return BaseConfig.OptionHostFeatures & HostFeatures::FEATURE_SHA512;
+  }
 
 private:
   FEX_CONFIG_OPT(ConfigDumpGPRs, DUMPGPRS);
@@ -609,6 +633,24 @@ public:
   bool RequiresAVXVNNI() const {
     return Config.RequiresAVXVNNI();
   }
+  bool RequiresSerialize() const {
+    return Config.RequiresSerialize();
+  }
+  bool RequiresAVXNEConvert() const {
+    return Config.RequiresAVXNEConvert();
+  }
+  bool RequiresAVXVNNIINT8() const {
+    return Config.RequiresAVXVNNIINT8();
+  }
+  bool RequiresAVXVNNIINT16() const {
+    return Config.RequiresAVXVNNIINT16();
+  }
+  bool RequiresAVXIFMA() const {
+    return Config.RequiresAVXIFMA();
+  }
+  bool RequiresSHA512() const {
+    return Config.RequiresSHA512();
+  }
 
 private:
   constexpr static uint64_t STACK_OFFSET = 0xc000'0000;

Source/Tools/FEXConfig/Main.cpp

@@ -107,7 +107,7 @@ void ConfigModel::erase(const QString& Name) {
 bool ConfigModel::getBool(const QString& Name, bool) const {
   auto ret = LoadedConfig->Get(NameToConfigLookup.at(Name.toStdString()));
   if (!ret || !*ret) {
-    throw std::runtime_error("Could not find setting");
+    ERROR_AND_DIE_FMT("Could not find setting");
   }
   return **ret == "1";
 }
@@ -139,7 +139,7 @@ void ConfigModel::setInt(const QString& Name, int Value) {
 QString ConfigModel::getString(const QString& Name, bool) const {
   auto ret = LoadedConfig->Get(NameToConfigLookup.at(Name.toStdString()));
   if (!ret || !*ret) {
-    throw std::runtime_error("Could not find setting");
+    ERROR_AND_DIE_FMT("Could not find setting");
   }
   return QString::fromUtf8((*ret)->c_str());
 }
@@ -159,12 +159,12 @@ QStringList ConfigModel::getStringList(const QString& Name, bool) const {
 int ConfigModel::getInt(const QString& Name, bool) const {
   auto ret = LoadedConfig->Get(NameToConfigLookup.at(Name.toStdString()));
   if (!ret || !*ret) {
-    throw std::runtime_error("Could not find setting");
+    ERROR_AND_DIE_FMT("Could not find setting");
   }
   int value;
   auto res = std::from_chars(&*(*ret)->begin(), &*(*ret)->end(), value);
   if (res.ptr != &*(*ret)->end()) {
-    throw std::runtime_error("Could not parse integer");
+    ERROR_AND_DIE_FMT("Could not parse integer");
   }
   return value;
 }

Source/Tools/FEXInterpreter/CMakeLists.txt

@@ -1,5 +1,5 @@
 list(APPEND LIBS FEXCore Common JemallocLibs LinuxEmulation
-  CommonTools ${PTHREAD_LIB} fmt::fmt)
+  CommonTools Threads::Threads fmt::fmt)
 
 set(DEFINES)
 if (ENABLE_VIXL_SIMULATOR)

Source/Tools/FEXInterpreter/FEXInterpreter.cpp

@@ -550,10 +550,10 @@ int main(int argc, char** argv, char** const envp) {
   bool SupportsSVE256 {};
   fextl::unique_ptr<FEXCore::Context::Context> CTX;
   {
-    auto HostFeatures = FEX::FetchHostFeatures();
+    auto HostFeatures = FEX::FetchHostFeatures(Loader.Is64BitMode());
     CTX = FEXCore::Context::Context::CreateNewContext(HostFeatures);
-    SupportsAVX = HostFeatures.SupportsAVX;
-    SupportsSVE256 = HostFeatures.SupportsSVE256;
+    SupportsAVX = HostFeatures.SupportsAVX();
+    SupportsSVE256 = HostFeatures.SupportsSVE256();
   }
 
   FEX::Kernel::Init(Loader.Is64BitMode(), CTX.get());

Source/Tools/FEXOfflineCompiler/CMakeLists.txt

@@ -10,9 +10,11 @@ target_link_libraries(FEXOfflineCompiler PRIVATE
 
 if (MINGW)
   if (ARCHITECTURE_arm64ec)
-    set_target_properties(FEXOfflineCompiler PROPERTIES OUTPUT_NAME "FEXOfflineCompiler64")
+    set_target_properties(FEXOfflineCompiler PROPERTIES OUTPUT_NAME
+      "FEXOfflineCompiler64")
   else()
-    set_target_properties(FEXOfflineCompiler PROPERTIES OUTPUT_NAME "FEXOfflineCompiler32")
+    set_target_properties(FEXOfflineCompiler PROPERTIES OUTPUT_NAME
+      "FEXOfflineCompiler32")
   endif()
   patch_library_wine(FEXOfflineCompiler)
   target_include_directories(FEXOfflineCompiler PRIVATE
@@ -22,7 +24,7 @@ if (MINGW)
   )
   target_link_libraries(FEXOfflineCompiler PRIVATE CommonWindows ntdll_ex)
 else()
-  target_link_libraries(FEXOfflineCompiler PRIVATE ${PTHREAD_LIB} LinuxEmulation)
+  target_link_libraries(FEXOfflineCompiler PRIVATE Threads::Threads LinuxEmulation)
 endif()
 
 LinkerGC(FEXOfflineCompiler)

Source/Tools/FEXOfflineCompiler/Main.cpp

@@ -410,7 +410,7 @@ static std::optional<std::string> GenerateSingleCache(FEXCore::ExecutableFileInf
 
   // Load HostFeatures
 #ifndef _WIN32
-  auto HostFeatures = FEX::FetchHostFeatures();
+  auto HostFeatures = FEX::FetchHostFeatures(Is64Bit);
 #else
   const auto NtDll = GetModuleHandleW(L"ntdll.dll");
   const bool IsWine = !!GetProcAddress(NtDll, "wine_get_version");
@@ -597,7 +597,7 @@ static int GenerateCache(int argc, const char** argv) {
   }
 
   if (!Data.contains(ProgramName)) {
-    throw std::runtime_error(fmt::format("Input code map {} did not contain {} ({:016x})", CodeMapPath, ProgramName.Filename, ProgramName.FileId));
+    ERROR_AND_DIE_FMT("Input code map {} did not contain {} ({:016x})", CodeMapPath, ProgramName.Filename, ProgramName.FileId);
   }
 
   fextl::string OutDir(Options.get("outdir"));

Source/Tools/FEXRootFSFetcher/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_executable(FEXRootFSFetcher Main.cpp XXFileHash.cpp)
-list(APPEND LIBS FEXCore Common JemallocDummy xxHash::xxhash ${PTHREAD_LIB})
+list(APPEND LIBS FEXCore Common JemallocDummy xxHash::xxhash Threads::Threads)
 
 LinkerGC(FEXRootFSFetcher)
 

Source/Tools/FEXServer/CMakeLists.txt

@@ -8,7 +8,7 @@ add_executable(FEXServer
 
 target_include_directories(FEXServer PRIVATE ${CMAKE_BINARY_DIR}/generated)
 
-target_link_libraries(FEXServer PRIVATE FEXCore Common CommonTools JemallocDummy ${PTHREAD_LIB})
+target_link_libraries(FEXServer PRIVATE FEXCore Common CommonTools JemallocDummy Threads::Threads)
 
 LinkerGC(FEXServer)
 

Source/Tools/LinuxEmulation/LinuxSyscalls/EmulatedFiles/EmulatedFiles.cpp

@@ -359,6 +359,7 @@ static fextl::string GenerateCPUInfo(FEXCore::Context::Context* ctx, uint32_t CP
     add_flag_if(res_d_1.eax & (1 << 3), "xsaves");
     add_flag_if(res_d_1.eax & (1 << 4), "xfd");
 
+    add_flag_if(res_7_1.eax & (1 << 0), "sha512");
     add_flag_if(res_7_1.eax & (1 << 4), "avx_vnni");
     add_flag_if(res_7_1.eax & (1 << 5), "avx512_bf16");
     add_flag_if(res_7_1.eax & (1 << 23), "avx_ifma");

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator.cpp

@@ -400,6 +400,20 @@ bool SignalDelegator::HandleDispatcherGuestSignal(FEXCore::Core::InternalThreadS
   // Set our state register to point to our guest thread data
   ArchHelpers::Context::SetState(ucontext, reinterpret_cast<uint64_t>(Frame));
 
+  if (GuestAction->sa_flags & SA_RESETHAND) {
+
+    {
+      // If SA_RESETHAND is set, then the signal handler is immediately reset to SIG_DFL.
+      std::lock_guard lk(GuestDelegatorMutex);
+      GuestAction->sigaction_handler.handler = SIG_DFL;
+    }
+
+    // If this signal is not required by FEX then we can fully uninstall the host handler as well.
+    if (HostHandlers[Signal].Required.load(std::memory_order_relaxed) != true) {
+      UninstallHostHandler(Signal);
+    }
+  }
+
   return true;
 }
 
@@ -586,6 +600,11 @@ bool SignalDelegator::HandleFrontendSIGSEGV(FEXCore::Core::InternalThreadState*
   }
 #endif
 
+  // If this fault is in the main thread stack guard region, we should try to grow the main thread stack.
+  if (SigInfo.si_code == SEGV_ACCERR && FEXCore::Allocator::TryGrowMainThreadStack(reinterpret_cast<uintptr_t>(SigInfo.si_addr))) {
+    return true;
+  }
+
   return false;
 }
 

Source/Tools/LinuxEmulation/LinuxSyscalls/SignalDelegator/GuestFramesManagement.cpp

@@ -438,7 +438,11 @@ uint64_t SignalDelegator::SetupFrame_x64(FEXCore::Core::InternalThreadState* Thr
 
     // Overwrite si_code and si_addr
     guest_siginfo->si_code = Thread->CurrentFrame->SynchronousFaultData.si_code;
-    guest_siginfo->si_addr = reinterpret_cast<void*>(ContextBackup->OriginalRIP);
+    if (Frame->SynchronousFaultData.FaultAddress) {
+      guest_siginfo->si_addr = reinterpret_cast<void*>(Frame->SynchronousFaultData.FaultAddress);
+    } else {
+      guest_siginfo->si_addr = reinterpret_cast<void*>(ContextBackup->OriginalRIP);
+    }
     Signal = Frame->SynchronousFaultData.Signal;
   } else {
     guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_TRAPNO] = ConvertSignalToTrapNo(Signal, HostSigInfo);
@@ -795,7 +799,14 @@ uint64_t SignalDelegator::SetupRTFrame_ia32(FEXCore::Core::InternalThreadState*
   case SigInfoLayout::LAYOUT_FAULT_RIP:
     // Macro expansion to get the si_addr
     // Can't really give a real result here. Pull from the context for now
-    guest_uctx->info._sifields._sigfault.addr = ContextBackup->OriginalRIP;
+    if (ContextBackup->FaultToTopAndGeneratedException && Frame->SynchronousFaultData.FaultAddress &&
+        Frame->SynchronousFaultData.Signal == FEXCore::Core::FAULT_SIGSEGV &&
+        Frame->SynchronousFaultData.TrapNo == FEXCore::X86State::X86_TRAPNO_PF) {
+      // Edge case for a synchronous fault due to partial instruction decode.
+      guest_uctx->info._sifields._sigfault.addr = static_cast<uint32_t>(Frame->SynchronousFaultData.FaultAddress);
+    } else {
+      guest_uctx->info._sifields._sigfault.addr = ContextBackup->OriginalRIP;
+    }
     break;
   case SigInfoLayout::LAYOUT_CHLD:
     guest_uctx->info._sifields._sigchld.pid = HostSigInfo->si_pid;

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls.h

@@ -408,6 +408,10 @@ private:
     if (Result == 0) return -errno; \
     return Result;                  \
   } while (0)
+#define SYSCALL_EFAULT_IF_NULL(Ptr)       \
+  do {                                    \
+    if ((Ptr) == nullptr) return -EFAULT; \
+  } while (0)
 
 extern FEX::HLE::SyscallHandler* _SyscallHandler;
 

Source/Tools/LinuxEmulation/LinuxSyscalls/Syscalls/Thread.cpp

@@ -546,11 +546,16 @@ void RegisterThread(FEX::HLE::SyscallHandler* Handler) {
       Result = -EINVAL; // We don't support CET, return EINVAL
       break;
     case 0x1011: // ARCH_GET_CPUID
-      return 1;
-      break;
-    case 0x1012:      // ARCH_SET_CPUID
-      return -ENODEV; // Claim we don't support faulting on CPUID
-      break;
+      return (Frame->State.CoreFlags & FEXCore::Core::CoreFlagsEnum::CPUID_ENABLED) == FEXCore::Core::CoreFlagsEnum::CPUID_ENABLED ? 1 : 0;
+    case 0x1012: // ARCH_SET_CPUID
+      if (addr == 0) {
+        // Disable CPUID (thus enabling CPUID faulting)
+        Frame->State.CoreFlags = Frame->State.CoreFlags & ~FEXCore::Core::CoreFlagsEnum::CPUID_ENABLED;
+      } else {
+        // Enable CPUID (thus disabling CPUID faulting)
+        Frame->State.CoreFlags = Frame->State.CoreFlags | FEXCore::Core::CoreFlagsEnum::CPUID_ENABLED;
+      }
+      return 0;
     default:
       LogMan::Msg::EFmt("Unknown prctl: 0x{:x}", code);
       Result = -EINVAL;

Source/Tools/LinuxEmulation/LinuxSyscalls/x32/FD.cpp

@@ -315,6 +315,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
       if (host_stat.st_nlink > std::numeric_limits<decltype(buf->st_nlink)>::max()) {
         return -EOVERFLOW;
       }
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
 
       *buf = host_stat;
@@ -332,6 +333,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
       if (host_stat.st_nlink > std::numeric_limits<decltype(buf->st_nlink)>::max()) {
         return -EOVERFLOW;
       }
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
 
       *buf = host_stat;
@@ -349,6 +351,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
       if (host_stat.st_nlink > std::numeric_limits<decltype(buf->st_nlink)>::max()) {
         return -EOVERFLOW;
       }
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
 
       *buf = host_stat;
@@ -360,6 +363,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Stat(pathname, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -370,6 +374,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat host_stat;
     uint64_t Result = ::fstat(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -380,6 +385,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Lstat(path, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -390,6 +396,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Stat(pathname, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -401,6 +408,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Lstat(path, &host_stat);
 
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -411,6 +419,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat64 host_stat;
     uint64_t Result = ::fstat64(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -421,6 +430,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct statfs host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Statfs(path, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -431,6 +441,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct statfs host_stat;
     uint64_t Result = ::fstatfs(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -443,6 +454,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct statfs64 host_stat;
     uint64_t Result = ::fstatfs64(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -455,6 +467,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct statfs host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Statfs(path, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
@@ -564,6 +577,7 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     struct stat64 host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.NewFSStatAt64(dirfd, pathname, &host_stat, flag);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
       FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }

Source/Tools/LinuxEmulation/LinuxSyscalls/x64/FD.cpp

@@ -63,32 +63,37 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
     });
 
   REGISTER_SYSCALL_IMPL_X64(stat, [](FEXCore::Core::CpuStateFrame* Frame, const char* pathname, FEX::HLE::x64::guest_stat* buf) -> uint64_t {
+    SYSCALL_EFAULT_IF_NULL(pathname);
     FaultSafeUserMemAccess::VerifyIsStringReadableMaxSize(pathname, PATH_MAX);
-    FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Stat(pathname, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
+      FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
     SYSCALL_ERRNO();
   });
 
   REGISTER_SYSCALL_IMPL_X64(fstat, [](FEXCore::Core::CpuStateFrame* Frame, int fd, FEX::HLE::x64::guest_stat* buf) -> uint64_t {
-    FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
     struct stat host_stat;
     uint64_t Result = ::fstat(fd, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
+      FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
     SYSCALL_ERRNO();
   });
 
   REGISTER_SYSCALL_IMPL_X64(lstat, [](FEXCore::Core::CpuStateFrame* Frame, const char* path, FEX::HLE::x64::guest_stat* buf) -> uint64_t {
+    SYSCALL_EFAULT_IF_NULL(path);
     FaultSafeUserMemAccess::VerifyIsStringReadableMaxSize(path, PATH_MAX);
-    FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
     struct stat host_stat;
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Lstat(path, &host_stat);
     if (Result != -1) {
+      SYSCALL_EFAULT_IF_NULL(buf);
+      FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       *buf = host_stat;
     }
     SYSCALL_ERRNO();
@@ -96,11 +101,13 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
 
   REGISTER_SYSCALL_IMPL_X64(
     newfstatat, [](FEXCore::Core::CpuStateFrame* Frame, int dirfd, const char* pathname, FEX::HLE::x64::guest_stat* buf, int flag) -> uint64_t {
+      SYSCALL_EFAULT_IF_NULL(pathname);
       FaultSafeUserMemAccess::VerifyIsStringReadableMaxSize(pathname, PATH_MAX);
-      FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
       struct stat host_stat;
       uint64_t Result = FEX::HLE::_SyscallHandler->FM.NewFSStatAt(dirfd, pathname, &host_stat, flag);
       if (Result != -1) {
+        SYSCALL_EFAULT_IF_NULL(buf);
+        FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
         *buf = host_stat;
       }
       SYSCALL_ERRNO();
@@ -133,8 +140,8 @@ void RegisterFD(FEX::HLE::SyscallHandler* Handler) {
   });
 
   REGISTER_SYSCALL_IMPL_X64(statfs, [](FEXCore::Core::CpuStateFrame* Frame, const char* path, struct statfs* buf) -> uint64_t {
+    SYSCALL_EFAULT_IF_NULL(path);
     FaultSafeUserMemAccess::VerifyIsStringReadableMaxSize(path, PATH_MAX);
-    FaultSafeUserMemAccess::VerifyIsWritable(buf, sizeof(*buf));
     uint64_t Result = FEX::HLE::_SyscallHandler->FM.Statfs(path, buf);
     SYSCALL_ERRNO();
   });

Source/Tools/TestHarnessRunner/CMakeLists.txt

@@ -1,4 +1,4 @@
-list(APPEND LIBS FEXCore Common JemallocLibs ${PTHREAD_LIB})
+list(APPEND LIBS FEXCore Common JemallocLibs Threads::Threads)
 
 set(SRCS TestHarnessRunner.cpp)
 if (NOT MINGW)

Source/Tools/TestHarnessRunner/TestHarnessRunner.cpp

@@ -18,6 +18,7 @@ $end_info$
 #endif
 
 #include "Common/HostFeatures.h"
+#include "Common/X86Features.h"
 #include "Common/Linux/SBRKAllocations.h"
 #include "HarnessHelpers.h"
 #include "TestHarnessRunner/HostRunner.h"
@@ -44,9 +45,6 @@ $end_info$
 #include <sys/types.h>
 #include <utility>
 
-#ifdef ARCHITECTURE_x86_64
-#include "Common/X86Features.h"
-#endif
 
 void MsgHandler(LogMan::DebugLevels Level, const char* Message) {
   fextl::fmt::print("{} {}\n", LogMan::DebugLevelStr(Level), Message);
@@ -245,11 +243,11 @@ int main(int argc, char** argv, char** const envp) {
   bool SupportsAVX = false;
   FEXCore::Core::CPUState State;
 
-  auto HostFeatures = FEX::FetchHostFeatures();
+  auto HostFeatures = FEX::FetchHostFeatures(Loader.Is64BitMode());
   auto CTX = FEXCore::Context::Context::CreateNewContext(HostFeatures);
 
 #ifndef _WIN32
-  auto SignalDelegation = FEX::HLE::CreateSignalDelegator(CTX.get(), {}, HostFeatures.SupportsAVX, HostFeatures.SupportsSVE256);
+  auto SignalDelegation = FEX::HLE::CreateSignalDelegator(CTX.get(), {}, HostFeatures.SupportsAVX(), HostFeatures.SupportsSVE256());
 #else
   // Enable exit on HLT while Wine's longjump is broken.
   //
@@ -259,20 +257,30 @@ int main(int argc, char** argv, char** const envp) {
 #endif
 
   // Skip any tests that the host doesn't support features for
-  SupportsAVX = HostFeatures.SupportsAVX;
+  SupportsAVX = HostFeatures.SupportsAVX();
 
   bool TestUnsupported = (!SupportsAVX && Loader.RequiresAVX()) || (!HostFeatures.SupportsRAND && Loader.RequiresRAND()) ||
                          (!HostFeatures.SupportsSHA && Loader.RequiresSHA()) || (!HostFeatures.SupportsCLZERO && Loader.RequiresCLZERO()) ||
                          (!HostFeatures.SupportsAES256 && Loader.RequiresAES256()) || (!HostFeatures.SupportsAFP && Loader.RequiresAFP());
-
+  TestUnsupported |= !HostFeatures.SupportsBF16 && Loader.RequiresAVXNEConvert();
 
   bool IsHostRunner = false;
 #if !defined(VIXL_SIMULATOR) && defined(ARCHITECTURE_x86_64)
   IsHostRunner = true;
-  ///< Features that are only unsupported when running using the HostRunner and the CI machine doesn't support the feature getting tested.
   FEX::X86::Features Feature {};
-  const bool Supports3DNow = Feature.Feat_3dnow;
+
   const bool SupportsSSE4A = Feature.Feat_sse4a;
+
+  // Hostrunner doesn't support EMMI while FEX does. CPUID doesn't have a query bit.
+  TestUnsupported |= Loader.RequiresEMMI();
+#else
+  FEX::X86::Features Feature {CTX.get()};
+
+  // Override features that FEX default disables.
+  const bool SupportsSSE4A = true;
+#endif
+
+  const bool Supports3DNow = Feature.Feat_3dnow;
   const bool SupportsBMI1 = Feature.Feat_bmi1;
   const bool SupportsBMI2 = Feature.Feat_bmi2;
   const bool SupportsCLWB = Feature.Feat_clwb;
@@ -288,6 +296,12 @@ int main(int argc, char** argv, char** const envp) {
   const bool SupportsCLFLOPT = Feature.Feat_clflopt;
   const bool SupportsFSGSBase = Feature.Feat_fsgsbase;
   const bool SupportsAVXVNNI = Feature.Feat_avx_vnni;
+  const bool SupportsSerialize = Feature.Feat_serialize;
+  const bool SupportsAVXNEConvert = Feature.Feat_avx_ne_convert;
+  const bool SupportsSHA512 = Feature.Feat_sha512;
+  const bool SupportsAVXIFMA = Feature.Feat_avx_ifma;
+  const bool SupportsAVXVNNIINT8 = Feature.Feat_avx_vnni_int8;
+  const bool SupportsAVXVNNIINT16 = Feature.Feat_avx_vnni_int16;
 
   TestUnsupported |=
     (!Supports3DNow && Loader.Requires3DNow()) || (!SupportsSSE4A && Loader.RequiresSSE4A()) || (!SupportsBMI1 && Loader.RequiresBMI1()) ||
@@ -295,9 +309,14 @@ int main(int argc, char** argv, char** const envp) {
     (!SupportsSSE4_1 && Loader.RequiresSSE4_1()) || (!SupportsSSE4_2 && Loader.RequiresSSE4_2()) ||
     (!SupportsAES && Loader.RequiresAES()) || (!SupportsPCLMUL && Loader.RequiresPCLMUL()) || (!SupportsMOVBE && Loader.RequiresMOVBE()) ||
     (!SupportsADX && Loader.RequiresADX()) || (!SupportsXSAVE && Loader.RequiresXSAVE()) || (!SupportsRDPID && Loader.RequiresRDPID()) ||
-    (!SupportsCLFLOPT && Loader.RequiresCLFLOPT()) || (!SupportsFSGSBase && Loader.RequiresFSGSBase()) || Loader.RequiresEMMI();
+    (!SupportsCLFLOPT && Loader.RequiresCLFLOPT()) || (!SupportsFSGSBase && Loader.RequiresFSGSBase());
   TestUnsupported |= !SupportsAVXVNNI && Loader.RequiresAVXVNNI();
-#endif
+  TestUnsupported |= !SupportsSerialize && Loader.RequiresSerialize();
+  TestUnsupported |= !SupportsAVXNEConvert && Loader.RequiresAVXNEConvert();
+  TestUnsupported |= !SupportsSHA512 && Loader.RequiresSHA512();
+  TestUnsupported |= !SupportsAVXIFMA && Loader.RequiresAVXIFMA();
+  TestUnsupported |= !SupportsAVXVNNIINT8 && Loader.RequiresAVXVNNIINT8();
+  TestUnsupported |= !SupportsAVXVNNIINT16 && Loader.RequiresAVXVNNIINT16();
 
 #ifdef _WIN32
   TestUnsupported |= Loader.RequiresLinux();

Source/Tools/TestHarnessRunner/TestHarnessRunner/HostRunner.cpp

@@ -221,8 +221,10 @@ public:
   }
 
   void Dispatch(uint64_t InitialRip) {
+#if !defined(VIXL_SIMULATOR) && defined(ARCHITECTURE_x86_64)
     FEX::X86::Features Feature {};
     Dispatcher(InitialRip, &ReturningStackLocation, CodeSegmentEntry, Feature.Feat_fsgsbase);
+#endif
   }
 
 private:

Source/Windows/ARM64EC/Module.S

@@ -158,9 +158,9 @@ DIRECT_SYSCALL_WRAPPER("#NtProtectVirtualMemoryNative", WineNtProtectVirtualMemo
   subs x16, x11, x9
   b.lo end
   adrp x17, NtDllRedirectionLUTSize
-  ldr x17, [x17, #:lo12:NtDllRedirectionLUTSize]
+  ldr w17, [x17, #:lo12:NtDllRedirectionLUTSize]
   cmp x16, x17
-  b.hi end
+  b.hs end
   adrp x17, NtDllRedirectionLUT
   ldr x17, [x17, #:lo12:NtDllRedirectionLUT]
   ldr w11, [x17, x16, lsl #2]

Source/Windows/ARM64EC/Module.cpp

@@ -450,6 +450,67 @@ static void LoadStateFromECContext(FEXCore::Core::InternalThreadState* Thread, C
     State.flags[FEXCore::X86State::X87FLAG_TOP_LOC] = (Context.FltSave.StatusWord >> 11) & 0b111;
     State.AbridgedFTW = Context.FltSave.TagWord;
   }
+
+  // TODO: These are super incorrect for debug registers.
+  if constexpr (false) {
+    if ((Context.ContextFlags & CONTEXT_DEBUG_REGISTERS) == CONTEXT_DEBUG_REGISTERS) {
+      // If any of the debug register state changed then we need to invalidate.
+      const bool Dr0Changed = (Context.Dr0 != State.Dr[0]) || ((Context.Dr7 ^ State.Dr7) & ((0b1111 << 16) | (0b11 << 0)));
+      const bool Dr1Changed = (Context.Dr1 != State.Dr[1]) || ((Context.Dr7 ^ State.Dr7) & ((0b1111 << 20) | (0b11 << 2)));
+      const bool Dr2Changed = (Context.Dr2 != State.Dr[2]) || ((Context.Dr7 ^ State.Dr7) & ((0b1111 << 24) | (0b11 << 4)));
+      const bool Dr3Changed = (Context.Dr3 != State.Dr[3]) || ((Context.Dr7 ^ State.Dr7) & ((0b1111 << 28) | (0b11 << 6)));
+
+      if (Dr0Changed || Dr1Changed || Dr2Changed || Dr3Changed) {
+        std::scoped_lock Lock(ThreadCreationMutex);
+        // Need to clear both context and state addresses if changed.
+        // Only if the BP type was instruction class.
+        if (Dr0Changed) {
+          if (Context.Dr0 && ((Context.Dr7 >> 16) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(Context.Dr0, 8, false);
+          }
+
+          if (State.Dr[0] && ((State.Dr7 >> 16) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(State.Dr[0], 8, false);
+          }
+        }
+        if (Dr1Changed) {
+          if (Context.Dr1 && ((Context.Dr7 >> 20) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(Context.Dr1, 8, false);
+          }
+
+          if (State.Dr[1] && ((State.Dr7 >> 20) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(State.Dr[1], 8, false);
+          }
+        }
+        if (Dr2Changed) {
+          if (Context.Dr2 && ((Context.Dr7 >> 24) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(Context.Dr2, 8, false);
+          }
+
+          if (State.Dr[2] && ((State.Dr7 >> 24) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(State.Dr[2], 8, false);
+          }
+        }
+        if (Dr3Changed) {
+          if (Context.Dr3 && ((Context.Dr7 >> 28) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(Context.Dr3, 8, false);
+          }
+
+          if (State.Dr[3] && ((State.Dr7 >> 28) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(State.Dr[3], 8, false);
+          }
+        }
+      }
+
+      // Update context now that we have invalidated.
+      State.Dr[0] = Context.Dr0;
+      State.Dr[1] = Context.Dr1;
+      State.Dr[2] = Context.Dr2;
+      State.Dr[3] = Context.Dr3;
+      State.Dr6 = Context.Dr6;
+      State.Dr7 = Context.Dr7;
+    }
+  }
 }
 
 static void ReconstructThreadState(FEXCore::Core::InternalThreadState* Thread, ARM64_NT_CONTEXT& Context) {
@@ -471,6 +532,16 @@ static void ReconstructThreadState(FEXCore::Core::InternalThreadState* Thread, A
   // Spill EFlags
   uint32_t EFlags = CTX->ReconstructCompactedEFLAGS(Thread, true, Context.X, Context.Cpsr);
   CTX->SetFlagsFromCompactedEFLAGS(Thread, EFlags);
+
+  // TODO: These are super incorrect for debug registers.
+  if constexpr (false) {
+    State.Dr[0] = Context.Bvr[0];
+    State.Dr[1] = Context.Bvr[1];
+    State.Dr[2] = Context.Bvr[2];
+    State.Dr[3] = Context.Bvr[3];
+    State.Dr6 = Context.Bcr[0];
+    State.Dr7 = Context.Bcr[1];
+  }
 }
 
 // Reconstructs an x64 context from the input thread's state, packed into a regular ARM64 context following the ARM64EC register mapping
@@ -539,20 +610,23 @@ static ARM64_NT_CONTEXT StoreStateToPackedECContext(FEXCore::Core::InternalThrea
   ECContext.Fpcr = FPCR;
   ECContext.Fpsr = FPSR;
 
+  // TODO: These are super incorrect for debug registers.
+  if constexpr (false) {
+    // Debug registers
+    ECContext.Bvr[0] = State.Dr[0];
+    ECContext.Bvr[1] = State.Dr[1];
+    ECContext.Bvr[2] = State.Dr[2];
+    ECContext.Bvr[3] = State.Dr[3];
+    ECContext.Bcr[0] = State.Dr6;
+    ECContext.Bcr[1] = State.Dr7;
+  }
+
   return ECContext;
 }
 
-static void RethrowGuestException(const EXCEPTION_RECORD& Rec, ARM64_NT_CONTEXT& Context) {
-  auto* Thread = GetCPUArea().ThreadState();
+static void RaiseException(FEXCore::Core::InternalThreadState* Thread, const EXCEPTION_RECORD& Rec, ARM64_NT_CONTEXT& GuestContext) {
   auto& Fault = Thread->CurrentFrame->SynchronousFaultData;
 
-  LogMan::Msg::DFmt("Reconstructing context");
-  if (!IsDispatcherAddress(Context.Pc)) {
-    ReconstructThreadState(Thread, Context);
-  }
-  ARM64_NT_CONTEXT GuestContext = StoreStateToPackedECContext(Thread, Context.Fpcr, Context.Fpsr);
-  LogMan::Msg::DFmt("pc: {:X} rip: {:X}", Context.Pc, GuestContext.Pc);
-
   // X64 Windows always clears TF, DF and AF when handling an exception, restoring after.
   // Current ARM64EC windows can only restore NZCV+SS when returning from an exception and other flags are left untouched from the handler context.
   // TODO: Can extend wine to support this by mapping the remaining EFlags into reserved cpsr members.
@@ -572,6 +646,47 @@ static void RethrowGuestException(const EXCEPTION_RECORD& Rec, ARM64_NT_CONTEXT&
   NtRaiseExceptionNative(&GuestRec, &GuestContext, FirstChance);
 }
 
+static void RethrowGuestException(FEXCore::Core::InternalThreadState* Thread, const EXCEPTION_RECORD& Rec, ARM64_NT_CONTEXT& HostContext) {
+  LogMan::Msg::DFmt("Reconstructing context");
+  if (!IsDispatcherAddress(HostContext.Pc)) {
+    ReconstructThreadState(Thread, HostContext);
+  }
+  auto GuestContext = StoreStateToPackedECContext(Thread, HostContext.Fpcr, HostContext.Fpsr);
+  LogMan::Msg::DFmt("pc: {:X} rip: {:X}", HostContext.Pc, GuestContext.Pc);
+
+  RaiseException(Thread, Rec, GuestContext);
+}
+
+[[noreturn]]
+static void SynchronousExceptionHandler(FEXCore::Context::Context* CTX, FEXCore::Core::CpuStateFrame* Frame) {
+  // Coming in to this function, the guest state is synchronized and the host thread is in a safe location.
+  auto Thread = Frame->Thread;
+  auto& Fault = Thread->CurrentFrame->SynchronousFaultData;
+
+  uint64_t FPCR, FPSR;
+  __asm volatile("mrs %[fpcr], fpcr" : [fpcr] "=r"(FPCR));
+  __asm volatile("mrs %[fpsr], fpsr" : [fpsr] "=r"(FPSR));
+
+  auto Context = StoreStateToPackedECContext(Thread, static_cast<uint32_t>(FPCR), static_cast<uint32_t>(FPSR));
+  EXCEPTION_RECORD Record = {};
+
+  switch (Fault.Signal) {
+  case FEXCore::Core::FAULT_SIGILL: Record.ExceptionCode = EXCEPTION_ILLEGAL_INSTRUCTION; break;
+  case FEXCore::Core::FAULT_SIGTRAP:
+    switch (Fault.TrapNo) {
+    case FEXCore::X86State::X86_TRAPNO_DB: Record.ExceptionCode = EXCEPTION_SINGLE_STEP; break;
+    case FEXCore::X86State::X86_TRAPNO_BP: Record.ExceptionCode = EXCEPTION_BREAKPOINT; break;
+    default: LogMan::Msg::EFmt("Unknown SIGTRAP trap: {}", Fault.TrapNo); break;
+    }
+    break;
+  case FEXCore::Core::FAULT_SIGSEGV: Record.ExceptionCode = EXCEPTION_ACCESS_VIOLATION; break;
+  default: ERROR_AND_DIE_FMT("Unknown fault signal"); break;
+  }
+
+  RaiseException(Thread, Record, Context);
+  ERROR_AND_DIE_FMT("This should always noreturn");
+}
+
 class ECSyscallHandler : public FEXCore::HLE::SyscallHandler, public FEXCore::Allocator::FEXAllocOperators {
 public:
   ECSyscallHandler() = default;
@@ -788,6 +903,11 @@ bool ResetToConsistentStateImpl(const ThreadCPUArea CPUArea, EXCEPTION_RECORD* E
     return true;
   }
 
+  if (NativeContext->Pc == reinterpret_cast<uint64_t>(&FEXCore::Assert::ForcedAssert)) {
+    LogMan::Msg::DFmt("FEX: Forced assert");
+    NtTerminateProcess(NtCurrentProcess(), 0x42);
+  }
+
   if (!IsJIT && !IsDispatcherAddress(NativeContext->Pc)) {
     LogMan::Msg::DFmt("Passing through exception");
     return false;
@@ -805,7 +925,7 @@ bool ResetToConsistentStateImpl(const ThreadCPUArea CPUArea, EXCEPTION_RECORD* E
   }
 
   if (IsEmulatorStackAddress(CPUArea, reinterpret_cast<uint64_t>(__builtin_frame_address(0)))) {
-    Exception::RethrowGuestException(*Exception, *NativeContext);
+    Exception::RethrowGuestException(Thread, *Exception, *NativeContext);
     LogMan::Msg::DFmt("Rethrowing onto guest stack: {:X}", NativeContext->Sp);
     return true;
   } else {
@@ -844,6 +964,70 @@ NTSTATUS ResetToConsistentState(EXCEPTION_RECORD* Exception, CONTEXT* GuestConte
     NtContinueNative(NativeContext, false);
   }
 
+  // TODO: These are super incorrect for debug registers.
+  if constexpr (false) {
+    auto Thread = CPUArea.ThreadState();
+    auto& State = Thread->CurrentFrame->State;
+
+    if ((GuestContext->ContextFlags & CONTEXT_DEBUG_REGISTERS) == CONTEXT_DEBUG_REGISTERS) {
+      // If any of the debug register state changed then we need to invalidate.
+      const bool Dr0Changed = (GuestContext->Dr0 != State.Dr[0]) || ((GuestContext->Dr7 ^ State.Dr7) & ((0b1111 << 16) | (0b11 << 0)));
+      const bool Dr1Changed = (GuestContext->Dr1 != State.Dr[1]) || ((GuestContext->Dr7 ^ State.Dr7) & ((0b1111 << 20) | (0b11 << 2)));
+      const bool Dr2Changed = (GuestContext->Dr2 != State.Dr[2]) || ((GuestContext->Dr7 ^ State.Dr7) & ((0b1111 << 24) | (0b11 << 4)));
+      const bool Dr3Changed = (GuestContext->Dr3 != State.Dr[3]) || ((GuestContext->Dr7 ^ State.Dr7) & ((0b1111 << 28) | (0b11 << 6)));
+
+      if (Dr0Changed || Dr1Changed || Dr2Changed || Dr3Changed) {
+        std::scoped_lock Lock(ThreadCreationMutex);
+        // Need to clear both context and state addresses if changed.
+        // Only if the BP type was instruction class.
+        if (Dr0Changed) {
+          if (GuestContext->Dr0 && ((GuestContext->Dr7 >> 16) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(GuestContext->Dr0, 8, false);
+          }
+
+          if (State.Dr[0] && ((State.Dr7 >> 16) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(State.Dr[0], 8, false);
+          }
+        }
+        if (Dr1Changed) {
+          if (GuestContext->Dr1 && ((GuestContext->Dr7 >> 20) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(GuestContext->Dr1, 8, false);
+          }
+
+          if (State.Dr[1] && ((State.Dr7 >> 20) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(State.Dr[1], 8, false);
+          }
+        }
+        if (Dr2Changed) {
+          if (GuestContext->Dr2 && ((GuestContext->Dr7 >> 24) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(GuestContext->Dr2, 8, false);
+          }
+
+          if (State.Dr[2] && ((State.Dr7 >> 24) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(State.Dr[2], 8, false);
+          }
+        }
+        if (Dr3Changed) {
+          if (GuestContext->Dr3 && ((GuestContext->Dr7 >> 28) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(GuestContext->Dr3, 8, false);
+          }
+
+          if (State.Dr[3] && ((State.Dr7 >> 28) & 0b11) == 0) {
+            InvalidationTracker->InvalidateAlignedInterval(State.Dr[3], 8, false);
+          }
+        }
+      }
+
+      // Update context now that we have invalidated.
+      State.Dr[0] = GuestContext->Dr0;
+      State.Dr[1] = GuestContext->Dr1;
+      State.Dr[2] = GuestContext->Dr2;
+      State.Dr[3] = GuestContext->Dr3;
+      State.Dr6 = GuestContext->Dr6;
+      State.Dr7 = GuestContext->Dr7;
+    }
+  }
+
   CPUArea.Area->InSimulation = false;
   CPUArea.Area->InSyscallCallback = false;
   return STATUS_SUCCESS;
@@ -1038,6 +1222,7 @@ NTSTATUS ThreadInit() {
 
   FEX::Windows::CallRetStack::InitializeThread(Thread);
   Thread->CurrentFrame->Pointers.ExitFunctionEC = reinterpret_cast<uintptr_t>(&ExitFunctionEC);
+  Thread->CurrentFrame->Pointers.SynchronousExceptionHandler = reinterpret_cast<uintptr_t>(&Exception::SynchronousExceptionHandler);
   CPUArea.StateFrame() = Thread->CurrentFrame;
 
   uint64_t EnterEC = Thread->CurrentFrame->Pointers.DispatcherLoopTopEnterEC;
@@ -1046,7 +1231,7 @@ NTSTATUS ThreadInit() {
   uint64_t EnterECFillSRA = Thread->CurrentFrame->Pointers.DispatcherLoopTopEnterECFillSRA;
   CPUArea.DispatcherLoopTopEnterECFillSRA() = EnterECFillSRA;
 
-  CPUArea.ContextAmd64() = {.ContextFlags = CONTEXT_CONTROL | CONTEXT_SEGMENTS | CONTEXT_INTEGER | CONTEXT_FLOATING_POINT,
+  CPUArea.ContextAmd64() = {.ContextFlags = CONTEXT_CONTROL | CONTEXT_SEGMENTS | CONTEXT_INTEGER | CONTEXT_FLOATING_POINT | CONTEXT_DEBUG_REGISTERS,
                             .AMD64_SegCs = (FEXCore::Core::CPUState::DEFAULT_USER_CS << 3) | 3,
                             .AMD64_SegDs = 0x2b,
                             .AMD64_SegEs = 0x2b,

Source/Windows/CMakeLists.txt

@@ -6,13 +6,16 @@ function(build_implib name)
     COMMENT "Building lib${name_ex}.a")
 
   add_library(${name_ex} SHARED IMPORTED)
-  set_property(TARGET ${name_ex} PROPERTY IMPORTED_IMPLIB ${CMAKE_CURRENT_BINARY_DIR}/lib${name_ex}.a)
+  set_property(TARGET ${name_ex} PROPERTY
+    IMPORTED_IMPLIB ${CMAKE_CURRENT_BINARY_DIR}/lib${name_ex}.a)
   add_dependencies(${name_ex} ${name_ex}lib)
 endfunction()
 
 function(patch_library_wine target)
   add_custom_command(TARGET ${target} POST_BUILD
-    COMMAND dd bs=32 count=1 seek=2 conv=notrunc if=${CMAKE_SOURCE_DIR}/Source/Windows/wine_builtin.bin of=$<TARGET_FILE:${target}>)
+    COMMAND dd bs=32 count=1 seek=2 conv=notrunc
+      if=${CMAKE_SOURCE_DIR}/Source/Windows/wine_builtin.bin
+      of=$<TARGET_FILE:${target}>)
 endfunction()
 
 separate_arguments(CXX_FLAGS_LIST NATIVE_COMMAND "${CMAKE_CXX_FLAGS}")

Source/Windows/Common/CPUFeatures.cpp

@@ -9,6 +9,7 @@
 #include <windows.h>
 
 #include "CPUFeatures.h"
+#include "FEXUnixLib.h"
 
 namespace {
 
@@ -75,21 +76,24 @@ FEXCore::HostFeatures CPUFeatures::FetchHostFeatures(bool IsWine, FEXCore::HostF
     RegCloseKey(Key);
   }
 
-  FEX::FetchHostFeatures(Features, HostFeatures, !IsWine, CTR, MIDR);
+  FEX::FetchHostFeatures(Features, HostFeatures, IsWine, CTR, MIDR);
 
   // Force-disable SVE until wine/windows gain support for SVE context save/restore
-  HostFeatures.SupportsSVE128 = false;
-  HostFeatures.SupportsSVE256 = false;
+  HostFeatures.ProcessFeatures.SupportsSVE128 = false;
+  HostFeatures.ProcessFeatures.SupportsSVE256 = false;
 
-  HostFeatures.SupportsCPUIndexInTPIDRRO = !IsWine;
+  HostFeatures.SupportsCPUIndexInTPIDRRO = !IsWine || FEX::Windows::UnixLib::SupportsTPIDRRO_CPU();
 
-  HostFeatures.HostType = HostType;
   HostFeatures.ProcessPID = ProcessPID;
 
-  if (HostType == FEXCore::HostFeatures::HostTypeEnum::Wow64) {
-    // AVX is unsupported for WOW64
-    HostFeatures.SupportsAVX = false;
-  }
+  // AVX is unsupported for WOW64
+  HostFeatures.ProcessFeatures.SupportsAVX = HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec;
+
+  HostFeatures.ProcessFeatures.HostType = HostType;
+  HostFeatures.ProcessFeatures.Is64BitMode = HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec;
+  // arm64ec supports synchronous faults without a double-fault dance.
+  HostFeatures.ProcessFeatures.SupportsFaultFreeSynchronousFaults = HostType == FEXCore::HostFeatures::HostTypeEnum::Arm64ec;
+
   return HostFeatures;
 }
 

Source/Windows/Common/CRT/CMakeLists.txt

@@ -1,2 +1,3 @@
-target_sources(CommonWindowsRuntime PRIVATE Alloc.cpp IO.cpp Math.cpp String.cpp Misc.cpp CRT.cpp)
+target_sources(CommonWindowsRuntime PRIVATE
+    Alloc.cpp IO.cpp Math.cpp String.cpp Misc.cpp CRT.cpp)
 add_subdirectory(Musl)

Source/Windows/Common/CRT/Musl/CMakeLists.txt

@@ -1 +1,4 @@
-target_sources(CommonWindowsRuntime PRIVATE exp2.c log2_data.c remainder.c strtoimax.c strtoull.c exp_data.c fmod.c log2.c isnan.c remquo.c strtoll.c strtoumax.c __math_uflow.c __math_oflow.c __math_xflow.c __math_invalid.c __math_divzero.c)
+target_sources(CommonWindowsRuntime PRIVATE
+    exp2.c log2_data.c remainder.c strtoimax.c strtoull.c exp_data.c fmod.c
+    log2.c isnan.c remquo.c strtoll.c strtoumax.c __math_uflow.c __math_oflow.c
+    __math_xflow.c __math_invalid.c __math_divzero.c)