Merge pull request #6106 from Sonicadvance1/310

Minor fixes

.github/workflows/mingw_build.yml

@@ -47,7 +47,8 @@ jobs:
       run: |
         cmake -S . -B build -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_TOOLCHAIN_FILE=$GITHUB_WORKSPACE/Data/CMake/toolchain_mingw.cmake \
         -DMINGW_TRIPLE=$MINGW_TRIPLE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DBUILD_TESTING=False \
-        -DCMAKE_INSTALL_PREFIX="$PWD"/build/install
+        -DCMAKE_INSTALL_PREFIX="$PWD"/build/install \
+        -DBUILD_FEX_MINGW_TESTS=True -DLLVM_MINGW="$HOME/llvm-mingw/build/"
 
     - name: Build
       run: cmake --build build

.github/workflows/pr-code-format.yml

@@ -61,3 +61,19 @@ jobs:
             --start-rev "$START_REV" \
             --end-rev "$END_REV" \
             --changed-files "$CHANGED_FILES"
+
+      - name: Check long CMake lines
+        if: always()
+        run: |
+          # check lines > 140 chars
+          # unit tests are left alone for now
+          # FEXInterpreter is also left alone until its long echo lines can be removed
+          echo "$CHANGED_FILES" | tr ',' '\n' | grep -ve "unittests\|External\|FEXInterpreter" \
+            | grep -e "CMakeLists.txt\|\.cmake" | xargs -L1 wc -L | while read -r line; do
+            if [ "$(echo "$line" | cut -d' ' -f1)" -gt 140 ]; then
+              echo "$line"
+            fi
+          done | tee lines.txt
+
+          # if any lines were emitted, fail
+          [ ! -s lines.txt ]

.gitlab-ci.yml

@@ -57,7 +57,7 @@ promote:
     - arm64
     - aarch64
   rules:
-    - if: '$PROMOTE_BRANCH'
+    - if: $PROMOTE_BRANCH && $CI_COMMIT_BRANCH == 'main'
   before_script:
     - apt-get -y update
     - apt-get install -y tmux curl

.neocmake.toml

@@ -0,0 +1 @@
+line_max_words = 140

CMakeLists.txt

@@ -5,40 +5,69 @@ include(CheckIncludeFiles)
 check_include_files("gdb/jit-reader.h" HAVE_GDB_JIT_READER_H)
 
 option(BUILD_FEX_LINUX_TESTS "Build FEXLinuxTests (requires x86 compiler)" FALSE)
+option(BUILD_FEX_MINGW_TESTS "Build FEXMingwTests (requires llvm-mingw compiler)" FALSE)
 option(BUILD_THUNKS "Build thunks" FALSE)
-option(BUILD_FEXCONFIG "Build FEXConfig" TRUE)
 option(ENABLE_CLANG_THUNKS "Build thunks with clang" TRUE)
+
+option(BUILD_FEXCONFIG "Build FEXConfig" TRUE)
 option(ENABLE_IWYU "Enable the Include What You Use sanitizer" FALSE)
 option(ENABLE_LTO "Enable LTO with compilation" TRUE)
-option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
+option(ENABLE_COVERAGE "Enable Code Coverage" FALSE)
 set(USE_LINKER "" CACHE STRING "Path to a custom linker program")
+
+option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
 option(ENABLE_UBSAN "Enable the Clang Undefined Behavior Sanitizer" FALSE)
 option(ENABLE_ASAN "Enable the Clang Address Sanitizer" FALSE)
 option(ENABLE_TSAN "Enable the Clang Thread Sanitizer" FALSE)
-option(ENABLE_COVERAGE "Enable Code Coverage" FALSE)
+
 option(ENABLE_ASSERTIONS "Enable debug assertions" FALSE)
 option(ENABLE_GDB_SYMBOLS "Enable GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H})
+
 option(ENABLE_STRICT_WERROR "Enable stricter -Werror" FALSE)
 option(ENABLE_WERROR "Enable -Werror" FALSE)
+
 option(ENABLE_FEX_ALLOCATOR "Enable allocator for FEX" TRUE)
 option(ENABLE_JEMALLOC_GLIBC_ALLOC "Enable jemalloc glibc allocator" TRUE)
+
 option(ENABLE_OFFLINE_TELEMETRY "Enable FEX offline telemetry" TRUE)
 option(ENABLE_COMPILE_TIME_TRACE "Enable time trace compile option" FALSE)
+
 option(ENABLE_LIBCXX "Use LLVM's libc++ instead of the GNU libstdc++" FALSE)
 option(ENABLE_CCACHE "Enable ccache for build caching" TRUE)
+
 option(ENABLE_VIXL_SIMULATOR "Use the VIXL simulator for emulation (only useful for CI testing)" FALSE)
 option(ENABLE_VIXL_DISASSEMBLER "Enable debug disassembler output with VIXL" FALSE)
 option(ENABLE_ZYDIS "Enable x86/x86-64 guest disassembler output with Zydis" FALSE)
+
 option(USE_LEGACY_BINFMTMISC "Use legacy method of setting up binfmt_misc" FALSE)
+
 option(ENABLE_FEXCORE_PROFILER "Enable FEXCore's timeline profiling capabilities" FALSE)
 set(FEXCORE_PROFILER_BACKEND "gpuvis" CACHE STRING "Set which backend to use for FEXCore's profiler")
 set_property(CACHE FEXCORE_PROFILER_BACKEND PROPERTY STRINGS gpuvis tracy)
+
 option(ENABLE_GLIBC_ALLOCATOR_HOOK_FAULT "Enables glibc memory allocation hooking with fault for CI testing")
+
 option(USE_PDB_DEBUGINFO "Build debug info in PDB format" FALSE)
 option(BUILD_STEAM_SUPPORT "Enable Steam integration" FALSE)
 
-set(X86_32_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
-set(X86_64_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake" CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
+set(X86_32_TOOLCHAIN_FILE
+  "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_32.cmake"
+  CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting i686")
+
+set(X86_64_TOOLCHAIN_FILE
+  "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_x86_64.cmake"
+  CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting x86_64")
+
+if (BUILD_FEX_MINGW_TESTS)
+  set(LLVM_MINGW "" CACHE FILEPATH "Path to llvm_mingw")
+  set(X86_32_MINGW_TOOLCHAIN_FILE
+    "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_mingw_x86_32.cmake"
+    CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting mingw_i686")
+  set(X86_64_MINGW_TOOLCHAIN_FILE
+    "${CMAKE_CURRENT_SOURCE_DIR}/Data/CMake/toolchain_mingw_x86_64.cmake"
+    CACHE FILEPATH "Toolchain file for the (cross-)compiler targeting mingw_x86_64")
+endif()
+
 set(X86_DEV_ROOTFS "/" CACHE FILEPATH "Path to the sysroot used for cross-compiling for i686 and x86_64")
 set(DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
 set(HOSTLIBS_DATA_DIRECTORY "" CACHE PATH "Global data directory (override)")
@@ -81,7 +110,8 @@ else ()
   message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
   set(CLANG_MINIMUM_VERSION 13.0)
   if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
-    message(FATAL_ERROR "Clang version too old for FEX. Need at least ${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}")
+    message(FATAL_ERROR "Clang version too old for FEX. Need at least "
+      "${CLANG_MINIMUM_VERSION} but has ${CMAKE_CXX_COMPILER_VERSION}")
   endif()
 endif()
 
@@ -137,7 +167,8 @@ if (ENABLE_FEXCORE_PROFILER)
     # Tracy can gather call stack samples in regular intervals, but this
     # isn't useful for us since it would usually sample opaque JIT code
     add_compile_definitions(TRACY_NO_SAMPLING=1)
-    # This pulls in libbacktrace which allocators in global constructors (before FEX can set up its allocator hooks)
+    # This pulls in libbacktrace which allocators in global constructors
+    # (before FEX can set up its allocator hooks)
     add_compile_definitions(TRACY_NO_CALLSTACK=1)
     if (MINGW)
       message(FATAL_ERROR "Tracy profiler not supported on MinGW")
@@ -174,8 +205,7 @@ set(OVERRIDE_HASH "detect" CACHE STRING "Override the FEX git hash")
 
 get_property(IS_MULTI_CONFIG GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
 if (NOT IS_MULTI_CONFIG AND NOT CMAKE_BUILD_TYPE)
-    set(CMAKE_BUILD_TYPE Release
-        CACHE STRING "Choose the type of build." FORCE)
+    set(CMAKE_BUILD_TYPE Release CACHE STRING "Choose the type of build." FORCE)
     message(STATUS "No build type set, defaulting to a Release build")
 endif()
 
@@ -224,11 +254,17 @@ check_cxx_source_compiles(
   HAS_CLANG_PRESERVE_ALL)
 unset(CMAKE_REQUIRED_FLAGS)
 if (HAS_CLANG_PRESERVE_ALL)
-  if (MINGW)
+  message(STATUS "Clang version ${CMAKE_CXX_COMPILER_VERSION}")
+  set(CLANG_MINIMUM_VERSION 22.1)
+  if (MINGW AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS ${CLANG_MINIMUM_VERSION})
     message(STATUS "Ignoring broken clang::preserve_all support")
     set(HAS_CLANG_PRESERVE_ALL FALSE)
   else()
     message(STATUS "Has clang::preserve_all")
+    if (MINGW)
+      message(STATUS "Disabling asynchronous-unwind-tables due to bug with preserve_all and mingw")
+      add_compile_options(-fno-asynchronous-unwind-tables)
+    endif()
   endif()
 endif()
 
@@ -246,7 +282,7 @@ check_cxx_source_compiles(
   return program_invocation_name == nullptr;
   }"
   HAS_PROGRAM_INVOCATION_NAME)
-add_compile_definitions("HAS_PROGRAM_INVOCATION_NAME=${HAS_PROGRAM_INVOCATION_NAME}")
+add_compile_definitions( "HAS_PROGRAM_INVOCATION_NAME=${HAS_PROGRAM_INVOCATION_NAME}")
 
 if (ENABLE_VIXL_SIMULATOR)
   # We can run the simulator on both x86-64 or AArch64 hosts
@@ -278,7 +314,8 @@ if (ENABLE_COMPILE_TIME_TRACE)
   link_libraries(-ftime-trace)
 endif()
 
-set(PTHREAD_LIB pthread)
+set(THREADS_PREFER_PTHREAD_FLAG ON)
+find_package(Threads REQUIRED)
 
 if (USE_LINKER)
   message(STATUS "Overriding linker to: ${USE_LINKER}")
@@ -287,8 +324,8 @@ endif()
 
 if (ENABLE_LIBCXX)
   message(WARNING "This is an unsupported configuration and should only be used for testing")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -stdlib=libc++")
-  set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi")
+  string(APPEND CMAKE_CXX_FLAGS " -std=c++11 -stdlib=libc++")
+  string(APPEND CMAKE_EXE_LINKER_FLAGS " -stdlib=libc++ -lc++abi")
 endif()
 
 if (NOT ENABLE_OFFLINE_TELEMETRY)
@@ -298,15 +335,16 @@ endif()
 
 if (ENABLE_UBSAN)
   # See https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
-  # and related discussion for the use of -fno-sanitize=alignment -fno-sanitize=function
-  # with UBSAN.
-  # alignment: we don't follow a strict alignment policy, for example IR uses packed structs
-  # that are regularly access unaligned.
-  # function: syscalls cast function pointers to void (*)(unsigned long...), causing warnings
-  # related to this access.
+  # and related discussion for the use of -fno-sanitize=alignment and -fno-sanitize=function with UBSAN.
+  # alignment: we don't follow a strict alignment policy, for example IR uses packed structs that are regularly access unaligned.
+  # function: syscalls cast function pointers to void (*)(unsigned long...), causing warnings related to this access.
   add_compile_definitions(ENABLE_UBSAN=1)
-  add_compile_options(-fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=function -fno-sanitize-recover=undefined)
-  link_libraries(-fno-omit-frame-pointer -fsanitize=undefined -fno-sanitize=alignment -fno-sanitize=function -fno-sanitize-recover=undefined)
+  add_compile_options(-fno-omit-frame-pointer -fsanitize=undefined
+    -fno-sanitize=alignment -fno-sanitize=function
+    -fno-sanitize-recover=undefined)
+  link_libraries(-fno-omit-frame-pointer -fsanitize=undefined
+    -fno-sanitize=alignment -fno-sanitize=function
+    -fno-sanitize-recover=undefined)
 endif()
 
 if (ENABLE_ASAN)
@@ -328,7 +366,8 @@ endif()
 if (ENABLE_JEMALLOC_GLIBC_ALLOC)
   # The glibc jemalloc subproject which hooks the glibc allocator.
   # Required for thunks to work.
-  # All host native libraries will use this allocator, while *most* other FEX internal allocations will use the other jemalloc allocator.
+  # All host native libraries will use this allocator, while *most*
+  # other FEX internal allocations will use the other jemalloc allocator.
   add_subdirectory(External/jemalloc_glibc/)
 elseif (NOT MINGW)
   message(STATUS
@@ -354,11 +393,11 @@ if (USE_PDB_DEBUGINFO)
   add_link_options(-g -Wl,--pdb=)
 endif()
 
-set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
-set(CMAKE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_LINKER_FLAGS_RELWITHDEBINFO} -fno-omit-frame-pointer")
+string(APPEND CMAKE_CXX_FLAGS_RELWITHDEBINFO " -fno-omit-frame-pointer")
+string(APPEND CMAKE_LINKER_FLAGS_RELWITHDEBINFO " -fno-omit-frame-pointer")
 
-set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fomit-frame-pointer")
-set(CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-pointer")
+string(APPEND CMAKE_CXX_FLAGS_RELEASE " -fomit-frame-pointer")
+string(APPEND CMAKE_LINKER_FLAGS_RELEASE " -fomit-frame-pointer")
 
 ## Modules ##
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/Data/CMake/)
@@ -367,7 +406,10 @@ include(LinkerGC)
 
 ## Externals ##
 
-find_package(unordered_dense QUIET CONFIG)
+if (NOT BUILD_STEAM_SUPPORT)
+  find_package(unordered_dense QUIET CONFIG)
+endif()
+
 if (NOT unordered_dense_FOUND)
   add_subdirectory(External/unordered_dense)
 endif()
@@ -378,8 +420,10 @@ if (BUILD_TESTING OR ENABLE_VIXL_DISASSEMBLER OR ENABLE_VIXL_SIMULATOR)
 endif()
 
 if (ENABLE_ZYDIS)
-  find_package(Zycore 1.5 MODULE QUIET)
-  find_package(Zydis 4.0 MODULE QUIET)
+  if (NOT BUILD_STEAM_SUPPORT)
+    find_package(Zycore 1.5 MODULE QUIET)
+    find_package(Zydis 4.0 MODULE QUIET)
+  endif()
 
   if (TARGET Zydis::Zydis AND TARGET Zycore::Zycore)
     message(STATUS "Using system Zydis")
@@ -400,7 +444,7 @@ find_package(Python 3.9 REQUIRED COMPONENTS Interpreter)
 
 set(BUILD_SHARED_LIBS OFF)
 
-if (NOT CMAKE_CROSSCOMPILING)
+if (NOT CMAKE_CROSSCOMPILING AND NOT BUILD_STEAM_SUPPORT)
   find_package(xxhash MODULE QUIET)
 endif()
 
@@ -419,7 +463,8 @@ if (BUILD_TESTING)
     add_subdirectory(External/Catch2/)
 
     # Pull in catch_discover_tests definition
-    list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/External/Catch2/contrib/")
+    list(APPEND CMAKE_MODULE_PATH
+      "${CMAKE_CURRENT_SOURCE_DIR}/External/Catch2/contrib/")
   endif()
 
   include(Catch)
@@ -428,7 +473,7 @@ else ()
   file(GENERATE OUTPUT CTestTestfile.cmake CONTENT "# No tests since BUILD_TESTING is disabled")
 endif()
 
-if (MINGW)
+if (MINGW OR BUILD_STEAM_SUPPORT)
   set(FMT_INSTALL OFF)
   add_subdirectory(External/fmt/)
 else()
@@ -440,10 +485,14 @@ else()
   endif()
 endif()
 
-find_package(range-v3 QUIET)
+if (NOT BUILD_STEAM_SUPPORT)
+  find_package(range-v3 QUIET)
+endif()
+
 if (NOT range-v3_FOUND)
   add_subdirectory(External/range-v3/)
-  target_compile_definitions(range-v3 INTERFACE RANGES_DISABLE_DEPRECATED_WARNINGS)
+  target_compile_definitions(range-v3 INTERFACE
+    RANGES_DISABLE_DEPRECATED_WARNINGS)
 endif()
 
 add_subdirectory(External/tiny-json/)
@@ -484,11 +533,13 @@ if(ENABLE_WERROR OR ENABLE_STRICT_WERROR)
 endif()
 
 set(FEX_TUNE_COMPILE_FLAGS)
+
 if (NOT TUNE_ARCH STREQUAL "generic")
   set(TUNE_ARCH_STRING "${TUNE_ARCH}")
   if(ARCHITECTURE_arm64)
     set(TUNE_ARCH_STRING "${TUNE_ARCH}+crc")
   endif()
+
   check_cxx_compiler_flag("-march=${TUNE_ARCH_STRING}" COMPILER_SUPPORTS_ARCH_TYPE)
   if(COMPILER_SUPPORTS_ARCH_TYPE)
     list(APPEND FEX_TUNE_COMPILE_FLAGS "-march=${TUNE_ARCH_STRING}")
@@ -505,19 +556,23 @@ if (TUNE_CPU STREQUAL "native")
   if(ARCHITECTURE_arm64)
     if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 999999.0)
       # Clang 12.0 fixed the -mcpu=native bug with mixed big.little implementers
-      # Clang can not currently check for native Apple M1 type in hypervisor. Currently disabled
+      # Clang can not currently check for native Apple M1 type in hypervisor.
+      # Currently disabled
       check_cxx_compiler_flag("-mcpu=native" COMPILER_SUPPORTS_CPU_TYPE)
       if(COMPILER_SUPPORTS_CPU_TYPE)
         list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=native")
       endif()
     else()
-      execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py" "/proc/cpuinfo" "${CMAKE_CXX_COMPILER_VERSION}"
+      execute_process(COMMAND python3
+        "${PROJECT_SOURCE_DIR}/Scripts/aarch64_fit_native.py"
+        "/proc/cpuinfo" "${CMAKE_CXX_COMPILER_VERSION}"
         OUTPUT_VARIABLE AARCH64_CPU)
 
       string(STRIP ${AARCH64_CPU} AARCH64_CPU)
 
       execute_process(COMMAND python3 "${PROJECT_SOURCE_DIR}/Scripts/NeedDisabledSVE.py"
         RESULT_VARIABLE NEEDS_SVE_DISABLED)
+
       if (NEEDS_SVE_DISABLED)
         message(STATUS "Platform has bugged SVE. Disabling")
         set(AARCH64_CPU "cortex-a78")
@@ -539,7 +594,8 @@ elseif (NOT TUNE_CPU STREQUAL "none")
   if(COMPILER_SUPPORTS_CPU_TYPE)
     list(APPEND FEX_TUNE_COMPILE_FLAGS "-mcpu=${TUNE_CPU}")
   else()
-    message(FATAL_ERROR "Trying to compile cpu type '${TUNE_CPU}' but the compiler doesn't support this")
+    message(FATAL_ERROR "Trying to compile cpu type '${TUNE_CPU}' "
+      "but the compiler doesn't support this")
   endif()
 endif()
 
@@ -602,13 +658,38 @@ add_compile_options(-Wall)
 if (BUILD_TESTING)
   message(STATUS "Unit tests are enabled")
 
-  set(TEST_JOB_COUNT "" CACHE STRING "Override number of parallel jobs to use while running tests")
+  set(TEST_JOB_COUNT "" CACHE STRING
+    "Override number of parallel jobs to use while running tests")
   if (TEST_JOB_COUNT)
     message(STATUS "Running tests with ${TEST_JOB_COUNT} jobs")
   elseif(CMAKE_VERSION VERSION_LESS "3.29")
-    execute_process(COMMAND "nproc" OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE TEST_JOB_COUNT)
+    execute_process(COMMAND "nproc"
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      OUTPUT_VARIABLE TEST_JOB_COUNT)
   endif()
+
   set(TEST_JOB_FLAG "-j${TEST_JOB_COUNT}")
+
+  # When running "ninja tests", defined below, it is useful to intentionally
+  # oversubscribe the CPU cores since many of the tests spend their time idle.
+  if ("$CACHE{TEST_JOB_COUNT}" STREQUAL "")
+    execute_process(COMMAND "nproc"
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+      OUTPUT_VARIABLE TESTS_TARGET_JOB_COUNT)
+
+    math(EXPR TESTS_TARGET_JOB_COUNT "${TESTS_TARGET_JOB_COUNT} * 3 / 2")
+    set(TESTS_TARGET_JOB_FLAG "-j${TESTS_TARGET_JOB_COUNT}")
+  else()
+    set(TESTS_TARGET_JOB_FLAG ${TEST_JOB_FLAG})
+  endif()
+
+  # Runs the whole test suite, with large pages enabled to
+  # reduce the cost of fork() in ctest.
+  add_custom_target(tests
+    WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
+    USES_TERMINAL
+    COMMAND ${CMAKE_COMMAND} -E env GLIBC_TUNABLES=glibc.malloc.hugetlb=1
+      ctest "--progress" "--timeout" "302" ${TESTS_TARGET_JOB_FLAG})
 endif()
 
 add_subdirectory(External/SoftFloat-3e/)
@@ -628,15 +709,15 @@ if (NOT BUILD_STEAM_SUPPORT)
   add_subdirectory(Data/AppConfig/)
 endif()
 
-# Install the ThunksDB file
-file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.json)
-
-# Any application configuration json file gets installed
-foreach(CONFIG_SRC ${CONFIG_SOURCES})
-  install(FILES ${CONFIG_SRC}
-    DESTINATION ${DATA_DIRECTORY}/
-    COMPONENT Runtime)
-endforeach()
+if (NOT MINGW)
+  # Install the ThunksDB file
+  file(GLOB CONFIG_SOURCES CONFIGURE_DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/Data/*.json)
+  foreach(CONFIG_SRC ${CONFIG_SOURCES})
+    install(FILES ${CONFIG_SRC}
+      DESTINATION ${DATA_DIRECTORY}/
+      COMPONENT Runtime)
+  endforeach()
+endif()
 
 if (BUILD_TESTING)
   add_subdirectory(unittests/)

CodeEmitter/CodeEmitter/ASIMDOps.inl

@@ -768,6 +768,16 @@ public:
       ASIMD2RegMisc(0, size, 0b11101, rd, rn);
     }
   }
+  ///< size is the destination size.
+  ///< source size is the next size up.
+  void bfcvtn(QRegister rd, QRegister rn) {
+    ASIMD2RegMisc(0, SubRegSize::i32Bit, 0b10110, rd.D(), rn.D());
+  }
+  ///< size is the destination size.
+  ///< source size is the next size up.
+  void bfcvtn2(QRegister rd, QRegister rn) {
+    ASIMD2RegMisc(0, SubRegSize::i32Bit, 0b10110, rd, rn);
+  }
 
   template<IsQOrDRegister T>
   void rev32(SubRegSize size, T rd, T rn) {

Data/CMake/LinkerGC.cmake

@@ -3,7 +3,7 @@
 # This applies some common linker options that reduce code size and linking time in Release mode. Namely:
 # --gc-sections: Linktime garbage collection, discards unused sections from the final output
 # --strip-all  : Similar to running `strip`, discards the symbol table from the final output
-# --as-needed  : Only includes libraries that are actually needed in the final output.
+# --as-needed  : Only includes libraries that are actually needed in the final output
 
 macro(LinkerGC target)
   if (CMAKE_BUILD_TYPE MATCHES "RELEASE")

Data/CMake/toolchain_mingw.cmake

@@ -6,14 +6,17 @@ set(CMAKE_CXX_COMPILER ${MINGW_TRIPLE}-clang++)
 set(CMAKE_DLLTOOL ${MINGW_TRIPLE}-dlltool)
 set(CMAKE_AR ${MINGW_TRIPLE}-ar)
 
-# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
-# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
+# Compile everything as static to avoid requiring the MinGW runtime libraries,
+# force page aligned sections so that debug symbols work correctly,
+# and disable loop alignment to workaround an LLVM bug
 # (https://github.com/llvm/llvm-project/issues/47432)
 set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
 set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+
 set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
 set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
 set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+
 set(CMAKE_SYSTEM_NAME Windows)
 set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
 

Data/CMake/toolchain_mingw_x86_32.cmake

@@ -0,0 +1,30 @@
+set(CMAKE_SYSTEM_PROCESSOR i686)
+
+set(LLVM_MINGW "" CACHE FILEPATH "Path to LLVM_MINGW")
+if (LLVM_MINGW)
+  set(LLVM_MINGW_BIN "${LLVM_MINGW}/bin/")
+else()
+  set(LLVM_MINGW_BIN "")
+endif()
+
+set(MINGW_TRIPLE "i686-w64-mingw32")
+
+set(CMAKE_RC_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-windres)
+set(CMAKE_C_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang)
+set(CMAKE_CXX_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang++)
+set(CMAKE_DLLTOOL ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-dlltool)
+set(CMAKE_AR ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-ar)
+
+# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
+# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
+# (https://github.com/llvm/llvm-project/issues/47432)
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_SYSTEM_NAME Windows)
+set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+

Data/CMake/toolchain_mingw_x86_64.cmake

@@ -0,0 +1,29 @@
+set(CMAKE_SYSTEM_PROCESSOR x86_64)
+
+set(LLVM_MINGW "" CACHE FILEPATH "Path to LLVM_MINGW")
+if (LLVM_MINGW)
+  set(LLVM_MINGW_BIN "${LLVM_MINGW}/bin/")
+else()
+  set(LLVM_MINGW_BIN "")
+endif()
+
+set(MINGW_TRIPLE "x86_64-w64-mingw32")
+
+set(CMAKE_RC_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-windres)
+set(CMAKE_C_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang)
+set(CMAKE_CXX_COMPILER ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-clang++)
+set(CMAKE_DLLTOOL ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-dlltool)
+set(CMAKE_AR ${LLVM_MINGW_BIN}${MINGW_TRIPLE}-ar)
+
+# Compile everything as static to avoid requiring the MinGW runtime libraries, force page aligned sections so that
+# debug symbols work correctly, and disable loop alignment to workaround an LLVM bug
+# (https://github.com/llvm/llvm-project/issues/47432)
+set(CMAKE_SHARED_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+set(CMAKE_EXE_LINKER_FLAGS_INIT "-static -Wl,--file-alignment=4096,/mllvm:-align-loops=1")
+set(CMAKE_C_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_CXX_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_STANDARD_LIBRARIES "" CACHE STRING "" FORCE)
+set(CMAKE_SYSTEM_NAME Windows)
+set(CMAKE_SYSTEM_PROCESSOR ${MINGW_TRIPLE})
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)

Data/binfmts/CMakeLists.txt

@@ -15,7 +15,8 @@ if (NOT USE_LEGACY_BINFMTMISC)
   configure_file(FEX-x86.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf)
   configure_file(FEX-x86_64.conf.in ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf)
 
-  install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf
+  install(FILES ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86.conf
+    ${CMAKE_BINARY_DIR}/Data/binfmts/FEX-x86_64.conf
     DESTINATION ${CMAKE_INSTALL_PREFIX}/lib/binfmt.d/
     COMPONENT Runtime)
 else()

External/code-format-helper/requirements_formatting.txt

@@ -4,40 +4,40 @@
 #
 #    pip-compile --generate-hashes --output-file=requirements_formatting.txt --strip-extras requirements_formatting.txt.in
 #
-black==26.3.1 \
-    --hash=sha256:0126ae5b7c09957da2bdbd91a9ba1207453feada9e9fe51992848658c6c8e01c \
-    --hash=sha256:0f76ff19ec5297dd8e66eb64deda23631e642c9393ab592826fd4bdc97a4bce7 \
-    --hash=sha256:28ef38aee69e4b12fda8dba75e21f9b4f979b490c8ac0baa7cb505369ac9e1ff \
-    --hash=sha256:2bd5aa94fc267d38bb21a70d7410a89f1a1d318841855f698746f8e7f51acd1b \
-    --hash=sha256:2c50f5063a9641c7eed7795014ba37b0f5fa227f3d408b968936e24bc0566b07 \
-    --hash=sha256:2d6bfaf7fd0993b420bed691f20f9492d53ce9a2bcccea4b797d34e947318a78 \
-    --hash=sha256:41cd2012d35b47d589cb8a16faf8a32ef7a336f56356babd9fcf70939ad1897f \
-    --hash=sha256:474c27574d6d7037c1bc875a81d9be0a9a4f9ee95e62800dab3cfaadbf75acd5 \
-    --hash=sha256:5602bdb96d52d2d0672f24f6ffe5218795736dd34807fd0fd55ccd6bf206168b \
-    --hash=sha256:5e9d0d86df21f2e1677cc4bd090cd0e446278bcbbe49bf3659c308c3e402843e \
-    --hash=sha256:5ed0ca58586c8d9a487352a96b15272b7fa55d139fc8496b519e78023a8dab0a \
-    --hash=sha256:6c54a4a82e291a1fee5137371ab488866b7c86a3305af4026bdd4dc78642e1ac \
-    --hash=sha256:6e131579c243c98f35bce64a7e08e87fb2d610544754675d4a0e73a070a5aa3a \
-    --hash=sha256:855822d90f884905362f602880ed8b5df1b7e3ee7d0db2502d4388a954cc8c54 \
-    --hash=sha256:86a8b5035fce64f5dcd1b794cf8ec4d31fe458cf6ce3986a30deb434df82a1d2 \
-    --hash=sha256:8a33d657f3276328ce00e4d37fe70361e1ec7614da5d7b6e78de5426cb56332f \
-    --hash=sha256:92c0ec1f2cc149551a2b7b47efc32c866406b6891b0ee4625e95967c8f4acfb1 \
-    --hash=sha256:9a5e9f45e5d5e1c5b5c29b3bd4265dcc90e8b92cf4534520896ed77f791f4da5 \
-    --hash=sha256:afc622538b430aa4c8c853f7f63bc582b3b8030fd8c80b70fb5fa5b834e575c2 \
-    --hash=sha256:b07fc0dab849d24a80a29cfab8d8a19187d1c4685d8a5e6385a5ce323c1f015f \
-    --hash=sha256:b5e6f89631eb88a7302d416594a32faeee9fb8fb848290da9d0a5f2903519fc1 \
-    --hash=sha256:bf9bf162ed91a26f1adba8efda0b573bc6924ec1408a52cc6f82cb73ec2b142c \
-    --hash=sha256:c7e72339f841b5a237ff14f7d3880ddd0fc7f98a1199e8c4327f9a4f478c1839 \
-    --hash=sha256:ddb113db38838eb9f043623ba274cfaf7d51d5b0c22ecb30afe58b1bb8322983 \
-    --hash=sha256:dfdd51fc3e64ea4f35873d1b3fb25326773d55d2329ff8449139ebaad7357efb \
-    --hash=sha256:f1cd08e99d2f9317292a311dfe578fd2a24b15dbce97792f9c4d752275c1fa56 \
-    --hash=sha256:f89f2ab047c76a9c03f78d0d66ca519e389519902fa27e7a91117ef7611c0568
+black==26.10.1 \
+    --hash=sha256:03c0ddd93bb392e71209903a691767eb366fe1a76deb9509ccbaae9e1f14bb52 \
+    --hash=sha256:0ce08b367307b0fd91c9dd1d4084e62b05b3055475f951f0f34a46b6e2393b64 \
+    --hash=sha256:182f6c32be38074b16d378498c498b32cb51928178ee611485344972c35ec9c6 \
+    --hash=sha256:1935b32f5326028019856e18cb42b4da63db23765dc84464cec723e0de478a9b \
+    --hash=sha256:19fa8f5beb5e77c54c9c7e21d00cc93ed6c8b6228ee385616906d6befe081143 \
+    --hash=sha256:2520037aa62f8a1454d0811b8f5c88b444445b03a4bfba480d8d220893b64c34 \
+    --hash=sha256:28842f9a8207cc1df6eb983a35a14c5a0dfcd603d214fe82d84bef552afd2e3a \
+    --hash=sha256:289282aa2e09d3162312a3be1788ff21b08e9ea9cc4a81e656024728b32428fb \
+    --hash=sha256:2ffbc023a12d0c729408823b8f10514490bd0baa301d0d4e21a7240249f9507f \
+    --hash=sha256:3414a0c52901964dceabd98c7c56beac0f964115a116ecedcce7247359b14017 \
+    --hash=sha256:4d9a90516db1d99c25dbb20cc0998e0e01531dd903466c7744e56d66f864220a \
+    --hash=sha256:51d5e417e700fe6ec0b0ecdc408c6f6cb5def80328f31f724993d82c6486b746 \
+    --hash=sha256:5cd88fd7b444ca51f3fc883b6f6657ea53a258b0b2eef6d9f2dfcfa17ce0e27b \
+    --hash=sha256:5f9f83beae62437e060dafd53d7f1fc327e3d3494f74d72ee5c2b73eb90fc4e7 \
+    --hash=sha256:70ccbd175b7f6be29d2b727ee7ca6b4c54053df59da653a6df80b175d20a94fa \
+    --hash=sha256:7bdade400bfe24d78a7762896acc2f9a8e1a17fb0fd0536bf6b7c7097cf3eec7 \
+    --hash=sha256:8375962579d537364cc0efa19b1474481915d3a793f9fc0774901814c5e5b5f4 \
+    --hash=sha256:978113a40223a6aaefc17364176a809a320e6b288683841427fff04c6d7b4130 \
+    --hash=sha256:9a0219b29cd70e49f920acb7081e6ce5025c719008447c521d0200dcad93206a \
+    --hash=sha256:b5347d760f0c02bb00dd249384cab71c3bf828b4f68d5b401eb116e0390f147d \
+    --hash=sha256:b6272cfd7e1e8e271f5b0e0207259fe2834687e5cb9b5f620b34a44db9754993 \
+    --hash=sha256:d42dd2fac7c342ae67e64ee99c9532e20b2a84e92c79ed3317fa2ef54c801d93 \
+    --hash=sha256:d5bd3518d8e97138fef295230b1e9804076d69fa4e3594071494a8c68abe6266 \
+    --hash=sha256:d8b3a9074a680b3c5749633714e9ae3992a1e5a23343a97ad61cd9b119b444d2 \
+    --hash=sha256:f6dba8138cdc99061ef07b958ac082d2aa057b6961d1936f9717c350f02bab5f \
+    --hash=sha256:fe85fc4019bee59bc495c0f2a8ee76c5cd02c7015508d94a967ba2376f39a52c \
+    --hash=sha256:ff57f63029aa1353fa8b1b0c8971fd88a6c92dc766608d2eee33ad2deb23270e
     # via
     #   -r requirements_formatting.txt.in
     #   darker
-certifi==2025.7.14 \
-    --hash=sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2 \
-    --hash=sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995
+certifi==2026.10.19 \
+    --hash=sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae \
+    --hash=sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b
     # via
     #   -r requirements_formatting.txt.in
     #   requests
@@ -308,9 +308,9 @@ pygithub==2.6.1 \
     --hash=sha256:6f2fa6d076ccae475f9fc392cc6cdbd54db985d4f69b8833a28397de75ed6ca3 \
     --hash=sha256:b5c035392991cca63959e9453286b41b54d83bf2de2daa7d7ff7e4312cebf3bf
     # via -r requirements_formatting.txt.in
-pyjwt==2.13.0 \
-    --hash=sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423 \
-    --hash=sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728
+pyjwt==2.15.1 \
+    --hash=sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193 \
+    --hash=sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8
     # via
     #   -r requirements_formatting.txt.in
     #   pygithub
@@ -403,9 +403,9 @@ typing-extensions==4.14.1 \
     --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \
     --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76
     # via pygithub
-urllib3==2.7.0 \
-    --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \
-    --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897
+urllib3==2.8.0 \
+    --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \
+    --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63
     # via
     #   -r requirements_formatting.txt.in
     #   pygithub

External/code-format-helper/requirements_formatting.txt.in

@@ -1,10 +1,10 @@
-black>=26.3.1
+black>=26.10.1
 darker==2.1.1
 PyGithub==2.6.1
 cryptography>=50.0.0
-urllib3>=2.7.0
+urllib3>=2.8.0
 requests>=2.33.0
 idna>=3.15
-certifi>=2024.7.4
+certifi>=2026.10.19
 PyNaCl>=1.6.2
-PyJWT>=2.13.0
+PyJWT>=2.15.1

FEXCore/CMakeLists.txt

@@ -33,9 +33,11 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/git_version.h.in
 include_directories(${CMAKE_BINARY_DIR}/generated)
 
 # Disable strict aliasing for all build modes
-# See discussion in https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
+# See discussion in
+# https://github.com/FEX-Emu/FEX/pull/4494#issuecomment-2800608944
 # for background context.
-add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fno-strict-aliasing> $<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>)
+add_compile_options($<$<COMPILE_LANGUAGE:CXX>:-fno-strict-aliasing>
+  $<$<COMPILE_LANGUAGE:CXX>:-fno-exceptions>)
 
 add_subdirectory(Source/)
 

FEXCore/Readme.md

@@ -8,8 +8,8 @@ This project aims to provide a fast and functional x86-64 emulation library that
   * Support a tiered recompiler to allow for fast runtime performance
   * Support offline compilation and offline tooling for inspection and performance analysis
   * Support threaded emulation. Including emulating x86-64's strong memory model on weak memory model architectures
-* Support a significant portion of the x86-64 instruction space.
-  * Including MMX, SSE, SSE2, SSE3, SSSE3, and SSE4*
+* Support a majority of the x86-64 instruction space.
+  * Including MMX, SSE, SSE2, SSE3, SSSE3, SSE4*, AVX, AVX2, F16C, and AVX-VNNI
 * Support fallback routines for uncommonly used x86-64 instructions
   * Including x87 and 3DNow!
 * Only support userspace emulation.

FEXCore/Source/CMakeLists.txt

@@ -4,6 +4,7 @@ set(FEXCORE_BASE_SRCS
   Interface/Config/Config.cpp
   Utils/Allocator.cpp
   Utils/FileLoading.cpp
+  Utils/FileUtils.cpp
   Utils/ForcedAssert.cpp
   Utils/LogManager.cpp
   Utils/SpinWaitLock.cpp
@@ -103,9 +104,13 @@ if (ENABLE_ZYDIS)
 endif()
 
 if (ARCHITECTURE_arm64 AND HAS_CLANG_PRESERVE_ALL)
-  list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all));-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
+  list(APPEND DEFINES
+    "-DFEXCORE_PRESERVE_ALL_ATTR=__attribute__((preserve_all))"
+    "-DFEXCORE_HAS_PRESERVE_ALL_ATTR=1")
 else()
-  list(APPEND DEFINES "-DFEXCORE_PRESERVE_ALL_ATTR=;-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0")
+  list(APPEND DEFINES
+    "-DFEXCORE_PRESERVE_ALL_ATTR="
+    "-DFEXCORE_HAS_PRESERVE_ALL_ATTR=0")
 endif()
 
 set(LIBS fmt::fmt xxHash::xxhash FEXHeaderUtils CodeEmitter cephes_128bit)
@@ -186,8 +191,8 @@ add_custom_command(
   OUTPUT "${OUTPUT_MAN_NAME}"
   DEPENDS "${INPUT_CONFIG_NAME}"
   DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py"
-  COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py" "${INPUT_CONFIG_NAME}" "${OUTPUT_CONFIG_NAME}" "${OUTPUT_MAN_NAME}"
-  "${OUTPUT_CONFIG_OPTION_NAME}")
+  COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py"
+    "${INPUT_CONFIG_NAME}" "${OUTPUT_CONFIG_NAME}" "${OUTPUT_MAN_NAME}" "${OUTPUT_CONFIG_OPTION_NAME}")
 
 add_custom_command(
   OUTPUT "${OUTPUT_MAN_NAME_COMPRESS}"
@@ -261,6 +266,10 @@ endfunction()
 # Build FEXCore_Base static library
 add_library(FEXCore_Base STATIC ${FEXCORE_BASE_SRCS})
 target_link_libraries(FEXCore_Base PUBLIC ${LIBS})
+if (MINGW)
+  target_link_libraries(FEXCore_Base PUBLIC ntdll)
+endif()
+
 AddDefaultOptionsToTarget(FEXCore_Base)
 
 if (ENABLE_FEXCORE_PROFILER AND FEXCORE_PROFILER_BACKEND STREQUAL "TRACY")
@@ -279,8 +288,8 @@ function(AddLibrary Name Type)
   add_library(${Name} ${Type} $<TARGET_OBJECTS:${PROJECT_NAME}_object>)
   set_target_properties(${Name} PROPERTIES OUTPUT_NAME FEXCore)
 
-  # During generation of the import library (dll.a), MinGW needs some extra symbols from libraries
-  # such as fmt, which are propagated by FEXCore_Base. Wonderful.
+  # During generation of the import library (dll.a), MinGW needs some extra
+  # symbols from libraries such as fmt, which are propagated by FEXCore_Base
   if (MINGW)
     target_link_libraries(${Name} PRIVATE FEXCore_Base)
   endif()
@@ -306,7 +315,8 @@ if (ENABLE_FEX_ALLOCATOR)
   target_include_directories(JemallocLibs PRIVATE "${PROJECT_SOURCE_DIR}/include/")
 endif()
 if (ENABLE_JEMALLOC_GLIBC_ALLOC)
-  set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1)
+  set_source_files_properties(Interface/HLE/Thunks/Thunks.cpp
+    PROPERTIES COMPILE_DEFINITIONS ENABLE_JEMALLOC_GLIBC=1)
   target_link_libraries(JemallocLibs INTERFACE FEX_jemalloc_glibc)
 endif()
 

FEXCore/Source/Common/JitSymbols.cpp

@@ -15,14 +15,9 @@ JITSymbols::~JITSymbols() {
   }
 }
 
-void JITSymbols::InitFile() {
+void JITSymbols::InitFile(uint32_t ProcessPID) {
   // We can't use FILE here since we must be robust against forking processes closing our FD from under us.
-#ifdef __ANDROID__
-  // Android simpleperf looks in /data/local/tmp instead of /tmp
-  const auto PerfMap = fextl::fmt::format("/data/local/tmp/perf-{}.map", getpid());
-#else
-  const auto PerfMap = fextl::fmt::format("/tmp/perf-{}.map", getpid());
-#endif
+  const auto PerfMap = fextl::fmt::format("/tmp/perf-{}.map", ProcessPID);
   fd = open(PerfMap.c_str(), O_CREAT | O_TRUNC | O_WRONLY | O_APPEND, 0644);
 }
 

FEXCore/Source/Common/JitSymbols.h

@@ -35,7 +35,7 @@ public:
   JITSymbols();
   ~JITSymbols();
 
-  void InitFile();
+  void InitFile(uint32_t ProcessPID);
   void RegisterNamedRegion(const void* HostAddr, uint32_t CodeSize, std::string_view Name);
   void RegisterJITSpace(const void* HostAddr, uint32_t CodeSize);
 

FEXCore/Source/Interface/Config/Config.json.in

@@ -42,6 +42,19 @@
           "Enable expensive validation when loading code caches"
         ]
       },
+      "JITFeatures": {
+        "Type": "strenum",
+        "Default": "FEXCore::Config::JITFeatures::OFF",
+        "AffectsCodeGen": "true",
+        "Enums": {
+          "ENABLEX87FIP": "enablex87fip"
+        },
+        "Desc": [
+          "Allows controlling JIT features in the unified location",
+          "\toff: Default JIT features.",
+          "\tenablex87fip: Turns on the x87 FIP emulation feature"
+        ]
+      },
       "HostFeatures": {
         "Type": "strenum",
         "Default": "FEXCore::Config::HostFeatures::OFF",
@@ -91,7 +104,15 @@
           "ENABLESSE4A": "enablesse4a",
           "DISABLESSE4A": "disablesse4a",
           "ENABLEMOPS": "enablemops",
-          "DISABLEMOPS": "disablemops"
+          "DISABLEMOPS": "disablemops",
+          "ENABLEI8MM": "enablei8mm",
+          "DISABLEI8MM": "disablei8mm",
+          "ENABLEDOTPROD": "enabledotprod",
+          "DISABLEDOTPROD": "disabledotprod",
+          "ENABLEBF16": "enablebf16",
+          "DISABLEBF16": "disablebf16",
+          "ENABLETPIDRRO": "enabletpidrro",
+          "DISABLETPIDRRO": "disabletpidrro"
         },
         "Desc": [
           "Allows controlling of the CPU features in the JIT.",
@@ -116,7 +137,11 @@
           "\t{enable,disable}wfxt: Will force enable or disable wfxt even if the host doesn't support it",
           "\t{enable,disable}3dnow: Will force enable or disable 3DNow! even if the host doesn't support it",
           "\t{enable,disable}sse4a: Will force enable or disable SSE4a even if the host doesn't support it",
-          "\t{enable,disable}mops: Will force enable or disable FEAT_MOPS even if the host doesn't support it"
+          "\t{enable,disable}mops: Will force enable or disable FEAT_MOPS even if the host doesn't support it",
+          "\t{enable,disable}i8mm: Will force enable or disable i8mm even if the host doesn't support it",
+          "\t{enable,disable}dotprod: Will force enable or disable dotprod even if the host doesn't support it",
+          "\t{enable,disable}bf16: Will force enable or disable bf16 even if the host doesn't support it",
+          "\t{enable,disable}tpidrro: Will force enable or disable tpidrro for CPU index even if the host doesn't support it"
         ]
       },
       "SmallTSCScale": {
@@ -135,6 +160,14 @@
           "Hides hybrid CPU core arrangement."
         ]
       },
+      "SoftwareRNG": {
+        "Type": "bool",
+        "Default": "false",
+        "AffectsCodeGen": "true",
+        "Desc": [
+          "Emulates RDRAND and RDSEED in software when the host does not implement FEAT_RNG."
+        ]
+      },
       "CPUFeatureRegisters": {
         "Type": "str",
         "Default": "",
@@ -184,6 +217,14 @@
           "Attempt to cache anonymous code"
         ]
       },
+      "DiskCacheMemorySize": {
+        "Type": "uint32",
+        "Default": "0",
+        "AffectsCodeGen": "false",
+        "Desc": [
+          "How much memory, if any, to use as a cache for the cache"
+        ]
+      },
       "DiskCachePath": {
         "Type": "str",
         "Default": "",
@@ -199,6 +240,23 @@
         "Desc": [
           "Optional list of extra read-only disk cache DBs to consider"
         ]
+      },
+      "DiskCacheMaxFileSize": {
+        "Type": "uint64",
+        "Default": "1073741824",
+        "AffectsCodeGen": "false",
+        "Desc": [
+          "Size limit on the main cache file - index not included. Default 1G"
+        ]
+      },
+      "DiskCachePruneStaleEntries": {
+        "Type": "bool",
+        "Default": "true",
+        "AffectsCodeGen": "false",
+        "Desc": [
+          "Allows FEX to prune stale disk cache entries on startup.",
+          "Frees space by removing cache entries associated with old FEX versions."
+        ]
       }
     },
     "Emulation": {
@@ -315,6 +373,15 @@
           "Can potentially introduce more stutters, more likely the higher the number.",
           "Don't have this number larger than the increase count!"
         ]
+      },
+      "UndefinedFlagAcidTest": {
+        "Type": "bool",
+        "Default": "false",
+        "AffectsCodeGen": "true",
+        "Desc": [
+          "Ensures that all undefined flags generate consistent results",
+          "This is a useful test to determine if undefined flags are causing problems"
+        ]
       }
     },
     "Debug": {

FEXCore/Source/Interface/Context/Context.h

@@ -4,13 +4,13 @@
 #include "Common/JitSymbols.h"
 #include "Interface/Core/CPUBackend.h"
 #include "Interface/Core/CPUID.h"
+#include "Interface/Core/DiskCache.h"
 #include "Interface/Core/SharedCodeBufferManager.h"
 #include <Interface/IR/IntrusiveIRList.h>
 #include <FEXCore/Config/Config.h>
 #include <FEXCore/Core/Context.h>
 #include <FEXCore/Core/CoreState.h>
 #include <FEXCore/Core/HostFeatures.h>
-#include <FEXCore/Core/DiskCache.h>
 #include <FEXCore/IR/IR.h>
 #include <FEXCore/Utils/CompilerDefs.h>
 #include <FEXCore/Utils/SignalScopeGuards.h>
@@ -83,7 +83,6 @@ public:
   FEX_CONFIG_OPT(EnableLazyCodeCaching, ENABLELAZYCODECACHINGWIP);
   FEX_CONFIG_OPT(EnableCodeCacheValidation, ENABLECODECACHEVALIDATION);
 
-  uint64_t ComputeCodeMapId(std::string_view Filename, int FD) override;
   bool SaveData(Core::InternalThreadState&, int TargetFD, const ExecutableFileSectionInfo&, uint64_t SerializedBaseAddress) override;
 
   fextl::unique_ptr<MappedCodeCacheFile> LoadCache(std::span<std::byte> CacheFile, const ExecutableFileInfo&, uint64_t FileStartVA) override;
@@ -369,10 +368,17 @@ public:
     FEX_CONFIG_OPT(DisableTelemetry, DISABLETELEMETRY);
     FEX_CONFIG_OPT(DisableVixlIndirectCalls, DISABLE_VIXL_INDIRECT_RUNTIME_CALLS);
     FEX_CONFIG_OPT(SmallTSCScale, SMALLTSCSCALE);
+    FEX_CONFIG_OPT(SoftwareRNG, SOFTWARERNG);
     FEX_CONFIG_OPT(StrictInProcessSplitLocks, STRICTINPROCESSSPLITLOCKS);
     FEX_CONFIG_OPT(MonoHacks, MONOHACKS);
+    FEX_CONFIG_OPT(JITFeatures, JITFEATURES);
   } Config;
 
+  bool SoftwareRNGEnabled() const {
+    return Config.SoftwareRNG() && HostRNGAvailable;
+  }
+  bool HostRNGAvailable {};
+
   FEXCore::Utils::WritePriorityMutex::Mutex CodeInvalidationMutex {};
 
   uint32_t StrictSplitLockMutex {};

FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.cpp

@@ -665,11 +665,11 @@ void Arm64Emitter::FillSpecialRegs(ARMEmitter::Register TmpReg, ARMEmitter::Regi
     // We don't bother spilling these in SpillStaticRegs,
     // since all that matters is we restore them on a fill.
     // It's not a concern if they get trounced by something else.
-    if (EmitterCTX->HostFeatures.SupportsSVE256) {
+    if (EmitterCTX->HostFeatures.SupportsSVE256()) {
       ptrue(ARMEmitter::SubRegSize::i8Bit, PRED_TMP_32B, ARMEmitter::PredicatePattern::SVE_VL32);
     }
 
-    if (EmitterCTX->HostFeatures.SupportsSVE128) {
+    if (EmitterCTX->HostFeatures.SupportsSVE128()) {
       ptrue(ARMEmitter::SubRegSize::i8Bit, PRED_TMP_16B, ARMEmitter::PredicatePattern::SVE_VL16);
     }
 
@@ -739,7 +739,7 @@ void Arm64Emitter::SpillStaticRegs(ARMEmitter::Register TmpReg, SpillStaticRegOp
   }
 
   if (Options.FPRs) {
-    if (EmitterCTX->HostFeatures.SupportsAVX && EmitterCTX->HostFeatures.SupportsSVE256) {
+    if (EmitterCTX->HostFeatures.SupportsAVX() && EmitterCTX->HostFeatures.SupportsSVE256()) {
       for (size_t i = 0; i < StaticFPRegisters.size(); i++) {
         const auto Reg = StaticFPRegisters[i];
 
@@ -826,7 +826,7 @@ void Arm64Emitter::FillStaticRegs(FillStaticRegOptions Options) {
   FillSpecialRegs(TmpReg, TmpReg2, {.SetFIZ = true, .SetPredRegs = Options.FPRs});
 
   if (Options.FPRs) {
-    if (EmitterCTX->HostFeatures.SupportsAVX && EmitterCTX->HostFeatures.SupportsSVE256) {
+    if (EmitterCTX->HostFeatures.SupportsAVX() && EmitterCTX->HostFeatures.SupportsSVE256()) {
       for (size_t i = 0; i < StaticFPRegisters.size(); i++) {
         const auto Reg = StaticFPRegisters[i];
         if (((1U << Reg.Idx()) & Options.FPRFillMask) != 0) {
@@ -990,7 +990,7 @@ void Arm64Emitter::PopGeneralRegisters(std::span<const ARMEmitter::Register> Reg
 }
 
 size_t Arm64Emitter::PushDynamicRegs(ARMEmitter::Register TmpReg) {
-  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
+  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
   const auto GPRSize = GeneralRegistersNotPreserved.size() * Core::CPUState::GPR_REG_SIZE;
   const auto FPRRegSize = CanUseSVE256 ? 32 : 16;
   const auto FPRSize = GeneralFPRegisters.size() * FPRRegSize;
@@ -1013,7 +1013,7 @@ size_t Arm64Emitter::PushDynamicRegs(ARMEmitter::Register TmpReg) {
 }
 
 void Arm64Emitter::PopDynamicRegs() {
-  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
+  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
 
   // Pop vectors first
   PopVectorRegisters(CanUseSVE256, GeneralFPRegisters);
@@ -1023,7 +1023,7 @@ void Arm64Emitter::PopDynamicRegs() {
 }
 
 size_t Arm64Emitter::SpillForPreserveAllABICall(ARMEmitter::Register TmpReg, bool FPRs) {
-  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
+  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
   const auto FPRRegSize = CanUseSVE256 ? 32 : 16;
 
   std::span<const ARMEmitter::Register> DynamicGPRs {};
@@ -1078,7 +1078,7 @@ size_t Arm64Emitter::SpillForPreserveAllABICall(ARMEmitter::Register TmpReg, boo
 }
 
 void Arm64Emitter::FillForPreserveAllABICall(bool FPRs) {
-  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256;
+  const auto CanUseSVE256 = EmitterCTX->HostFeatures.SupportsSVE256();
 
   std::span<const ARMEmitter::Register> DynamicGPRs {};
   std::span<const ARMEmitter::VRegister> DynamicFPRs {};
@@ -1129,4 +1129,9 @@ void Arm64Emitter::Align16B() {
   }
 }
 
+ARMEmitter::Register Arm64Emitter::GetX86GPRMapping(FEXCore::X86State::X86Reg Reg) {
+  LOGMAN_THROW_A_FMT(Reg <= FEXCore::X86State::X86Reg::REG_R15, "Incorrect register");
+  return StaticRegisters[Reg];
+}
+
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/ArchHelpers/Arm64Emitter.h

@@ -235,6 +235,8 @@ protected:
 
   void Align16B();
 
+  ARMEmitter::Register GetX86GPRMapping(FEXCore::X86State::X86Reg Reg);
+
 #ifdef VIXL_SIMULATOR
   // Generates a vixl simulator runtime call.
   //

FEXCore/Source/Interface/Core/CPUBackend.cpp

@@ -16,6 +16,7 @@ namespace CPU {
   constexpr static uint64_t NamedVectorConstants[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX][2] = {
     {0x0003'0002'0001'0000ULL, 0x0007'0006'0005'0004ULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX
     {0x000B'000A'0009'0008ULL, 0x000F'000E'000D'000CULL}, // NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER
+    {0x0706'0504'0302'0100ULL, 0x0F0E'0D0C'0B0A'0908ULL}, // NAMED_VECTOR_INCREMENTAL_U8_INDEX
     {0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT
     {0x0000'0000'8000'0000ULL, 0x0000'0000'8000'0000ULL}, // NAMED_VECTOR_PADDSUBPS_INVERT_UPPER
     {0x8000'0000'0000'0000ULL, 0x0000'0000'0000'0000ULL}, // NAMED_VECTOR_PADDSUBPD_INVERT

FEXCore/Source/Interface/Core/CPUBackend.h

@@ -115,7 +115,7 @@ namespace CPU {
      * @return Information about the compiled code block.
      */
     [[nodiscard]]
-    virtual CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
+    virtual CompiledCode EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                      FEXCore::Core::DebugData* DebugData, bool CheckTF) = 0;
 
     virtual CompiledCode LoadCachedCode(std::span<const uint8_t> HostBytes) {

FEXCore/Source/Interface/Core/CPUID.cpp

@@ -89,6 +89,18 @@ namespace ProductNames {
   static const char ARM_Blizzard_M2Pro[] = "Apple Blizzard (M2 Pro)";
   static const char ARM_Avalanche_M2Max[] = "Apple Avalanche (M2 Max)";
   static const char ARM_Blizzard_M2Max[] = "Apple Blizzard (M2 Max)";
+  static const char ARM_Sawtooth_M3[] = "Apple Sawtooth (M3)";
+  static const char ARM_Everest_M3[] = "Apple Everest (M3)";
+  static const char ARM_Sawtooth_M3Pro[] = "Apple Sawtooth (M3 Pro)";
+  static const char ARM_Everest_M3Pro[] = "Apple Everest (M3 Pro)";
+  static const char ARM_Sawtooth_M3Max[] = "Apple Sawtooth (M3 Max)";
+  static const char ARM_Everest_M3Max[] = "Apple Everest (M3 Max)";
+  static const char ARM_DonanP_M4[] = "Apple DonanP (M4)";
+  static const char ARM_DonanE_M4[] = "Apple DonanE (M4)";
+  static const char ARM_TahitiP_A18[] = "Apple TahitiP (A18)";
+  static const char ARM_TahitiE_A18[] = "Apple TahitiE (A18)";
+  static const char ARM_SotraP_M5[] = "Apple SotraP (M5)";
+  static const char ARM_SotraE_M5[] = "Apple SotraE (M5)";
   static const char ARM_AppleSilicon[] = "Apple Silicon";
 
   static const char ARM_ORYON_1[] = "Oryon-1";
@@ -187,11 +199,17 @@ void CPUIDEmu::SetupHostHybridFlag() {
   // CPU priority order
   // This is mostly arbitrary but will sort by some sort of CPU priority by performance
   // Relative list so things they will commonly end up in big.little configurations sort of relate
-  static constexpr std::array<CPUMIDR, 68> CPUMIDRs = {{
+  static constexpr std::array<CPUMIDR, 80> CPUMIDRs = {{
     // Typically big CPU cores
     {0x51, 0x002, 1, ProductNames::ARM_ORYON_3}, // Qualcomm Oryon-3
     {0x51, 0x001, 1, ProductNames::ARM_ORYON_1}, // Qualcomm Oryon-1
 
+    {0x61, 0x065, 1, ProductNames::ARM_SotraP_M5},       // Apple SotraP (M5)
+    {0x61, 0x061, 1, ProductNames::ARM_TahitiP_A18},     // Apple TahitiP (A18)
+    {0x61, 0x053, 1, ProductNames::ARM_DonanP_M4},       // Apple DonanP (M4)
+    {0x61, 0x048, 1, ProductNames::ARM_Sawtooth_M3Max},  // Apple Sawtooth (M3 Max)
+    {0x61, 0x044, 1, ProductNames::ARM_Sawtooth_M3Pro},  // Apple Sawtooth (M3 Pro)
+    {0x61, 0x042, 1, ProductNames::ARM_Sawtooth_M3},     // Apple Sawtooth (M3)
     {0x61, 0x039, 1, ProductNames::ARM_Avalanche_M2Max}, // Apple Avalanche (M2 Max)
     {0x61, 0x035, 1, ProductNames::ARM_Avalanche_M2Pro}, // Apple Avalanche (M2 Pro)
     {0x61, 0x033, 1, ProductNames::ARM_Avalanche_M2},    // Apple Avalanche (M2)
@@ -248,6 +266,12 @@ void CPUIDEmu::SetupHostHybridFlag() {
     {0x41, 0xd07, 1, ProductNames::ARM_A57}, // A57
 
     // Typically Little CPU cores
+    {0x61, 0x064, 1, ProductNames::ARM_SotraE_M5},      // Apple SotraE (M5)
+    {0x61, 0x060, 1, ProductNames::ARM_TahitiE_A18},    // Apple TahitiE (A18)
+    {0x61, 0x052, 1, ProductNames::ARM_DonanE_M4},      // Apple DonanE (M4)
+    {0x61, 0x049, 0, ProductNames::ARM_Everest_M3Max},  // Apple Everest (M3 Max)
+    {0x61, 0x045, 0, ProductNames::ARM_Everest_M3Pro},  // Apple Everest (M3 Pro)
+    {0x61, 0x043, 0, ProductNames::ARM_Everest_M3},     // Apple Everest (M3)
     {0x61, 0x038, 0, ProductNames::ARM_Blizzard_M2Max}, // Apple Blizzard (M2 Max)
     {0x61, 0x034, 0, ProductNames::ARM_Blizzard_M2Pro}, // Apple Blizzard (M2 Pro)
     {0x61, 0x032, 0, ProductNames::ARM_Blizzard_M2},    // Apple Blizzard (M2)
@@ -419,7 +443,7 @@ void CPUIDEmu::SetupHostHybridFlag() {}
 
 
 void CPUIDEmu::SetupFeatures() {
-  if (CTX->HostFeatures.SupportsAVX) {
+  if (CTX->HostFeatures.SupportsAVX()) {
     XCR0 |= XCR0_AVX;
   }
 
@@ -458,6 +482,8 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) const {
             (Cores << 16) |     // Number of addressable IDs for the logical cores in the physical CPU
             (GetCPUID() << 24); // Local APIC ID
 
+  const uint32_t SupportsRAND = CTX->HostFeatures.SupportsRAND || CTX->SoftwareRNGEnabled();
+
   Res.ecx = (1 << 0) |                                      // SSE3
             (CTX->HostFeatures.SupportsPMULL_128Bit << 1) | // PCLMULQDQ
             (1 << 2) |                                      // DS area supports 64bit layout
@@ -488,7 +514,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) const {
             (SupportsAVX() << 27) |                         // OSXSAVE
             (SupportsAVX() << 28) |                         // AVX
             (SupportsAVX() << 29) |                         // F16C
-            (CTX->HostFeatures.SupportsRAND << 30) |        // RDRAND
+            (SupportsRAND << 30) |                          // RDRAND
             (Hypervisor << 31);
 
   Res.edx = (1 << 0) |  // FPU
@@ -648,6 +674,12 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_06h(uint32_t Leaf) const {
 
 FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
   FEXCore::CPUID::FunctionResults Res {};
+
+  // AVX-VNNI is only advertised when the CPU supports I8MM or Dot Product.
+  // Without these features the implementation is so slow that it is likely
+  // to harm performance.
+  const uint32_t SupportsAVXVNNI = SupportsAVX() && (CTX->HostFeatures.SupportsI8MM || CTX->HostFeatures.SupportsDotProd);
+
   if (Leaf == 0) {
 #ifndef _WIN32
     constexpr uint32_t SUPPORTS_RDPID = 1;
@@ -664,40 +696,47 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
     const uint32_t SupportsVPCLMULQDQ = CTX->HostFeatures.SupportsPMULL_128Bit && SupportsAVX();
     const uint32_t SupportsWFXT = CTX->HostFeatures.SupportsWFXT;
 
+    const uint32_t SupportsRAND = CTX->HostFeatures.SupportsRAND || CTX->SoftwareRNGEnabled();
+    // When the X87 FIP feature is enabled, we update the state pointers on every instruction.
+    // We don't even update them on exception so this isn't quite true the other way anyway.
+    const uint32_t X87DataPtrExcept = (CTX->Config.JITFeatures & FEXCore::Config::JITFeatures::ENABLEX87FIP) == 0;
+
     // Number of subfunctions
-    Res.eax = 0x0;
-    Res.ebx = (1 << 0) |                               // FS/GS support
-              (0 << 1) |                               // TSC adjust MSR
-              (0 << 2) |                               // SGX
-              (SupportsAVX() << 3) |                   // BMI1
-              (0 << 4) |                               // Intel Hardware Lock Elison
-              (SupportsAVX() << 5) |                   // AVX2 support
-              (1 << 6) |                               // FPU data pointer updated only on exception
-              (1 << 7) |                               // SMEP support
-              (SupportsAVX() << 8) |                   // BMI2
-              (SupportsEnhancedREPMOVS << 9) |         // Enhanced REP MOVSB/STOSB
-              (1 << 10) |                              // INVPCID for system software control of process-context
-              (0 << 11) |                              // Restricted transactional memory
-              (0 << 12) |                              // Intel resource directory technology Monitoring
-              (1 << 13) |                              // Deprecates FPU CS and DS
-              (0 << 14) |                              // Intel MPX
-              (0 << 15) |                              // Intel Resource Directory Technology Allocation
-              (0 << 16) |                              // AVX512-F
-              (0 << 17) |                              // AVX512-DQ
-              (CTX->HostFeatures.SupportsRAND << 18) | // RDSEED
-              (1 << 19) |                              // ADCX and ADOX instructions
-              (0 << 20) |                              // SMAP Supervisor mode access prevention and CLAC/STAC instructions
-              (0 << 21) |                              // AVX512-IFMA
-              (0 << 22) |                              // PCOMMIT (deprecated?)
-              (1 << 23) |                              // CLFLUSHOPT instruction
-              (1 << 24) |                              // CLWB instruction
-              (0 << 25) |                              // Intel processor trace
-              (0 << 26) |                              // AVX512-PF
-              (0 << 27) |                              // AVX512-ER
-              (0 << 28) |                              // AVX512-CD
-              (Features.SHA << 29) |                   // SHA instructions
-              (0 << 30) |                              // AVX512-BW
-              (0 << 31);                               // AVX512-VL
+    // TODO: For now, subfunction 1 only exposes AVX-VNNI so we make it conditional
+    // on AVX-VNNI support. We should revisit this if/when we add more to this leaf.
+    Res.eax = SupportsAVXVNNI;
+    Res.ebx = (1 << 0) |                       // FS/GS support
+              (0 << 1) |                       // TSC adjust MSR
+              (0 << 2) |                       // SGX
+              (SupportsAVX() << 3) |           // BMI1
+              (0 << 4) |                       // Intel Hardware Lock Elison
+              (SupportsAVX() << 5) |           // AVX2 support
+              (X87DataPtrExcept << 6) |        // FPU data pointer updated only on exception
+              (1 << 7) |                       // SMEP support
+              (SupportsAVX() << 8) |           // BMI2
+              (SupportsEnhancedREPMOVS << 9) | // Enhanced REP MOVSB/STOSB
+              (1 << 10) |                      // INVPCID for system software control of process-context
+              (0 << 11) |                      // Restricted transactional memory
+              (0 << 12) |                      // Intel resource directory technology Monitoring
+              (1 << 13) |                      // Deprecates FPU CS and DS
+              (0 << 14) |                      // Intel MPX
+              (0 << 15) |                      // Intel Resource Directory Technology Allocation
+              (0 << 16) |                      // AVX512-F
+              (0 << 17) |                      // AVX512-DQ
+              (SupportsRAND << 18) |           // RDSEED
+              (1 << 19) |                      // ADCX and ADOX instructions
+              (0 << 20) |                      // SMAP Supervisor mode access prevention and CLAC/STAC instructions
+              (0 << 21) |                      // AVX512-IFMA
+              (0 << 22) |                      // PCOMMIT (deprecated?)
+              (1 << 23) |                      // CLFLUSHOPT instruction
+              (1 << 24) |                      // CLWB instruction
+              (0 << 25) |                      // Intel processor trace
+              (0 << 26) |                      // AVX512-PF
+              (0 << 27) |                      // AVX512-ER
+              (0 << 28) |                      // AVX512-CD
+              (Features.SHA << 29) |           // SHA instructions
+              (0 << 30) |                      // AVX512-BW
+              (0 << 31);                       // AVX512-VL
 
     Res.ecx = (1 << 0) |                                // PREFETCHWT1
               (0 << 1) |                                // AVX512VBMI
@@ -746,7 +785,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 11) |                  // rtm-always-abort
               (0 << 12) |                  // Reserved
               (0 << 13) |                  // TSX Force Abort (TSX will force abort if attempted)
-              (0 << 14) |                  // SERIALIZE instruction
+              (1 << 14) |                  // SERIALIZE instruction
               ((Hybrid ? 1U : 0U) << 15) | // Hybrid
               (0 << 16) |                  // TSXLDTRK (TSX Suspend load address tracking) - Allows untracked memory loads inside TSX region
               (0 << 17) |                  // Reserved
@@ -765,38 +804,40 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0 << 30) |                  // Arch capabilities - MSR module specific
               (0 << 31);                   // SSBD - Speculative Store Bypass Disable
   } else if (Leaf == 1) {
-    Res.eax = (0U << 0) |  // SHA512
-              (0U << 1) |  // SM3
-              (0U << 2) |  // SM4
-              (0U << 3) |  // RAO_INT
-              (0U << 4) |  // AVX_VNNI
-              (0U << 5) |  // AVX512_BF16
-              (0U << 6) |  // LASS (Linear Address Space Separation)
-              (0U << 7) |  // CMPCCXADD
-              (0U << 8) |  // ARCH_PERFMON_EXT
-              (0U << 9) |  // Reserved
-              (0U << 10) | // FAST_REP_MOVSB
-              (0U << 11) | // FAST_REP_STOSB
-              (0U << 12) | // FAST_REP_CMPSB_SCASB
-              (0U << 13) | // Reserved
-              (0U << 14) | // Reserved
-              (0U << 15) | // Reserved
-              (0U << 16) | // Reserved
-              (0U << 17) | // FRED (Flexible Return and Event Delivery)
-              (0U << 18) | // LKGS (Load into Kernel GS Base)
-              (0U << 19) | // WRMSRNS
-              (0U << 20) | // NMI_SRC
-              (0U << 21) | // AMX_FP16
-              (0U << 22) | // HRESET
-              (0U << 23) | // AVX_IFMA
-              (0U << 24) | // Reserved
-              (0U << 25) | // Reserved
-              (0U << 26) | // LAM (Linear Address Masking)
-              (0U << 27) | // MSRLIST
-              (0U << 28) | // Reserved
-              (0U << 29) | // Reserved
-              (0U << 30) | // INVD_DISABLE_POST_BIOS_DONE
-              (0U << 31);  // MOVRS
+    const uint32_t SupportsAVXConvert = SupportsAVX() && CTX->HostFeatures.SupportsBF16;
+
+    Res.eax = (0U << 0) |              // SHA512
+              (0U << 1) |              // SM3
+              (0U << 2) |              // SM4
+              (0U << 3) |              // RAO_INT
+              (SupportsAVXVNNI << 4) | // AVX_VNNI
+              (0U << 5) |              // AVX512_BF16
+              (0U << 6) |              // LASS (Linear Address Space Separation)
+              (0U << 7) |              // CMPCCXADD
+              (0U << 8) |              // ARCH_PERFMON_EXT
+              (0U << 9) |              // Reserved
+              (0U << 10) |             // FAST_REP_MOVSB
+              (0U << 11) |             // FAST_REP_STOSB
+              (0U << 12) |             // FAST_REP_CMPSB_SCASB
+              (0U << 13) |             // Reserved
+              (0U << 14) |             // Reserved
+              (0U << 15) |             // Reserved
+              (0U << 16) |             // Reserved
+              (0U << 17) |             // FRED (Flexible Return and Event Delivery)
+              (0U << 18) |             // LKGS (Load into Kernel GS Base)
+              (0U << 19) |             // WRMSRNS
+              (0U << 20) |             // NMI_SRC
+              (0U << 21) |             // AMX_FP16
+              (0U << 22) |             // HRESET
+              (0U << 23) |             // AVX_IFMA
+              (0U << 24) |             // Reserved
+              (0U << 25) |             // Reserved
+              (0U << 26) |             // LAM (Linear Address Masking)
+              (0U << 27) |             // MSRLIST
+              (0U << 28) |             // Reserved
+              (0U << 29) |             // Reserved
+              (0U << 30) |             // INVD_DISABLE_POST_BIOS_DONE
+              (0U << 31);              // MOVRS
 
     // Bits 4-31 currently reserved.
     Res.ebx = (0U << 0) | // PPIN
@@ -813,31 +854,31 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) const {
               (0U << 5);  // MSR_IMM
 
     // Bits 25-31 also reserved.
-    Res.edx = (0U << 0) |  // Reserved
-              (0U << 1) |  // Reserved
-              (0U << 2) |  // Reserved
-              (0U << 3) |  // Reserved
-              (0U << 4) |  // AVX_VNNI_INT8
-              (0U << 5) |  // AVX_NE_CONVERT
-              (0U << 6) |  // Reserved
-              (0U << 7) |  // Reserved
-              (0U << 8) |  // AMX_COMPLEX
-              (0U << 9) |  // Reserved
-              (0U << 10) | // AVX_VNNI_INT16
-              (0U << 11) | // Reserved
-              (0U << 12) | // Reserved
-              (0U << 13) | // UTMR (User-timer events)
-              (0U << 14) | // PREFETCHI
-              (0U << 15) | // USER_MSR
-              (0U << 16) | // Reserved
-              (0U << 17) | // UIRET_UIF
-              (0U << 18) | // CET_SSS
-              (0U << 19) | // AVX10
-              (0U << 20) | // Reserved
-              (0U << 21) | // APX_F
-              (0U << 22) | // SEC-TEE_ATTESTATION
-              (0U << 23) | // MWAIT
-              (0U << 24);  // SLSM (Static LSM)
+    Res.edx = (0U << 0) |                 // Reserved
+              (0U << 1) |                 // Reserved
+              (0U << 2) |                 // Reserved
+              (0U << 3) |                 // Reserved
+              (0U << 4) |                 // AVX_VNNI_INT8
+              (SupportsAVXConvert << 5) | // AVX_NE_CONVERT
+              (0U << 6) |                 // Reserved
+              (0U << 7) |                 // Reserved
+              (0U << 8) |                 // AMX_COMPLEX
+              (0U << 9) |                 // Reserved
+              (0U << 10) |                // AVX_VNNI_INT16
+              (0U << 11) |                // Reserved
+              (0U << 12) |                // Reserved
+              (0U << 13) |                // UTMR (User-timer events)
+              (0U << 14) |                // PREFETCHI
+              (0U << 15) |                // USER_MSR
+              (0U << 16) |                // Reserved
+              (0U << 17) |                // UIRET_UIF
+              (0U << 18) |                // CET_SSS
+              (0U << 19) |                // AVX10
+              (0U << 20) |                // Reserved
+              (0U << 21) |                // APX_F
+              (0U << 22) |                // SEC-TEE_ATTESTATION
+              (0U << 23) |                // MWAIT
+              (0U << 24);                 // SLSM (Static LSM)
   } else if (Leaf == 2) {
     // All bits are reserved except for EDX
     Res.eax = 0;
@@ -1121,14 +1162,14 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf) con
 
 FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0002h(uint32_t Leaf, uint32_t CPU) const {
   FEXCore::CPUID::FunctionResults Res {};
-  auto& Data = PerCPUData[CPU];
+  auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
   memcpy(&Res, Data.ProductName, std::min(strlen(Data.ProductName), sizeof(FEXCore::CPUID::FunctionResults)));
   return Res;
 }
 
 FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf, uint32_t CPU) const {
   FEXCore::CPUID::FunctionResults Res {};
-  auto& Data = PerCPUData[CPU];
+  auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
   const auto RemainingStringSize = std::max<ssize_t>(0, strlen(Data.ProductName) - 16);
   memcpy(&Res, Data.ProductName + 16, std::min<size_t>(RemainingStringSize, sizeof(FEXCore::CPUID::FunctionResults)));
   return Res;
@@ -1136,7 +1177,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0003h(uint32_t Leaf, uin
 
 FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0004h(uint32_t Leaf, uint32_t CPU) const {
   FEXCore::CPUID::FunctionResults Res {};
-  auto& Data = PerCPUData[CPU];
+  auto& Data = PerCPUData[std::min<uint32_t>(CPU, PerCPUData.size() - 1)];
   const auto RemainingStringSize = std::max<ssize_t>(0, strlen(Data.ProductName) - 32);
   memcpy(&Res, Data.ProductName + 32, std::min<size_t>(RemainingStringSize, sizeof(FEXCore::CPUID::FunctionResults)));
   return Res;

FEXCore/Source/Interface/Core/CPUID.h

@@ -119,6 +119,7 @@ private:
   bool Hybrid {};
   uint32_t Cores {};
   FEX_CONFIG_OPT(HideHypervisorBit, HIDEHYPERVISORBIT);
+  FEX_CONFIG_OPT(JITFeatures, JITFEATURES);
 
   // XFEATURE_ENABLED_MASK
   // Mask that configures what features are enabled on the CPU.

FEXCore/Source/Interface/Core/CodeCache.cpp

@@ -1,4 +1,5 @@
 // SPDX-License-Identifier: MIT
+#include "Utils/crc32.h"
 #include "FEXCore/Utils/LogManager.h"
 #include "FEXCore/Utils/MathUtils.h"
 #include "FEXCore/Utils/TypeDefines.h"
@@ -272,16 +273,6 @@ CodeCache::CodeCache(ContextImpl& CTX_)
   : CTX(CTX_) {}
 CodeCache::~CodeCache() = default;
 
-uint64_t CodeCache::ComputeCodeMapId(std::string_view Filename, int FD) {
-  if (Filename.empty()) {
-    return 0xffff'ffff'ffff'ffff;
-  }
-
-  // For now, we just use the file path as an identifier.
-  // TODO: Ensure the hash is unique enough to distinguish executables while remaining independent of the installation location
-  return XXH3_64bits(Filename.data(), Filename.size());
-}
-
 struct CodeCacheHeader {
   std::array<char, 4> Magic = ExpectedMagic;
   // Version history:
@@ -538,30 +529,37 @@ ApplyRIPMoveRelocation(ContextImpl& CTX, uint64_t GuestRIP, uint8_t RegisterInde
   Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Pointer, CPU::Arm64Emitter::PadType::DOPAD);
 }
 
-static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
-  uint64_t Value = 0;
-  memcpy(&Value, reinterpret_cast<const void*>(SiteAddress), ValueSize);
-  Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Value, CPU::Arm64Emitter::PadType::DOPAD);
-}
-
-static inline int64_t ReadLiveGuestDisplacement(uint64_t SiteAddress, uint8_t ValueSize) {
+static inline int64_t ReadLiveGuestData(uint64_t SiteAddress, uint8_t ValueSize) {
   uint64_t Raw = 0;
   memcpy(&Raw, reinterpret_cast<const void*>(SiteAddress), ValueSize);
   // manual sign-extension from guest live bytes
-  // 1/2 sizes not permitted in DetectDataMasks currently
-  if (ValueSize == 4) {
+  if (ValueSize == 1) {
+    return (int8_t)Raw;
+  } else if (ValueSize == 2) {
+    return (int16_t)Raw;
+  } else if (ValueSize == 4) {
     return (int32_t)Raw;
   } else {
     return (int64_t)Raw;
   }
 }
 
+static inline void ApplyPatchableDataRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
+  Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), ReadLiveGuestData(SiteAddress, ValueSize),
+                       CPU::Arm64Emitter::PadType::DOPAD);
+}
+
 static inline void ApplyPatchableRIPLiteralRelocation(uint64_t SiteAddress, uint8_t ValueSize, CPU::Arm64Emitter& Emitter) {
-  Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestDisplacement(SiteAddress, ValueSize));
+  Emitter.dc64(SiteAddress + ValueSize + ReadLiveGuestData(SiteAddress, ValueSize));
 }
 
 static inline void ApplyPatchableRIPMoveRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
-  const uint64_t Target = SiteAddress + ValueSize + ReadLiveGuestDisplacement(SiteAddress, ValueSize);
+  const uint64_t Target = SiteAddress + ValueSize + ReadLiveGuestData(SiteAddress, ValueSize);
+  Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD);
+}
+
+static inline void ApplyPatchableCRCMoveRelocation(uint64_t SiteAddress, uint8_t ValueSize, uint8_t RegisterIndex, CPU::Arm64Emitter& Emitter) {
+  const uint64_t Target = FEXCore::Utils::crc32(reinterpret_cast<const uint8_t*>(SiteAddress), ValueSize);
   Emitter.LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Register(RegisterIndex), Target, CPU::Arm64Emitter::PadType::DOPAD);
 }
 
@@ -599,6 +597,11 @@ bool CodeCache::ApplyPackedCodeRelocations(uint64_t GuestEntry, std::span<std::b
                                       Reloc.PatchableData.RegisterIndex, Emitter);
       break;
     }
+    case FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
+      ApplyPatchableCRCMoveRelocation(GuestEntry + Reloc.PatchableData.SiteOffset, Reloc.PatchableData.ValueSize,
+                                      Reloc.PatchableData.RegisterIndex, Emitter);
+      break;
+    }
     default: ERROR_AND_DIE_FMT("Unknown packed relocation type {}", ToUnderlying((CPU::RelocationTypes)Reloc.Type));
     }
   }

FEXCore/Source/Interface/Core/Core.cpp

@@ -30,6 +30,7 @@ $end_info$
 #include "Interface/IR/RegisterAllocationData.h"
 #include "Utils/Allocator.h"
 #include "Utils/Allocator/HostAllocator.h"
+#include "Utils/crc32.h"
 #include <FEXCore/Utils/SpinWaitLock.h>
 #include "Utils/variable_length_integer.h"
 
@@ -76,9 +77,6 @@ $end_info$
 #include <unordered_map>
 #include <utility>
 #include <xxhash.h>
-#if defined(ARCHITECTURE_arm64)
-#include <arm_acle.h>
-#endif
 
 namespace FEXCore::Context {
 ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
@@ -92,7 +90,7 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
 
   if (Config.BlockJITNaming() || Config.GlobalJITNaming() || Config.LibraryJITNaming()) {
     // Only initialize symbols file if enabled. Ensures we don't pollute /tmp with empty files.
-    Symbols.InitFile();
+    Symbols.InitFile(Features.ProcessPID);
   }
 
   uint64_t FrequencyCounter = FEXCore::GetCycleCounterFrequency();
@@ -107,6 +105,14 @@ ContextImpl::ContextImpl(const FEXCore::HostFeatures& Features)
   // Track atomic TSO emulation configuration.
   UpdateAtomicTSOEmulationConfig();
 
+#ifndef _WIN32
+  // Check if the kernel supports getrandom().
+  uint64_t Probe {};
+  HostRNGAvailable = FHU::Syscalls::getrandom(&Probe, sizeof(Probe), 0) == sizeof(Probe);
+#else
+  HostRNGAvailable = true;
+#endif
+
   DiskCache.Init(this);
 }
 
@@ -258,8 +264,8 @@ uint32_t ContextImpl::ReconstructCompactedEFLAGS(FEXCore::Core::InternalThreadSt
 void ContextImpl::ReconstructXMMRegisters(const FEXCore::Core::InternalThreadState* Thread, __uint128_t* XMM_Low, __uint128_t* YMM_High) {
   const size_t MaximumRegisters = Config.Is64BitMode ? FEXCore::Core::CPUState::NUM_XMMS : 8;
 
-  if (YMM_High != nullptr && HostFeatures.SupportsAVX) {
-    const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256;
+  if (YMM_High != nullptr && HostFeatures.SupportsAVX()) {
+    const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256();
 
     if (SupportsConvergedRegisters) {
       ///< Output wants to de-interleave
@@ -282,8 +288,8 @@ void ContextImpl::ReconstructXMMRegisters(const FEXCore::Core::InternalThreadSta
 
 void ContextImpl::SetXMMRegistersFromState(FEXCore::Core::InternalThreadState* Thread, const __uint128_t* XMM_Low, const __uint128_t* YMM_High) {
   const size_t MaximumRegisters = Config.Is64BitMode ? FEXCore::Core::CPUState::NUM_XMMS : 8;
-  if (YMM_High != nullptr && HostFeatures.SupportsAVX) {
-    const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256;
+  if (YMM_High != nullptr && HostFeatures.SupportsAVX()) {
+    const bool SupportsConvergedRegisters = HostFeatures.SupportsSVE256();
 
     if (SupportsConvergedRegisters) {
       ///< Output wants to de-interleave
@@ -358,11 +364,6 @@ bool ContextImpl::InitCore() {
   // Set up the SignalDelegator config since core is initialized.
   SignalDelegation->SetConfig(Dispatcher->MakeSignalDelegatorConfig());
 
-#if defined(_WIN32) && !defined(ARCHITECTURE_arm64ec)
-  // WOW64 always needs the interrupt fault check to be enabled.
-  Config.NeedsPendingInterruptFaultCheck = true;
-#endif
-
   if (Config.GdbServer) {
     // If gdbserver is enabled then this needs to be enabled.
     Config.NeedsPendingInterruptFaultCheck = true;
@@ -528,6 +529,8 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
 
   bool HasCustomIR {};
 
+  bool WantsDiskCachePatching = DiskCache.IsReadingDiskCache() || DiskCache.IsWritingDiskCache();
+
   if (HasCustomIRHandlers.load(std::memory_order_relaxed)) {
     std::shared_lock lk(CustomIRMutex);
     auto Handler = CustomIRHandlers.find(GuestRIP);
@@ -642,27 +645,11 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
           auto ExistingCodePtr = reinterpret_cast<uint8_t*>(Block.Entry + BlockInstructionsLength);
           auto InstAddressReg = Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP);
 
-          auto crc32 = [](const uint8_t* Ptr, size_t Size) -> uint32_t {
-#if defined(ARCHITECTURE_arm64)
-            uint32_t Result {};
-#define do_crc(type, suffix)                                               \
-  while (Size >= sizeof(type)) {                                           \
-    Result = __crc32##suffix(Result, *reinterpret_cast<const type*>(Ptr)); \
-    Ptr += sizeof(type);                                                   \
-    Size -= sizeof(type);                                                  \
-  }
-            do_crc(uint64_t, d);
-            do_crc(uint32_t, w);
-            do_crc(uint16_t, h);
-            do_crc(uint8_t, b);
-            return Result;
-#else
-            // Unsupported on non-arm.
-            return 0;
-#endif
-          };
-          auto CodeChanged = Thread->OpDispatcher->_ValidateCode(
-            Thread->OpDispatcher->Constant(crc32(ExistingCodePtr, DecodedInfo->InstSize)), InstAddressReg, DecodedInfo->InstSize);
+          auto Value = FEXCore::Utils::crc32(ExistingCodePtr, DecodedInfo->InstSize);
+          auto CRC = WantsDiskCachePatching ?
+                       Thread->OpDispatcher->_PatchableGuestCRC(IR::OpSize::i64Bit, Value, (int64_t)ExistingCodePtr, DecodedInfo->InstSize) :
+                       Thread->OpDispatcher->Constant(Value);
+          auto CodeChanged = Thread->OpDispatcher->_ValidateCode(CRC, InstAddressReg, DecodedInfo->InstSize);
 
           auto InvalidateCodeCond = Thread->OpDispatcher->CondJump(CodeChanged);
 
@@ -674,11 +661,10 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
           Thread->OpDispatcher->StartNewBlock();
 
           // Generate a relocatable entry for invalidation purposes.
-          auto EntryReg = Thread->OpDispatcher->_EntrypointOffset(GPRSize, 0);
-          Thread->OpDispatcher->_ThreadRemoveCodeEntry(EntryReg);
-
-          // Exit the function at this instruction after invalidation.
-          Thread->OpDispatcher->ExitFunction(Thread->OpDispatcher->_InlineEntrypointOffset(GPRSize, InstAddress - GuestRIP));
+          auto EntryToInvalidate = Thread->OpDispatcher->_EntrypointOffset(GPRSize, 0);
+          auto NewRIP = Thread->OpDispatcher->_EntrypointOffset(GPRSize, InstAddress - GuestRIP);
+          // Invalidate and exit the function
+          Thread->OpDispatcher->_ThreadRemoveCodeEntry(EntryToInvalidate, NewRIP);
 
           auto NextOpBlock = Thread->OpDispatcher->CreateNewCodeBlockAfter(CurrentBlock);
 
@@ -729,13 +715,19 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
               LogMan::Msg::EFmt("Invalid or Unknown instruction: {} 0x{:x}", TableInfo->Name ?: "UND", Block.Entry - GuestRIP);
             }
 
-            if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::INVALID_INST ||
-                Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BAD_RELOCATION) {
+            if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_0 ||
+                Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_1 ||
+                Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_2 ||
+                Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_3) {
+              Thread->OpDispatcher->BreakpointOp(DecodedInfo, FEXCore::ToUnderlying(Block.BlockStatus) -
+                                                                FEXCore::ToUnderlying(Frontend::Decoder::DecodedBlockStatus::BREAKPOINT_0));
+            } else if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::INVALID_INST ||
+                       Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::BAD_RELOCATION) {
               Thread->OpDispatcher->InvalidOp(DecodedInfo);
             } else if (Block.BlockStatus == Frontend::Decoder::DecodedBlockStatus::UNIMPLEMENTED_INST) {
               Thread->OpDispatcher->UnimplementedOp(DecodedInfo);
             } else {
-              Thread->OpDispatcher->NoExecOp(DecodedInfo);
+              Thread->OpDispatcher->NoExecOrPartialOp(DecodedInfo);
             }
           }
 
@@ -773,10 +765,10 @@ ContextImpl::GenerateIR(FEXCore::Core::InternalThreadState* Thread, uint64_t Gue
 #endif
 
     Thread->OpDispatcher->Finalize();
-
-    Thread->FrontendDecoder->DelayedDisownBuffer();
   }
 
+  Thread->FrontendDecoder->DelayedDisownBuffer();
+
   IR::IREmitter* IREmitter = Thread->OpDispatcher.get();
 
   auto ShouldDump = Thread->OpDispatcher->ShouldDumpIR();
@@ -846,7 +838,7 @@ ContextImpl::CompileCodeResult ContextImpl::CompileCode(FEXCore::Core::InternalT
   // If the trap flag is set we generate single instruction blocks that each check to generate a single step exception.
   bool TFSet = Thread->CurrentFrame->State.flags[X86State::RFLAG_TF_RAW_LOC];
 
-  auto CompiledCode = Thread->CPUBackend->CompileCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), TFSet);
+  auto CompiledCode = Thread->CPUBackend->EmitJITCode(GuestRIP, Length, TotalInstructions == 1, &*IRView, DebugData.get(), TFSet);
 
   // Release the IR
   Thread->OpDispatcher->DelayedDisownBuffer();

FEXCore/Source/Interface/Core/DiskCache.cpp

@@ -2,18 +2,22 @@
 
 #define XXH_STATIC_LINKING_ONLY
 
-#include "FEXCore/Utils/TypeDefines.h"
 #include "Interface/Core/Frontend.h"
-#include "FEXCore/Config/Config.h"
-#include "FEXCore/fextl/string.h"
-#include "FEXHeaderUtils/Filesystem.h"
-#include "FEXCore/Core/DiskCache.h"
-#include "FEXCore/Core/DiskCacheFileMapper.h"
-#include "FEXCore/Utils/LogManager.h"
 #include "Interface/Context/Context.h"
-#include "FEXCore/HLE/SyscallHandler.h"
-#include "FEXCore/Utils/File.h"
-#include "FEXCore/fextl/memory.h"
+
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Core/DiskCache.h>
+#include <FEXCore/Core/DiskCacheFileMapper.h>
+#include <FEXCore/fextl/memory.h>
+#include <FEXCore/fextl/string.h>
+#include <FEXCore/HLE/SyscallHandler.h>
+#include <FEXCore/Utils/File.h>
+#include <FEXCore/Utils/FileUtils.h>
+#include <FEXCore/Utils/LogManager.h>
+#include <FEXCore/Utils/Profiler.h>
+#include <FEXCore/Utils/TypeDefines.h>
+#include <FEXHeaderUtils/Filesystem.h>
+
 #include <cstdint>
 #include <cstring>
 #include <atomic>
@@ -269,37 +273,16 @@ namespace DiskCache {
     }
   }
 
-  bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob, Index& Index,
-                                 std::mutex& IndexMutex, std::span<const uint8_t> IndexBlob) {
+  bool IndexedDB::StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob,
+                                 MesaFOZ::mesa_index_db_file_entry& IndexEntry, std::span<const uint8_t> IndexBlob) {
     if (ReadOnly) {
       // shouldn't happen
       return false;
     }
 
-    {
-      std::lock_guard Guard(IndexMutex);
-      auto IndexIt = Index.find(LookupKey);
-      bool Dupe = false;
-      if (IndexIt != Index.end()) {
-        if (IndexIt->second.MoreEntries.get()) {
-          for (auto& [Key, Elem] : *IndexIt->second.MoreEntries) {
-            if (XXH128_isEqual(Elem.GuestHash, *(const XXH128_hash_t*)(IndexBlob.data()))) {
-              Dupe = true;
-              break;
-            }
-          }
-        }
-        if (!Dupe && XXH128_isEqual(IndexIt->second.MainEntry.GuestHash, *(const XXH128_hash_t*)(IndexBlob.data()))) {
-          Dupe = true;
-        }
-        // could happen if it's seen again while in flight in the store queue
-        if (Dupe) {
-          return true;
-        }
-        if (IndexIt->second.MoreEntries.get() && IndexIt->second.MoreEntries->size() >= LOOKUP_KEY_MAX_BUCKET_DEPTH) {
-          return true;
-        }
-      }
+    if (MaxSizeReached || CacheFileSize + Blob.size() >= MaxFileSize) {
+      MaxSizeReached = true;
+      return false;
     }
 
     if (!CacheFOZ.Lock(STORE_LOCK_TIMEOUT_MS) || !IndexFOZ.Lock(STORE_LOCK_TIMEOUT_MS)) {
@@ -317,10 +300,10 @@ namespace DiskCache {
       return false;
     }
 
-    MesaFOZ::mesa_index_db_file_entry IndexEntry {.hash = LookupKey,
-                                                  .size = (uint32_t)Blob.size(),
-                                                  .last_access_time = 0, // todo..
-                                                  .cache_db_file_offset = BlobOffset};
+    IndexEntry = {.hash = LookupKey,
+                  .size = (uint32_t)Blob.size(),
+                  .last_access_time = 0, // todo..
+                  .cache_db_file_offset = BlobOffset};
 
     std::span<const uint8_t> IndexBlobChunks[] = {
       {(const uint8_t*)&IndexEntry, sizeof(IndexEntry)},
@@ -341,22 +324,6 @@ namespace DiskCache {
       CacheFileSize = BlobOffset + Blob.size();
     }
 
-    const IndexExtraBlobHeader* IndexBlobHeader = reinterpret_cast<const IndexExtraBlobHeader*>(IndexBlob.data());
-
-    struct IndexEntry NewEntry {this, BlobOffset, (uint32_t)Blob.size(), IndexBlobHeader->GuestSize, IndexBlobHeader->GuestHash};
-    NewEntry.GuestExtents.resize(IndexBlobHeader->GuestExtentsCount);
-    memcpy(NewEntry.GuestExtents.data(), reinterpret_cast<const uint32_t*>(IndexBlob.data() + sizeof(IndexExtraBlobHeader)),
-           IndexBlobHeader->GuestExtentsCount * sizeof(uint32_t));
-    std::lock_guard Guard(IndexMutex);
-    auto It = Index.find(LookupKey);
-    if (It == Index.end()) {
-      Index.emplace(LookupKey, IndexCacheHead {std::move(NewEntry), IndexBlobHeader->GuestFootprint, nullptr});
-    } else {
-      if (!It->second.MoreEntries.get()) {
-        It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, struct IndexEntry>>();
-      }
-      It->second.MoreEntries->insert({IndexBlobHeader->GuestFootprint, std::move(NewEntry)});
-    }
     return true;
   }
 
@@ -393,6 +360,39 @@ namespace DiskCache {
     FileMapper = Func;
   }
 
+  static inline void PruneStaleEntries(std::string_view CacheBase, std::string_view MachineBucketHash) {
+    FEXCORE_PROFILE_SCOPED("DiskCache::PruneStaleEntries");
+    FEX_CONFIG_OPT(DiskCachePruneStaleEntries, DISKCACHEPRUNESTALEENTRIES);
+
+    if (!DiskCachePruneStaleEntries) {
+      return;
+    }
+
+    struct SimpleCapture {
+      std::string_view CacheBase, MachineBucketHash;
+    } const SimpleCapture {
+      .CacheBase = CacheBase,
+      .MachineBucketHash = MachineBucketHash,
+    };
+
+    FEXCore::FileUtils::WalkDirectory(
+      CacheBase,
+      [](std::string_view name, bool is_dir, const void* user_data) {
+        if (!is_dir) {
+          return;
+        }
+
+        auto capture = reinterpret_cast<const struct SimpleCapture*>(user_data);
+
+        // Current behaviour is to remove entries that no longer match the MachineBucketHash.
+        // This means that if the Disk Cache version no longer matches, or the FEXCore::HostFeatures differ, then they get removed.
+        if (name != capture->MachineBucketHash) {
+          FEXCore::FileUtils::RecursiveRemoveDirectory(fextl::fmt::format("{}/{}", capture->CacheBase, name));
+        }
+      },
+      &SimpleCapture);
+  }
+
   void DiskCache::Init(FEXCore::Context::ContextImpl* CTX) {
     this->CTX = CTX;
 
@@ -402,30 +402,50 @@ namespace DiskCache {
 
     fextl::string SerializedConfig = FEXCore::Config::SerializeForCache();
 
+    const auto HostFeatureHash = CTX->HostFeatures.HashForCaching();
+
     struct __attribute__((packed)) {
       uint16_t FormatVersion;
-      uint8_t Is64BitMode;
       uint64_t HostFeaturesHash;
-    } BucketHeader = {FormatVersion, CTX->Config.Is64BitMode, CTX->HostFeatures.HashForCaching()};
+    } MachineBucketData = {FormatVersion, HostFeatureHash.HostFeaturesHash};
 
-    fextl::vector<uint8_t> BucketBytes;
-    BucketBytes.resize(sizeof(BucketHeader) + SerializedConfig.size());
-    memcpy(BucketBytes.data(), &BucketHeader, sizeof(BucketHeader));
-    memcpy(BucketBytes.data() + sizeof(BucketHeader), SerializedConfig.data(), SerializedConfig.size());
-    BucketHash = XXH3_128bits(BucketBytes.data(), BucketBytes.size());
+    uint8_t ProcessFeaturesHash {};
+    memcpy(&ProcessFeaturesHash, &HostFeatureHash.ProcessFeaturesHash, sizeof(ProcessFeaturesHash));
+    LogMan::Msg::IFmt("Disk Cache features:");
+    LogMan::Msg::IFmt("HostFeatures Hash: 0x{:016x}", HostFeatureHash.HostFeaturesHash);
+    LogMan::Msg::IFmt("ProcessFeaturesHash: 0x{:02x}", ProcessFeaturesHash);
+
+    fextl::vector<uint8_t> BucketBytes(sizeof(MachineBucketData) + sizeof(HostFeatureHash.ProcessFeaturesHash) + SerializedConfig.size());
+    memcpy(BucketBytes.data(), &MachineBucketData, sizeof(MachineBucketData));
+
+    // 64-bit mode and HostType is in the ProcessBucket hash only instead of the MachineBucketHash
+    // These effect code-gen, but they aren't part of the MachineBucketHash, as it comes from Process state.
+    memcpy(&BucketBytes[sizeof(MachineBucketData)], &HostFeatureHash.ProcessFeaturesHash, sizeof(HostFeatureHash.ProcessFeaturesHash));
+    memcpy(BucketBytes.data() + sizeof(MachineBucketData) + sizeof(HostFeatureHash.ProcessFeaturesHash), SerializedConfig.data(),
+           SerializedConfig.size());
+
+    uint64_t MachineBucketHash = XXH3_64bits(BucketBytes.data(), sizeof(MachineBucketData));
+    uint64_t ProcessBucketHash =
+      XXH3_64bits(BucketBytes.data() + sizeof(MachineBucketData), sizeof(HostFeatureHash.ProcessFeaturesHash) + SerializedConfig.size());
+    BucketHash.high64 = MachineBucketHash;
+    BucketHash.low64 = ProcessBucketHash;
 
     fextl::string BasePath = BasePathOverride();
     if (BasePath.empty()) {
       BasePath = FEXCore::Config::GetCacheDirectory() + "DiskCache/";
-      BasePath += fextl::fmt::format("{:016x}{:016x}", BucketHash.high64, BucketHash.low64) + "/";
     }
+
+    const auto MachineBucketHashAsString = fextl::fmt::format("{:016x}", MachineBucketHash);
+    PruneStaleEntries(BasePath, MachineBucketHashAsString);
+
+    BasePath += MachineBucketHashAsString + "/";
     FHU::Filesystem::CreateDirectories(BasePath);
 
     if (!MapDiskCacheFiles) {
       FileMapper = nullptr;
     }
 
-    fextl::string RWDBBasePath = BasePath + "RWCacheDB";
+    const auto RWDBBasePath = fextl::fmt::format("{}RWCacheDB_{:016x}", BasePath, ProcessBucketHash);
     OpenCacheDB(RWDBBasePath, false);
 
     if (RWCacheDB && !FoundMetadata) {
@@ -433,7 +453,8 @@ namespace DiskCache {
       MesaFOZ::foz_payload_key MetadataKey;
       memset(MetadataKey.bytes, 0xFF, sizeof(MetadataKey));
       IndexExtraBlobHeader MetaDataHeader = {};
-      RWCacheDB->StoreCacheBlob(MetadataKey, ~0, {BucketBytes.data(), BucketBytes.size()}, Index, IndexLock,
+      MesaFOZ::mesa_index_db_file_entry IndexEntry = {};
+      RWCacheDB->StoreCacheBlob(MetadataKey, ~0, {BucketBytes.data(), BucketBytes.size()}, IndexEntry,
                                 {reinterpret_cast<uint8_t*>(&MetaDataHeader), sizeof(MetaDataHeader)});
       Index.erase(~0);
     }
@@ -459,7 +480,7 @@ namespace DiskCache {
 
     if (IsWritingDiskCache()) {
       FEXCore::Threads::Flags WriterThreadFlags = {.LowPriority = true, .Internal = true};
-      Writer = fextl::make_unique<WorkQueueThread>(WriterThreadFlags);
+      Writer = fextl::make_unique<WorkQueueThread>(WriterThreadFlags, "FEX:DiskCache");
     }
   }
 
@@ -486,6 +507,49 @@ namespace DiskCache {
     return XXH3_64bits(&BlobKeyBytes, sizeof(BlobKeyBytes));
   }
 
+  struct DiskCache::PruneMemoryLRUWorkItem final : WorkQueueThread::WorkItem {
+    DiskCache* Self;
+    PruneMemoryLRUWorkItem(DiskCache* Self)
+      : Self(Self) {}
+    void Run() override {
+      if (Self->MemoryLRUCurrentSize <= Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
+        return;
+      }
+      std::lock_guard IndexGuard(Self->IndexLock);
+      std::lock_guard LRUGuard(Self->MemoryLRULock);
+      if (Self->MemoryLRU.empty()) {
+        return;
+      }
+      auto Last = std::prev(Self->MemoryLRU.end());
+
+      while (Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize && !Self->MemoryLRU.empty()) {
+        auto LastKey = *Last;
+        auto IndexEntry = Self->LookupLocked(LastKey.LookupKey, LastKey.GuestHash, LastKey.GuestFootprint);
+        bool AtFront = (Last == Self->MemoryLRU.begin());
+        if (IndexEntry && IndexEntry->MemoryBlob.use_count() > 1) {
+          // being read rn, keep moving
+          if (AtFront) {
+            break;
+          }
+          Last--;
+          continue;
+        }
+        if (IndexEntry) {
+          IndexEntry->MemoryBlob.reset();
+          IndexEntry->LRUEntry.reset();
+        }
+        Self->MemoryLRUCurrentSize -= LastKey.Size;
+        auto Deleted = Last;
+        if (AtFront) {
+          Self->MemoryLRU.erase(Deleted);
+          break;
+        }
+        Last--;
+        Self->MemoryLRU.erase(Deleted);
+      }
+    }
+  };
+
   std::optional<CodeHitData> DiskCache::Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region,
                                                uint64_t GuestRIP, std::optional<uint64_t>& GuestCodeKey) {
     if (!IsReadingDiskCache()) {
@@ -546,6 +610,8 @@ namespace DiskCache {
     fextl::multimap<uint64_t, IndexEntry>::iterator MoreEntriesIt;
     fextl::multimap<uint64_t, IndexEntry>* MapPointer = nullptr;
     IndexEntry* EntryUnderReview;
+    fextl::shared_ptr<fextl::vector<uint8_t>> BlobRef;
+    std::optional<fextl::list<MemoryLRUKey>::iterator> LRUIter;
     uint64_t CurrentFootprint = 0;
     {
       std::lock_guard Guard(IndexLock);
@@ -563,9 +629,13 @@ namespace DiskCache {
       }
       if (MapPointer && MoreEntriesIt != It->second.MoreEntries->end()) {
         EntryUnderReview = &MoreEntriesIt->second;
+        BlobRef = EntryUnderReview->MemoryBlob;
+        LRUIter = EntryUnderReview->LRUEntry;
         CurrentFootprint = MoreEntriesIt->first;
       } else {
         EntryUnderReview = &MainEntry;
+        BlobRef = EntryUnderReview->MemoryBlob;
+        LRUIter = EntryUnderReview->LRUEntry;
         CurrentFootprint = MainEntryFootprint;
         TriedMainEntry = true;
       }
@@ -582,6 +652,8 @@ namespace DiskCache {
           // if the current footprint is also the main entry's footprint, give main entry a shot next
           if (!TriedMainEntry && MoreEntriesIt->first == MainEntryFootprint) {
             EntryUnderReview = &MainEntry;
+            BlobRef = EntryUnderReview->MemoryBlob;
+            LRUIter = EntryUnderReview->LRUEntry;
             CurrentFootprint = MainEntryFootprint;
             TriedMainEntry = true;
           } else {
@@ -594,17 +666,27 @@ namespace DiskCache {
             break;
           } else {
             EntryUnderReview = &MainEntry;
+            BlobRef = EntryUnderReview->MemoryBlob;
+            LRUIter = EntryUnderReview->LRUEntry;
             CurrentFootprint = MainEntryFootprint;
             TriedMainEntry = true;
           }
         }
         if (!EntryUnderReview && MapPointer && MoreEntriesIt != MapPointer->end()) {
           EntryUnderReview = &MoreEntriesIt->second;
+          BlobRef = EntryUnderReview->MemoryBlob;
+          LRUIter = EntryUnderReview->LRUEntry;
           CurrentFootprint = MoreEntriesIt->first;
         }
       }
 
       Advance = true;
+
+      // entry not backed by anything right now - todo prune..
+      if (!EntryUnderReview->DB && !BlobRef) {
+        continue;
+      }
+
       // do we have enough room in our live code to even hash GuestSize worth?
       if (Available < EntryUnderReview->GuestSize) {
         continue;
@@ -635,7 +717,7 @@ namespace DiskCache {
           break;
         } else if (Validation) {
           fextl::vector<uint8_t> GuestCode(Entry.GuestSize);
-          if (Entry.Size >= Entry.GuestSize && Entry.DB->ReadCacheBlob(Entry.Offset + Entry.Size - Entry.GuestSize, GuestCode)) {
+          if (Entry.Size >= Entry.GuestSize && Entry.DB && Entry.DB->ReadCacheBlob(Entry.Offset + Entry.Size - Entry.GuestSize, GuestCode)) {
             const uint8_t* CachedGuest = GuestCode.data();
             const uint8_t* LiveGuest = reinterpret_cast<const uint8_t*>(GuestRIP);
             uint64_t DiffCount = 0;
@@ -699,7 +781,17 @@ namespace DiskCache {
     HitData.Blob.resize(GuestPages.size() * sizeof(uint64_t) + EntrySizeWithoutGuestCode);
     memcpy(HitData.Blob.data(), GuestPages.data(), GuestPages.size() * sizeof(uint64_t));
     uint32_t BlobOffset = GuestPages.size() * sizeof(uint64_t);
-    if (!Entry.DB->ReadCacheBlob(Entry.Offset, {HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode})) {
+    bool FoundInLRU = false;
+    if (BlobRef && BlobRef->size() >= EntrySizeWithoutGuestCode) {
+      FoundInLRU = true;
+      memcpy(HitData.Blob.data() + BlobOffset, BlobRef->data(), EntrySizeWithoutGuestCode);
+      if (LRUIter) {
+        std::lock_guard Guard(MemoryLRULock);
+        // avoided disk by nabbing from lru, bump to front
+        // LogMan::Msg::IFmt("lru hit! {}", MemoryLRUCurrentSize);
+        MemoryLRU.splice(MemoryLRU.begin(), MemoryLRU, *LRUIter);
+      }
+    } else if (!Entry.DB->ReadCacheBlob(Entry.Offset, {HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode})) {
       return std::nullopt;
     }
 
@@ -720,6 +812,29 @@ namespace DiskCache {
       return std::nullopt;
     }
 
+    bool StoreInMemory = !FoundInLRU;
+    if (EntrySizeWithoutGuestCode > MemoryLRUMaxSize) {
+      StoreInMemory = false;
+    }
+
+    if (StoreInMemory) {
+      auto NewBlobRef = fextl::make_shared<fextl::vector<uint8_t>>(EntrySizeWithoutGuestCode);
+      memcpy(NewBlobRef->data(), HitData.Blob.data() + BlobOffset, EntrySizeWithoutGuestCode);
+      std::lock_guard Guard(IndexLock);
+      auto CurrentIndexEntry = LookupLocked(LookupKey, Header.GuestHash, LastFootprintHashed);
+      if (CurrentIndexEntry && !CurrentIndexEntry->MemoryBlob) {
+        std::lock_guard LRUGuard(MemoryLRULock);
+        MemoryLRU.push_front({LookupKey, Header.GuestHash, LastFootprintHashed, EntrySizeWithoutGuestCode});
+        CurrentIndexEntry->LRUEntry = MemoryLRU.begin();
+        CurrentIndexEntry->MemoryBlob = NewBlobRef;
+        MemoryLRUCurrentSize += EntrySizeWithoutGuestCode;
+      }
+    }
+
+    if (StoreInMemory && MemoryLRUCurrentSize > MemoryLRUMaxSize + MemoryLRUEvictThreshold) {
+      Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(this));
+    }
+
     HitData.HostCode = {HitData.Blob.data() + BlobOffset, Header.HostSize};
     BlobOffset += Header.HostSize;
     HitData.EntryPointRIPs = {reinterpret_cast<uint64_t*>(HitData.Blob.data() + BlobOffset), Header.EntryPointCount};
@@ -785,23 +900,81 @@ namespace DiskCache {
     }
   }
 
+  IndexEntry* DiskCache::LookupLocked(const uint64_t LookupKey, const XXH128_hash_t& GuestHash, const uint64_t GuestFootprint) {
+    auto It = Index.find(LookupKey);
+    if (It != Index.end()) {
+      if (XXH128_isEqual(It->second.MainEntry.GuestHash, GuestHash)) {
+        return &It->second.MainEntry;
+      }
+      if (It->second.MoreEntries.get()) {
+        auto Range = It->second.MoreEntries->equal_range(GuestFootprint);
+        for (auto MoreEntriesIt = Range.first; MoreEntriesIt != Range.second; MoreEntriesIt++) {
+          if (XXH128_isEqual(MoreEntriesIt->second.GuestHash, GuestHash)) {
+            return &MoreEntriesIt->second;
+          }
+        }
+      }
+    }
+    return nullptr;
+  }
+
   struct DiskCache::CacheStoreWorkItem final : WorkQueueThread::WorkItem {
     DiskCache* Self;
     IndexedDB* DB;
     MesaFOZ::foz_payload_key UniqueKey;
     uint64_t LookupKey;
-    fextl::vector<uint8_t> Blob;
+    std::span<uint8_t> Blob;
     fextl::vector<uint8_t> IndexBlob;
+    bool StoreDisk;
     CacheStoreWorkItem(DiskCache* Self, IndexedDB* DB, const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey,
-                       fextl::vector<uint8_t>&& Blob, fextl::vector<uint8_t>&& IndexBlob)
+                       std::span<uint8_t> Blob, fextl::vector<uint8_t>&& IndexBlob, bool StoreDisk)
       : Self(Self)
       , DB(DB)
       , UniqueKey(UniqueKey)
       , LookupKey(LookupKey)
-      , Blob(std::move(Blob))
-      , IndexBlob(std::move(IndexBlob)) {}
+      , Blob(Blob)
+      , IndexBlob(std::move(IndexBlob))
+      , StoreDisk(StoreDisk) {}
     void Run() override {
-      DB->StoreCacheBlob(UniqueKey, LookupKey, Blob, Self->Index, Self->IndexLock, IndexBlob);
+      struct MesaFOZ::mesa_index_db_file_entry IndexHeader;
+      bool DiskSuccess = !StoreDisk || DB->StoreCacheBlob(UniqueKey, LookupKey, Blob, IndexHeader, IndexBlob);
+
+      bool KeepEntryInMemory = true;
+      // todo possible other lru condition here like entry size?
+      if (Blob.size() > Self->MemoryLRUMaxSize) {
+        KeepEntryInMemory = false;
+      } else {
+        Self->MemoryLRUCurrentSize += Blob.size();
+      }
+
+      const IndexExtraBlobHeader* IndexAfterHeader = reinterpret_cast<const IndexExtraBlobHeader*>(IndexBlob.data());
+
+      fextl::list<MemoryLRUKey>::iterator NewLRUEntry;
+      if (KeepEntryInMemory) {
+        std::lock_guard Guard(Self->MemoryLRULock);
+        Self->MemoryLRU.push_front({LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint, (uint32_t)Blob.size()});
+        NewLRUEntry = Self->MemoryLRU.begin();
+      }
+      {
+        std::lock_guard Guard(Self->IndexLock);
+        auto IndexEntry = Self->LookupLocked(LookupKey, IndexAfterHeader->GuestHash, IndexAfterHeader->GuestFootprint);
+        LOGMAN_THROW_A_FMT(IndexEntry != nullptr, "Stored Index entry not found?");
+        if (IndexEntry) {
+          if (StoreDisk && DiskSuccess) {
+            IndexEntry->DB = DB;
+            IndexEntry->Offset = IndexHeader.cache_db_file_offset;
+          }
+          if (!KeepEntryInMemory) {
+            IndexEntry->MemoryBlob.reset();
+          } else {
+            IndexEntry->LRUEntry = NewLRUEntry;
+          }
+        }
+      }
+
+      if (KeepEntryInMemory && Self->MemoryLRUCurrentSize > Self->MemoryLRUMaxSize + Self->MemoryLRUEvictThreshold) {
+        Self->Writer->QueueWork(fextl::make_unique<PruneMemoryLRUWorkItem>(Self));
+      }
     }
   };
 
@@ -825,6 +998,11 @@ namespace DiskCache {
         if (Target >= Region->BeginVA && Target < Region->EndVA) {
           continue;
         }
+        // let it through if it's inside the same ELF image? (like bss)
+        if (Region->FileInfo.MappedSize && Target >= Region->FileStartVA && Target < Region->FileStartVA + Region->FileInfo.MappedSize) {
+          continue;
+        }
+
         auto TargetSection = CTX->SyscallHandler->LookupExecutableFileSection(Thread, Target);
         if (!TargetSection || TargetSection->FileInfo.FileId != Region->FileInfo.FileId) {
           // we don't know where it's pointing, so we don't know how to encode the offset, so we can't cache atm
@@ -843,61 +1021,6 @@ namespace DiskCache {
       }
     }
 
-    fextl::set<uint64_t> DataMaskAddresses;
-
-    fextl::vector<uint32_t> ExactGuestCodeExtents;
-    uint64_t CurStartExtent = 0, CurEndExtent = 0;
-    const Frontend::Decoder::DecodedBlocks* LastBlock = nullptr;
-    for (auto& SubBlock : DecodedBlockInfo->Blocks) {
-      if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
-        return false;
-      }
-      if (!CurStartExtent) {
-        CurStartExtent = SubBlock.Entry;
-        CurEndExtent = SubBlock.Entry + SubBlock.Size;
-      } else {
-        LOGMAN_THROW_A_FMT(SubBlock.Entry >= CurEndExtent, "DecodedBlocks not sorted or overlapping?");
-        if (SubBlock.Entry == CurEndExtent) {
-          CurEndExtent = SubBlock.Entry + SubBlock.Size;
-        } else {
-          ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
-          ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
-          CurStartExtent = SubBlock.Entry;
-          CurEndExtent = SubBlock.Entry + SubBlock.Size;
-        }
-      }
-      // split extents according to data masks as well
-      for (auto& Mask : SubBlock.DataMasks) {
-        if (Mask.FieldAddress > CurStartExtent) {
-          ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
-          ExactGuestCodeExtents.push_back(Mask.FieldAddress - CurStartExtent);
-        }
-        CurStartExtent = Mask.FieldAddress + Mask.ValueSize;
-
-        DataMaskAddresses.insert(Mask.FieldAddress);
-      }
-      LastBlock = &SubBlock;
-    }
-    if (LastBlock && (CurStartExtent != GuestRIP || CurEndExtent != GuestRIP + GuestCode.size())) {
-      ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
-      ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
-    }
-
-    if (ExactGuestCodeExtents.size() == 0) {
-      ExactGuestCodeExtents.reserve(2);
-      ExactGuestCodeExtents.push_back(0);
-      ExactGuestCodeExtents.push_back(GuestCode.size());
-    }
-
-    uint64_t GuestFootprint = XXH3_64bits(ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
-
-    // if (ExactGuestCodeExtents.size()) {
-    //   LogMan::Msg::IFmt("store! length {:d}", GuestCode.size());
-    //   for(uint32_t i = 0; i < ExactGuestCodeExtents.size(); i+=2 ) {
-    //     LogMan::Msg::IFmt("extent {} {}", ExactGuestCodeExtents[i], ExactGuestCodeExtents[i]+ExactGuestCodeExtents[i+1]);
-    //   }
-    // }
-
     const uint32_t EntryPointCount = (uint32_t)CompiledCode.EntryPoints.size();
 
     const size_t HeaderOffset = 0;
@@ -910,9 +1033,8 @@ namespace DiskCache {
     const size_t TotalSize = GuestCodeOffset + GuestCode.size();
 
     // we'll copy everything into here and pass it to the Writer, then return to caller quickly
-    fextl::vector<uint8_t> Blob;
-    Blob.resize(TotalSize);
-    uint8_t* BlobData = Blob.data();
+    auto BlobRef = fextl::make_shared<fextl::vector<uint8_t>>(TotalSize);
+    uint8_t* BlobData = BlobRef->data();
 
     BlobFixedHeader Header {
       .GuestSize = (uint32_t)GuestCode.size(),
@@ -922,15 +1044,6 @@ namespace DiskCache {
       .ThunkRelocCount = ThunkRelocCount,
     };
 
-    {
-      XXH3_state_t HashState;
-      XXH3_128bits_reset(&HashState);
-      for (uint32_t i = 0; i < ExactGuestCodeExtents.size(); i += 2) {
-        XXH3_128bits_update(&HashState, GuestCode.data() + ExactGuestCodeExtents[i], ExactGuestCodeExtents[i + 1]);
-      }
-      Header.GuestHash = XXH3_128bits_digest(&HashState);
-    }
-    memcpy(BlobData + HeaderOffset, &Header, sizeof(Header));
     memcpy(BlobData + HostCodeOffset, CompiledCode.BlockBegin, CompiledCode.Size);
 
     // pack and relocate entrypoints
@@ -943,6 +1056,8 @@ namespace DiskCache {
       EntryIdx++;
     }
 
+    fextl::set<uint64_t> DataMasksConsumed;
+
     // pack relocations
     auto* SmallRelocs = reinterpret_cast<BlobSmallRelocation*>(BlobData + SmallRelocsOffset);
     auto* ThunkRelocs = reinterpret_cast<BlobThunkRelocation*>(BlobData + ThunkRelocsOffset);
@@ -979,7 +1094,8 @@ namespace DiskCache {
       }
       case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE:
       case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE:
-      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL: {
+      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_LITERAL:
+      case CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE: {
         // same data for all, relative vs. not and register vs. literal will depend on type on apply
         BlobSmallRelocation SmallReloc = {};
         SmallReloc.Offset = Reloc.Header.Offset;
@@ -989,12 +1105,8 @@ namespace DiskCache {
         SmallReloc.PatchableData.SiteOffset = uint32_t(Reloc.GuestPatchableData.SiteAddress - GuestRIP);
         SmallRelocs[SmallIdx++] = SmallReloc;
 
-        // mark the corresponding data mask consumed - we might not find one if they got removed due to the smc workaround
-        // the hash will just fail on lookup later
-        auto It = DataMaskAddresses.find(Reloc.GuestPatchableData.SiteAddress);
-        if (It != DataMaskAddresses.end()) {
-          DataMaskAddresses.erase(It);
-        }
+        // mark the corresponding data mask consumed
+        DataMasksConsumed.insert(Reloc.GuestPatchableData.SiteAddress);
 
         break;
       }
@@ -1009,18 +1121,108 @@ namespace DiskCache {
       }
     }
 
-    if (!DataMaskAddresses.empty()) {
-      LogMan::Msg::IFmt("DiskCache: DataMask unaccounted for! {:x}", GuestCodeKey);
-      // this would mean we omitted contents in the hash that we're not going to patch, which would be loading corrupt code
-      return false;
+    fextl::vector<uint32_t> ExactGuestCodeExtents;
+    uint64_t CurStartExtent = 0, CurEndExtent = 0;
+    const Frontend::Decoder::DecodedBlocks* LastBlock = nullptr;
+    for (auto& SubBlock : DecodedBlockInfo->Blocks) {
+      if (SubBlock.BlockStatus != Frontend::Decoder::DecodedBlockStatus::SUCCESS) {
+        return false;
+      }
+      if (!CurStartExtent) {
+        CurStartExtent = SubBlock.Entry;
+        CurEndExtent = SubBlock.Entry + SubBlock.Size;
+      } else {
+        LOGMAN_THROW_A_FMT(SubBlock.Entry >= CurEndExtent, "DecodedBlocks not sorted or overlapping?");
+        if (SubBlock.Entry == CurEndExtent) {
+          CurEndExtent = SubBlock.Entry + SubBlock.Size;
+        } else {
+          ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
+          ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
+          CurStartExtent = SubBlock.Entry;
+          CurEndExtent = SubBlock.Entry + SubBlock.Size;
+        }
+      }
+      // split extents according to data masks as well
+      for (auto& Mask : SubBlock.DataMasks) {
+        if (Mask.Type == Frontend::Decoder::DataMaskType::NOP || DataMasksConsumed.find(Mask.FieldAddress) != DataMasksConsumed.end()) {
+          if (Mask.FieldAddress > CurStartExtent) {
+            ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
+            ExactGuestCodeExtents.push_back(Mask.FieldAddress - CurStartExtent);
+          }
+          CurStartExtent = Mask.FieldAddress + Mask.ValueSize;
+        }
+      }
+      LastBlock = &SubBlock;
+    }
+    if (LastBlock && (CurStartExtent != GuestRIP || CurEndExtent != GuestRIP + GuestCode.size())) {
+      ExactGuestCodeExtents.push_back(CurStartExtent - GuestRIP);
+      ExactGuestCodeExtents.push_back(CurEndExtent - CurStartExtent);
     }
 
-    memcpy(BlobData + GuestCodeOffset, GuestCode.data(), GuestCode.size());
+    if (ExactGuestCodeExtents.size() == 0) {
+      ExactGuestCodeExtents.reserve(2);
+      ExactGuestCodeExtents.push_back(0);
+      ExactGuestCodeExtents.push_back(GuestCode.size());
+    }
+
+    uint64_t GuestFootprint = XXH3_64bits(ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
+
+    // if (ExactGuestCodeExtents.size()) {
+    //   LogMan::Msg::IFmt("store! length {:d}", GuestCode.size());
+    //   for(uint32_t i = 0; i < ExactGuestCodeExtents.size(); i+=2 ) {
+    //     LogMan::Msg::IFmt("extent {} {}", ExactGuestCodeExtents[i], ExactGuestCodeExtents[i]+ExactGuestCodeExtents[i+1]);
+    //   }
+    // }
+    {
+      XXH3_state_t HashState;
+      XXH3_128bits_reset(&HashState);
+      for (uint32_t i = 0; i < ExactGuestCodeExtents.size(); i += 2) {
+        XXH3_128bits_update(&HashState, GuestCode.data() + ExactGuestCodeExtents[i], ExactGuestCodeExtents[i + 1]);
+      }
+      Header.GuestHash = XXH3_128bits_digest(&HashState);
+    }
+    memcpy(BlobData + HeaderOffset, &Header, sizeof(Header));
 
     auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, GuestRIP);
     uint64_t LookupKey =
       MakeLookupKey(Thread, GuestCodeKey, RangeInfo.Writable, GuestRIP == CTX->GetMonoBackPatcherBlock().load(std::memory_order_relaxed));
 
+    // blob done, publish to index as in-memory for now, flush to disk below
+    struct IndexEntry NewEntry {nullptr, 0, (uint32_t)TotalSize, Header.GuestSize, Header.GuestHash, BlobRef};
+    NewEntry.GuestExtents = ExactGuestCodeExtents;
+    {
+      std::lock_guard Guard(IndexLock);
+      auto It = Index.find(LookupKey);
+      if (It == Index.end()) {
+        Index.emplace(LookupKey, IndexCacheHead {std::move(NewEntry), GuestFootprint, nullptr});
+      } else {
+        bool Dupe = false;
+        if (It->second.MoreEntries.get()) {
+          for (auto& [Key, Elem] : *It->second.MoreEntries) {
+            if (XXH128_isEqual(Elem.GuestHash, Header.GuestHash)) {
+              Dupe = true;
+              break;
+            }
+          }
+        }
+        if (!Dupe && XXH128_isEqual(It->second.MainEntry.GuestHash, Header.GuestHash)) {
+          Dupe = true;
+        }
+        // could happen if it's seen again while in flight in the store queue
+        if (Dupe) {
+          return true;
+        }
+        if (!It->second.MoreEntries.get()) {
+          It->second.MoreEntries = fextl::make_unique<fextl::multimap<uint64_t, struct IndexEntry>>();
+        } else if (It->second.MoreEntries->size() >= LOOKUP_KEY_MAX_BUCKET_DEPTH) {
+          return false;
+        }
+        It->second.MoreEntries->insert({GuestFootprint, std::move(NewEntry)});
+      }
+    }
+
+    memcpy(BlobData + GuestCodeOffset, GuestCode.data(), GuestCode.size());
+
     MesaFOZ::foz_payload_key Key = {};
     {
       XXH3_state_t HashState;
@@ -1039,8 +1241,15 @@ namespace DiskCache {
     memcpy(IndexBlob.data(), &IndexBlobHeader, sizeof(IndexExtraBlobHeader));
     memcpy(IndexBlob.data() + sizeof(IndexExtraBlobHeader), ExactGuestCodeExtents.data(), ExactGuestCodeExtents.size() * sizeof(uint32_t));
 
+    bool StoreDisk = true;
+
+    if (RWCacheDB->Full()) {
+      StoreDisk = false;
+    }
+
     // hand the rest off to the writer thread
-    Writer->QueueWork(fextl::make_unique<CacheStoreWorkItem>(this, RWCacheDB.get(), Key, LookupKey, std::move(Blob), std::move(IndexBlob)));
+    Writer->QueueWork(fextl::make_unique<CacheStoreWorkItem>(this, RWCacheDB.get(), Key, LookupKey, std::span(BlobData, TotalSize),
+                                                             std::move(IndexBlob), StoreDisk));
     return true;
   }
 

FEXCore/Source/Interface/Core/DiskCache.h

@@ -0,0 +1,258 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include "Interface/Core/JIT/Relocations.h"
+#include "Interface/Core/Frontend.h"
+#include "Interface/Core/CPUBackend.h"
+
+#include <FEXCore/Core/CodeCache.h>
+#include <FEXCore/Core/Context.h>
+#include <FEXCore/Config/Config.h>
+#include <FEXCore/Utils/File.h>
+#include <FEXCore/Utils/WorkQueueThread.h>
+#include <FEXCore/fextl/memory.h>
+#include <FEXCore/fextl/string.h>
+#include <FEXCore/fextl/unordered_set.h>
+#include <FEXCore/fextl/robin_map.h>
+#include <FEXCore/fextl/map.h>
+#include <FEXCore/fextl/vector.h>
+#include <stdint.h>
+#include <mutex>
+#include <optional>
+#include <span>
+#include <xxhash.h>
+
+namespace FEXCore {
+
+namespace Context {
+  class ContextImpl;
+}
+
+namespace DiskCache {
+
+  namespace MesaFOZ {
+
+#define FOSSILIZE_BLOB_HASH_LENGTH 40 /* SHA1 hexadecimal string length */
+
+    struct __attribute__((packed)) foz_payload_key {
+      uint8_t bytes[FOSSILIZE_BLOB_HASH_LENGTH];
+    };
+
+    struct __attribute__((packed)) foz_payload_header {
+      uint32_t payload_size;
+      uint32_t format;
+      uint32_t crc;
+      uint32_t uncompressed_size;
+    };
+
+    struct mesa_index_db_file_entry;
+
+  } // namespace MesaFOZ
+
+  class IndexedDB;
+
+  struct MemoryLRUKey {
+    uint64_t LookupKey;
+    XXH128_hash_t GuestHash;
+    uint64_t GuestFootprint;
+    uint32_t Size;
+  };
+
+  struct IndexEntry {
+    IndexedDB* DB;
+    uint64_t Offset;
+    uint32_t Size;
+    uint32_t GuestSize;
+    XXH128_hash_t GuestHash;
+    fextl::shared_ptr<fextl::vector<uint8_t>> MemoryBlob;
+    fextl::vector<uint32_t> GuestExtents;
+    std::optional<fextl::list<MemoryLRUKey>::iterator> LRUEntry;
+  };
+
+  struct IndexCacheHead {
+    struct IndexEntry MainEntry;
+    uint64_t MainEntryFootprint;
+    fextl::unique_ptr<fextl::multimap<uint64_t, IndexEntry>> MoreEntries; // sorted by guest footprint
+  };
+
+  struct __attribute__((packed)) BlobFixedHeader {
+    uint32_t GuestSize;
+    uint32_t HostSize;
+    uint32_t EntryPointCount;
+    uint32_t SmallRelocCount;
+    uint32_t ThunkRelocCount;
+    XXH128_hash_t GuestHash;
+  };
+
+  // packed struct for types 0, 2 and 3. type 1 is bigger and separate below
+  struct __attribute__((packed)) BlobSmallRelocation {
+    uint32_t Offset;
+    uint8_t Type;
+    union {
+      struct __attribute__((packed)) {
+        uint32_t Symbol;
+      } Named;
+      struct __attribute__((packed)) {
+        uint64_t GuestRIP;
+      } RIPLiteral;
+      struct __attribute__((packed)) {
+        uint8_t RegisterIndex;
+        uint64_t GuestRIP;
+      } RIPMove;
+      struct __attribute__((packed)) {
+        uint8_t RegisterIndex;
+        uint8_t ValueSize;
+        uint32_t SiteOffset;
+      } PatchableData;
+    };
+  };
+
+  // type 1, implicit
+  struct __attribute__((packed)) BlobThunkRelocation {
+    uint32_t Offset;
+    uint8_t RegisterIndex;
+    uint8_t SymbolHash[32]; // sha256sum in the real RelocNamedThunkMove
+  };
+
+  struct CodeHitData {
+    fextl::vector<uint8_t> Blob;
+    std::span<uint8_t> HostCode;
+    std::span<const uint64_t> GuestPages;
+    std::span<uint64_t> EntryPointRIPs;
+    std::span<const uint32_t> EntryPointHostOffsets;
+
+    // the spans above point to memory owned by the Blob vec, so it's important this can't be copied
+    CodeHitData() = default;
+    CodeHitData(CodeHitData&&) = default;
+    CodeHitData& operator=(CodeHitData&&) = default;
+    CodeHitData(const CodeHitData&) = delete;
+    CodeHitData& operator=(const CodeHitData&) = delete;
+  };
+
+  using Index = fextl::robin_map<uint64_t, IndexCacheHead>;
+
+  class FOZFile {
+  public:
+    bool Open(const fextl::string& CacheFileName, bool ReadOnly);
+    bool Lock(uint32_t TimeoutMS) {
+      if (!FD) {
+        return false;
+      }
+      return FD->Lock(TimeoutMS);
+    }
+    bool Unlock() {
+      if (!FD) {
+        return false;
+      }
+      return FD->Unlock();
+    }
+    File::File::FileHandleType GetHandle() {
+      return FD ? FD->GetHandle() : (File::File::FileHandleType)-1;
+    }
+    ssize_t Size();
+    bool ReadAll(fextl::vector<uint8_t>& Out); // from first blob
+    bool ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
+    bool WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset);
+
+  private:
+    static constexpr uint32_t OPEN_LOCK_TIMEOUT_MS = 100;
+
+    fextl::string FileName;
+    fextl::unique_ptr<File::File> FD;
+    bool ReadOnly = false;
+  };
+
+  class IndexedDB {
+  public:
+    bool Open(const fextl::string& CacheDBName, bool ReadOnly);
+    void PopulateIndex(Index& CacheIndex, bool& FoundMetadata);
+    bool ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
+    bool StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob,
+                        MesaFOZ::mesa_index_db_file_entry& IndexEntry, std::span<const uint8_t> IndexBlob);
+    bool Full() const {
+      return MaxSizeReached;
+    }
+
+  private:
+    // stores run on the Writer, so returning quick isn't as important
+    static constexpr uint32_t STORE_LOCK_TIMEOUT_MS = 1000;
+    static constexpr uint64_t BIG_MAPPING_SIZE = 1ULL << 33;
+
+    FOZFile CacheFOZ;
+    uint8_t* CacheFileMapping = nullptr;
+    std::atomic<uint64_t> CacheFileSize;
+    FOZFile IndexFOZ;
+    bool ReadOnly = false;
+    bool MaxSizeReached = false;
+
+    FEX_CONFIG_OPT(MaxFileSize, DISKCACHEMAXFILESIZE);
+  };
+
+  class DiskCache {
+  public:
+    void Init(FEXCore::Context::ContextImpl* CTX);
+
+    std::optional<CodeHitData> Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP,
+                                      std::optional<uint64_t>& GuestCodeKey);
+    void Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
+                  std::optional<ExecutableFileSectionInfo> Region);
+    bool Store(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP, uint64_t GuestCodeKey,
+               std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
+               std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo);
+
+    bool IsWritingDiskCache() const {
+      return WritingDiskCache;
+    }
+    bool IsReadingDiskCache() const {
+      return ReadingDiskCache;
+    }
+    bool IsValidating() const {
+      return Validation;
+    }
+
+  private:
+    bool OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly);
+    uint64_t MakeLookupKey(Core::InternalThreadState* Thread, const uint64_t ModuleOffset, bool Writable, bool MonoBackpatcher);
+    IndexEntry* LookupLocked(const uint64_t LookupKey, const XXH128_hash_t& GuestHash, const uint64_t GuestFootprint);
+
+    bool ReadingDiskCache {};
+    bool WritingDiskCache {};
+    FEXCore::Context::ContextImpl* CTX;
+    XXH128_hash_t BucketHash;
+    fextl::vector<fextl::unique_ptr<IndexedDB>> ROCacheDBs;
+    fextl::unique_ptr<IndexedDB> RWCacheDB;
+    Index Index;
+    std::mutex IndexLock;
+    bool FoundMetadata = false;
+    struct CacheStoreWorkItem;
+
+    struct PruneMemoryLRUWorkItem;
+    std::atomic<uint64_t> MemoryLRUCurrentSize {};
+    std::mutex MemoryLRULock;
+    fextl::list<MemoryLRUKey> MemoryLRU;
+
+    // the Writer holds references to all this stuff above and needs to be last
+    fextl::unique_ptr<WorkQueueThread> Writer;
+
+    FEX_CONFIG_OPT(EnableDiskCache, DISKCACHE);
+    FEX_CONFIG_OPT(Validation, DISKCACHEVALIDATION);
+    FEX_CONFIG_OPT(MapDiskCacheFiles, DISKCACHEFILEMAPPING);
+    FEX_CONFIG_OPT(RelocationFilter, DISKCACHERELOCATIONFILTER);
+    FEX_CONFIG_OPT(AnonCaching, DISKCACHEANONCACHING);
+    FEX_CONFIG_OPT(BasePathOverride, DISKCACHEPATH);
+    FEX_CONFIG_OPT(RODBNames, DISKCACHERODBNAMES);
+    FEX_CONFIG_OPT(MemoryLRUMaxSize, DISKCACHEMEMORYSIZE);
+
+    uint64_t MemoryLRUEvictThreshold = MemoryLRUMaxSize / 25;
+  };
+
+  static constexpr uint16_t AnonPrefixGuestBytes = 64;
+  // The current version of the diskcache.
+  // This must be changed any time codegen changes occur!
+  // Be aware of the impact of changing this frequently!
+  static constexpr uint16_t FormatVersion = 43;
+
+  static constexpr uint32_t LOOKUP_KEY_MAX_BUCKET_DEPTH = 500;
+
+} // namespace DiskCache
+
+} // namespace FEXCore

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.cpp

@@ -276,8 +276,15 @@ void Dispatcher::EmitDispatcher() {
     str(TMP2, STATE, offsetof(FEXCore::Core::CPUState, DeferredSignalRefCount));
 
     // Trigger segfault if any deferred signals are pending
-    strb(ARMEmitter::XReg::zr, STATE,
-         offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState));
+    constexpr size_t InterruptPageOffset =
+      offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState);
+    if constexpr (InterruptPageOffset <= 32760) {
+      str(ARMEmitter::XReg::zr, STATE, InterruptPageOffset);
+    } else {
+      // Need to use vector 128-bit store for this range.
+      // Doesn't matter which register we use to store.
+      str(ARMEmitter::QReg::q0, STATE, InterruptPageOffset);
+    }
 #endif
   };
 
@@ -410,6 +417,7 @@ void Dispatcher::EmitDispatcher() {
     brk(0);
   }
 
+  ARMEmitter::BackwardLabel GuestSIGSEGVLabelPostSpill;
   {
     // Guest Overflow handler
     // Needs to be distinct from the SignalHandlerReturnAddress
@@ -417,6 +425,8 @@ void Dispatcher::EmitDispatcher() {
 
     SpillStaticRegs(TMP1);
 
+    (void)Bind(&GuestSIGSEGVLabelPostSpill);
+
     // hlt/udf = SIGILL
     // brk = SIGTRAP
     // ??? = SIGSEGV
@@ -432,6 +442,26 @@ void Dispatcher::EmitDispatcher() {
     }
   }
 
+  {
+    // Faultless synchronous signal.
+    // Some games abuse SIGILL/SIGTRAP/SIGSEGV heavily and the fault overhead is visible.
+    // Instead of double faulting, clean up state and jump to the frontend to handle this.
+    // `FEXCore::Core::CpuStateFrame::SynchronousFaultData` needs to be setup at this point.
+    GuestSignal_SynchronousFault = GetCursorAddress<uint64_t>();
+
+    SpillStaticRegs(TMP1);
+
+    ldr(ARMEmitter::XReg::x0, &l_CTX);
+    mov(ARMEmitter::XReg::x1, STATE);
+    ldr(ARMEmitter::XReg::x2, STATE_PTR(CpuStateFrame, Pointers.SynchronousExceptionHandler));
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
+    } else {
+      br(ARMEmitter::Reg::r2);
+    }
+  }
+
   {
     ThreadPauseHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
     SpillStaticRegs(TMP1);
@@ -507,43 +537,167 @@ void Dispatcher::EmitDispatcher() {
     (void)b(&LoopTop);
   }
 
-  auto EmitLongALUOpHandler = [&](auto R, auto Offset) {
-    auto Address = GetCursorAddress<uint64_t>();
+  auto EmitL1Lookup = [&](int rip_offset = 0) {
+    // L1 lookup after syscall. Mirrors the implementation in `ExitFunction`.
+    // Most syscall instructions just return to the following RIP, so it is highly likely to be in the L1 in tight loops.
+    ldr(TMP3, STATE_PTR(CpuStateFrame, State.rip));
+    ldp<ARMEmitter::IndexType::OFFSET>(TMP1, TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, State.L1Pointer));
 
-    PushDynamicRegs(TMP4);
-    SpillStaticRegs(TMP4);
-
-    if (!TMP_ABIARGS) {
-      mov(ARMEmitter::XReg::x0, TMP1);
-      mov(ARMEmitter::XReg::x1, TMP2);
-      mov(ARMEmitter::XReg::x2, TMP3);
+    if (rip_offset) {
+      add(ARMEmitter::Size::i64Bit, TMP3, TMP3, rip_offset);
     }
 
-    ldr(ARMEmitter::XReg::x3, R, Offset);
+    // Calculate (tmp1 + ((ripreg & L1_ENTRIES_MASK) << 4)) for the address
+    // L1Mask is pre-shifted.
+    and_(ARMEmitter::Size::i64Bit, TMP2, TMP2, TMP3, ARMEmitter::ShiftType::LSL, FEXCore::ilog2(sizeof(LookupCache::LookupCacheEntry)));
+    add(TMP1, TMP1, TMP2);
+
+    ldp<ARMEmitter::IndexType::OFFSET>(TMP2, TMP1, TMP1, 0);
+
+    sub(TMP1, TMP1, TMP3);
+    (void)cbnz(ARMEmitter::Size::i64Bit, TMP1, &LoopTop);
+    br(TMP2);
+  };
+
+  {
+    // All dynamic and static registers are spilled coming in to this handler.
+    // It's also the end of block and RIP might have changed, so we jump directly to the top of the loop.
+    ThreadDispatchSyscallHandler = GetCursorAddress<uint64_t>();
+
+    // Store in the state that we are in a syscall
+    // 16bit LoadConstant to be a single instruction
+    // This gives the signal handler a value to check to see if we are in a syscall at all
+    LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, 0xFFFF);
+    str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
+
+    ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerObj));
+    ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerFunc));
+    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, STATE.R());
+
     if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-      GenerateIndirectRuntimeCall<__uint128_t, uint64_t, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
+      GenerateIndirectRuntimeCall<uint64_t, void*, void*, void*>(ARMEmitter::Reg::r3);
     } else {
       blr(ARMEmitter::Reg::r3);
     }
 
-    // Result is now in x0, x1
+    // Fix the stack and any values that were stepped on
+    // Syscall result is in any static register that the frontend desired.
+    FillStaticRegs({
+      .OptionalReg = ARMEmitter::Reg::r1,
+      .OptionalReg2 = ARMEmitter::Reg::r2,
+    });
+
+    // Now the registers we've spilled are back in their original host registers
+    // We can safely claim we are no longer in a syscall
+    str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
+
+    EmitL1Lookup();
+  }
+
+  {
+    // All dynamic and static registers are spilled coming in to this handler.
+    // It's also the end of block and RIP might have changed, so we jump directly to the top of the loop.
+    ThreadDispatchCPUIDHandler = GetCursorAddress<uint64_t>();
+
+    // CPUID behaves like an ISB.
+    isb();
+
+    auto RAXReg = GetX86GPRMapping(FEXCore::X86State::REG_RAX);
+    auto RBXReg = GetX86GPRMapping(FEXCore::X86State::REG_RBX);
+    auto RCXReg = GetX86GPRMapping(FEXCore::X86State::REG_RCX);
+    auto RDXReg = GetX86GPRMapping(FEXCore::X86State::REG_RDX);
+
+    ARMEmitter::ForwardLabel SetupCPUIDFault {};
+    ldr(TMP2, STATE, offsetof(FEXCore::Core::CPUState, CoreFlags));
+    (void)tbz(TMP2, FEXCore::ilog2(FEXCore::ToUnderlying(FEXCore::Core::CoreFlagsEnum::CPUID_ENABLED)), &SetupCPUIDFault);
+
+    // TMP1 passes the instruction size. Usually only 2-bytes but prefixes can make it 15.
+    ldr(TMP2, STATE_PTR(CpuStateFrame, State.rip));
+    add(TMP2, TMP2, TMP1);
+    str(TMP2, STATE_PTR(CpuStateFrame, State.rip));
+
+    mov(ARMEmitter::XReg::x1, RAXReg.X());
+    mov(ARMEmitter::XReg::x2, RCXReg.X());
+
+    // x0 = CPUID Handler
+    // x1 = CPUID Function (RAX)
+    // x2 = CPUID Leaf (RCX)
+    ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));
+    ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDFunction));
+
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
+    } else {
+      blr(ARMEmitter::Reg::r3);
+    }
+
+    // Results are in x0, x1
+    auto Res0 = ARMEmitter::Reg::r0;
+    auto Res1 = ARMEmitter::Reg::r1;
+
     if (!TMP_ABIARGS) {
-      mov(TMP1, ARMEmitter::XReg::x0);
-      mov(TMP2, ARMEmitter::XReg::x1);
+      // SRA Overlaps, need to move.
+      mov(ARMEmitter::Size::i64Bit, TMP1, Res0);
+      mov(ARMEmitter::Size::i64Bit, TMP2, Res1);
+      Res0 = TMP1;
+      Res1 = TMP2;
     }
 
     FillStaticRegs();
 
-    // Fix the stack and any values that were stepped on
-    PopDynamicRegs();
+    // Results need to be moved to 32-bit EAX, ECX, EBX, EDX.
+    mov(ARMEmitter::Size::i32Bit, RAXReg, Res0);
+    mov(ARMEmitter::Size::i32Bit, RCXReg, Res1);
+    ubfx(ARMEmitter::Size::i64Bit, RBXReg, Res0, 32, 32);
+    ubfx(ARMEmitter::Size::i64Bit, RDXReg, Res1, 32, 32);
 
-    // Go back to our code block
-    ret();
-    return Address;
-  };
+    EmitL1Lookup();
 
-  LUDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LUDIV));
-  LDIVHandlerAddress = EmitLongALUOpHandler(STATE_PTR(CpuStateFrame, Pointers.LDIV));
+    // CPUID fault handling
+    (void)Bind(&SetupCPUIDFault);
+
+    constexpr Core::CpuStateFrame::SynchronousFaultDataStruct State = {
+      .FaultToTopAndGeneratedException = 1,
+      .Signal = Core::FAULT_SIGSEGV,
+      .TrapNo = X86State::X86_TRAPNO_GP,
+      .si_code = 0x80,
+      .err_code = 0,
+      .FaultAddress = 0, // Comes from register.
+    };
+
+    uint64_t Constant {};
+    memcpy(&Constant, &State, sizeof(uint64_t));
+
+    LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
+
+    str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
+    ldr(TMP2, STATE_PTR(CpuStateFrame, State.rip));
+    str(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
+
+    (void)b(&GuestSIGSEGVLabelPostSpill);
+  }
+
+  {
+    // All dynamic and static registers are spilled coming in to this handler.
+    // It's also the end of block and RIP might have changed, so we jump directly to the top of the loop.
+    ThreadDispatchRemoveCodeEntry = GetCursorAddress<uint64_t>();
+
+    // Arguments are already in x0, x1. Just jump to the handler.
+    ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadRemoveCodeEntryFromJIT));
+    if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+      GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
+    } else {
+      blr(ARMEmitter::Reg::r2);
+    }
+
+    FillStaticRegs({
+      .OptionalReg = ARMEmitter::Reg::r1,
+      .OptionalReg2 = ARMEmitter::Reg::r2,
+    });
+
+    // Now go back to the regular dispatcher loop
+    (void)b(&LoopTop);
+  }
 
   EmitF64Sin();
   EmitF64Cos();
@@ -2620,13 +2774,15 @@ void Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState* Thread)
     Ptrs.ExitFunctionLinker = ExitFunctionLinkerAddress;
     Ptrs.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddressSpillSRA;
     Ptrs.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddressSpillSRA;
+    Ptrs.ThreadDispatchSyscallHandler = ThreadDispatchSyscallHandler;
+    Ptrs.ThreadDispatchCPUIDHandler = ThreadDispatchCPUIDHandler;
+    Ptrs.ThreadDispatchRemoveCodeEntry = ThreadDispatchRemoveCodeEntry;
     Ptrs.GuestSignal_SIGILL = GuestSignal_SIGILL;
     Ptrs.GuestSignal_SIGTRAP = GuestSignal_SIGTRAP;
     Ptrs.GuestSignal_SIGSEGV = GuestSignal_SIGSEGV;
+    Ptrs.GuestSignal_SynchronousFault = GuestSignal_SynchronousFault;
     Ptrs.SignalReturnHandler = SignalHandlerReturnAddress;
     Ptrs.SignalReturnHandlerRT = SignalHandlerReturnAddressRT;
-    Ptrs.LUDIVHandler = LUDIVHandlerAddress;
-    Ptrs.LDIVHandler = LDIVHandlerAddress;
     Ptrs.F64SinHandler = F64SinHandlerAddress;
     Ptrs.F64CosHandler = F64CosHandlerAddress;
     Ptrs.F64TanHandler = F64TanHandlerAddress;

FEXCore/Source/Interface/Core/Dispatcher/Dispatcher.h

@@ -81,12 +81,16 @@ private:
   uint64_t AbsoluteLoopTopAddressEnterECFillSRA {};
   uint64_t ThreadPauseHandlerAddress {};
   uint64_t ThreadPauseHandlerAddressSpillSRA {};
+  uint64_t ThreadDispatchSyscallHandler {};
+  uint64_t ThreadDispatchCPUIDHandler {};
+  uint64_t ThreadDispatchRemoveCodeEntry {};
   uint64_t ExitFunctionLinkerAddress {};
   uint64_t SignalHandlerReturnAddress {};
   uint64_t SignalHandlerReturnAddressRT {};
   uint64_t GuestSignal_SIGILL {};
   uint64_t GuestSignal_SIGTRAP {};
   uint64_t GuestSignal_SIGSEGV {};
+  uint64_t GuestSignal_SynchronousFault {};
 
   uint64_t PauseReturnInstruction {};
   std::array<uint64_t, FallbackABI::FABI_UNKNOWN> ABIPointers {};
@@ -95,10 +99,6 @@ private:
   uint64_t Start {};
   uint64_t End {};
 
-  // Long division helpers
-  uint64_t LUDIVHandlerAddress {};
-  uint64_t LDIVHandlerAddress {};
-
   // F64 reduced-precision shared handlers
   uint64_t F64SinHandlerAddress {};
   uint64_t F64CosHandlerAddress {};

FEXCore/Source/Interface/Core/Frontend.cpp

@@ -8,6 +8,7 @@ $end_info$
 
 #include "Interface/Context/Context.h"
 #include "Interface/Core/Frontend.h"
+#include "Interface/Core/OpcodeDispatcher.h"
 #include "Interface/Core/X86Tables/X86Tables.h"
 #include "Interface/Core/LookupCache.h"
 
@@ -78,39 +79,56 @@ Decoder::Decoder(FEXCore::Core::InternalThreadState* Thread)
     X87Table = &FEXCore::X86Tables::X87F80Ops;
   }
 
-  if (CTX->HostFeatures.SupportsAVX && CTX->HostFeatures.SupportsSVE256) {
+  if (CTX->HostFeatures.SupportsAVX() && CTX->HostFeatures.SupportsSVE256()) {
     VEXTable = &FEXCore::X86Tables::VEXTableOps;
     VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps;
-  } else if (CTX->HostFeatures.SupportsAVX) {
+  } else if (CTX->HostFeatures.SupportsAVX()) {
     VEXTable = &FEXCore::X86Tables::VEXTableOps_AVX128;
     VEXTableGroup = &FEXCore::X86Tables::VEXTableGroupOps_AVX128;
   }
 }
 
-bool Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {
+int32_t Decoder::CheckRangeExecutable(uint64_t Address, uint64_t Size) {
   // Check for wraparound
   if (Address + Size < Address) {
-    return false;
+    // Say that any bytes that wrapped around are unreadable
+    return -(Address + Size);
   }
 
-  while (Address < ExecutableRangeBase || Address + Size > ExecutableRangeEnd) {
+  if (Thread->CurrentFrame->State.Dr7) [[unlikely]] {
+    // If Dr7 has anything enabled then check it.
+    for (size_t i = 0; i < Core::CPUState::NUM_DEBUG_ADDRESSES; ++i) {
+      if (Core::CPUState::IsInstructionBPEnabled(i, Thread->CurrentFrame->State.Dr7)) {
+        size_t DrSize = Core::CPUState::BreakpointSize(i, Thread->CurrentFrame->State.Dr7);
+        if (Address >= Thread->CurrentFrame->State.Dr[i] && (Address + Size - 1) < (Thread->CurrentFrame->State.Dr[i] + DrSize)) {
+          HitBreakpoint = i;
+          return false;
+        }
+      }
+    }
+  }
+
+  uint64_t RemainingSize = Size;
+  while (Address < ExecutableRangeBase || Address + RemainingSize > ExecutableRangeEnd) {
     auto RangeInfo = CTX->SyscallHandler->QueryGuestExecutableRange(Thread, Address);
     ExecutableRangeBase = RangeInfo.Base;
     ExecutableRangeEnd = RangeInfo.Base + RangeInfo.Size;
     ExecutableRangeWritable = RangeInfo.Writable;
 
     if (RangeInfo.Size == 0) {
-      return false;
+      // Return the number of bytes that couldn't be decoded.
+      return -(Size - RemainingSize);
     }
 
     uint64_t RangeRemainingSize = ExecutableRangeEnd - Address;
-    if (Size > RangeRemainingSize) {
-      Size -= RangeRemainingSize;
+    if (RemainingSize > RangeRemainingSize) {
+      RemainingSize -= RangeRemainingSize;
       Address += RangeRemainingSize;
     }
   }
 
-  return true;
+  // Say all bytes are readable.
+  return Size;
 }
 
 uint8_t Decoder::ReadByte() {
@@ -128,7 +146,7 @@ uint8_t Decoder::ReadByte() {
 
 std::optional<uint8_t> Decoder::PeekByte(uint8_t Offset) {
   uint64_t ByteAddress = reinterpret_cast<uint64_t>(InstStream.InstStream + InstructionSize + Offset);
-  if (CheckRangeExecutable(ByteAddress, 1)) {
+  if (CheckRangeExecutable(ByteAddress, 1) == 1) {
     return InstStream.AdjustedInstStream[InstructionSize + Offset];
   } else {
     return std::nullopt;
@@ -142,7 +160,8 @@ std::pair<uint64_t, bool> Decoder::ReadData(uint8_t Size) {
   uint64_t Address = reinterpret_cast<uint64_t>(InstStream.InstStream + InstructionSize);
   LastFieldReadOffset = (uint8_t)InstructionSize;
   LastFieldReadSize = Size;
-  if (CheckRangeExecutable(Address, Size)) {
+  auto ReadableSize = CheckRangeExecutable(Address, Size);
+  if (ReadableSize == Size) {
     std::memcpy(&Res, &InstStream.AdjustedInstStream[InstructionSize], Size);
   } else {
     HitNonExecutableRange = true;
@@ -150,13 +169,14 @@ std::pair<uint64_t, bool> Decoder::ReadData(uint8_t Size) {
     Res = 0;
   }
 
-
+  // Skip the number of decoded bytes regardless to get instruction size.
+  // Even partially decoded instruction.
 #if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
-  for (size_t i = 0; i < Size; ++i) {
+  for (size_t i = 0; i < std::abs(ReadableSize); ++i) {
     ReadByte();
   }
 #else
-  SkipBytes(Size);
+  SkipBytes(std::abs(ReadableSize));
 #endif
 
   if (Relocations) {
@@ -216,6 +236,7 @@ void Decoder::DecodeModRM_16(X86Tables::DecodedOperand* Operand, X86Tables::ModR
   Operand->Type = DecodedOperand::OpType::SIB;
   Operand->Data.SIB.Scale = 1;
   Operand->Data.SIB.Offset = Literal;
+  Operand->Data.SIB.PatchableDisp = false;
 
   // Only called when ModRM.mod != 0b11
   struct Encodings {
@@ -291,6 +312,7 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR
     // SIB
     Operand->Type = DecodedOperand::OpType::SIB;
     Operand->Data.SIB.Scale = 1 << SIB.scale;
+    Operand->Data.SIB.PatchableDisp = false;
 
     // The invalid encoding types are described at Table 1-12. "promoted nsigned is always non-zero"
     {
@@ -329,6 +351,7 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR
       auto [Literal, IsRelocation] = ReadData(4);
       Operand->Type = IsRelocation ? DecodedOperand::OpType::RIPRelativeRelocation : DecodedOperand::OpType::RIPRelative;
       Operand->Data.RIPLiteral.Value = Literal;
+      Operand->Data.RIPLiteral.PatchableDisp = false;
     } else {
       // Register-direct addressing
       Operand->Type = DecodedOperand::OpType::GPRDirect;
@@ -344,6 +367,7 @@ void Decoder::DecodeModRM_64(X86Tables::DecodedOperand* Operand, X86Tables::ModR
     Operand->Type = IsRelocation ? DecodedOperand::OpType::GPRIndirectRelocation : DecodedOperand::OpType::GPRIndirect;
     Operand->Data.GPRIndirect.GPR = MapModRMToReg(DecodeInst->Flags & DecodeFlags::FLAG_REX_XGPR_B ? 1 : 0, ModRM.rm, false, false, false, false);
     Operand->Data.GPRIndirect.Displacement = Literal;
+    Operand->Data.GPRIndirect.PatchableDisp = false;
   }
 }
 
@@ -496,15 +520,7 @@ Decoder::DecodedBlockStatus Decoder::NormalOp(const FEXCore::X86Tables::X86InstI
 
   auto* CurrentDest = &DecodeInst->Dest;
 
-  if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ||
-      HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RDX)) {
-    // Some instructions hardcode their destination as RAX
-    CurrentDest->Type = DecodedOperand::OpType::GPR;
-    CurrentDest->Data.GPR.HighBits = false;
-    CurrentDest->Data.GPR.GPR =
-      HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_DST_RAX) ? FEXCore::X86State::REG_RAX : FEXCore::X86State::REG_RDX;
-    CurrentDest = &DecodeInst->Src[0];
-  } else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {
+  if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_REX_IN_BYTE)) {
     LOGMAN_THROW_A_FMT(!HasMODRM, "This instruction shouldn't have ModRM!");
 
     // If the REX is in the byte that means the lower nibble of the OP contains the destination GPR
@@ -621,18 +637,6 @@ Decoder::DecodedBlockStatus Decoder::NormalOp(const FEXCore::X86Tables::X86InstI
     ++CurrentSrc;
   }
 
-  if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RAX)) {
-    DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
-    DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;
-    DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RAX;
-    ++CurrentSrc;
-  } else if (HAS_NON_XMM_SUBFLAG(Info->Flags, FEXCore::X86Tables::InstFlags::FLAGS_SF_SRC_RCX)) {
-    DecodeInst->Src[CurrentSrc].Type = DecodedOperand::OpType::GPR;
-    DecodeInst->Src[CurrentSrc].Data.GPR.HighBits = false;
-    DecodeInst->Src[CurrentSrc].Data.GPR.GPR = FEXCore::X86State::REG_RCX;
-    ++CurrentSrc;
-  }
-
   if (VEXOperand == FEXCore::X86Tables::InstFlags::FLAGS_VEX_DST) {
     CurrentDest->Type = DecodedOperand::OpType::GPR;
     CurrentDest->Data.GPR.HighBits = false;
@@ -957,23 +961,24 @@ Decoder::DecodedBlockStatus Decoder::DecodeInstructionImpl(uint64_t PC) {
           // If any of these three prefixes are used then it falls down the subtable
           // Additionally: If you hit repeat of differnt prefixes then only the LAST one before this one works for subtable selection
 
-          bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;
-          bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
+          const bool NoOverlay = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY) != 0;
+          const bool NoOverlay66 = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_OVERLAY66) != 0;
+          const bool NoPrefixOrder = (FEXCore::X86Tables::SecondBaseOps[EscapeOp].Flags & InstFlags::FLAGS_NO_PREFIX_ORDER) != 0;
 
           DecodeREXIfValid(-2);
           if (NoOverlay) { // This section of the table ignores prefix extention
             return NormalOpHeader(&FEXCore::X86Tables::SecondBaseOps[EscapeOp], EscapeOp);
-          } else if (LastEscapePrefix == 0xF3) { // REP
+          } else if (LastEscapePrefix == 0xF3 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_REP_PREFIX)) { // REP
             // Remove prefix so it doesn't effect calculations.
             // This is only an escape prefix rather tan modifier now
             DecodeInst->Flags &= ~DecodeFlags::FLAG_REP_PREFIX;
             return NormalOpHeader(&FEXCore::X86Tables::RepModOps[EscapeOp], EscapeOp);
-          } else if (LastEscapePrefix == 0xF2) { // REPNE
+          } else if (LastEscapePrefix == 0xF2 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_REPNE_PREFIX)) { // REPNE
             // Remove prefix so it doesn't effect calculations.
             // This is only an escape prefix rather tan modifier now
             DecodeInst->Flags &= ~DecodeFlags::FLAG_REPNE_PREFIX;
             return NormalOpHeader(&FEXCore::X86Tables::RepNEModOps[EscapeOp], EscapeOp);
-          } else if (LastEscapePrefix == 0x66 && !NoOverlay66) { // Operand Size
+          } else if ((LastEscapePrefix == 0x66 || (NoPrefixOrder && DecodeInst->Flags & DecodeFlags::FLAG_OPERAND_SIZE)) && !NoOverlay66) { // Operand Size
             // Remove prefix so it doesn't effect calculations.
             // This is only an escape prefix rather tan modifier now
             DecodeInst->Flags &= ~DecodeFlags::FLAG_OPERAND_SIZE;
@@ -1093,21 +1098,27 @@ Decoder::DecodedBlockStatus Decoder::DecodeInstruction(uint64_t PC) {
   // Will be set if DecodeInstructionImpl tries to read non-executable memory
   HitNonExecutableRange = false;
   HitBadRelocation = false;
+  HitBreakpoint = ~0U;
   auto ErrorDuringDecoding = DecodeInstructionImpl(PC);
 
   if (ErrorDuringDecoding != DecodedBlockStatus::SUCCESS || HitNonExecutableRange || HitBadRelocation) [[unlikely]] {
     // Put an invalid instruction in the stream so the core can raise SIGILL if hit
     // Error while decoding instruction. We don't know the table or instruction size
-    const auto InstSize = DecodeInst->InstSize;
     DecodeInst->TableInfo = nullptr;
-    DecodeInst->InstSize = 0;
+
+    if (HitBreakpoint != ~0U) {
+      return static_cast<Decoder::DecodedBlockStatus>(FEXCore::ToUnderlying(DecodedBlockStatus::BREAKPOINT_0) + HitBreakpoint);
+    }
 
     // A decode error can be caused by substituting zero for an inaccessible
     // instruction byte, so the instruction fetch fault takes priority.
     if (HitNonExecutableRange) {
-      return InstSize ? DecodedBlockStatus::PARTIAL_DECODE_INST : DecodedBlockStatus::NOEXEC_INST;
+      return DecodeInst->InstSize ? DecodedBlockStatus::PARTIAL_DECODE_INST : DecodedBlockStatus::NOEXEC_INST;
     }
 
+    // Keep InstSize around for PARTIAL_DECODE_INST so we can get the offset where it partial decoded.
+    DecodeInst->InstSize = 0;
+
     if (HitBadRelocation) {
       return DecodedBlockStatus::BAD_RELOCATION;
     }
@@ -1367,8 +1378,8 @@ const Decoder::DecodeStream Decoder::AdjustAddrForSpecialRegion(const uint8_t* _
   constexpr uint64_t VSyscall_Base = 0xFFFF'FFFF'FF60'0000ULL;
   constexpr uint64_t VSyscall_End = VSyscall_Base + 0x1000;
 
-  if (BlockInfo.Is64BitMode && CTX->HostFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux && RIP >= VSyscall_Base &&
-      RIP < VSyscall_End) {
+  if (BlockInfo.Is64BitMode && CTX->HostFeatures.ProcessFeatures.HostType == FEXCore::HostFeatures::HostTypeEnum::Linux &&
+      RIP >= VSyscall_Base && RIP < VSyscall_End) {
     // VSyscall
     // This doesn't exist on AArch64 and on x86_64 hosts this is emulated with faults to a region mapped with --xp permissions
     // Offset     0: vgettimeofday
@@ -1396,45 +1407,126 @@ bool Decoder::CheckIfCacheable(FEXCore::Core::InternalThreadState& Thread, const
 }
 
 void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) {
+  FEXCore::X86Tables::DecodedOperand* LiteralToPatch = nullptr;
+
+  // cmp *, imm8 - seen varying in mono jitted code
+  {
+    FEXCore::X86Tables::ModRMDecoded ModRM;
+    ModRM.Hex = DecodeInst->ModRM;
+    if ((DecodeInst->OPRaw == 0x80 || DecodeInst->OPRaw == 0x83) && ModRM.reg == 7 && LastFieldReadSize == 1) {
+      for (auto& Src : DecodeInst->Src) {
+        if (Src.IsLiteral()) {
+          LiteralToPatch = &Src;
+          break;
+        }
+      }
+      if (LiteralToPatch && LiteralToPatch->Literal() != 0) {
+        Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, DataMaskType::MOV, LastFieldReadSize});
+
+        LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable;
+        LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset;
+        LiteralToPatch->Data.LiteralPatchable.Width = LastFieldReadSize;
+      }
+    }
+  }
+
   if (LastFieldReadSize < 4) {
     return;
   }
 
-  FEXCore::X86Tables::DecodedOperand* LiteralToPatch = nullptr;
-  DataMaskType Type;
+  DataMaskType Type = DataMaskType::NONE;
 
-  // mov reg,imm
-  if (DecodeInst->OP >= 0xB8 && DecodeInst->OP <= 0xBF) {
+  // imm32 or imm64 at the end type instructions
+  if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_LITERAL_PATCHABLE) {
     for (auto& Src : DecodeInst->Src) {
       if (Src.IsLiteral()) {
         LiteralToPatch = &Src;
         break;
       }
     }
+    if (DecodeInst->Dest.IsLiteral()) {
+      LiteralToPatch = &DecodeInst->Dest;
+    }
 
-    // we could filter to certain high values that are more likely to be pointers/etc?
-    // const uint64_t Value = Lit->Data.Literal.Value;
-    // if (LiteralToPatch && Value < 0x1000000ULL) {
-    //   LiteralToPatch = nullptr;
-    // }
-    Type = DataMaskType::MOV;
+    // heuristic: if it's a small value, assume it's more likely to be part of the code around it
+    bool IsMOV = (DecodeInst->OPRaw >= 0xB8 && DecodeInst->OPRaw <= 0xBF) || DecodeInst->OPRaw == 0xC7;
+    if (LiteralToPatch && IsMOV && LiteralToPatch->Literal() < 0x10000ULL) {
+      LiteralToPatch = nullptr;
+    }
+    if (LiteralToPatch && LiteralToPatch->Literal() != 0) {
+      Type = DataMaskType::MOV;
+    }
   }
 
+  // anything that has a patchable disp32
+  bool TryDisp = DecodeInst->Flags & X86Tables::DecodeFlags::FLAG_DECODED_MODRM;
+  if (TryDisp) {
+    FEXCore::X86Tables::ModRMDecoded ModRM;
+    ModRM.Hex = DecodeInst->ModRM;
+    {
+      // todo we could handle both imm + disp with more load tracking
+      bool FoundLiteral = false;
+      FEXCore::X86Tables::DecodedOperand* OpToPatch = nullptr;
+      for (auto& Src : DecodeInst->Src) {
+        if (Src.IsLiteral()) {
+          FoundLiteral = true;
+          break;
+        }
+        if (Src.IsRIPRelative() || Src.IsGPRIndirect() || Src.IsSIB()) {
+          OpToPatch = &Src;
+        }
+      }
+      if (DecodeInst->Dest.IsLiteral()) {
+        FoundLiteral = true;
+      }
+      if (DecodeInst->Dest.IsRIPRelative() || DecodeInst->Dest.IsGPRIndirect() || DecodeInst->Dest.IsSIB()) {
+        OpToPatch = &DecodeInst->Dest;
+      }
+      if (!FoundLiteral && OpToPatch) {
+        if (DecodeInst->TableInfo->OpcodeDispatcher.OpDispatch == &IR::OpDispatchBuilder::NOPOp) {
+          // if it's a nop disp, only mask data out, no other action required
+          Type = DataMaskType::NOP;
+        } else if (OpToPatch->IsRIPRelative()) {
+          Type = DataMaskType::DISP;
+          OpToPatch->Data.RIPLiteral.PatchableDisp = true;
+          OpToPatch->Data.RIPLiteral.DispOffset = LastFieldReadOffset;
+        } else if (OpToPatch->IsGPRIndirect()) {
+          // filter out disp8
+          if (!(ModRM.mod == 1)) {
+            Type = DataMaskType::DISP;
+            OpToPatch->Data.GPRIndirect.PatchableDisp = true;
+            OpToPatch->Data.GPRIndirect.DispOffset = LastFieldReadOffset;
+          }
+        } else if (OpToPatch->IsSIB()) {
+          FEXCore::X86Tables::SIBDecoded SIB;
+          SIB.Hex = DecodeInst->SIB;
+          // two disp32 cases here
+          if (ModRM.mod == 0b10 || (ModRM.mod == 0 && SIB.base == 0b101)) {
+            Type = DataMaskType::DISP;
+            OpToPatch->Data.SIB.PatchableDisp = true;
+            OpToPatch->Data.SIB.DispOffset = LastFieldReadOffset;
+          }
+        }
+      }
+    }
+  }
   // jmp/call branches that use a literal rip-relative offset
   // some of those may be inlined by multiblock and will be cleaned up at decode end
-  if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_SETS_RIP && DecodeInst->Src[0].IsLiteral()) {
+  if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_SETS_RIP && DecodeInst->Src[0].IsLiteral() && DecodeInst->Src[0].Literal() != 0) {
     LiteralToPatch = &DecodeInst->Src[0];
     Type = DataMaskType::BRANCH;
   }
 
   // todo add a bunch more
 
-  if (LiteralToPatch) {
+  if (Type != DataMaskType::NONE) {
     Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, Type, LastFieldReadSize});
 
-    LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable;
-    LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset;
-    LiteralToPatch->Data.LiteralPatchable.Width = LastFieldReadSize;
+    if (LiteralToPatch) {
+      LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable;
+      LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset;
+      LiteralToPatch->Data.LiteralPatchable.Width = LastFieldReadSize;
+    }
   }
 }
 
@@ -1464,10 +1556,6 @@ void Decoder::PruneInlinedBranchDataMasks() {
 void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
   // counter-intuitively, the masks are also needed for lookup on anon prefix decodes, not just stores
   bool WantsDataMasks = CTX->DiskCache.IsReadingDiskCache() || CTX->DiskCache.IsWritingDiskCache();
-  // remove this if we ever fixup ValidateCode crc constant after relocations
-  if (CTX->Config.SMCChecks == FEXCore::Config::CONFIG_SMC_FULL) {
-    WantsDataMasks = false;
-  }
 
   while (!FinalInstruction && (Paused || !BlocksToDecode.empty())) {
     bool Pausing = false;
@@ -1588,6 +1676,10 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
           EraseBlock = true;
         } else {
           LogMan::Msg::EFmt("{} instruction in entry block: {:X}",
+                            BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_0       ? "Breakpoint" :
+                            BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_1       ? "Breakpoint" :
+                            BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_2       ? "Breakpoint" :
+                            BlockIt->BlockStatus == DecodedBlockStatus::BREAKPOINT_3       ? "Breakpoint" :
                             BlockIt->BlockStatus == DecodedBlockStatus::INVALID_INST       ? "Invalid" :
                             BlockIt->BlockStatus == DecodedBlockStatus::NOEXEC_INST        ? "NoExec" :
                             BlockIt->BlockStatus == DecodedBlockStatus::BAD_RELOCATION     ? "BadRelocation" :
@@ -1621,9 +1713,6 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
           // NOTE: This will invalidate BlockIt, this is fine as we immediately break from the loop and EraseBlock cannot be true
           if (CTX->AreMonoHacksActive() && IsBranchMonoTailcall(BlockIt->NumInstructions)) {
             BlockIt->ForceFullSMCDetection = true;
-            // todo abandon patching this for now, as the crc will fail and it will lock up redoing it over and over
-            // we should fix the crc at relocation if this is important
-            BlockIt->DataMasks.clear();
           }
           BranchTargetInMultiblockRange();
         }

FEXCore/Source/Interface/Core/Frontend.h

@@ -30,9 +30,13 @@ public:
     PARTIAL_DECODE_INST,
     BAD_RELOCATION,
     UNIMPLEMENTED_INST,
+    BREAKPOINT_0,
+    BREAKPOINT_1,
+    BREAKPOINT_2,
+    BREAKPOINT_3,
   };
 
-  enum class DataMaskType : uint8_t { MOV, BRANCH };
+  enum class DataMaskType : uint8_t { NONE, MOV, BRANCH, DISP, NOP };
 
   struct DataMask final {
     uint64_t FieldAddress;
@@ -115,7 +119,10 @@ private:
   void DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block);
   void PruneInlinedBranchDataMasks();
 
-  bool CheckRangeExecutable(uint64_t Address, uint64_t Size);
+  // Check for the number of bytes that are able to be decoded as executable
+  // Size: Returns Size if able to decode all bytes.
+  // Negative: Number of bytes that failed to decode.
+  int32_t CheckRangeExecutable(uint64_t Address, uint64_t Size);
 
   uint8_t ReadByte();
   std::optional<uint8_t> PeekByte(uint8_t Offset);
@@ -153,6 +160,7 @@ private:
   bool ExecutableRangeWritable {};
   bool HitNonExecutableRange {};
   bool HitBadRelocation {};
+  uint32_t HitBreakpoint {~0U};
 
   struct DecodeStream {
     // Original instruction stream RIP location.

FEXCore/Source/Interface/Core/Interpreter/Fallbacks/InterpreterFallbacks.cpp

@@ -100,6 +100,7 @@ void InterpreterOps::FillFallbackIndexPointers(Core::FallbackABIInfo* Info, uint
                                   reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_F64SCALE>::handle)};
 
   // SSE4.2 string instructions
+  // NOTE: Currently unused. See VectorFallbacks.h.
   Info[Core::OPINDEX_VPCMPESTRX] = {ABIHandlers[FABI_I32_I64_I64_V128_V128_I16],
                                     reinterpret_cast<uint64_t>(&FEXCore::CPU::OpHandlers<IR::OP_VPCMPESTRX>::handle)};
   Info[Core::OPINDEX_VPCMPISTRX] = {ABIHandlers[FABI_I32_V128_V128_I16],

FEXCore/Source/Interface/Core/Interpreter/Fallbacks/StringCompareFallbacks.cpp

@@ -8,6 +8,7 @@
 
 #include <cstring>
 
+// NOTE: Currently unused. See VectorFallbacks.h.
 namespace FEXCore::CPU {
 #ifdef ARCHITECTURE_arm64
 FEXCORE_PRESERVE_ALL_ATTR static int32_t GetImplicitLength(FEXCore::VectorRegType data, uint16_t control) {

FEXCore/Source/Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h

@@ -13,29 +13,33 @@
 
 namespace FEXCore::CPU {
 
+// PCMPXSTRX control byte fields
+enum class AggregationOp {
+  EqualAny = 0b00,
+  Ranges = 0b01,
+  EqualEach = 0b10,
+  EqualOrdered = 0b11,
+};
+
+enum class SourceData {
+  U8,
+  U16,
+  S8,
+  S16,
+};
+
+enum class Polarity {
+  Positive,
+  Negative,
+  PositiveMasked,
+  NegativeMasked,
+};
+
+// NB: The fallback handlers for the *PMCP*STR* instructions
+// are no longer used since we now emit inline ASM for them.
+// We preserve this as a reference implementation.
 template<>
 struct OpHandlers<IR::OP_VPCMPESTRX> {
-  enum class AggregationOp {
-    EqualAny = 0b00,
-    Ranges = 0b01,
-    EqualEach = 0b10,
-    EqualOrdered = 0b11,
-  };
-
-  enum class SourceData {
-    U8,
-    U16,
-    S8,
-    S16,
-  };
-
-  enum class Polarity {
-    Positive,
-    Negative,
-    PositiveMasked,
-    NegativeMasked,
-  };
-
   FEXCORE_PRESERVE_ALL_ATTR static uint32_t handle(uint64_t RAX, uint64_t RDX, VectorRegType lhs_v, VectorRegType rhs_v, uint16_t control) {
     __uint128_t lhs;
     memcpy(&lhs, &lhs_v, sizeof(lhs));

FEXCore/Source/Interface/Core/JIT/ALUOps.cpp

@@ -9,6 +9,7 @@ $end_info$
 #include "FEXCore/IR/IR.h"
 #include "Interface/Context/Context.h"
 #include "Interface/Core/JIT/JITClass.h"
+#include "Interface/Core/JIT/Relocations.h"
 #include "Interface/IR/Passes/RegisterAllocationPass.h"
 
 namespace FEXCore::CPU {
@@ -66,7 +67,17 @@ DEF_OP(EntrypointOffset) {
 
 DEF_OP(PatchableGuestData) {
   auto Op = IROp->C<IR::IROp_PatchableGuestData>();
-  InsertGuestPatchableDataMove(GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
+  InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
+}
+
+DEF_OP(PatchableGuestRIP) {
+  auto Op = IROp->C<IR::IROp_PatchableGuestRIP>();
+  InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
+}
+
+DEF_OP(PatchableGuestCRC) {
+  auto Op = IROp->C<IR::IROp_PatchableGuestRIP>();
+  InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_CRC_MOVE, GetReg(Node), Op->Value, Op->SiteAddress, (uint8_t)Op->SiteSize);
 }
 
 DEF_OP(InlineConstant) {
@@ -854,6 +865,159 @@ DEF_OP(PExt) {
   }
 }
 
+// Divide a uint128_t by a uint64_t, producing quotient and remainder.
+//
+// Adapted from divllu() in libdivide: https://github.com/ridiculousfish/libdivide/blob/master/doc/divlu.c
+// divllu() specifically is licensed as public domain, despite the zlib license that covers the rest of libdivide.
+// divllu() is itself an adaptation of Knuth's classic Algorithm D for long division, with some optimizations.
+// Where possible we have followed divllu() to the letter, including comments, but some changes were needed
+// due to our limited set of temp registers. Some comments below represent the original C code from divllu()
+// which is intended to match the assembly we emit.
+//
+// Additional reading:
+// https://ridiculousfish.com/blog/posts/labor-of-division-episode-iv.html
+// https://ridiculousfish.com/blog/posts/labor-of-division-episode-v.html
+void Arm64JITCore::EmitDivllu(ARMEmitter::Register Quotient, ARMEmitter::Register Remainder, ARMEmitter::Register Upper,
+                              ARMEmitter::Register Lower, ARMEmitter::Register Divisor) {
+  const auto EmitSize = ARMEmitter::Size::i64Bit;
+  ARMEmitter::ForwardLabel UpperReduced {};
+  ARMEmitter::ForwardLabel HighDigitDone {};
+  ARMEmitter::ForwardLabel LowDigitDone {};
+
+  // We work in base 2**32.
+  // A uint32 holds a single digit. A uint64 holds two digits.
+  // Our numerator is conceptually [num3, num2, num1, num0].
+  // Our denominator is [den1, den0].
+
+  // The normalization shift factor.
+  const auto Shift = TMP1;
+
+  // The divisor, and the high and low words of the dividend.
+  const auto Den = TMP2;
+  const auto NumHi = TMP3;
+  const auto NumLo = TMP4;
+
+  // A partial remainder.
+  const auto Rem = TMP3;
+
+  // Check for overflow and divide by 0.
+  // TODO: IF FEX handled #DE, we could simplify this I think.
+  if (NumHi != Upper) {
+    mov(EmitSize, NumHi, Upper);
+  }
+  cmp(EmitSize, Upper, Divisor);
+  (void)b(ARMEmitter::Condition::CC_LO, &UpperReduced);
+  udiv(EmitSize, Quotient, Upper, Divisor);
+  msub(EmitSize, NumHi, Quotient, Divisor, Upper);
+  (void)Bind(&UpperReduced);
+
+  // Determine the normalization factor. We multiply den by this, so that its leading digit is at
+  // least half b. In binary this means just shifting left by the number of leading zeros, so that
+  // there's a 1 in the MSB.
+  // We also shift numer by the same amount. This cannot overflow because numhi < den.
+  // The expression (-shift & 63) is the same as (64 - shift), except it avoids the UB of shifting
+  // by 64. The funny bitwise 'and' ensures that numlo does not get shifted into numhi if shift is 0.
+
+  // shift = divlu_count_leading_zeros64(den);
+  clz(EmitSize, Shift, Divisor);
+  // den <<= shift;
+  lslv(EmitSize, Den, Divisor, Shift);
+  // numhi <<= shift;
+  lslv(EmitSize, NumHi, NumHi, Shift);
+  // numhi |= (numlo >> (-shift & 63)) & (-(int64_t)shift >> 63);
+  mvn(EmitSize, Quotient, Shift);
+  lsr(EmitSize, Remainder, Lower, 1);
+  lsrv(EmitSize, Remainder, Remainder, Quotient);
+  orr(EmitSize, NumHi, NumHi, Remainder);
+  // numlo <<= shift;
+  lslv(EmitSize, NumLo, Lower, Shift);
+
+  // We wish to compute q1 = [n3 n2 n1] / [d1 d0].
+  // Estimate q1 as [n3 n2] / [d1], and then correct it.
+  // Note while qhat may be 2 digits, q1 is always 1 digit.
+  {
+    const auto Den1 = TMP1;
+    const auto Den0 = TMP1;
+
+    // The estimated quotient, and estimated remainder.
+    const auto QHat = Quotient;
+    const auto RHat = Remainder;
+
+    // Variables used to correct the estimated quotient.
+    const auto C1 = TMP1;
+    const auto C2 = Remainder;
+
+    // qhat = numhi / den1;
+    lsr(EmitSize, Den1, Den, 32);
+    udiv(EmitSize, QHat, NumHi, Den1);
+    // rhat = numhi % den1;
+    msub(EmitSize, RHat, QHat, Den1, NumHi);
+    // c1 = qhat * den0;
+    mov(ARMEmitter::Size::i32Bit, Den0, Den);
+    mul(EmitSize, C1, QHat, Den0);
+    // c2 = rhat * b + num1;
+    extr(EmitSize, C2, RHat, NumLo, 32);
+    // if (c1 > c2)
+    //     qhat -= (c1 - c2 > den) ? 2 : 1;
+    // C1 holds c1 - c2 after the subs.
+    subs(EmitSize, C1, C1, C2);
+    (void)b(ARMEmitter::Condition::CC_LS, &HighDigitDone);
+    sub(EmitSize, QHat, QHat, 1);
+    cmp(EmitSize, C1, Den);
+    (void)b(ARMEmitter::Condition::CC_LS, &HighDigitDone);
+    sub(EmitSize, QHat, QHat, 1);
+    (void)Bind(&HighDigitDone);
+    // q1 = (uint32_t)qhat;
+    // This move is a nop for us.
+  }
+
+  // Compute the true (partial) remainder.
+  // rem = numhi * b + num1 - q1 * den;
+  extr(EmitSize, Rem, NumHi, NumLo, 32);
+  msub(EmitSize, Rem, Quotient, Den, Rem);
+
+  // We wish to compute q0 = [rem1 rem0 n0] / [d1 d0].
+  // Estimate q0 as [rem1 rem0] / [d1] and correct it.
+  {
+    const auto Den1 = TMP1;
+    const auto Den0 = TMP3;
+
+    // The estimated quotient, and its corresponding remainder
+    const auto QHat = Remainder;
+    const auto RHat = TMP1;
+
+    // Variables used to correct the estimated quotient.
+    const auto C1 = TMP3;
+    const auto C2 = NumLo;
+
+    // qhat = rem / den1;
+    lsr(EmitSize, Den1, Den, 32);
+    udiv(EmitSize, QHat, Rem, Den1);
+    // rhat = rem % den1;
+    msub(EmitSize, RHat, QHat, Den1, Rem);
+    // c1 = qhat * den0;
+    mov(ARMEmitter::Size::i32Bit, Den0, Den);
+    mul(EmitSize, C1, QHat, Den0);
+    // c2 = rhat * b + num0;
+    bfi(EmitSize, C2, RHat, 32, 32);
+    // if (c1 > c2)
+    //     qhat -= (c1 - c2 > den) ? 2 : 1;
+    // C1 holds c1 - c2 after the subs.
+    subs(EmitSize, C1, C1, C2);
+    (void)b(ARMEmitter::Condition::CC_LS, &LowDigitDone);
+    sub(EmitSize, QHat, QHat, 1);
+    cmp(EmitSize, C1, Den);
+    (void)b(ARMEmitter::Condition::CC_LS, &LowDigitDone);
+    sub(EmitSize, QHat, QHat, 1);
+    (void)Bind(&LowDigitDone);
+    // q0 = (uint32_t)qhat;
+    // This move is a nop for us.
+  }
+
+  // q = ((uint64_t)q1 << 32) | q0;
+  orr(EmitSize, Quotient, Remainder, Quotient, ARMEmitter::ShiftType::LSL, 32);
+}
+
 DEF_OP(Div) {
   auto Op = IROp->C<IR::IROp_Div>();
   const auto OpSize = IROp->Size;
@@ -919,22 +1083,48 @@ DEF_OP(Div) {
 
     // If the sign bit matches then the result is zero
     (void)cbz(EmitSize, TMP1, &Only64Bit);
+    // The x86 instruction raises #DE when the divisor is zero which we don't do.
+    // TODO: #DE
+    (void)cbz(EmitSize, Divisor, &Only64Bit);
 
     // Long divide
+    // EmitDivllu() is the unsigned case, so we need to "cast" to unsigned first,
+    // and then apply the correct sign back to the results afterwards.
     {
-      mov(EmitSize, TMP1, Upper);
-      mov(EmitSize, TMP2, Lower);
-      mov(EmitSize, TMP3, Divisor);
+      const auto Sign = TMP1;
+      const auto AbsUpper = TMP3;
+      const auto AbsLower = TMP4;
+      const auto AbsDivisor = TMP2;
 
-      ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LDIVHandler));
+      mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
+      str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
 
-      str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
-      blr(TMP4);
-      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
+      // Extract sign bit of the dividend.
+      asr(EmitSize, Sign, Upper, 63);
 
-      // Move results to the destination registers
-      mov(EmitSize, Quotient, TMP1);
-      mov(EmitSize, Remainder, TMP2);
+      // Compute absolute value of the 128 bit dividend.
+      eor(EmitSize, AbsLower, Lower, Sign);
+      eor(EmitSize, AbsUpper, Upper, Sign);
+      subs(EmitSize, AbsLower, AbsLower, Sign);
+      sbc(EmitSize, AbsUpper, AbsUpper, Sign);
+
+      // Absolute value of the divisor.
+      cmp(EmitSize, Divisor, 0);
+      cneg(EmitSize, AbsDivisor, Divisor, ARMEmitter::Condition::CC_LT);
+
+      // Now do the unsigned divide.
+      EmitDivllu(Quotient, Remainder, AbsUpper, AbsLower, AbsDivisor);
+
+      // The quotient is negative when the dividend and the divisor have different signs.
+      eor(EmitSize, TMP1, Upper, Divisor);
+      cmp(EmitSize, TMP1, 0);
+      cneg(EmitSize, Quotient, Quotient, ARMEmitter::Condition::CC_LT);
+
+      ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
+      msr(ARMEmitter::SystemRegister::NZCV, TMP1);
+
+      // Compute the remainder.
+      msub(EmitSize, Remainder, Quotient, Divisor, Lower);
 
       // Skip 64-bit path
       (void)b(&LongDIVRet);
@@ -1002,22 +1192,22 @@ DEF_OP(UDiv) {
     // Check the upper bits for zero
     // If the upper bits are zero then we can do a 64-bit divide
     (void)cbz(EmitSize, Upper, &Only64Bit);
+    // The x86 instruction raises #DE when the divisor is zero which we don't do.
+    // TODO: #DE
+    (void)cbz(EmitSize, Divisor, &Only64Bit);
 
     // Long divide
     {
-      mov(EmitSize, TMP1, Upper);
-      mov(EmitSize, TMP2, Lower);
-      mov(EmitSize, TMP3, Divisor);
+      mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
+      str(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
 
-      ldr(TMP4, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.LUDIVHandler));
+      EmitDivllu(Quotient, Remainder, Upper, Lower, Divisor);
 
-      str<ARMEmitter::IndexType::PRE>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, -16);
-      blr(TMP4);
-      ldr<ARMEmitter::IndexType::POST>(ARMEmitter::XReg::lr, ARMEmitter::Reg::rsp, 16);
+      ldr(TMP1.W(), STATE.R(), offsetof(FEXCore::Core::CpuStateFrame, State.flags[X86State::RFLAG_NZCV_LOC]));
+      msr(ARMEmitter::SystemRegister::NZCV, TMP1);
 
-      // Move results to the destination registers
-      mov(EmitSize, Quotient, TMP1);
-      mov(EmitSize, Remainder, TMP2);
+      // Compute the remainder.
+      msub(EmitSize, Remainder, Quotient, Divisor, Lower);
 
       // Skip 64-bit path
       (void)b(&LongDIVRet);

FEXCore/Source/Interface/Core/JIT/Arm64Relocations.cpp

@@ -121,21 +121,10 @@ auto Arm64JITCore::InsertGuestPatchableRIPLiteral(uint64_t GuestRIP, uint64_t Si
   };
 }
 
-void Arm64JITCore::InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) {
+void Arm64JITCore::InsertGuestPatchableMove(FEXCore::CPU::RelocationTypes Type, ARMEmitter::Register Reg, uint64_t Value,
+                                            uint64_t SiteAddress, uint8_t ValueSize) {
   Relocation MoveABI = Relocation::Default();
-  MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_DATA_MOVE};
-  MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx();
-  MoveABI.GuestPatchableData.ValueSize = ValueSize;
-  MoveABI.GuestPatchableData.SiteAddress = SiteAddress;
-
-  // this might get patched on disk cache load
-  LoadConstant(ARMEmitter::Size::i64Bit, Reg, Value, FEXCore::CPU::Arm64Emitter::PadType::DOPAD);
-  Relocations.emplace_back(MoveABI);
-}
-
-void Arm64JITCore::InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize) {
-  Relocation MoveABI = Relocation::Default();
-  MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE};
+  MoveABI.GuestPatchableData.Header = {.Offset = GetCursorOffset(), .Type = Type};
   MoveABI.GuestPatchableData.RegisterIndex = Reg.Idx();
   MoveABI.GuestPatchableData.ValueSize = ValueSize;
   MoveABI.GuestPatchableData.SiteAddress = SiteAddress;

FEXCore/Source/Interface/Core/JIT/BranchOps.cpp

@@ -84,7 +84,8 @@ DEF_OP(ExitFunction) {
       str(REG_CALLRET_SP, STATE_PTR(CpuStateFrame, State.callret_sp));
       add(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::rsp, StaticRegisters[X86State::REG_RSP], 0);
       if (Op->PatchSiteAddress) {
-        InsertGuestPatchableRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress, Op->PatchSiteSize);
+        InsertGuestPatchableMove(RelocationTypes::RELOC_GUEST_PATCHABLE_RIP_MOVE, EC_CALL_CHECKER_PC_REG, NewRIP, Op->PatchSiteAddress,
+                                 Op->PatchSiteSize);
       } else {
         InsertGuestRIPMove(EC_CALL_CHECKER_PC_REG, NewRIP);
       }
@@ -282,51 +283,11 @@ DEF_OP(CondJump) {
 }
 
 DEF_OP(Syscall) {
-  // Arguments are passed as follows:
-  // X0: SyscallHandler
-  // X1: ThreadState
+  SpillStaticRegs(TMP1);
 
-  PushDynamicRegs(TMP1);
-
-  uint32_t GPRSpillMask = ~0U;
-  uint32_t FPRSpillMask = ~0U;
-
-  SpillStaticRegs(TMP1, {
-                          .GPRSpillMask = GPRSpillMask,
-                          .FPRSpillMask = FPRSpillMask,
-                        });
-
-  // Now that we are spilled, store in the state that we are in a syscall
-  // Still without overwriting registers that matter
-  // 16bit LoadConstant to be a single instruction
-  // This gives the signal handler a value to check to see if we are in a syscall at all
-  LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, GPRSpillMask & 0xFFFF);
-  str(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
-
-  ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerObj));
-  ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.SyscallHandlerFunc));
-  mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, STATE.R());
-
-  if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-    GenerateIndirectRuntimeCall<uint64_t, void*, void*, void*>(ARMEmitter::Reg::r3);
-  } else {
-    blr(ARMEmitter::Reg::r3);
-  }
-
-  // Fix the stack and any values that were stepped on
-  // Syscall result is in any static register that the frontend desired.
-  FillStaticRegs({
-    .OptionalReg = ARMEmitter::Reg::r1,
-    .OptionalReg2 = ARMEmitter::Reg::r2,
-    .GPRFillMask = GPRSpillMask,
-    .FPRFillMask = FPRSpillMask,
-  });
-
-  // Now the registers we've spilled are back in their original host registers
-  // We can safely claim we are no longer in a syscall
-  str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo));
-
-  PopDynamicRegs();
+  // Jump to the syscall dispatch handler. We won't return after this.
+  ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadDispatchSyscallHandler));
+  br(TMP1);
 }
 
 DEF_OP(Thunk) {
@@ -413,71 +374,33 @@ DEF_OP(ValidateCode) {
 DEF_OP(ThreadRemoveCodeEntry) {
   auto Op = IROp->C<IR::IROp_ThreadRemoveCodeEntry>();
 
-  // Move the entry to ABI before saving state.
-  mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetReg(Op->Entry));
+  SpillStaticRegs(TMP1);
 
-  PushDynamicRegs(TMP4);
-  SpillStaticRegs(TMP4);
+  // Store the new RIP to go to.
+  str(GetReg(Op->NewRIP).X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip));
+
+  // Move the entry to ABI before saving state.
+  mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, GetReg(Op->EntryToInvalidate));
 
   // Arguments are passed as follows:
   // X0: Thread
-  // X1: RIP
+  // X1: RIPToInvalidate
   mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, STATE.R());
 
-  ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadRemoveCodeEntryFromJIT));
-  if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-    GenerateIndirectRuntimeCall<void, void*, void*>(ARMEmitter::Reg::r2);
-  } else {
-    blr(ARMEmitter::Reg::r2);
-  }
-  FillStaticRegs();
-
-  // Fix the stack and any values that were stepped on
-  PopDynamicRegs();
+  // Jump to the invalidate dispatch handler. We won't return after this.
+  ldr(ARMEmitter::XReg::x2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadDispatchRemoveCodeEntry));
+  br(ARMEmitter::XReg::x2);
 }
 
 DEF_OP(CPUID) {
   auto Op = IROp->C<IR::IROp_CPUID>();
 
-  isb();
-  mov(ARMEmitter::Size::i64Bit, TMP2, GetReg(Op->Function));
-  mov(ARMEmitter::Size::i64Bit, TMP3, GetReg(Op->Leaf));
+  SpillStaticRegs(TMP1);
 
-  PushDynamicRegs(TMP4);
-  SpillStaticRegs(TMP4);
-
-  // x0 = CPUID Handler
-  // x1 = CPUID Function
-  // x2 = CPUID Leaf
-  ldr(ARMEmitter::XReg::x0, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDObj));
-  ldr(ARMEmitter::XReg::x3, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.CPUIDFunction));
-
-  if (!TMP_ABIARGS) {
-    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r1, TMP2);
-    mov(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r2, TMP3);
-  }
-
-  if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
-    GenerateIndirectRuntimeCall<__uint128_t, void*, uint64_t, uint64_t>(ARMEmitter::Reg::r3);
-  } else {
-    blr(ARMEmitter::Reg::r3);
-  }
-
-  if (!TMP_ABIARGS) {
-    mov(ARMEmitter::Size::i64Bit, TMP1, ARMEmitter::Reg::r0);
-    mov(ARMEmitter::Size::i64Bit, TMP2, ARMEmitter::Reg::r1);
-  }
-
-  FillStaticRegs();
-
-  PopDynamicRegs();
-
-  // Results are in x0, x1
-  // Results want to be 4xi32 scalars
-  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutEAX), TMP1);
-  mov(ARMEmitter::Size::i32Bit, GetReg(Op->OutECX), TMP2);
-  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEBX), TMP1, 32, 32);
-  ubfx(ARMEmitter::Size::i64Bit, GetReg(Op->OutEDX), TMP2, 32, 32);
+  LoadConstant(ARMEmitter::Size::i32Bit, TMP1, Op->InstSize);
+  // Jump to the syscall dispatch handler. We won't return after this.
+  ldr(TMP2, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.ThreadDispatchCPUIDHandler));
+  br(TMP2);
 }
 
 DEF_OP(XGetBV) {

FEXCore/Source/Interface/Core/JIT/ConversionOps.cpp

@@ -405,6 +405,37 @@ DEF_OP(VFCVTN2) {
   }
 }
 
+DEF_OP(VBFCVTN) {
+  LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsBF16, "Need BF16 for VBFCVTN");
+  const auto Op = IROp->C<IR::IROp_VBFCVTN>();
+
+  const auto Dst = GetVReg(Node);
+  const auto Vector = GetVReg(Op->Vector);
+
+  bfcvtn(Dst.Q(), Vector.Q());
+}
+
+DEF_OP(VBFCVTN2) {
+  LOGMAN_THROW_A_FMT(CTX->HostFeatures.SupportsBF16, "Need BF16 for VBFCVTN");
+  const auto Op = IROp->C<IR::IROp_VBFCVTN2>();
+
+  const auto Dst = GetVReg(Node);
+  const auto VectorLower = GetVReg(Op->VectorLower);
+  const auto VectorUpper = GetVReg(Op->VectorUpper);
+
+  auto Lower = VectorLower;
+  if (Dst != VectorLower) {
+    mov(VTMP1.Q(), VectorLower.Q());
+    Lower = VTMP1;
+  }
+
+  bfcvtn2(Lower.Q(), VectorUpper.Q());
+
+  if (Dst != VectorLower) {
+    mov(Dst.Q(), Lower.Q());
+  }
+}
+
 DEF_OP(Vector_FToI) {
   const auto Op = IROp->C<IR::IROp_Vector_FToI>();
   const auto OpSize = IROp->Size;

FEXCore/Source/Interface/Core/JIT/JIT.cpp

@@ -36,35 +36,44 @@ $end_info$
 
 #include <cstdio>
 #include <cstring>
+#include <optional>
+#include <type_traits>
 #include <unistd.h>
+#ifdef _WIN32
+#include <atomic>
+#else
+#include <FEXHeaderUtils/Syscalls.h>
+#endif
 
 namespace {
-struct DivRem {
-  uint64_t Quotient;
-  uint64_t Remainder;
-};
+#ifndef _WIN32
 
-static struct DivRem LUDIV(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {
-  __uint128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
-
-  return {
-    .Quotient = (uint64_t)(Source / Divisor),
-    .Remainder = (uint64_t)(Source % Divisor),
-  };
+static std::optional<uint64_t> RDRANDFallback(uint64_t Reseed) {
+  uint64_t Value {};
+  FHU::Syscalls::getrandom(&Value, sizeof(Value), 0);
+  return Value;
 }
 
-static struct DivRem
-LDIV(uint64_t SrcHigh, uint64_t SrcLow, int64_t Divisor) {
-  __int128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
+#else
 
-  return {
-    .Quotient = (uint64_t)(Source / Divisor),
-    .Remainder = (uint64_t)(Source % Divisor),
-  };
+// Windows does not have an equivalent to getrandom() without dynamically linking
+// bcrypt et al. Since this fallback is just for compat it does not need to be
+// cryptographic, so instead we vendor SplitMix64 as a naive fallback.
+
+// Reference implementation by Sebastiano Vigna, public domain (CC0)
+// https://prng.di.unimi.it/splitmix64.c
+static std::atomic<uint64_t> RNGState {static_cast<uint64_t>(__builtin_readcyclecounter())};
+
+static std::optional<uint64_t> RDRANDFallback(uint64_t Reseed) {
+  const uint64_t State = RNGState.load(std::memory_order_relaxed) + 0x9E3779B97F4A7C15ULL;
+  RNGState.store(State, std::memory_order_relaxed);
+  uint64_t Value = (State ^ (State >> 30)) * 0xBF58476D1CE4E5B9ULL;
+  Value = (Value ^ (Value >> 27)) * 0x94D049BB133111EBULL;
+  return Value ^ (Value >> 31);
 }
+#endif
 
-static void
-PrintValue(uint64_t Value) {
+static void PrintValue(uint64_t Value) {
   LogMan::Msg::DFmt("Value: 0x{:x}", Value);
 }
 
@@ -616,9 +625,9 @@ void Arm64JITCore::Op_NoOp(const IR::IROp_Header* IROp, IR::Ref Node) {}
 Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::InternalThreadState* Thread)
   : CPUBackend(*ctx, Thread)
   , Arm64Emitter(ctx)
-  , HostSupportsSVE128 {ctx->HostFeatures.SupportsSVE128 != 0}
-  , HostSupportsSVE256 {ctx->HostFeatures.SupportsSVE256 != 0}
-  , HostSupportsAVX256 {ctx->HostFeatures.SupportsAVX && ctx->HostFeatures.SupportsSVE256}
+  , HostSupportsSVE128 {ctx->HostFeatures.SupportsSVE128()}
+  , HostSupportsSVE256 {ctx->HostFeatures.SupportsSVE256()}
+  , HostSupportsAVX256 {ctx->HostFeatures.SupportsAVX() && ctx->HostFeatures.SupportsSVE256()}
   , HostSupportsRPRES {ctx->HostFeatures.SupportsRPRES != 0}
   , HostSupportsAFP {ctx->HostFeatures.SupportsAFP != 0}
   , CTX {ctx}
@@ -662,8 +671,7 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::ContextImpl* ctx, FEXCore::Core::In
       Ptrs.SyscallHandlerFunc = PMF.GetVTableEntry(CTX->SyscallHandler);
     }
     Ptrs.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Arm64JITCore::ExitFunctionLink);
-    Ptrs.LUDIV = reinterpret_cast<uint64_t>(LUDIV);
-    Ptrs.LDIV = reinterpret_cast<uint64_t>(LDIV);
+    Ptrs.RDRANDFallback = reinterpret_cast<uint64_t>(RDRANDFallback);
   }
 
   CurrentCodeBuffer = SharedCodeBuffers.GetLatest();
@@ -746,13 +754,17 @@ void Arm64JITCore::EmitTFCheck() {
     .TrapNo = X86State::X86_TRAPNO_DB,
     .si_code = 2,
     .err_code = 0,
+    .FaultAddress = 0,
   };
 
   uint64_t Constant {};
-  memcpy(&Constant, &State, sizeof(State));
+  memcpy(&Constant, &State, sizeof(uint64_t));
 
-  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
+  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
+  // Can't use stp, offset is too large.
   str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
+  str(ARMEmitter::XReg::zr, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
+
   ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
   br(TMP1);
 
@@ -766,7 +778,7 @@ void Arm64JITCore::EmitTFCheck() {
 void Arm64JITCore::EmitSuspendInterruptCheck() {
   if (CTX->Config.NeedsPendingInterruptFaultCheck) {
     // Trigger a fault if there are any pending interrupts
-    // Used only for suspend on WIN32 at the moment
+    // Used only for gdbserver at the moment
     constexpr size_t InterruptPageOffset =
       offsetof(FEXCore::Core::InternalThreadState, InterruptFaultPage) - offsetof(FEXCore::Core::InternalThreadState, BaseFrameState);
     if constexpr (InterruptPageOffset <= 32760) {
@@ -778,7 +790,7 @@ void Arm64JITCore::EmitSuspendInterruptCheck() {
     }
   }
 
-#ifdef ARCHITECTURE_arm64ec
+#ifdef _WIN32
   static constexpr uint16_t SuspendMagic {0xCAFE};
 
   ldr(TMP2.W(), STATE_PTR(CpuStateFrame, SuspendDoorbell));
@@ -840,9 +852,9 @@ CodeBuffer::CodeBufferAllocation Arm64JITCore::AllocateCodeBufferInSharedCache(s
   return AllocatedInfo;
 }
 
-CPUBackend::CompiledCode Arm64JITCore::CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
+CPUBackend::CompiledCode Arm64JITCore::EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                                    FEXCore::Core::DebugData* DebugData, bool CheckTF) {
-  FEXCORE_PROFILE_SCOPED("Arm64::CompileCode");
+  FEXCORE_PROFILE_SCOPED("Arm64::EmitJITCode");
 
   const auto PrevNumAllocations = Relocations.size();
 

FEXCore/Source/Interface/Core/JIT/JITClass.h

@@ -51,7 +51,7 @@ public:
   ~Arm64JITCore() override;
 
   [[nodiscard]]
-  CPUBackend::CompiledCode CompileCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
+  CPUBackend::CompiledCode EmitJITCode(uint64_t Entry, uint64_t Size, bool SingleInst, const FEXCore::IR::IRListView* IR,
                                        FEXCore::Core::DebugData* DebugData, bool CheckTF) override;
 
   [[nodiscard]]
@@ -564,8 +564,8 @@ private:
    */
   void InsertGuestRIPMove(ARMEmitter::Register Reg, uint64_t Constant);
 
-  void InsertGuestPatchableDataMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize);
-  void InsertGuestPatchableRIPMove(ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress, uint8_t ValueSize);
+  void InsertGuestPatchableMove(FEXCore::CPU::RelocationTypes Type, ARMEmitter::Register Reg, uint64_t Value, uint64_t SiteAddress,
+                                uint8_t ValueSize);
 
   /**
    * @brief Inserts a named symbol as a literal in memory
@@ -629,10 +629,14 @@ private:
   void Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize, ARMEmitter::VRegister Dst, ARMEmitter::VRegister IncomingDst,
                            std::optional<ARMEmitter::Register> BaseAddr, ARMEmitter::VRegister VectorIndexLow,
                            std::optional<ARMEmitter::VRegister> VectorIndexHigh, ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize,
-                           size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize);
+                           size_t DataElementOffsetStart, size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize,
+                           bool MaskAllOnes);
 
   void EmitTFCheck();
 
+  void EmitDivllu(ARMEmitter::Register Quotient, ARMEmitter::Register Remainder, ARMEmitter::Register Upper, ARMEmitter::Register Lower,
+                  ARMEmitter::Register Divisor);
+
   void EmitSuspendInterruptCheck();
 
   void EmitEntryPoint(ARMEmitter::BackwardLabel& HeaderLabel, bool CheckTF);

FEXCore/Source/Interface/Core/JIT/MemoryOps.cpp

@@ -1058,7 +1058,7 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
                                        ARMEmitter::VRegister IncomingDst, std::optional<ARMEmitter::Register> BaseAddr,
                                        ARMEmitter::VRegister VectorIndexLow, std::optional<ARMEmitter::VRegister> VectorIndexHigh,
                                        ARMEmitter::VRegister MaskReg, IR::OpSize VectorIndexSize, size_t DataElementOffsetStart,
-                                       size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize) {
+                                       size_t IndexElementOffsetStart, uint8_t OffsetScale, IR::OpSize AddrSize, bool MaskAllOnes) {
   LOGMAN_THROW_A_FMT(ElementSize >= IR::OpSize::i8Bit && ElementSize <= IR::OpSize::i64Bit, "Invalid element size");
 
   const auto PerformSMove = [this](IR::OpSize ElementSize, const ARMEmitter::Register Dst, const ARMEmitter::VRegister Vector, int index) {
@@ -1116,11 +1116,13 @@ void Arm64JITCore::Emulate128BitGather(IR::OpSize Size, IR::OpSize ElementSize,
 
   for (size_t i = DataElementOffsetStart, IndexElement = IndexElementOffsetStart; i < NumDataElements; ++i, ++IndexElement) {
     ARMEmitter::ForwardLabel Skip {};
-    // Extract mask element
-    PerformMove(ElementSize, WorkingReg, MaskReg, i);
+    if (!MaskAllOnes) {
+      // Extract mask element
+      PerformMove(ElementSize, WorkingReg, MaskReg, i);
 
-    // Skip if the mask's sign bit isn't set
-    (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+      // Skip if the mask's sign bit isn't set
+      (void)tbz(WorkingReg, ElementSizeInBits - 1, &Skip);
+    }
 
     // Extract Index Element
     if ((IndexElement * IR::OpSizeToSize(VectorIndexSize)) >= 16) {
@@ -1216,12 +1218,15 @@ DEF_OP(VLoadVectorGatherMasked) {
 
     const auto SubRegSize = ConvertSubRegSize8(IROp);
 
-    const auto CMPPredicate = ARMEmitter::PReg::p0;
     const auto GoverningPredicate = Is256Bit ? PRED_TMP_32B : PRED_TMP_16B;
 
-    // Check if the sign bit is set for the given element size.
-    cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
-    auto TempDst = VTMP1;
+    // A mask known to be all-ones loads every element straight into the destination under the governing predicate.
+    const ARMEmitter::PRegister CMPPredicate = Op->MaskAllOnes ? GoverningPredicate : ARMEmitter::PReg::p0;
+    auto TempDst = Op->MaskAllOnes ? Dst : VTMP1;
+    if (!Op->MaskAllOnes) {
+      // Check if the sign bit is set for the given element size.
+      cmplt(SubRegSize, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
+    }
 
     // No need to load a temporary register in the case that we weren't provided a base address and there is no scaling.
     ARMEmitter::SVEMemOperand MemDst {ARMEmitter::SVEMemOperand(VectorIndexLow.Z(), 0)};
@@ -1256,12 +1261,14 @@ DEF_OP(VLoadVectorGatherMasked) {
     default: break;
     }
 
-    ///< Merge elements based on predicate.
-    sel(SubRegSize, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
+    if (!Op->MaskAllOnes) {
+      ///< Merge elements based on predicate.
+      sel(SubRegSize, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
+    }
   } else {
     LOGMAN_THROW_A_FMT(!Is256Bit, "Can't emulate this gather load in the backend! Programming error!");
     Emulate128BitGather(IROp->Size, IROp->ElementSize, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,
-                        VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale, Op->AddrSize);
+                        VectorIndexSize, DataElementOffsetStart, IndexElementOffsetStart, OffsetScale, Op->AddrSize, Op->MaskAllOnes);
   }
 }
 
@@ -1295,16 +1302,20 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
     }
 
     const auto CMPPredicate = ARMEmitter::PReg::p0;
-    const auto CMPPredicate2 = ARMEmitter::PReg::p1;
+    // A mask known to be all-ones loads every element under the governing predicate and narrows straight into the destination.
+    const ARMEmitter::PRegister CMPPredicate2 = Op->MaskAllOnes ? PRED_TMP_16B : ARMEmitter::PReg::p1;
 
     const auto GoverningPredicate = PRED_TMP_16B;
 
-    // Check if the sign bit is set for the given element size.
-    // This will set the predicate bits for elements [0, 1, 2, 3]
-    // We then use punpklo to extend the low results to be for 64-bit elements.
-    cmplt(ARMEmitter::SubRegSize::i32Bit, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
-    punpklo(CMPPredicate2, CMPPredicate);
+    if (!Op->MaskAllOnes) {
+      // Check if the sign bit is set for the given element size.
+      // This will set the predicate bits for elements [0, 1, 2, 3]
+      // We then use punpklo to extend the low results to be for 64-bit elements.
+      cmplt(ARMEmitter::SubRegSize::i32Bit, CMPPredicate, GoverningPredicate.Zeroing(), MaskReg.Z(), 0);
+      punpklo(CMPPredicate2, CMPPredicate);
+    }
     auto TempDst = VTMP1;
+    const auto NarrowDst = Op->MaskAllOnes ? Dst : TempDst;
 
     auto GatherExtend = [this](ARMEmitter::VRegister Dst, std::optional<ARMEmitter::Register> BaseAddr, ARMEmitter::VRegister VectorIndex,
                                ARMEmitter::PRegister CMPPredicate, ARMEmitter::SVEModType ModType, uint8_t OffsetScale) {
@@ -1328,21 +1339,23 @@ DEF_OP(VLoadVectorGatherMaskedQPS) {
     GatherExtend(TempDst, BaseAddr, VectorIndexLow, CMPPredicate2, ModType, OffsetScale);
 
     if (VectorIndexHigh.has_value()) {
-      punpkhi(CMPPredicate2, CMPPredicate);
+      if (!Op->MaskAllOnes) {
+        punpkhi(CMPPredicate2, CMPPredicate);
+      }
       GatherExtend(VTMP2, BaseAddr, *VectorIndexHigh, CMPPredicate2, ModType, OffsetScale);
       // Move elements to the lower half.
-      uzp1(ARMEmitter::SubRegSize::i32Bit, TempDst.Q(), TempDst.Q(), VTMP2.Q());
-      ///< Merge elements based on predicate.
-      sel(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
+      uzp1(ARMEmitter::SubRegSize::i32Bit, NarrowDst.Q(), TempDst.Q(), VTMP2.Q());
     } else {
       // Move elements to the lower half.
-      xtn(ARMEmitter::SubRegSize::i32Bit, TempDst.Q(), TempDst.Q());
+      xtn(ARMEmitter::SubRegSize::i32Bit, NarrowDst.Q(), TempDst.Q());
+    }
+    if (!Op->MaskAllOnes) {
       ///< Merge elements based on predicate.
       sel(ARMEmitter::SubRegSize::i32Bit, Dst.Z(), CMPPredicate, TempDst.Z(), IncomingDst.Z());
     }
   } else {
     Emulate128BitGather(IR::OpSize::i128Bit, IR::OpSize::i32Bit, Dst, IncomingDst, BaseAddr, VectorIndexLow, VectorIndexHigh, MaskReg,
-                        IR::OpSize::i64Bit, 0, 0, OffsetScale, Op->AddrSize);
+                        IR::OpSize::i64Bit, 0, 0, OffsetScale, Op->AddrSize, Op->MaskAllOnes);
   }
 }
 

FEXCore/Source/Interface/Core/JIT/MiscOps.cpp

@@ -68,31 +68,42 @@ DEF_OP(Break) {
     .TrapNo = Op->Reason.TrapNumber,
     .si_code = Op->Reason.si_code,
     .err_code = Op->Reason.ErrorRegister,
+    .FaultAddress = 0, // Comes from SSA argument.
   };
 
   uint64_t Constant {};
-  memcpy(&Constant, &State, sizeof(State));
+  memcpy(&Constant, &State, sizeof(uint64_t));
 
-  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, Constant);
+  LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Constant));
+
+  auto FaultGPR = GetZeroableReg(Op->FaultAddress);
+
+  // Can't use stp, offset is too large.
   str(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData));
+  str(FaultGPR.X(), STATE, offsetof(FEXCore::Core::CpuStateFrame, SynchronousFaultData) + 8);
 
-  switch (Op->Reason.Signal) {
-  case Core::FAULT_SIGILL:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
+  if (CTX->HostFeatures.ProcessFeatures.SupportsFaultFreeSynchronousFaults) {
+    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SynchronousFault));
     br(TMP1);
-    break;
-  case Core::FAULT_SIGTRAP:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
-    br(TMP1);
-    break;
-  case Core::FAULT_SIGSEGV:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
-    br(TMP1);
-    break;
-  default:
-    ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
-    br(TMP1);
-    break;
+  } else {
+    switch (Op->Reason.Signal) {
+    case Core::FAULT_SIGILL:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGILL));
+      br(TMP1);
+      break;
+    case Core::FAULT_SIGTRAP:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
+      br(TMP1);
+      break;
+    case Core::FAULT_SIGSEGV:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGSEGV));
+      br(TMP1);
+      break;
+    default:
+      ldr(TMP1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.GuestSignal_SIGTRAP));
+      br(TMP1);
+      break;
+    }
   }
 }
 
@@ -309,7 +320,39 @@ DEF_OP(ProcessorID) {
 DEF_OP(RDRAND) {
   auto Op = IROp->C<IR::IROp_RDRAND>();
 
-  mrs(GetReg(Node), Op->GetReseeded ? ARMEmitter::SystemRegister::RNDRRS : ARMEmitter::SystemRegister::RNDR);
+  if (CTX->HostFeatures.SupportsRAND) {
+    mrs(GetReg(Node), Op->GetReseeded ? ARMEmitter::SystemRegister::RNDRRS : ARMEmitter::SystemRegister::RNDR);
+    return;
+  }
+
+  // Software fallback, call the host RNG generator.
+  PushDynamicRegs(TMP4);
+  SpillStaticRegs(TMP4);
+
+  // x0 = Reseed
+  // x1 = Generator
+  LoadConstant(ARMEmitter::Size::i64Bit, ARMEmitter::Reg::r0, Op->GetReseeded ? 1 : 0);
+  ldr(ARMEmitter::XReg::x1, STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.RDRANDFallback));
+
+  if (!CTX->Config.DisableVixlIndirectCalls) [[unlikely]] {
+    GenerateIndirectRuntimeCall<__uint128_t, uint64_t>(ARMEmitter::Reg::r1);
+  } else {
+    blr(ARMEmitter::Reg::r1);
+  }
+
+  if (!TMP_ABIARGS) {
+    mov(ARMEmitter::Size::i64Bit, TMP1, ARMEmitter::Reg::r0);
+    mov(ARMEmitter::Size::i64Bit, TMP2, ARMEmitter::Reg::r1);
+  }
+
+  FillStaticRegs();
+  PopDynamicRegs();
+
+  // Results are in x0, x1
+  // std::optional<uint64_t>: value in x0, engaged flag in the low byte of x1. Match the hardware behaviour of setting Z when
+  // no number was produced.
+  mov(ARMEmitter::Size::i64Bit, GetReg(Node), TMP1);
+  tst(ARMEmitter::Size::i64Bit, TMP2, 0xFF);
 }
 
 DEF_OP(Yield) {

FEXCore/Source/Interface/Core/JIT/Relocations.h

@@ -37,6 +37,10 @@ enum class RelocationTypes : uint32_t {
   // Like PATCHABLE_RIP_LITERAL but puts it in a register
   // Aligned to struct RelocGuestPatchableData
   RELOC_GUEST_PATCHABLE_RIP_MOVE,
+
+  // Patchable guest CRC
+  // Aligned to struct RelocGuestPatchableData
+  RELOC_GUEST_PATCHABLE_CRC_MOVE,
 };
 
 struct FEX_PACKED RelocationHeader final {

FEXCore/Source/Interface/Core/JIT/VectorOps.cpp

@@ -930,6 +930,9 @@ DEF_OP(VectorImm) {
 
   const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
   LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
+  if (Op->Do64BitExpansion) {
+    LOGMAN_THROW_A_FMT(!Is256Bit, "64-bit expansion unsupported on 256-bit operations.");
+  }
   const auto ElementSize = Op->Header.ElementSize;
   const auto SubRegSize = ConvertSubRegSize8(IROp);
 
@@ -945,12 +948,22 @@ DEF_OP(VectorImm) {
       dup_imm(SubRegSize, Dst.Z(), static_cast<int8_t>(Op->Immediate));
     }
   } else {
-    if (ElementSize == IR::OpSize::i64Bit) {
+    if (!Op->Do64BitExpansion && ElementSize == IR::OpSize::i64Bit) {
       // movi with 64bit element size doesn't do what we want here
       LoadConstant(ARMEmitter::Size::i64Bit, TMP1, static_cast<uint64_t>(Op->Immediate) << Op->ShiftAmount);
       dup(SubRegSize, Dst.Q(), TMP1.R());
     } else {
-      movi(SubRegSize, Dst.Q(), Op->Immediate, Op->ShiftAmount);
+      uint64_t Imm = Op->Immediate;
+      if (Op->Do64BitExpansion) {
+        // Need to manually expand the immediate for this case.
+        Imm = 0;
+        for (size_t i = 0; i < 8; ++i) {
+          if ((Op->Immediate >> i) & 1) {
+            Imm |= (0xFFULL << (i * 8));
+          }
+        }
+      }
+      movi(SubRegSize, Dst.Q(), Imm, Op->ShiftAmount);
     }
   }
 }
@@ -1112,8 +1125,10 @@ DEF_OP(VAddP) {
     // pairwise addition, the SVE version actually interleaves the
     // results of the pairwise addition (gross!), so we need to undo that.
     addp(SubRegSize, LHS.Z(), Pred, LHS.Z(), VectorUpper.Z());
-    uzp1(SubRegSize, Dst.Z(), LHS.Z(), LHS.Z());
+
+    // Extract the upper half first, since Dst may alias the LHS.
     uzp2(SubRegSize, VTMP2.Z(), LHS.Z(), LHS.Z());
+    uzp1(SubRegSize, Dst.Z(), LHS.Z(), LHS.Z());
 
     // Merge upper half with lower half.
     splice<ARMEmitter::OpType::Destructive>(ARMEmitter::SubRegSize::i64Bit, Dst.Z(), PRED_TMP_16B, Dst.Z(), VTMP2.Z());
@@ -2167,6 +2182,46 @@ DEF_OP(VCMPGT) {
   }
 }
 
+DEF_OP(VUCMPGT) {
+  const auto Op = IROp->C<IR::IROp_VUCMPGT>();
+  const auto OpSize = IROp->Size;
+
+  const auto ElementSize = Op->Header.ElementSize;
+  const auto SubRegSize = ConvertSubRegSizePair16(IROp);
+  const auto IsScalar = ElementSize == OpSize;
+  const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
+  LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
+
+  const auto Dst = GetVReg(Node);
+  const auto Vector1 = GetVReg(Op->Vector1);
+  const auto Vector2 = GetVReg(Op->Vector2);
+
+  if (HostSupportsSVE256 && Is256Bit) {
+    const auto Mask = PRED_TMP_32B.Zeroing();
+    const auto ComparePred = ARMEmitter::PReg::p0;
+
+    // FIXME: We should rework this op to avoid the NZCV spill/fill dance.
+    mrs(TMP1, ARMEmitter::SystemRegister::NZCV);
+
+    // General idea is to compare for unsigned greater-than, bitwise NOT
+    // the valid values, then ORR the NOTed values with the original
+    // values to form entries that are all 1s.
+    cmphi(SubRegSize.Vector, ComparePred, Mask, Vector1.Z(), Vector2.Z());
+    not_(SubRegSize.Vector, VTMP1.Z(), ComparePred.Merging(), Vector1.Z());
+    movprfx(SubRegSize.Vector, Dst.Z(), ComparePred.Zeroing(), Vector1.Z());
+    orr(SubRegSize.Vector, Dst.Z(), ComparePred.Merging(), Dst.Z(), VTMP1.Z());
+
+    // Restore NZCV
+    msr(ARMEmitter::SystemRegister::NZCV, TMP1);
+  } else {
+    if (IsScalar) {
+      cmhi(SubRegSize.Scalar, Dst, Vector1, Vector2);
+    } else {
+      cmhi(SubRegSize.Vector, Dst.Q(), Vector1.Q(), Vector2.Q());
+    }
+  }
+}
+
 DEF_OP(VCMPGTZ) {
   const auto Op = IROp->C<IR::IROp_VCMPGTZ>();
   const auto OpSize = IROp->Size;
@@ -3833,6 +3888,171 @@ DEF_OP(VMul) {
   }
 }
 
+DEF_OP(VUSDot) {
+  ///< Dest = Acc + dot(Vector1 (unsigned 8-bit), Vector2 (signed 8-bit))
+  // Matches:
+  // - SVE    - USDOT
+  // - ASIMD  - USDOT
+  const auto Op = IROp->C<IR::IROp_VUSDot>();
+  const auto OpSize = IROp->Size;
+
+  const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
+  LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
+
+  const auto Dst = GetVReg(Node);
+  const auto Acc = GetVReg(Op->Acc);
+  const auto Vector1 = GetVReg(Op->Vector1);
+  const auto Vector2 = GetVReg(Op->Vector2);
+
+  // USDOT accumulates in to its destination register,
+  // so we need to emit a move if Acc != Dst
+  ARMEmitter::VRegister DestTmp = Dst;
+  if (Dst != Acc) {
+    if (Dst != Vector1 && Dst != Vector2) {
+      DestTmp = Dst;
+    } else {
+      DestTmp = VTMP1;
+    }
+  }
+
+  if (HostSupportsSVE256 && Is256Bit) {
+    if (Dst != Acc) {
+      mov(DestTmp.Z(), Acc.Z());
+    }
+
+    usdot(DestTmp.Z(), Vector1.Z(), Vector2.Z());
+    if (Dst != DestTmp) {
+      mov(Dst.Z(), DestTmp.Z());
+    }
+  } else {
+    if (Dst != Acc) {
+      mov(DestTmp.Q(), Acc.Q());
+    }
+
+    usdot(DestTmp.Q(), Vector1.Q(), Vector2.Q());
+    if (Dst != DestTmp) {
+      mov(Dst.Q(), DestTmp.Q());
+    }
+  }
+}
+
+DEF_OP(VSDot) {
+  ///< Dest = Acc + dot(Vector1 (signed 8-bit), Vector2 (signed 8-bit))
+  // Matches:
+  // - SVE    - SDOT
+  // - ASIMD  - SDOT
+  const auto Op = IROp->C<IR::IROp_VSDot>();
+  const auto OpSize = IROp->Size;
+
+  const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
+  LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
+
+  const auto Dst = GetVReg(Node);
+  const auto Acc = GetVReg(Op->Acc);
+  const auto Vector1 = GetVReg(Op->Vector1);
+  const auto Vector2 = GetVReg(Op->Vector2);
+
+  // SDOT accumulates in to its destination register,
+  // so we need to emit a move if Acc != Dst
+  ARMEmitter::VRegister DestTmp = Dst;
+  if (Dst != Acc) {
+    if (Dst != Vector1 && Dst != Vector2) {
+      DestTmp = Dst;
+    } else {
+      DestTmp = VTMP1;
+    }
+  }
+
+  if (HostSupportsSVE256 && Is256Bit) {
+    if (Dst != Acc) {
+      mov(DestTmp.Z(), Acc.Z());
+    }
+
+    sdot(ARMEmitter::SubRegSize::i32Bit, DestTmp.Z(), Vector1.Z(), Vector2.Z());
+    if (Dst != DestTmp) {
+      mov(Dst.Z(), DestTmp.Z());
+    }
+  } else {
+    if (Dst != Acc) {
+      mov(DestTmp.Q(), Acc.Q());
+    }
+
+    sdot(ARMEmitter::SubRegSize::i32Bit, DestTmp.Q(), Vector1.Q(), Vector2.Q());
+    if (Dst != DestTmp) {
+      mov(Dst.Q(), DestTmp.Q());
+    }
+  }
+}
+
+DEF_OP(VSAddLP) {
+  const auto Op = IROp->C<IR::IROp_VSAddLP>();
+  const auto OpSize = IROp->Size;
+
+  const auto SubRegSize = ConvertSubRegSize248(IROp);
+  const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
+  LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
+
+  const auto Dst = GetVReg(Node);
+  const auto Vector = GetVReg(Op->Vector);
+
+  if (HostSupportsSVE256 && Is256Bit) {
+    // SVE only has the accumulating form, so accumulate in to a zeroed register.
+    // Zero a temporary instead if Dst aliases the source.
+    const auto DestTmp = Dst == Vector ? VTMP1 : Dst;
+    dup_imm(SubRegSize, DestTmp.Z(), 0);
+    sadalp(SubRegSize, DestTmp.Z(), PRED_TMP_32B.Merging(), Vector.Z());
+    if (Dst != DestTmp) {
+      mov(Dst.Z(), DestTmp.Z());
+    }
+  } else {
+    saddlp(SubRegSize, Dst.Q(), Vector.Q());
+  }
+}
+
+DEF_OP(VSAdALP) {
+  const auto Op = IROp->C<IR::IROp_VSAdALP>();
+  const auto OpSize = IROp->Size;
+
+  const auto SubRegSize = ConvertSubRegSize248(IROp);
+  const auto Is256Bit = OpSize == IR::OpSize::i256Bit;
+  LOGMAN_THROW_A_FMT(!Is256Bit || HostSupportsSVE256, "Need SVE256 support in order to use {} with 256-bit operation", __func__);
+
+  const auto Dst = GetVReg(Node);
+  const auto Acc = GetVReg(Op->Acc);
+  const auto Vector = GetVReg(Op->Vector);
+
+  // SADALP accumulates in to its destination register,
+  // so we need to emit a move if Acc != Dst
+  ARMEmitter::VRegister DestTmp = Dst;
+  if (Dst != Acc) {
+    DestTmp = Dst != Vector ? Dst : VTMP1;
+  }
+
+  if (HostSupportsSVE256 && Is256Bit) {
+    if (Dst != Acc) {
+      mov(DestTmp.Z(), Acc.Z());
+    }
+
+    sadalp(SubRegSize, DestTmp.Z(), PRED_TMP_32B.Merging(), Vector.Z());
+    if (Dst != DestTmp) {
+      mov(Dst.Z(), DestTmp.Z());
+    }
+  } else {
+    if (Dst == Vector) {
+      // ASIMD has the non-accumulating form, which is cheaper than shuffling through a temporary.
+      saddlp(SubRegSize, VTMP1.Q(), Vector.Q());
+      add(SubRegSize, Dst.Q(), Acc.Q(), VTMP1.Q());
+      return;
+    }
+
+    if (Dst != Acc) {
+      mov(Dst.Q(), Acc.Q());
+    }
+
+    sadalp(SubRegSize, Dst.Q(), Vector.Q());
+  }
+}
+
 DEF_OP(VUMull) {
   const auto Op = IROp->C<IR::IROp_VUMull>();
   const auto OpSize = IROp->Size;

FEXCore/Source/Interface/Core/OpcodeDispatcher.h

@@ -207,9 +207,12 @@ public:
     FlushRegisterCache();
     return _ExitFunction(GetOpSize(NewRIP), NewRIP, Hint, CallReturnAddress, CallReturnBlock, PatchSiteAddress, PatchSiteSize);
   }
-  IRPair<IROp_Break> Break(BreakDefinition Reason) {
+  IRPair<IROp_Break> Break(BreakDefinition Reason, Ref RelocatedPC = nullptr) {
     FlushRegisterCache();
-    return _Break(Reason);
+    if (!RelocatedPC) {
+      RelocatedPC = Constant(0);
+    }
+    return _Break(Reason, RelocatedPC);
   }
   IRPair<IROp_Thunk> Thunk(Ref ArgPtr, SHA256Sum ThunkNameHash) {
     FlushRegisterCache();
@@ -217,6 +220,8 @@ public:
   }
 
   bool FinishOp(uint64_t NextRIP, bool LastOp) {
+    ++RegCache.Index;
+
     // If we are switching to a new block and this current block has yet to set a RIP
     // Then we need to insert an unconditional jump from the current block to the one we are going to
     // This happens most frequently when an instruction jumps backwards to another location
@@ -363,7 +368,8 @@ public:
   void MOVVectorUnalignedOp(OpcodeArgs);
   void MOVVectorUnalignedNoNopOp(OpcodeArgs);
   void MOVVectorNTOp(OpcodeArgs, bool IsAVX);
-  void ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx);
+  void ALURAXOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp);
+  void ALUOp(OpcodeArgs, FEXCore::IR::IROps ALUIROp, FEXCore::IR::IROps AtomicFetchOp, unsigned SrcIdx, bool DestRAX);
   void LSLOp(OpcodeArgs);
   void INTOp(OpcodeArgs);
   void SyscallOp(OpcodeArgs, bool IsSyscallInst);
@@ -374,8 +380,8 @@ public:
   void IRETOp(OpcodeArgs);
   void CallbackReturnOp(OpcodeArgs);
   void SecondaryALUOp(OpcodeArgs);
-  void ADCOp(OpcodeArgs, uint32_t SrcIndex);
-  void SBBOp(OpcodeArgs, uint32_t SrcIndex);
+  void ADCOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX);
+  void SBBOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX);
   void SALCOp(OpcodeArgs);
   void PUSHOp(OpcodeArgs);
   void PUSHREGOp(OpcodeArgs);
@@ -395,16 +401,18 @@ public:
   void JUMPFARIndirectOp(OpcodeArgs);
   void CALLFARIndirectOp(OpcodeArgs);
   void RETFARIndirectOp(OpcodeArgs);
-  void TESTOp(OpcodeArgs, uint32_t SrcIndex);
+  void TESTOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX);
   void ARPLOp(OpcodeArgs);
   void MOVSXDOp(OpcodeArgs);
   void MOVSXOp(OpcodeArgs);
   void MOVZXOp(OpcodeArgs);
-  void CMPOp(OpcodeArgs, uint32_t SrcIndex);
+  void CMPOp(OpcodeArgs, uint32_t SrcIndex, bool DestRAX);
   void SETccOp(OpcodeArgs);
   void CQOOp(OpcodeArgs);
   void CDQOp(OpcodeArgs);
+  std::optional<Ref> XCHGOpImpl(OpcodeArgs, Ref Src);
   void XCHGOp(OpcodeArgs);
+  void XCHGRAXOp(OpcodeArgs);
   void SAHFOp(OpcodeArgs);
   void LAHFOp(OpcodeArgs);
   void MOVSegOp(OpcodeArgs, bool ToSeg);
@@ -426,11 +434,11 @@ public:
   void RotateOp(OpcodeArgs, bool Left, bool IsImmediate, bool Is1Bit);
   void RCROp1Bit(OpcodeArgs);
   void RCROp8x1Bit(OpcodeArgs);
-  void RCROp(OpcodeArgs);
-  void RCRSmallerOp(OpcodeArgs);
+  void RCROp(OpcodeArgs, bool UseRCX);
+  void RCRSmallerOp(OpcodeArgs, bool UseRCX);
   void RCLOp1Bit(OpcodeArgs);
-  void RCLOp(OpcodeArgs);
-  void RCLSmallerOp(OpcodeArgs);
+  void RCLOp(OpcodeArgs, bool UseRCX);
+  void RCLSmallerOp(OpcodeArgs, bool UseRCX);
 
   void BTOp(OpcodeArgs, uint32_t SrcIndex, enum BTAction Action);
 
@@ -485,6 +493,7 @@ public:
   void SGDTOp(OpcodeArgs);
   void SIDTOp(OpcodeArgs);
   void SMSWOp(OpcodeArgs);
+  void SLDTOp(OpcodeArgs);
 
   enum class VectorOpType {
     MMX,
@@ -665,6 +674,8 @@ public:
 
   void VPMADDUBSWOp(OpcodeArgs);
   void VPMADDWDOp(OpcodeArgs);
+  void VPDPBUSDOp(OpcodeArgs, bool Saturating);
+  void VPDPWSSDOp(OpcodeArgs, bool Saturating);
 
   void VPMASKMOVOp(OpcodeArgs, bool IsStore);
 
@@ -741,7 +752,7 @@ public:
   void X87FLDCW(OpcodeArgs);
   void X87FNSAVE(OpcodeArgs);
   void X87FNSTENV(OpcodeArgs);
-  void X87FNSTSW(OpcodeArgs);
+  void X87FNSTSW(OpcodeArgs, bool DestRAX);
   void X87FRSTOR(OpcodeArgs);
   void X87FSTCW(OpcodeArgs);
   void X87FXAM(OpcodeArgs);
@@ -750,6 +761,7 @@ public:
   void X87LDENV(OpcodeArgs);
   void X87ModifySTP(OpcodeArgs, bool Inc);
   void X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2);
+  void X87NOP(OpcodeArgs);
 
   enum class FCOMIFlags {
     FLAGS_X87,
@@ -823,6 +835,7 @@ public:
   void LoadFenceOrXRSTOR(OpcodeArgs);
   void MemFenceOrXSAVEOPT(OpcodeArgs);
   void StoreFenceOrCLFlush(OpcodeArgs);
+  void SerializeOp(OpcodeArgs);
   void UMonitorOrCLRSSBSY(OpcodeArgs);
   void UMWaitOp(OpcodeArgs);
   void CLZeroOp(OpcodeArgs);
@@ -889,7 +902,8 @@ public:
   void Extrq(OpcodeArgs);
   void Insertq(OpcodeArgs);
 
-  void BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition);
+  void BreakOp(OpcodeArgs, FEXCore::IR::BreakDefinition BreakDefinition, Ref RelocatedPC = nullptr);
+  void BreakpointOp(OpcodeArgs, uint32_t BP);
   void UnimplementedOp(OpcodeArgs);
   void PermissionRestrictedOp(OpcodeArgs);
 
@@ -1036,6 +1050,9 @@ public:
 
   void AVX128_VPMADDUBSW(OpcodeArgs);
   void AVX128_VPMADDWD(OpcodeArgs);
+  void AVX128_VPDPImpl(OpcodeArgs, std::function<Ref(Ref Acc, Ref Src1, Ref Src2)> Helper);
+  void AVX128_VPDPBUSD(OpcodeArgs, bool Saturating);
+  void AVX128_VPDPWSSD(OpcodeArgs, bool Saturating);
 
   void AVX128_VBLEND(OpcodeArgs, IR::OpSize ElementSize);
 
@@ -1076,13 +1093,19 @@ public:
   void AVX128_VFMAScalarImpl(OpcodeArgs, IROps IROp, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
   void AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Src1Idx, uint8_t Src2Idx, uint8_t AddendIdx);
 
-  RefPair AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB);
-  RefPair AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask, RefVSIB VSIB);
+  RefPair AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB, bool MaskAllOnes);
+  RefPair AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize, RefPair Dest, RefPair Mask,
+                              RefVSIB VSIB, bool MaskAllOnesLow = false, bool MaskAllOnesHigh = false);
 
   void AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize);
 
   void AVX128_VCVTPH2PS(OpcodeArgs);
   void AVX128_VCVTPS2PH(OpcodeArgs);
+  void AVX128_VBCSTNEBF162PS(OpcodeArgs);
+  void AVX128_VBCSTNESH2PS(OpcodeArgs);
+  void AVX128_VCVTNEEBF162PS(OpcodeArgs, bool Even);
+  void AVX128_VCVTNEEPH2PS(OpcodeArgs, bool Even);
+  void AVX128_VCVTNEPS2BF16(OpcodeArgs);
 
   // End of AVX 128-bit implementation
 
@@ -1117,10 +1140,16 @@ public:
   void AVXVectorUnaryOp(OpcodeArgs, IROps IROp, IR::OpSize ElementSize);
   void AVXVectorVariableBlend(OpcodeArgs, IR::OpSize ElementSize);
 
+  void AVXVBCSTNEBF162PS(OpcodeArgs);
+  void AVXVBCSTNESH2PS(OpcodeArgs);
+  void AVXVCVTNEEBF162PS(OpcodeArgs, bool Even);
+  void AVXVCVTNEEPH2PS(OpcodeArgs, bool Even);
+  void AVXVCVTNEPS2BF16(OpcodeArgs);
+
   // End of AVX 256-bit implementation
 
   void InvalidOp(OpcodeArgs);
-  void NoExecOp(OpcodeArgs);
+  void NoExecOrPartialOp(OpcodeArgs);
 
   void SetPackedRFLAG(bool Lower8, Ref Src);
   Ref GetPackedRFLAG(uint32_t FlagsMask = ~0U);
@@ -1170,6 +1199,21 @@ public:
     // At block boundaries, fix up the carry flag.
     if (!SRAOnly) {
       RectifyCarryInvert(CFInvertedABI);
+
+      // Also we can't trust the LastDef or PrevDef arrays accross block boundaries, so clear them as well.
+      for (auto& Def : RegCache.LastDef) {
+        Def = nullptr;
+      }
+      for (auto& Def : RegCache.PrevDef) {
+        Def = nullptr;
+      }
+      for (auto& DefIndex : RegCache.LastDefIndex) {
+        DefIndex = 0;
+      }
+      for (auto& DefIndex : RegCache.PrevDefIndex) {
+        DefIndex = 0;
+      }
+      RegCache.Index = 0;
     }
 
     if (!MMXOnly) {
@@ -1286,7 +1330,7 @@ protected:
       /* On ASIMD platforms, the emulation happens to preserve NZCV, unlike the
        * more optimal SVE implementation that clobbers.
        */
-      if (!CTX->HostFeatures.SupportsSVE128 && !CTX->HostFeatures.SupportsSVE256) {
+      if (!CTX->HostFeatures.SupportsSVE()) {
         return;
       }
 
@@ -1307,6 +1351,7 @@ protected:
 
 private:
   FEX_CONFIG_OPT(ReducedPrecisionMode, X87REDUCEDPRECISION);
+  FEX_CONFIG_OPT(UndefinedFlagAcidTest, UNDEFINEDFLAGACIDTEST);
 
   struct JumpTargetInfo {
     Ref BlockEntry;
@@ -1384,6 +1429,11 @@ private:
                     const X86Tables::DecodedOperand& Imm, bool IsAVX);
 
   void PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask, bool IsAVX);
+  Ref PCMPXSTRXSaturateExplicitLength(IR::OpSize ElementSize, IR::OpSize LengthSize, Ref RawLength);
+  Ref PCMPXSTRXEqualAny(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements);
+  Ref PCMPXSTRXRanges(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements, bool IsSigned);
+  Ref PCMPXSTRXEqualEach(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements);
+  Ref PCMPXSTRXEqualOrdered(IR::OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1Length, Ref Src2Length, Ref Src1ValidElements, Ref Indices);
 
   Ref PHADDSOpImpl(OpSize Size, Ref Src1, Ref Src2);
 
@@ -1400,6 +1450,10 @@ private:
 
   Ref PMADDUBSWOpImpl(IR::OpSize Size, Ref Src1, Ref Src2);
 
+  Ref VPDPBUSDOpImpl(IR::OpSize Size, Ref Acc, Ref Src1, Ref Src2, bool Saturating);
+
+  Ref VPDPWSSDOpImpl(IR::OpSize Size, Ref Acc, Ref Src1, Ref Src2, bool Saturating);
+
   Ref PMULHRSWOpImpl(OpSize Size, Ref Src1, Ref Src2);
 
   Ref PMULHWOpImpl(OpcodeArgs, bool Signed, Ref Src1, Ref Src2);
@@ -1480,7 +1534,7 @@ private:
   void SaveAVXState(Ref MemBase);
 
   void XRstorOpImpl(OpcodeArgs);
-  void RestoreX87State(Ref MemBase);
+  void RestoreX87State(OpcodeArgs, Ref MemBase);
   void RestoreSSEState(Ref MemBase);
   void RestoreMXCSRState(Ref MXCSR);
   void RestoreAVXState(Ref MemBase);
@@ -1501,6 +1555,20 @@ private:
   void StoreGPRRegister(uint32_t GPR, const Ref Src, IR::OpSize Size = OpSize::iInvalid, uint8_t Offset = 0);
   void StoreXMMRegister(uint32_t XMM, const Ref Src);
 
+  // Matches semantics around GPR storing that matches `StoreResult_WithOpSize` behaviour.
+  void StoreGPRResultWithZExtSemantics(uint32_t GPR, Ref Src, IR::OpSize OpSize) {
+    const auto GPRSize = GetGPROpSize();
+
+    if (GPRSize == OpSize::i64Bit && OpSize == OpSize::i32Bit) {
+      // If the Source IR op is 64 bits, we need to zext the upper bits
+      // For all other sizes, the upper bits are guaranteed to already be zero
+      Src = GetOpSize(Src) == OpSize::i64Bit ? ARef(Src).Bfe(0, 32).Ref() : Src;
+      StoreGPRRegister(GPR, Src, GPRSize);
+    } else {
+      StoreGPRRegister(GPR, Src, std::min(GPRSize, OpSize));
+    }
+  }
+
   Ref _GetRelocatedPC(const FEXCore::X86Tables::DecodedOp& Op, int64_t Offset, bool Inline) {
     const auto GPRSize = GetGPROpSize();
     const auto Offs = Op->PC + Op->InstSize + Offset - Entry;
@@ -1609,7 +1677,7 @@ private:
   }
 
   constexpr OpSize GetGuestVectorLength() const {
-    return (CTX->HostFeatures.SupportsSVE256 && CTX->HostFeatures.SupportsAVX) ? OpSize::i256Bit : OpSize::i128Bit;
+    return (CTX->HostFeatures.SupportsSVE256() && CTX->HostFeatures.SupportsAVX()) ? OpSize::i256Bit : OpSize::i128Bit;
   }
 
   [[nodiscard]]
@@ -1867,6 +1935,55 @@ private:
     InvalidateReg(Core::CPUState::AF_AS_GREG);
   }
 
+  // Sets the desired flags to zero when acid test is enabled.
+  // Not meant to be fast, just meant to produce consistent results.
+  void AcidTestFlags(unsigned BitOffset = 0, unsigned BitOffset2 = 0, unsigned BitOffset3 = 0, unsigned BitOffset4 = 0,
+                     unsigned BitOffset5 = 0, unsigned BitOffset6 = 0) {
+    if (!UndefinedFlagAcidTest) [[likely]] {
+      return;
+    }
+
+    unsigned BitFlags {};
+    if (BitOffset) {
+      BitFlags |= 1U << BitOffset;
+    }
+    if (BitOffset2) {
+      BitFlags |= 1U << BitOffset2;
+    }
+    if (BitOffset3) {
+      BitFlags |= 1U << BitOffset3;
+    }
+    if (BitOffset4) {
+      BitFlags |= 1U << BitOffset4;
+    }
+    if (BitOffset5) {
+      BitFlags |= 1U << BitOffset5;
+    }
+    if (BitOffset6) {
+      BitFlags |= 1U << BitOffset6;
+    }
+
+    auto Flag = Constant(0);
+    if (BitFlags & (1U << X86State::RFLAG_SF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_SF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_ZF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_ZF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_CF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_CF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_OF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_OF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_AF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_AF_RAW_LOC);
+    }
+    if (BitFlags & (1U << X86State::RFLAG_AF_RAW_LOC)) {
+      SetRFLAG(Flag, FEXCore::X86State::RFLAG_AF_RAW_LOC);
+    }
+  }
+
   [[nodiscard]]
   static CondClass CondForNZCVBit(unsigned BitOffset, bool Invert) {
     switch (BitOffset) {
@@ -1932,12 +2049,25 @@ private:
     uint64_t Partial;
 
     Ref Value[64];
+
+    // Keep track of where each register was last written to within the current block,
+    // for use in peephole optimizations. LastDefIndex is a parallel structure,
+    // which tracks the index of each Op in LastDef.
+    Ref LastDef[64];
+    uint32_t LastDefIndex[64];
+    // Similar to LastDef, but tracks the write before LastDef. Think of it as LastDef^2.
+    Ref PrevDef[64];
+    uint32_t PrevDefIndex[64];
+
+    // The index of the current instruction in the current block.
+    uint32_t Index;
   } RegCache {};
 
   void InvalidateReg(uint8_t Index) {
     uint64_t Bit = (1ull << (uint64_t)Index);
     RegCache.Cached &= ~Bit;
     RegCache.Written &= ~Bit;
+    RegCache.LastDef[Index] = nullptr;
   }
 
   Ref LoadRegCache(uint64_t Offset, uint8_t Index, RegClass Class, IR::OpSize Size) {
@@ -2053,6 +2183,10 @@ private:
     RegCache.Value[Index] = Value;
     RegCache.Cached |= Bit;
     RegCache.Written |= Bit;
+    RegCache.PrevDef[Index] = RegCache.LastDef[Index];
+    RegCache.PrevDefIndex[Index] = RegCache.LastDefIndex[Index];
+    RegCache.LastDef[Index] = Value;
+    RegCache.LastDefIndex[Index] = RegCache.Index;
   }
 
   void InvalidateHighAVXRegisters() {
@@ -2164,8 +2298,7 @@ private:
     Ref V_inv = GetRFLAG(FEXCore::X86State::RFLAG_OF_RAW_LOC, true);
     SetRFLAG<FEXCore::X86State::RFLAG_PF_RAW_LOC>(V_inv);
 
-    // Intel: OF, SF, and AF set to zero
-    // AMD:   no mention of OF, SF and AF but actual hardware seems to always zero
+    // Both Intel and AMD manual state that OF, SF, and AF are set to zero.
     //
     // Zero AF. Note that the comparison sets the raw PF to 0/1 above, so
     // PF[4] is 0 so the XOR with PF will have no effect, so setting the AF
@@ -2396,6 +2529,12 @@ private:
   void SetX87FTW(Ref FTW);
   Ref GetX87FTW_Helper();
   void SetX87Top(Ref Value);
+  void StoreX87FIP(FEXCore::X86Tables::DecodedOp Op) {
+    if (CTX->Config.JITFeatures & FEXCore::Config::JITFeatures::ENABLEX87FIP) {
+      // Only store FIP if the feature is enabled.
+      _StoreContextGPR(OpSize::i64Bit, GetRelocatedPC(Op, -Op->InstSize), offsetof(FEXCore::Core::CPUState, FIP));
+    }
+  }
 
   void ChgStateX87_MMX() override {
     LOGMAN_THROW_A_FMT(MMXState == MMXState_X87, "Expected state to be x87");

FEXCore/Source/Interface/Core/OpcodeDispatcher/AVX_128.cpp

@@ -1470,6 +1470,35 @@ void OpDispatchBuilder::AVX128_VPMADDWD(OpcodeArgs) {
                           [this](IR::OpSize _ElementSize, Ref Src1, Ref Src2) { return PMADDWDOpImpl(OpSize::i128Bit, Src1, Src2); });
 }
 
+void OpDispatchBuilder::AVX128_VPDPImpl(OpcodeArgs, std::function<Ref(Ref Acc, Ref Src1, Ref Src2)> Helper) {
+  const auto Size = OpSizeFromDst(Op);
+  const auto Is128Bit = Size == OpSize::i128Bit;
+
+  auto Acc = AVX128_LoadSource_WithOpSize(Op, Op->Dest, Op->Flags, !Is128Bit);
+  auto Src1 = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);
+  auto Src2 = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);
+
+  RefPair Result {};
+  Result.Low = Helper(Acc.Low, Src1.Low, Src2.Low);
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Helper(Acc.High, Src1.High, Src2.High);
+  }
+
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VPDPBUSD(OpcodeArgs, bool Saturating) {
+  AVX128_VPDPImpl(Op,
+                  [this, Saturating](Ref Acc, Ref Src1, Ref Src2) { return VPDPBUSDOpImpl(OpSize::i128Bit, Acc, Src1, Src2, Saturating); });
+}
+
+void OpDispatchBuilder::AVX128_VPDPWSSD(OpcodeArgs, bool Saturating) {
+  AVX128_VPDPImpl(Op,
+                  [this, Saturating](Ref Acc, Ref Src1, Ref Src2) { return VPDPWSSDOpImpl(OpSize::i128Bit, Acc, Src1, Src2, Saturating); });
+}
+
 void OpDispatchBuilder::AVX128_VBLEND(OpcodeArgs, IR::OpSize ElementSize) {
   const auto SrcSize = OpSizeFromSrc(Op);
   const auto Is128Bit = SrcSize == OpSize::i128Bit;
@@ -2023,7 +2052,8 @@ void OpDispatchBuilder::AVX128_VFMAddSubImpl(OpcodeArgs, bool AddSub, uint8_t Sr
 }
 
 OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, OpSize Size, OpSize ElementLoadSize, OpSize AddrElementSize,
-                                                                  RefPair Dest, RefPair Mask, RefVSIB VSIB) {
+                                                                  RefPair Dest, RefPair Mask, RefVSIB VSIB, bool MaskAllOnesLow,
+                                                                  bool MaskAllOnesHigh) {
   LOGMAN_THROW_A_FMT(AddrElementSize == OpSize::i32Bit || AddrElementSize == OpSize::i64Bit, "Unknown address element size");
   const auto Is128Bit = Size == OpSize::i128Bit;
 
@@ -2037,7 +2067,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
     BaseAddr = Invalid();
   }
 
-  if (CTX->HostFeatures.SupportsSVE128) {
+  if (CTX->HostFeatures.SupportsSVE128()) {
     if (ElementLoadSize == OpSize::i64Bit && AddrElementSize == OpSize::i32Bit) {
       // In the case that FEX is loading double the amount of data than the number of address bits then we can optimize this case.
       // For 256-bits of data we need to sign extend all four 32-bit address elements to be 64-bit.
@@ -2073,7 +2103,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
   RefPair Result {};
   ///< Calculate the low-half.
   Result.Low = _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, Dest.Low, Mask.Low, BaseAddr, VSIB.Low, VSIB.High,
-                                        AddrElementSize, VSIB.Scale, 0, 0, AddrSize);
+                                        AddrElementSize, VSIB.Scale, 0, 0, AddrSize, MaskAllOnesLow);
 
   if (Is128Bit) {
     Result.High = LoadZeroVector(OpSize::i128Bit);
@@ -2088,6 +2118,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
 
     Ref DestReg = Dest.High;
     Ref MaskReg = Mask.High;
+    bool MaskAllOnes = MaskAllOnesHigh;
     uint8_t IndexElementOffset {};
     uint8_t DataElementOffset {};
     if (AddrElementSize == ElementLoadSize) {
@@ -2104,13 +2135,15 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
       AddrAddressing.High = Invalid();
       DestReg = Result.Low; ///< Start mixing with the low register.
       MaskReg = Mask.Low;   ///< Mask starts with the low mask here.
+      MaskAllOnes = MaskAllOnesLow;
       IndexElementOffset = 0;
       DataElementOffset = IR::NumElements(OpSize::i128Bit, ElementLoadSize) / 2;
     }
 
     ///< Calculate the high-half.
-    auto ResultHigh = _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, DestReg, MaskReg, BaseAddr, AddrAddressing.Low,
-                                               AddrAddressing.High, AddrElementSize, VSIB.Scale, DataElementOffset, IndexElementOffset, AddrSize);
+    auto ResultHigh =
+      _VLoadVectorGatherMasked(OpSize::i128Bit, ElementLoadSize, DestReg, MaskReg, BaseAddr, AddrAddressing.Low, AddrAddressing.High,
+                               AddrElementSize, VSIB.Scale, DataElementOffset, IndexElementOffset, AddrSize, MaskAllOnes);
 
     if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {
       // If we only fetched 128-bits worth of data then the upper-result is all zero.
@@ -2123,7 +2156,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherImpl(OpcodeArgs, Op
   return Result;
 }
 
-OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB) {
+OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs, Ref Dest, Ref Mask, RefVSIB VSIB, bool MaskAllOnes) {
 
   ///< BaseAddr doesn't need to exist, calculate that here.
   Ref BaseAddr = VSIB.BaseAddr;
@@ -2137,7 +2170,7 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs,
 
   bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (BaseAddr == Invalid() && VSIB.Scale != 1);
 
-  if (CTX->HostFeatures.SupportsSVE128 && NeedsSVEScale) {
+  if (CTX->HostFeatures.SupportsSVE128() && NeedsSVEScale) {
     // SVE gather instructions don't support scaling their vector elements by anything other than 1 or the address element size.
     // Pre-scale 64-bit addresses in the case that scale doesn't match in-order to hit SVE code paths more frequently.
     // Only hit this path if the host supports SVE. Otherwise it's a degradation for the ASIMD codepath.
@@ -2155,7 +2188,8 @@ OpDispatchBuilder::RefPair OpDispatchBuilder::AVX128_VPGatherQPSImpl(OpcodeArgs,
   auto AddrSize = (Op->Flags & X86Tables::DecodeFlags::FLAG_ADDRESS_SIZE) != 0 ? (GPRSize >> 1) : GPRSize;
 
   ///< Calculate the low-half.
-  Result.Low = _VLoadVectorGatherMaskedQPS(OpSize::i128Bit, OpSize::i32Bit, Dest, Mask, BaseAddr, VSIB.Low, VSIB.High, VSIB.Scale, AddrSize);
+  Result.Low =
+    _VLoadVectorGatherMaskedQPS(OpSize::i128Bit, OpSize::i32Bit, Dest, Mask, BaseAddr, VSIB.Low, VSIB.High, VSIB.Scale, AddrSize, MaskAllOnes);
   Result.High = LoadZeroVector(OpSize::i128Bit);
   if (VSIB.High == Invalid()) {
     // Special case for only loading two floats.
@@ -2184,10 +2218,25 @@ void OpDispatchBuilder::AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize) {
   auto VSIB = AVX128_LoadVSIB(Op, Op->Src[0], Op->Flags, NeedsHighAddrBytes);
   auto Mask = AVX128_LoadSource_WithOpSize(Op, Op->Src[1], Op->Flags, !Is128Bit);
 
+  // Check if the mask was computed from a self comparison, which indicates an all-ones mask.
+  // This is a compiler idiom for an unmasked gather. In this case, we can apply a peephole and skip
+  // all mask checks in the codegen, reducing codesize substantially.
+  bool MaskAllOnesLow = false;
+  bool MaskAllOnesHigh = false;
+  if (Op->Src[1].IsGPR()) {
+    const auto MaskXMM = Op->Src[1].Data.GPR.GPR - X86State::REG_XMM_0;
+
+    const Ref LowDef = RegCache.LastDef[FPR0Index + MaskXMM];
+    const Ref HighDef = RegCache.LastDef[AVXHigh0Index + MaskXMM];
+
+    MaskAllOnesLow = LowDef && IsValueVectorAllOnes(WrapNode(LowDef));
+    MaskAllOnesHigh = !Is128Bit && HighDef && IsValueVectorAllOnes(WrapNode(HighDef));
+  }
+
   bool NeedsSVEScale = (VSIB.Scale == 2 || VSIB.Scale == 8) || (VSIB.BaseAddr == Invalid() && VSIB.Scale != 1);
 
   const bool NeedsExplicitSVEPath =
-    CTX->HostFeatures.SupportsSVE128 && AddrElementSize == OpSize::i32Bit && ElementLoadSize == OpSize::i32Bit && NeedsSVEScale;
+    CTX->HostFeatures.SupportsSVE128() && AddrElementSize == OpSize::i32Bit && ElementLoadSize == OpSize::i32Bit && NeedsSVEScale;
 
   RefPair Result {};
   if (NeedsExplicitSVEPath) {
@@ -2214,15 +2263,15 @@ void OpDispatchBuilder::AVX128_VPGATHER(OpcodeArgs, OpSize AddrElementSize) {
     }
 
     ///< AddressElementSize is now OpSize::i64Bit
-    Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIBLow);
+    Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIBLow, MaskAllOnesLow);
     if (NeedsHighAddrBytes) {
-      auto Res = AVX128_VPGatherQPSImpl(Op, Dest.High, Mask.High, VSIBHigh);
+      auto Res = AVX128_VPGatherQPSImpl(Op, Dest.High, Mask.High, VSIBHigh, MaskAllOnesHigh);
       Result.High = Res.Low;
     }
   } else if (AddrElementSize == OpSize::i64Bit && ElementLoadSize == OpSize::i32Bit) {
-    Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIB);
+    Result = AVX128_VPGatherQPSImpl(Op, Dest.Low, Mask.Low, VSIB, MaskAllOnesLow);
   } else {
-    Result = AVX128_VPGatherImpl(Op, Size, ElementLoadSize, AddrElementSize, Dest, Mask, VSIB);
+    Result = AVX128_VPGatherImpl(Op, Size, ElementLoadSize, AddrElementSize, Dest, Mask, VSIB, MaskAllOnesLow, MaskAllOnesHigh);
   }
   AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
 
@@ -2306,4 +2355,124 @@ void OpDispatchBuilder::AVX128_VCVTPS2PH(OpcodeArgs) {
   }
 }
 
+void OpDispatchBuilder::AVX128_VBCSTNEBF162PS(OpcodeArgs) {
+  const auto DstSize = GetDstSize(Op);
+  const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
+
+  auto Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags);
+
+  // bfloat16 is converted to fp32 and broadcast through the vector.
+  // bfloat16->float32 is left-shift by 16.
+  Src1 = _VShlI(OpSize::i128Bit, OpSize::i32Bit, Src1, 16);
+  Src1 = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src1, 0);
+
+  RefPair Result {};
+  Result.Low = Src1;
+
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Result.Low;
+  }
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VBCSTNESH2PS(OpcodeArgs) {
+  const auto DstSize = GetDstSize(Op);
+  const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
+
+  auto Src1 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i16Bit, Op->Flags);
+
+  // float16 is converted to fp32 and broadcast through the vector.
+  Src1 = _Float_FToF(OpSize::i32Bit, OpSize::i16Bit, Src1);
+  Src1 = _VDupElement(OpSize::i128Bit, OpSize::i32Bit, Src1, 0);
+
+  RefPair Result {};
+  Result.Low = Src1;
+
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Result.Low;
+  }
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VCVTNEEBF162PS(OpcodeArgs, bool Even) {
+  const auto DstSize = GetDstSize(Op);
+  const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
+
+  auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);
+
+  RefPair Result {};
+
+  auto Convert = [&](auto Src, bool Even) -> Ref {
+    // Even or odd elements of each 128-bit register is converted from bfloat16 to float32.
+    // For even, just left shift the elements.
+    if (Even) {
+      return _VShlI(OpSize::i128Bit, OpSize::i32Bit, Src, 16);
+    }
+
+    // For Odds, we just need to mask off the lower 16-bits of each 32-bit element.
+    // Easiest way to do this is with a right then left shift.
+    auto Tmp = _VUShrI(OpSize::i128Bit, OpSize::i32Bit, Src, 16);
+    return _VShlI(OpSize::i128Bit, OpSize::i32Bit, Tmp, 16);
+  };
+
+  Result.Low = Convert(Src.Low, Even);
+
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Convert(Src.High, Even);
+  }
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VCVTNEEPH2PS(OpcodeArgs, bool Even) {
+  const auto DstSize = GetDstSize(Op);
+  const auto Is128Bit = DstSize == Core::CPUState::XMM_SSE_REG_SIZE;
+
+  auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128Bit);
+
+  RefPair Result {};
+
+  auto Convert = [&](auto Src, bool Even) -> Ref {
+    // Transpose even or odd elements and then fcvtl.
+    if (Even) {
+      Src = _VUnZip(OpSize::i128Bit, OpSize::i16Bit, Src, Src);
+    } else {
+      Src = _VUnZip2(OpSize::i128Bit, OpSize::i16Bit, Src, Src);
+    }
+    return _Vector_FToF(OpSize::i128Bit, OpSize::i32Bit, Src, OpSize::i16Bit);
+  };
+
+  Result.Low = Convert(Src.Low, Even);
+
+  if (Is128Bit) {
+    Result.High = LoadZeroVector(OpSize::i128Bit);
+  } else {
+    Result.High = Convert(Src.High, Even);
+  }
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
+void OpDispatchBuilder::AVX128_VCVTNEPS2BF16(OpcodeArgs) {
+  const auto SrcSize = OpSizeFromSrc(Op);
+  const auto Is128BitSrc = SrcSize == OpSize::i128Bit;
+  auto Src = AVX128_LoadSource_WithOpSize(Op, Op->Src[0], Op->Flags, !Is128BitSrc);
+
+  // Convert 4 or 8 32-bits floats in to bfloat16.
+  // This instruction only operates with rounding mode set to `Rounds to Nearest (even)`.
+
+  RefPair Result {};
+  Result.Low = _VBFCVTN(OpSize::i128Bit, Src.Low);
+  if (!Is128BitSrc) {
+    Result.Low = _VBFCVTN2(OpSize::i128Bit, Result.Low, Src.High);
+  }
+
+  Result.High = LoadZeroVector(OpSize::i128Bit);
+  AVX128_StoreResult_WithOpSize(Op, Op->Dest, Result);
+}
+
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/BaseTables.h

@@ -5,21 +5,30 @@
 namespace FEXCore::IR {
 constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
   // Instructions
-  {0x00, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0>},
+  {0x00, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0, false>},
+  {0x04, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ADD, FEXCore::IR::IROps::OP_ATOMICFETCHADD, 0, true>},
 
-  {0x08, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_OR, FEXCore::IR::IROps::OP_ATOMICFETCHOR, 0>},
+  {0x08, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_OR, FEXCore::IR::IROps::OP_ATOMICFETCHOR, 0, false>},
+  {0x0c, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_OR, FEXCore::IR::IROps::OP_ATOMICFETCHOR, 0, true>},
 
-  {0x10, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 0>},
+  {0x10, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 0, false>},
+  {0x14, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 0, true>},
 
-  {0x18, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 0>},
+  {0x18, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 0, false>},
+  {0x1c, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 0, true>},
 
-  {0x20, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ANDWITHFLAGS, FEXCore::IR::IROps::OP_ATOMICFETCHAND, 0>},
+  {0x20, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ANDWITHFLAGS, FEXCore::IR::IROps::OP_ATOMICFETCHAND, 0, false>},
+  {0x24, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_ANDWITHFLAGS, FEXCore::IR::IROps::OP_ATOMICFETCHAND, 0, true>},
 
-  {0x28, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_SUB, FEXCore::IR::IROps::OP_ATOMICFETCHSUB, 0>},
+  {0x28, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_SUB, FEXCore::IR::IROps::OP_ATOMICFETCHSUB, 0, false>},
+  {0x2c, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_SUB, FEXCore::IR::IROps::OP_ATOMICFETCHSUB, 0, true>},
 
-  {0x30, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_XOR, FEXCore::IR::IROps::OP_ATOMICFETCHXOR, 0>},
+  {0x30, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_XOR, FEXCore::IR::IROps::OP_ATOMICFETCHXOR, 0, false>},
+  {0x34, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ALUOp, FEXCore::IR::IROps::OP_XOR, FEXCore::IR::IROps::OP_ATOMICFETCHXOR, 0, true>},
+
+  {0x38, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 0, false>},
+  {0x3c, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 0, true>},
 
-  {0x38, 6, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 0>},
   {0x50, 8, &OpDispatchBuilder::PUSHREGOp},
   {0x58, 8, &OpDispatchBuilder::POPOp},
   {0x68, 1, &OpDispatchBuilder::PUSHOp},
@@ -29,7 +38,7 @@ constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
   {0x6C, 4, &OpDispatchBuilder::PermissionRestrictedOp},
 
   {0x70, 16, &OpDispatchBuilder::CondJUMPOp},
-  {0x84, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0>},
+  {0x84, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0, false>},
   {0x86, 2, &OpDispatchBuilder::XCHGOp},
   {0x88, 4, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVGPROp, 0>},
 
@@ -37,7 +46,7 @@ constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
   {0x8D, 1, &OpDispatchBuilder::LEAOp},
   {0x8E, 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::MOVSegOp, true>},
   {0x8F, 1, &OpDispatchBuilder::POPOp},
-  {0x90, 8, &OpDispatchBuilder::XCHGOp},
+  {0x90, 8, &OpDispatchBuilder::XCHGRAXOp},
 
   {0x98, 1, &OpDispatchBuilder::CDQOp},
   {0x99, 1, &OpDispatchBuilder::CQOOp},
@@ -49,7 +58,7 @@ constexpr inline DispatchTableEntry OpDispatch_BaseOpTable[] = {
   {0xA4, 2, &OpDispatchBuilder::MOVSOp},
 
   {0xA6, 2, &OpDispatchBuilder::CMPSOp},
-  {0xA8, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0>},
+  {0xA8, 2, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 0, true>},
   {0xAA, 2, &OpDispatchBuilder::STOSOp},
   {0xAC, 2, &OpDispatchBuilder::LODSOp},
   {0xAE, 2, &OpDispatchBuilder::SCASOp},

FEXCore/Source/Interface/Core/OpcodeDispatcher/Crypto.cpp

@@ -27,7 +27,7 @@ void OpDispatchBuilder::SHA1NEXTEOp(OpcodeArgs) {
   Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
 
   Ref Result {};
-  if (CTX->HostFeatures.SupportsSVE128) {
+  if (CTX->HostFeatures.SupportsSVE128()) {
     auto ZeroVec = LoadZeroVector(OpSize::i128Bit);
     auto Tmp = _VInsElement(OpSize::i128Bit, OpSize::i32Bit, 3, 3, ZeroVec, Dest);
     auto Xar = _VXar(OpSize::i128Bit, OpSize::i32Bit, ZeroVec, Tmp, 2);
@@ -158,11 +158,6 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
     UnimplementedOp(Op);
     return;
   }
-  Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);
-  Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
-  // Hardcoded to XMM0
-  auto XMM0 = LoadXMMRegister(0);
-
   auto shuffle_abcd = [this](Ref Src1, Ref Src2) -> Ref {
     // Generates a suitable SHA256 `abcd` configuration from x86 format.
     auto Tmp = _VZip2(OpSize::i128Bit, OpSize::i64Bit, Src2, Src1);
@@ -175,14 +170,99 @@ void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
     return _VRev64(OpSize::i128Bit, OpSize::i32Bit, Tmp);
   };
 
+  // Peephole: Attempt to identify this pattern:
+  //   sha256rnds2 xmm1, xmm2
+  //   pshufd xmm0, xmm0, 0xe (or psrldq xmm0, 8)
+  //   sha256rnds2 xmm2, xmm1
+  // This is an idiom used by various crypto libraries (OpenSSL etc).
+  // Since the ARM64 SHA256 instructions compute all four rounds,
+  // we can elide the second SHA256RNDS2 and the key shuffle.
+  // Since this matching is done on IR nodes, this is somewhat fragile,
+  // since changes to the translation of the constitiuent instructions
+  // would break the pattern.
+  //
+  // NB: xmm1 and xmm2 are used here as examples, but note that
+  // xmm0 is hardcoded as the key register in SHA256RNDS2.
+
+  if (Op->Dest.IsGPR() && Op->Src[0].IsGPR()) {
+    // xmm1 in our example above.
+    const auto X = Op->Src[0].Data.GPR.GPR - X86State::REG_XMM_0;
+    // xmm2 in our example above.
+    const auto Y = Op->Dest.Data.GPR.GPR - X86State::REG_XMM_0;
+
+    const int KeyIndex = FPR0Index;
+    const int XIndex = FPR0Index + X;
+    const int YIndex = FPR0Index + Y;
+
+    // It is helpful to remember that the current Op
+    // is the "second" SHA256RNDS2 in the example, so we are looking
+    // backwards to find the rest of the pattern.
+    const Ref XLastWritten = RegCache.LastDef[XIndex];
+    const auto XLastWrittenIndex = RegCache.LastDefIndex[XIndex];
+    const Ref KeyLastWritten = RegCache.LastDef[KeyIndex];
+    // xmm0 and X must have been written to in in this block.
+    if (!XLastWritten || !KeyLastWritten) {
+      goto no_peephole;
+    }
+
+    // True if no instruction wrote Y after the first SHA256RNDS2.
+    const bool YUnchanged = !RegCache.LastDef[YIndex] || RegCache.LastDefIndex[YIndex] < XLastWrittenIndex;
+    // True if the key shift is after the first SHA256RNDS2, and no instruction wrote XMM0 afterwards.
+    const bool KeyUnchanged = RegCache.LastDefIndex[KeyIndex] > XLastWrittenIndex &&
+                              (!RegCache.PrevDef[KeyIndex] || RegCache.PrevDefIndex[KeyIndex] < XLastWrittenIndex);
+    if (!YUnchanged || !KeyUnchanged) {
+      goto no_peephole;
+    }
+
+    // Now make sure that the op that wrote the key is what we expect.
+    // This should match the translation of a `pshufd xmm0, xmm0, 0xe` or `psrldq xmm0, 8` instruction.
+    const auto KeyShift = GetOpHeader(WrapNode(KeyLastWritten));
+    const bool IsVExtr = KeyShift->Op == OP_VEXTR && KeyShift->Size == OpSize::i128Bit;
+    if (!IsVExtr) {
+      goto no_peephole;
+    }
+
+    // Compute shift in bytes, since pshufd and psrldq use different element sizes.
+    const auto ShiftInBytes = KeyShift->C<IR::IROp_VExtr>()->Index * IR::OpSizeToSize(KeyShift->ElementSize);
+    const bool IsKeyShift = ShiftInBytes == 8 && IsValueLoadRegister(KeyShift->C<IR::IROp_VExtr>()->VectorUpper, RegClass::FPR, 0);
+    if (!IsKeyShift) {
+      goto no_peephole;
+    }
+
+    // Now check that the rest of the OPs match the pattern.
+    const Ref FirstZip = GetArgIfOp(XLastWritten, OP_VREV64, 0);
+    // A and B are the SHA256H and SHA256H2 output of the first SHA256RNDS2 instruction
+    const Ref A = GetArgIfOp(FirstZip, OP_VZIP2, 1);
+    const Ref B = GetArgIfOp(FirstZip, OP_VZIP2, 0);
+    // Figure out based on A what the source of the first SHA256RNDS2 was.
+    const Ref FirstSrc = GetArgIfOp(GetArgIfOp(GetArgIfOp(A, OP_VSHA256H, 0), OP_VREV64, 0), OP_VZIP2, 0);
+    // True if X holds the result of the first SHA256RNDS2, and its source register was Y.
+    const bool IsFirstResult = FirstSrc && IsValueLoadRegister(WrapNode(FirstSrc), RegClass::FPR, Y);
+    if (!IsFirstResult) {
+      goto no_peephole;
+    }
+
+    // The pattern matched, so we can elide the second SHA256RNDS2,
+    // and instead use the result of the first pair of SHA256H and SHA256H2 instructions
+    // to populate the final result.
+    StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, shuffle_efgh(A, B));
+    return;
+  }
+
+no_peephole:
+  Ref Dest = LoadSourceFPR(Op, Op->Dest, Op->Flags);
+  Ref Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+  // Hardcoded to XMM0
+  auto XMM0 = LoadXMMRegister(0);
+
   auto ABCD = shuffle_abcd(Dest, Src);
   auto EFGH = shuffle_efgh(Dest, Src);
 
-  // x86 uses only the bottom 64-bits of the key, so duplicate to match ARM64 semantics.
-  auto Key = _VDupElement(OpSize::i128Bit, OpSize::i64Bit, XMM0, 0);
-
-  auto A = _VSha256H(ABCD, EFGH, Key);
-  auto B = _VSha256H2(EFGH, ABCD, Key);
+  // x86 uses only the lower 64 bits of the key for two rounds.
+  // ARM64 does four rounds, so in order to support the peephole optimization,
+  // we use the full key always.
+  auto A = _VSha256H(ABCD, EFGH, XMM0);
+  auto B = _VSha256H2(EFGH, ABCD, XMM0);
   auto Result = shuffle_abcd(A, B);
 
   StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result);

FEXCore/Source/Interface/Core/OpcodeDispatcher/PrimaryGroupTables.h

@@ -9,36 +9,36 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
   // GROUP 1
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 0), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 1), 1, &OpDispatchBuilder::SecondaryALUOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1, false>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 4), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 5), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 6), 1, &OpDispatchBuilder::SecondaryALUOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>}, // CMP
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x80), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1, false>},
 
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 0), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 1), 1, &OpDispatchBuilder::SecondaryALUOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1, false>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 4), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 5), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 6), 1, &OpDispatchBuilder::SecondaryALUOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x81), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1, false>},
 
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 0), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 1), 1, &OpDispatchBuilder::SecondaryALUOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ADCOp, 1, false>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SBBOp, 1, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 4), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 5), 1, &OpDispatchBuilder::SecondaryALUOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 6), 1, &OpDispatchBuilder::SecondaryALUOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_1, OpToIndex(0x83), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::CMPOp, 1, false>},
 
   // GROUP 2
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, true, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, true, false>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 2), 1, &OpDispatchBuilder::RCLOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 3), 1, &OpDispatchBuilder::RCROp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCLOp, false>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCROp, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHRImmediateOp, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC0), 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>}, // SAL
@@ -46,8 +46,8 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
 
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, true, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, true, false>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 2), 1, &OpDispatchBuilder::RCLOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 3), 1, &OpDispatchBuilder::RCROp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCLOp, false>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCROp, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 4), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 5), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHRImmediateOp, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xC1), 6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::SHLImmediateOp, false>}, // SAL
@@ -73,8 +73,8 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
 
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, false, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, false, false>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, &OpDispatchBuilder::RCLSmallerOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, &OpDispatchBuilder::RCRSmallerOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCLSmallerOp, true>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCRSmallerOp, true>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, &OpDispatchBuilder::SHLOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, &OpDispatchBuilder::SHROp}, // SHR by CL
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD2), 6), 1, &OpDispatchBuilder::SHLOp}, // SAL
@@ -82,16 +82,16 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
 
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, true, false, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RotateOp, false, false, false>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, &OpDispatchBuilder::RCLOp},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, &OpDispatchBuilder::RCROp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCLOp, true>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::RCROp, true>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, &OpDispatchBuilder::SHLOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, &OpDispatchBuilder::SHROp}, // SHR by CL
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 6), 1, &OpDispatchBuilder::SHLOp}, // SAL
   {OPD(FEXCore::X86Tables::TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::ASHROp, false, false>}, // SAR
 
   // GROUP 3
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1, false>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 2), 1, &OpDispatchBuilder::NOTOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 3), 1, &OpDispatchBuilder::NEGOp}, // NEG
   {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 4), 1, &OpDispatchBuilder::MULOp},
@@ -99,8 +99,8 @@ constexpr DispatchTableEntry OpDispatch_PrimaryGroupTables[] = {
   {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 6), 1, &OpDispatchBuilder::DIVOp},  // DIV
   {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF6), 7), 1, &OpDispatchBuilder::IDIVOp}, // IDIV
 
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},
-  {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1, false>},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 1), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::TESTOp, 1, false>},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 2), 1, &OpDispatchBuilder::NOTOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_3, OpToIndex(0xF7), 3), 1, &OpDispatchBuilder::NEGOp}, // NEG
 

FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryGroupTables.h

@@ -10,6 +10,11 @@ constexpr uint16_t PF_66 = 2;
 constexpr uint16_t PF_F2 = 3;
 constexpr DispatchTableEntry OpDispatch_SecondaryGroupTables[] = {
   // GROUP 6
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_NONE, 0), 1, &OpDispatchBuilder::SLDTOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F3, 0), 1, &OpDispatchBuilder::SLDTOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_66, 0), 1, &OpDispatchBuilder::SLDTOp},
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F2, 0), 1, &OpDispatchBuilder::SLDTOp},
+
   {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_NONE, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_F3, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_6, PF_66, 3), 1, &OpDispatchBuilder::PermissionRestrictedOp},
@@ -36,6 +41,8 @@ constexpr DispatchTableEntry OpDispatch_SecondaryGroupTables[] = {
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 4), 1, &OpDispatchBuilder::SMSWOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F2, 4), 1, &OpDispatchBuilder::SMSWOp},
 
+  {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 5), 1, &OpDispatchBuilder::SerializeOp},
+
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_NONE, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_F3, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {OPD(FEXCore::X86Tables::TYPE_GROUP_7, PF_66, 6), 1, &OpDispatchBuilder::PermissionRestrictedOp},

FEXCore/Source/Interface/Core/OpcodeDispatcher/SecondaryModRMTables.h

@@ -10,12 +10,12 @@ constexpr DispatchTableEntry OpDispatch_SecondaryModRMTables[] = {
 
   // REG /2
   {((1 << 3) | 0), 1, &OpDispatchBuilder::XGetBVOp},
+  {((1 << 3) | 1), 1, &OpDispatchBuilder::PermissionRestrictedOp},
 
   // REG /3
   {((2 << 3) | 7), 1, &OpDispatchBuilder::PermissionRestrictedOp},
 
   // REG /7
-  {((3 << 3) | 0), 1, &OpDispatchBuilder::PermissionRestrictedOp},
   {((3 << 3) | 1), 1, &OpDispatchBuilder::RDTSCPOp},
   {((3 << 3) | 4), 1, &OpDispatchBuilder::CLZeroOp},
 };

FEXCore/Source/Interface/Core/OpcodeDispatcher/Vector.cpp

@@ -8,6 +8,7 @@ $end_info$
 
 #include "Interface/Context/Context.h"
 #include "Interface/Core/OpcodeDispatcher.h"
+#include "Interface/Core/Interpreter/Fallbacks/VectorFallbacks.h"
 #include "Interface/Core/X86Tables/X86Tables.h"
 #include "Interface/IR/IR.h"
 
@@ -3024,7 +3025,7 @@ void OpDispatchBuilder::XSaveOpImpl(OpcodeArgs) {
     StoreIfFlagSet(1, [this, Op] { SaveSSEState(XSaveBase(Op)); });
   }
   // AVX
-  if (CTX->HostFeatures.SupportsAVX) {
+  if (CTX->HostFeatures.SupportsAVX()) {
     StoreIfFlagSet(2, [this, Op] { std::invoke(SaveAVXStateFunc, this, XSaveBase(Op)); });
   }
 
@@ -3071,6 +3072,12 @@ void OpDispatchBuilder::SaveX87State(OpcodeArgs, Ref MemBase) {
 
   { _StoreMemGPR(OpSize::i16Bit, ReconstructFSW_Helper(), MemBase, Constant(2), OpSize::i16Bit, MemOffsetType::SXTX, 1); }
 
+  {
+    auto FIPSize = Op->Flags & X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
+    auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
+    _StoreMemGPR(FIPSize, FIP, MemBase, Constant(8), FIPSize, MemOffsetType::SXTX, 1);
+  }
+
   {
     // Abridged FTW
     auto FTW = _LoadContextGPR(OpSize::i8Bit, offsetof(FEXCore::Core::CPUState, AbridgedFTW));
@@ -3172,7 +3179,7 @@ Ref OpDispatchBuilder::GetMXCSR() {
 void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {
   Ref Mem = MakeSegmentAddress(Op, Op->Src[0]);
 
-  RestoreX87State(Mem);
+  RestoreX87State(Op, Mem);
   RestoreSSEState(Mem);
 
   Ref MXCSR = _LoadMemGPR(OpSize::i32Bit, Mem, Constant(24), OpSize::i32Bit, MemOffsetType::SXTX, 1);
@@ -3218,14 +3225,14 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
 
   // x87
   {
-    RestoreIfFlagSetOrDefault(0, [this, Op] { RestoreX87State(XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });
+    RestoreIfFlagSetOrDefault(0, [this, Op] { RestoreX87State(Op, XSaveBase(Op)); }, [this, Op] { DefaultX87State(Op); });
   }
   // SSE
   {
     RestoreIfFlagSetOrDefault(1, [this, Op] { RestoreSSEState(XSaveBase(Op)); }, [this] { DefaultSSEState(); });
   }
   // AVX
-  if (CTX->HostFeatures.SupportsAVX) {
+  if (CTX->HostFeatures.SupportsAVX()) {
     RestoreIfFlagSetOrDefault(
       2, [this, Op] { std::invoke(RestoreAVXStateFunc, this, XSaveBase(Op)); }, [this] { std::invoke(DefaultAVXStateFunc, this); });
   }
@@ -3243,15 +3250,22 @@ void OpDispatchBuilder::XRstorOpImpl(OpcodeArgs) {
   }
 }
 
-void OpDispatchBuilder::RestoreX87State(Ref MemBase) {
+void OpDispatchBuilder::RestoreX87State(OpcodeArgs, Ref MemBase) {
   _StackForceSlow();
 
   auto NewFCW = _LoadMemGPR(OpSize::i16Bit, MemBase, OpSize::i16Bit);
   _StoreContextGPR(OpSize::i16Bit, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
 
+  Ref Top {};
   {
     auto NewFSW = _LoadMemGPR(OpSize::i16Bit, MemBase, Constant(2), OpSize::i16Bit, MemOffsetType::SXTX, 1);
-    ReconstructX87StateFromFSW_Helper(NewFSW);
+    Top = ReconstructX87StateFromFSW_Helper(NewFSW);
+  }
+
+  {
+    auto FIPSize = Op->Flags & X86Tables::DecodeFlags::FLAG_REX_WIDENING ? OpSize::i64Bit : OpSize::i32Bit;
+    auto FIP = _LoadMemGPR(FIPSize, MemBase, Constant(8), FIPSize, MemOffsetType::SXTX, 1);
+    _StoreContextGPR(OpSize::i64Bit, FIP, offsetof(FEXCore::Core::CPUState, FIP));
   }
 
   {
@@ -3260,10 +3274,21 @@ void OpDispatchBuilder::RestoreX87State(Ref MemBase) {
     _StoreContextGPR(OpSize::i8Bit, NewFTW, offsetof(FEXCore::Core::CPUState, AbridgedFTW));
   }
 
-  for (uint32_t i = 0; i < Core::CPUState::NUM_MMS; i += 2) {
-    auto MMRegs = LoadMemPairFPR(OpSize::i128Bit, MemBase, i * 16 + 32);
-    _StoreContextFPR(OpSize::i128Bit, MMRegs.Low, MMBaseOffset() + i * 16);
-    _StoreContextFPR(OpSize::i128Bit, MMRegs.High, MMBaseOffset() + (i + 1) * 16);
+  auto SevenConst = Constant(7);
+  auto low = Constant(~0ULL);
+  auto high = Constant(0xFFFF);
+  Ref Mask = _VLoadTwoGPRs(low, high);
+  const auto StoreSize = ReducedPrecisionMode ? OpSize::i64Bit : OpSize::i128Bit;
+  for (uint32_t i = 0; i < Core::CPUState::NUM_MMS; ++i) {
+    Ref Reg = _LoadMemFPR(OpSize::i128Bit, MemBase, Constant(16 * i + 32), OpSize::i8Bit, MemOffsetType::SXTX, 1);
+    // Mask off the top bits
+    Reg = _VAnd(OpSize::i128Bit, Reg, Mask);
+    if (ReducedPrecisionMode) {
+      // Convert to double precision
+      Reg = _F80CVT(OpSize::i64Bit, Reg);
+    }
+    _StoreContextFPRIndexed(Reg, Top, StoreSize, MMBaseOffset(), IR::OpSizeToSize(OpSize::i128Bit));
+    Top = _And(OpSize::i32Bit, Add(OpSize::i32Bit, Top, 1), SevenConst);
   }
 }
 
@@ -3760,6 +3785,84 @@ void OpDispatchBuilder::VPMADDUBSWOp(OpcodeArgs) {
   StoreResultFPR(Op, Result);
 }
 
+Ref OpDispatchBuilder::VPDPBUSDOpImpl(IR::OpSize Size, Ref Acc, Ref Src1, Ref Src2, bool Saturating) {
+  // Does four 8-bit unsigned * signed byte multiplies per 32-bit element, sums them and accumulates in to the destination
+
+  if (CTX->HostFeatures.SupportsI8MM) {
+    // The I8MM extension maps onto VPDP* almost directly.
+    if (!Saturating) {
+      return _VUSDot(Size, Acc, Src1, Src2);
+    }
+
+    auto DotProduct = _VUSDot(Size, LoadZeroVector(Size), Src1, Src2);
+    return _VSQAdd(Size, OpSize::i32Bit, Acc, DotProduct);
+  }
+
+  if (CTX->HostFeatures.SupportsDotProd) {
+    // VSDOT assumes signed input, so we need to convert Src1 to signed, and then
+    // perform correction afterwards using 0x40 to account for the unsigned input.
+    auto Src1Signed = _VXor(Size, Src1, _VectorImm(Size, OpSize::i8Bit, 0x80));
+    auto SixtyFour = _VectorImm(Size, OpSize::i8Bit, 0x40);
+
+    auto DotProduct = _VSDot(Size, Saturating ? LoadZeroVector(Size) : Acc, Src2, SixtyFour);
+    DotProduct = _VSDot(Size, DotProduct, Src2, SixtyFour);
+    DotProduct = _VSDot(Size, DotProduct, Src1Signed, Src2);
+    if (Saturating) {
+      return _VSQAdd(Size, OpSize::i32Bit, Acc, DotProduct);
+    }
+    return DotProduct;
+  }
+
+  // Naive software implementation.
+  auto Even = _VUnZip(Size, OpSize::i16Bit, Src1, Src2);
+  auto Even1_16b = _VUXTL(Size, OpSize::i8Bit, Even);
+  auto Even2_16b = _VSXTL2(Size, OpSize::i8Bit, Even);
+  auto ResMul_Even = _VMul(Size, OpSize::i16Bit, Even1_16b, Even2_16b);
+
+  auto Odd = _VUnZip2(Size, OpSize::i16Bit, Src1, Src2);
+  auto Odd1_16b = _VUXTL(Size, OpSize::i8Bit, Odd);
+  auto Odd2_16b = _VSXTL2(Size, OpSize::i8Bit, Odd);
+  auto ResMul_Odd = _VMul(Size, OpSize::i16Bit, Odd1_16b, Odd2_16b);
+
+  auto DotProduct = _VSAdALP(Size, OpSize::i16Bit, _VSAddLP(Size, OpSize::i16Bit, ResMul_Even), ResMul_Odd);
+  if (Saturating) {
+    return _VSQAdd(Size, OpSize::i32Bit, Acc, DotProduct);
+  }
+  return _VAdd(Size, OpSize::i32Bit, Acc, DotProduct);
+}
+
+Ref OpDispatchBuilder::VPDPWSSDOpImpl(IR::OpSize Size, Ref Acc, Ref Src1, Ref Src2, bool Saturating) {
+  auto DotProduct = PMADDWDOpImpl(Size, Src1, Src2);
+  if (!Saturating) {
+    return _VAdd(Size, OpSize::i32Bit, Acc, DotProduct);
+  }
+
+  auto NegDotProduct = _VNeg(Size, OpSize::i32Bit, DotProduct);
+  return _VSQSub(Size, OpSize::i32Bit, Acc, NegDotProduct);
+}
+
+void OpDispatchBuilder::VPDPBUSDOp(OpcodeArgs, bool Saturating) {
+  const auto Size = OpSizeFromSrc(Op);
+
+  Ref Acc = LoadSourceFPR(Op, Op->Dest, Op->Flags);
+  Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+  Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);
+
+  Ref Result = VPDPBUSDOpImpl(Size, Acc, Src1, Src2, Saturating);
+  StoreResultFPR(Op, Result);
+}
+
+void OpDispatchBuilder::VPDPWSSDOp(OpcodeArgs, bool Saturating) {
+  const auto Size = OpSizeFromSrc(Op);
+
+  Ref Acc = LoadSourceFPR(Op, Op->Dest, Op->Flags);
+  Ref Src1 = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+  Ref Src2 = LoadSourceFPR(Op, Op->Src[1], Op->Flags);
+
+  Ref Result = VPDPWSSDOpImpl(Size, Acc, Src1, Src2, Saturating);
+  StoreResultFPR(Op, Result);
+}
+
 Ref OpDispatchBuilder::PMULHWOpImpl(OpcodeArgs, bool Signed, Ref Src1, Ref Src2) {
   const auto Size = OpSizeFromSrc(Op);
   if (Signed) {
@@ -5360,6 +5463,132 @@ void OpDispatchBuilder::VPERMILRegOp(OpcodeArgs, IR::OpSize ElementSize) {
   StoreResultFPR(Op, Result);
 }
 
+// Computes the number of valid elements in a string for the explicit case,
+// where "valid" means that the character is not after the Nth character in the string.
+Ref OpDispatchBuilder::PCMPXSTRXSaturateExplicitLength(OpSize ElementSize, OpSize LengthSize, Ref RawLength) {
+  const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
+
+  SaveNZCV();
+  _SubNZCV(LengthSize, RawLength, Constant(0));
+  Ref AbsLength = _Neg(LengthSize, RawLength, CondClass::MI);
+  return _Select(OpSize::i32Bit, LengthSize, CondClass::ULT, AbsLength, Constant(NumElements), AbsLength, Constant(NumElements));
+}
+
+// Equal Any is a "is character in set" operation.
+// The set of characters is passed in Src1, and we then check if every
+// character in Src2 is in that set.
+Ref OpDispatchBuilder::PCMPXSTRXEqualAny(OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements) {
+  const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
+
+  // First we sanitize Src1, and zero out elements after the end of the string,
+  // removing them from the matching set.
+  Ref Src1Element0 = _VDupElement(OpSize::i128Bit, ElementSize, Src1, 0);
+  Ref SanitizedSrc1 = _VBSL(OpSize::i128Bit, Src1ValidElements, Src1, Src1Element0);
+
+  // Now walk through every element in Src1, broadcast it into a temporary vector,
+  // and compare it against every element in Src2, then OR the results
+  // together to get the final match vector.
+  Ref Matches = _VCMPEQ(OpSize::i128Bit, ElementSize, Src2, Src1Element0);
+  for (uint32_t i = 1; i < NumElements; i++) {
+    Ref Src1Element = _VDupElement(OpSize::i128Bit, ElementSize, SanitizedSrc1, i);
+    Ref ElementMatches = _VCMPEQ(OpSize::i128Bit, ElementSize, Src2, Src1Element);
+    Matches = _VOr(OpSize::i128Bit, Matches, ElementMatches);
+  }
+
+  // Handle the case where Src1 was entirely empty, and also sanitize the results
+  // for any NULLs in Src2, since those don't count towards matching.
+  Ref Src1NotEmpty = _VDupElement(OpSize::i128Bit, ElementSize, Src1ValidElements, 0);
+  Matches = _VAnd(OpSize::i128Bit, Matches, Src1NotEmpty);
+  return _VAnd(OpSize::i128Bit, Matches, Src2ValidElements);
+}
+
+
+// The ranges aggregation is a weird one. It checks if every character in Src2 is
+// within a character range (ie. a-z or A-Z) similar to regex.
+// Ranges are passed in Src1 as pairs of characters in adjacent lanes.
+Ref OpDispatchBuilder::PCMPXSTRXRanges(OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements, bool IsSigned) {
+  const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
+
+  // Signed or unsigned greater than comparison, based on the Imm8 value
+  const auto GreaterThan = [&](Ref Lhs, Ref Rhs) {
+    return IsSigned ? _VCMPGT(OpSize::i128Bit, ElementSize, Lhs, Rhs) : _VUCMPGT(OpSize::i128Bit, ElementSize, Lhs, Rhs);
+  };
+
+  // First we walk through the ranges from Src1, and construct
+  // temporary vectors to represent the lower and upper bounds
+  // of the comparison. Then check if both comparisons are true,
+  // zero out invalid lanes, and OR the result into the final result vector.
+  Ref Result {};
+  for (uint32_t i = 0; i < NumElements; i += 2) {
+    Ref LowerBound = _VDupElement(OpSize::i128Bit, ElementSize, Src1, i);
+    Ref UpperBound = _VDupElement(OpSize::i128Bit, ElementSize, Src1, i + 1);
+    Ref BelowLower = GreaterThan(LowerBound, Src2);
+    Ref AboveUpper = GreaterThan(Src2, UpperBound);
+
+    // A range is only valid if its upper bound is a valid Src1 element.
+    Ref RangeValid = _VDupElement(OpSize::i128Bit, ElementSize, Src1ValidElements, i + 1);
+
+    // RangeValid & ~BelowLower & ~AboveUpper
+    Ref InRange = _VAndn(OpSize::i128Bit, RangeValid, BelowLower);
+    InRange = _VAndn(OpSize::i128Bit, InRange, AboveUpper);
+    Result = Result ? _VOr(OpSize::i128Bit, Result, InRange) : InRange;
+  }
+
+  // Invalid (ie. NULL) Src2 characters don't count towards matching.
+  return _VAnd(OpSize::i128Bit, Result, Src2ValidElements);
+}
+
+// This aggregation is the simplest, and is the most similar
+// to strcmp(). It Just compares lanewise which characters are
+// equal in each string.
+Ref OpDispatchBuilder::PCMPXSTRXEqualEach(OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1ValidElements, Ref Src2ValidElements) {
+  // Elements match when both are valid and equal, or when both are invalid.
+  Ref Equal = _VCMPEQ(OpSize::i128Bit, ElementSize, Src1, Src2);
+  Ref BothValid = _VAnd(OpSize::i128Bit, Src1ValidElements, Src2ValidElements);
+  Ref EitherValid = _VOr(OpSize::i128Bit, Src1ValidElements, Src2ValidElements);
+  Ref ValidMatches = _VAnd(OpSize::i128Bit, Equal, BothValid);
+  return _VOrn(OpSize::i128Bit, ValidMatches, EitherValid);
+}
+
+// EqualOrdered implements a strstr()-like semantic finding all instances
+// of the string Src1 in Src2.
+Ref OpDispatchBuilder::PCMPXSTRXEqualOrdered(OpSize ElementSize, Ref Src1, Ref Src2, Ref Src1Length, Ref Src2Length, Ref Src1ValidElements,
+                                             Ref Indices) {
+  const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
+
+  // Broadcast needle[i] into every lane of a temp vector, then compare it to
+  // the haystack vector. On succesive iterations, we shift the haystack over
+  // by one element, and compare it against the next needle element.
+  // AND together the results of all comparisons, and what is left should
+  // be a vector where the only 'true' elements are lanes where the full
+  // needle was found in the haystack, or those positions after the end of the string.
+  Ref Result {};
+  for (uint32_t i = 0; i < NumElements; i++) {
+    Ref Needle = _VDupElement(OpSize::i128Bit, ElementSize, Src1, i);
+    Ref Haystack = i == 0 ? Src2 : _VExtr(OpSize::i128Bit, ElementSize, Needle, Src2, i);
+    Ref ElementsEqual = _VCMPEQ(OpSize::i128Bit, ElementSize, Haystack, Needle);
+
+    // TODO: I think a more optimal version of this is possible, perhaps using
+    // MATCH from SVE2?
+    Ref ElementsValid = _VDupElement(OpSize::i128Bit, ElementSize, Src1ValidElements, i);
+    Ref ElementsEqualValid = _VOrn(OpSize::i128Bit, ElementsEqual, ElementsValid);
+
+    // Ternary to handle the first row case
+    Result = Result ? _VAnd(OpSize::i128Bit, Result, ElementsEqualValid) : ElementsEqualValid;
+  }
+
+  // Clear positions in the result after the end of the string.
+  Ref FirstClearedPosition = Add(OpSize::i32Bit, _Sub(OpSize::i32Bit, Src2Length, Src1Length), 1);
+  FirstClearedPosition =
+    _Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::ULT, Src2Length, Constant(NumElements), FirstClearedPosition, Constant(NumElements));
+  FirstClearedPosition =
+    _Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::EQ, Src1Length, Constant(0), Constant(NumElements), FirstClearedPosition);
+
+  Ref FirstClearedPositionVector = _VDupFromGPR(OpSize::i128Bit, ElementSize, FirstClearedPosition);
+  Ref KeptPositions = _VCMPGT(OpSize::i128Bit, ElementSize, FirstClearedPositionVector, Indices);
+  return _VAnd(OpSize::i128Bit, Result, KeptPositions);
+}
+
 void OpDispatchBuilder::PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask, bool IsAVX) {
   const uint16_t Control = Op->Src[1].Literal();
 
@@ -5372,84 +5601,166 @@ void OpDispatchBuilder::PCMPXSTRXOpImpl(OpcodeArgs, bool IsExplicit, bool IsMask
   Ref Src1 = LoadSourceFPR_WithOpSize(Op, Op->Dest, OpSize::i128Bit, Op->Flags);
   Ref Src2 = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::i128Bit, Op->Flags, {.Align = OpSize::i8Bit});
 
-  Ref IntermediateResult {};
+  // Parse the immediate control bits.
+  // See section 4.1 in Intel SDM for details.
+  const auto Format = static_cast<CPU::SourceData>(Control & 0b11);
+  const auto Aggregation = static_cast<CPU::AggregationOp>((Control >> 2) & 0b11);
+  const auto Polarity = static_cast<CPU::Polarity>((Control >> 4) & 0b11);
+  const bool OutputSelection = (Control & (1 << 6)) != 0;
+
+  // Helper constants
+  const bool IsWords = Format == CPU::SourceData::U16 || Format == CPU::SourceData::S16;
+  const bool IsSigned = Format == CPU::SourceData::S8 || Format == CPU::SourceData::S16;
+  const auto ElementSize = IsWords ? OpSize::i16Bit : OpSize::i8Bit;
+  const bool IsEqualOrdered = Aggregation == CPU::AggregationOp::EqualOrdered;
+  const bool NeedsSrc1ValidElements = true;
+  const bool NeedsSrc2ValidElements = !IsEqualOrdered || Polarity == CPU::Polarity::NegativeMasked;
+  const uint32_t NumElements = IR::NumElements(OpSize::i128Bit, ElementSize);
+
+  // Load a vector where each element contains the value of its own index
+  Ref Indices = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, IsWords ? NAMED_VECTOR_INCREMENTAL_U16_INDEX : NAMED_VECTOR_INCREMENTAL_U8_INDEX);
+
+  // Helper lambda that computes the lowest index element set "all high" in mask.
+  Ref VecNumElements {};
+  const auto LowestIndex = [&](Ref Mask) {
+    if (!VecNumElements) {
+      VecNumElements = _VectorImm(OpSize::i128Bit, ElementSize, NumElements);
+    }
+    Ref Candidates = _VBSL(OpSize::i128Bit, Mask, Indices, VecNumElements);
+    return _VUMinV(OpSize::i128Bit, ElementSize, Candidates);
+  };
+
+  Ref Src1Length {};
+  Ref Src2Length {};
+  Ref Src2LengthVector {};
+  Ref Src1ValidElementsMask {};
+
+  // Compute the valid element masks for the two source strings.
   if (IsExplicit) {
-    // Will be 4 in the absence of a REX.W bit and 8 in the presence of a REX.W bit.
-    //
-    // While the control bit immediate for the instruction itself is only ever 8 bits
-    // in size, we use it as a 16-bit value so that we can use the 8th bit to signify
-    // whether or not RAX and RDX should be interpreted as a 64-bit value.
-    const auto SrcSize = OpSizeFromSrc(Op);
-    const auto Is64Bit = SrcSize == OpSize::i64Bit;
-    const auto NewControl = uint16_t(Control | (uint16_t(Is64Bit) << 8));
+    const auto LengthSize = OpSizeFromSrc(Op);
 
-    Ref SrcRAX = LoadGPRRegister(X86State::REG_RAX);
-    Ref SrcRDX = LoadGPRRegister(X86State::REG_RDX);
+    Src1Length = PCMPXSTRXSaturateExplicitLength(ElementSize, LengthSize, LoadGPRRegister(X86State::REG_RAX));
+    Src2Length = PCMPXSTRXSaturateExplicitLength(ElementSize, LengthSize, LoadGPRRegister(X86State::REG_RDX));
 
-    IntermediateResult = _VPCMPESTRX(Src1, Src2, SrcRAX, SrcRDX, NewControl);
-  } else {
-    IntermediateResult = _VPCMPISTRX(Src1, Src2, Control);
-  }
-
-  if (IsMask) {
-    // For the masked variant of the instructions, if control[6] is set, then we
-    // need to expand the intermediate result into a byte or word mask (depending
-    // on data size specified in control[1]) along the entire length of XMM0,
-    // where set bits in the intermediate result set the corresponding entry
-    // in XMM0 to all 1s and unset bits set the corresponding entry to all 0s.
-    //
-    // If control[6] is not set, then we just store the intermediate result as-is
-    // into the least significant bits of XMM0 and zero extend it.
-    const auto IsExpandedMask = (Control & 0b0100'0000) != 0;
-
-    if (IsExpandedMask) {
-      // We need to iterate over the intermediate result and
-      // expand the mask into XMM0 elements.
-      const auto ElementSize = 1U << (Control & 1);
-      const auto NumElements = 16U >> (Control & 1);
-
-      Ref Result = LoadZeroVector(OpSize::i128Bit);
-      for (uint32_t i = 0; i < NumElements; i++) {
-        Ref SignBit = _Sbfe(OpSize::i64Bit, 1, i, IntermediateResult);
-        Result = _VInsGPR(OpSize::i128Bit, IR::SizeToOpSize(ElementSize), i, Result, SignBit);
-      }
-
-      if (IsAVX) {
-        StoreXMMRegister(0, Result);
-      } else {
-        StoreXMMRegister_WithAVXInsert(VectorOpType::SSE, 0, Result);
-      }
-    } else {
-      // We insert the intermediate result as-is.
-      Ref Result = _VCastFromGPR(OpSize::i128Bit, OpSize::i16Bit, IntermediateResult);
-
-      if (IsAVX) {
-        StoreXMMRegister(0, Result);
-      } else {
-        StoreXMMRegister_WithAVXInsert(VectorOpType::SSE, 0, Result);
-      }
+    if (NeedsSrc1ValidElements) {
+      Src1ValidElementsMask = _VCMPGT(OpSize::i128Bit, ElementSize, _VDupFromGPR(OpSize::i128Bit, ElementSize, Src1Length), Indices);
+    }
+    if (NeedsSrc2ValidElements) {
+      Src2LengthVector = _VDupFromGPR(OpSize::i128Bit, ElementSize, Src2Length);
     }
   } else {
-    Ref ZeroConst = Constant(0);
+    // Get index of the first NUL element, to determine length of input strings.
+    Ref Src1FirstNull = LowestIndex(_VCMPEQZ(OpSize::i128Bit, ElementSize, Src1));
+    Ref Src2FirstNull = LowestIndex(_VCMPEQZ(OpSize::i128Bit, ElementSize, Src2));
+    Src1Length = _VExtractToGPR(OpSize::i128Bit, ElementSize, Src1FirstNull, 0);
+    Src2Length = _VExtractToGPR(OpSize::i128Bit, ElementSize, Src2FirstNull, 0);
 
-    // For the indexed variant of the instructions, if control[6] is set, then we
-    // store the index of the most significant bit into ECX. If it's not set,
-    // then we store the least significant bit.
-    const auto UseMSBIndex = (Control & 0b0100'0000) != 0;
+    if (NeedsSrc1ValidElements) {
+      // If the index in the lane is less than the index of the first NUL, then it's a valid element.
+      Src1ValidElementsMask = _VCMPGT(OpSize::i128Bit, ElementSize, _VDupElement(OpSize::i128Bit, ElementSize, Src1FirstNull, 0), Indices);
+    }
+    if (NeedsSrc2ValidElements) {
+      Src2LengthVector = _VDupElement(OpSize::i128Bit, ElementSize, Src2FirstNull, 0);
+    }
+  }
 
-    Ref ResultNoFlags = _Bfe(OpSize::i32Bit, 16, 0, IntermediateResult);
+  Ref Src1HasInvalidElements = Select01(OpSize::i32Bit, CondClass::ULT, Src1Length, Constant(NumElements));
+  Ref Src2HasInvalidElements = Select01(OpSize::i32Bit, CondClass::ULT, Src2Length, Constant(NumElements));
+  Ref Src2ValidElementsMask = NeedsSrc2ValidElements ? _VCMPGT(OpSize::i128Bit, ElementSize, Src2LengthVector, Indices) : nullptr;
 
-    Ref IfZero = Constant(16 >> (Control & 1));
-    Ref IfNotZero = UseMSBIndex ? _FindMSB(IR::OpSize::i32Bit, ResultNoFlags) : _FindLSB(IR::OpSize::i32Bit, ResultNoFlags);
-    Ref Result = _Select(OpSize::i64Bit, OpSize::i64Bit, CondClass::EQ, ResultNoFlags, ZeroConst, IfZero, IfNotZero);
+  // Perform the aggregation operations, see section 4.1.3 in the Intel SDM.
+  Ref Matches {};
+  switch (Aggregation) {
+  case CPU::AggregationOp::EqualAny:
+    Matches = PCMPXSTRXEqualAny(ElementSize, Src1, Src2, Src1ValidElementsMask, Src2ValidElementsMask);
+    break;
+  case CPU::AggregationOp::Ranges:
+    Matches = PCMPXSTRXRanges(ElementSize, Src1, Src2, Src1ValidElementsMask, Src2ValidElementsMask, IsSigned);
+    break;
+  case CPU::AggregationOp::EqualEach:
+    Matches = PCMPXSTRXEqualEach(ElementSize, Src1, Src2, Src1ValidElementsMask, Src2ValidElementsMask);
+    break;
+  case CPU::AggregationOp::EqualOrdered:
+    Matches = PCMPXSTRXEqualOrdered(ElementSize, Src1, Src2, Src1Length, Src2Length, Src1ValidElementsMask, Indices);
+    break;
+  }
+
+  // Invert the matches if the polarity is negative or negative masked,
+  // see section 4.1.4 in the Intel SDM.
+  if (Polarity == CPU::Polarity::Negative) {
+    Matches = _VNot(OpSize::i128Bit, Matches);
+  } else if (Polarity == CPU::Polarity::NegativeMasked) {
+    Matches = _VXor(OpSize::i128Bit, Matches, Src2ValidElementsMask);
+  }
+
+  // Compute the flags and result
+  Ref CF {};
+  Ref OF {};
+
+  // PCMPESTRM and PCMPISTRM
+  // These instructions return a mask of the matching elements.
+  if (IsMask) {
+    Ref Result = Matches;
+    // Byte/word mask rather than a bit mask.
+    if (OutputSelection) {
+      // Result is already a byte/word mask, so just compute flags.
+      // CF set when no elements matched
+      // OF is set when the first element matched.
+      Ref AnyMatch = _VExtractToGPR(OpSize::i128Bit, ElementSize, _VUMaxV(OpSize::i128Bit, ElementSize, Matches), 0);
+      CF = Select01(OpSize::i32Bit, CondClass::EQ, AnyMatch, Constant(0));
+      OF = _And(OpSize::i64Bit, _VExtractToGPR(OpSize::i128Bit, ElementSize, Matches, 0), Constant(1));
+
+    } else {
+      // Convert per element vector to a bit mask with one bit per lane.
+      Ref BitPositions = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, NAMED_VECTOR_MOVMASKB);
+      Ref Zero = LoadZeroVector(OpSize::i128Bit);
+
+      // If the elements are words, truncate to bytes first.
+      Result = IsWords ? _VUnZip(OpSize::i128Bit, OpSize::i8Bit, Matches, Zero) : Matches;
+      // Now repeat this pattern until we collapse each byte into a single bit.
+      Result = _VAnd(OpSize::i128Bit, Result, BitPositions);
+      Result = _VAddP(OpSize::i128Bit, OpSize::i8Bit, Result, Zero);
+      Result = _VAddP(OpSize::i128Bit, OpSize::i8Bit, Result, Zero);
+      Result = _VAddP(OpSize::i128Bit, OpSize::i8Bit, Result, Zero);
+
+      // Compute the flags from the bit mask in Result
+      Ref BitMask = _VExtractToGPR(OpSize::i128Bit, IsWords ? OpSize::i8Bit : OpSize::i16Bit, Result, 0);
+      CF = Select01(OpSize::i32Bit, CondClass::EQ, BitMask, Constant(0));
+      OF = _And(OpSize::i64Bit, BitMask, Constant(1));
+    }
+
+    StoreXMMRegister_WithAVXInsert(IsAVX ? VectorOpType::AVX : VectorOpType::SSE, 0, Result);
+
+    // PCMPESTRI and PCMPISTRI
+    // These instructions return either the index of the first match
+    // or the number of elements in the string if no match.
+  } else {
+    Ref LowestMatch = _VExtractToGPR(OpSize::i128Bit, ElementSize, LowestIndex(Matches), 0);
+    CF = Select01(OpSize::i32Bit, CondClass::EQ, LowestMatch, Constant(NumElements));
+    OF = Select01(OpSize::i32Bit, CondClass::EQ, LowestMatch, Constant(0));
+
+    Ref Result = LowestMatch;
+    // Most significant rather than least significant index.
+    if (OutputSelection) {
+      // The max is also 0 when nothing matched, LowestMatch tells those apart.
+      Ref Candidates = _VAnd(OpSize::i128Bit, Indices, Matches);
+      Ref HighestMatchVector = _VUMaxV(OpSize::i128Bit, ElementSize, Candidates);
+      Ref HighestMatch = _VExtractToGPR(OpSize::i128Bit, ElementSize, HighestMatchVector, 0);
+      Result = _Select(OpSize::i32Bit, OpSize::i32Bit, CondClass::EQ, LowestMatch, Constant(NumElements), Constant(NumElements), HighestMatch);
+    }
 
     // Store the result, it is already zero-extended to 64-bit implicitly.
     StoreGPRRegister(X86State::REG_RCX, Result);
   }
 
-  // Set all of the necessary flags. NZCV stored in bits 28...31 like the hw op.
-  SetNZCV(IntermediateResult);
-  CFInverted = false;
+  // SF/ZF are set when Src1/Src2 respectively have invalid elements.
+  Ref SF = Src1HasInvalidElements;
+  Ref ZF = Src2HasInvalidElements;
+  Ref NZCV = _Orlshl(OpSize::i64Bit, _Lshl(OpSize::i64Bit, OF, Constant(28)), CF, 29);
+  NZCV = _Orlshl(OpSize::i64Bit, NZCV, ZF, 30);
+  NZCV = _Orlshl(OpSize::i64Bit, NZCV, SF, 31);
+  SetNZCV(NZCV);
+  CFInverted = true;
   ZeroPF_AF();
 }
 
@@ -5757,4 +6068,86 @@ void OpDispatchBuilder::Insertq(OpcodeArgs) {
   StoreResult_WithAVXInsert(VectorOpType::SSE, RegClass::FPR, Op, Result);
 }
 
+void OpDispatchBuilder::AVXVBCSTNEBF162PS(OpcodeArgs) {
+  const auto DstSize = OpSizeFromDst(Op);
+
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  // bfloat16 is converted to fp32 and broadcast through the vector.
+  // bfloat16->float32 is left-shift by 16.
+  Src = _VShlI(DstSize, OpSize::i32Bit, Src, 16);
+  Src = _VDupElement(DstSize, OpSize::i32Bit, Src, 0);
+
+  StoreResultFPR(Op, Src);
+}
+
+void OpDispatchBuilder::AVXVBCSTNESH2PS(OpcodeArgs) {
+  const auto DstSize = OpSizeFromDst(Op);
+
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  // float16 is converted to fp32 and broadcast through the vector.
+  Src = _Float_FToF(OpSize::i32Bit, OpSize::i16Bit, Src);
+  Src = _VDupElement(DstSize, OpSize::i32Bit, Src, 0);
+
+  StoreResultFPR(Op, Src);
+}
+
+void OpDispatchBuilder::AVXVCVTNEEPH2PS(OpcodeArgs, bool Even) {
+  const auto DstSize = OpSizeFromDst(Op);
+
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  // float16 is converted to fp32 and broadcast through the vector.
+  if (Even) {
+    Src = _VUnZip(DstSize, OpSize::i16Bit, Src, Src);
+  } else {
+    Src = _VUnZip2(DstSize, OpSize::i16Bit, Src, Src);
+  }
+  Src = _Vector_FToF(DstSize, OpSize::i32Bit, Src, OpSize::i16Bit);
+
+  StoreResultFPR(Op, Src);
+}
+
+void OpDispatchBuilder::AVXVCVTNEEBF162PS(OpcodeArgs, bool Even) {
+  const auto DstSize = OpSizeFromDst(Op);
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  auto Convert = [&](auto Src, bool Even) -> Ref {
+    // Even or odd elements of each 128-bit register is converted from bfloat16 to float32.
+    // For even, just left shift the elements.
+    if (Even) {
+      return _VShlI(DstSize, OpSize::i32Bit, Src, 16);
+    }
+
+    // For Odds, we just need to mask off the lower 16-bits of each 32-bit element.
+    // Can't do the same movi+vbsl instruction as ASIMD. Just use shifts instead.
+    auto Tmp = _VUShrI(DstSize, OpSize::i32Bit, Src, 16);
+    return _VShlI(DstSize, OpSize::i32Bit, Tmp, 16);
+  };
+  StoreResultFPR(Op, Convert(Src, Even));
+}
+
+void OpDispatchBuilder::AVXVCVTNEPS2BF16(OpcodeArgs) {
+  const auto DstSize = OpSizeFromDst(Op);
+  const auto Is128Bit = DstSize == OpSize::i128Bit;
+
+  auto Src = LoadSourceFPR(Op, Op->Src[0], Op->Flags);
+
+  // Convert 4 or 8 32-bits floats in to bfloat16.
+  // This instruction only operates with rounding mode set to `Rounds to Nearest (even)`.
+  Ref Result {};
+  if (Is128Bit) {
+    // Just do the operation in the lower half of the register.
+    Result = _VBFCVTN(OpSize::i128Bit, Src);
+  } else {
+    // Split the operation in to two and recombine.
+    Ref UpperLane = _VDupElement(OpSize::i256Bit, OpSize::i128Bit, Src, 1);
+    auto ResultLow = _VBFCVTN(OpSize::i128Bit, Src);
+    Result = _VBFCVTN2(OpSize::i128Bit, ResultLow, UpperLane);
+  }
+
+  StoreResultFPR(Op, Result);
+}
+
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/X87.cpp

@@ -67,11 +67,13 @@ void OpDispatchBuilder::FLD(OpcodeArgs, IR::OpSize Width) {
     ConvertedData = _F80CVTTo(Data, Width);
   }
   _PushStack(ConvertedData, Data, Width);
+  StoreX87FIP(Op);
 }
 
 // Float LoaD operation with memory operand
 void OpDispatchBuilder::FLDFromStack(OpcodeArgs) {
   _CopyPushStack(Op->OP & 7);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FBLD(OpcodeArgs) {
@@ -79,18 +81,21 @@ void OpDispatchBuilder::FBLD(OpcodeArgs) {
   Ref Data = LoadSourceFPR_WithOpSize(Op, Op->Src[0], OpSize::f80Bit, Op->Flags);
   Ref ConvertedData = _F80BCDLoad(Data);
   _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FBSTP(OpcodeArgs) {
   Ref converted = _F80BCDStore(_ReadStackValue(0));
   StoreResultFPR_WithOpSize(Op, Op->Dest, converted, OpSize::f80Bit, OpSize::i8Bit);
   _PopStackDestroy();
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FLD_Const(OpcodeArgs, NamedVectorConstant K) {
   // Update TOP
   Ref Data = LoadAndCacheNamedVectorConstant(OpSize::i128Bit, K);
   _PushStack(Data, Data, OpSize::f80Bit);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FILD(OpcodeArgs) {
@@ -122,6 +127,7 @@ void OpDispatchBuilder::FILD(OpcodeArgs) {
 
   Ref ConvertedData = _VLoadTwoGPRs(shifted, upper);
   _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
@@ -135,6 +141,7 @@ void OpDispatchBuilder::FST(OpcodeArgs, IR::OpSize Width) {
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FSTToStack(OpcodeArgs) {
@@ -144,6 +151,7 @@ void OpDispatchBuilder::FSTToStack(OpcodeArgs) {
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 // Store integer to memory (possibly with truncation)
@@ -183,6 +191,7 @@ void OpDispatchBuilder::FIST(OpcodeArgs, bool Truncate) {
   if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -197,6 +206,7 @@ void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -213,6 +223,7 @@ void OpDispatchBuilder::FADD(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
 
   // top of stack is at offset zero
   _F80AddValue(0, Arg);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -227,6 +238,7 @@ void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -247,6 +259,7 @@ void OpDispatchBuilder::FMUL(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispa
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -264,6 +277,7 @@ void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -288,6 +302,7 @@ void OpDispatchBuilder::FDIV(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -305,6 +320,7 @@ void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -329,6 +345,7 @@ void OpDispatchBuilder::FSUB(OpcodeArgs, IR::OpSize Width, bool Integer, bool Re
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 Ref OpDispatchBuilder::GetX87FTW_Helper() {
@@ -398,7 +415,8 @@ void OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {
 
   {
     // Instruction Offset
-    _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
+    auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
+    _StoreMemGPR(Size, FIP, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
   }
 
   {
@@ -496,7 +514,8 @@ void OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {
 
   {
     // Instruction Offset
-    _StoreMemGPR(Size, ZeroConst, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
+    auto FIP = _LoadContextGPR(OpSize::i64Bit, offsetof(FEXCore::Core::CPUState, FIP));
+    _StoreMemGPR(Size, FIP, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
   }
 
   {
@@ -566,6 +585,12 @@ void OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {
     SetX87FTW(_LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 2), Size, MemOffsetType::SXTX, 1));
   }
 
+  {
+    // FIP
+    auto FIP = _LoadMemGPR(Size, Mem, Constant(IR::OpSizeToSize(Size) * 3), Size, MemOffsetType::SXTX, 1);
+    _StoreContextGPR(OpSize::i64Bit, FIP, offsetof(FEXCore::Core::CPUState, FIP));
+  }
+
   auto SevenConst = Constant(7);
   auto low = Constant(~0ULL);
   auto high = Constant(0xFFFF);
@@ -619,6 +644,7 @@ void OpDispatchBuilder::FXCH(OpcodeArgs) {
     _F80StackXchange(Offset);
   }
   SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FYL2X(OpcodeArgs, bool IsFYL2XP1) {
@@ -627,6 +653,7 @@ void OpDispatchBuilder::X87FYL2X(OpcodeArgs, bool IsFYL2XP1) {
   } else {
     _F80FYL2XStack();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::FCOMIFlags WhichFlags, bool PopTwice) {
@@ -691,6 +718,7 @@ void OpDispatchBuilder::FCOMI(OpcodeArgs, IR::OpSize Width, bool Integer, OpDisp
   } else if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FTST(OpcodeArgs) {
@@ -710,6 +738,7 @@ void OpDispatchBuilder::FTST(OpcodeArgs) {
   // Set Invalid Operation flag when unordered (NaN comparison)
   // The x87 exception flags are sticky. Preserve earlier result
   SetRFLAG<FEXCore::X86State::X87FLAG_IE_LOC>(_Or(OpSize::i32Bit, GetRFLAG(FEXCore::X86State::X87FLAG_IE_LOC), HostFlag_Unordered));
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool ZeroC2) {
@@ -717,6 +746,7 @@ void OpDispatchBuilder::X87OpHelper(OpcodeArgs, FEXCore::IR::IROps IROp, bool Ze
   if (ZeroC2) {
     SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(Constant(0));
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87ModifySTP(OpcodeArgs, bool Inc) {
@@ -728,6 +758,7 @@ void OpDispatchBuilder::X87ModifySTP(OpcodeArgs, bool Inc) {
 
   // C1 set to 0
   SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Constant(0));
+  StoreX87FIP(Op);
 }
 
 // Operations dealing with loading and storing environment pieces
@@ -768,10 +799,14 @@ Ref OpDispatchBuilder::ReconstructFSW_Helper(Ref T) {
 // Store Status Word
 // There's no load Status Word instruction but you can load it through frstor
 // or fldenv.
-void OpDispatchBuilder::X87FNSTSW(OpcodeArgs) {
+void OpDispatchBuilder::X87FNSTSW(OpcodeArgs, bool DestRAX) {
   Ref TopValue = _SyncStackToSlow();
   Ref StatusWord = ReconstructFSW_Helper(TopValue);
-  StoreResultGPR(Op, StatusWord);
+  if (DestRAX) {
+    StoreGPRRegister(X86State::REG_RAX, StatusWord, OpSize::i16Bit);
+  } else {
+    StoreResultGPR(Op, StatusWord);
+  }
 }
 
 void OpDispatchBuilder::FNCLEX(OpcodeArgs) {
@@ -805,15 +840,20 @@ void OpDispatchBuilder::FNINIT(OpcodeArgs) {
   SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(Zero);
   SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(Zero);
   SetRFLAG<FEXCore::X86State::X87FLAG_IE_LOC>(Zero);
+
+  // fninit resets the FIP.
+  _StoreContextGPR(OpSize::i64Bit, _Constant(0), offsetof(FEXCore::Core::CPUState, FIP));
 }
 
 void OpDispatchBuilder::X87FFREE(OpcodeArgs) {
   _InvalidateStack(Op->OP & 7);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87EMMS(OpcodeArgs) {
   // Tags all get set to 0b11
   _InvalidateStack(0xff);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
@@ -852,29 +892,112 @@ void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
 
   Ref VecCond = _VDupFromGPR(OpSize::i128Bit, OpSize::i64Bit, SelectCC0All1(CC));
   _F80VBSLStack(OpSize::i128Bit, VecCond, Op->OP & 7, 0);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FXAM(OpcodeArgs) {
   auto a = _ReadStackValue(0);
-  Ref Result =
-    ReducedPrecisionMode ? _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, a, 0) : _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, a, 1);
+  Ref Value = ReducedPrecisionMode ? _VExtractToGPR(OpSize::i64Bit, OpSize::i64Bit, a, 0) : _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, a, 1);
 
-  // Extract the sign bit
-  Result = ReducedPrecisionMode ? _Bfe(OpSize::i64Bit, 1, 63, Result) : _Bfe(OpSize::i64Bit, 1, 15, Result);
+  // Extract the sign bit, which goes in C1
+  Ref Result = ReducedPrecisionMode ? _Bfe(OpSize::i64Bit, 1, 63, Value) : _Bfe(OpSize::i64Bit, 1, 15, Value);
   SetRFLAG<FEXCore::X86State::X87FLAG_C1_LOC>(Result);
 
-  // Claim this is a normal number
-  // We don't support anything else
-  auto TopValid = _StackValidTag(0);
+  auto NotEmpty = _StackValidTag(0);
+  Ref IsEmpty = _Xor(OpSize::i64Bit, NotEmpty, Constant(1));
+  Ref IsNaN {};
+  Ref IsDenormal {};
+  Ref IsInf {};
+  Ref IsZero {};
+  Ref IsUnsupported {};
+  Ref NoSignBit {};
 
-  // In the case of top being invalid then C3:C2:C0 is 0b101
-  auto C3 = Select01(OpSize::i32Bit, CondClass::NEQ, TopValid, Constant(1));
+  // TODO: The codegen for this is not optimal, and can probably be improved
+  // if FXAM ends up on the hot path for some workload.
+
+  if (ReducedPrecisionMode) {
+    constexpr uint64_t ExponentMask = 0x7FF0'0000'0000'0000ULL;
+    NoSignBit = _Bfe(OpSize::i64Bit, 63, 0, Value);
+    IsInf = Select01(OpSize::i64Bit, CondClass::EQ, NoSignBit, Constant(ExponentMask));
+    IsNaN = Select01(OpSize::i64Bit, CondClass::UGT, NoSignBit, Constant(ExponentMask));
+
+    IsZero = Select01(OpSize::i64Bit, CondClass::EQ, NoSignBit, Constant(0));
+    // 64 bit floats can't represent an x87 denormal, nor any of the
+    // unsupported encodings.
+    IsDenormal = Constant(0);
+    IsUnsupported = Constant(0);
+  } else {
+    Ref Mantissa = _VExtractToGPR(OpSize::i128Bit, OpSize::i64Bit, a, 0);
+
+    // "J" is the name given to the msb of the mantissa in the SDM.
+    Ref JBit = _Bfe(OpSize::i64Bit, 1, 63, Mantissa);
+    Ref Exponent = _Bfe(OpSize::i64Bit, 15, 0, Value);
+    Ref IsExponentZero = Select01(OpSize::i64Bit, CondClass::EQ, Exponent, Constant(0));
+    Ref IsExponentMax = Select01(OpSize::i64Bit, CondClass::EQ, Exponent, Constant(0x7FFF));
+
+    // Inf is when mantissa only has the J bit set, exponent is all 1's.
+    Ref IsOnlyJBit = Select01(OpSize::i64Bit, CondClass::EQ, Mantissa, Constant(1ULL << 63));
+    IsInf = _And(OpSize::i64Bit, IsExponentMax, IsOnlyJBit);
+
+    // NaN is when the low 63 bits of the mantissa are non-zero
+    // and exponent is max, and the J bit is set.
+    Ref Fraction = _Bfe(OpSize::i64Bit, 63, 0, Mantissa);
+    Ref FractionNonZero = Select01(OpSize::i64Bit, CondClass::NEQ, Fraction, Constant(0));
+    Ref IsExponentMaxWithJBit = _And(OpSize::i64Bit, IsExponentMax, JBit);
+    IsNaN = _And(OpSize::i64Bit, IsExponentMaxWithJBit, FractionNonZero);
+
+    // Zero and Denormal are basically the same as the 64-bit case.
+    Ref MantissaNonZero = Select01(OpSize::i64Bit, CondClass::NEQ, Mantissa, Constant(0));
+    Ref MantissaZero = _Xor(OpSize::i64Bit, MantissaNonZero, Constant(1));
+    IsZero = _And(OpSize::i64Bit, IsExponentZero, MantissaZero);
+    IsDenormal = _And(OpSize::i64Bit, IsExponentZero, MantissaNonZero);
+
+    // This is where things are weird. If the J bit is not set
+    // and the exponent is non-zero, then this is an "unsupported"
+    // encoding, which I believe is left in for legacy reasons.
+    Ref IsSupported = _Or(OpSize::i64Bit, IsExponentZero, JBit);
+    IsUnsupported = _Xor(OpSize::i64Bit, IsSupported, Constant(1));
+  }
+
+  // NormalFiniteNumber = !Zero && !Denormal && !Inf && !NaN && !Empty && !Unsupported
+  Ref temp1 = _Or(OpSize::i64Bit, IsZero, IsDenormal);
+  Ref temp2 = _Or(OpSize::i64Bit, IsInf, IsNaN);
+  Ref temp3 = _Or(OpSize::i64Bit, IsUnsupported, IsEmpty);
+  temp1 = _Or(OpSize::i64Bit, temp1, temp2);
+  temp2 = _Or(OpSize::i64Bit, temp1, temp3);
+  Ref NormalFiniteNumber = _Xor(OpSize::i64Bit, temp2, Constant(1));
+
+  // Set C3, C2, C0 based on the class of the FP value
+  // Table is from "FXAM" page in the SDM.
+  // +----------------------+----+----+----+
+  // | Class                | C3 | C2 | C0 |
+  // +----------------------+----+----+----+
+  // | Unsupported          | 0  | 0  | 0  |
+  // | NaN                  | 0  | 0  | 1  |
+  // | Normal finite number | 0  | 1  | 0  |
+  // | Infinity             | 0  | 1  | 1  |
+  // | Zero                 | 1  | 0  | 0  |
+  // | Empty                | 1  | 0  | 1  |
+  // | Denormal number      | 1  | 1  | 0  |
+  // +----------------------+----+----+----+
+
+  // C0 = IsNaN || IsInf || IsEmpty
+  Ref C0 = _Or(OpSize::i64Bit, IsNaN, IsInf);
+  C0 = _Or(OpSize::i64Bit, C0, IsEmpty);
+
+  // C2 = (IsInf || Denormal || NormalFiniteNumber) && !IsEmpty
+  Ref C2 = _Or(OpSize::i64Bit, IsInf, IsDenormal);
+  C2 = _Or(OpSize::i64Bit, C2, NormalFiniteNumber);
+  C2 = _And(OpSize::i64Bit, C2, NotEmpty);
+
+  // C3 = Zero || IsEmpty || Denormal
+  Ref C3 = _Or(OpSize::i64Bit, IsZero, IsEmpty);
+  C3 = _Or(OpSize::i64Bit, C3, IsDenormal);
 
-  auto C2 = TopValid;
-  auto C0 = C3; // Mirror C3 until something other than zero is supported
   SetRFLAG<FEXCore::X86State::X87FLAG_C0_LOC>(C0);
   SetRFLAG<FEXCore::X86State::X87FLAG_C2_LOC>(C2);
   SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(C3);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FXTRACT(OpcodeArgs) {
@@ -885,6 +1008,11 @@ void OpDispatchBuilder::X87FXTRACT(OpcodeArgs) {
   auto Sig = _F80XTRACT_SIG(Top);
   _PushStack(Exp, Invalid(), OpSize::iInvalid);
   _PushStack(Sig, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
+}
+
+void OpDispatchBuilder::X87NOP(OpcodeArgs) {
+  StoreX87FIP(Op);
 }
 
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/OpcodeDispatcher/X87F64.cpp

@@ -68,6 +68,7 @@ void OpDispatchBuilder::FLDF64(OpcodeArgs, IR::OpSize Width) {
     ConvertedData = _F80CVT(OpSize::i64Bit, Data);
   }
   _PushStack(ConvertedData, Data, Width);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
@@ -76,6 +77,7 @@ void OpDispatchBuilder::FBLDF64(OpcodeArgs) {
   Ref ConvertedData = _F80BCDLoad(Data);
   ConvertedData = _F80CVT(OpSize::i64Bit, ConvertedData);
   _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
@@ -83,11 +85,13 @@ void OpDispatchBuilder::FBSTPF64(OpcodeArgs) {
   converted = _F80BCDStore(converted);
   StoreResultFPR_WithOpSize(Op, Op->Dest, converted, OpSize::f80Bit, OpSize::i8Bit);
   _PopStackDestroy();
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FLDF64_Const(OpcodeArgs, uint64_t Num) {
   auto Data = _VCastFromGPR(OpSize::i64Bit, OpSize::i64Bit, Constant(Num));
   _PushStack(Data, Data, OpSize::i64Bit);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FILDF64(OpcodeArgs) {
@@ -100,6 +104,7 @@ void OpDispatchBuilder::FILDF64(OpcodeArgs) {
   }
   auto ConvertedData = _Float_FromGPR_S(OpSize::i64Bit, ReadWidth == OpSize::i32Bit ? OpSize::i32Bit : OpSize::i64Bit, Data);
   _PushStack(ConvertedData, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FISTF64(OpcodeArgs, bool Truncate) {
@@ -128,6 +133,7 @@ void OpDispatchBuilder::FISTF64(OpcodeArgs, bool Truncate) {
   if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::OpResult ResInST0) {
@@ -142,6 +148,7 @@ void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -165,6 +172,7 @@ void OpDispatchBuilder::FADDF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
 
   // top of stack is at offset zero
   _F80AddValue(0, arg);
+  StoreX87FIP(Op);
 }
 
 // FIXME: following is very similar to FADDF64
@@ -180,6 +188,7 @@ void OpDispatchBuilder::FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -207,6 +216,7 @@ void OpDispatchBuilder::FMULF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDi
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -231,6 +241,7 @@ void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
     return;
   }
 
@@ -264,6 +275,7 @@ void OpDispatchBuilder::FDIVF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool Reverse, OpDispatchBuilder::OpResult ResInST0) {
@@ -288,6 +300,8 @@ void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
     if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
       _PopStackDestroy();
     }
+    StoreX87FIP(Op);
+
     return;
   }
 
@@ -321,6 +335,7 @@ void OpDispatchBuilder::FSUBF64(OpcodeArgs, IR::OpSize Width, bool Integer, bool
   if (Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FTSTF64(OpcodeArgs) {
@@ -330,6 +345,7 @@ void OpDispatchBuilder::FTSTF64(OpcodeArgs) {
   // Now we do our comparison.
   _F80StackTest(0);
   ConvertNZCVToX87();
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::FCOMIF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpDispatchBuilder::FCOMIFlags WhichFlags, bool PopTwice) {
@@ -376,6 +392,7 @@ void OpDispatchBuilder::FCOMIF64(OpcodeArgs, IR::OpSize Width, bool Integer, OpD
   } else if ((Op->TableInfo->Flags & X86Tables::InstFlags::FLAGS_POP) != 0) {
     _PopStackDestroy();
   }
+  StoreX87FIP(Op);
 }
 
 void OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {
@@ -429,5 +446,6 @@ void OpDispatchBuilder::X87FXTRACTF64(OpcodeArgs) {
   _PopStackDestroy();
   _PushStack(Exp, Invalid(), OpSize::iInvalid);
   _PushStack(Sig, Invalid(), OpSize::iInvalid);
+  StoreX87FIP(Op);
 }
 } // namespace FEXCore::IR

FEXCore/Source/Interface/Core/SharedCodeBufferManager.cpp

@@ -16,7 +16,7 @@ static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
 // We don't want to move above 128MB atm because that means we will have to encode longer jumps
 static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
 
-CodeBuffer::CodeBuffer(size_t Size)
+CodeBuffer::CodeBuffer(size_t Size, bool ShouldBeNamed)
   : AllocatedSize(Size) {
   Ptr = static_cast<uint8_t*>(FEXCore::Allocator::VirtualAlloc(Size, true));
   LOGMAN_THROW_A_FMT(!!Ptr, "Couldn't allocate code buffer");
@@ -28,7 +28,9 @@ CodeBuffer::CodeBuffer(size_t Size)
     LogMan::Msg::EFmt("Failed to mprotect last page of code buffer.");
   }
 
-  FEXCore::Allocator::VirtualName("FEXMemJIT", Ptr, Size);
+  if (ShouldBeNamed) {
+    FEXCore::Allocator::VirtualName("FEXMemJIT", Ptr, Size);
+  }
 
   // Huge-pages reduce the amount of iTLB misses dramatically when it works.
   FEXCore::Allocator::VirtualTHPControl(Ptr, Size, FEXCore::Allocator::THPControl::Enable);
@@ -43,6 +45,17 @@ CodeBuffer::~CodeBuffer() {
   FEXCore::Allocator::VirtualFree(Ptr, AllocatedSize);
 }
 
+SharedCodeBufferManager::SharedCodeBufferManager() {
+  FEX_CONFIG_OPT(GlobalJITNaming, GLOBALJITNAMING);
+  FEX_CONFIG_OPT(LibraryJITNaming, LIBRARYJITNAMING);
+  FEX_CONFIG_OPT(BlockJITNaming, BLOCKJITNAMING);
+
+  // Only name the JIT buffers if perf JIT naming is disabled.
+  // `perf top` prefers VMA names over the JIT symbols file for some reason.
+  // Breaks memory tracking when naming is enabled, but it's a debug feature so it isn't expected to be enabled by default.
+  NameJITBuffers = !(GlobalJITNaming || LibraryJITNaming || BlockJITNaming);
+}
+
 fextl::shared_ptr<CodeBuffer> SharedCodeBufferManager::AllocateNew(size_t Size) {
 #ifndef _WIN32
   // MDWE (Memory-Deny-Write-Execute) is a new Linux 6.3 feature.
@@ -66,7 +79,7 @@ fextl::shared_ptr<CodeBuffer> SharedCodeBufferManager::AllocateNew(size_t Size)
   }
 #endif
 
-  auto Buffer = fextl::make_shared<CodeBuffer>(Size);
+  auto Buffer = fextl::make_shared<CodeBuffer>(Size, NameJITBuffers);
 
   Latest = Buffer;
 

FEXCore/Source/Interface/Core/SharedCodeBufferManager.h

@@ -23,7 +23,7 @@ namespace FEXCore::CPU {
 struct CodeBuffer {
   fextl::unique_ptr<GuestToHostMap> LookupCache;
 
-  CodeBuffer(size_t Size);
+  CodeBuffer(size_t Size, bool ShouldBeNamed);
   CodeBuffer(const CodeBuffer&) = delete;
   CodeBuffer& operator=(const CodeBuffer&) = delete;
   CodeBuffer(CodeBuffer&& oth) = delete;
@@ -111,6 +111,7 @@ private:
  */
 class SharedCodeBufferManager {
 public:
+  SharedCodeBufferManager();
   virtual ~SharedCodeBufferManager() = default;
 
   // Get the CodeBuffer that was most recently allocated.
@@ -131,5 +132,7 @@ private:
   fextl::shared_ptr<CodeBuffer> Latest;
 
   fextl::shared_ptr<CodeBuffer> AllocateNew(size_t Size);
+
+  bool NameJITBuffers {true};
 };
 } // namespace FEXCore::CPU

FEXCore/Source/Interface/Core/X86Tables/BaseTables.cpp

@@ -83,22 +83,22 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> Primary_ArchSelect_LUT = {{
   },
   // ENTRY_27
   {
-    {"DAA",      TYPE_INST, GenFlagsDstSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::DAAOp } },
+    {"DAA",      TYPE_INST, GenFlagsDstSize(SIZE_8BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::DAAOp } },
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   // ENTRY_2F
   {
-    {"DAS",      TYPE_INST, GenFlagsDstSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::DASOp } },
+    {"DAS",      TYPE_INST, GenFlagsDstSize(SIZE_8BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::DASOp } },
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   // ENTRY_37
   {
-    {"AAA",      TYPE_INST, GenFlagsDstSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::AAAOp } },
+    {"AAA",      TYPE_INST, GenFlagsDstSize(SIZE_16BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::AAAOp } },
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   // ENTRY_3F
   {
-    {"AAS",      TYPE_INST, GenFlagsDstSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::AASOp } },
+    {"AAS",      TYPE_INST, GenFlagsDstSize(SIZE_16BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::AASOp } },
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   // ENTRY_40
@@ -134,23 +134,23 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> Primary_ArchSelect_LUT = {{
   },
   // ENTRY_A0
   {
-    {"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
-    {"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
+    {"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
+    {"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
   },
   // ENTRY_A1
   {
-    {"MOV",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
-    {"MOV",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
+    {"MOV",    TYPE_INST, FLAGS_MEM_OFFSET | FLAGS_LITERAL_PATCHABLE, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
+    {"MOV",    TYPE_INST, FLAGS_MEM_OFFSET | FLAGS_LITERAL_PATCHABLE, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
   },
   // ENTRY_A2
   {
-    {"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
-    {"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
+    {"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
+    {"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
   },
   // ENTRY_A3
   {
-    {"MOV",    TYPE_INST, FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
-    {"MOV",    TYPE_INST, FLAGS_SF_SRC_RAX | FLAGS_MEM_OFFSET, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
+    {"MOV",    TYPE_INST, FLAGS_MEM_OFFSET | FLAGS_LITERAL_PATCHABLE, 4, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
+    {"MOV",    TYPE_INST, FLAGS_MEM_OFFSET | FLAGS_LITERAL_PATCHABLE, 8, { .OpDispatch = &IR::OpDispatchBuilder::MOVOffsetOp } },
   },
   // ENTRY_CE
   {
@@ -159,17 +159,17 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> Primary_ArchSelect_LUT = {{
   },
   // ENTRY_D4
   {
-    {"AAM",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 1, { .OpDispatch = &IR::OpDispatchBuilder::AAMOp } },
+    {"AAM",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1, { .OpDispatch = &IR::OpDispatchBuilder::AAMOp } },
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   // ENTRY_D5
   {
-    {"AAD",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX, 1, { .OpDispatch = &IR::OpDispatchBuilder::AADOp } },
-    {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
+    {"AAD",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 1, { .OpDispatch = &IR::OpDispatchBuilder::AADOp } },
+    {"REX2", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   // ENTRY_D6
   {
-    {"SALC",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_SF_SRC_RAX, 0, { .OpDispatch = &IR::OpDispatchBuilder::SALCOp } },
+    {"SALC",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT), 0, { .OpDispatch = &IR::OpDispatchBuilder::SALCOp } },
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   // ENTRY_EA
@@ -204,8 +204,8 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0x01, 1, X86InstInfo{"ADD",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                                       0}},
     {0x02, 1, X86InstInfo{"ADD",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM,                                                   0}},
     {0x03, 1, X86InstInfo{"ADD",    TYPE_INST, FLAGS_MODRM,                                                                   0}},
-    {0x04, 1, X86InstInfo{"ADD",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX  ,                              1}},
-    {0x05, 1, X86InstInfo{"ADD",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
+    {0x04, 1, X86InstInfo{"ADD",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                              1}},
+    {0x05, 1, X86InstInfo{"ADD",    TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
 
     {0x06, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_06] }}},
     {0x07, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_07] }}},
@@ -214,16 +214,16 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0x09, 1, X86InstInfo{"OR",     TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                                   0}},
     {0x0A, 1, X86InstInfo{"OR",     TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM,                                                   0}},
     {0x0B, 1, X86InstInfo{"OR",     TYPE_INST, FLAGS_MODRM,                                                                   0}},
-    {0x0C, 1, X86InstInfo{"OR",     TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX ,                              1}},
-    {0x0D, 1, X86InstInfo{"OR",     TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
+    {0x0C, 1, X86InstInfo{"OR",     TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                              1}},
+    {0x0D, 1, X86InstInfo{"OR",     TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
     {0x0E, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_0E] }}},
 
     {0x10, 1, X86InstInfo{"ADC",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                   0}},
     {0x11, 1, X86InstInfo{"ADC",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                                       0}},
     {0x12, 1, X86InstInfo{"ADC",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM,                                                   0}},
     {0x13, 1, X86InstInfo{"ADC",    TYPE_INST, FLAGS_MODRM,                                                                   0}},
-    {0x14, 1, X86InstInfo{"ADC",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX  ,                              1}},
-    {0x15, 1, X86InstInfo{"ADC",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
+    {0x14, 1, X86InstInfo{"ADC",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                              1}},
+    {0x15, 1, X86InstInfo{"ADC",    TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
     {0x16, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_16] }}},
     {0x17, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_17] }}},
 
@@ -231,8 +231,8 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0x19, 1, X86InstInfo{"SBB",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                                       0}},
     {0x1A, 1, X86InstInfo{"SBB",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM,                                                   0}},
     {0x1B, 1, X86InstInfo{"SBB",    TYPE_INST, FLAGS_MODRM,                                                                   0}},
-    {0x1C, 1, X86InstInfo{"SBB",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX  ,                              1}},
-    {0x1D, 1, X86InstInfo{"SBB",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
+    {0x1C, 1, X86InstInfo{"SBB",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                              1}},
+    {0x1D, 1, X86InstInfo{"SBB",    TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
     {0x1E, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_1E] }}},
     {0x1F, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_1F] }}},
 
@@ -240,32 +240,32 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0x21, 1, X86InstInfo{"AND",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                                   0}},
     {0x22, 1, X86InstInfo{"AND",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM,                                                   0}},
     {0x23, 1, X86InstInfo{"AND",    TYPE_INST, FLAGS_MODRM,                                                                   0}},
-    {0x24, 1, X86InstInfo{"AND",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX  ,                              1}},
-    {0x25, 1, X86InstInfo{"AND",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
+    {0x24, 1, X86InstInfo{"AND",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                              1}},
+    {0x25, 1, X86InstInfo{"AND",    TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
 
     {0x27, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_27] }}},
     {0x28, 1, X86InstInfo{"SUB",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                   0}},
     {0x29, 1, X86InstInfo{"SUB",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                                   0}},
     {0x2A, 1, X86InstInfo{"SUB",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM,                                                   0}},
     {0x2B, 1, X86InstInfo{"SUB",    TYPE_INST, FLAGS_MODRM,                                                                   0}},
-    {0x2C, 1, X86InstInfo{"SUB",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX ,                              1}},
-    {0x2D, 1, X86InstInfo{"SUB",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
+    {0x2C, 1, X86InstInfo{"SUB",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                              1}},
+    {0x2D, 1, X86InstInfo{"SUB",    TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
     {0x2F, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_2F] }}},
 
     {0x30, 1, X86InstInfo{"XOR",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                   0}},
     {0x31, 1, X86InstInfo{"XOR",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                                   0}},
     {0x32, 1, X86InstInfo{"XOR",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM,                                                   0}},
     {0x33, 1, X86InstInfo{"XOR",    TYPE_INST, FLAGS_MODRM,                                                                   0}},
-    {0x34, 1, X86InstInfo{"XOR",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX  ,                              1}},
-    {0x35, 1, X86InstInfo{"XOR",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
+    {0x34, 1, X86InstInfo{"XOR",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                              1}},
+    {0x35, 1, X86InstInfo{"XOR",    TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
 
     {0x37, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_37] }}},
     {0x38, 1, X86InstInfo{"CMP",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                                                   0}},
     {0x39, 1, X86InstInfo{"CMP",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                                                   0}},
     {0x3A, 1, X86InstInfo{"CMP",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM,                                                   0}},
     {0x3B, 1, X86InstInfo{"CMP",    TYPE_INST, FLAGS_MODRM,                                                                   0}},
-    {0x3C, 1, X86InstInfo{"CMP",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX  ,                              1}},
-    {0x3D, 1, X86InstInfo{"CMP",    TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2, 4}},
+    {0x3C, 1, X86InstInfo{"CMP",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                              1}},
+    {0x3D, 1, X86InstInfo{"CMP",    TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE, 4}},
     {0x3F, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_3F] }}},
 
     {0x40, 8, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_40] }}},
@@ -321,9 +321,9 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0x8D, 1, X86InstInfo{"LEA",    TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_MODRM | FLAGS_SF_MOD_MEM_ONLY, 0}},
     {0x8E, 1, X86InstInfo{"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_16BIT) | FLAGS_MODRM,                      0}},
     {0x8F, 1, X86InstInfo{"POP",    TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_MOD_ZERO_REG | FLAGS_DEBUG_MEM_ACCESS, 0}},
-    {0x90, 8, X86InstInfo{"XCHG",   TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_SF_SRC_RAX, 0}},
-    {0x98, 1, X86InstInfo{"CDQE",   TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SF_SRC_RAX,     0}},
-    {0x99, 1, X86InstInfo{"CQO",    TYPE_INST, FLAGS_SF_DST_RDX | FLAGS_SF_SRC_RAX,     0}},
+    {0x90, 8, X86InstInfo{"XCHG",   TYPE_INST, FLAGS_SF_REX_IN_BYTE, 0}},
+    {0x98, 1, X86InstInfo{"CDQE",   TYPE_INST, FLAGS_NONE,     0}},
+    {0x99, 1, X86InstInfo{"CQO",    TYPE_INST, FLAGS_NONE,     0}},
 
     // These three are all X87 instructions
     {0x9B, 1, X86InstInfo{"FWAIT",  TYPE_INST, FLAGS_NONE,                              0}},
@@ -343,17 +343,17 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0xA6, 1, X86InstInfo{"CMPSB",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS,                                            0}},
     {0xA7, 1, X86InstInfo{"CMPS",   TYPE_INST, FLAGS_DEBUG_MEM_ACCESS,                                                            0}},
 
-    {0xA8, 1, X86InstInfo{"TEST",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX ,                                             1}},
-    {0xA9, 1, X86InstInfo{"TEST",   TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2,                4}},
-    {0xAA, 1, X86InstInfo{"STOS",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX,                   0}},
-    {0xAB, 1, X86InstInfo{"STOS",   TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX,                                   0}},
-    {0xAC, 1, X86InstInfo{"LODS",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_DST_RAX | FLAGS_DEBUG_MEM_ACCESS,                                                      0}},
-    {0xAD, 1, X86InstInfo{"LODS",   TYPE_INST, FLAGS_SF_DST_RAX | FLAGS_DEBUG_MEM_ACCESS,                                                      0}},
-    {0xAE, 1, X86InstInfo{"SCAS",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX,                                   0}},
-    {0xAF, 1, X86InstInfo{"SCAS",   TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_SF_SRC_RAX,                                   0}},
+    {0xA8, 1, X86InstInfo{"TEST",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT),                                             1}},
+    {0xA9, 1, X86InstInfo{"TEST",   TYPE_INST, FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE,                4}},
+    {0xAA, 1, X86InstInfo{"STOS",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS,                   0}},
+    {0xAB, 1, X86InstInfo{"STOS",   TYPE_INST, FLAGS_DEBUG_MEM_ACCESS,                                   0}},
+    {0xAC, 1, X86InstInfo{"LODS",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS,                                                      0}},
+    {0xAD, 1, X86InstInfo{"LODS",   TYPE_INST, FLAGS_DEBUG_MEM_ACCESS,                                                      0}},
+    {0xAE, 1, X86InstInfo{"SCAS",   TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_DEBUG_MEM_ACCESS,                                   0}},
+    {0xAF, 1, X86InstInfo{"SCAS",   TYPE_INST, FLAGS_DEBUG_MEM_ACCESS,                                   0}},
 
     {0xB0, 8, X86InstInfo{"MOV",    TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_SF_REX_IN_BYTE ,                                         1}},
-    {0xB8, 8, X86InstInfo{"MOV",    TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_DISPLACE_SIZE_MUL_2, 4}},
+    {0xB8, 8, X86InstInfo{"MOV",    TYPE_INST, FLAGS_SF_REX_IN_BYTE | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_DISPLACE_SIZE_MUL_2 | FLAGS_LITERAL_PATCHABLE, 4}},
 
     {0xC2, 1, X86InstInfo{"RET",    TYPE_INST, FLAGS_SETS_RIP | FLAGS_BLOCK_END,                                             2}},
     {0xC3, 1, X86InstInfo{"RET",    TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END ,                                                0}},
@@ -362,7 +362,7 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0xCA, 1, X86InstInfo{"RETF",   TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END,                                                              2}},
     {0xCB, 1, X86InstInfo{"RETF",   TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END,                                                              0}},
     {0xCC, 1, X86InstInfo{"INT3",   TYPE_INST, FLAGS_BLOCK_END,                                                                                      0}},
-    {0xCD, 1, X86InstInfo{"INT",    TYPE_INST, DEFAULT_SYSCALL_FLAGS,                                                                  1}},
+    {0xCD, 1, X86InstInfo{"INT",    TYPE_INST, FLAGS_NO_OVERLAY | FLAGS_BLOCK_END,                                                                  1}},
     {0xCE, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_CE] }}},
     {0xCF, 1, X86InstInfo{"IRET",   TYPE_INST, FLAGS_SETS_RIP | FLAGS_BLOCK_END,                                                                                    0}},
 
@@ -371,9 +371,9 @@ const std::array<X86InstInfo, MAX_PRIMARY_TABLE_SIZE> BaseOps = []() consteval {
     {0xD6, 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = Primary_ArchSelect_LUT[ENTRY_D6] }}},
     {0xD7, 1, X86InstInfo{"XLAT",   TYPE_INST, FLAGS_DEBUG_MEM_ACCESS,                                                                           0}},
 
-    {0xE0, 1, X86InstInfo{"LOOPNE", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX,                             1}},
-    {0xE1, 1, X86InstInfo{"LOOPE",  TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX,                             1}},
-    {0xE2, 1, X86InstInfo{"LOOP",   TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT | FLAGS_SF_SRC_RCX,                             1}},
+    {0xE0, 1, X86InstInfo{"LOOPNE", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT,                             1}},
+    {0xE1, 1, X86InstInfo{"LOOPE",  TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT,                             1}},
+    {0xE2, 1, X86InstInfo{"LOOP",   TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT,                             1}},
     {0xE3, 1, X86InstInfo{"JrCXZ",  TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_SRC_SEXT ,                             1}},
 
     // Should just throw GP

FEXCore/Source/Interface/Core/X86Tables/PrimaryGroupTables.cpp

@@ -36,11 +36,11 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> PrimaryGroup_ArchSelect_LUT = {{
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   {
-    {"ADC",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::ADCOp, 1> }},
+    {"ADC",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::ADCOp, 1, false> }},
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   {
-    {"SBB",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::SBBOp, 1> }},
+    {"SBB",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::SBBOp, 1, false> }},
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   {
@@ -56,7 +56,7 @@ constexpr std::array<X86InstInfo[2], ENTRY_MAX> PrimaryGroup_ArchSelect_LUT = {{
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
   {
-    {"CMP",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::CMPOp, 1> }},
+    {"CMP",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST, 1, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::CMPOp, 1, false> }},
     {"", TYPE_INVALID, FLAGS_NONE, 0, { .OpDispatch = nullptr } },
   },
 }};
@@ -75,14 +75,14 @@ constexpr std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps
     {OPD(TYPE_GROUP_1, OpToIndex(0x80), 6), 1, X86InstInfo{"XOR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                      1}},
     {OPD(TYPE_GROUP_1, OpToIndex(0x80), 7), 1, X86InstInfo{"CMP",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                      1}},
 
-    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 0), 1, X86InstInfo{"ADD",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                          4}},
-    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 1), 1, X86InstInfo{"OR",   TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                          4}},
-    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 2), 1, X86InstInfo{"ADC",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                          4}},
-    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 3), 1, X86InstInfo{"SBB",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                          4}},
-    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 4), 1, X86InstInfo{"AND",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                          4}},
-    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 5), 1, X86InstInfo{"SUB",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                          4}},
-    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 6), 1, X86InstInfo{"XOR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                          4}},
-    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 7), 1, X86InstInfo{"CMP",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK,                          4}},
+    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 0), 1, X86InstInfo{"ADD",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE,                          4}},
+    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 1), 1, X86InstInfo{"OR",   TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE,                          4}},
+    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 2), 1, X86InstInfo{"ADC",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE,                          4}},
+    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 3), 1, X86InstInfo{"SBB",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE,                          4}},
+    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 4), 1, X86InstInfo{"AND",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE,                          4}},
+    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 5), 1, X86InstInfo{"SUB",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE,                          4}},
+    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 6), 1, X86InstInfo{"XOR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE,                          4}},
+    {OPD(TYPE_GROUP_1, OpToIndex(0x81), 7), 1, X86InstInfo{"CMP",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_SUPPORTS_LOCK | FLAGS_LITERAL_PATCHABLE,                          4}},
 
     // Duplicates the 0x80 opcode group
     {OPD(TYPE_GROUP_1, OpToIndex(0x82), 0), 1, X86InstInfo{"",  TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = PrimaryGroup_ArchSelect_LUT[ENTRY_1_82_0] }}},
@@ -140,23 +140,23 @@ constexpr std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps
     {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 6), 1, X86InstInfo{"SHL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                                      0}},
     {OPD(TYPE_GROUP_2, OpToIndex(0xD1), 7), 1, X86InstInfo{"SAR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                                      0}},
 
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, X86InstInfo{"ROL",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, X86InstInfo{"ROR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, X86InstInfo{"RCL",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, X86InstInfo{"RCR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, X86InstInfo{"SHL",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, X86InstInfo{"SHR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 6), 1, X86InstInfo{"SHL",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 7), 1, X86InstInfo{"SAR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 0), 1, X86InstInfo{"ROL",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 1), 1, X86InstInfo{"ROR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 2), 1, X86InstInfo{"RCL",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 3), 1, X86InstInfo{"RCR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 4), 1, X86InstInfo{"SHL",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 5), 1, X86InstInfo{"SHR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 6), 1, X86InstInfo{"SHL",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD2), 7), 1, X86InstInfo{"SAR",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                   0}},
 
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, X86InstInfo{"ROL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, X86InstInfo{"ROR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, X86InstInfo{"RCL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, X86InstInfo{"RCR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, X86InstInfo{"SHL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, X86InstInfo{"SHR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 6), 1, X86InstInfo{"SHL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                                   0}},
-    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, X86InstInfo{"SAR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX,                                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 0), 1, X86InstInfo{"ROL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 1), 1, X86InstInfo{"ROR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 2), 1, X86InstInfo{"RCL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 3), 1, X86InstInfo{"RCR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 4), 1, X86InstInfo{"SHL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 5), 1, X86InstInfo{"SHR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 6), 1, X86InstInfo{"SHL",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                   0}},
+    {OPD(TYPE_GROUP_2, OpToIndex(0xD3), 7), 1, X86InstInfo{"SAR",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,                                   0}},
 
     // GROUP 3
     {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 0), 1, X86InstInfo{"TEST", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                                      1}},
@@ -168,7 +168,7 @@ constexpr std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps
     {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 6), 1, X86InstInfo{"DIV",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                                      0}},
     {OPD(TYPE_GROUP_3, OpToIndex(0xF6), 7), 1, X86InstInfo{"IDIV", TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST,                                      0}},
 
-    {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, X86InstInfo{"TEST", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2,                          4}},
+    {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 0), 1, X86InstInfo{"TEST", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE,                          4}},
     {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 1), 1, X86InstInfo{"TEST", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT64BIT | FLAGS_DISPLACE_SIZE_DIV_2,                          4}},
     {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 2), 1, X86InstInfo{"NOT",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                      0}},
     {OPD(TYPE_GROUP_3, OpToIndex(0xF7), 3), 1, X86InstInfo{"NEG",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SUPPORTS_LOCK,                                                      0}},
@@ -196,7 +196,7 @@ constexpr std::array<X86InstInfo, MAX_INST_GROUP_TABLE_SIZE> PrimaryInstGroupOps
     {OPD(TYPE_GROUP_11, OpToIndex(0xC6), 0), 1, X86InstInfo{"MOV",  TYPE_INST, GenFlagsSameSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_SF_MOD_DST  | FLAGS_SRC_SEXT,                   1}},
     {OPD(TYPE_GROUP_11, OpToIndex(0xC6), 1), 5, X86InstInfo{"",     TYPE_INVALID, FLAGS_NONE,                                                       0}},
     {OPD(TYPE_GROUP_11, OpToIndex(0xC6), 7), 1, X86InstInfo{"XABORT", TYPE_INST, FLAGS_MODRM,                                                       1}},
-    {OPD(TYPE_GROUP_11, OpToIndex(0xC7), 0), 1, X86InstInfo{"MOV",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2,                                   4}},
+    {OPD(TYPE_GROUP_11, OpToIndex(0xC7), 0), 1, X86InstInfo{"MOV",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SRC_SEXT | FLAGS_DISPLACE_SIZE_DIV_2 | FLAGS_LITERAL_PATCHABLE,                                   4}},
     {OPD(TYPE_GROUP_11, OpToIndex(0xC7), 1), 5, X86InstInfo{"",     TYPE_INVALID, FLAGS_NONE,                                                       0}},
     {OPD(TYPE_GROUP_11, OpToIndex(0xC7), 7), 1, X86InstInfo{"XBEGIN", TYPE_INST, FLAGS_MODRM | FLAGS_SRC_SEXT | FLAGS_SETS_RIP | FLAGS_DISPLACE_SIZE_DIV_2,                                                       4}},
   };

FEXCore/Source/Interface/Core/X86Tables/SecondaryGroupTables.cpp

@@ -61,7 +61,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     // Pulls from other MODRM table
 
     // GROUP 6
-    {OPD(TYPE_GROUP_6, PF_NONE, 0), 1, X86InstInfo{"SLDT",  TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
+    {OPD(TYPE_GROUP_6, PF_NONE, 0), 1, X86InstInfo{"SLDT",  TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
     {OPD(TYPE_GROUP_6, PF_NONE, 1), 1, X86InstInfo{"STR",   TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST,  0}},
     {OPD(TYPE_GROUP_6, PF_NONE, 2), 1, X86InstInfo{"LLDT",  TYPE_PRIV, FLAGS_NONE,       0}},
     {OPD(TYPE_GROUP_6, PF_NONE, 3), 1, X86InstInfo{"LTR",   TYPE_INST, FLAGS_NONE,       0}},
@@ -70,7 +70,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     {OPD(TYPE_GROUP_6, PF_NONE, 6), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,    0}},
     {OPD(TYPE_GROUP_6, PF_NONE, 7), 1, X86InstInfo{"",      TYPE_INVALID, FLAGS_NONE,    0}},
 
-    {OPD(TYPE_GROUP_6, PF_F3, 0), 1, X86InstInfo{"SLDT",    TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
+    {OPD(TYPE_GROUP_6, PF_F3, 0), 1, X86InstInfo{"SLDT",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
     {OPD(TYPE_GROUP_6, PF_F3, 1), 1, X86InstInfo{"STR",     TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST,  0}},
     {OPD(TYPE_GROUP_6, PF_F3, 2), 1, X86InstInfo{"LLDT",    TYPE_PRIV, FLAGS_NONE,       0}},
     {OPD(TYPE_GROUP_6, PF_F3, 3), 1, X86InstInfo{"LTR",     TYPE_INST, FLAGS_NONE,       0}},
@@ -79,7 +79,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     {OPD(TYPE_GROUP_6, PF_F3, 6), 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,    0}},
     {OPD(TYPE_GROUP_6, PF_F3, 7), 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,    0}},
 
-    {OPD(TYPE_GROUP_6, PF_66, 0), 1, X86InstInfo{"SLDT",    TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
+    {OPD(TYPE_GROUP_6, PF_66, 0), 1, X86InstInfo{"SLDT",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
     {OPD(TYPE_GROUP_6, PF_66, 1), 1, X86InstInfo{"STR",     TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST,  0}},
     {OPD(TYPE_GROUP_6, PF_66, 2), 1, X86InstInfo{"LLDT",    TYPE_PRIV, FLAGS_NONE,       0}},
     {OPD(TYPE_GROUP_6, PF_66, 3), 1, X86InstInfo{"LTR",     TYPE_INST, FLAGS_NONE,       0}},
@@ -88,7 +88,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     {OPD(TYPE_GROUP_6, PF_66, 6), 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,    0}},
     {OPD(TYPE_GROUP_6, PF_66, 7), 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,    0}},
 
-    {OPD(TYPE_GROUP_6, PF_F2, 0), 1, X86InstInfo{"SLDT",    TYPE_UNDEC, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
+    {OPD(TYPE_GROUP_6, PF_F2, 0), 1, X86InstInfo{"SLDT",    TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
     {OPD(TYPE_GROUP_6, PF_F2, 1), 1, X86InstInfo{"STR",     TYPE_PRIV, FLAGS_MODRM | FLAGS_SF_MOD_DST,  0}},
     {OPD(TYPE_GROUP_6, PF_F2, 2), 1, X86InstInfo{"LLDT",    TYPE_PRIV, FLAGS_NONE,       0}},
     {OPD(TYPE_GROUP_6, PF_F2, 3), 1, X86InstInfo{"LTR",     TYPE_INST, FLAGS_NONE,       0}},
@@ -103,7 +103,7 @@ constexpr std::array<X86InstInfo, MAX_INST_SECOND_GROUP_TABLE_SIZE> SecondInstGr
     {OPD(TYPE_GROUP_7, PF_NONE, 2), 1, X86InstInfo{"",     TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
     {OPD(TYPE_GROUP_7, PF_NONE, 3), 1, X86InstInfo{"",     TYPE_SECOND_GROUP_MODRM, FLAGS_NONE, 0}},
     {OPD(TYPE_GROUP_7, PF_NONE, 4), 1, X86InstInfo{"SMSW", TYPE_INST, FLAGS_MODRM | FLAGS_SF_MOD_DST,          0}},
-    {OPD(TYPE_GROUP_7, PF_NONE, 5), 1, X86InstInfo{"",     TYPE_INVALID, FLAGS_NONE,            0}},
+    {OPD(TYPE_GROUP_7, PF_NONE, 5), 1, X86InstInfo{"SERIALIZE", TYPE_INST, FLAGS_NONE,            0}},
     {OPD(TYPE_GROUP_7, PF_NONE, 6), 1, X86InstInfo{"LMSW", TYPE_INST, FLAGS_MODRM,          0}},
     {OPD(TYPE_GROUP_7, PF_NONE, 7), 1, X86InstInfo{"INVLPG", TYPE_SECOND_GROUP_MODRM, FLAGS_MODRM | FLAGS_SF_MOD_DST, 0}},
 

FEXCore/Source/Interface/Core/X86Tables/SecondaryModRMTables.cpp

@@ -12,6 +12,19 @@ $end_info$
 
 namespace FEXCore::X86Tables {
 using namespace InstFlags;
+enum ModRM_LUT {
+  ENTRY_3_0,
+  ENTRY_MAX,
+};
+
+constexpr std::array<X86InstInfo[2], ENTRY_MAX> ModRM_ArchSelect_LUT = {{
+  // ENTRY_3_0
+  {
+    {"SWAPGS",   TYPE_INST,    FLAGS_NONE, 0, { .OpDispatch = &IR::OpDispatchBuilder::UnimplementedOp }},
+    {"SWAPGS",   TYPE_INST,    FLAGS_NONE, 0, { .OpDispatch = &IR::OpDispatchBuilder::PermissionRestrictedOp }},
+  },
+}};
+
 constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableOps = []() consteval {
   std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> Table{};
   constexpr U8U8InfoStruct SecondaryModRMExtensionOpTable[] = {
@@ -27,7 +40,7 @@ constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableO
 
     // REG /2
     {((1 << 3) | 0), 1, X86InstInfo{"XGETBV",   TYPE_INST,    FLAGS_NONE, 0}},
-    {((1 << 3) | 1), 1, X86InstInfo{"XSETBV",   TYPE_PRIV,    FLAGS_NONE, 0}},
+    {((1 << 3) | 1), 1, X86InstInfo{"XSETBV",   TYPE_INST,    FLAGS_NONE, 0}},
     {((1 << 3) | 2), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},
     {((1 << 3) | 3), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},
     {((1 << 3) | 4), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},
@@ -46,11 +59,11 @@ constexpr std::array<X86InstInfo, MAX_SECOND_MODRM_TABLE_SIZE> SecondModRMTableO
     {((2 << 3) | 7), 1, X86InstInfo{"INVLPGA",  TYPE_INST,    FLAGS_NONE, 0}},
 
     // REG /7
-    {((3 << 3) | 0), 1, X86InstInfo{"SWAPGS",   TYPE_INST,    FLAGS_NONE, 0}},
+    {((3 << 3) | 0), 1, X86InstInfo{"", TYPE_ARCH_DISPATCHER, FLAGS_NONE, 0, { .Indirect = ModRM_ArchSelect_LUT[ENTRY_3_0] }}},
     {((3 << 3) | 1), 1, X86InstInfo{"RDTSCP",   TYPE_INST,    FLAGS_NONE, 0}},
     {((3 << 3) | 2), 1, X86InstInfo{"MONITORX", TYPE_PRIV,    FLAGS_NONE, 0}},
     {((3 << 3) | 3), 1, X86InstInfo{"MWAITX",   TYPE_PRIV,    FLAGS_NONE, 0}},
-    {((3 << 3) | 4), 1, X86InstInfo{"CLZERO",   TYPE_INST,    GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SF_SRC_RAX | FLAGS_DEBUG_MEM_ACCESS, 0}},
+    {((3 << 3) | 4), 1, X86InstInfo{"CLZERO",   TYPE_INST,    GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS, 0}},
     {((3 << 3) | 5), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},
     {((3 << 3) | 6), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},
     {((3 << 3) | 7), 1, X86InstInfo{"",         TYPE_INVALID, FLAGS_NONE, 0}},

FEXCore/Source/Interface/Core/X86Tables/SecondaryTables.cpp

@@ -26,8 +26,8 @@ enum Secondary_LUT {
 
 constexpr std::array<X86InstInfo[2], ENTRY_MAX> Secondary_ArchSelect_LUT = {{
   {
-    {"SYSCALL", TYPE_INST, DEFAULT_SYSCALL_FLAGS, 0, { .OpDispatch = &IR::OpDispatchBuilder::NOPOp } },
-    {"SYSCALL", TYPE_INST, DEFAULT_SYSCALL_FLAGS, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::SyscallOp, true> } },
+    {"SYSCALL", TYPE_INST, FLAGS_NO_OVERLAY | FLAGS_BLOCK_END, 0, { .OpDispatch = &IR::OpDispatchBuilder::NOPOp } },
+    {"SYSCALL", TYPE_INST, FLAGS_NO_OVERLAY | FLAGS_BLOCK_END, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::SyscallOp, true> } },
   },
   {
     {"PUSH FS", TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .OpDispatch = &IR::OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::PUSHSegmentOp, FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX> } },
@@ -204,17 +204,17 @@ constexpr std::array<X86InstInfo, MAX_SECOND_TABLE_SIZE> SecondBaseOps = []() co
 
     {0xA0, 1, X86InstInfo{"",        TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A0] }}},
     {0xA1, 1, X86InstInfo{"",        TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A1] }}},
-    {0xA2, 1, X86InstInfo{"CPUID",   TYPE_INST,     FLAGS_SF_SRC_RAX | FLAGS_NO_OVERLAY,                                              0}},
+    {0xA2, 1, X86InstInfo{"CPUID",   TYPE_INST,     FLAGS_NO_OVERLAY | FLAGS_BLOCK_END,                                              0}},
     {0xA3, 1, X86InstInfo{"BT",      TYPE_INST,     FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY | FLAGS_SUPPORTS_LOCK, 0}},
     {0xA4, 1, X86InstInfo{"SHLD",    TYPE_INST,     FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY,                                              1}},
-    {0xA5, 1, X86InstInfo{"SHLD",    TYPE_INST,     FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX | FLAGS_NO_OVERLAY,                           0}},
+    {0xA5, 1, X86InstInfo{"SHLD",    TYPE_INST,     FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY,                           0}},
     {0xA6, 2, X86InstInfo{"",        TYPE_INVALID,  FLAGS_NO_OVERLAY,                                                                               0}},
     {0xA8, 1, X86InstInfo{"",        TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A8] }}},
     {0xA9, 1, X86InstInfo{"",        TYPE_ARCH_DISPATCHER, FLAGS_DEBUG_MEM_ACCESS | FLAGS_NO_OVERLAY, 0, { .Indirect = Secondary_ArchSelect_LUT[ENTRY_A9] }}},
     {0xAA, 1, X86InstInfo{"RSM",     TYPE_PRIV,     FLAGS_NO_OVERLAY,                                                                               0}},
     {0xAB, 1, X86InstInfo{"BTS",     TYPE_INST,     FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY | FLAGS_SUPPORTS_LOCK, 0}},
     {0xAC, 1, X86InstInfo{"SHRD",    TYPE_INST,     FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY,                                              1}},
-    {0xAD, 1, X86InstInfo{"SHRD",    TYPE_INST,     FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_SF_SRC_RCX | FLAGS_NO_OVERLAY,                           0}},
+    {0xAD, 1, X86InstInfo{"SHRD",    TYPE_INST,     FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY,                           0}},
     {0xAE, 1, X86InstInfo{"",        TYPE_GROUP_15, FLAGS_NO_OVERLAY,                                                                               0}},
     {0xAF, 1, X86InstInfo{"IMUL",    TYPE_INST,     FLAGS_MODRM | FLAGS_NO_OVERLAY,                                                                 0}},
 
@@ -226,12 +226,12 @@ constexpr std::array<X86InstInfo, MAX_SECOND_TABLE_SIZE> SecondBaseOps = []() co
     {0xB5, 1, X86InstInfo{"LGS",     TYPE_INVALID, FLAGS_NO_OVERLAY,                                                                                0}},
     {0xB6, 1, X86InstInfo{"MOVZX",   TYPE_INST, GenFlagsSrcSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY,                                        0}},
     {0xB7, 1, X86InstInfo{"MOVZX",   TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY,                                       0}},
-    {0xB8, 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE,                                                                                      0}},
+    {0xB8, 1, X86InstInfo{"",        TYPE_INVALID, FLAGS_NO_PREFIX_ORDER,                                                                           0}},
     {0xB9, 1, X86InstInfo{"",        TYPE_GROUP_10, FLAGS_NO_OVERLAY,                                                                               0}},
     {0xBA, 1, X86InstInfo{"",        TYPE_GROUP_8, FLAGS_NO_OVERLAY,                                                                                0}},
     {0xBB, 1, X86InstInfo{"BTC",     TYPE_INST, FLAGS_DEBUG_MEM_ACCESS | FLAGS_MODRM | FLAGS_SF_MOD_DST | FLAGS_NO_OVERLAY | FLAGS_SUPPORTS_LOCK,   0}},
-    {0xBC, 1, X86InstInfo{"BSF",     TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66,                                                                   0}},
-    {0xBD, 1, X86InstInfo{"BSR",     TYPE_INST, FLAGS_MODRM | FLAGS_NO_OVERLAY66,                                                                   0}},
+    {0xBC, 1, X86InstInfo{"BSF",     TYPE_INST, FLAGS_MODRM | FLAGS_NO_PREFIX_ORDER | FLAGS_NO_OVERLAY66,                                           0}},
+    {0xBD, 1, X86InstInfo{"BSR",     TYPE_INST, FLAGS_MODRM | FLAGS_NO_PREFIX_ORDER | FLAGS_NO_OVERLAY66,                                           0}},
     {0xBE, 1, X86InstInfo{"MOVSX",   TYPE_INST, GenFlagsSrcSize(SIZE_8BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY,                                        0}},
     {0xBF, 1, X86InstInfo{"MOVSX",   TYPE_INST, GenFlagsSrcSize(SIZE_16BIT) | FLAGS_MODRM | FLAGS_NO_OVERLAY,                                       0}},
 

FEXCore/Source/Interface/Core/X86Tables/VEXTables.cpp

@@ -312,10 +312,16 @@ namespace AVX128 {
     {OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VariableShiftImpl, IROps::OP_VSSHR>}, // VPSRAVD
     {OPD(2, 0b01, 0x47), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VariableShiftImpl, IROps::OP_VUSHL>}, // VPSLLV
 
+    {OPD(2, 0b01, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPDPBUSD, false>},
+    {OPD(2, 0b01, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPDPBUSD, true>},
+    {OPD(2, 0b01, 0x52), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPDPWSSD, false>},
+    {OPD(2, 0b01, 0x53), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VPDPWSSD, true>},
+
     {OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i32Bit>},
     {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i64Bit>},
     {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i128Bit>},
 
+    {OPD(2, 0b10, 0x72), 1, &OpDispatchBuilder::AVX128_VCVTNEPS2BF16},
     {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i8Bit>},
     {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VBROADCAST, OpSize::i16Bit>},
 
@@ -360,6 +366,13 @@ namespace AVX128 {
     {OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 1, 3>},  // VFMADDSUB
     {OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD
 
+    {OPD(2, 0b00, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEPH2PS, false>},
+    {OPD(2, 0b01, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEPH2PS, true>},
+    {OPD(2, 0b10, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, true>},
+    {OPD(2, 0b11, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVX128_VCVTNEEBF162PS, false>},
+    {OPD(2, 0b01, 0xB1), 1, &IR::OpDispatchBuilder::AVX128_VBCSTNESH2PS},
+    {OPD(2, 0b10, 0xB1), 1, &IR::OpDispatchBuilder::AVX128_VBCSTNEBF162PS},
+
     {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, true, 2, 3, 1>},  // VFMADDSUB
     {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::AVX128_VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD
 
@@ -757,10 +770,16 @@ namespace AVX256 {
     {OPD(2, 0b01, 0x46), 1, &OpDispatchBuilder::VPSRAVDOp},
     {OPD(2, 0b01, 0x47), 1, &OpDispatchBuilder::VPSLLVOp},
 
+    {OPD(2, 0b01, 0x50), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPDPBUSDOp, false>},
+    {OPD(2, 0b01, 0x51), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPDPBUSDOp, true>},
+    {OPD(2, 0b01, 0x52), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPDPWSSDOp, false>},
+    {OPD(2, 0b01, 0x53), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VPDPWSSDOp, true>},
+
     {OPD(2, 0b01, 0x58), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i32Bit>},
     {OPD(2, 0b01, 0x59), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i64Bit>},
     {OPD(2, 0b01, 0x5A), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i128Bit>},
 
+    {OPD(2, 0b10, 0x72), 1, &OpDispatchBuilder::AVXVCVTNEPS2BF16},
     {OPD(2, 0b01, 0x78), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i8Bit>},
     {OPD(2, 0b01, 0x79), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VBROADCASTOp, OpSize::i16Bit>},
 
@@ -805,6 +824,13 @@ namespace AVX256 {
     {OPD(2, 0b01, 0xA6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 1, 3>},  // VFMADDSUB
     {OPD(2, 0b01, 0xA7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 1, 3>}, // VFMSUBADD
 
+    {OPD(2, 0b00, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEPH2PS, false>},
+    {OPD(2, 0b01, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEPH2PS, true>},
+    {OPD(2, 0b10, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEBF162PS, true>},
+    {OPD(2, 0b11, 0xB0), 1, &OpDispatchBuilder::Bind<&IR::OpDispatchBuilder::AVXVCVTNEEBF162PS, false>},
+    {OPD(2, 0b01, 0xB1), 1, &IR::OpDispatchBuilder::AVXVBCSTNESH2PS},
+    {OPD(2, 0b10, 0xB1), 1, &IR::OpDispatchBuilder::AVXVBCSTNEBF162PS},
+
     {OPD(2, 0b01, 0xB6), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, true, 2, 3, 1>},  // VFMADDSUB
     {OPD(2, 0b01, 0xB7), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::VFMAddSubImpl, false, 2, 3, 1>}, // VFMSUBADD
 
@@ -1207,10 +1233,16 @@ auto BaseTableLambda = [](const auto RuntimeTable) consteval {
     {OPD(2, 0b01, 0x46), 1, X86InstInfo{"VPSRAVD", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0x47), 1, X86InstInfo{"VPSLLV", TYPE_INST, GenFlagsSizes(SIZE_128BIT, SIZE_32BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
 
+    {OPD(2, 0b01, 0x50), 1, X86InstInfo{"VPDPBUSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b01, 0x51), 1, X86InstInfo{"VPDPBUSDS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b01, 0x52), 1, X86InstInfo{"VPDPWSSD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b01, 0x53), 1, X86InstInfo{"VPDPWSSDS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
+
     {OPD(2, 0b01, 0x58), 1, X86InstInfo{"VPBROADCASTD", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0x59), 1, X86InstInfo{"VPBROADCASTQ", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0x5A), 1, X86InstInfo{"VBROADCASTI128", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_L_1 | FLAGS_SF_MOD_MEM_ONLY | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
 
+    {OPD(2, 0b10, 0x72), 1, X86InstInfo{"VCVTNEPS2BF16", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0x78), 1, X86InstInfo{"VPBROADCASTB", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0x79), 1, X86InstInfo{"VPBROADCASTW", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_REX_W_0 | FLAGS_XMM_FLAGS, 0}},
 
@@ -1255,6 +1287,13 @@ auto BaseTableLambda = [](const auto RuntimeTable) consteval {
     {OPD(2, 0b01, 0xA6), 1, X86InstInfo{"VFMADDSUB213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0xA7), 1, X86InstInfo{"VFMSUBADD213", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
 
+    {OPD(2, 0b00, 0xB0), 1, X86InstInfo{"VCVTNEOPH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b01, 0xB0), 1, X86InstInfo{"VCVTNEEPH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b10, 0xB0), 1, X86InstInfo{"VCVTNEEBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b11, 0xB0), 1, X86InstInfo{"VCVTNEOBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b01, 0xB1), 1, X86InstInfo{"VBCSTNESH2PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+    {OPD(2, 0b10, 0xB1), 1, X86InstInfo{"VBCSTNEBF162PS", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS, 0}},
+
     {OPD(2, 0b01, 0xB6), 1, X86InstInfo{"VFMADDSUB231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
     {OPD(2, 0b01, 0xB7), 1, X86InstInfo{"VFMSUBADD231", TYPE_INST, GenFlagsSameSize(SIZE_128BIT) | FLAGS_MODRM | FLAGS_VEX_1ST_SRC | FLAGS_XMM_FLAGS, 0}},
 

FEXCore/Source/Interface/Core/X86Tables/X86Tables.h

@@ -185,10 +185,14 @@ struct DecodedOperand {
     struct {
       int64_t Displacement;
       uint8_t GPR;
+      bool PatchableDisp;
+      uint8_t DispOffset;
     } GPRIndirect; // Shared with GPRIndirectRelocation
 
     struct {
       int64_t Value;
+      bool PatchableDisp;
+      uint8_t DispOffset;
     } RIPLiteral; // Shared with RIPLiteralRelocation
 
     struct LiteralType {
@@ -211,7 +215,9 @@ struct DecodedOperand {
       uint8_t Scale;
       uint8_t Index; // ~0 invalid
       uint8_t Base;  // ~0 invalid
-    } SIB;           // Shared with SIBRelocation
+      bool PatchableDisp;
+      uint8_t DispOffset;
+    } SIB; // Shared with SIBRelocation
   };
 
   TypeUnion Data;
@@ -349,11 +355,8 @@ namespace InstFlags {
   constexpr InstFlagType FLAGS_X87_FLAGS = (1ULL << 10);
 
   // Non-XMM subflags
-  constexpr InstFlagType FLAGS_SF_DST_RAX = (1ULL << 11);
-  constexpr InstFlagType FLAGS_SF_DST_RDX = (1ULL << 12);
-  constexpr InstFlagType FLAGS_SF_SRC_RAX = (1ULL << 13);
-  constexpr InstFlagType FLAGS_SF_SRC_RCX = (1ULL << 14);
-  constexpr InstFlagType FLAGS_SF_REX_IN_BYTE = (1ULL << 15);
+  constexpr InstFlagType FLAGS_SF_REX_IN_BYTE = (1ULL << 11);
+  // subflag [15:12] unused
 
   // XMM subflags
   constexpr InstFlagType FLAGS_SF_UNUSED = (1ULL << 11); // No assigned behavior yet
@@ -403,7 +406,10 @@ namespace InstFlags {
 
   constexpr InstFlagType FLAGS_CALL = (1ULL << 30);
   constexpr InstFlagType FLAGS_SUPPORTS_LOCK = (1ULL << 31);
-  // Flags [57..32]: Undefined
+  constexpr InstFlagType FLAGS_LITERAL_PATCHABLE = (1ULL << 32);
+  constexpr InstFlagType FLAGS_NO_PREFIX_ORDER = (1ULL << 33);
+
+  // Flags [57..34]: Undefined
   // Flags [60..58]: Dst size
   constexpr InstFlagType FLAGS_SIZE_DST_OFF = 58;
   // Flags [63..61]: Src size
@@ -419,13 +425,6 @@ namespace InstFlags {
   constexpr InstFlagType SIZE_256BIT = 0b110;
   constexpr InstFlagType SIZE_64BITDEF = 0b111; // Default mode is 64bit instead of typical 32bit
 
-#ifndef _WIN32
-  constexpr uint32_t DEFAULT_SYSCALL_FLAGS = FLAGS_NO_OVERLAY;
-#else
-                                                // Syscall ends a block on WIN32 because the instruction can update the CPU's RIP.
-  constexpr uint32_t DEFAULT_SYSCALL_FLAGS = FLAGS_NO_OVERLAY | FLAGS_BLOCK_END;
-#endif
-
   constexpr InstFlagType GetSizeDstFlags(InstFlagType Flags) {
     return (Flags >> FLAGS_SIZE_DST_OFF) & SIZE_MASK;
   }

FEXCore/Source/Interface/Core/X86Tables/X87Tables.cpp

@@ -69,7 +69,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F64OpTable = {{
 
   {OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},
   {OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},
-  {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP
+  {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::X87NOP}, // FNOP
   // D1 = Invalid
   // D8 = Invalid
   {OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},
@@ -216,7 +216,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F64OpTable = {{
   // 5 = Invalid
   {OPDReg(0xDD, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSAVE},
 
-  {OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::X87FNSTSW},
+  {OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FNSTSW, false>},
 
   {OPD(0xDD, 0xC0), 8, &OpDispatchBuilder::X87FFREE},
   {OPD(0xDD, 0xC8), 8, &OpDispatchBuilder::FXCH},
@@ -284,7 +284,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F64OpTable = {{
   {OPD(0xDF, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
   {OPD(0xDF, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
 
-  {OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::X87FNSTSW},
+  {OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FNSTSW, true>},
   {OPD(0xDF, 0xE8), 8,
    &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMIF64, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
   {OPD(0xDF, 0xF0), 8,
@@ -341,7 +341,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F80OpTable = {{
 
   {OPD(0xD9, 0xC0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FLDFromStack>},
   {OPD(0xD9, 0xC8), 8, &OpDispatchBuilder::FXCH},
-  {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::NOPOp}, // FNOP
+  {OPD(0xD9, 0xD0), 1, &OpDispatchBuilder::X87NOP}, // FNOP
   // D1 = Invalid
   // D8 = Invalid
   {OPD(0xD9, 0xE0), 1, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87OpHelper, OP_F80STACKCHANGESIGN, false>},
@@ -483,7 +483,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F80OpTable = {{
   // 5 = Invalid
   {OPDReg(0xDD, 6) | 0x00, 8, &OpDispatchBuilder::X87FNSAVE},
 
-  {OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::X87FNSTSW},
+  {OPDReg(0xDD, 7) | 0x00, 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FNSTSW, false>},
 
   {OPD(0xDD, 0xC0), 8, &OpDispatchBuilder::X87FFREE},
   {OPD(0xDD, 0xC8), 8, &OpDispatchBuilder::FXCH},
@@ -545,7 +545,7 @@ constexpr std::array<DispatchTableEntry, 140> X87F80OpTable = {{
   {OPD(0xDF, 0xD0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
   {OPD(0xDF, 0xD8), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::FSTToStack>},
 
-  {OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::X87FNSTSW},
+  {OPD(0xDF, 0xE0), 8, &OpDispatchBuilder::Bind<&OpDispatchBuilder::X87FNSTSW, true>},
   {OPD(0xDF, 0xE8), 8,
    &OpDispatchBuilder::Bind<&OpDispatchBuilder::FCOMI, OpSize::f80Bit, false, OpDispatchBuilder::FCOMIFlags::FLAGS_RFLAGS, false>},
   {OPD(0xDF, 0xF0), 8,
@@ -794,7 +794,7 @@ auto GenerateX87TableLambda = [](const auto DispatchTable) consteval {
       //  / 3
       {OPD(0xDF, 0xD8), 8, X86InstInfo{"FSTP",    TYPE_X87, FLAGS_SF_MOD_DST | FLAGS_POP, 0}},
       //  / 4
-      {OPD(0xDF, 0xE0), 1, X86InstInfo{"FNSTSW",  TYPE_INST, GenFlagsSameSize(SIZE_16BIT) | FLAGS_SF_DST_RAX, 0}},
+      {OPD(0xDF, 0xE0), 1, X86InstInfo{"FNSTSW",  TYPE_INST, GenFlagsSameSize(SIZE_16BIT), 0}},
       {OPD(0xDF, 0xE1), 7, X86InstInfo{"",        TYPE_INVALID, FLAGS_NONE, 0}},
       //  / 5
       {OPD(0xDF, 0xE8), 8, X86InstInfo{"FUCOMIP", TYPE_INST,    FLAGS_POP, 0}},

FEXCore/Source/Interface/IR/IR.json

@@ -201,7 +201,7 @@
         "DestSize": "OpSize::i64Bit"
       },
 
-      "ThreadRemoveCodeEntry GPR:$Entry": {
+      "ThreadRemoveCodeEntry GPR:$EntryToInvalidate, GPR:$NewRIP": {
         "HasSideEffects": true
       },
 
@@ -275,7 +275,8 @@
                  "The boolean argument asks if we should be reading the reseeded number or not",
                  "Reseeded RNG calculation is more expensive and will be heavier to use",
                  "Returns the 64-bit number",
-                 "Sets the Z flag if the number is valid.",
+                 "Falls back to a host RNG call when the hardware doesn't support it",
+                 "Sets the Z flag if the number is invalid.",
                  "RNG hardware is allowed to fail early and return. Software must always check this"
                 ],
         "HasSideEffects": true,
@@ -320,7 +321,7 @@
         "DestSize": "Size",
         "RAOverride": "2"
       },
-      "Break BreakDefinition:$Reason": {
+      "Break BreakDefinition:$Reason, GPR:$FaultAddress": {
         "HasSideEffects": true
       },
       "CallbackReturn": {
@@ -336,9 +337,8 @@
         "HasSideEffects": true
       },
 
-      "GPR:$EAX, GPR:$EBX, GPR:$ECX, GPR:$EDX = CPUID GPR:$Function, GPR:$Leaf": {
+      "CPUID i8:$InstSize": {
         "Desc": ["Calls in to the CPUID handler function to return emulated CPUID"],
-        "DestSize": "OpSize::i32Bit",
         "HasSideEffects": true
       },
       "GPR:$EAX, GPR:$EDX = XGetBV GPR:$Function": {
@@ -529,6 +529,7 @@
       },
 
       "SSA = LoadMem RegisterClass:$Class, OpSize:#Size, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale": {
+        "HasSideEffects": true,
         "Inline": ["", "Mem"],
         "DestSize": "Size"
       },
@@ -574,6 +575,7 @@
       },
 
       "SSA = LoadMemTSO RegisterClass:$Class, OpSize:#Size, GPR:$Addr, GPR:$Offset, OpSize:$Align, MemOffsetType:$OffsetType, u8:$OffsetScale": {
+        "HasSideEffects": true,
         "Desc": ["Does a x86 TSO compatible load from memory. Offset must be Invalid()."
                 ],
         "Inline": ["", "Memtso"],
@@ -589,6 +591,7 @@
       },
 
       "FPR = VLoadVectorMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Mask, GPR:$Addr, GPR:$Offset, MemOffsetType:$OffsetType, u8:$OffsetScale": {
+        "HasSideEffects": true,
         "Desc": ["Does a masked load similar to VPMASKMOV/VMASKMOV where the upper bit of each element",
                  "determines whether or not that element will be loaded from memory"],
         "ImplicitFlagClobber": true,
@@ -603,11 +606,13 @@
         "DestSize": "RegisterSize",
         "ElementSize": "ElementSize"
       },
-      "FPR = VLoadVectorGatherMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$Mask, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, OpSize:$VectorIndexElementSize, u8:$OffsetScale, u8:$DataElementOffsetStart, u8:$IndexElementOffsetStart, OpSize:$AddrSize": {
+      "FPR = VLoadVectorGatherMasked OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$Mask, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, OpSize:$VectorIndexElementSize, u8:$OffsetScale, u8:$DataElementOffsetStart, u8:$IndexElementOffsetStart, OpSize:$AddrSize, i1:$MaskAllOnes{false}": {
+        "HasSideEffects": true,
         "Desc": [
           "Does a masked load similar to VPGATHERD* where the upper bit of each element",
           "determines whether or not that element will be loaded from memory.",
-          "Most of VSIB encoding is passed directly through to the IR operation."
+          "Most of VSIB encoding is passed directly through to the IR operation.",
+          "MaskAllOnes is set when every mask element is provably all ones."
         ],
         "TiedSource": 0,
         "ImplicitFlagClobber": true,
@@ -617,12 +622,14 @@
           "$VectorIndexElementSize == OpSize::i32Bit || $VectorIndexElementSize == OpSize::i64Bit"
         ]
       },
-      "FPR = VLoadVectorGatherMaskedQPS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$MaskReg, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, u8:$OffsetScale, OpSize:$AddrSize": {
+      "FPR = VLoadVectorGatherMaskedQPS OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Incoming, FPR:$MaskReg, GPR:$AddrBase, FPR:$VectorIndexLow, FPR:$VectorIndexHigh, u8:$OffsetScale, OpSize:$AddrSize, i1:$MaskAllOnes{false}": {
+        "HasSideEffects": true,
         "Desc": [
           "Does a masked load similar to VPGATHERQPS where the upper bit of each element",
           "determines whether or not that element will be loaded from memory.",
           "Most of VSIB encoding is passed directly through to the IR operation.",
-          "Only supports the case of 32-bit data element sizes from 64-bit addresses"
+          "Only supports the case of 32-bit data element sizes from 64-bit addresses",
+          "MaskAllOnes is set when every mask element is provably all ones."
         ],
         "TiedSource": 0,
         "ImplicitFlagClobber": true,
@@ -634,6 +641,7 @@
         ]
       },
       "FPR = VLoadVectorElement OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$DstSrc, u8:$Index, GPR:$Addr": {
+        "HasSideEffects": true,
         "Desc": ["Does a memory load to a single element of a vector.",
                  "Leaves the rest of the vector's data intact.",
                  "Matches arm64 ld1 semantics"],
@@ -773,6 +781,7 @@
         ]
       },
       "FPR = VLoadNonTemporal OpSize:#RegisterSize, GPR:$Addr, i8:$Offset": {
+        "HasSideEffects": true,
         "Desc": ["Does a non-temporal memory load of a vector.",
                  "Matches arm64 SVE ldnt1b semantics.",
                  "Specifically weak-memory model ordered to match x86 non-temporal stores."
@@ -960,6 +969,20 @@
         "DestSize": "Size"
       },
 
+      "GPR = PatchableGuestRIP OpSize:#Size, i64:$Value, i64:$SiteAddress, i64:$SiteSize": {
+        "Desc": ["Loads GuestRIP-relative Value in a patchable way",
+                 "On disk cache load the value is patched from live guest PC and live displacement at SiteAddress"
+                ],
+        "DestSize": "Size"
+      },
+
+      "GPR = PatchableGuestCRC OpSize:#Size, i64:$Value, i64:$GuestAddress, i64:$GuestSize": {
+        "Desc": ["Loads Guest CRC in a patchable way",
+                 "On disk cache load the value is recomputed from live guest bytes and patched"
+                ],
+        "DestSize": "Size"
+      },
+
       "GPR = Constant i64:$Constant, ConstPad:$Pad{IR::ConstPad::NoPad}, i32:$MaxBytes{0}": {
         "Desc": ["Generates a 64bit constant inside of a GPR",
                  "Unsupported to create a constant in FPR"
@@ -1846,7 +1869,7 @@
         "DestSize": "RegisterSize"
       },
 
-      "FPR = VectorImm OpSize:#RegisterSize, OpSize:#ElementSize, u8:$Immediate, u8:$ShiftAmount{0}": {
+      "FPR = VectorImm OpSize:#RegisterSize, OpSize:#ElementSize, u8:$Immediate, u8:$ShiftAmount{0}, i1:$Do64BitExpansion{false}": {
         "Desc": ["Generates a vector with each element containg the immediate zexted"
                 ],
         "DestSize": "RegisterSize",
@@ -2363,6 +2386,43 @@
         "DestSize": "RegisterSize",
         "ElementSize": "ElementSize << 1"
       },
+      "FPR = VUSDot OpSize:#RegisterSize, FPR:$Acc, FPR:$Vector1, FPR:$Vector2": {
+        "Desc": ["Unsigned by signed 8-bit dot product, accumulating four products in to each 32-bit element of Acc.",
+                 "Requires FEAT_I8MM."
+                ],
+        "DestSize": "RegisterSize",
+        "ElementSize": "OpSize::i32Bit",
+        "TiedSource": 0,
+        "EmitValidation": [
+          "RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i256Bit"
+        ]
+      },
+      "FPR = VSDot OpSize:#RegisterSize, FPR:$Acc, FPR:$Vector1, FPR:$Vector2": {
+        "Desc": ["Signed 8-bit dot product, accumulating four products in to each 32-bit element of Acc.",
+                 "Requires FEAT_DotProd."
+                ],
+        "DestSize": "RegisterSize",
+        "ElementSize": "OpSize::i32Bit",
+        "TiedSource": 0,
+        "EmitValidation": [
+          "RegisterSize == FEXCore::IR::OpSize::i128Bit || RegisterSize == FEXCore::IR::OpSize::i256Bit"
+        ]
+      },
+      "FPR = VSAddLP OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector": {
+        "Desc": ["Signed add long pairwise. Adds adjacent pairs of elements in to elements of twice the size.",
+                 "ElementSize is the source size"
+                ],
+        "DestSize": "RegisterSize",
+        "ElementSize": "ElementSize << 1"
+      },
+      "FPR = VSAdALP OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Acc, FPR:$Vector": {
+        "Desc": ["Signed add and accumulate long pairwise. Adds adjacent pairs of elements in to the elements of Acc, which are twice the size.",
+                 "ElementSize is the source size"
+                ],
+        "DestSize": "RegisterSize",
+        "ElementSize": "ElementSize << 1",
+        "TiedSource": 0
+      },
       "FPR = VUMulH OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2": {
         "Desc": ["Wide unsigned multiply returning the high results"],
         "DestSize": "RegisterSize",
@@ -2466,6 +2526,14 @@
         "DestSize": "RegisterSize",
         "ElementSize": "ElementSize"
       },
+      "FPR = VUCMPGT OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2": {
+        "Desc": ["Vector compare unsigned greater than",
+                 "Each element is compared, if the result is true then the resulting element is ~0, else zero"
+                ],
+
+        "DestSize": "RegisterSize",
+        "ElementSize": "ElementSize"
+      },
       "FPR = VFCMPEQ OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector1, FPR:$Vector2": {
         "DestSize": "RegisterSize",
         "ElementSize": "ElementSize"
@@ -2533,7 +2601,8 @@
       },
 
       "GPR = VPCMPESTRX FPR:$LHS, FPR:$RHS, GPR:$RAX, GPR:$RDX, u16:$Control": {
-        "Desc": ["Performs intermediate behavior analogous to the x86 PCMPESTRI/PCMPESTRM instruction",
+        "Desc": ["NOTE: Currently unused. The OpcodeDispatcher implements the SSE4.2 string instructions inline.",
+                 "Performs intermediate behavior analogous to the x86 PCMPESTRI/PCMPESTRM instruction",
                  "This will return the intermediate result of a PCMPESTR-type operation, but NOT the final",
                  "result. This must be derived from the intermediate result",
 
@@ -2545,7 +2614,8 @@
         "JITDispatch": false
       },
       "GPR = VPCMPISTRX FPR:$LHS, FPR:$RHS, u8:$Control": {
-        "Desc": ["Performs intermediate behavior analogous to the x86 PCMPISTRI/PCMPISTRM instruction",
+        "Desc": ["NOTE: Currently unused. The OpcodeDispatcher implements the SSE4.2 string instructions inline.",
+                 "Performs intermediate behavior analogous to the x86 PCMPISTRI/PCMPISTRM instruction",
                  "This will return the intermediate result of a PCMPISTR-type operation, but NOT the final",
                  "result. This must be derived from the intermediate result",
 
@@ -2697,6 +2767,28 @@
           "RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
         ]
       },
+      "FPR = VBFCVTN OpSize:#RegisterSize, FPR:$Vector": {
+        "Desc": [
+          "Converts fp32 to bf16."
+        ],
+        "DestSize": "RegisterSize",
+        "ElementSize": "FEXCore::IR::OpSize::i16Bit",
+        "EmitValidation": [
+          "RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
+        ]
+      },
+      "FPR = VBFCVTN2 OpSize:#RegisterSize, FPR:$VectorLower, FPR:$VectorUpper": {
+        "TiedSource": 0,
+        "Desc": [
+          "Converts fp32 to bf16 and inserting in to the high bits.",
+          "Bottom half is untouched"
+        ],
+        "DestSize": "RegisterSize",
+        "ElementSize": "FEXCore::IR::OpSize::i16Bit",
+        "EmitValidation": [
+          "RegisterSize != FEXCore::IR::OpSize::i256Bit && \"What does 256-bit mean in this context?\""
+        ]
+      },
       "FPR = Vector_FToI OpSize:#RegisterSize, OpSize:#ElementSize, FPR:$Vector, RoundType:$Round": {
         "Desc": ["Vector op: Rounds float to integral",
                  "Rounding mode determined by argument"

FEXCore/Source/Interface/IR/IRDumper.cpp

@@ -172,6 +172,8 @@ static void PrintArg(fextl::ostringstream* out, const IRListView*, NamedVectorCo
         return "u16_incremental_index";
       case NamedVectorConstant::NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER:
         return "u16_incremental_index_upper";
+      case NamedVectorConstant::NAMED_VECTOR_INCREMENTAL_U8_INDEX:
+        return "u8_incremental_index";
       case NamedVectorConstant::NAMED_VECTOR_PADDSUBPS_INVERT:
         return "addsubps_invert";
       case NamedVectorConstant::NAMED_VECTOR_PADDSUBPS_INVERT_UPPER:

FEXCore/Source/Interface/IR/IREmitter.cpp

@@ -19,6 +19,9 @@ namespace FEXCore::IR {
 
 static bool IsFragmentExit(FEXCore::IR::IROps Op) {
   switch (Op) {
+  case OP_THREADREMOVECODEENTRY:
+  case OP_SYSCALL:
+  case OP_CPUID:
   case OP_EXITFUNCTION:
   case OP_BREAK: return true;
   default: return false;

FEXCore/Source/Interface/IR/IREmitter.h

@@ -341,6 +341,27 @@ public:
     return false;
   }
 
+  // Returns true if every bit of the vector value is set.
+  // Currently this only recognizes the self compare equal idiom.
+  // TODO: Should this recognize other patterns?
+  bool IsValueVectorAllOnes(OrderedNodeWrapper ssa) const {
+    auto RealNode = ssa.GetNode(DualListData.ListBegin());
+    const auto* IROp = RealNode->Op(DualListData.DataBegin());
+    return IROp->Op == OP_VCMPEQ && IROp->Args[0].ID() == IROp->Args[1].ID();
+  }
+
+  // Returns true if the value is a load of register Reg of class Class.
+  bool IsValueLoadRegister(OrderedNodeWrapper ssa, RegClass Class, uint32_t Reg) const {
+    auto RealNode = ssa.GetNode(DualListData.ListBegin());
+    const auto* IROp = RealNode->Op(DualListData.DataBegin());
+    if (IROp->Op != OP_LOADREGISTER) {
+      return false;
+    }
+
+    auto Op = IROp->C<IR::IROp_LoadRegister>();
+    return Op->Class == Class && Op->Reg == Reg;
+  }
+
   FEXCore::IR::IROp_Header* GetOpHeader(OrderedNodeWrapper ssa) {
     Ref RealNode = ssa.GetNode(DualListData.ListBegin());
     return RealNode->Op(DualListData.DataBegin());
@@ -354,6 +375,15 @@ public:
     return node->Wrapped(DualListData.ListBegin());
   }
 
+  // Returns argument Index of Node if the op of Node is Op. Else returns nullptr.
+  Ref GetArgIfOp(Ref Node, IROps Op, unsigned Index) {
+    if (!Node || GetOpHeader(WrapNode(Node))->Op != Op) {
+      return nullptr;
+    }
+
+    return UnwrapNode(GetOpHeader(WrapNode(Node))->Args[Index]);
+  }
+
   NodeIterator GetIterator(OrderedNodeWrapper wrapper) {
     return NodeIterator(DualListData.ListBegin(), DualListData.DataBegin(), wrapper);
   }

FEXCore/Source/Interface/IR/Passes/RegisterAllocationPass.cpp

@@ -527,27 +527,6 @@ bool ConstrainedRAPass::TryPostRAMerge(Ref LastNode, Ref CodeNode, IROp_Header*
       IREmit->RemovePostRA(CodeNode);
       return false;
     }
-  } else if (IROp->Op == OP_CPUID && PhysicalRegister(IROp->Args[0]) == PhysicalRegister(LastNode) && LastOp->Op == OP_CONSTANT) {
-    // Try to constant fold. As a limitation of merging only 2 instructions, we
-    // can only handle constant functions, not constant leafs. This could be
-    // lifted if we generalized at a (significant) complexity cost.
-    uint64_t ConstantFunction = LastOp->C<IROp_Constant>()->Constant;
-    auto Op = IROp->CW<IR::IROp_CPUID>();
-
-    const auto SupportsConstant = CPUID->DoesFunctionReportConstantData(ConstantFunction);
-    if (SupportsConstant.SupportsConstantFunction == CPUIDEmu::SupportsConstant::CONSTANT &&
-        SupportsConstant.NeedsLeaf != CPUIDEmu::NeedsLeafConstant::NEEDSLEAFCONSTANT) {
-      const auto Result = CPUID->RunFunction(ConstantFunction, 0 /* leaf */);
-
-      IREmit->SetWriteCursorBefore(CodeNode);
-      IREmit->_Fence(IR::FenceType::Inst);
-      IREmit->_Constant(Result.eax).Node->Reg = PhysicalRegister(Op->OutEAX).Raw;
-      IREmit->_Constant(Result.ebx).Node->Reg = PhysicalRegister(Op->OutEBX).Raw;
-      IREmit->_Constant(Result.ecx).Node->Reg = PhysicalRegister(Op->OutECX).Raw;
-      IREmit->_Constant(Result.edx).Node->Reg = PhysicalRegister(Op->OutEDX).Raw;
-      IREmit->RemovePostRA(CodeNode);
-      return false;
-    }
   }
 
   // Merge moves that are immediately consumed.

FEXCore/Source/Utils/Allocator.cpp

@@ -23,7 +23,9 @@
 #include <fcntl.h>
 #ifndef _WIN32
 #include <sys/mman.h>
+#include <sys/resource.h>
 #include <sys/user.h>
+#include <unistd.h>
 #endif
 
 namespace fextl::pmr {
@@ -214,11 +216,62 @@ fextl::vector<MemoryRegion> CollectMemoryGaps(uintptr_t Begin, uintptr_t End, in
   FEX_UNREACHABLE;
 }
 
+// Tracks the region below the main thread stack, which is reserved for stack growth,
+// as well as the guard region directly below the stack window.
+static uintptr_t MainThreadStackReserveBegin {};
+static uintptr_t MainThreadStackWindowBegin {};
+static uintptr_t MainThreadStackWindowEnd {};
+static size_t MainThreadStackGuardSize {};
+static uintptr_t MainThreadStackGuardBegin {};
+
+// Currently, guest signal handlers leak space on the main stack
+// if they exit via longjmp instead of sigreturn. Because of this memory leak
+// we must allow the main thread stack to grow without bound. This technically
+// violates RLIMIT_STACK, but until we fix the leak, we don't have much choice.
+// TODO: Once the leak is fixed, remove this function entirely.
+bool TryGrowMainThreadStack(uintptr_t FaultAddress) {
+  // Only grow the stack when the fault is in the guard region.
+  if (FaultAddress < MainThreadStackGuardBegin || FaultAddress >= MainThreadStackWindowBegin) {
+    return false;
+  }
+
+  // Compute the required size of the stack region.
+  // Double the current stack, up to 1GB, then just add 1GB at a time.
+  constexpr size_t OneGB = 1ULL << 30;
+  const size_t RequiredSize = MainThreadStackWindowEnd - FaultAddress;
+  size_t NewSize = MainThreadStackWindowEnd - MainThreadStackWindowBegin;
+
+  while (NewSize < RequiredSize) {
+    if (NewSize < OneGB) {
+      NewSize = std::min(NewSize * 2, OneGB);
+    } else {
+      NewSize += OneGB;
+    }
+  }
+
+  // Don't grow past the bottom of the reserved region,
+  // which is where the first dynamically loaded libraries live.
+  const uintptr_t NewWindowBegin = std::max(MainThreadStackWindowEnd - NewSize, MainThreadStackReserveBegin);
+  if (mprotect(reinterpret_cast<void*>(NewWindowBegin), MainThreadStackWindowBegin - NewWindowBegin, PROT_READ | PROT_WRITE) != 0) {
+    return false;
+  }
+
+  // Move the window and the guard region below it down.
+  MainThreadStackWindowBegin = NewWindowBegin;
+  MainThreadStackGuardBegin = std::max(NewWindowBegin - MainThreadStackGuardSize, MainThreadStackReserveBegin);
+  return true;
+}
+
 fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
   const uintptr_t StackLocation_u64 = reinterpret_cast<uintptr_t>(alloca(0));
 
   const int MapsFD = open("/proc/self/maps", O_RDONLY);
-  LogMan::Throw::AFmt(MapsFD != -1, "Failed to open /proc/self/maps");
+  if (MapsFD == -1) {
+    // No procfs, as in BuildKit's emulator probe (an empty chroot). The assert
+    // above it compiles out in release, and CollectMemoryGaps then spins on
+    // read(-1) forever; reserve nothing instead.
+    return {};
+  }
 
   auto Regions = CollectMemoryGaps(Begin, End, MapsFD);
   close(MapsFD);
@@ -226,7 +279,7 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
   // If the memory bounds include the stack, blocking all memory regions will
   // limit the stack size to the current value. To allow some stack growth,
   // we don't block the memory gap directly below the stack memory but
-  // instead map it as readable+writable.
+  // instead reserve it for the stack, see TryGrowMainThreadStack.
   {
     auto StackRegionIt = std::find_if(Regions.begin(), Regions.end(), [StackLocation_u64](auto& Region) {
       return reinterpret_cast<uintptr_t>(Region.Ptr) + Region.Size > StackLocation_u64;
@@ -238,16 +291,40 @@ fextl::vector<MemoryRegion> StealMemoryRegion(uintptr_t Begin, uintptr_t End) {
 
     if (IsStackMapping && StackRegionIt != Regions.begin() &&
         reinterpret_cast<uintptr_t>(std::prev(StackRegionIt)->Ptr) + std::prev(StackRegionIt)->Size <= End) {
-      // Allocate the region under the stack as READ | WRITE so the stack can still grow
       --StackRegionIt;
 
-      auto Alloc =
-        ::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);
+      // Get the page size, so we can make sure to align on page boundaries.
+      const auto PageSize = sysconf(_SC_PAGESIZE);
+      const size_t HostPageSize = PageSize > 0 ? PageSize : FEXCore::Utils::FEX_PAGE_SIZE;
+
+      // Allocate RLIMIT_STACK worth of memory for the main thread stack, or 64MB if no limit is set.
+      struct rlimit StackLimit {};
+      size_t Size = 1ULL << 26;
+      if (getrlimit(RLIMIT_STACK, &StackLimit) == 0 && StackLimit.rlim_cur != RLIM_INFINITY) {
+        Size = std::max<size_t>(FEXCore::AlignUp(StackLimit.rlim_cur, HostPageSize), HostPageSize);
+      }
+      Size = std::min(Size, StackRegionIt->Size);
+
+      // Reserve all of the memory between the top of the stack, and the start of the next region.
+      // We must map it with PROT_NONE, otherwise the MM will assume these pages are "in use"
+      // and charge this usage towards VmData. In practice this is multiple TBs of VA space.
+      auto Alloc = ::mmap(StackRegionIt->Ptr, StackRegionIt->Size, PROT_NONE, MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE | MAP_FIXED, -1, 0);
 
       LogMan::Throw::AFmt(Alloc != MAP_FAILED, "StealMemoryRegion:Stack: mmap({}, {:x}) failed: {}", fmt::ptr(StackRegionIt->Ptr),
                           StackRegionIt->Size, errno);
       LogMan::Throw::AFmt(Alloc == StackRegionIt->Ptr, "mmap returned {} instead of {}", Alloc, fmt::ptr(StackRegionIt->Ptr));
 
+      // Now determine the bounds for the stack region, and its guard region, which is used by TryGrowMainThreadStack().
+      MainThreadStackReserveBegin = reinterpret_cast<uintptr_t>(StackRegionIt->Ptr);
+      MainThreadStackWindowEnd = MainThreadStackReserveBegin + StackRegionIt->Size;
+      MainThreadStackWindowBegin = MainThreadStackWindowEnd - Size;
+      MainThreadStackGuardSize = 256 * HostPageSize;
+      MainThreadStackGuardBegin = std::max(MainThreadStackWindowBegin - MainThreadStackGuardSize, MainThreadStackReserveBegin);
+
+      // Protect the stack window.
+      const auto Result = mprotect(reinterpret_cast<void*>(MainThreadStackWindowBegin), Size, PROT_READ | PROT_WRITE);
+      LogMan::Throw::AFmt(Result == 0, "StealMemoryRegion:Stack: mprotect({:x}, {:x}) failed: {}", MainThreadStackWindowBegin, Size, errno);
+
       Regions.erase(StackRegionIt);
     }
   }
@@ -282,6 +359,9 @@ fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists(size_t PageSize) {
   uintptr_t Begin48BitVA = 0x0'8000'0000'0000ULL;
   uintptr_t End48BitVA = 0x1'0000'0000'0000ULL;
   auto Regions = StealMemoryRegion(Begin48BitVA, End48BitVA);
+  if (Regions.empty()) {
+    return {};
+  }
 
   Alloc64 = Alloc::OSAllocator::Create64BitAllocatorWithRegions(Regions);
   AssignHookOverrides(PageSize);

FEXCore/Source/Utils/AllocatorHooks.cpp

@@ -27,9 +27,13 @@ using vma_name_hook_type = void (*)(const char* name, const void* address, size_
 #ifdef ENABLE_FEX_ALLOCATOR
 typedef void* (*rp_mmap_hook_type)(size_t size, size_t alignment, size_t* offset, size_t* mapped_size);
 typedef void (*rp_munmap_hook_type)(void* address, size_t offset, size_t mapped_size);
+typedef void (*rp_mcommit_hook_type)(void* address, size_t size);
+typedef void (*rp_mdecommit_hook_type)(void* address, size_t size);
 typedef void (*vma_name_hook_type)(const char* name, const void* address, size_t size);
 extern "C" rp_mmap_hook_type rp_mmap_hook;
 extern "C" rp_munmap_hook_type rp_munmap_hook;
+extern "C" rp_mcommit_hook_type rp_mcommit_hook;
+extern "C" rp_mdecommit_hook_type rp_mdecommit_hook;
 extern "C" vma_name_hook_type rp_name_hook;
 
 #ifndef _WIN32
@@ -112,8 +116,11 @@ static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t*
     alignment = 0;
   }
 
+  // Protect with PROT_NONE so that the whole heap is not charged against VmData.
+  // In the memory commit callback, FEX_rp_memory_commit, we reprotect with
+  // PROT_READ | PROT_WRITE to commit memory on demand.
   size_t map_size = AlignUp(size + alignment, global_config.page_size);
-  auto ptr = fex_mmap_hook(0, map_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
+  auto ptr = fex_mmap_hook(0, map_size, PROT_NONE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 
   if (ptr == MAP_FAILED) {
     ptr = nullptr;
@@ -149,7 +156,9 @@ static void* FEX_rp_mmap(size_t size, size_t alignment, size_t* offset, size_t*
 }
 
 static void FEX_rp_memory_commit(void* address, size_t size) {
-  // NOP-implementation.
+  if (mprotect(address, size, PROT_READ | PROT_WRITE)) {
+    fprintf(stderr, "Failed to commit VMA region.");
+  }
 }
 
 static void FEX_rp_memory_decommit(void* address, size_t size) {
@@ -160,6 +169,11 @@ static void FEX_rp_memory_decommit(void* address, size_t size) {
   if (madvise(address, size, MADV_DONTNEED)) {
     fprintf(stderr, "Failed to decommit VMA region.");
   }
+
+  // Drop the protections back to PROT_NONE so the pages are no longer charged towards VmData.
+  if (mprotect(address, size, PROT_NONE)) {
+    fprintf(stderr, "Failed to decommit VMA region.");
+  }
 }
 
 static void FEX_rp_memory_unmap(void* address, size_t offset, size_t mapped_size) {
@@ -190,6 +204,8 @@ void InitializeAllocator(size_t PageSize) {
   rpmalloc_initialize_config(&global_interface, &global_config);
   rp_mmap_hook = FEX_rp_mmap;
   rp_munmap_hook = FEX_rp_memory_unmap;
+  rp_mcommit_hook = FEX_rp_memory_commit;
+  rp_mdecommit_hook = FEX_rp_memory_decommit;
   rp_name_hook = LocalVirtualName;
 }
 #else

FEXCore/Source/Utils/FileUtils.cpp

@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: MIT
+#include <FEXCore/fextl/fmt.h>
+#include <FEXCore/fextl/functional.h>
+#include <FEXCore/fextl/string.h>
+#include <FEXCore/Utils/CompilerDefs.h>
+#include <FEXCore/Utils/LogManager.h>
+
+#ifndef _WIN32
+#include <dirent.h>
+#include <fcntl.h>
+#include <unistd.h>
+#else
+#include <windows.h>
+#include <winnt.h>
+#include <winternl.h>
+#include <ntstatus.h>
+
+// FEX today doesn't have Windows include in FEXCore.
+extern "C" NTSTATUS WINAPI NtQueryDirectoryFile(HANDLE, HANDLE, PIO_APC_ROUTINE, PVOID, PIO_STATUS_BLOCK, PVOID, ULONG,
+                                                FILE_INFORMATION_CLASS, BOOLEAN, PUNICODE_STRING, BOOLEAN);
+
+extern "C" NTSTATUS RtlUnicodeToUTF8N(OUT PCHAR UTF8StringDestination, IN ULONG UTF8StringMaxByteCount,
+                                      OUT PULONG UTF8StringActualByteCount, IN PCWCH UnicodeStringSource, IN ULONG UnicodeStringByteCount);
+#endif
+
+namespace FEXCore::FileUtils {
+#ifndef _WIN32
+static inline bool unlinkat(int fd, const char* path, bool dir) {
+  if (::unlinkat(fd, path, dir ? AT_REMOVEDIR : 0) == -1) {
+    return errno == ENOENT;
+  }
+  return true;
+}
+
+static bool RecursiveRemoveDirectory(int parent_fd, const char* Directory) {
+  // Don't follow symlinks and ensure it closes on exec.
+  constexpr int DIR_FLAGS = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC;
+  int dir_fd = ::openat(parent_fd, Directory, DIR_FLAGS);
+
+  if (dir_fd == -1) {
+    if (errno == ENOENT) {
+      // Probably raced something. Non-error.
+      return true;
+    }
+  }
+
+  // Walk the directory listing.
+  bool Result = true;
+  // Four pages arbitrary chosen to be a balance between NFS wanting to return data in page-size granules and
+  // local filesystems returning arbitrary sizes.
+  size_t dirent_size = 4096 * 4;
+  uint8_t* dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
+
+  if (!dirent_buffer) {
+    // Ran out of memory?
+    Result = false;
+    goto end;
+  }
+
+  while (true) {
+    ssize_t read = getdents64(dir_fd, dirent_buffer, dirent_size);
+
+    if (read == -1) {
+      if (errno == EINVAL) {
+        // Buffer too small? Scale and try again.
+        dirent_size *= 2;
+        FEXCore::Allocator::free(dirent_buffer);
+        dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
+        if (!dirent_buffer) {
+          // Ran out of memory?
+          Result = false;
+          goto end;
+        }
+
+        continue;
+      }
+
+      // Anything else just exit.
+      Result = false;
+      goto end;
+    }
+
+    if (read == 0) {
+      // Done.
+      break;
+    }
+
+    for (size_t dirent_offset = 0; dirent_offset < read;) {
+      auto path_dirent = reinterpret_cast<const struct dirent*>(dirent_buffer + dirent_offset);
+      std::string_view path_name_view = path_dirent->d_name;
+
+      if (path_name_view == "." || path_name_view == "..") {
+        // Skip these two special files.
+        dirent_offset += path_dirent->d_reclen;
+        continue;
+      }
+
+      if (path_dirent->d_type == DT_DIR) {
+        // Recurse directories as we find them and remove them.
+        if (!RecursiveRemoveDirectory(dir_fd, path_dirent->d_name)) {
+          Result = false;
+          goto end;
+        }
+      }
+
+      // Remove anything possible.
+      if (!unlinkat(dir_fd, path_dirent->d_name, path_dirent->d_type == DT_DIR)) {
+        // Couldn't unlink the file for some reason.
+        LogMan::Msg::IFmt("Failed to remove file: {}", path_name_view);
+        Result = false;
+        goto end;
+      }
+
+      // dirent is a VLA so we need to increment by reported size.
+      dirent_offset += path_dirent->d_reclen;
+    }
+  }
+
+end:
+  if (dirent_buffer) {
+    FEXCore::Allocator::free(dirent_buffer);
+  }
+
+  close(dir_fd);
+  return Result;
+}
+
+FEX_DEFAULT_VISIBILITY bool RecursiveRemoveDirectory(const fextl::string& Directory) {
+  return RecursiveRemoveDirectory(AT_FDCWD, Directory.c_str()) && unlinkat(AT_FDCWD, Directory.c_str(), true);
+}
+
+FEX_DEFAULT_VISIBILITY void WalkDirectory(std::string_view Directory,
+                                          fextl::move_only_function<void(std::string_view name, bool is_dir, const void* user_data)> Callback,
+                                          const void* user_data) {
+  constexpr int DIR_FLAGS = O_RDONLY | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC;
+  int dir_fd = ::openat(AT_FDCWD, fextl::string(Directory).c_str(), DIR_FLAGS);
+  if (dir_fd == -1) {
+    if (errno == ENOENT) {
+      // Probably raced something. Non-error.
+      return;
+    }
+  }
+  // Four pages arbitrary chosen to be a balance between NFS wanting to return data in page-size granules and
+  // local filesystems returning arbitrary sizes.
+  size_t dirent_size = 4096 * 4;
+  uint8_t* dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
+
+  if (!dirent_buffer) {
+    // Ran out of memory?
+    goto end;
+  }
+
+  while (true) {
+    ssize_t read = getdents64(dir_fd, dirent_buffer, dirent_size);
+
+    if (read == -1) {
+      if (errno == EINVAL) {
+        // Buffer too small? Scale and try again.
+        dirent_size *= 2;
+        FEXCore::Allocator::free(dirent_buffer);
+        dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
+        if (!dirent_buffer) {
+          // Ran out of memory?
+          goto end;
+        }
+
+        continue;
+      }
+
+      // Anything else just exit.
+      goto end;
+    }
+
+    if (read == 0) {
+      // Done.
+      break;
+    }
+
+    for (size_t dirent_offset = 0; dirent_offset < read;) {
+      auto path_dirent = reinterpret_cast<const struct dirent*>(dirent_buffer + dirent_offset);
+      std::string_view path_name_view = path_dirent->d_name;
+
+      if (path_name_view == "." || path_name_view == "..") {
+        // Skip these two special files.
+        dirent_offset += path_dirent->d_reclen;
+        continue;
+      }
+
+      Callback(path_name_view, path_dirent->d_type == DT_DIR, user_data);
+
+      // dirent is a VLA so we need to increment by reported size.
+      dirent_offset += path_dirent->d_reclen;
+    }
+  }
+
+end:
+  if (dirent_buffer) {
+    FEXCore::Allocator::free(dirent_buffer);
+  }
+
+  close(dir_fd);
+}
+
+#else
+static inline std::optional<UNICODE_STRING> PathToNTPath(std::string_view Path) {
+  UNICODE_STRING PathW;
+  if (!RtlCreateUnicodeStringFromAsciiz(&PathW, fextl::string(Path).c_str())) {
+    return std::nullopt;
+  }
+  UNICODE_STRING NTPath;
+  bool Success = RtlDosPathNameToNtPathName_U(PathW.Buffer, &NTPath, nullptr, nullptr);
+  RtlFreeUnicodeString(&PathW);
+  if (!Success) {
+    return std::nullopt;
+  }
+
+  return NTPath;
+}
+
+static inline void FreeNTPath(UNICODE_STRING Path) {
+  RtlFreeUnicodeString(&Path);
+}
+
+FEX_DEFAULT_VISIBILITY void WalkDirectory(std::string_view Directory,
+                                          fextl::move_only_function<void(std::string_view name, bool is_dir, const void* user_data)> Callback,
+                                          const void* user_data) {
+  auto NTPath = PathToNTPath(Directory);
+  if (!NTPath) {
+    return;
+  }
+
+  NTSTATUS Status {};
+
+  HANDLE dir_fd {};
+  OBJECT_ATTRIBUTES attr {};
+  IO_STATUS_BLOCK io {};
+
+  InitializeObjectAttributes(&attr, &*NTPath, OBJ_CASE_INSENSITIVE, nullptr, nullptr);
+  Status = NtOpenFile(&dir_fd, FILE_LIST_DIRECTORY | SYNCHRONIZE, &attr, &io, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,
+                      FILE_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT);
+
+  FreeNTPath(*NTPath);
+
+  if (!NT_SUCCESS(Status)) {
+    return;
+  }
+
+  BOOLEAN FirstQuery = TRUE;
+  size_t dirent_size = 4096 * 4;
+  uint8_t* dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
+
+  if (!dirent_buffer) {
+    // Ran out of memory?
+    goto end;
+  }
+
+  while (true) {
+    Status = NtQueryDirectoryFile(dir_fd,
+                                  nullptr, // Event
+                                  nullptr, // ApcRoutine
+                                  nullptr, // ApcContext
+                                  &io, dirent_buffer, dirent_size, FileDirectoryInformation,
+                                  FALSE,   // ReturnSingleEntry
+                                  nullptr, // FileName
+                                  FirstQuery);
+    FirstQuery = FALSE;
+
+    if (Status == STATUS_NO_MORE_FILES) {
+      // No more files
+      break;
+    }
+
+    if (!NT_SUCCESS(Status)) {
+      if (Status == STATUS_BUFFER_TOO_SMALL || Status == STATUS_INFO_LENGTH_MISMATCH) {
+        // Buffer too small? Scale and try again.
+        dirent_size *= 2;
+        FEXCore::Allocator::free(dirent_buffer);
+        uint8_t* dirent_buffer = reinterpret_cast<uint8_t*>(FEXCore::Allocator::malloc(dirent_size));
+
+        if (!dirent_buffer) {
+          // Ran out of memory?
+          goto end;
+        }
+
+        continue;
+      }
+
+      // Any other failure, exit loop.
+      break;
+    }
+
+    // Iterate the entries returned.
+    for (size_t dirent_offset = 0;;) {
+      auto Info = reinterpret_cast<FILE_DIRECTORY_INFORMATION*>(dirent_buffer + dirent_offset);
+
+      std::wstring_view EntryName(Info->FileName, Info->FileNameLength / sizeof(wchar_t));
+
+      if (EntryName != L"." && EntryName != L"..") {
+        ULONG utf8_bytes_needed = 0;
+        ULONG actual_utf8_bytes = 0;
+        RtlUnicodeToUTF8N(nullptr, 0, &utf8_bytes_needed, Info->FileName, Info->FileNameLength);
+        char* dynamic_buf = reinterpret_cast<char*>(FEXCore::Allocator::malloc(utf8_bytes_needed + 1));
+        if (dynamic_buf) {
+          RtlUnicodeToUTF8N(dynamic_buf, utf8_bytes_needed + 1, &actual_utf8_bytes, Info->FileName, Info->FileNameLength);
+          std::string_view name_view(dynamic_buf, actual_utf8_bytes);
+          bool is_dir = (Info->FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0;
+          Callback(name_view, is_dir, user_data);
+          FEXCore::Allocator::free(dynamic_buf);
+        }
+      }
+
+      if (Info->NextEntryOffset == 0) {
+        break;
+      }
+
+      dirent_offset += Info->NextEntryOffset;
+    }
+  }
+
+end:
+  if (dirent_buffer) {
+    FEXCore::Allocator::free(dirent_buffer);
+  }
+
+  NtClose(dir_fd);
+}
+
+FEX_DEFAULT_VISIBILITY bool RecursiveRemoveDirectory(const fextl::string& Directory) {
+  using CallbackType = void (*)(std::string_view name, bool is_dir, const void* user_data);
+  struct UserData {
+    CallbackType RemoveFile {};
+    std::string_view base_path;
+  };
+
+  CallbackType RemoveFile = [](std::string_view name, bool is_dir, const void* user_data) {
+    auto Data = reinterpret_cast<const UserData*>(user_data);
+
+    auto full_path = std::format("{}/{}", Data->base_path, name);
+    if (is_dir) {
+      UserData NewData {
+        .RemoveFile = Data->RemoveFile,
+        .base_path = full_path,
+      };
+
+      WalkDirectory(full_path, Data->RemoveFile, &NewData);
+    }
+
+    if (is_dir) {
+      RemoveDirectoryA(full_path.c_str());
+    } else {
+      DeleteFileA(full_path.c_str());
+    }
+  };
+
+  UserData Data {
+    .RemoveFile = RemoveFile,
+    .base_path = Directory,
+  };
+
+  WalkDirectory(Directory, RemoveFile, &Data);
+
+  return RemoveDirectoryA(Directory.c_str()) != 0;
+}
+
+#endif
+} // namespace FEXCore::FileUtils

FEXCore/Source/Utils/Threads.cpp

@@ -7,7 +7,8 @@
 #include <unistd.h>
 
 namespace FEXCore::Threads {
-static fextl::unique_ptr<FEXCore::Threads::Thread> CreateThread_Default(ThreadFunc Func, void* Arg, FEXCore::Threads::Flags Flags) {
+static fextl::unique_ptr<FEXCore::Threads::Thread>
+CreateThread_Default(ThreadFunc Func, void* Arg, FEXCore::Threads::Flags Flags, const char* ThreadName) {
   ERROR_AND_DIE_FMT("Frontend didn't setup thread creation!");
 }
 
@@ -20,8 +21,9 @@ static FEXCore::Threads::Pointers Ptrs = {
   .CleanupAfterFork = CleanupAfterFork_Default,
 };
 
-fextl::unique_ptr<FEXCore::Threads::Thread> FEXCore::Threads::Thread::Create(ThreadFunc Func, void* Arg, FEXCore::Threads::Flags Flags) {
-  return Ptrs.CreateThread(Func, Arg, Flags);
+fextl::unique_ptr<FEXCore::Threads::Thread>
+FEXCore::Threads::Thread::Create(ThreadFunc Func, void* Arg, FEXCore::Threads::Flags Flags, const char* ThreadName) {
+  return Ptrs.CreateThread(Func, Arg, Flags, ThreadName);
 }
 
 void FEXCore::Threads::Thread::CleanupAfterFork() {

FEXCore/Source/Utils/WorkQueueThread.cpp

@@ -4,8 +4,8 @@
 
 namespace FEXCore {
 
-WorkQueueThread::WorkQueueThread(FEXCore::Threads::Flags ThreadFlags) {
-  Thread = FEXCore::Threads::Thread::Create(ThreadEntry, this, ThreadFlags);
+WorkQueueThread::WorkQueueThread(FEXCore::Threads::Flags ThreadFlags, const char* ThreadName) {
+  Thread = FEXCore::Threads::Thread::Create(ThreadEntry, this, ThreadFlags, ThreadName);
 }
 
 WorkQueueThread::~WorkQueueThread() {

FEXCore/Source/Utils/crc32.h

@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include <cstddef>
+#include <cstdint>
+#if defined(ARCHITECTURE_arm64)
+#include <arm_acle.h>
+#endif
+
+namespace FEXCore::Utils {
+template<typename T>
+static inline uint32_t crc32(const T* Ptr, size_t Size) {
+  uint32_t Result {};
+#if defined(ARCHITECTURE_arm64)
+#define do_crc(type, suffix)                                               \
+  while (Size >= sizeof(type)) {                                           \
+    Result = __crc32##suffix(Result, *reinterpret_cast<const type*>(Ptr)); \
+    Ptr += sizeof(type);                                                   \
+    Size -= sizeof(type);                                                  \
+  }
+  do_crc(uint64_t, d);
+  do_crc(uint32_t, w);
+  do_crc(uint16_t, h);
+  do_crc(uint8_t, b);
+#undef do_crc
+#else
+  // Emulate arm64 crc32.
+  // This is basically just the pseudo-code for crc32b.
+  // Doesn't need to be fast, just needs to match.
+  auto reverse_bits = [](auto bits) {
+    decltype(bits) Result {};
+    for (size_t i = 0; i < (sizeof(decltype(bits)) * 8); ++i) {
+      Result = (Result << 1) | ((bits >> i) & 1);
+    }
+    return Result;
+  };
+
+  auto Poly32Mod2 = [](uint64_t data) -> uint32_t {
+    constexpr static size_t bits = 40;
+    constexpr static uint64_t poly = 0x04C11DB7U;
+    for (size_t i = (bits - 1); i >= 32; --i) {
+      if (((data >> i) & 1) != 0) {
+        const uint64_t poly_shift = poly << (i - 32);
+        const uint64_t data_mask = (1ULL << i) - 1;
+        data = (data & data_mask) ^ poly_shift;
+      }
+    }
+
+    return data;
+  };
+
+  for (size_t i = 0; i < Size; ++i) {
+    uint64_t TempAcc = static_cast<uint64_t>(reverse_bits(Result)) << 8;
+    uint64_t TempVal = static_cast<uint64_t>(reverse_bits(reinterpret_cast<const uint8_t*>(Ptr)[i])) << 32;
+    Result = reverse_bits(Poly32Mod2(TempAcc ^ TempVal));
+  }
+#endif
+  return Result;
+}
+} // namespace FEXCore::Utils

FEXCore/include/FEXCore/Config/Config.h

@@ -297,6 +297,16 @@ struct FEX_DEFAULT_VISIBILITY Getter : public Value<typename detail::ConfigOptio
   using OptionInfo = detail::ConfigOptionInfo<Option>;
   Getter()
     : Value<typename OptionInfo::Type> {Option, OptionInfo::Default()} {}
+
+  // Check if the config option is actually the default.
+  bool IsDefault() const {
+    auto Value = FEXCore::Config::GetConv<typename detail::ConfigOptionInfo<Option>::Type>(Option);
+    if (!Value) {
+      return true;
+    }
+
+    return *Value == OptionInfo::Default();
+  }
 };
 
 /**

FEXCore/include/FEXCore/Core/CodeCache.h

@@ -51,6 +51,7 @@ struct ExecutableFileInfo {
   uint64_t FileId = 0;
   fextl::string Filename;
   fextl::robin_map<uint32_t, GuestRelocationType> Relocations;
+  uint64_t MappedSize = 0;
 };
 
 // Information associated with a specific section of an executable file
@@ -234,14 +235,6 @@ class AbstractCodeCache {
 public:
   virtual ~AbstractCodeCache() = default;
 
-  /**
-   * Computes a unique identifier for the referenced binary file to be used for
-   * generating the code map.
-   * This identifier is independent of FEX build/runtime configuration and
-   * stable across FEX updates.
-   */
-  virtual uint64_t ComputeCodeMapId(std::string_view Filename, int FD) = 0;
-
   /**
    * Bundles the current Core state (CodeBuffer, GuestToHostMapping, ...) to a code cache and writes it to the given file descriptor.
    * Returns true on success.

FEXCore/include/FEXCore/Core/CoreState.h

@@ -71,6 +71,12 @@ static_assert(std::is_standard_layout_v<NonAtomicRefCounter<uint64_t>>, "Needs t
 static_assert(std::is_trivially_copyable_v<NonAtomicRefCounter<uint64_t>>, "needs to be trivially copyable");
 static_assert(sizeof(NonAtomicRefCounter<uint64_t>) == sizeof(uint64_t), "Needs to be correct size");
 
+// Flags about the core state that the JIT can query.
+enum class CoreFlagsEnum : uint64_t {
+  CPUID_ENABLED = (1ULL << 0),
+};
+FEX_DEF_NUM_OPS(CoreFlagsEnum)
+
 struct alignas(64) CPUState {
   // Allows more efficient handling of the register
   // file in the event AVX is not supported.
@@ -107,7 +113,9 @@ struct alignas(64) CPUState {
   uint64_t L1Pointer {};
   uint64_t L1Mask {};
   uint64_t callret_sp {};
-  uint64_t _pad1 {};
+
+  // CPUID is default enabled.
+  CoreFlagsEnum CoreFlags {CoreFlagsEnum::CPUID_ENABLED};
 
   // Cacheline: 1,2,3,4
   // The high 128-bits of AVX registers when not being emulated by SVE256.
@@ -188,10 +196,41 @@ struct alignas(64) CPUState {
     GDT->Limit1 = Limit >> 16;
   }
 
+  static bool IsInstructionBPEnabled(unsigned BPIndex, uint64_t Dr7) {
+    LOGMAN_THROW_A_FMT(BPIndex < NUM_DEBUG_ADDRESSES, "Too large of an index");
+
+    // If the GE/LE flags and Transaction type is instruction then it is enabled.
+    uint64_t GELEEnabledMask = 0b11ULL << (2 * BPIndex);
+    uint64_t TransactionTypeMask = 0b11ULL << (16 + (4 * BPIndex));
+    return (Dr7 & GELEEnabledMask) && (Dr7 & TransactionTypeMask) == 0;
+  };
+
+  static size_t BreakpointSize(unsigned BPIndex, uint64_t Dr7) {
+    LOGMAN_THROW_A_FMT(BPIndex < NUM_DEBUG_ADDRESSES, "Too large of an index");
+
+    switch ((Dr7 >> (18 + (4 * BPIndex))) & 0b11) {
+    case 0: return 1;
+    case 1: return 2;
+    case 2: return 8;
+    case 3: return 4;
+    default: FEX_UNREACHABLE;
+    }
+  }
+
   uint16_t FCW {0x37F};
   uint8_t AbridgedFTW {};
 
   uint8_t _pad2[5];
+  uint64_t FIP {};
+
+  // Debug registers 0-3 are addresses.
+  static constexpr size_t NUM_DEBUG_ADDRESSES = 4;
+  uint64_t Dr[NUM_DEBUG_ADDRESSES] {};
+  // Debug register 6 is a status return register.
+  uint64_t Dr6 {};
+  // Debug register 7 is the control register.
+  uint64_t Dr7 {};
+
   // PF/AF are statically mapped as-if they were r16/r17 (which do not exist in
   // x86 otherwise). This allows a straightforward mapping for SRA.
   static constexpr uint8_t PF_AS_GREG = 16;
@@ -247,7 +286,6 @@ struct alignas(64) CPUState {
   // Follows encoding of the TI bit in segment selector encoding.
   constexpr static uint32_t SEGMENT_ARRAY_INDEX_GDT = 0;
   constexpr static uint32_t SEGMENT_ARRAY_INDEX_LDT = 1;
-  Core::CPUState::gdt_segment private_gdt[32] {};
 };
 static_assert(std::is_trivially_copyable_v<CPUState>, "Needs to be trivial");
 static_assert(std::is_standard_layout_v<CPUState>, "This needs to be standard layout");
@@ -348,12 +386,17 @@ struct JITPointers {
   uint64_t SyscallHandlerFunc {};
   uint64_t ExitFunctionLink {};
   uint64_t MonoBackpatcherWrite {};
-  uint64_t LUDIV {};
-  uint64_t LDIV {};
+  uint64_t RDRANDFallback {};
   uint64_t ThunkCallbackRet {};
 
+  /**
+   * @name Frontend handler pointers.
+   * @{ */
   // Handles returning/calling ARM64EC code from the JIT, expects the target PC in TMP3
   uint64_t ExitFunctionEC {};
+  // Handles a synchronous exception without faulting to jump in to the handler. Frontend specific.
+  uint64_t SynchronousExceptionHandler {};
+  /**  @} */
 
   FallbackABIInfo FallbackHandlerPointers[FallbackHandlerIndex::OPINDEX_MAX];
   uint64_t NamedVectorConstantPointers[FEXCore::IR::NamedVectorConstant::NAMED_VECTOR_CONST_POOL_MAX];
@@ -370,14 +413,16 @@ struct JITPointers {
   uint64_t ExitFunctionLinker {};
   uint64_t ThreadStopHandlerSpillSRA {};
   uint64_t ThreadPauseHandlerSpillSRA {};
+  uint64_t ThreadDispatchSyscallHandler {};
+  uint64_t ThreadDispatchCPUIDHandler {};
+  uint64_t ThreadDispatchRemoveCodeEntry {};
   uint64_t GuestSignal_SIGILL {};
   uint64_t GuestSignal_SIGTRAP {};
   uint64_t GuestSignal_SIGSEGV {};
+  uint64_t GuestSignal_SynchronousFault {};
   uint64_t SignalReturnHandler {};
   uint64_t SignalReturnHandlerRT {};
   uint64_t L2Pointer {};
-  uint64_t LUDIVHandler {};
-  uint64_t LDIVHandler {};
   uint64_t F64SinHandler {};
   uint64_t F64CosHandler {};
   uint64_t F64TanHandler {};
@@ -414,20 +459,22 @@ struct CpuStateFrame {
    */
   uint64_t InSyscallInfo {};
 
-  uint32_t SignalHandlerRefCounter {};
-
-  struct alignas(8) SynchronousFaultDataStruct {
+  struct SynchronousFaultDataStruct {
     bool FaultToTopAndGeneratedException {};
     uint8_t Signal;
     uint8_t TrapNo;
     uint8_t si_code;
     uint16_t err_code;
     uint16_t _pad : 16;
+    // Fault address can be different from the FaultRIP so need to distinguish.
+    uint64_t FaultAddress;
   } SynchronousFaultData;
 
+  uint32_t SignalHandlerRefCounter {};
+
   InternalThreadState* Thread;
 
-#ifdef ARCHITECTURE_arm64ec
+#ifdef _WIN32
   // Set by the kernel on ARM64EC whenever the JIT should cooperatively suspend running guest code.
   uint32_t SuspendDoorbell {};
 #endif
@@ -441,7 +488,7 @@ static_assert(offsetof(CpuStateFrame, Pointers) + sizeof(CpuStateFrame::Pointers
                                                                                             "than architecture maximum 32768");
 
 static_assert(std::is_standard_layout<CpuStateFrame>::value, "This needs to be standard layout");
-static_assert(sizeof(CpuStateFrame::SynchronousFaultData) == 8, "This needs to be 8 bytes");
+static_assert(sizeof(CpuStateFrame::SynchronousFaultData) == 16, "This needs to be 16 bytes");
 static_assert(alignof(CpuStateFrame::SynchronousFaultDataStruct) == 8, "This needs to be 8 bytes");
 static_assert(offsetof(CpuStateFrame, SynchronousFaultData) % 8 == 0, "This needs to be aligned");
 } // namespace FEXCore::Core

FEXCore/include/FEXCore/Core/DiskCache.h

@@ -1,231 +1,10 @@
 // SPDX-License-Identifier: MIT
 #pragma once
-#include "FEXCore/Core/CodeCache.h"
-#include "FEXCore/Core/Context.h"
-#include "Interface/Core/JIT/Relocations.h"
-#include "Interface/Core/Frontend.h"
-#include "Interface/Core/CPUBackend.h"
-#include "FEXCore/Config/Config.h"
-#include "FEXCore/Utils/File.h"
-#include "FEXCore/Utils/WorkQueueThread.h"
-#include "FEXCore/fextl/memory.h"
-#include <FEXCore/fextl/string.h>
-#include <FEXCore/fextl/unordered_set.h>
-#include <FEXCore/fextl/robin_map.h>
-#include <FEXCore/fextl/map.h>
-#include <FEXCore/fextl/vector.h>
-#include <stdint.h>
-#include <mutex>
-#include <optional>
-#include <span>
-#include <xxhash.h>
 
-namespace FEXCore {
+#include "FEXCore/Utils/CompilerDefs.h"
 
-namespace Context {
-  class ContextImpl;
-}
+#include <cstdint>
 
-namespace DiskCache {
-
-  namespace MesaFOZ {
-
-#define FOSSILIZE_BLOB_HASH_LENGTH 40 /* SHA1 hexadecimal string length */
-
-    struct __attribute__((packed)) foz_payload_key {
-      uint8_t bytes[FOSSILIZE_BLOB_HASH_LENGTH];
-    };
-
-    struct __attribute__((packed)) foz_payload_header {
-      uint32_t payload_size;
-      uint32_t format;
-      uint32_t crc;
-      uint32_t uncompressed_size;
-    };
-
-  } // namespace MesaFOZ
-
-  class IndexedDB;
-
-  struct IndexEntry {
-    IndexedDB* DB;
-    uint64_t Offset;
-    uint32_t Size;
-    uint32_t GuestSize;
-    XXH128_hash_t GuestHash;
-    fextl::vector<uint32_t> GuestExtents;
-  };
-
-  struct IndexCacheHead {
-    struct IndexEntry MainEntry;
-    uint64_t MainEntryFootprint;
-    fextl::unique_ptr<fextl::multimap<uint64_t, IndexEntry>> MoreEntries; // sorted by guest footprint
-  };
-
-  struct __attribute__((packed)) BlobFixedHeader {
-    uint32_t GuestSize;
-    uint32_t HostSize;
-    uint32_t EntryPointCount;
-    uint32_t SmallRelocCount;
-    uint32_t ThunkRelocCount;
-    XXH128_hash_t GuestHash;
-  };
-
-  // packed struct for types 0, 2 and 3. type 1 is bigger and separate below
-  struct __attribute__((packed)) BlobSmallRelocation {
-    uint32_t Offset;
-    uint8_t Type;
-    union {
-      struct __attribute__((packed)) {
-        uint32_t Symbol;
-      } Named;
-      struct __attribute__((packed)) {
-        uint64_t GuestRIP;
-      } RIPLiteral;
-      struct __attribute__((packed)) {
-        uint8_t RegisterIndex;
-        uint64_t GuestRIP;
-      } RIPMove;
-      struct __attribute__((packed)) {
-        uint8_t RegisterIndex;
-        uint8_t ValueSize;
-        uint32_t SiteOffset;
-      } PatchableData;
-    };
-  };
-
-  // type 1, implicit
-  struct __attribute__((packed)) BlobThunkRelocation {
-    uint32_t Offset;
-    uint8_t RegisterIndex;
-    uint8_t SymbolHash[32]; // sha256sum in the real RelocNamedThunkMove
-  };
-
-  struct CodeHitData {
-    fextl::vector<uint8_t> Blob;
-    std::span<uint8_t> HostCode;
-    std::span<const uint64_t> GuestPages;
-    std::span<uint64_t> EntryPointRIPs;
-    std::span<const uint32_t> EntryPointHostOffsets;
-
-    // the spans above point to memory owned by the Blob vec, so it's important this can't be copied
-    CodeHitData() = default;
-    CodeHitData(CodeHitData&&) = default;
-    CodeHitData& operator=(CodeHitData&&) = default;
-    CodeHitData(const CodeHitData&) = delete;
-    CodeHitData& operator=(const CodeHitData&) = delete;
-  };
-
-  using Index = fextl::robin_map<uint64_t, IndexCacheHead>;
-
-  class FOZFile {
-  public:
-    bool Open(const fextl::string& CacheFileName, bool ReadOnly);
-    bool Lock(uint32_t TimeoutMS) {
-      if (!FD) {
-        return false;
-      }
-      return FD->Lock(TimeoutMS);
-    }
-    bool Unlock() {
-      if (!FD) {
-        return false;
-      }
-      return FD->Unlock();
-    }
-    File::File::FileHandleType GetHandle() {
-      return FD ? FD->GetHandle() : (File::File::FileHandleType)-1;
-    }
-    ssize_t Size();
-    bool ReadAll(fextl::vector<uint8_t>& Out); // from first blob
-    bool ReadBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
-    bool WriteBlob(const MesaFOZ::foz_payload_key& Key, std::span<const std::span<const uint8_t>> BlobChunks, uint64_t& OutBlobOffset);
-
-  private:
-    static constexpr uint32_t OPEN_LOCK_TIMEOUT_MS = 100;
-
-    fextl::string FileName;
-    fextl::unique_ptr<File::File> FD;
-    bool ReadOnly = false;
-  };
-
-  class IndexedDB {
-  public:
-    bool Open(const fextl::string& CacheDBName, bool ReadOnly);
-    void PopulateIndex(Index& CacheIndex, bool& FoundMetadata);
-    bool ReadCacheBlob(uint64_t Offset, std::span<uint8_t> OutBlob);
-    bool StoreCacheBlob(const MesaFOZ::foz_payload_key& UniqueKey, uint64_t LookupKey, std::span<const uint8_t> Blob, Index& CacheIndex,
-                        std::mutex& IndexMutex, std::span<const uint8_t> IndexBlob);
-
-  private:
-    // stores run on the Writer, so returning quick isn't as important
-    static constexpr uint32_t STORE_LOCK_TIMEOUT_MS = 1000;
-    static constexpr uint64_t BIG_MAPPING_SIZE = 1ULL << 33;
-    static constexpr uint32_t LOOKUP_KEY_MAX_BUCKET_DEPTH = 20;
-
-    FOZFile CacheFOZ;
-    uint8_t* CacheFileMapping = nullptr;
-    std::atomic<uint64_t> CacheFileSize;
-    FOZFile IndexFOZ;
-    bool ReadOnly = false;
-  };
-
-  class DiskCache {
-  public:
-    void Init(FEXCore::Context::ContextImpl* CTX);
-
-    std::optional<CodeHitData> Lookup(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP,
-                                      std::optional<uint64_t>& GuestCodeKey);
-    void Validate(uint64_t GuestCodeKey, const CodeHitData& Hit, const CPU::CPUBackend::CompiledCode& CompiledCode,
-                  std::optional<ExecutableFileSectionInfo> Region);
-    bool Store(Core::InternalThreadState* Thread, std::optional<ExecutableFileSectionInfo> Region, uint64_t GuestRIP, uint64_t GuestCodeKey,
-               std::span<const uint8_t> GuestCode, const CPU::CPUBackend::CompiledCode& CompiledCode,
-               std::span<const FEXCore::CPU::Relocation> Relocations, const Frontend::Decoder::DecodedBlockInformation* DecodedBlockInfo);
-
-    bool IsWritingDiskCache() const {
-      return WritingDiskCache;
-    }
-    bool IsReadingDiskCache() const {
-      return ReadingDiskCache;
-    }
-    bool IsValidating() const {
-      return Validation;
-    }
-
-  private:
-    bool OpenCacheDB(const fextl::string& CacheDBName, bool ReadOnly);
-    uint64_t MakeLookupKey(Core::InternalThreadState* Thread, const uint64_t ModuleOffset, bool Writable, bool MonoBackpatcher);
-
-    bool ReadingDiskCache {};
-    bool WritingDiskCache {};
-    FEXCore::Context::ContextImpl* CTX;
-    XXH128_hash_t BucketHash;
-    fextl::vector<fextl::unique_ptr<IndexedDB>> ROCacheDBs;
-    fextl::unique_ptr<IndexedDB> RWCacheDB;
-    Index Index;
-    std::mutex IndexLock;
-    bool FoundMetadata = false;
-    struct CacheStoreWorkItem;
-
-    // the Writer holds references to all this stuff above and needs to be last
-    fextl::unique_ptr<WorkQueueThread> Writer;
-
-    FEX_CONFIG_OPT(EnableDiskCache, DISKCACHE);
-    FEX_CONFIG_OPT(Validation, DISKCACHEVALIDATION);
-    FEX_CONFIG_OPT(MapDiskCacheFiles, DISKCACHEFILEMAPPING);
-    FEX_CONFIG_OPT(RelocationFilter, DISKCACHERELOCATIONFILTER);
-    FEX_CONFIG_OPT(AnonCaching, DISKCACHEANONCACHING);
-    FEX_CONFIG_OPT(BasePathOverride, DISKCACHEPATH);
-    FEX_CONFIG_OPT(RODBNames, DISKCACHERODBNAMES);
-  };
-
-  static constexpr uint16_t AnonPrefixGuestBytes = 64;
-
-  // TODO: This header is in global installed header path, but uses internal headers.
-  // Migrate this once that is fixed.
-  static constexpr uint16_t FormatVersion = 19;
-  FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion();
-
-} // namespace DiskCache
-
-} // namespace FEXCore
+namespace FEXCore::DiskCache {
+FEX_DEFAULT_VISIBILITY uint16_t GetFormatVersion();
+} // namespace FEXCore::DiskCache

FEXCore/include/FEXCore/Core/HostFeatures.h

@@ -14,7 +14,7 @@ namespace FEXCore {
  */
 struct HostFeatures {
   // Changes code generation slightly.
-  enum class HostTypeEnum : uint32_t {
+  enum class HostTypeEnum : uint8_t {
     Unknown,
     Linux,
     Wow64,
@@ -24,7 +24,22 @@ struct HostFeatures {
   // Whether or not the host supports any kind of SVE implementation.
   [[nodiscard]]
   bool SupportsSVE() const {
-    return SupportsSVE128 || SupportsSVE256;
+    return ProcessFeatures.SupportsSVE128 || ProcessFeatures.SupportsSVE256;
+  }
+
+  [[nodiscard]]
+  bool SupportsSVE128() const {
+    return ProcessFeatures.SupportsSVE128;
+  }
+
+  [[nodiscard]]
+  bool SupportsSVE256() const {
+    return ProcessFeatures.SupportsSVE256;
+  }
+
+  [[nodiscard]]
+  bool SupportsAVX() const {
+    return ProcessFeatures.SupportsAVX;
   }
 
   [[nodiscard]]
@@ -32,55 +47,92 @@ struct HostFeatures {
     return 4 << DCacheLineLog2;
   }
 
+  /**
+   * @brief This structure contains all the data required to determine hashing at the "process" level.
+   * Unlike the `HostFeaturesHash` below, this data changes codegen depending on if the process is Linux, Arm64EC, or Wow64.
+   */
+  struct ProcessFeaturesHash {
+    HostTypeEnum HostType                      : 2 {};
+    uint8_t SupportsFaultFreeSynchronousFaults : 1 {};
+    // This only determines the starting mode a process started in.
+    // The process can technically long jump to-from the other modes.
+    uint8_t Is64BitMode : 1 {};
+
+    // These are technically HostFeatures, but they differ between Linux, wow64, arm64.
+    uint8_t SupportsAVX    : 1 {};
+    uint8_t SupportsSVE128 : 1 {};
+    uint8_t SupportsSVE256 : 1 {};
+
+    uint8_t pad : 1 {};
+  };
+  static_assert(sizeof(ProcessFeaturesHash) == 1);
+
+  struct CacheHash {
+    // This hash is used to determine the "bucket" a machine goes in to.
+    // This should match between Linux, Arm64EC, and Wow64.
+    uint64_t HostFeaturesHash;
+
+    // This hash is used to determine features that are different between process types.
+    ProcessFeaturesHash ProcessFeaturesHash;
+  };
+
   [[nodiscard]]
-  uint64_t HashForCaching() const {
-    // As long as the number of options is 64-bit or below, we can just return it.
-    // Skip CPUMIDRs as it doesn't affect codegen.
-    static_assert(offsetof(HostFeatures, CPUMIDRs) == 8);
-    uint64_t Result {};
-    memcpy(&Result, this, sizeof(Result));
+  CacheHash HashForCaching() const {
+    static_assert(offsetof(HostFeatures, ProcessFeatures) == 8);
+
+    CacheHash Result {};
+    memcpy(&Result.HostFeaturesHash, this, sizeof(uint64_t));
+    Result.ProcessFeaturesHash = ProcessFeatures;
+
     return Result;
   }
 
-  uint32_t DCacheLineLog2              : 4 {};
-  uint32_t SupportsCacheMaintenanceOps : 1 {};
-  uint32_t SupportsAES                 : 1 {};
-  uint32_t SupportsCRC                 : 1 {};
-  uint32_t SupportsCLZERO              : 1 {};
-  uint32_t SupportsAtomics             : 1 {};
-  uint32_t SupportsRCPC                : 1 {};
-  uint32_t SupportsTSOImm9             : 1 {};
-  uint32_t SupportsRAND                : 1 {};
-  uint32_t SupportsAVX                 : 1 {};
-  uint32_t SupportsSVE128              : 1 {};
-  uint32_t SupportsSVE256              : 1 {};
-  uint32_t SupportsSHA                 : 1 {};
-  uint32_t SupportsPMULL_128Bit        : 1 {};
-  uint32_t SupportsCSSC                : 1 {};
-  uint32_t SupportsFCMA                : 1 {};
-  uint32_t SupportsFlagM               : 1 {};
-  uint32_t SupportsFlagM2              : 1 {};
-  uint32_t SupportsRPRES               : 1 {};
-  uint32_t SupportsPreserveAllABI      : 1 {};
-  uint32_t SupportsAES256              : 1 {};
-  uint32_t SupportsSVEBitPerm          : 1 {};
-  uint32_t SupportsCPUIndexInTPIDRRO   : 1 {};
-  uint32_t SupportsFRINTTS             : 1 {};
-  uint32_t SupportsECV                 : 1 {};
-  uint32_t SupportsWFXT                : 1 {};
-  uint32_t Supports3DNow               : 1 {};
-  uint32_t SupportsSSE4a               : 1 {};
-  uint32_t SupportsMOPS                : 1 {};
-  uint32_t PreferZVAForVZero           : 1 {};
-  uint32_t SupportsAFP                 : 1 {};
-  uint32_t SupportsFloatExceptions     : 1 {};
+  // Affects codegen and is basically machine description.
+  uint32_t DCacheLineLog2              : 4 {}; // [3:0]
+  uint32_t SupportsCacheMaintenanceOps : 1 {}; // [4]
+  uint32_t SupportsAES                 : 1 {}; // [5]
+  uint32_t SupportsCRC                 : 1 {}; // [6]
+  uint32_t SupportsCLZERO              : 1 {}; // [7]
+  uint32_t SupportsAtomics             : 1 {}; // [8]
+  uint32_t SupportsRCPC                : 1 {}; // [9]
+  uint32_t SupportsTSOImm9             : 1 {}; // [10]
+  uint32_t SupportsRAND                : 1 {}; // [11]
+  uint32_t SupportsSHA                 : 1 {}; // [12]
+  uint32_t SupportsPMULL_128Bit        : 1 {}; // [13]
+  uint32_t SupportsCSSC                : 1 {}; // [14]
+  uint32_t SupportsFCMA                : 1 {}; // [15]
+  uint32_t SupportsFlagM               : 1 {}; // [16]
+  uint32_t SupportsFlagM2              : 1 {}; // [17]
+  uint32_t SupportsRPRES               : 1 {}; // [18]
+  uint32_t SupportsPreserveAllABI      : 1 {}; // [19]
+  uint32_t SupportsAES256              : 1 {}; // [20]
+  uint32_t SupportsSVEBitPerm          : 1 {}; // [21]
+  uint32_t SupportsCPUIndexInTPIDRRO   : 1 {}; // [22]
+  uint32_t SupportsFRINTTS             : 1 {}; // [23]
+  uint32_t SupportsECV                 : 1 {}; // [24]
+  uint32_t SupportsWFXT                : 1 {}; // [25]
+  uint32_t Supports3DNow               : 1 {}; // [26]
+  uint32_t SupportsSSE4a               : 1 {}; // [27]
+  uint32_t SupportsMOPS                : 1 {}; // [28]
+  uint32_t SupportsI8MM                : 1 {}; // [29]
+  uint32_t SupportsDotProd             : 1 {}; // [30]
+  uint32_t PreferZVAForVZero           : 1 {}; // [31]
+  uint32_t SupportsAFP                 : 1 {}; // [32]
+  uint32_t SupportsFloatExceptions     : 1 {}; // [33]
+  uint32_t SupportsBF16                : 1 {}; // [34]
   // Flag if this is InstCountCI
-  uint32_t IsInstCountCI : 1 {};
-  HostTypeEnum HostType  : 2 {};
-  uint32_t pad           : 26 {};
+  uint32_t IsInstCountCI : 1 {};  // [38]
+  uint32_t pad           : 28 {}; // [63:35]
+
+  // This affects codegen, but it isn't machine state
+  ProcessFeaturesHash ProcessFeatures {};
 
   // MIDR information
   // Also used for determining number of CPU cores for CPUID
   fextl::vector<uint32_t> CPUMIDRs;
+
+  // The Linux PID of this process. Useful for punching through perf-top information.
+  uint32_t ProcessPID {};
+  uint32_t pad2 {};
 };
 } // namespace FEXCore

FEXCore/include/FEXCore/IR/IR.h

@@ -15,6 +15,7 @@ namespace FEXCore::IR {
 enum NamedVectorConstant : uint8_t {
   NAMED_VECTOR_INCREMENTAL_U16_INDEX = 0,
   NAMED_VECTOR_INCREMENTAL_U16_INDEX_UPPER,
+  NAMED_VECTOR_INCREMENTAL_U8_INDEX,
   NAMED_VECTOR_PADDSUBPS_INVERT,
   NAMED_VECTOR_PADDSUBPS_INVERT_UPPER,
   NAMED_VECTOR_PADDSUBPD_INVERT,

FEXCore/include/FEXCore/Utils/Allocator.h

@@ -79,5 +79,6 @@ FEX_DEFAULT_VISIBILITY fextl::vector<MemoryRegion> Setup48BitAllocatorIfExists(s
 #ifndef _WIN32
 FEX_DEFAULT_VISIBILITY void RegisterTLSData(FEXCore::Core::InternalThreadState* Thread);
 FEX_DEFAULT_VISIBILITY void UninstallTLSData(FEXCore::Core::InternalThreadState* Thread);
+FEX_DEFAULT_VISIBILITY bool TryGrowMainThreadStack(uintptr_t FaultAddress);
 #endif
 } // namespace FEXCore::Allocator

FEXCore/include/FEXCore/Utils/FileUtils.h

@@ -0,0 +1,19 @@
+// SPDX-License-Identifier: MIT
+#pragma once
+#include <FEXCore/fextl/functional.h>
+#include <FEXCore/fextl/string.h>
+
+#include <FEXCore/Utils/CompilerDefs.h>
+
+namespace FEXCore::FileUtils {
+/**
+ * @brief Removes an entirely directory passed in. Including the path itself.
+ *
+ * @return True if the directory didn't exist or was deleted.
+ */
+FEX_DEFAULT_VISIBILITY bool RecursiveRemoveDirectory(const fextl::string& Directory);
+
+FEX_DEFAULT_VISIBILITY void WalkDirectory(std::string_view Directory,
+                                          fextl::move_only_function<void(std::string_view name, bool is_dir, const void* user_data)> Callback,
+                                          const void* user_data);
+} // namespace FEXCore::FileUtils

FEXCore/include/FEXCore/Utils/PrctlUtils.h

@@ -53,4 +53,8 @@
 #define PR_GET_MDWE 66
 #endif
 
+#ifndef PR_GET_TPIDRRO_CPU
+#define PR_GET_TPIDRRO_CPU 0x7F1DBB0
+#endif
+
 #endif // ifndef _WIN32

FEXCore/include/FEXCore/Utils/Threads.h

@@ -13,7 +13,7 @@ struct Flags {
 using ThreadFunc = void* (*)(void* user_ptr);
 
 class Thread;
-using CreateThreadFunc = fextl::unique_ptr<Thread> (*)(ThreadFunc Func, void* Arg, Flags Flags);
+using CreateThreadFunc = fextl::unique_ptr<Thread> (*)(ThreadFunc Func, void* Arg, Flags Flags, const char* ThreadName);
 using CleanupAfterForkFunc = void (*)();
 
 struct Pointers {
@@ -34,7 +34,7 @@ public:
    * @name Calls provided API functions
    * @{ */
 
-  static fextl::unique_ptr<Thread> Create(ThreadFunc Func, void* Arg, Flags Flags = {});
+  static fextl::unique_ptr<Thread> Create(ThreadFunc Func, void* Arg, Flags Flags = {}, const char* ThreadName = nullptr);
 
   static void CleanupAfterFork();
 

FEXCore/include/FEXCore/Utils/WorkQueueThread.h

@@ -18,7 +18,7 @@ public:
     virtual void Run() = 0;
   };
 
-  WorkQueueThread(FEXCore::Threads::Flags Flags = {});
+  WorkQueueThread(FEXCore::Threads::Flags Flags = {}, const char* ThreadName = nullptr);
   ~WorkQueueThread();
 
   void QueueWork(fextl::unique_ptr<WorkItem> Work);

FEXCore/include/FEXCore/fextl/functional.h

@@ -47,7 +47,7 @@ public:
 
       // Second, wrap the relocated argument in a single-capture lambda
       auto wrapped_lambda = [moved_lambda](Args... args) {
-        return (*moved_lambda)(std::forward<Args>(args)...);
+        return std::invoke(*moved_lambda, std::forward<Args>(args)...);
       };
 
       // Third, assign the result to std::function, ensuring it's indeed

FEXCore/unittests/APITests/CMakeLists.txt

@@ -4,8 +4,11 @@ set(LIBS fmt::fmt vixl::vixl Catch2::Catch2WithMain FEXCore_Base JemallocLibs)
 foreach(TEST ${TESTS})
   get_filename_component(TEST_NAME ${TEST} NAME_WLE)
   add_executable(FEXCore_Tests_${TEST_NAME} ${TEST})
+
   target_link_libraries(FEXCore_Tests_${TEST_NAME} PRIVATE ${LIBS})
   target_include_directories(FEXCore_Tests_${TEST_NAME} PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/../../Source/")
+  target_compile_options(FEXCore_Tests_${TEST_NAME} PRIVATE ${FEX_TUNE_COMPILE_FLAGS})
+
   set_target_properties(FEXCore_Tests_${TEST_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/FEXCore_Tests")
   catch_discover_tests(FEXCore_Tests_${TEST_NAME} TEST_SUFFIX ".${TEST_NAME}.FEXCore_Tests")
 endforeach()

FEXCore/unittests/APITests/crc32.cpp

@@ -0,0 +1,9 @@
+// SPDX-License-Identifier: MIT
+#include <catch2/catch_all.hpp>
+
+#include "Utils/crc32.h"
+
+TEST_CASE("Simple") {
+  uint32_t data = 0x41424344U;
+  CHECK(FEXCore::Utils::crc32(&data, sizeof(data)) == 0xa53ea072);
+}

FEXCore/unittests/Emitter/ASIMD_Tests.cpp

@@ -644,6 +644,9 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: ASIMD: Advanced SIMD two-register m
   TEST_SINGLE(frecpe(SubRegSize::i32Bit, DReg::d30, DReg::d29), "frecpe v30.2s, v29.2s");
   // TEST_SINGLE(frecpe(SubRegSize::i64Bit, DReg::d30, DReg::d29), "frecpe v30.1d, v29.1d");
 
+  TEST_SINGLE(bfcvtn(QReg::q30, QReg::q29), "bfcvtn v30.4h, v29.4s");
+  TEST_SINGLE(bfcvtn2(QReg::q30, QReg::q29), "bfcvtn2 v30.8h, v29.4s");
+
   TEST_SINGLE(rev32(SubRegSize::i8Bit, QReg::q30, QReg::q29), "rev32 v30.16b, v29.16b");
   TEST_SINGLE(rev32(SubRegSize::i16Bit, QReg::q30, QReg::q29), "rev32 v30.8h, v29.8h");
   // TEST_SINGLE(rev32(SubRegSize::i32Bit, QReg::q30, QReg::q29), "rev32 v30.4s, v29.4s");

FEXCore/unittests/Emitter/SVE_Tests.cpp

@@ -3937,12 +3937,9 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: SVE: SVE load and broadcast element
   TEST_SINGLE(ld1rb(SubRegSize::i8Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 31), "ld1rb {z30.b}, p6/z, [x29, #31]");
   TEST_SINGLE(ld1rb(SubRegSize::i8Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 63), "ld1rb {z30.b}, p6/z, [x29, #63]");
 
-  // TODO: Several instances are commented out due to a reported bug in the vixl dissassembler.
-  //       Uncomment these when it's fixed.
-
   TEST_SINGLE(ld1rb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 0), "ld1rb {z30.h}, p6/z, [x29]");
-  // TEST_SINGLE(ld1rb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 31), "ld1rb {z30.h}, p6/z, [x29, #31]");
-  // TEST_SINGLE(ld1rb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 63), "ld1rb {z30.h}, p6/z, [x29, #63]");
+  TEST_SINGLE(ld1rb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 31), "ld1rb {z30.h}, p6/z, [x29, #31]");
+  TEST_SINGLE(ld1rb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 63), "ld1rb {z30.h}, p6/z, [x29, #63]");
 
   TEST_SINGLE(ld1rb(SubRegSize::i32Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 0), "ld1rb {z30.s}, p6/z, [x29]");
   TEST_SINGLE(ld1rb(SubRegSize::i32Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 31), "ld1rb {z30.s}, p6/z, [x29, #31]");
@@ -3953,8 +3950,8 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: SVE: SVE load and broadcast element
   TEST_SINGLE(ld1rb(SubRegSize::i64Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 63), "ld1rb {z30.d}, p6/z, [x29, #63]");
 
   TEST_SINGLE(ld1rsb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 0), "ld1rsb {z30.h}, p6/z, [x29]");
-  // TEST_SINGLE(ld1rsb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 31), "ld1rsb {z30.h}, p6/z, [x29, #31]");
-  // TEST_SINGLE(ld1rsb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 63), "ld1rsb {z30.h}, p6/z, [x29, #63]");
+  TEST_SINGLE(ld1rsb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 31), "ld1rsb {z30.h}, p6/z, [x29, #31]");
+  TEST_SINGLE(ld1rsb(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 63), "ld1rsb {z30.h}, p6/z, [x29, #63]");
 
   TEST_SINGLE(ld1rsb(SubRegSize::i32Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 0), "ld1rsb {z30.s}, p6/z, [x29]");
   TEST_SINGLE(ld1rsb(SubRegSize::i32Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 31), "ld1rsb {z30.s}, p6/z, [x29, #31]");
@@ -3965,8 +3962,8 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: SVE: SVE load and broadcast element
   TEST_SINGLE(ld1rsb(SubRegSize::i64Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 63), "ld1rsb {z30.d}, p6/z, [x29, #63]");
 
   TEST_SINGLE(ld1rh(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 0), "ld1rh {z30.h}, p6/z, [x29]");
-  // TEST_SINGLE(ld1rh(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 64), "ld1rh {z30.h}, p6/z, [x29, #64]");
-  // TEST_SINGLE(ld1rh(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 126), "ld1rh {z30.h}, p6/z, [x29, #126]");
+  TEST_SINGLE(ld1rh(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 64), "ld1rh {z30.h}, p6/z, [x29, #64]");
+  TEST_SINGLE(ld1rh(SubRegSize::i16Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 126), "ld1rh {z30.h}, p6/z, [x29, #126]");
 
   TEST_SINGLE(ld1rh(SubRegSize::i32Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 0), "ld1rh {z30.s}, p6/z, [x29]");
   TEST_SINGLE(ld1rh(SubRegSize::i32Bit, ZReg::z30, PReg::p6.Zeroing(), Reg::r29, 64), "ld1rh {z30.s}, p6/z, [x29, #64]");

FEXCore/unittests/Emitter/Scalar_Tests.cpp

@@ -660,25 +660,22 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Scalar: Advanced SIMD scalar x inde
   TEST_SINGLE(sqrdmulh(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 0), "sqrdmulh s30, s29, v28.s[0]");
   TEST_SINGLE(sqrdmulh(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 3), "sqrdmulh s30, s29, v28.s[3]");
 
-  // TODO: Commented out due to a bug in vixl's decoder (which has been reported).
-  //       Uncomment these when fixed.
-
-  // TEST_SINGLE(fmla(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 4), "fmla h30, h29, v15.h[4]");
-  // TEST_SINGLE(fmla(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 7), "fmla h30, h29, v15.h[7]");
+  TEST_SINGLE(fmla(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 4), "fmla h30, h29, v15.h[4]");
+  TEST_SINGLE(fmla(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 7), "fmla h30, h29, v15.h[7]");
   TEST_SINGLE(fmla(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 0), "fmla s30, s29, v28.s[0]");
   TEST_SINGLE(fmla(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 3), "fmla s30, s29, v28.s[3]");
   TEST_SINGLE(fmla(ScalarRegSize::i64Bit, VReg::v30, VReg::v29, VReg::v28, 0), "fmla d30, d29, v28.d[0]");
   TEST_SINGLE(fmla(ScalarRegSize::i64Bit, VReg::v30, VReg::v29, VReg::v28, 1), "fmla d30, d29, v28.d[1]");
 
-  // TEST_SINGLE(fmls(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 4), "fmls h30, h29, v15.h[4]");
-  // TEST_SINGLE(fmls(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 7), "fmls h30, h29, v15.h[7]");
+  TEST_SINGLE(fmls(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 4), "fmls h30, h29, v15.h[4]");
+  TEST_SINGLE(fmls(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 7), "fmls h30, h29, v15.h[7]");
   TEST_SINGLE(fmls(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 0), "fmls s30, s29, v28.s[0]");
   TEST_SINGLE(fmls(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 3), "fmls s30, s29, v28.s[3]");
   TEST_SINGLE(fmls(ScalarRegSize::i64Bit, VReg::v30, VReg::v29, VReg::v28, 0), "fmls d30, d29, v28.d[0]");
   TEST_SINGLE(fmls(ScalarRegSize::i64Bit, VReg::v30, VReg::v29, VReg::v28, 1), "fmls d30, d29, v28.d[1]");
 
-  // TEST_SINGLE(fmul(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 4), "fmul h30, h29, v15.h[4]");
-  // TEST_SINGLE(fmul(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 7), "fmul h30, h29, v15.h[7]");
+  TEST_SINGLE(fmul(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 4), "fmul h30, h29, v15.h[4]");
+  TEST_SINGLE(fmul(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 7), "fmul h30, h29, v15.h[7]");
   TEST_SINGLE(fmul(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 0), "fmul s30, s29, v28.s[0]");
   TEST_SINGLE(fmul(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 3), "fmul s30, s29, v28.s[3]");
   TEST_SINGLE(fmul(ScalarRegSize::i64Bit, VReg::v30, VReg::v29, VReg::v28, 0), "fmul d30, d29, v28.d[0]");
@@ -694,8 +691,8 @@ TEST_CASE_METHOD(TestDisassembler, "Emitter: Scalar: Advanced SIMD scalar x inde
   TEST_SINGLE(sqrdmlsh(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 0), "sqrdmlsh s30, s29, v28.s[0]");
   TEST_SINGLE(sqrdmlsh(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 3), "sqrdmlsh s30, s29, v28.s[3]");
 
-  // TEST_SINGLE(fmulx(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 4), "fmulx h30, h29, v15.h[4]");
-  // TEST_SINGLE(fmulx(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 7), "fmulx h30, h29, v15.h[7]");
+  TEST_SINGLE(fmulx(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 4), "fmulx h30, h29, v15.h[4]");
+  TEST_SINGLE(fmulx(ScalarRegSize::i16Bit, VReg::v30, VReg::v29, VReg::v15, 7), "fmulx h30, h29, v15.h[7]");
   TEST_SINGLE(fmulx(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 0), "fmulx s30, s29, v28.s[0]");
   TEST_SINGLE(fmulx(ScalarRegSize::i32Bit, VReg::v30, VReg::v29, VReg::v28, 3), "fmulx s30, s29, v28.s[3]");
   TEST_SINGLE(fmulx(ScalarRegSize::i64Bit, VReg::v30, VReg::v29, VReg::v28, 0), "fmulx d30, d29, v28.d[0]");

FEXHeaderUtils/FEXHeaderUtils/Syscalls.h

@@ -96,6 +96,10 @@ inline int32_t renameat2(int olddirfd, const char* oldpath, int newdirfd, const
 inline int32_t pidfd_open(pid_t pid, unsigned int flags) {
   return ::syscall(SYS_pidfd_open, pid, flags);
 }
+
+inline ssize_t getrandom(void* buf, size_t buflen, unsigned int flags) {
+  return ::syscall(SYS_getrandom, buf, buflen, flags);
+}
 #else
 
 inline int32_t getcpu(uint32_t* cpu, uint32_t* node) {